dolysis
/
rocksdb
mirror of https://github.com/facebook/rocksdb.git
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Replace deprecated RocksDB#addFile with RocksDB#ingestExternalFile 

Summary:
Previously the Java implementation of `RocksDB#addFile` was both incomplete and not inline with the C++ API.

Rather than fix it, as I see that `rocksdb::DB::AddFile` is now deprecated in favour of `rocksdb::DB::IngestExternalFile`, I have removed the old broken implementation and implemented `RocksDB#ingestExternalFile`.

Closes https://github.com/facebook/rocksdb/issues/2261
Closes https://github.com/facebook/rocksdb/pull/2291

Differential Revision: D5061264

Pulled By: sagar0

fbshipit-source-id: 85df0899fa1b1fc3535175cac4f52353511d4104
This commit is contained in:
Adam Retter 2017-05-22 10:22:49 -07:00 committed by Facebook Github Bot
parent 228f49d20a
commit 88c818e437
11 changed files with 461 additions and 633 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
java/Makefile
View File

@ -19,9 +19,9 @@ NATIVE_JAVA_CLASSES = org.rocksdb.AbstractCompactionFilter\
org.rocksdb.DirectSlice\
org.rocksdb.Env\
org.rocksdb.EnvOptions\
org.rocksdb.ExternalSstFileInfo\
org.rocksdb.FlushOptions\
org.rocksdb.Filter\
org.rocksdb.IngestExternalFileOptions\
org.rocksdb.HashLinkedListMemTableConfig\
org.rocksdb.HashSkipListMemTableConfig\
org.rocksdb.Logger\
@ -87,7 +87,7 @@ JAVA_TESTS = org.rocksdb.BackupableDBOptionsTest\
org.rocksdb.DirectComparatorTest\
org.rocksdb.DirectSliceTest\
org.rocksdb.EnvOptionsTest\
org.rocksdb.ExternalSstFileInfoTest\
org.rocksdb.IngestExternalFileOptionsTest\
org.rocksdb.util.EnvironmentTest\
org.rocksdb.FilterTest\
org.rocksdb.FlushTest\

274
java/rocksjni/external_sst_file_info.cc
View File

@ -1,274 +0,0 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
//
// This file implements the "bridge" between Java and C++ and enables
// calling C++ rocksdb::ExternalSstFileInfo methods
// from Java side.
#include <jni.h>
#include "include/org_rocksdb_ExternalSstFileInfo.h"
#include "rocksdb/sst_file_writer.h"
/*
* Class: org_rocksdb_ExternalSstFileInfo
* Method: newExternalSstFileInfo
* Signature: ()J
*/
jlong Java_org_rocksdb_ExternalSstFileInfo_newExternalSstFileInfo__(
JNIEnv *env, jclass jcls) {
return reinterpret_cast<jlong>(new rocksdb::ExternalSstFileInfo());
}
/*
* Class: org_rocksdb_ExternalSstFileInfo
* Method: newExternalSstFileInfo
* Signature: (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;JJII)J
*/
jlong Java_org_rocksdb_ExternalSstFileInfo_newExternalSstFileInfo__Ljava_lang_String_2Ljava_lang_String_2Ljava_lang_String_2JJII(
JNIEnv *env, jclass jcls, jstring jfile_path, jstring jsmallest_key,
jstring jlargest_key, jlong jsequence_number, jlong jfile_size,
jint jnum_entries, jint jversion) {
const char *file_path = env->GetStringUTFChars(jfile_path, nullptr);
if(file_path == nullptr) {
// exception thrown: OutOfMemoryError
return 0;
}
const char *smallest_key = env->GetStringUTFChars(jsmallest_key, nullptr);
if(smallest_key == nullptr) {
// exception thrown: OutOfMemoryError
env->ReleaseStringUTFChars(jfile_path, file_path);
return 0;
}
const char *largest_key = env->GetStringUTFChars(jlargest_key, nullptr);
if(largest_key == nullptr) {
// exception thrown: OutOfMemoryError
env->ReleaseStringUTFChars(jsmallest_key, smallest_key);
env->ReleaseStringUTFChars(jfile_path, file_path);
return 0;
}
auto *external_sst_file_info = new rocksdb::ExternalSstFileInfo(
file_path, smallest_key, largest_key,
static_cast<rocksdb::SequenceNumber>(jsequence_number),
static_cast<uint64_t>(jfile_size), static_cast<int32_t>(jnum_entries),
static_cast<int32_t>(jversion));
env->ReleaseStringUTFChars(jlargest_key, largest_key);
env->ReleaseStringUTFChars(jsmallest_key, smallest_key);
env->ReleaseStringUTFChars(jfile_path, file_path);
return reinterpret_cast<jlong>(external_sst_file_info);
}
/*
* Class: org_rocksdb_ExternalSstFileInfo
* Method: setFilePath
* Signature: (JLjava/lang/String;)V
*/
void Java_org_rocksdb_ExternalSstFileInfo_setFilePath(JNIEnv *env, jobject jobj,
jlong jhandle,
jstring jfile_path) {
auto *external_sst_file_info =
reinterpret_cast<rocksdb::ExternalSstFileInfo *>(jhandle);
const char *file_path = env->GetStringUTFChars(jfile_path, nullptr);
if(file_path == nullptr) {
// exception thrown: OutOfMemoryError
return;
}
external_sst_file_info->file_path = file_path;
env->ReleaseStringUTFChars(jfile_path, file_path);
}
/*
* Class: org_rocksdb_ExternalSstFileInfo
* Method: filePath
* Signature: (J)Ljava/lang/String;
*/
jstring Java_org_rocksdb_ExternalSstFileInfo_filePath(JNIEnv *env, jobject jobj,
jlong jhandle) {
auto *external_sst_file_info =
reinterpret_cast<rocksdb::ExternalSstFileInfo *>(jhandle);
return env->NewStringUTF(external_sst_file_info->file_path.data());
}
/*
* Class: org_rocksdb_ExternalSstFileInfo
* Method: setSmallestKey
* Signature: (JLjava/lang/String;)V
*/
void Java_org_rocksdb_ExternalSstFileInfo_setSmallestKey(
JNIEnv *env, jobject jobj, jlong jhandle, jstring jsmallest_key) {
auto *external_sst_file_info =
reinterpret_cast<rocksdb::ExternalSstFileInfo *>(jhandle);
const char *smallest_key = env->GetStringUTFChars(jsmallest_key, nullptr);
if(smallest_key == nullptr) {
// exception thrown: OutOfMemoryError
return;
}
external_sst_file_info->smallest_key = smallest_key;
env->ReleaseStringUTFChars(jsmallest_key, smallest_key);
}
/*
* Class: org_rocksdb_ExternalSstFileInfo
* Method: smallestKey
* Signature: (J)Ljava/lang/String;
*/
jstring Java_org_rocksdb_ExternalSstFileInfo_smallestKey(JNIEnv *env,
jobject jobj,
jlong jhandle) {
auto *external_sst_file_info =
reinterpret_cast<rocksdb::ExternalSstFileInfo *>(jhandle);
return env->NewStringUTF(external_sst_file_info->smallest_key.data());
}
/*
* Class: org_rocksdb_ExternalSstFileInfo
* Method: setLargestKey
* Signature: (JLjava/lang/String;)V
*/
void Java_org_rocksdb_ExternalSstFileInfo_setLargestKey(JNIEnv *env,
jobject jobj,
jlong jhandle,
jstring jlargest_key) {
auto *external_sst_file_info =
reinterpret_cast<rocksdb::ExternalSstFileInfo *>(jhandle);
const char *largest_key = env->GetStringUTFChars(jlargest_key, NULL);
if(largest_key == nullptr) {
// exception thrown: OutOfMemoryError
return;
}
external_sst_file_info->largest_key = largest_key;
env->ReleaseStringUTFChars(jlargest_key, largest_key);
}
/*
* Class: org_rocksdb_ExternalSstFileInfo
* Method: largestKey
* Signature: (J)Ljava/lang/String;
*/
jstring Java_org_rocksdb_ExternalSstFileInfo_largestKey(JNIEnv *env,
jobject jobj,
jlong jhandle) {
auto *external_sst_file_info =
reinterpret_cast<rocksdb::ExternalSstFileInfo *>(jhandle);
return env->NewStringUTF(external_sst_file_info->largest_key.data());
}
/*
* Class: org_rocksdb_ExternalSstFileInfo
* Method: setSequenceNumber
* Signature: (JJ)V
*/
void Java_org_rocksdb_ExternalSstFileInfo_setSequenceNumber(
JNIEnv *env, jobject jobj, jlong jhandle, jlong jsequence_number) {
auto *external_sst_file_info =
reinterpret_cast<rocksdb::ExternalSstFileInfo *>(jhandle);
external_sst_file_info->sequence_number =
static_cast<rocksdb::SequenceNumber>(jsequence_number);
}
/*
* Class: org_rocksdb_ExternalSstFileInfo
* Method: sequenceNumber
* Signature: (J)J
*/
jlong Java_org_rocksdb_ExternalSstFileInfo_sequenceNumber(JNIEnv *env,
jobject jobj,
jlong jhandle) {
auto *external_sst_file_info =
reinterpret_cast<rocksdb::ExternalSstFileInfo *>(jhandle);
return static_cast<jlong>(external_sst_file_info->sequence_number);
}
/*
* Class: org_rocksdb_ExternalSstFileInfo
* Method: setFileSize
* Signature: (JJ)V
*/
void Java_org_rocksdb_ExternalSstFileInfo_setFileSize(JNIEnv *env, jobject jobj,
jlong jhandle,
jlong jfile_size) {
auto *external_sst_file_info =
reinterpret_cast<rocksdb::ExternalSstFileInfo *>(jhandle);
external_sst_file_info->file_size = static_cast<uint64_t>(jfile_size);
}
/*
* Class: org_rocksdb_ExternalSstFileInfo
* Method: fileSize
* Signature: (J)J
*/
jlong Java_org_rocksdb_ExternalSstFileInfo_fileSize(JNIEnv *env, jobject jobj,
jlong jhandle) {
auto *external_sst_file_info =
reinterpret_cast<rocksdb::ExternalSstFileInfo *>(jhandle);
return static_cast<jlong>(external_sst_file_info->file_size);
}
/*
* Class: org_rocksdb_ExternalSstFileInfo
* Method: setNumEntries
* Signature: (JI)V
*/
void Java_org_rocksdb_ExternalSstFileInfo_setNumEntries(JNIEnv *env,
jobject jobj,
jlong jhandle,
jint jnum_entries) {
auto *external_sst_file_info =
reinterpret_cast<rocksdb::ExternalSstFileInfo *>(jhandle);
external_sst_file_info->num_entries = static_cast<uint64_t>(jnum_entries);
}
/*
* Class: org_rocksdb_ExternalSstFileInfo
* Method: numEntries
* Signature: (J)I
*/
jint Java_org_rocksdb_ExternalSstFileInfo_numEntries(JNIEnv *env, jobject jobj,
jlong jhandle) {
auto *external_sst_file_info =
reinterpret_cast<rocksdb::ExternalSstFileInfo *>(jhandle);
return static_cast<jint>(external_sst_file_info->num_entries);
}
/*
* Class: org_rocksdb_ExternalSstFileInfo
* Method: setVersion
* Signature: (JI)V
*/
void Java_org_rocksdb_ExternalSstFileInfo_setVersion(JNIEnv *env, jobject jobj,
jlong jhandle,
jint jversion) {
auto *external_sst_file_info =
reinterpret_cast<rocksdb::ExternalSstFileInfo *>(jhandle);
external_sst_file_info->version = static_cast<int32_t>(jversion);
}
/*
* Class: org_rocksdb_ExternalSstFileInfo
* Method: version
* Signature: (J)I
*/
jint Java_org_rocksdb_ExternalSstFileInfo_version(JNIEnv *env, jobject jobj,
jlong jhandle) {
auto *external_sst_file_info =
reinterpret_cast<rocksdb::ExternalSstFileInfo *>(jhandle);
return static_cast<jint>(external_sst_file_info->version);
}
/*
* Class: org_rocksdb_ExternalSstFileInfo
* Method: disposeInternal
* Signature: (J)V
*/
void Java_org_rocksdb_ExternalSstFileInfo_disposeInternal(JNIEnv *env,
jobject jobj,
jlong jhandle) {
auto* esfi = reinterpret_cast<rocksdb::ExternalSstFileInfo *>(jhandle);
assert(esfi != nullptr);
delete esfi;
}

149
java/rocksjni/ingest_external_file_options.cc Normal file
View File

@ -0,0 +1,149 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
//
// This file implements the "bridge" between Java and C++ for
// rocksdb::FilterPolicy.
#include <jni.h>
#include "include/org_rocksdb_IngestExternalFileOptions.h"
#include "rocksdb/options.h"
/*
* Class: org_rocksdb_IngestExternalFileOptions
* Method: newIngestExternalFileOptions
* Signature: ()J
*/
jlong Java_org_rocksdb_IngestExternalFileOptions_newIngestExternalFileOptions__(
JNIEnv* env, jclass jclazz) {
auto* options = new rocksdb::IngestExternalFileOptions();
return reinterpret_cast<jlong>(options);
}
/*
* Class: org_rocksdb_IngestExternalFileOptions
* Method: newIngestExternalFileOptions
* Signature: (ZZZZ)J
*/
jlong Java_org_rocksdb_IngestExternalFileOptions_newIngestExternalFileOptions__ZZZZ(
JNIEnv* env, jclass jcls, jboolean jmove_files,
jboolean jsnapshot_consistency, jboolean jallow_global_seqno,
jboolean jallow_blocking_flush) {
auto* options = new rocksdb::IngestExternalFileOptions();
options->move_files = static_cast<bool>(jmove_files);
options->snapshot_consistency = static_cast<bool>(jsnapshot_consistency);
options->allow_global_seqno = static_cast<bool>(jallow_global_seqno);
options->allow_blocking_flush = static_cast<bool>(jallow_blocking_flush);
return reinterpret_cast<jlong>(options);
}
/*
* Class: org_rocksdb_IngestExternalFileOptions
* Method: moveFiles
* Signature: (J)Z
*/
jboolean Java_org_rocksdb_IngestExternalFileOptions_moveFiles(
JNIEnv* env, jobject jobj, jlong jhandle) {
auto* options =
reinterpret_cast<rocksdb::IngestExternalFileOptions*>(jhandle);
return static_cast<jboolean>(options->move_files);
}
/*
* Class: org_rocksdb_IngestExternalFileOptions
* Method: setMoveFiles
* Signature: (JZ)V
*/
void Java_org_rocksdb_IngestExternalFileOptions_setMoveFiles(
JNIEnv* env, jobject jobj, jlong jhandle, jboolean jmove_files) {
auto* options =
reinterpret_cast<rocksdb::IngestExternalFileOptions*>(jhandle);
options->move_files = static_cast<bool>(jmove_files);
}
/*
* Class: org_rocksdb_IngestExternalFileOptions
* Method: snapshotConsistency
* Signature: (J)Z
*/
jboolean Java_org_rocksdb_IngestExternalFileOptions_snapshotConsistency(
JNIEnv* env, jobject jobj, jlong jhandle) {
auto* options =
reinterpret_cast<rocksdb::IngestExternalFileOptions*>(jhandle);
return static_cast<jboolean>(options->snapshot_consistency);
}
/*
* Class: org_rocksdb_IngestExternalFileOptions
* Method: setSnapshotConsistency
* Signature: (JZ)V
*/
void Java_org_rocksdb_IngestExternalFileOptions_setSnapshotConsistency(
JNIEnv* env, jobject jobj, jlong jhandle,
jboolean jsnapshot_consistency) {
auto* options =
reinterpret_cast<rocksdb::IngestExternalFileOptions*>(jhandle);
options->snapshot_consistency = static_cast<bool>(jsnapshot_consistency);
}
/*
* Class: org_rocksdb_IngestExternalFileOptions
* Method: allowGlobalSeqNo
* Signature: (J)Z
*/
jboolean Java_org_rocksdb_IngestExternalFileOptions_allowGlobalSeqNo(
JNIEnv* env, jobject jobj, jlong jhandle) {
auto* options =
reinterpret_cast<rocksdb::IngestExternalFileOptions*>(jhandle);
return static_cast<jboolean>(options->allow_global_seqno);
}
/*
* Class: org_rocksdb_IngestExternalFileOptions
* Method: setAllowGlobalSeqNo
* Signature: (JZ)V
*/
void Java_org_rocksdb_IngestExternalFileOptions_setAllowGlobalSeqNo(
JNIEnv* env, jobject jobj, jlong jhandle, jboolean jallow_global_seqno) {
auto* options =
reinterpret_cast<rocksdb::IngestExternalFileOptions*>(jhandle);
options->allow_global_seqno = static_cast<bool>(jallow_global_seqno);
}
/*
* Class: org_rocksdb_IngestExternalFileOptions
* Method: allowBlockingFlush
* Signature: (J)Z
*/
jboolean Java_org_rocksdb_IngestExternalFileOptions_allowBlockingFlush(
JNIEnv* env, jobject jobj, jlong jhandle) {
auto* options =
reinterpret_cast<rocksdb::IngestExternalFileOptions*>(jhandle);
return static_cast<jboolean>(options->allow_blocking_flush);
}
/*
* Class: org_rocksdb_IngestExternalFileOptions
* Method: setAllowBlockingFlush
* Signature: (JZ)V
*/
void Java_org_rocksdb_IngestExternalFileOptions_setAllowBlockingFlush(
JNIEnv* env, jobject jobj, jlong jhandle, jboolean jallow_blocking_flush) {
auto* options =
reinterpret_cast<rocksdb::IngestExternalFileOptions*>(jhandle);
options->allow_blocking_flush = static_cast<bool>(jallow_blocking_flush);
}
/*
* Class: org_rocksdb_IngestExternalFileOptions
* Method: disposeInternal
* Signature: (J)V
*/
void Java_org_rocksdb_IngestExternalFileOptions_disposeInternal(
JNIEnv* env, jobject jobj, jlong jhandle) {
auto* options =
reinterpret_cast<rocksdb::IngestExternalFileOptions*>(jhandle);
delete options;
}

21
java/rocksjni/rocksjni.cc
View File

@ -2164,18 +2164,17 @@ void Java_org_rocksdb_RocksDB_setOptions(JNIEnv* env, jobject jdb,
}
//////////////////////////////////////////////////////////////////////////////
// rocksdb::DB::AddFile
// rocksdb::DB::IngestExternalFile
/*
* Class: org_rocksdb_RocksDB
* Method: addFile
* Signature: (JJ[Ljava/lang/String;IZ)V
* Method: ingestExternalFile
* Signature: (JJ[Ljava/lang/String;IJ)V
*/
void Java_org_rocksdb_RocksDB_addFile__JJ_3Ljava_lang_String_2IZ(
void Java_org_rocksdb_RocksDB_ingestExternalFile(
JNIEnv* env, jobject jdb, jlong jdb_handle, jlong jcf_handle,
jobjectArray jfile_path_list, jint jfile_path_list_len,
jboolean jmove_file) {
jlong jingest_external_file_options_handle) {
jboolean has_exception = JNI_FALSE;
std::vector<std::string> file_path_list =
rocksdb::JniUtil::copyStrings(env, jfile_path_list, jfile_path_list_len,
@ -2188,13 +2187,11 @@ void Java_org_rocksdb_RocksDB_addFile__JJ_3Ljava_lang_String_2IZ(
auto* db = reinterpret_cast<rocksdb::DB*>(jdb_handle);
auto* column_family =
reinterpret_cast<rocksdb::ColumnFamilyHandle*>(jcf_handle);
rocksdb::IngestExternalFileOptions ifo;
ifo.move_files = static_cast<bool>(jmove_file);
ifo.snapshot_consistency = true;
ifo.allow_global_seqno = false;
ifo.allow_blocking_flush = false;
auto* ifo =
reinterpret_cast<rocksdb::IngestExternalFileOptions*>(
jingest_external_file_options_handle);
rocksdb::Status s =
db->IngestExternalFile(column_family, file_path_list, ifo);
db->IngestExternalFile(column_family, file_path_list, *ifo);
if (!s.ok()) {
rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
}

135
java/src/main/java/org/rocksdb/ExternalSstFileInfo.java
View File

@ -1,135 +0,0 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
package org.rocksdb;
import java.io.File;
public class ExternalSstFileInfo extends RocksObject {
public ExternalSstFileInfo() {
super(newExternalSstFileInfo());
}
public ExternalSstFileInfo(final String filePath, final String smallestKey,
final String largestKey, final long sequenceNumber, final long fileSize, final int numEntries,
final int version) {
super(newExternalSstFileInfo(ensureNotNullFilePath(filePath),
ensureNotNullSmallestKey(smallestKey), ensureNotNullLargestKey(largestKey), sequenceNumber,
fileSize, numEntries, version));
}
private static String ensureNotNullFilePath(final String filePath) {
if (filePath == null) {
throw new NullPointerException("filePath is null.");
}
return filePath;
}
private static String ensureNotNullSmallestKey(final String smallestKey) {
if (smallestKey == null) {
throw new NullPointerException("smallestKey is null.");
}
return smallestKey;
}
private static String ensureNotNullLargestKey(final String largestKey) {
if (largestKey == null) {
throw new NullPointerException("largestKey is null.");
}
return largestKey;
}
public void setFilePath(final String filePath) {
setFilePath(nativeHandle_, filePath);
}
public String filePath() {
return filePath(nativeHandle_);
}
public void setSmallestKey(final String smallestKey) {
setSmallestKey(nativeHandle_, smallestKey);
}
public String smallestKey() {
return smallestKey(nativeHandle_);
}
public void setLargestKey(final String largestKey) {
setLargestKey(nativeHandle_, largestKey);
}
public String largestKey() {
return largestKey(nativeHandle_);
}
public void setSequenceNumber(final long sequenceNumber) {
setSequenceNumber(nativeHandle_, sequenceNumber);
}
public long sequenceNumber() {
return sequenceNumber(nativeHandle_);
}
public void setFileSize(final long fileSize) {
setFileSize(nativeHandle_, fileSize);
}
public long fileSize() {
return fileSize(nativeHandle_);
}
public void setNumEntries(final int numEntries) {
setNumEntries(nativeHandle_, numEntries);
}
public int numEntries() {
return numEntries(nativeHandle_);
}
public void setVersion(final int version) {
setVersion(nativeHandle_, version);
}
public int version() {
return version(nativeHandle_);
}
private native static long newExternalSstFileInfo();
private native static long newExternalSstFileInfo(final String filePath, final String smallestKey,
final String largestKey, final long sequenceNumber, final long fileSize, final int numEntries,
final int version);
private native void setFilePath(final long handle, final String filePath);
private native String filePath(final long handle);
private native void setSmallestKey(final long handle, final String smallestKey);
private native String smallestKey(final long handle);
private native void setLargestKey(final long handle, final String largestKey);
private native String largestKey(final long handle);
private native void setSequenceNumber(final long handle, final long sequenceNumber);
private native long sequenceNumber(final long handle);
private native void setFileSize(final long handle, final long fileSize);
private native long fileSize(final long handle);
private native void setNumEntries(final long handle, final int numEntries);
private native int numEntries(final long handle);
private native void setVersion(final long handle, final int version);
private native int version(final long handle);
@Override protected final native void disposeInternal(final long handle);
}

125
java/src/main/java/org/rocksdb/IngestExternalFileOptions.java Normal file
View File

@ -0,0 +1,125 @@
package org.rocksdb;
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
import java.util.List;
/**
* IngestExternalFileOptions is used by {@link RocksDB#ingestExternalFile(ColumnFamilyHandle, List, IngestExternalFileOptions)}
*/
public class IngestExternalFileOptions extends RocksObject {
public IngestExternalFileOptions() {
super(newIngestExternalFileOptions());
}
/**
* @param moveFiles {@link #setMoveFiles(boolean)}
* @param snapshotConsistency {@link #setSnapshotConsistency(boolean)}
* @param allowGlobalSeqNo {@link #setAllowGlobalSeqNo(boolean)}
* @param allowBlockingFlush {@link #setAllowBlockingFlush(boolean)}
*/
public IngestExternalFileOptions(final boolean moveFiles,
final boolean snapshotConsistency, final boolean allowGlobalSeqNo,
final boolean allowBlockingFlush) {
super(newIngestExternalFileOptions(moveFiles, snapshotConsistency,
allowGlobalSeqNo, allowBlockingFlush));
}
/**
* Can be set to true to move the files instead of copying them.
*
* @return true if files will be moved
*/
public boolean moveFiles() {
return moveFiles(nativeHandle_);
}
/**
* Can be set to true to move the files instead of copying them.
*
* @param moveFiles true if files should be moved instead of copied
*/
public void setMoveFiles(final boolean moveFiles) {
setMoveFiles(nativeHandle_, moveFiles);
}
/**
* If set to false, an ingested file keys could appear in existing snapshots
* that where created before the file was ingested.
*
* @return true if snapshot consistency is assured
*/
public boolean snapshotConsistency() {
return snapshotConsistency(nativeHandle_);
}
/**
* If set to false, an ingested file keys could appear in existing snapshots
* that where created before the file was ingested.
*
* @param snapshotConsistency true if snapshot consistency is required
*/
public void setSnapshotConsistency(final boolean snapshotConsistency) {
setSnapshotConsistency(nativeHandle_, snapshotConsistency);
}
/**
* If set to false, {@link RocksDB#ingestExternalFile(ColumnFamilyHandle, List, IngestExternalFileOptions)}
* will fail if the file key range overlaps with existing keys or tombstones in the DB.
*
* @return true if global seq numbers are assured
*/
public boolean allowGlobalSeqNo() {
return allowGlobalSeqNo(nativeHandle_);
}
/**
* If set to false, {@link RocksDB#ingestExternalFile(ColumnFamilyHandle, List, IngestExternalFileOptions)}
* will fail if the file key range overlaps with existing keys or tombstones in the DB.
*
* @param allowGlobalSeqNo true if global seq numbers are required
*/
public void setAllowGlobalSeqNo(final boolean allowGlobalSeqNo) {
setAllowGlobalSeqNo(nativeHandle_, allowGlobalSeqNo);
}
/**
* If set to false and the file key range overlaps with the memtable key range
* (memtable flush required), IngestExternalFile will fail.
*
* @return true if blocking flushes may occur
*/
public boolean allowBlockingFlush() {
return allowBlockingFlush(nativeHandle_);
}
/**
* If set to false and the file key range overlaps with the memtable key range
* (memtable flush required), IngestExternalFile will fail.
*
* @param allowBlockingFlush true if blocking flushes are allowed
*/
public void setAllowBlockingFlush(final boolean allowBlockingFlush) {
setAllowBlockingFlush(nativeHandle_, allowBlockingFlush);
}
private native static long newIngestExternalFileOptions();
private native static long newIngestExternalFileOptions(
final boolean moveFiles, final boolean snapshotConsistency,
final boolean allowGlobalSeqNo, final boolean allowBlockingFlush);
private native boolean moveFiles(final long handle);
private native void setMoveFiles(final long handle, final boolean move_files);
private native boolean snapshotConsistency(final long handle);
private native void setSnapshotConsistency(final long handle,
final boolean snapshotConsistency);
private native boolean allowGlobalSeqNo(final long handle);
private native void setAllowGlobalSeqNo(final long handle,
final boolean allowGloablSeqNo);
private native boolean allowBlockingFlush(final long handle);
private native void setAllowBlockingFlush(final long handle,
final boolean allowBlockingFlush);
@Override protected final native void disposeInternal(final long handle);
}

137
java/src/main/java/org/rocksdb/RocksDB.java
View File

@ -2122,100 +2122,50 @@ public class RocksDB extends RocksObject {
return handleList;
}
public void addFileWithFilePath(final ColumnFamilyHandle columnFamilyHandle,
final List<String> filePathList) throws RocksDBException {
addFile(nativeHandle_, columnFamilyHandle.nativeHandle_,
filePathList.toArray(new String[filePathList.size()]), filePathList.size(), false);
}
public void addFileWithFilePath(final ColumnFamilyHandle columnFamilyHandle,
final List<String> filePathList, final boolean moveFile) throws RocksDBException {
addFile(nativeHandle_, columnFamilyHandle.nativeHandle_,
filePathList.toArray(new String[filePathList.size()]), filePathList.size(), moveFile);
}
public void addFileWithFilePath(final List<String> filePathList) throws RocksDBException {
addFile(nativeHandle_, getDefaultColumnFamily().nativeHandle_,
filePathList.toArray(new String[filePathList.size()]), filePathList.size(), false);
}
public void addFileWithFilePath(final List<String> filePathList, final boolean moveFile)
/**
* ingestExternalFile will load a list of external SST files (1) into the DB
* We will try to find the lowest possible level that the file can fit in, and
* ingest the file into this level (2). A file that have a key range that
* overlap with the memtable key range will require us to Flush the memtable
* first before ingesting the file.
*
* (1) External SST files can be created using {@link SstFileWriter}
* (2) We will try to ingest the files to the lowest possible level
* even if the file compression doesn't match the level compression
*
* @param filePathList The list of files to ingest
* @param ingestExternalFileOptions the options for the ingestion
*/
public void ingestExternalFile(final List<String> filePathList,
final IngestExternalFileOptions ingestExternalFileOptions)
throws RocksDBException {
addFile(nativeHandle_, getDefaultColumnFamily().nativeHandle_,
filePathList.toArray(new String[filePathList.size()]), filePathList.size(), moveFile);
ingestExternalFile(nativeHandle_, getDefaultColumnFamily().nativeHandle_,
filePathList.toArray(new String[filePathList.size()]),
filePathList.size(), ingestExternalFileOptions.nativeHandle_);
}
public void addFileWithFilePath(
final ColumnFamilyHandle columnFamilyHandle, final String filePath) throws RocksDBException {
addFile(nativeHandle_, columnFamilyHandle.nativeHandle_, new String[] {filePath}, 1, false);
}
public void addFileWithFilePath(final ColumnFamilyHandle columnFamilyHandle,
final String filePath, final boolean moveFile) throws RocksDBException {
addFile(nativeHandle_, columnFamilyHandle.nativeHandle_, new String[] {filePath}, 1, moveFile);
}
public void addFileWithFilePath(final String filePath) throws RocksDBException {
addFile(
nativeHandle_, getDefaultColumnFamily().nativeHandle_, new String[] {filePath}, 1, false);
}
public void addFileWithFilePath(final String filePath, final boolean moveFile)
/**
* ingestExternalFile will load a list of external SST files (1) into the DB
* We will try to find the lowest possible level that the file can fit in, and
* ingest the file into this level (2). A file that have a key range that
* overlap with the memtable key range will require us to Flush the memtable
* first before ingesting the file.
*
* (1) External SST files can be created using {@link SstFileWriter}
* (2) We will try to ingest the files to the lowest possible level
* even if the file compression doesn't match the level compression
*
* @param columnFamilyHandle The column family for the ingested files
* @param filePathList The list of files to ingest
* @param ingestExternalFileOptions the options for the ingestion
*/
public void ingestExternalFile(final ColumnFamilyHandle columnFamilyHandle,
final List<String> filePathList,
final IngestExternalFileOptions ingestExternalFileOptions)
throws RocksDBException {
addFile(nativeHandle_, getDefaultColumnFamily().nativeHandle_, new String[] {filePath}, 1,
moveFile);
}
public void addFileWithFileInfo(final ColumnFamilyHandle columnFamilyHandle,
final List<ExternalSstFileInfo> fileInfoList) throws RocksDBException {
final long[] fiHandleList = toNativeHandleList(fileInfoList);
addFile(
nativeHandle_, columnFamilyHandle.nativeHandle_, fiHandleList, fiHandleList.length, false);
}
public void addFileWithFileInfo(final ColumnFamilyHandle columnFamilyHandle,
final List<ExternalSstFileInfo> fileInfoList, final boolean moveFile)
throws RocksDBException {
final long[] fiHandleList = toNativeHandleList(fileInfoList);
addFile(nativeHandle_, columnFamilyHandle.nativeHandle_, fiHandleList, fiHandleList.length,
moveFile);
}
public void addFileWithFileInfo(final List<ExternalSstFileInfo> fileInfoList)
throws RocksDBException {
final long[] fiHandleList = toNativeHandleList(fileInfoList);
addFile(nativeHandle_, getDefaultColumnFamily().nativeHandle_, fiHandleList,
fiHandleList.length, false);
}
public void addFileWithFileInfo(final List<ExternalSstFileInfo> fileInfoList,
final boolean moveFile) throws RocksDBException {
final long[] fiHandleList = toNativeHandleList(fileInfoList);
addFile(nativeHandle_, getDefaultColumnFamily().nativeHandle_, fiHandleList,
fiHandleList.length, moveFile);
}
public void addFileWithFileInfo(final ColumnFamilyHandle columnFamilyHandle,
final ExternalSstFileInfo fileInfo) throws RocksDBException {
addFile(nativeHandle_, columnFamilyHandle.nativeHandle_, new long[] {fileInfo.nativeHandle_}, 1,
false);
}
public void addFileWithFileInfo(final ColumnFamilyHandle columnFamilyHandle,
final ExternalSstFileInfo fileInfo, final boolean moveFile) throws RocksDBException {
addFile(nativeHandle_, columnFamilyHandle.nativeHandle_, new long[] {fileInfo.nativeHandle_}, 1,
moveFile);
}
public void addFileWithFileInfo(final ExternalSstFileInfo fileInfo) throws RocksDBException {
addFile(nativeHandle_, getDefaultColumnFamily().nativeHandle_,
new long[] {fileInfo.nativeHandle_}, 1, false);
}
public void addFileWithFileInfo(final ExternalSstFileInfo fileInfo, final boolean moveFile)
throws RocksDBException {
addFile(nativeHandle_, getDefaultColumnFamily().nativeHandle_,
new long[] {fileInfo.nativeHandle_}, 1, moveFile);
ingestExternalFile(nativeHandle_, columnFamilyHandle.nativeHandle_,
filePathList.toArray(new String[filePathList.size()]),
filePathList.size(), ingestExternalFileOptions.nativeHandle_);
}
/**
@ -2421,9 +2371,8 @@ public class RocksDB extends RocksObject {
throws RocksDBException;
private native void setOptions(long handle, long cfHandle, String[] keys,
String[] values) throws RocksDBException;
private native void addFile(long handle, long cfHandle, String[] filePathList,
int filePathListLen, boolean moveFile) throws RocksDBException;
private native void addFile(long handle, long cfHandle, long[] fiHandleList, int fiHandleListLen,
boolean moveFile) throws RocksDBException;
private native void ingestExternalFile(long handle, long cfHandle,
String[] filePathList, int filePathListLen,
long ingest_external_file_options_handle) throws RocksDBException;
protected DBOptionsInterface options_;
}

105
java/src/test/java/org/rocksdb/ExternalSstFileInfoTest.java
View File

@ -1,105 +0,0 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
package org.rocksdb;
import org.junit.ClassRule;
import org.junit.Test;
import java.util.Random;
import static org.assertj.core.api.Assertions.assertThat;
public class ExternalSstFileInfoTest {
@ClassRule
public static final RocksMemoryResource rocksMemoryResource = new RocksMemoryResource();
public static final Random rand = PlatformRandomHelper.getPlatformSpecificRandomFactory();
@Test
public void createExternalSstFileInfoWithoutParameters() {
try (final ExternalSstFileInfo info = new ExternalSstFileInfo()) {
assertThat(info).isNotNull();
}
}
@Test
public void createExternalSstFileInfoWithParameters() {
final String filePath = "path/to/sst";
final String smallestKey = "min";
final String largestKey = "max";
final long sequenceNumber = rand.nextLong();
final long fileSize = rand.nextLong();
final int numEntries = rand.nextInt();
final int version = rand.nextInt();
try (final ExternalSstFileInfo info = new ExternalSstFileInfo(
filePath, smallestKey, largestKey, sequenceNumber, fileSize, numEntries, version)) {
assertThat(info).isNotNull();
}
}
@Test
public void filePath() {
try (final ExternalSstFileInfo info = new ExternalSstFileInfo()) {
final String stringVale = "path/to/sst";
info.setFilePath(stringVale);
assertThat(info.filePath()).isEqualTo(stringVale);
}
}
@Test
public void smallestKey() {
try (final ExternalSstFileInfo info = new ExternalSstFileInfo()) {
final String stringValue = "min";
info.setSmallestKey(stringValue);
assertThat(info.smallestKey()).isEqualTo(stringValue);
}
}
@Test
public void largestKey() {
try (final ExternalSstFileInfo info = new ExternalSstFileInfo()) {
final String stringValue = "max";
info.setLargestKey(stringValue);
assertThat(info.largestKey()).isEqualTo(stringValue);
}
}
@Test
public void sequenceNumber() {
try (final ExternalSstFileInfo info = new ExternalSstFileInfo()) {
final long longValue = rand.nextLong();
info.setSequenceNumber(longValue);
assertThat(info.sequenceNumber()).isEqualTo(longValue);
}
}
@Test
public void fileSize() {
try (final ExternalSstFileInfo info = new ExternalSstFileInfo()) {
final long longValue = Math.max(1, rand.nextLong());
info.setFileSize(longValue);
assertThat(info.fileSize()).isEqualTo(longValue);
}
}
@Test
public void numEntries() {
try (final ExternalSstFileInfo info = new ExternalSstFileInfo()) {
final int intValue = Math.max(1, rand.nextInt());
info.setNumEntries(intValue);
assertThat(info.numEntries()).isEqualTo(intValue);
}
}
@Test
public void version() {
try (final ExternalSstFileInfo info = new ExternalSstFileInfo()) {
final int intValue = rand.nextInt();
info.setVersion(intValue);
assertThat(info.version()).isEqualTo(intValue);
}
}
}

87
java/src/test/java/org/rocksdb/IngestExternalFileOptionsTest.java Normal file
View File

@ -0,0 +1,87 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
package org.rocksdb;
import org.junit.ClassRule;
import org.junit.Test;
import java.util.Random;
import static org.assertj.core.api.Assertions.assertThat;
public class IngestExternalFileOptionsTest {
@ClassRule
public static final RocksMemoryResource rocksMemoryResource
= new RocksMemoryResource();
public static final Random rand =
PlatformRandomHelper.getPlatformSpecificRandomFactory();
@Test
public void createExternalSstFileInfoWithoutParameters() {
try (final IngestExternalFileOptions options =
new IngestExternalFileOptions()) {
assertThat(options).isNotNull();
}
}
@Test
public void createExternalSstFileInfoWithParameters() {
final boolean moveFiles = rand.nextBoolean();
final boolean snapshotConsistency = rand.nextBoolean();
final boolean allowGlobalSeqNo = rand.nextBoolean();
final boolean allowBlockingFlush = rand.nextBoolean();
try (final IngestExternalFileOptions options =
new IngestExternalFileOptions(moveFiles, snapshotConsistency,
allowGlobalSeqNo, allowBlockingFlush)) {
assertThat(options).isNotNull();
assertThat(options.moveFiles()).isEqualTo(moveFiles);
assertThat(options.snapshotConsistency()).isEqualTo(snapshotConsistency);
assertThat(options.allowGlobalSeqNo()).isEqualTo(allowGlobalSeqNo);
assertThat(options.allowBlockingFlush()).isEqualTo(allowBlockingFlush);
}
}
@Test
public void moveFiles() {
try (final IngestExternalFileOptions options =
new IngestExternalFileOptions()) {
final boolean moveFiles = rand.nextBoolean();
options.setMoveFiles(moveFiles);
assertThat(options.moveFiles()).isEqualTo(moveFiles);
}
}
@Test
public void snapshotConsistency() {
try (final IngestExternalFileOptions options =
new IngestExternalFileOptions()) {
final boolean snapshotConsistency = rand.nextBoolean();
options.setSnapshotConsistency(snapshotConsistency);
assertThat(options.snapshotConsistency()).isEqualTo(snapshotConsistency);
}
}
@Test
public void allowGlobalSeqNo() {
try (final IngestExternalFileOptions options =
new IngestExternalFileOptions()) {
final boolean allowGlobalSeqNo = rand.nextBoolean();
options.setAllowGlobalSeqNo(allowGlobalSeqNo);
assertThat(options.allowGlobalSeqNo()).isEqualTo(allowGlobalSeqNo);
}
}
@Test
public void allowBlockingFlush() {
try (final IngestExternalFileOptions options =
new IngestExternalFileOptions()) {
final boolean allowBlockingFlush = rand.nextBoolean();
options.setAllowBlockingFlush(allowBlockingFlush);
assertThat(options.allowBlockingFlush()).isEqualTo(allowBlockingFlush);
}
}
}

55
java/src/test/java/org/rocksdb/SstFileWriterTest.java
View File

@ -13,6 +13,7 @@ import org.rocksdb.util.BytewiseComparator;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.Map;
import java.util.TreeMap;
@ -23,7 +24,8 @@ public class SstFileWriterTest {
private static final String DB_DIRECTORY_NAME = "test_db";
@ClassRule
public static final RocksMemoryResource rocksMemoryResource = new RocksMemoryResource();
public static final RocksMemoryResource rocksMemoryResource
= new RocksMemoryResource();
@Rule public TemporaryFolder parentFolder = new TemporaryFolder();
@ -71,7 +73,8 @@ public class SstFileWriterTest {
}
@Test
public void generateSstFileWithJavaComparator() throws RocksDBException, IOException {
public void generateSstFileWithJavaComparator()
throws RocksDBException, IOException {
final TreeMap<String, String> keyValues = new TreeMap<>();
keyValues.put("key1", "value1");
keyValues.put("key2", "value2");
@ -79,7 +82,8 @@ public class SstFileWriterTest {
}
@Test
public void generateSstFileWithNativeComparator() throws RocksDBException, IOException {
public void generateSstFileWithNativeComparator()
throws RocksDBException, IOException {
final TreeMap<String, String> keyValues = new TreeMap<>();
keyValues.put("key1", "value1");
keyValues.put("key2", "value2");
@ -93,14 +97,45 @@ public class SstFileWriterTest {
keyValues.put("key2", "value2");
final File sstFile = newSstFile(keyValues, false);
final File dbFolder = parentFolder.newFolder(DB_DIRECTORY_NAME);
final Options options = new Options().setCreateIfMissing(true);
final RocksDB db = RocksDB.open(options, dbFolder.getAbsolutePath());
db.addFileWithFilePath(sstFile.getAbsolutePath());
try(final Options options = new Options().setCreateIfMissing(true);
final RocksDB db = RocksDB.open(options, dbFolder.getAbsolutePath());
final IngestExternalFileOptions ingestExternalFileOptions
= new IngestExternalFileOptions()) {
db.ingestExternalFile(Arrays.asList(sstFile.getAbsolutePath()),
ingestExternalFileOptions);
assertThat(db.get("key1".getBytes())).isEqualTo("value1".getBytes());
assertThat(db.get("key2".getBytes())).isEqualTo("value2".getBytes());
assertThat(db.get("key1".getBytes())).isEqualTo("value1".getBytes());
assertThat(db.get("key2".getBytes())).isEqualTo("value2".getBytes());
}
}
options.close();
db.close();
@Test
public void ingestSstFile_cf() throws RocksDBException, IOException {
final TreeMap<String, String> keyValues = new TreeMap<>();
keyValues.put("key1", "value1");
keyValues.put("key2", "value2");
final File sstFile = newSstFile(keyValues, false);
final File dbFolder = parentFolder.newFolder(DB_DIRECTORY_NAME);
try(final Options options = new Options()
.setCreateIfMissing(true)
.setCreateMissingColumnFamilies(true);
final RocksDB db = RocksDB.open(options, dbFolder.getAbsolutePath());
final IngestExternalFileOptions ingestExternalFileOptions =
new IngestExternalFileOptions()) {
try(final ColumnFamilyOptions cf_opts = new ColumnFamilyOptions();
final ColumnFamilyHandle cf_handle = db.createColumnFamily(
new ColumnFamilyDescriptor("new_cf".getBytes(), cf_opts))) {
db.ingestExternalFile(cf_handle,
Arrays.asList(sstFile.getAbsolutePath()),
ingestExternalFileOptions);
assertThat(db.get(cf_handle,
"key1".getBytes())).isEqualTo("value1".getBytes());
assertThat(db.get(cf_handle,
"key2".getBytes())).isEqualTo("value2".getBytes());
}
}
}
}

2
src.mk
View File

@ -355,7 +355,7 @@ JNI_NATIVE_SOURCES = \
java/rocksjni/compression_options.cc \
java/rocksjni/env.cc \
java/rocksjni/env_options.cc \
java/rocksjni/external_sst_file_info.cc \
java/rocksjni/ingest_external_file_options.cc \
java/rocksjni/filter.cc \
java/rocksjni/iterator.cc \
java/rocksjni/loggerjnicallback.cc \