From 88c818e4375a22ac3bd61c15e63c81318d158e47 Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Mon, 22 May 2017 10:22:49 -0700 Subject: [PATCH] Replace deprecated RocksDB#addFile with RocksDB#ingestExternalFile Summary: Previously the Java implementation of `RocksDB#addFile` was both incomplete and not inline with the C++ API. Rather than fix it, as I see that `rocksdb::DB::AddFile` is now deprecated in favour of `rocksdb::DB::IngestExternalFile`, I have removed the old broken implementation and implemented `RocksDB#ingestExternalFile`. Closes https://github.com/facebook/rocksdb/issues/2261 Closes https://github.com/facebook/rocksdb/pull/2291 Differential Revision: D5061264 Pulled By: sagar0 fbshipit-source-id: 85df0899fa1b1fc3535175cac4f52353511d4104 --- java/Makefile | 4 +- java/rocksjni/external_sst_file_info.cc | 274 ------------------ java/rocksjni/ingest_external_file_options.cc | 149 ++++++++++ java/rocksjni/rocksjni.cc | 21 +- .../java/org/rocksdb/ExternalSstFileInfo.java | 135 --------- .../rocksdb/IngestExternalFileOptions.java | 125 ++++++++ java/src/main/java/org/rocksdb/RocksDB.java | 137 +++------ .../org/rocksdb/ExternalSstFileInfoTest.java | 105 ------- .../IngestExternalFileOptionsTest.java | 87 ++++++ .../java/org/rocksdb/SstFileWriterTest.java | 55 +++- src.mk | 2 +- 11 files changed, 461 insertions(+), 633 deletions(-) delete mode 100644 java/rocksjni/external_sst_file_info.cc create mode 100644 java/rocksjni/ingest_external_file_options.cc delete mode 100644 java/src/main/java/org/rocksdb/ExternalSstFileInfo.java create mode 100644 java/src/main/java/org/rocksdb/IngestExternalFileOptions.java delete mode 100644 java/src/test/java/org/rocksdb/ExternalSstFileInfoTest.java create mode 100644 java/src/test/java/org/rocksdb/IngestExternalFileOptionsTest.java diff --git a/java/Makefile b/java/Makefile index ab20546020..b64955a07e 100644 --- a/java/Makefile +++ b/java/Makefile @@ -19,9 +19,9 @@ NATIVE_JAVA_CLASSES = org.rocksdb.AbstractCompactionFilter\ org.rocksdb.DirectSlice\ org.rocksdb.Env\ org.rocksdb.EnvOptions\ - org.rocksdb.ExternalSstFileInfo\ org.rocksdb.FlushOptions\ org.rocksdb.Filter\ + org.rocksdb.IngestExternalFileOptions\ org.rocksdb.HashLinkedListMemTableConfig\ org.rocksdb.HashSkipListMemTableConfig\ org.rocksdb.Logger\ @@ -87,7 +87,7 @@ JAVA_TESTS = org.rocksdb.BackupableDBOptionsTest\ org.rocksdb.DirectComparatorTest\ org.rocksdb.DirectSliceTest\ org.rocksdb.EnvOptionsTest\ - org.rocksdb.ExternalSstFileInfoTest\ + org.rocksdb.IngestExternalFileOptionsTest\ org.rocksdb.util.EnvironmentTest\ org.rocksdb.FilterTest\ org.rocksdb.FlushTest\ diff --git a/java/rocksjni/external_sst_file_info.cc b/java/rocksjni/external_sst_file_info.cc deleted file mode 100644 index 3bbcd4801c..0000000000 --- a/java/rocksjni/external_sst_file_info.cc +++ /dev/null @@ -1,274 +0,0 @@ -// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. -// -// This file implements the "bridge" between Java and C++ and enables -// calling C++ rocksdb::ExternalSstFileInfo methods -// from Java side. - -#include - -#include "include/org_rocksdb_ExternalSstFileInfo.h" -#include "rocksdb/sst_file_writer.h" - -/* - * Class: org_rocksdb_ExternalSstFileInfo - * Method: newExternalSstFileInfo - * Signature: ()J - */ -jlong Java_org_rocksdb_ExternalSstFileInfo_newExternalSstFileInfo__( - JNIEnv *env, jclass jcls) { - return reinterpret_cast(new rocksdb::ExternalSstFileInfo()); -} - -/* - * Class: org_rocksdb_ExternalSstFileInfo - * Method: newExternalSstFileInfo - * Signature: (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;JJII)J - */ -jlong Java_org_rocksdb_ExternalSstFileInfo_newExternalSstFileInfo__Ljava_lang_String_2Ljava_lang_String_2Ljava_lang_String_2JJII( - JNIEnv *env, jclass jcls, jstring jfile_path, jstring jsmallest_key, - jstring jlargest_key, jlong jsequence_number, jlong jfile_size, - jint jnum_entries, jint jversion) { - const char *file_path = env->GetStringUTFChars(jfile_path, nullptr); - if(file_path == nullptr) { - // exception thrown: OutOfMemoryError - return 0; - } - const char *smallest_key = env->GetStringUTFChars(jsmallest_key, nullptr); - if(smallest_key == nullptr) { - // exception thrown: OutOfMemoryError - env->ReleaseStringUTFChars(jfile_path, file_path); - return 0; - } - const char *largest_key = env->GetStringUTFChars(jlargest_key, nullptr); - if(largest_key == nullptr) { - // exception thrown: OutOfMemoryError - env->ReleaseStringUTFChars(jsmallest_key, smallest_key); - env->ReleaseStringUTFChars(jfile_path, file_path); - return 0; - } - - auto *external_sst_file_info = new rocksdb::ExternalSstFileInfo( - file_path, smallest_key, largest_key, - static_cast(jsequence_number), - static_cast(jfile_size), static_cast(jnum_entries), - static_cast(jversion)); - - env->ReleaseStringUTFChars(jlargest_key, largest_key); - env->ReleaseStringUTFChars(jsmallest_key, smallest_key); - env->ReleaseStringUTFChars(jfile_path, file_path); - - return reinterpret_cast(external_sst_file_info); -} - -/* - * Class: org_rocksdb_ExternalSstFileInfo - * Method: setFilePath - * Signature: (JLjava/lang/String;)V - */ -void Java_org_rocksdb_ExternalSstFileInfo_setFilePath(JNIEnv *env, jobject jobj, - jlong jhandle, - jstring jfile_path) { - auto *external_sst_file_info = - reinterpret_cast(jhandle); - const char *file_path = env->GetStringUTFChars(jfile_path, nullptr); - if(file_path == nullptr) { - // exception thrown: OutOfMemoryError - return; - } - external_sst_file_info->file_path = file_path; - env->ReleaseStringUTFChars(jfile_path, file_path); -} - -/* - * Class: org_rocksdb_ExternalSstFileInfo - * Method: filePath - * Signature: (J)Ljava/lang/String; - */ -jstring Java_org_rocksdb_ExternalSstFileInfo_filePath(JNIEnv *env, jobject jobj, - jlong jhandle) { - auto *external_sst_file_info = - reinterpret_cast(jhandle); - return env->NewStringUTF(external_sst_file_info->file_path.data()); -} - -/* - * Class: org_rocksdb_ExternalSstFileInfo - * Method: setSmallestKey - * Signature: (JLjava/lang/String;)V - */ -void Java_org_rocksdb_ExternalSstFileInfo_setSmallestKey( - JNIEnv *env, jobject jobj, jlong jhandle, jstring jsmallest_key) { - auto *external_sst_file_info = - reinterpret_cast(jhandle); - const char *smallest_key = env->GetStringUTFChars(jsmallest_key, nullptr); - if(smallest_key == nullptr) { - // exception thrown: OutOfMemoryError - return; - } - external_sst_file_info->smallest_key = smallest_key; - env->ReleaseStringUTFChars(jsmallest_key, smallest_key); -} - -/* - * Class: org_rocksdb_ExternalSstFileInfo - * Method: smallestKey - * Signature: (J)Ljava/lang/String; - */ -jstring Java_org_rocksdb_ExternalSstFileInfo_smallestKey(JNIEnv *env, - jobject jobj, - jlong jhandle) { - auto *external_sst_file_info = - reinterpret_cast(jhandle); - return env->NewStringUTF(external_sst_file_info->smallest_key.data()); -} - -/* - * Class: org_rocksdb_ExternalSstFileInfo - * Method: setLargestKey - * Signature: (JLjava/lang/String;)V - */ -void Java_org_rocksdb_ExternalSstFileInfo_setLargestKey(JNIEnv *env, - jobject jobj, - jlong jhandle, - jstring jlargest_key) { - auto *external_sst_file_info = - reinterpret_cast(jhandle); - const char *largest_key = env->GetStringUTFChars(jlargest_key, NULL); - if(largest_key == nullptr) { - // exception thrown: OutOfMemoryError - return; - } - external_sst_file_info->largest_key = largest_key; - env->ReleaseStringUTFChars(jlargest_key, largest_key); -} - -/* - * Class: org_rocksdb_ExternalSstFileInfo - * Method: largestKey - * Signature: (J)Ljava/lang/String; - */ -jstring Java_org_rocksdb_ExternalSstFileInfo_largestKey(JNIEnv *env, - jobject jobj, - jlong jhandle) { - auto *external_sst_file_info = - reinterpret_cast(jhandle); - return env->NewStringUTF(external_sst_file_info->largest_key.data()); -} - -/* - * Class: org_rocksdb_ExternalSstFileInfo - * Method: setSequenceNumber - * Signature: (JJ)V - */ -void Java_org_rocksdb_ExternalSstFileInfo_setSequenceNumber( - JNIEnv *env, jobject jobj, jlong jhandle, jlong jsequence_number) { - auto *external_sst_file_info = - reinterpret_cast(jhandle); - external_sst_file_info->sequence_number = - static_cast(jsequence_number); -} - -/* - * Class: org_rocksdb_ExternalSstFileInfo - * Method: sequenceNumber - * Signature: (J)J - */ -jlong Java_org_rocksdb_ExternalSstFileInfo_sequenceNumber(JNIEnv *env, - jobject jobj, - jlong jhandle) { - auto *external_sst_file_info = - reinterpret_cast(jhandle); - return static_cast(external_sst_file_info->sequence_number); -} - -/* - * Class: org_rocksdb_ExternalSstFileInfo - * Method: setFileSize - * Signature: (JJ)V - */ -void Java_org_rocksdb_ExternalSstFileInfo_setFileSize(JNIEnv *env, jobject jobj, - jlong jhandle, - jlong jfile_size) { - auto *external_sst_file_info = - reinterpret_cast(jhandle); - external_sst_file_info->file_size = static_cast(jfile_size); -} - -/* - * Class: org_rocksdb_ExternalSstFileInfo - * Method: fileSize - * Signature: (J)J - */ -jlong Java_org_rocksdb_ExternalSstFileInfo_fileSize(JNIEnv *env, jobject jobj, - jlong jhandle) { - auto *external_sst_file_info = - reinterpret_cast(jhandle); - return static_cast(external_sst_file_info->file_size); -} - -/* - * Class: org_rocksdb_ExternalSstFileInfo - * Method: setNumEntries - * Signature: (JI)V - */ -void Java_org_rocksdb_ExternalSstFileInfo_setNumEntries(JNIEnv *env, - jobject jobj, - jlong jhandle, - jint jnum_entries) { - auto *external_sst_file_info = - reinterpret_cast(jhandle); - external_sst_file_info->num_entries = static_cast(jnum_entries); -} - -/* - * Class: org_rocksdb_ExternalSstFileInfo - * Method: numEntries - * Signature: (J)I - */ -jint Java_org_rocksdb_ExternalSstFileInfo_numEntries(JNIEnv *env, jobject jobj, - jlong jhandle) { - auto *external_sst_file_info = - reinterpret_cast(jhandle); - return static_cast(external_sst_file_info->num_entries); -} - -/* - * Class: org_rocksdb_ExternalSstFileInfo - * Method: setVersion - * Signature: (JI)V - */ -void Java_org_rocksdb_ExternalSstFileInfo_setVersion(JNIEnv *env, jobject jobj, - jlong jhandle, - jint jversion) { - auto *external_sst_file_info = - reinterpret_cast(jhandle); - external_sst_file_info->version = static_cast(jversion); -} - -/* - * Class: org_rocksdb_ExternalSstFileInfo - * Method: version - * Signature: (J)I - */ -jint Java_org_rocksdb_ExternalSstFileInfo_version(JNIEnv *env, jobject jobj, - jlong jhandle) { - auto *external_sst_file_info = - reinterpret_cast(jhandle); - return static_cast(external_sst_file_info->version); -} - -/* - * Class: org_rocksdb_ExternalSstFileInfo - * Method: disposeInternal - * Signature: (J)V - */ -void Java_org_rocksdb_ExternalSstFileInfo_disposeInternal(JNIEnv *env, - jobject jobj, - jlong jhandle) { - auto* esfi = reinterpret_cast(jhandle); - assert(esfi != nullptr); - delete esfi; -} diff --git a/java/rocksjni/ingest_external_file_options.cc b/java/rocksjni/ingest_external_file_options.cc new file mode 100644 index 0000000000..006ce14e42 --- /dev/null +++ b/java/rocksjni/ingest_external_file_options.cc @@ -0,0 +1,149 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. +// +// This file implements the "bridge" between Java and C++ for +// rocksdb::FilterPolicy. + +#include + +#include "include/org_rocksdb_IngestExternalFileOptions.h" +#include "rocksdb/options.h" + +/* + * Class: org_rocksdb_IngestExternalFileOptions + * Method: newIngestExternalFileOptions + * Signature: ()J + */ +jlong Java_org_rocksdb_IngestExternalFileOptions_newIngestExternalFileOptions__( + JNIEnv* env, jclass jclazz) { + auto* options = new rocksdb::IngestExternalFileOptions(); + return reinterpret_cast(options); +} + +/* + * Class: org_rocksdb_IngestExternalFileOptions + * Method: newIngestExternalFileOptions + * Signature: (ZZZZ)J + */ +jlong Java_org_rocksdb_IngestExternalFileOptions_newIngestExternalFileOptions__ZZZZ( + JNIEnv* env, jclass jcls, jboolean jmove_files, + jboolean jsnapshot_consistency, jboolean jallow_global_seqno, + jboolean jallow_blocking_flush) { + auto* options = new rocksdb::IngestExternalFileOptions(); + options->move_files = static_cast(jmove_files); + options->snapshot_consistency = static_cast(jsnapshot_consistency); + options->allow_global_seqno = static_cast(jallow_global_seqno); + options->allow_blocking_flush = static_cast(jallow_blocking_flush); + return reinterpret_cast(options); +} + +/* + * Class: org_rocksdb_IngestExternalFileOptions + * Method: moveFiles + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_IngestExternalFileOptions_moveFiles( + JNIEnv* env, jobject jobj, jlong jhandle) { + auto* options = + reinterpret_cast(jhandle); + return static_cast(options->move_files); +} + +/* + * Class: org_rocksdb_IngestExternalFileOptions + * Method: setMoveFiles + * Signature: (JZ)V + */ +void Java_org_rocksdb_IngestExternalFileOptions_setMoveFiles( + JNIEnv* env, jobject jobj, jlong jhandle, jboolean jmove_files) { + auto* options = + reinterpret_cast(jhandle); + options->move_files = static_cast(jmove_files); +} + +/* + * Class: org_rocksdb_IngestExternalFileOptions + * Method: snapshotConsistency + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_IngestExternalFileOptions_snapshotConsistency( + JNIEnv* env, jobject jobj, jlong jhandle) { + auto* options = + reinterpret_cast(jhandle); + return static_cast(options->snapshot_consistency); +} + +/* + * Class: org_rocksdb_IngestExternalFileOptions + * Method: setSnapshotConsistency + * Signature: (JZ)V + */ +void Java_org_rocksdb_IngestExternalFileOptions_setSnapshotConsistency( + JNIEnv* env, jobject jobj, jlong jhandle, + jboolean jsnapshot_consistency) { + auto* options = + reinterpret_cast(jhandle); + options->snapshot_consistency = static_cast(jsnapshot_consistency); +} + +/* + * Class: org_rocksdb_IngestExternalFileOptions + * Method: allowGlobalSeqNo + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_IngestExternalFileOptions_allowGlobalSeqNo( + JNIEnv* env, jobject jobj, jlong jhandle) { + auto* options = + reinterpret_cast(jhandle); + return static_cast(options->allow_global_seqno); +} + +/* + * Class: org_rocksdb_IngestExternalFileOptions + * Method: setAllowGlobalSeqNo + * Signature: (JZ)V + */ +void Java_org_rocksdb_IngestExternalFileOptions_setAllowGlobalSeqNo( + JNIEnv* env, jobject jobj, jlong jhandle, jboolean jallow_global_seqno) { + auto* options = + reinterpret_cast(jhandle); + options->allow_global_seqno = static_cast(jallow_global_seqno); +} + +/* + * Class: org_rocksdb_IngestExternalFileOptions + * Method: allowBlockingFlush + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_IngestExternalFileOptions_allowBlockingFlush( + JNIEnv* env, jobject jobj, jlong jhandle) { + auto* options = + reinterpret_cast(jhandle); + return static_cast(options->allow_blocking_flush); +} + +/* + * Class: org_rocksdb_IngestExternalFileOptions + * Method: setAllowBlockingFlush + * Signature: (JZ)V + */ +void Java_org_rocksdb_IngestExternalFileOptions_setAllowBlockingFlush( + JNIEnv* env, jobject jobj, jlong jhandle, jboolean jallow_blocking_flush) { + auto* options = + reinterpret_cast(jhandle); + options->allow_blocking_flush = static_cast(jallow_blocking_flush); +} + +/* + * Class: org_rocksdb_IngestExternalFileOptions + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_IngestExternalFileOptions_disposeInternal( + JNIEnv* env, jobject jobj, jlong jhandle) { + auto* options = + reinterpret_cast(jhandle); + delete options; +} \ No newline at end of file diff --git a/java/rocksjni/rocksjni.cc b/java/rocksjni/rocksjni.cc index 05ed30dd05..0dfe7f0a15 100644 --- a/java/rocksjni/rocksjni.cc +++ b/java/rocksjni/rocksjni.cc @@ -2164,18 +2164,17 @@ void Java_org_rocksdb_RocksDB_setOptions(JNIEnv* env, jobject jdb, } ////////////////////////////////////////////////////////////////////////////// -// rocksdb::DB::AddFile +// rocksdb::DB::IngestExternalFile /* * Class: org_rocksdb_RocksDB - * Method: addFile - * Signature: (JJ[Ljava/lang/String;IZ)V + * Method: ingestExternalFile + * Signature: (JJ[Ljava/lang/String;IJ)V */ -void Java_org_rocksdb_RocksDB_addFile__JJ_3Ljava_lang_String_2IZ( +void Java_org_rocksdb_RocksDB_ingestExternalFile( JNIEnv* env, jobject jdb, jlong jdb_handle, jlong jcf_handle, jobjectArray jfile_path_list, jint jfile_path_list_len, - jboolean jmove_file) { - + jlong jingest_external_file_options_handle) { jboolean has_exception = JNI_FALSE; std::vector file_path_list = rocksdb::JniUtil::copyStrings(env, jfile_path_list, jfile_path_list_len, @@ -2188,13 +2187,11 @@ void Java_org_rocksdb_RocksDB_addFile__JJ_3Ljava_lang_String_2IZ( auto* db = reinterpret_cast(jdb_handle); auto* column_family = reinterpret_cast(jcf_handle); - rocksdb::IngestExternalFileOptions ifo; - ifo.move_files = static_cast(jmove_file); - ifo.snapshot_consistency = true; - ifo.allow_global_seqno = false; - ifo.allow_blocking_flush = false; + auto* ifo = + reinterpret_cast( + jingest_external_file_options_handle); rocksdb::Status s = - db->IngestExternalFile(column_family, file_path_list, ifo); + db->IngestExternalFile(column_family, file_path_list, *ifo); if (!s.ok()) { rocksdb::RocksDBExceptionJni::ThrowNew(env, s); } diff --git a/java/src/main/java/org/rocksdb/ExternalSstFileInfo.java b/java/src/main/java/org/rocksdb/ExternalSstFileInfo.java deleted file mode 100644 index a35c15539e..0000000000 --- a/java/src/main/java/org/rocksdb/ExternalSstFileInfo.java +++ /dev/null @@ -1,135 +0,0 @@ -// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. - -package org.rocksdb; - -import java.io.File; - -public class ExternalSstFileInfo extends RocksObject { - public ExternalSstFileInfo() { - super(newExternalSstFileInfo()); - } - - public ExternalSstFileInfo(final String filePath, final String smallestKey, - final String largestKey, final long sequenceNumber, final long fileSize, final int numEntries, - final int version) { - super(newExternalSstFileInfo(ensureNotNullFilePath(filePath), - ensureNotNullSmallestKey(smallestKey), ensureNotNullLargestKey(largestKey), sequenceNumber, - fileSize, numEntries, version)); - } - - private static String ensureNotNullFilePath(final String filePath) { - if (filePath == null) { - throw new NullPointerException("filePath is null."); - } - return filePath; - } - - private static String ensureNotNullSmallestKey(final String smallestKey) { - if (smallestKey == null) { - throw new NullPointerException("smallestKey is null."); - } - return smallestKey; - } - - private static String ensureNotNullLargestKey(final String largestKey) { - if (largestKey == null) { - throw new NullPointerException("largestKey is null."); - } - return largestKey; - } - - public void setFilePath(final String filePath) { - setFilePath(nativeHandle_, filePath); - } - - public String filePath() { - return filePath(nativeHandle_); - } - - public void setSmallestKey(final String smallestKey) { - setSmallestKey(nativeHandle_, smallestKey); - } - - public String smallestKey() { - return smallestKey(nativeHandle_); - } - - public void setLargestKey(final String largestKey) { - setLargestKey(nativeHandle_, largestKey); - } - - public String largestKey() { - return largestKey(nativeHandle_); - } - - public void setSequenceNumber(final long sequenceNumber) { - setSequenceNumber(nativeHandle_, sequenceNumber); - } - - public long sequenceNumber() { - return sequenceNumber(nativeHandle_); - } - - public void setFileSize(final long fileSize) { - setFileSize(nativeHandle_, fileSize); - } - - public long fileSize() { - return fileSize(nativeHandle_); - } - - public void setNumEntries(final int numEntries) { - setNumEntries(nativeHandle_, numEntries); - } - - public int numEntries() { - return numEntries(nativeHandle_); - } - - public void setVersion(final int version) { - setVersion(nativeHandle_, version); - } - - public int version() { - return version(nativeHandle_); - } - - private native static long newExternalSstFileInfo(); - - private native static long newExternalSstFileInfo(final String filePath, final String smallestKey, - final String largestKey, final long sequenceNumber, final long fileSize, final int numEntries, - final int version); - - private native void setFilePath(final long handle, final String filePath); - - private native String filePath(final long handle); - - private native void setSmallestKey(final long handle, final String smallestKey); - - private native String smallestKey(final long handle); - - private native void setLargestKey(final long handle, final String largestKey); - - private native String largestKey(final long handle); - - private native void setSequenceNumber(final long handle, final long sequenceNumber); - - private native long sequenceNumber(final long handle); - - private native void setFileSize(final long handle, final long fileSize); - - private native long fileSize(final long handle); - - private native void setNumEntries(final long handle, final int numEntries); - - private native int numEntries(final long handle); - - private native void setVersion(final long handle, final int version); - - private native int version(final long handle); - - @Override protected final native void disposeInternal(final long handle); -} diff --git a/java/src/main/java/org/rocksdb/IngestExternalFileOptions.java b/java/src/main/java/org/rocksdb/IngestExternalFileOptions.java new file mode 100644 index 0000000000..21aab069e0 --- /dev/null +++ b/java/src/main/java/org/rocksdb/IngestExternalFileOptions.java @@ -0,0 +1,125 @@ +package org.rocksdb; +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +import java.util.List; + +/** + * IngestExternalFileOptions is used by {@link RocksDB#ingestExternalFile(ColumnFamilyHandle, List, IngestExternalFileOptions)} + */ +public class IngestExternalFileOptions extends RocksObject { + + public IngestExternalFileOptions() { + super(newIngestExternalFileOptions()); + } + + /** + * @param moveFiles {@link #setMoveFiles(boolean)} + * @param snapshotConsistency {@link #setSnapshotConsistency(boolean)} + * @param allowGlobalSeqNo {@link #setAllowGlobalSeqNo(boolean)} + * @param allowBlockingFlush {@link #setAllowBlockingFlush(boolean)} + */ + public IngestExternalFileOptions(final boolean moveFiles, + final boolean snapshotConsistency, final boolean allowGlobalSeqNo, + final boolean allowBlockingFlush) { + super(newIngestExternalFileOptions(moveFiles, snapshotConsistency, + allowGlobalSeqNo, allowBlockingFlush)); + } + + /** + * Can be set to true to move the files instead of copying them. + * + * @return true if files will be moved + */ + public boolean moveFiles() { + return moveFiles(nativeHandle_); + } + + /** + * Can be set to true to move the files instead of copying them. + * + * @param moveFiles true if files should be moved instead of copied + */ + public void setMoveFiles(final boolean moveFiles) { + setMoveFiles(nativeHandle_, moveFiles); + } + + /** + * If set to false, an ingested file keys could appear in existing snapshots + * that where created before the file was ingested. + * + * @return true if snapshot consistency is assured + */ + public boolean snapshotConsistency() { + return snapshotConsistency(nativeHandle_); + } + + /** + * If set to false, an ingested file keys could appear in existing snapshots + * that where created before the file was ingested. + * + * @param snapshotConsistency true if snapshot consistency is required + */ + public void setSnapshotConsistency(final boolean snapshotConsistency) { + setSnapshotConsistency(nativeHandle_, snapshotConsistency); + } + + /** + * If set to false, {@link RocksDB#ingestExternalFile(ColumnFamilyHandle, List, IngestExternalFileOptions)} + * will fail if the file key range overlaps with existing keys or tombstones in the DB. + * + * @return true if global seq numbers are assured + */ + public boolean allowGlobalSeqNo() { + return allowGlobalSeqNo(nativeHandle_); + } + + /** + * If set to false, {@link RocksDB#ingestExternalFile(ColumnFamilyHandle, List, IngestExternalFileOptions)} + * will fail if the file key range overlaps with existing keys or tombstones in the DB. + * + * @param allowGlobalSeqNo true if global seq numbers are required + */ + public void setAllowGlobalSeqNo(final boolean allowGlobalSeqNo) { + setAllowGlobalSeqNo(nativeHandle_, allowGlobalSeqNo); + } + + /** + * If set to false and the file key range overlaps with the memtable key range + * (memtable flush required), IngestExternalFile will fail. + * + * @return true if blocking flushes may occur + */ + public boolean allowBlockingFlush() { + return allowBlockingFlush(nativeHandle_); + } + + /** + * If set to false and the file key range overlaps with the memtable key range + * (memtable flush required), IngestExternalFile will fail. + * + * @param allowBlockingFlush true if blocking flushes are allowed + */ + public void setAllowBlockingFlush(final boolean allowBlockingFlush) { + setAllowBlockingFlush(nativeHandle_, allowBlockingFlush); + } + + private native static long newIngestExternalFileOptions(); + private native static long newIngestExternalFileOptions( + final boolean moveFiles, final boolean snapshotConsistency, + final boolean allowGlobalSeqNo, final boolean allowBlockingFlush); + private native boolean moveFiles(final long handle); + private native void setMoveFiles(final long handle, final boolean move_files); + private native boolean snapshotConsistency(final long handle); + private native void setSnapshotConsistency(final long handle, + final boolean snapshotConsistency); + private native boolean allowGlobalSeqNo(final long handle); + private native void setAllowGlobalSeqNo(final long handle, + final boolean allowGloablSeqNo); + private native boolean allowBlockingFlush(final long handle); + private native void setAllowBlockingFlush(final long handle, + final boolean allowBlockingFlush); + @Override protected final native void disposeInternal(final long handle); +} diff --git a/java/src/main/java/org/rocksdb/RocksDB.java b/java/src/main/java/org/rocksdb/RocksDB.java index c7a79edb00..1b516753d4 100644 --- a/java/src/main/java/org/rocksdb/RocksDB.java +++ b/java/src/main/java/org/rocksdb/RocksDB.java @@ -2122,100 +2122,50 @@ public class RocksDB extends RocksObject { return handleList; } - public void addFileWithFilePath(final ColumnFamilyHandle columnFamilyHandle, - final List filePathList) throws RocksDBException { - addFile(nativeHandle_, columnFamilyHandle.nativeHandle_, - filePathList.toArray(new String[filePathList.size()]), filePathList.size(), false); - } - - public void addFileWithFilePath(final ColumnFamilyHandle columnFamilyHandle, - final List filePathList, final boolean moveFile) throws RocksDBException { - addFile(nativeHandle_, columnFamilyHandle.nativeHandle_, - filePathList.toArray(new String[filePathList.size()]), filePathList.size(), moveFile); - } - - public void addFileWithFilePath(final List filePathList) throws RocksDBException { - addFile(nativeHandle_, getDefaultColumnFamily().nativeHandle_, - filePathList.toArray(new String[filePathList.size()]), filePathList.size(), false); - } - - public void addFileWithFilePath(final List filePathList, final boolean moveFile) + /** + * ingestExternalFile will load a list of external SST files (1) into the DB + * We will try to find the lowest possible level that the file can fit in, and + * ingest the file into this level (2). A file that have a key range that + * overlap with the memtable key range will require us to Flush the memtable + * first before ingesting the file. + * + * (1) External SST files can be created using {@link SstFileWriter} + * (2) We will try to ingest the files to the lowest possible level + * even if the file compression doesn't match the level compression + * + * @param filePathList The list of files to ingest + * @param ingestExternalFileOptions the options for the ingestion + */ + public void ingestExternalFile(final List filePathList, + final IngestExternalFileOptions ingestExternalFileOptions) throws RocksDBException { - addFile(nativeHandle_, getDefaultColumnFamily().nativeHandle_, - filePathList.toArray(new String[filePathList.size()]), filePathList.size(), moveFile); + ingestExternalFile(nativeHandle_, getDefaultColumnFamily().nativeHandle_, + filePathList.toArray(new String[filePathList.size()]), + filePathList.size(), ingestExternalFileOptions.nativeHandle_); } - public void addFileWithFilePath( - final ColumnFamilyHandle columnFamilyHandle, final String filePath) throws RocksDBException { - addFile(nativeHandle_, columnFamilyHandle.nativeHandle_, new String[] {filePath}, 1, false); - } - - public void addFileWithFilePath(final ColumnFamilyHandle columnFamilyHandle, - final String filePath, final boolean moveFile) throws RocksDBException { - addFile(nativeHandle_, columnFamilyHandle.nativeHandle_, new String[] {filePath}, 1, moveFile); - } - - public void addFileWithFilePath(final String filePath) throws RocksDBException { - addFile( - nativeHandle_, getDefaultColumnFamily().nativeHandle_, new String[] {filePath}, 1, false); - } - - public void addFileWithFilePath(final String filePath, final boolean moveFile) + /** + * ingestExternalFile will load a list of external SST files (1) into the DB + * We will try to find the lowest possible level that the file can fit in, and + * ingest the file into this level (2). A file that have a key range that + * overlap with the memtable key range will require us to Flush the memtable + * first before ingesting the file. + * + * (1) External SST files can be created using {@link SstFileWriter} + * (2) We will try to ingest the files to the lowest possible level + * even if the file compression doesn't match the level compression + * + * @param columnFamilyHandle The column family for the ingested files + * @param filePathList The list of files to ingest + * @param ingestExternalFileOptions the options for the ingestion + */ + public void ingestExternalFile(final ColumnFamilyHandle columnFamilyHandle, + final List filePathList, + final IngestExternalFileOptions ingestExternalFileOptions) throws RocksDBException { - addFile(nativeHandle_, getDefaultColumnFamily().nativeHandle_, new String[] {filePath}, 1, - moveFile); - } - - public void addFileWithFileInfo(final ColumnFamilyHandle columnFamilyHandle, - final List fileInfoList) throws RocksDBException { - final long[] fiHandleList = toNativeHandleList(fileInfoList); - addFile( - nativeHandle_, columnFamilyHandle.nativeHandle_, fiHandleList, fiHandleList.length, false); - } - - public void addFileWithFileInfo(final ColumnFamilyHandle columnFamilyHandle, - final List fileInfoList, final boolean moveFile) - throws RocksDBException { - final long[] fiHandleList = toNativeHandleList(fileInfoList); - addFile(nativeHandle_, columnFamilyHandle.nativeHandle_, fiHandleList, fiHandleList.length, - moveFile); - } - - public void addFileWithFileInfo(final List fileInfoList) - throws RocksDBException { - final long[] fiHandleList = toNativeHandleList(fileInfoList); - addFile(nativeHandle_, getDefaultColumnFamily().nativeHandle_, fiHandleList, - fiHandleList.length, false); - } - - public void addFileWithFileInfo(final List fileInfoList, - final boolean moveFile) throws RocksDBException { - final long[] fiHandleList = toNativeHandleList(fileInfoList); - addFile(nativeHandle_, getDefaultColumnFamily().nativeHandle_, fiHandleList, - fiHandleList.length, moveFile); - } - - public void addFileWithFileInfo(final ColumnFamilyHandle columnFamilyHandle, - final ExternalSstFileInfo fileInfo) throws RocksDBException { - addFile(nativeHandle_, columnFamilyHandle.nativeHandle_, new long[] {fileInfo.nativeHandle_}, 1, - false); - } - - public void addFileWithFileInfo(final ColumnFamilyHandle columnFamilyHandle, - final ExternalSstFileInfo fileInfo, final boolean moveFile) throws RocksDBException { - addFile(nativeHandle_, columnFamilyHandle.nativeHandle_, new long[] {fileInfo.nativeHandle_}, 1, - moveFile); - } - - public void addFileWithFileInfo(final ExternalSstFileInfo fileInfo) throws RocksDBException { - addFile(nativeHandle_, getDefaultColumnFamily().nativeHandle_, - new long[] {fileInfo.nativeHandle_}, 1, false); - } - - public void addFileWithFileInfo(final ExternalSstFileInfo fileInfo, final boolean moveFile) - throws RocksDBException { - addFile(nativeHandle_, getDefaultColumnFamily().nativeHandle_, - new long[] {fileInfo.nativeHandle_}, 1, moveFile); + ingestExternalFile(nativeHandle_, columnFamilyHandle.nativeHandle_, + filePathList.toArray(new String[filePathList.size()]), + filePathList.size(), ingestExternalFileOptions.nativeHandle_); } /** @@ -2421,9 +2371,8 @@ public class RocksDB extends RocksObject { throws RocksDBException; private native void setOptions(long handle, long cfHandle, String[] keys, String[] values) throws RocksDBException; - private native void addFile(long handle, long cfHandle, String[] filePathList, - int filePathListLen, boolean moveFile) throws RocksDBException; - private native void addFile(long handle, long cfHandle, long[] fiHandleList, int fiHandleListLen, - boolean moveFile) throws RocksDBException; + private native void ingestExternalFile(long handle, long cfHandle, + String[] filePathList, int filePathListLen, + long ingest_external_file_options_handle) throws RocksDBException; protected DBOptionsInterface options_; } diff --git a/java/src/test/java/org/rocksdb/ExternalSstFileInfoTest.java b/java/src/test/java/org/rocksdb/ExternalSstFileInfoTest.java deleted file mode 100644 index 2e8b2368bd..0000000000 --- a/java/src/test/java/org/rocksdb/ExternalSstFileInfoTest.java +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. - -package org.rocksdb; - -import org.junit.ClassRule; -import org.junit.Test; - -import java.util.Random; - -import static org.assertj.core.api.Assertions.assertThat; - -public class ExternalSstFileInfoTest { - @ClassRule - public static final RocksMemoryResource rocksMemoryResource = new RocksMemoryResource(); - - public static final Random rand = PlatformRandomHelper.getPlatformSpecificRandomFactory(); - - @Test - public void createExternalSstFileInfoWithoutParameters() { - try (final ExternalSstFileInfo info = new ExternalSstFileInfo()) { - assertThat(info).isNotNull(); - } - } - - @Test - public void createExternalSstFileInfoWithParameters() { - final String filePath = "path/to/sst"; - final String smallestKey = "min"; - final String largestKey = "max"; - final long sequenceNumber = rand.nextLong(); - final long fileSize = rand.nextLong(); - final int numEntries = rand.nextInt(); - final int version = rand.nextInt(); - try (final ExternalSstFileInfo info = new ExternalSstFileInfo( - filePath, smallestKey, largestKey, sequenceNumber, fileSize, numEntries, version)) { - assertThat(info).isNotNull(); - } - } - - @Test - public void filePath() { - try (final ExternalSstFileInfo info = new ExternalSstFileInfo()) { - final String stringVale = "path/to/sst"; - info.setFilePath(stringVale); - assertThat(info.filePath()).isEqualTo(stringVale); - } - } - - @Test - public void smallestKey() { - try (final ExternalSstFileInfo info = new ExternalSstFileInfo()) { - final String stringValue = "min"; - info.setSmallestKey(stringValue); - assertThat(info.smallestKey()).isEqualTo(stringValue); - } - } - - @Test - public void largestKey() { - try (final ExternalSstFileInfo info = new ExternalSstFileInfo()) { - final String stringValue = "max"; - info.setLargestKey(stringValue); - assertThat(info.largestKey()).isEqualTo(stringValue); - } - } - - @Test - public void sequenceNumber() { - try (final ExternalSstFileInfo info = new ExternalSstFileInfo()) { - final long longValue = rand.nextLong(); - info.setSequenceNumber(longValue); - assertThat(info.sequenceNumber()).isEqualTo(longValue); - } - } - - @Test - public void fileSize() { - try (final ExternalSstFileInfo info = new ExternalSstFileInfo()) { - final long longValue = Math.max(1, rand.nextLong()); - info.setFileSize(longValue); - assertThat(info.fileSize()).isEqualTo(longValue); - } - } - - @Test - public void numEntries() { - try (final ExternalSstFileInfo info = new ExternalSstFileInfo()) { - final int intValue = Math.max(1, rand.nextInt()); - info.setNumEntries(intValue); - assertThat(info.numEntries()).isEqualTo(intValue); - } - } - - @Test - public void version() { - try (final ExternalSstFileInfo info = new ExternalSstFileInfo()) { - final int intValue = rand.nextInt(); - info.setVersion(intValue); - assertThat(info.version()).isEqualTo(intValue); - } - } -} diff --git a/java/src/test/java/org/rocksdb/IngestExternalFileOptionsTest.java b/java/src/test/java/org/rocksdb/IngestExternalFileOptionsTest.java new file mode 100644 index 0000000000..e7a50317ba --- /dev/null +++ b/java/src/test/java/org/rocksdb/IngestExternalFileOptionsTest.java @@ -0,0 +1,87 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +package org.rocksdb; + +import org.junit.ClassRule; +import org.junit.Test; + +import java.util.Random; + +import static org.assertj.core.api.Assertions.assertThat; + +public class IngestExternalFileOptionsTest { + @ClassRule + public static final RocksMemoryResource rocksMemoryResource + = new RocksMemoryResource(); + + public static final Random rand = + PlatformRandomHelper.getPlatformSpecificRandomFactory(); + + @Test + public void createExternalSstFileInfoWithoutParameters() { + try (final IngestExternalFileOptions options = + new IngestExternalFileOptions()) { + assertThat(options).isNotNull(); + } + } + + @Test + public void createExternalSstFileInfoWithParameters() { + final boolean moveFiles = rand.nextBoolean(); + final boolean snapshotConsistency = rand.nextBoolean(); + final boolean allowGlobalSeqNo = rand.nextBoolean(); + final boolean allowBlockingFlush = rand.nextBoolean(); + try (final IngestExternalFileOptions options = + new IngestExternalFileOptions(moveFiles, snapshotConsistency, + allowGlobalSeqNo, allowBlockingFlush)) { + assertThat(options).isNotNull(); + assertThat(options.moveFiles()).isEqualTo(moveFiles); + assertThat(options.snapshotConsistency()).isEqualTo(snapshotConsistency); + assertThat(options.allowGlobalSeqNo()).isEqualTo(allowGlobalSeqNo); + assertThat(options.allowBlockingFlush()).isEqualTo(allowBlockingFlush); + } + } + + @Test + public void moveFiles() { + try (final IngestExternalFileOptions options = + new IngestExternalFileOptions()) { + final boolean moveFiles = rand.nextBoolean(); + options.setMoveFiles(moveFiles); + assertThat(options.moveFiles()).isEqualTo(moveFiles); + } + } + + @Test + public void snapshotConsistency() { + try (final IngestExternalFileOptions options = + new IngestExternalFileOptions()) { + final boolean snapshotConsistency = rand.nextBoolean(); + options.setSnapshotConsistency(snapshotConsistency); + assertThat(options.snapshotConsistency()).isEqualTo(snapshotConsistency); + } + } + + @Test + public void allowGlobalSeqNo() { + try (final IngestExternalFileOptions options = + new IngestExternalFileOptions()) { + final boolean allowGlobalSeqNo = rand.nextBoolean(); + options.setAllowGlobalSeqNo(allowGlobalSeqNo); + assertThat(options.allowGlobalSeqNo()).isEqualTo(allowGlobalSeqNo); + } + } + + @Test + public void allowBlockingFlush() { + try (final IngestExternalFileOptions options = + new IngestExternalFileOptions()) { + final boolean allowBlockingFlush = rand.nextBoolean(); + options.setAllowBlockingFlush(allowBlockingFlush); + assertThat(options.allowBlockingFlush()).isEqualTo(allowBlockingFlush); + } + } +} diff --git a/java/src/test/java/org/rocksdb/SstFileWriterTest.java b/java/src/test/java/org/rocksdb/SstFileWriterTest.java index 5f2a4b10ef..373d27af1f 100644 --- a/java/src/test/java/org/rocksdb/SstFileWriterTest.java +++ b/java/src/test/java/org/rocksdb/SstFileWriterTest.java @@ -13,6 +13,7 @@ import org.rocksdb.util.BytewiseComparator; import java.io.File; import java.io.IOException; +import java.util.Arrays; import java.util.Map; import java.util.TreeMap; @@ -23,7 +24,8 @@ public class SstFileWriterTest { private static final String DB_DIRECTORY_NAME = "test_db"; @ClassRule - public static final RocksMemoryResource rocksMemoryResource = new RocksMemoryResource(); + public static final RocksMemoryResource rocksMemoryResource + = new RocksMemoryResource(); @Rule public TemporaryFolder parentFolder = new TemporaryFolder(); @@ -71,7 +73,8 @@ public class SstFileWriterTest { } @Test - public void generateSstFileWithJavaComparator() throws RocksDBException, IOException { + public void generateSstFileWithJavaComparator() + throws RocksDBException, IOException { final TreeMap keyValues = new TreeMap<>(); keyValues.put("key1", "value1"); keyValues.put("key2", "value2"); @@ -79,7 +82,8 @@ public class SstFileWriterTest { } @Test - public void generateSstFileWithNativeComparator() throws RocksDBException, IOException { + public void generateSstFileWithNativeComparator() + throws RocksDBException, IOException { final TreeMap keyValues = new TreeMap<>(); keyValues.put("key1", "value1"); keyValues.put("key2", "value2"); @@ -93,14 +97,45 @@ public class SstFileWriterTest { keyValues.put("key2", "value2"); final File sstFile = newSstFile(keyValues, false); final File dbFolder = parentFolder.newFolder(DB_DIRECTORY_NAME); - final Options options = new Options().setCreateIfMissing(true); - final RocksDB db = RocksDB.open(options, dbFolder.getAbsolutePath()); - db.addFileWithFilePath(sstFile.getAbsolutePath()); + try(final Options options = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(options, dbFolder.getAbsolutePath()); + final IngestExternalFileOptions ingestExternalFileOptions + = new IngestExternalFileOptions()) { + db.ingestExternalFile(Arrays.asList(sstFile.getAbsolutePath()), + ingestExternalFileOptions); - assertThat(db.get("key1".getBytes())).isEqualTo("value1".getBytes()); - assertThat(db.get("key2".getBytes())).isEqualTo("value2".getBytes()); + assertThat(db.get("key1".getBytes())).isEqualTo("value1".getBytes()); + assertThat(db.get("key2".getBytes())).isEqualTo("value2".getBytes()); + } + } - options.close(); - db.close(); + @Test + public void ingestSstFile_cf() throws RocksDBException, IOException { + final TreeMap keyValues = new TreeMap<>(); + keyValues.put("key1", "value1"); + keyValues.put("key2", "value2"); + final File sstFile = newSstFile(keyValues, false); + final File dbFolder = parentFolder.newFolder(DB_DIRECTORY_NAME); + try(final Options options = new Options() + .setCreateIfMissing(true) + .setCreateMissingColumnFamilies(true); + final RocksDB db = RocksDB.open(options, dbFolder.getAbsolutePath()); + final IngestExternalFileOptions ingestExternalFileOptions = + new IngestExternalFileOptions()) { + + try(final ColumnFamilyOptions cf_opts = new ColumnFamilyOptions(); + final ColumnFamilyHandle cf_handle = db.createColumnFamily( + new ColumnFamilyDescriptor("new_cf".getBytes(), cf_opts))) { + + db.ingestExternalFile(cf_handle, + Arrays.asList(sstFile.getAbsolutePath()), + ingestExternalFileOptions); + + assertThat(db.get(cf_handle, + "key1".getBytes())).isEqualTo("value1".getBytes()); + assertThat(db.get(cf_handle, + "key2".getBytes())).isEqualTo("value2".getBytes()); + } + } } } diff --git a/src.mk b/src.mk index c7fa3660a8..fa1c376fe0 100644 --- a/src.mk +++ b/src.mk @@ -355,7 +355,7 @@ JNI_NATIVE_SOURCES = \ java/rocksjni/compression_options.cc \ java/rocksjni/env.cc \ java/rocksjni/env_options.cc \ - java/rocksjni/external_sst_file_info.cc \ + java/rocksjni/ingest_external_file_options.cc \ java/rocksjni/filter.cc \ java/rocksjni/iterator.cc \ java/rocksjni/loggerjnicallback.cc \