[RocksJava] TTL-Support

Summary:
TTLDB Support exposed in Java-API. It is now
possible to open a datbase using the RocksDB time
to live feature.

Test Plan:
make rocksdbjava
make test
mvn -f rocksjni.pom package

@Adam please test mac osx compile

Reviewers: yhchiang, adamretter, ankgup87

Subscribers: dhruba, adam

Differential Revision: https://reviews.facebook.net/D31449
This commit is contained in:
fyrz 2015-01-13 22:56:37 +01:00
parent 5ff8aec4db
commit 859c54a03d
3 changed files with 319 additions and 134 deletions

View file

@ -89,18 +89,40 @@ public class TtlDB extends RocksDB {
return ttldb;
}
//static Status Open(const DBOptions& db_options, const std::string& dbname,
// const std::vector<ColumnFamilyDescriptor>& column_families,
// std::vector<ColumnFamilyHandle*>* handles,
// DBWithTTL** dbptr, std::vector<int32_t> ttls,
// bool read_only = false);
/**
* <p>Opens a TtlDB.</p>
*
* @param options {@link org.rocksdb.Options} instance.
* @param db_path path to database.
* @param columnFamilyDescriptors list of column family descriptors
* @param columnFamilyHandles will be filled with ColumnFamilyHandle instances
* on open.
* @param ttlValues time to live values per column family handle
* @param readOnly boolean value indicating if database if db is
* opened read-only.
*
* @return TtlDB instance.
*
* @throws RocksDBException thrown if an error occurs within the native
* part of the library.
* @throws java.lang.IllegalArgumentException when there is not a ttl value
* per given column family handle.
*/
public static TtlDB open(DBOptions options, String db_path,
List<ColumnFamilyDescriptor> columnFamilyDescriptors,
List<ColumnFamilyHandle> columnFamilyHandles,
List<Integer> ttlValues, boolean readOnly){
return null;
List<Integer> ttlValues, boolean readOnly) throws RocksDBException {
if (columnFamilyDescriptors.size() != ttlValues.size()) {
throw new IllegalArgumentException("There must be a ttl value per column" +
"family handle.");
}
TtlDB ttlDB = new TtlDB();
List<Long> cfReferences = ttlDB.open(options.nativeHandle_, db_path, columnFamilyDescriptors,
columnFamilyDescriptors.size(), ttlValues, readOnly);
for (int i=0; i<columnFamilyDescriptors.size(); i++) {
columnFamilyHandles.add(new ColumnFamilyHandle(ttlDB, cfReferences.get(i)));
}
return ttlDB;
}
/**
@ -146,10 +168,10 @@ public class TtlDB extends RocksDB {
/**
* <p>A protected constructor that will be used in the static
* factory method
* {@link #open(DBOptions, String, java.util.List, java.util.List)}
* {@link #open(Options, String, int, boolean)}
* and
* {@link #open(DBOptions, String, java.util.List,
* java.util.List, java.util.List, boolean)}.
* {@link #open(DBOptions, String, java.util.List, java.util.List,
* java.util.List, boolean)}.
* </p>
*/
protected TtlDB() {
@ -163,6 +185,10 @@ public class TtlDB extends RocksDB {
private native void open(long optionsHandle, String db_path, int ttl,
boolean readOnly) throws RocksDBException;
private native List<Long> open(long optionsHandle, String db_path,
List<ColumnFamilyDescriptor> columnFamilyDescriptors,
int columnFamilyDescriptorsLength, List<Integer> ttlValues,
boolean readOnly) throws RocksDBException;
private native long createColumnFamilyWithTtl(long handle,
ColumnFamilyDescriptor columnFamilyDescriptor, int ttl)
throws RocksDBException;

View file

@ -1,113 +1,169 @@
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
package org.rocksdb.test;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.rocksdb.*;
import java.util.concurrent.TimeUnit;
import static org.assertj.core.api.Assertions.assertThat;
public class TtlDBTest {
@ClassRule
public static final RocksMemoryResource rocksMemoryResource =
new RocksMemoryResource();
@Rule
public TemporaryFolder dbFolder = new TemporaryFolder();
@Test
public void ttlDBOpen() throws RocksDBException,
InterruptedException {
Options options = null;
TtlDB ttlDB = null;
try {
options = new Options().
setCreateIfMissing(true).
setMaxGrandparentOverlapFactor(0).
setMaxMemCompactionLevel(0);
ttlDB = TtlDB.open(options,
dbFolder.getRoot().getAbsolutePath());
ttlDB.put("key".getBytes(), "value".getBytes());
assertThat(ttlDB.get("key".getBytes())).
isEqualTo("value".getBytes());
assertThat(ttlDB.get("key".getBytes())).isNotNull();
} finally {
if (ttlDB != null) {
ttlDB.close();
}
if (options != null) {
options.dispose();
}
}
}
@Test
public void ttlDBOpenWithTtl() throws RocksDBException,
InterruptedException {
Options options = null;
TtlDB ttlDB = null;
try {
options = new Options().
setCreateIfMissing(true).
setMaxGrandparentOverlapFactor(0).
setMaxMemCompactionLevel(0);
ttlDB = TtlDB.open(options, dbFolder.getRoot().getAbsolutePath(),
1, false);
ttlDB.put("key".getBytes(), "value".getBytes());
assertThat(ttlDB.get("key".getBytes())).
isEqualTo("value".getBytes());
TimeUnit.SECONDS.sleep(2);
ttlDB.compactRange();
assertThat(ttlDB.get("key".getBytes())).isNull();
} finally {
if (ttlDB != null) {
ttlDB.close();
}
if (options != null) {
options.dispose();
}
}
}
@Test
public void createTtlColumnFamily() throws RocksDBException,
InterruptedException {
Options options = null;
TtlDB ttlDB = null;
ColumnFamilyHandle columnFamilyHandle = null;
try {
options = new Options().setCreateIfMissing(true);
ttlDB = TtlDB.open(options,
dbFolder.getRoot().getAbsolutePath());
columnFamilyHandle = ttlDB.createColumnFamilyWithTtl(
new ColumnFamilyDescriptor("new_cf"), 1);
ttlDB.put(columnFamilyHandle, "key".getBytes(),
"value".getBytes());
assertThat(ttlDB.get(columnFamilyHandle, "key".getBytes())).
isEqualTo("value".getBytes());
Thread.sleep(2500);
ttlDB.compactRange(columnFamilyHandle);
assertThat(ttlDB.get(columnFamilyHandle, "key".getBytes())).isNull();
} finally {
if (columnFamilyHandle != null) {
columnFamilyHandle.dispose();
}
if (ttlDB != null) {
ttlDB.close();
}
if (options != null) {
options.dispose();
}
}
}
}
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
package org.rocksdb.test;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.rocksdb.*;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
import static org.assertj.core.api.Assertions.assertThat;
public class TtlDBTest {
@ClassRule
public static final RocksMemoryResource rocksMemoryResource =
new RocksMemoryResource();
@Rule
public TemporaryFolder dbFolder = new TemporaryFolder();
@Test
public void ttlDBOpen() throws RocksDBException,
InterruptedException {
Options options = null;
TtlDB ttlDB = null;
try {
options = new Options().
setCreateIfMissing(true).
setMaxGrandparentOverlapFactor(0).
setMaxMemCompactionLevel(0);
ttlDB = TtlDB.open(options,
dbFolder.getRoot().getAbsolutePath());
ttlDB.put("key".getBytes(), "value".getBytes());
assertThat(ttlDB.get("key".getBytes())).
isEqualTo("value".getBytes());
assertThat(ttlDB.get("key".getBytes())).isNotNull();
} finally {
if (ttlDB != null) {
ttlDB.close();
}
if (options != null) {
options.dispose();
}
}
}
@Test
public void ttlDBOpenWithTtl() throws RocksDBException,
InterruptedException {
Options options = null;
TtlDB ttlDB = null;
try {
options = new Options().
setCreateIfMissing(true).
setMaxGrandparentOverlapFactor(0).
setMaxMemCompactionLevel(0);
ttlDB = TtlDB.open(options, dbFolder.getRoot().getAbsolutePath(),
1, false);
ttlDB.put("key".getBytes(), "value".getBytes());
assertThat(ttlDB.get("key".getBytes())).
isEqualTo("value".getBytes());
TimeUnit.SECONDS.sleep(2);
ttlDB.compactRange();
assertThat(ttlDB.get("key".getBytes())).isNull();
} finally {
if (ttlDB != null) {
ttlDB.close();
}
if (options != null) {
options.dispose();
}
}
}
@Test
public void ttlDbOpenWithColumnFamilies() throws RocksDBException, InterruptedException {
DBOptions dbOptions = null;
TtlDB ttlDB = null;
List<ColumnFamilyDescriptor> cfNames =
new ArrayList<>();
List<ColumnFamilyHandle> columnFamilyHandleList =
new ArrayList<>();
cfNames.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY));
cfNames.add(new ColumnFamilyDescriptor("new_cf".getBytes()));
List<Integer> ttlValues = new ArrayList<>();
// Default column family with infinite lifetime
ttlValues.add(0);
// new column family with 1 second ttl
ttlValues.add(1);
try {
dbOptions = new DBOptions().
setCreateMissingColumnFamilies(true).
setCreateIfMissing(true);
ttlDB = TtlDB.open(dbOptions, dbFolder.getRoot().getAbsolutePath(),
cfNames, columnFamilyHandleList, ttlValues, false);
ttlDB.put("key".getBytes(), "value".getBytes());
assertThat(ttlDB.get("key".getBytes())).
isEqualTo("value".getBytes());
ttlDB.put(columnFamilyHandleList.get(1), "key".getBytes(),
"value".getBytes());
assertThat(ttlDB.get(columnFamilyHandleList.get(1),
"key".getBytes())).isEqualTo("value".getBytes());
TimeUnit.SECONDS.sleep(2);
ttlDB.compactRange();
ttlDB.compactRange(columnFamilyHandleList.get(1));
assertThat(ttlDB.get("key".getBytes())).isNotNull();
assertThat(ttlDB.get(columnFamilyHandleList.get(1),
"key".getBytes())).isNull();
} finally {
for (ColumnFamilyHandle columnFamilyHandle :
columnFamilyHandleList) {
columnFamilyHandle.dispose();
}
if (ttlDB != null) {
ttlDB.close();
}
if (dbOptions != null) {
dbOptions.dispose();
}
}
}
@Test
public void createTtlColumnFamily() throws RocksDBException,
InterruptedException {
Options options = null;
TtlDB ttlDB = null;
ColumnFamilyHandle columnFamilyHandle = null;
try {
options = new Options().setCreateIfMissing(true);
ttlDB = TtlDB.open(options,
dbFolder.getRoot().getAbsolutePath());
columnFamilyHandle = ttlDB.createColumnFamilyWithTtl(
new ColumnFamilyDescriptor("new_cf".getBytes()), 1);
ttlDB.put(columnFamilyHandle, "key".getBytes(),
"value".getBytes());
assertThat(ttlDB.get(columnFamilyHandle, "key".getBytes())).
isEqualTo("value".getBytes());
TimeUnit.SECONDS.sleep(2);
ttlDB.compactRange(columnFamilyHandle);
assertThat(ttlDB.get(columnFamilyHandle, "key".getBytes())).isNull();
} finally {
if (columnFamilyHandle != null) {
columnFamilyHandle.dispose();
}
if (ttlDB != null) {
ttlDB.close();
}
if (options != null) {
options.dispose();
}
}
}
}

View file

@ -7,22 +7,22 @@
// calling c++ rocksdb::TtlDB methods.
// from Java side.
#include <jni.h>
#include <stdio.h>
#include <stdlib.h>
#include <jni.h>
#include <string>
#include <vector>
#include "include/org_rocksdb_TtlDB.h"
#include "rocksjni/portal.h"
#include "rocksdb/utilities/db_ttl.h"
#include "rocksjni/portal.h"
/*
* Class: org_rocksdb_TtlDB
* Method: open
* Signature: (JJ)V
* Signature: (JLjava/lang/String;IZ)V
*/
void Java_org_rocksdb_TtlDB_open(JNIEnv* env, jobject jttldb,
void Java_org_rocksdb_TtlDB_open__JLjava_lang_String_2IZ(JNIEnv* env, jobject jttldb,
jlong joptions_handle, jstring jdb_path, jint jttl,
jboolean jread_only) {
auto opt = reinterpret_cast<rocksdb::Options*>(joptions_handle);
@ -41,6 +41,107 @@ void Java_org_rocksdb_TtlDB_open(JNIEnv* env, jobject jttldb,
rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
}
/*
* Class: org_rocksdb_TtlDB
* Method: open
* Signature: (JLjava/lang/String;Ljava/util/List;ILjava/util/List;Z)Ljava/util/List;
*/
jobject
Java_org_rocksdb_TtlDB_open__JLjava_lang_String_2Ljava_util_List_2ILjava_util_List_2Z(
JNIEnv* env, jobject jdb, jlong jopt_handle, jstring jdb_path,
jobject jcfdesc_list, jint jcfdesc_count, jobject jttl_list,
jboolean jread_only) {
auto opt = reinterpret_cast<rocksdb::Options*>(jopt_handle);
rocksdb::DBWithTTL* db = nullptr;
const char* db_path = env->GetStringUTFChars(jdb_path, 0);
std::vector<jbyte*> cfnames_to_free;
std::vector<jbyteArray> jcfnames_for_free;
std::vector<rocksdb::ColumnFamilyDescriptor> column_families;
std::vector<int32_t> ttl_values;
std::vector<rocksdb::ColumnFamilyHandle* > handles;
// get iterator for ColumnFamilyDescriptors
jobject iteratorObj = env->CallObjectMethod(
jcfdesc_list, rocksdb::ListJni::getIteratorMethod(env));
// iterate over ColumnFamilyDescriptors
while (env->CallBooleanMethod(
iteratorObj, rocksdb::ListJni::getHasNextMethod(env)) == JNI_TRUE) {
// get ColumnFamilyDescriptor
jobject jcf_descriptor = env->CallObjectMethod(iteratorObj,
rocksdb::ListJni::getNextMethod(env));
// get ColumnFamilyName
jbyteArray byteArray = static_cast<jbyteArray>(env->CallObjectMethod(
jcf_descriptor,
rocksdb::ColumnFamilyDescriptorJni::getColumnFamilyNameMethod(
env)));
// get CF Options
jobject jcf_opt_obj = env->CallObjectMethod(jcf_descriptor,
rocksdb::ColumnFamilyDescriptorJni::getColumnFamilyOptionsMethod(
env));
rocksdb::ColumnFamilyOptions* cfOptions =
rocksdb::ColumnFamilyOptionsJni::getHandle(env, jcf_opt_obj);
jbyte* cfname = env->GetByteArrayElements(byteArray, 0);
// free allocated cfnames after call to open
cfnames_to_free.push_back(cfname);
jcfnames_for_free.push_back(byteArray);
column_families.push_back(rocksdb::ColumnFamilyDescriptor(
reinterpret_cast<const char *>(cfname),
*cfOptions));
}
// get iterator for TTL values
iteratorObj = env->CallObjectMethod(
jttl_list, rocksdb::ListJni::getIteratorMethod(env));
// iterate over TTL values
while (env->CallBooleanMethod(
iteratorObj, rocksdb::ListJni::getHasNextMethod(env)) == JNI_TRUE) {
// get TTL object
jobject jttl_object = env->CallObjectMethod(iteratorObj,
rocksdb::ListJni::getNextMethod(env));
// get Integer value
jclass jIntClazz = env->FindClass("java/lang/Integer");
jmethodID getVal = env->GetMethodID(jIntClazz, "intValue", "()I");
ttl_values.push_back(env->CallIntMethod(jttl_object, getVal));
}
rocksdb::Status s = rocksdb::DBWithTTL::Open(*opt, db_path, column_families,
&handles, &db, ttl_values, jread_only);
env->ReleaseStringUTFChars(jdb_path, db_path);
// free jbyte allocations
for (std::vector<jbyte*>::size_type i = 0;
i != cfnames_to_free.size(); i++) {
// free cfnames
env->ReleaseByteArrayElements(jcfnames_for_free[i], cfnames_to_free[i], 0);
}
// check if open operation was successful
if (s.ok()) {
rocksdb::RocksDBJni::setHandle(env, jdb, db);
jclass jListClazz = env->FindClass("java/util/ArrayList");
jmethodID midList = rocksdb::ListJni::getArrayListConstructorMethodId(
env, jListClazz);
jobject jcfhandle_list = env->NewObject(jListClazz,
midList, handles.size());
// insert in java list
for (std::vector<rocksdb::ColumnFamilyHandle*>::size_type i = 0;
i != handles.size(); i++) {
// jlong must be converted to Long due to collections restrictions
jclass jLongClazz = env->FindClass("java/lang/Long");
jmethodID midLong = env->GetMethodID(jLongClazz, "<init>", "(J)V");
jobject obj = env->NewObject(jLongClazz, midLong,
reinterpret_cast<jlong>(handles[i]));
env->CallBooleanMethod(jcfhandle_list,
rocksdb::ListJni::getListAddMethodId(env), obj);
}
return jcfhandle_list;
}
rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
return nullptr;
}
/*
* Class: org_rocksdb_TtlDB
* Method: createColumnFamilyWithTtl
@ -52,9 +153,11 @@ jlong Java_org_rocksdb_TtlDB_createColumnFamilyWithTtl(
rocksdb::ColumnFamilyHandle* handle;
auto db_handle = reinterpret_cast<rocksdb::DBWithTTL*>(jdb_handle);
jstring jstr = (jstring) env->CallObjectMethod(jcf_descriptor,
// get ColumnFamilyName
jbyteArray byteArray = static_cast<jbyteArray>(env->CallObjectMethod(
jcf_descriptor,
rocksdb::ColumnFamilyDescriptorJni::getColumnFamilyNameMethod(
env));
env)));
// get CF Options
jobject jcf_opt_obj = env->CallObjectMethod(jcf_descriptor,
rocksdb::ColumnFamilyDescriptorJni::getColumnFamilyOptionsMethod(
@ -62,10 +165,10 @@ jlong Java_org_rocksdb_TtlDB_createColumnFamilyWithTtl(
rocksdb::ColumnFamilyOptions* cfOptions =
rocksdb::ColumnFamilyOptionsJni::getHandle(env, jcf_opt_obj);
const char* cfname = env->GetStringUTFChars(jstr, 0);
jbyte* cfname = env->GetByteArrayElements(byteArray, 0);
rocksdb::Status s = db_handle->CreateColumnFamilyWithTtl(
*cfOptions, cfname, &handle, jttl);
env->ReleaseStringUTFChars(jstr, cfname);
*cfOptions, reinterpret_cast<char *>(cfname), &handle, jttl);
env->ReleaseByteArrayElements(byteArray, cfname, 0);
if (s.ok()) {
return reinterpret_cast<jlong>(handle);