From cfee7fb51a41a8af5d46679b0702b3c5654f644b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20P=C3=A9pin-Perreault?= Date: Wed, 11 Jul 2018 15:44:52 -0700 Subject: [PATCH] Allow storing metadata with backups for Java API (#4111) Summary: Exposes BackupEngine::CreateNewBackupWithMetadata and BackupInfo metadata to the Java API. Full disclaimer, I'm not familiar with JNI stuff, so I might have forgotten something (hopefully no memory leaks!). I also tried to find contributing guidelines but didn't see any, but I hope the PR style is consistent with the rest of the code base. Pull Request resolved: https://github.com/facebook/rocksdb/pull/4111 Differential Revision: D8811180 Pulled By: ajkr fbshipit-source-id: e38b3e396c7574328c2a1a0e55acc8d092b6a569 --- java/rocksjni/backupenginejni.cc | 29 ++++++++++++++++ java/rocksjni/portal.h | 20 ++++++++--- .../main/java/org/rocksdb/BackupEngine.java | 33 +++++++++++++++++++ .../src/main/java/org/rocksdb/BackupInfo.java | 12 ++++++- .../java/org/rocksdb/BackupEngineTest.java | 24 ++++++++++++++ 5 files changed, 113 insertions(+), 5 deletions(-) diff --git a/java/rocksjni/backupenginejni.cc b/java/rocksjni/backupenginejni.cc index 3b23dc85f3..34ee6d1969 100644 --- a/java/rocksjni/backupenginejni.cc +++ b/java/rocksjni/backupenginejni.cc @@ -56,6 +56,35 @@ void Java_org_rocksdb_BackupEngine_createNewBackup( rocksdb::RocksDBExceptionJni::ThrowNew(env, status); } +/* + * Class: org_rocksdb_BackupEngine + * Method: createNewBackupWithMetadata + * Signature: (JJLjava/lang/String;Z)V + */ +void Java_org_rocksdb_BackupEngine_createNewBackupWithMetadata( + JNIEnv* env, jobject /*jbe*/, jlong jbe_handle, jlong db_handle, + jstring japp_metadata, jboolean jflush_before_backup) { + auto* db = reinterpret_cast(db_handle); + auto* backup_engine = reinterpret_cast(jbe_handle); + + jboolean has_exception = JNI_FALSE; + std::string app_metadata = rocksdb::JniUtil::copyStdString( + env, japp_metadata, &has_exception); + if (has_exception == JNI_TRUE) { + rocksdb::RocksDBExceptionJni::ThrowNew(env, "Could not copy jstring to std::string"); + return; + } + + auto status = backup_engine->CreateNewBackupWithMetadata( + db, app_metadata, static_cast(jflush_before_backup)); + + if (status.ok()) { + return; + } + + rocksdb::RocksDBExceptionJni::ThrowNew(env, status); +} + /* * Class: org_rocksdb_BackupEngine * Method: getBackupInfo diff --git a/java/rocksjni/portal.h b/java/rocksjni/portal.h index aa41eff336..e6c9da7ee1 100644 --- a/java/rocksjni/portal.h +++ b/java/rocksjni/portal.h @@ -2364,27 +2364,38 @@ class BackupInfoJni : public JavaClass { * @param timestamp timestamp of the backup * @param size size of the backup * @param number_files number of files related to the backup + * @param app_metadata application specific metadata * * @return A reference to a Java BackupInfo object, or a nullptr if an * exception occurs */ static jobject construct0(JNIEnv* env, uint32_t backup_id, int64_t timestamp, - uint64_t size, uint32_t number_files) { + uint64_t size, uint32_t number_files, const std::string& app_metadata) { jclass jclazz = getJClass(env); if(jclazz == nullptr) { // exception occurred accessing class return nullptr; } - static jmethodID mid = env->GetMethodID(jclazz, "", "(IJJI)V"); + static jmethodID mid = env->GetMethodID(jclazz, "", "(IJJILjava/lang/String;)V"); if(mid == nullptr) { // exception occurred accessing method return nullptr; } + jstring japp_metadata = nullptr; + if (app_metadata != nullptr) { + japp_metadata = env->NewStringUTF(app_metadata.c_str()); + if (japp_metadata == nullptr) { + // exception occurred creating java string + return nullptr; + } + } + jobject jbackup_info = - env->NewObject(jclazz, mid, backup_id, timestamp, size, number_files); + env->NewObject(jclazz, mid, backup_id, timestamp, size, number_files, japp_metadata); if(env->ExceptionCheck()) { + env->DeleteLocalRef(japp_metadata); return nullptr; } @@ -2441,7 +2452,8 @@ class BackupInfoListJni { backup_info.backup_id, backup_info.timestamp, backup_info.size, - backup_info.number_files); + backup_info.number_files, + backup_info.app_metadata); if(env->ExceptionCheck()) { // exception occurred constructing object if(obj != nullptr) { diff --git a/java/src/main/java/org/rocksdb/BackupEngine.java b/java/src/main/java/org/rocksdb/BackupEngine.java index 763994575c..e41c265f43 100644 --- a/java/src/main/java/org/rocksdb/BackupEngine.java +++ b/java/src/main/java/org/rocksdb/BackupEngine.java @@ -81,6 +81,36 @@ public class BackupEngine extends RocksObject implements AutoCloseable { createNewBackup(nativeHandle_, db.nativeHandle_, flushBeforeBackup); } + /** + * Captures the state of the database in the latest backup along with + * application specific metadata. + * + * @param db The database to backup + * @param metadata Application metadata + * @param flushBeforeBackup When true, the Backup Engine will first issue a + * memtable flush and only then copy the DB files to + * the backup directory. Doing so will prevent log + * files from being copied to the backup directory + * (since flush will delete them). + * When false, the Backup Engine will not issue a + * flush before starting the backup. In that case, + * the backup will also include log files + * corresponding to live memtables. The backup will + * always be consistent with the current state of the + * database regardless of the flushBeforeBackup + * parameter. + * + * Note - This method is not thread safe + * + * @throws RocksDBException thrown if a new backup could not be created + */ + public void createNewBackupWithMetadata( + final RocksDB db, final String metadata, final boolean flushBeforeBackup) + throws RocksDBException { + assert (isOwningHandle()); + createNewBackupWithMetadata(nativeHandle_, db.nativeHandle_, metadata, flushBeforeBackup); + } + /** * Gets information about the available * backups @@ -197,6 +227,9 @@ public class BackupEngine extends RocksObject implements AutoCloseable { private native void createNewBackup(final long handle, final long dbHandle, final boolean flushBeforeBackup) throws RocksDBException; + private native void createNewBackupWithMetadata(final long handle, final long dbHandle, + final String metadata, final boolean flushBeforeBackup) throws RocksDBException; + private native List getBackupInfo(final long handle); private native int[] getCorruptedBackups(final long handle); diff --git a/java/src/main/java/org/rocksdb/BackupInfo.java b/java/src/main/java/org/rocksdb/BackupInfo.java index 10f418629a..c1c955b94b 100644 --- a/java/src/main/java/org/rocksdb/BackupInfo.java +++ b/java/src/main/java/org/rocksdb/BackupInfo.java @@ -20,11 +20,12 @@ public class BackupInfo { * @param numberFiles number of files related to this backup. */ BackupInfo(final int backupId, final long timestamp, final long size, - final int numberFiles) { + final int numberFiles, final String app_metadata) { backupId_ = backupId; timestamp_ = timestamp; size_ = size; numberFiles_ = numberFiles; + app_metadata_ = app_metadata; } /** @@ -59,8 +60,17 @@ public class BackupInfo { return numberFiles_; } + /** + * + * @return the associated application metadata, or null + */ + public String appMetadata() { + return app_metadata_; + } + private int backupId_; private long timestamp_; private long size_; private int numberFiles_; + private String app_metadata_; } diff --git a/java/src/test/java/org/rocksdb/BackupEngineTest.java b/java/src/test/java/org/rocksdb/BackupEngineTest.java index 1caae5098e..03eb60384a 100644 --- a/java/src/test/java/org/rocksdb/BackupEngineTest.java +++ b/java/src/test/java/org/rocksdb/BackupEngineTest.java @@ -11,6 +11,7 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; import java.util.List; +import java.util.concurrent.ThreadLocalRandom; import static org.assertj.core.api.Assertions.assertThat; @@ -205,6 +206,29 @@ public class BackupEngineTest { } } + @Test + public void backupDbWithMetadata() + throws RocksDBException { + // Open empty database. + try(final Options opt = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(opt, + dbFolder.getRoot().getAbsolutePath())) { + + // Fill database with some test values + prepareDatabase(db); + + // Create two backups + try(final BackupableDBOptions bopt = new BackupableDBOptions( + backupFolder.getRoot().getAbsolutePath()); + final BackupEngine be = BackupEngine.open(opt.getEnv(), bopt)) { + final String metadata = String.valueOf(ThreadLocalRandom.current().nextInt()); + be.createNewBackupWithMetadata(db, metadata, true); + final List backupInfoList = verifyNumberOfValidBackups(be, 1); + assertThat(backupInfoList.get(0).appMetadata()).isEqualTo(metadata); + } + } + } + /** * Verify backups. *