Merge pull request #356 from fyrz/RocksJava-TableOptions-3.6

[RocksJava] BlockBasedTableConfig & PlainTableConfig enhancements

java/Makefile

@@ -39,11 +39,13 @@ test: java
 	javac org/rocksdb/test/*.java
 	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.WriteBatchTest
 	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.BackupableDBTest
+	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.BlockBasedTableConfigTest
 	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.ColumnFamilyTest
 	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.FilterTest
 	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.KeyMayExistTest
 	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.MemTableTest
 	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.OptionsTest
+	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.PlainTableConfigTest
 	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.ReadOnlyTest
 	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.MergeTest
 	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.ReadOptionsTest

java/org/rocksdb/BlockBasedTableConfig.java

@@ -14,6 +14,7 @@ public class BlockBasedTableConfig extends TableFormatConfig {
   public BlockBasedTableConfig() {
     noBlockCache_ = false;
     blockCacheSize_ = 8 * 1024 * 1024;
+    blockCacheNumShardBits_ = 0;
     blockSize_ = 4 * 1024;
     blockSizeDeviation_ = 10;
     blockRestartInterval_ = 16;
@@ -22,6 +23,9 @@ public class BlockBasedTableConfig extends TableFormatConfig {
     cacheIndexAndFilterBlocks_ = false;
     hashIndexAllowCollision_ = true;
     blockCacheCompressedSize_ = 0;
+    blockCacheCompressedNumShardBits_ = 0;
+    checksumType_ = ChecksumType.kCRC32c;
+    indexType_ = IndexType.kBinarySearch;
   }
 
   /**
@@ -293,6 +297,44 @@ public class BlockBasedTableConfig extends TableFormatConfig {
     return this;
   }
 
+  /**
+   * Sets the checksum type to be used with this table.
+   *
+   * @param checksumType {@link org.rocksdb.ChecksumType} value.
+   * @return the reference to the current option.
+   */
+  public BlockBasedTableConfig setChecksumType(ChecksumType checksumType) {
+    checksumType_ = checksumType;
+    return this;
+  }
+
+  /**
+   *
+   * @return the currently set checksum type
+   */
+  public ChecksumType checksumType() {
+    return checksumType_;
+  }
+
+  /**
+   * Sets the index type to used with this table.
+   *
+   * @param indexType {@link org.rocksdb.IndexType} value
+   * @return the reference to the current option.
+   */
+  public BlockBasedTableConfig setIndexType(IndexType indexType) {
+    indexType_ = indexType;
+    return this;
+  }
+
+  /**
+   *
+   * @return the currently set index type
+   */
+  public IndexType indexType() {
+    return indexType_;
+  }
+
   @Override protected long newTableFactoryHandle() {
     long filterHandle = 0;
     if (filter_ != null) {
@@ -304,7 +346,8 @@ public class BlockBasedTableConfig extends TableFormatConfig {
         blockRestartInterval_, wholeKeyFiltering_,
         filterHandle, cacheIndexAndFilterBlocks_,
         hashIndexAllowCollision_, blockCacheCompressedSize_,
-        blockCacheCompressedNumShardBits_);
+        blockCacheCompressedNumShardBits_,
+        checksumType_.getValue(), indexType_.getValue());
   }
 
   private native long newTableFactoryHandle(
@@ -312,19 +355,21 @@ public class BlockBasedTableConfig extends TableFormatConfig {
       long blockSize, int blockSizeDeviation, int blockRestartInterval,
       boolean wholeKeyFiltering, long filterPolicyHandle,
       boolean cacheIndexAndFilterBlocks, boolean hashIndexAllowCollision,
-      long blockCacheCompressedSize, int blockCacheCompressedNumShardBits);
+      long blockCacheCompressedSize, int blockCacheCompressedNumShardBits,
+      byte checkSumType, byte indexType);
 
+  private boolean cacheIndexAndFilterBlocks_;
+  private IndexType indexType_;
+  private boolean hashIndexAllowCollision_;
+  private ChecksumType checksumType_;
   private boolean noBlockCache_;
+  private long blockSize_;
   private long blockCacheSize_;
   private int blockCacheNumShardBits_;
-  private long shard;
-  private long blockSize_;
-  private int blockSizeDeviation_;
-  private int blockRestartInterval_;
-  private boolean wholeKeyFiltering_;
-  private Filter filter_;
-  private boolean cacheIndexAndFilterBlocks_;
-  private boolean hashIndexAllowCollision_;
   private long blockCacheCompressedSize_;
   private int blockCacheCompressedNumShardBits_;
+  private int blockSizeDeviation_;
+  private int blockRestartInterval_;
+  private Filter filter_;
+  private boolean wholeKeyFiltering_;
 }

java/org/rocksdb/ChecksumType.java

@@ -0,0 +1,39 @@
+// Copyright (c) 2014, Facebook, Inc.  All rights reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+package org.rocksdb;
+
+/**
+ * Checksum types used in conjunction with BlockBasedTable.
+ */
+public enum ChecksumType {
+  /**
+   * Not implemented yet.
+   */
+  kNoChecksum((byte) 0),
+  /**
+   * CRC32 Checksum
+   */
+  kCRC32c((byte) 1),
+  /**
+   * XX Hash
+   */
+  kxxHash((byte) 2);
+
+  /**
+   * Returns the byte value of the enumerations value
+   *
+   * @return byte representation
+   */
+  public byte getValue() {
+    return value_;
+  }
+
+  private ChecksumType(byte value) {
+    value_ = value;
+  }
+
+  private final byte value_;
+}

java/org/rocksdb/EncodingType.java

@@ -0,0 +1,55 @@
+// Copyright (c) 2014, Facebook, Inc.  All rights reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+package org.rocksdb;
+
+/**
+ * EncodingType
+ *
+ * <p>The value will determine how to encode keys
+ * when writing to a new SST file.</p>
+ *
+ * <p>This value will be stored
+ * inside the SST file which will be used when reading from
+ * the file, which makes it possible for users to choose
+ * different encoding type when reopening a DB. Files with
+ * different encoding types can co-exist in the same DB and
+ * can be read.</p>
+ */
+public enum EncodingType {
+  /**
+   * Always write full keys without any special encoding.
+   */
+  kPlain((byte) 0),
+  /**
+   * <p>Find opportunity to write the same prefix once for multiple rows.
+   * In some cases, when a key follows a previous key with the same prefix,
+   * instead of writing out the full key, it just writes out the size of the
+   * shared prefix, as well as other bytes, to save some bytes.</p>
+   *
+   * <p>When using this option, the user is required to use the same prefix
+   * extractor to make sure the same prefix will be extracted from the same key.
+   * The Name() value of the prefix extractor will be stored in the file. When
+   * reopening the file, the name of the options.prefix_extractor given will be
+   * bitwise compared to the prefix extractors stored in the file. An error
+   * will be returned if the two don't match.</p>
+   */
+  kPrefix((byte) 1);
+
+  /**
+   * Returns the byte value of the enumerations value
+   *
+   * @return byte representation
+   */
+  public byte getValue() {
+    return value_;
+  }
+
+  private EncodingType(byte value) {
+    value_ = value;
+  }
+
+  private final byte value_;
+}

java/org/rocksdb/IndexType.java

@@ -0,0 +1,37 @@
+// Copyright (c) 2014, Facebook, Inc.  All rights reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+package org.rocksdb;
+
+/**
+ * IndexType used in conjunction with BlockBasedTable.
+ */
+public enum IndexType {
+  /**
+   * A space efficient index block that is optimized for
+   * binary-search-based index.
+   */
+  kBinarySearch((byte) 0),
+  /**
+   * The hash index, if enabled, will do the hash lookup when
+   * {@code Options.prefix_extractor} is provided.
+   */
+  kHashSearch((byte) 1);
+
+  /**
+   * Returns the byte value of the enumerations value
+   *
+   * @return byte representation
+   */
+  public byte getValue() {
+    return value_;
+  }
+
+  private IndexType(byte value) {
+    value_ = value;
+  }
+
+  private final byte value_;
+}

java/org/rocksdb/PlainTableConfig.java

@@ -7,28 +7,43 @@ package org.rocksdb;
 /**
  * The config for plain table sst format.
  *
- * PlainTable is a RocksDB's SST file format optimized for low query latency
- * on pure-memory or really low-latency media.  It also support prefix
- * hash feature.
+ * <p>PlainTable is a RocksDB's SST file format optimized for low query
+ * latency on pure-memory or really low-latency media.</p>
+ *
+ * <p>It also support prefix hash feature.</p>
  */
 public class PlainTableConfig extends TableFormatConfig {
   public static final int VARIABLE_LENGTH = 0;
   public static final int DEFAULT_BLOOM_BITS_PER_KEY = 10;
   public static final double DEFAULT_HASH_TABLE_RATIO = 0.75;
   public static final int DEFAULT_INDEX_SPARSENESS = 16;
+  public static final int DEFAULT_HUGE_TLB_SIZE = 0;
+  public static final EncodingType DEFAULT_ENCODING_TYPE =
+      EncodingType.kPlain;
+  public static final boolean DEFAULT_FULL_SCAN_MODE = false;
+  public static final boolean DEFAULT_STORE_INDEX_IN_FILE
+      = false;
 
   public PlainTableConfig() {
     keySize_ = VARIABLE_LENGTH;
     bloomBitsPerKey_ = DEFAULT_BLOOM_BITS_PER_KEY;
     hashTableRatio_ = DEFAULT_HASH_TABLE_RATIO;
     indexSparseness_ = DEFAULT_INDEX_SPARSENESS;
+    hugePageTlbSize_ = DEFAULT_HUGE_TLB_SIZE;
+    encodingType_ = DEFAULT_ENCODING_TYPE;
+    fullScanMode_ = DEFAULT_FULL_SCAN_MODE;
+    storeIndexInFile_ = DEFAULT_STORE_INDEX_IN_FILE;
   }
 
   /**
-   * Set the length of the user key. If it is set to be VARIABLE_LENGTH,
-   * then it indicates the user keys are variable-lengthed.  Otherwise,
-   * all the keys need to have the same length in byte.
-   * DEFAULT: VARIABLE_LENGTH
+   * <p>Set the length of the user key. If it is set to be
+   * VARIABLE_LENGTH, then it indicates the user keys are
+   * of variable length.</p>
+   *
+   * <p>Otherwise,all the keys need to have the same length
+   * in byte.</p>
+   *
+   * <p>DEFAULT: VARIABLE_LENGTH</p>
    *
    * @param keySize the length of the user key.
    * @return the reference to the current config.
@@ -103,21 +118,134 @@ public class PlainTableConfig extends TableFormatConfig {
   /**
    * @return the index sparseness.
    */
-  public int indexSparseness() {
+  public long indexSparseness() {
     return indexSparseness_;
   }
 
+  /**
+   * <p>huge_page_tlb_size: if <=0, allocate hash indexes and blooms
+   * from malloc otherwise from huge page TLB.</p>
+   *
+   * <p>The user needs to reserve huge pages for it to be allocated,
+   * like: {@code sysctl -w vm.nr_hugepages=20}</p>
+   *
+   * <p>See linux doc Documentation/vm/hugetlbpage.txt</p>
+   *
+   * @param hugePageTlbSize
+   * @return the reference to the current config.
+   */
+  public PlainTableConfig setHugePageTlbSize(int hugePageTlbSize) {
+    this.hugePageTlbSize_ = hugePageTlbSize;
+    return this;
+  }
+
+  /**
+   * Returns the value for huge page tlb size
+   *
+   * @return hugePageTlbSize
+   */
+  public int hugePageTlbSize() {
+    return hugePageTlbSize_;
+  }
+
+  /**
+   * Sets the encoding type.
+   *
+   * <p>This setting determines how to encode
+   * the keys. See enum {@link EncodingType} for
+   * the choices.</p>
+   *
+   * <p>The value will determine how to encode keys
+   * when writing to a new SST file. This value will be stored
+   * inside the SST file which will be used when reading from
+   * the file, which makes it possible for users to choose
+   * different encoding type when reopening a DB. Files with
+   * different encoding types can co-exist in the same DB and
+   * can be read.</p>
+   *
+   * @param encodingType {@link org.rocksdb.EncodingType} value.
+   * @return the reference to the current config.
+   */
+  public PlainTableConfig setEncodingType(EncodingType encodingType) {
+    this.encodingType_ = encodingType;
+    return this;
+  }
+
+  /**
+   * Returns the active EncodingType
+   *
+   * @return currently set encoding type
+   */
+  public EncodingType encodingType() {
+    return encodingType_;
+  }
+
+  /**
+   * Set full scan mode, if true the whole file will be read
+   * one record by one without using the index.
+   *
+   * @param fullScanMode boolean value indicating if full
+   *     scan mode shall be enabled.
+   * @return the reference to the current config.
+   */
+  public PlainTableConfig setFullScanMode(boolean fullScanMode) {
+    this.fullScanMode_ = fullScanMode;
+    return this;
+  }
+
+  /**
+   * Return if full scan mode is active
+   * @return boolean value indicating if the full scan mode is
+   *     enabled.
+   */
+  public boolean fullScanMode() {
+    return fullScanMode_;
+  }
+
+  /**
+   * <p>If set to true: compute plain table index and bloom
+   * filter during file building and store it in file.
+   * When reading file, index will be mmaped instead
+   * of doing recomputation.</p>
+   *
+   * @param storeIndexInFile value indicating if index shall
+   *     be stored in a file
+   * @return the reference to the current config.
+   */
+  public PlainTableConfig setStoreIndexInFile(boolean storeIndexInFile) {
+    this.storeIndexInFile_ = storeIndexInFile;
+    return this;
+  }
+
+  /**
+   * Return a boolean value indicating if index shall be stored
+   * in a file.
+   *
+   * @return currently set value for store index in file.
+   */
+  public boolean storeIndexInFile() {
+    return storeIndexInFile_;
+  }
+
   @Override protected long newTableFactoryHandle() {
     return newTableFactoryHandle(keySize_, bloomBitsPerKey_,
-        hashTableRatio_, indexSparseness_);
+        hashTableRatio_, indexSparseness_, hugePageTlbSize_,
+        encodingType_.getValue(), fullScanMode_,
+        storeIndexInFile_);
   }
 
   private native long newTableFactoryHandle(
       int keySize, int bloomBitsPerKey,
-      double hashTableRatio, int indexSparseness);
+      double hashTableRatio, int indexSparseness,
+      int hugePageTlbSize, byte encodingType,
+      boolean fullScanMode, boolean storeIndexInFile);
 
   private int keySize_;
   private int bloomBitsPerKey_;
   private double hashTableRatio_;
   private int indexSparseness_;
+  private int hugePageTlbSize_;
+  private EncodingType encodingType_;
+  private boolean fullScanMode_;
+  private boolean storeIndexInFile_;
 }

java/org/rocksdb/test/BlockBasedTableConfigTest.java

@@ -0,0 +1,47 @@
+// Copyright (c) 2014, Facebook, Inc.  All rights reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+package org.rocksdb.test;
+
+import org.rocksdb.BlockBasedTableConfig;
+import org.rocksdb.ChecksumType;
+import org.rocksdb.IndexType;
+
+public class BlockBasedTableConfigTest {
+
+  public static void main(String[] args) {
+    BlockBasedTableConfig blockBasedTableConfig =
+        new BlockBasedTableConfig();
+    blockBasedTableConfig.setNoBlockCache(true);
+    assert(blockBasedTableConfig.noBlockCache());
+    blockBasedTableConfig.setBlockCacheSize(8*1024);
+    assert(blockBasedTableConfig.blockCacheSize() == (8*1024));
+    blockBasedTableConfig.setBlockSizeDeviation(12);
+    assert(blockBasedTableConfig.blockSizeDeviation() == 12);
+    blockBasedTableConfig.setBlockRestartInterval(15);
+    assert(blockBasedTableConfig.blockRestartInterval() == 15);
+    blockBasedTableConfig.setWholeKeyFiltering(false);
+    assert(!blockBasedTableConfig.wholeKeyFiltering());
+    blockBasedTableConfig.setCacheIndexAndFilterBlocks(true);
+    assert(blockBasedTableConfig.cacheIndexAndFilterBlocks());
+    blockBasedTableConfig.setHashIndexAllowCollision(false);
+    assert(!blockBasedTableConfig.hashIndexAllowCollision());
+    blockBasedTableConfig.setBlockCacheCompressedSize(40);
+    assert(blockBasedTableConfig.blockCacheCompressedSize() == 40);
+    blockBasedTableConfig.setChecksumType(ChecksumType.kNoChecksum);
+    blockBasedTableConfig.setChecksumType(ChecksumType.kxxHash);
+    assert(blockBasedTableConfig.checksumType().equals(
+        ChecksumType.kxxHash));
+    blockBasedTableConfig.setIndexType(IndexType.kHashSearch);
+    assert(blockBasedTableConfig.indexType().equals(
+        IndexType.kHashSearch));
+    blockBasedTableConfig.setBlockCacheCompressedNumShardBits(4);
+    assert(blockBasedTableConfig.blockCacheCompressedNumShardBits()
+        == 4);
+    blockBasedTableConfig.setCacheNumShardBits(5);
+    assert(blockBasedTableConfig.cacheNumShardBits() == 5);
+    System.out.println("BlockBasedTableConfig test passed");
+  }
+}

java/org/rocksdb/test/PlainTableConfigTest.java

@@ -0,0 +1,34 @@
+// Copyright (c) 2014, Facebook, Inc.  All rights reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+package org.rocksdb.test;
+
+import org.rocksdb.EncodingType;
+import org.rocksdb.PlainTableConfig;
+
+public class PlainTableConfigTest {
+
+  public static void main(String[] args) {
+    PlainTableConfig plainTableConfig = new PlainTableConfig();
+    plainTableConfig.setKeySize(5);
+    assert(plainTableConfig.keySize() == 5);
+    plainTableConfig.setBloomBitsPerKey(11);
+    assert(plainTableConfig.bloomBitsPerKey() == 11);
+    plainTableConfig.setHashTableRatio(0.95);
+    assert(plainTableConfig.hashTableRatio() == 0.95);
+    plainTableConfig.setIndexSparseness(18);
+    assert(plainTableConfig.indexSparseness() == 18);
+    plainTableConfig.setHugePageTlbSize(1);
+    assert(plainTableConfig.hugePageTlbSize() == 1);
+    plainTableConfig.setEncodingType(EncodingType.kPrefix);
+    assert(plainTableConfig.encodingType().equals(
+        EncodingType.kPrefix));
+    plainTableConfig.setFullScanMode(true);
+    assert(plainTableConfig.fullScanMode());
+    plainTableConfig.setStoreIndexInFile(true);
+    assert(plainTableConfig.storeIndexInFile());
+    System.out.println("PlainTableConfig test passed");
+  }
+}

java/rocksjni/table.cc

@@ -15,23 +15,30 @@
 /*
  * Class:     org_rocksdb_PlainTableConfig
  * Method:    newTableFactoryHandle
- * Signature: (IIDI)J
+ * Signature: (IIDIIBZZ)J
  */
 jlong Java_org_rocksdb_PlainTableConfig_newTableFactoryHandle(
     JNIEnv* env, jobject jobj, jint jkey_size, jint jbloom_bits_per_key,
-    jdouble jhash_table_ratio, jint jindex_sparseness) {
+    jdouble jhash_table_ratio, jint jindex_sparseness,
+    jint jhuge_page_tlb_size, jbyte jencoding_type,
+    jboolean jfull_scan_mode, jboolean jstore_index_in_file) {
   rocksdb::PlainTableOptions options = rocksdb::PlainTableOptions();
   options.user_key_len = jkey_size;
   options.bloom_bits_per_key = jbloom_bits_per_key;
   options.hash_table_ratio = jhash_table_ratio;
   options.index_sparseness = jindex_sparseness;
+  options.huge_page_tlb_size = jhuge_page_tlb_size;
+  options.encoding_type = static_cast<rocksdb::EncodingType>(
+      jencoding_type);
+  options.full_scan_mode = jfull_scan_mode;
+  options.store_index_in_file = jstore_index_in_file;
   return reinterpret_cast<jlong>(rocksdb::NewPlainTableFactory(options));
 }
 
 /*
  * Class:     org_rocksdb_BlockBasedTableConfig
  * Method:    newTableFactoryHandle
- * Signature: (ZJIJIIZIZZJI)J
+ * Signature: (ZJIJIIZIZZJIBB)J
  */
 jlong Java_org_rocksdb_BlockBasedTableConfig_newTableFactoryHandle(
     JNIEnv* env, jobject jobj, jboolean no_block_cache, jlong block_cache_size,
@@ -39,7 +46,8 @@ jlong Java_org_rocksdb_BlockBasedTableConfig_newTableFactoryHandle(
     jint block_restart_interval, jboolean whole_key_filtering,
     jlong jfilterPolicy, jboolean cache_index_and_filter_blocks,
     jboolean hash_index_allow_collision, jlong block_cache_compressed_size,
-    jint block_cache_compressd_num_shard_bits) {
+    jint block_cache_compressd_num_shard_bits, jbyte jchecksum_type,
+    jbyte jindex_type) {
   rocksdb::BlockBasedTableOptions options;
   options.no_block_cache = no_block_cache;
 
@@ -72,6 +80,9 @@ jlong Java_org_rocksdb_BlockBasedTableConfig_newTableFactoryHandle(
       options.block_cache = rocksdb::NewLRUCache(block_cache_compressed_size);
     }
   }
+  options.checksum = static_cast<rocksdb::ChecksumType>(jchecksum_type);
+  options.index_type = static_cast<
+      rocksdb::BlockBasedTableOptions::IndexType>(jindex_type);
 
   return reinterpret_cast<jlong>(rocksdb::NewBlockBasedTableFactory(options));
 }