From 143ee5a1c1d2788ff46bf0cc03087be0f8a8c273 Mon Sep 17 00:00:00 2001 From: Ankit Gupta Date: Sun, 22 Jun 2014 13:23:10 -0700 Subject: [PATCH 1/5] Add stats collector --- java/org/rocksdb/StatisticsCollector.java | 86 +++++++++++++++++++ .../rocksdb/StatisticsCollectorCallback.java | 27 ++++++ .../rocksdb/test/StatisticsCollectorTest.java | 39 +++++++++ java/org/rocksdb/test/StatsCallbackMock.java | 22 +++++ 4 files changed, 174 insertions(+) create mode 100644 java/org/rocksdb/StatisticsCollector.java create mode 100644 java/org/rocksdb/StatisticsCollectorCallback.java create mode 100644 java/org/rocksdb/test/StatisticsCollectorTest.java create mode 100644 java/org/rocksdb/test/StatsCallbackMock.java diff --git a/java/org/rocksdb/StatisticsCollector.java b/java/org/rocksdb/StatisticsCollector.java new file mode 100644 index 0000000000..b771ce15db --- /dev/null +++ b/java/org/rocksdb/StatisticsCollector.java @@ -0,0 +1,86 @@ +// Copyright (c) 2014, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +package org.rocksdb; + +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Helper class to collect DB statistics periodically at a period specified in + * constructor. Callback function (provided in constructor) is called with + * every statistics collection. + * + * Caller should call start() to start statistics collection. Shutdown() should + * be called to stop stats collection and should be called before statistics ( + * provided in constructor) reference has been disposed. + */ +public class StatisticsCollector { + private final Statistics _statistics; + private final ThreadPoolExecutor _threadPoolExecutor; + private final int _statsCollectionInterval; + private final StatisticsCollectorCallback _statsCallback; + private volatile boolean _isRunning = true; + + public StatisticsCollector(Statistics statistics, + int statsCollectionIntervalInMilliSeconds, + StatisticsCollectorCallback statsCallback) { + _statistics = statistics; + _statsCollectionInterval = statsCollectionIntervalInMilliSeconds; + _statsCallback = statsCallback; + + _threadPoolExecutor = new ThreadPoolExecutor(1, 1, 0, TimeUnit.SECONDS, + new ArrayBlockingQueue(1)); + } + + public void start() { + _threadPoolExecutor.submit(collectStatistics()); + } + + public void shutDown() throws InterruptedException { + _isRunning = false; + + _threadPoolExecutor.shutdown(); + // Wait for collectStatistics runnable to finish so that disposal of + // statistics does not cause any exceptions to be thrown. + _threadPoolExecutor.awaitTermination(Integer.MAX_VALUE, TimeUnit.SECONDS); + } + + private Runnable collectStatistics() { + return new Runnable() { + + @Override + public void run() { + while (_isRunning) { + try { + // Collect ticker data + for(TickerType ticker : TickerType.values()) { + long tickerValue = _statistics.getTickerCount(ticker); + _statsCallback.tickerCallback(ticker, tickerValue); + } + + // Collect histogram data + for(HistogramType histogramType : HistogramType.values()) { + HistogramData histogramData = + _statistics.geHistogramData(histogramType); + _statsCallback.histogramCallback(histogramType, histogramData); + } + + Thread.sleep(_statsCollectionInterval); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Thread got interrupted!", e); + } + catch (Exception e) { + throw new RuntimeException("Error while calculating statistics", e); + } + } + } + }; + } +} \ No newline at end of file diff --git a/java/org/rocksdb/StatisticsCollectorCallback.java b/java/org/rocksdb/StatisticsCollectorCallback.java new file mode 100644 index 0000000000..37f53684e7 --- /dev/null +++ b/java/org/rocksdb/StatisticsCollectorCallback.java @@ -0,0 +1,27 @@ +// Copyright (c) 2014, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +package org.rocksdb; + +/** + * Callback interface provided to StatisticsCollector. + * @param tickerType + * @param tickerCount +*/ +public interface StatisticsCollectorCallback { + /** + * Callback function to get ticker values. + * @param tickerType Ticker type. + * @param tickerCount Value of ticker type. + */ + void tickerCallback(TickerType tickerType, long tickerCount); + + /** + * Callback function to get histogram values. + * @param histType Histogram type. + * @param histData Histogram data. + */ + void histogramCallback(HistogramType histType, HistogramData histData); +} \ No newline at end of file diff --git a/java/org/rocksdb/test/StatisticsCollectorTest.java b/java/org/rocksdb/test/StatisticsCollectorTest.java new file mode 100644 index 0000000000..e706528b15 --- /dev/null +++ b/java/org/rocksdb/test/StatisticsCollectorTest.java @@ -0,0 +1,39 @@ +// Copyright (c) 2014, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +package org.rocksdb.test; + +import org.rocksdb.*; + +public class StatisticsCollectorTest { + static final String db_path = "/tmp/backupablejni_db"; + static { + RocksDB.loadLibrary(); + } + + public static void main(String[] args) throws InterruptedException, RocksDBException { + Options opt = new Options().createStatistics().setCreateIfMissing(true); + Statistics stats = opt.statisticsPtr(); + + RocksDB db = RocksDB.open(db_path); + + StatsCallbackMock callback = new StatsCallbackMock(); + StatisticsCollector statsCollector = new StatisticsCollector(stats, 100, + callback); + statsCollector.start(); + + Thread.sleep(1000); + + assert(callback.tickerCallbackCount > 0); + assert(callback.histCallbackCount > 0); + + statsCollector.shutDown(); + + db.close(); + opt.dispose(); + + System.out.println("Stats collector test passed.!"); + } +} \ No newline at end of file diff --git a/java/org/rocksdb/test/StatsCallbackMock.java b/java/org/rocksdb/test/StatsCallbackMock.java new file mode 100644 index 0000000000..2cb70f799c --- /dev/null +++ b/java/org/rocksdb/test/StatsCallbackMock.java @@ -0,0 +1,22 @@ +// Copyright (c) 2014, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +package org.rocksdb.test; + +import org.rocksdb.*; + +public class StatsCallbackMock implements StatisticsCollectorCallback { + public int tickerCallbackCount = 0; + public int histCallbackCount = 0; + + public void tickerCallback(TickerType tickerType, long tickerCount) { + tickerCallbackCount++; + } + + public void histogramCallback(HistogramType histType, + HistogramData histData) { + histCallbackCount++; + } +} \ No newline at end of file From e0ebea6cc21de82240be63dd2a2f7a4f5cc68cfc Mon Sep 17 00:00:00 2001 From: Ankit Gupta Date: Sun, 22 Jun 2014 13:27:22 -0700 Subject: [PATCH 2/5] Add doc and end of line --- Makefile | 2 +- java/Makefile | 1 + java/org/rocksdb/RocksDB.java | 5 +---- java/org/rocksdb/StatisticsCollector.java | 4 ++-- java/org/rocksdb/StatisticsCollectorCallback.java | 2 +- java/org/rocksdb/test/StatisticsCollectorTest.java | 2 +- java/org/rocksdb/test/StatsCallbackMock.java | 2 +- 7 files changed, 8 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index c31e1e8cac..ca419f08b5 100644 --- a/Makefile +++ b/Makefile @@ -439,7 +439,7 @@ ROCKSDBJNILIB = ./java/librocksdbjni.jnilib JAVA_INCLUDE = -I/System/Library/Frameworks/JavaVM.framework/Headers/ endif -rocksdbjava: clean +rocksdbjava: OPT="-fPIC -DNDEBUG -O2" $(MAKE) $(LIBRARY) -j32 cd java;$(MAKE) java; rm -f $(ROCKSDBJNILIB) diff --git a/java/Makefile b/java/Makefile index 36452d3241..7a9c64732b 100644 --- a/java/Makefile +++ b/java/Makefile @@ -29,6 +29,7 @@ test: java java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.BackupableDBTest java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.OptionsTest java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.ReadOptionsTest + java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.StatisticsCollectorTest db_bench: java javac org/rocksdb/benchmark/*.java diff --git a/java/org/rocksdb/RocksDB.java b/java/org/rocksdb/RocksDB.java index 1b758e1a2e..cec73ed498 100644 --- a/java/org/rocksdb/RocksDB.java +++ b/java/org/rocksdb/RocksDB.java @@ -93,10 +93,7 @@ public class RocksDB extends RocksObject { // This allows to use the rocksjni default Options instead of // the c++ one. Options options = new Options(); - db.open(options.nativeHandle_, options.cacheSize_, path); - db.transferCppRawPointersOwnershipFrom(options); - options.dispose(); - return db; + return open(options, path); } /** diff --git a/java/org/rocksdb/StatisticsCollector.java b/java/org/rocksdb/StatisticsCollector.java index b771ce15db..e642e4f0b1 100644 --- a/java/org/rocksdb/StatisticsCollector.java +++ b/java/org/rocksdb/StatisticsCollector.java @@ -57,7 +57,7 @@ public class StatisticsCollector { public void run() { while (_isRunning) { try { - // Collect ticker data + // Collect ticker data for(TickerType ticker : TickerType.values()) { long tickerValue = _statistics.getTickerCount(ticker); _statsCallback.tickerCallback(ticker, tickerValue); @@ -83,4 +83,4 @@ public class StatisticsCollector { } }; } -} \ No newline at end of file +} diff --git a/java/org/rocksdb/StatisticsCollectorCallback.java b/java/org/rocksdb/StatisticsCollectorCallback.java index 37f53684e7..95671915bc 100644 --- a/java/org/rocksdb/StatisticsCollectorCallback.java +++ b/java/org/rocksdb/StatisticsCollectorCallback.java @@ -24,4 +24,4 @@ public interface StatisticsCollectorCallback { * @param histData Histogram data. */ void histogramCallback(HistogramType histType, HistogramData histData); -} \ No newline at end of file +} diff --git a/java/org/rocksdb/test/StatisticsCollectorTest.java b/java/org/rocksdb/test/StatisticsCollectorTest.java index e706528b15..ed89938408 100644 --- a/java/org/rocksdb/test/StatisticsCollectorTest.java +++ b/java/org/rocksdb/test/StatisticsCollectorTest.java @@ -36,4 +36,4 @@ public class StatisticsCollectorTest { System.out.println("Stats collector test passed.!"); } -} \ No newline at end of file +} diff --git a/java/org/rocksdb/test/StatsCallbackMock.java b/java/org/rocksdb/test/StatsCallbackMock.java index 2cb70f799c..355b7368f1 100644 --- a/java/org/rocksdb/test/StatsCallbackMock.java +++ b/java/org/rocksdb/test/StatsCallbackMock.java @@ -19,4 +19,4 @@ public class StatsCallbackMock implements StatisticsCollectorCallback { HistogramData histData) { histCallbackCount++; } -} \ No newline at end of file +} From 718029fc388b6640729d5425c291f0c072c88586 Mon Sep 17 00:00:00 2001 From: Ankit Gupta Date: Sun, 22 Jun 2014 13:30:26 -0700 Subject: [PATCH 3/5] Arc lint fixes --- java/org/rocksdb/BackupableDBOptions.java | 4 ++-- java/org/rocksdb/StatisticsCollector.java | 22 +++++++++---------- .../rocksdb/StatisticsCollectorCallback.java | 2 +- .../rocksdb/test/StatisticsCollectorTest.java | 18 +++++++-------- java/org/rocksdb/test/StatsCallbackMock.java | 4 ++-- 5 files changed, 25 insertions(+), 25 deletions(-) diff --git a/java/org/rocksdb/BackupableDBOptions.java b/java/org/rocksdb/BackupableDBOptions.java index c72b44f8c9..2c5047f773 100644 --- a/java/org/rocksdb/BackupableDBOptions.java +++ b/java/org/rocksdb/BackupableDBOptions.java @@ -38,10 +38,10 @@ public class BackupableDBOptions extends RocksObject { boolean destroyOldData, boolean backupLogFiles, long backupRateLimit, long restoreRateLimit) { super(); - + backupRateLimit = (backupRateLimit <= 0) ? 0 : backupRateLimit; restoreRateLimit = (restoreRateLimit <= 0) ? 0 : restoreRateLimit; - + newBackupableDBOptions(path, shareTableFiles, sync, destroyOldData, backupLogFiles, backupRateLimit, restoreRateLimit); } diff --git a/java/org/rocksdb/StatisticsCollector.java b/java/org/rocksdb/StatisticsCollector.java index e642e4f0b1..eb3121a129 100644 --- a/java/org/rocksdb/StatisticsCollector.java +++ b/java/org/rocksdb/StatisticsCollector.java @@ -14,7 +14,7 @@ import java.util.concurrent.atomic.AtomicBoolean; * Helper class to collect DB statistics periodically at a period specified in * constructor. Callback function (provided in constructor) is called with * every statistics collection. - * + * * Caller should call start() to start statistics collection. Shutdown() should * be called to stop stats collection and should be called before statistics ( * provided in constructor) reference has been disposed. @@ -25,31 +25,31 @@ public class StatisticsCollector { private final int _statsCollectionInterval; private final StatisticsCollectorCallback _statsCallback; private volatile boolean _isRunning = true; - - public StatisticsCollector(Statistics statistics, + + public StatisticsCollector(Statistics statistics, int statsCollectionIntervalInMilliSeconds, StatisticsCollectorCallback statsCallback) { _statistics = statistics; _statsCollectionInterval = statsCollectionIntervalInMilliSeconds; _statsCallback = statsCallback; - + _threadPoolExecutor = new ThreadPoolExecutor(1, 1, 0, TimeUnit.SECONDS, new ArrayBlockingQueue(1)); } - + public void start() { _threadPoolExecutor.submit(collectStatistics()); } - + public void shutDown() throws InterruptedException { _isRunning = false; - + _threadPoolExecutor.shutdown(); - // Wait for collectStatistics runnable to finish so that disposal of + // Wait for collectStatistics runnable to finish so that disposal of // statistics does not cause any exceptions to be thrown. _threadPoolExecutor.awaitTermination(Integer.MAX_VALUE, TimeUnit.SECONDS); } - + private Runnable collectStatistics() { return new Runnable() { @@ -65,11 +65,11 @@ public class StatisticsCollector { // Collect histogram data for(HistogramType histogramType : HistogramType.values()) { - HistogramData histogramData = + HistogramData histogramData = _statistics.geHistogramData(histogramType); _statsCallback.histogramCallback(histogramType, histogramData); } - + Thread.sleep(_statsCollectionInterval); } catch (InterruptedException e) { diff --git a/java/org/rocksdb/StatisticsCollectorCallback.java b/java/org/rocksdb/StatisticsCollectorCallback.java index 95671915bc..a2630db369 100644 --- a/java/org/rocksdb/StatisticsCollectorCallback.java +++ b/java/org/rocksdb/StatisticsCollectorCallback.java @@ -17,7 +17,7 @@ public interface StatisticsCollectorCallback { * @param tickerCount Value of ticker type. */ void tickerCallback(TickerType tickerType, long tickerCount); - + /** * Callback function to get histogram values. * @param histType Histogram type. diff --git a/java/org/rocksdb/test/StatisticsCollectorTest.java b/java/org/rocksdb/test/StatisticsCollectorTest.java index ed89938408..25306f2f2f 100644 --- a/java/org/rocksdb/test/StatisticsCollectorTest.java +++ b/java/org/rocksdb/test/StatisticsCollectorTest.java @@ -12,28 +12,28 @@ public class StatisticsCollectorTest { static { RocksDB.loadLibrary(); } - + public static void main(String[] args) throws InterruptedException, RocksDBException { Options opt = new Options().createStatistics().setCreateIfMissing(true); Statistics stats = opt.statisticsPtr(); - + RocksDB db = RocksDB.open(db_path); - + StatsCallbackMock callback = new StatsCallbackMock(); - StatisticsCollector statsCollector = new StatisticsCollector(stats, 100, + StatisticsCollector statsCollector = new StatisticsCollector(stats, 100, callback); statsCollector.start(); - + Thread.sleep(1000); - + assert(callback.tickerCallbackCount > 0); assert(callback.histCallbackCount > 0); - + statsCollector.shutDown(); - + db.close(); opt.dispose(); - + System.out.println("Stats collector test passed.!"); } } diff --git a/java/org/rocksdb/test/StatsCallbackMock.java b/java/org/rocksdb/test/StatsCallbackMock.java index 355b7368f1..4ad2fb7b79 100644 --- a/java/org/rocksdb/test/StatsCallbackMock.java +++ b/java/org/rocksdb/test/StatsCallbackMock.java @@ -10,11 +10,11 @@ import org.rocksdb.*; public class StatsCallbackMock implements StatisticsCollectorCallback { public int tickerCallbackCount = 0; public int histCallbackCount = 0; - + public void tickerCallback(TickerType tickerType, long tickerCount) { tickerCallbackCount++; } - + public void histogramCallback(HistogramType histType, HistogramData histData) { histCallbackCount++; From 4938032549cc41511d91f7b8dd6dd70d26c2c3a5 Mon Sep 17 00:00:00 2001 From: Ankit Gupta Date: Sun, 22 Jun 2014 13:31:10 -0700 Subject: [PATCH 4/5] Fix line width --- java/org/rocksdb/test/StatisticsCollectorTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/java/org/rocksdb/test/StatisticsCollectorTest.java b/java/org/rocksdb/test/StatisticsCollectorTest.java index 25306f2f2f..973a5d383d 100644 --- a/java/org/rocksdb/test/StatisticsCollectorTest.java +++ b/java/org/rocksdb/test/StatisticsCollectorTest.java @@ -13,7 +13,8 @@ public class StatisticsCollectorTest { RocksDB.loadLibrary(); } - public static void main(String[] args) throws InterruptedException, RocksDBException { + public static void main(String[] args) + throws InterruptedException, RocksDBException { Options opt = new Options().createStatistics().setCreateIfMissing(true); Statistics stats = opt.statisticsPtr(); From b2625a4cc10baf5b2a85632113b46fbd70da0aab Mon Sep 17 00:00:00 2001 From: Ankit Gupta Date: Sun, 29 Jun 2014 07:52:27 -0700 Subject: [PATCH 5/5] Add java docs for StatsCollector constructor and StatsCollectorCallback thread safety --- java/org/rocksdb/StatisticsCollector.java | 7 +++++++ java/org/rocksdb/StatisticsCollectorCallback.java | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/java/org/rocksdb/StatisticsCollector.java b/java/org/rocksdb/StatisticsCollector.java index eb3121a129..059e63cb54 100644 --- a/java/org/rocksdb/StatisticsCollector.java +++ b/java/org/rocksdb/StatisticsCollector.java @@ -26,6 +26,13 @@ public class StatisticsCollector { private final StatisticsCollectorCallback _statsCallback; private volatile boolean _isRunning = true; + /** + * Constructor for statistics collector. + * @param statistics Reference of DB statistics. + * @param statsCollectionIntervalInMilliSeconds Statistics collection time + * period (specified in milliseconds) + * @param statsCallback Reference of statistics callback interface. + */ public StatisticsCollector(Statistics statistics, int statsCollectionIntervalInMilliSeconds, StatisticsCollectorCallback statsCallback) { diff --git a/java/org/rocksdb/StatisticsCollectorCallback.java b/java/org/rocksdb/StatisticsCollectorCallback.java index a2630db369..d50b109d2c 100644 --- a/java/org/rocksdb/StatisticsCollectorCallback.java +++ b/java/org/rocksdb/StatisticsCollectorCallback.java @@ -7,6 +7,13 @@ package org.rocksdb; /** * Callback interface provided to StatisticsCollector. + * + * Thread safety: + * StatisticsCollector doesn't make any guarantees about thread safety. + * If the same reference of StatisticsCollectorCallback is passed to multiple + * StatisticsCollector references, then its the responsibility of the + * user to make StatisticsCollectorCallback' implementation thread-safe. + * * @param tickerType * @param tickerCount */