From 80c663882adb27af5b8e8b6ae3e4639b534c5c8a Mon Sep 17 00:00:00 2001 From: Dhruba Borthakur Date: Mon, 2 Jul 2012 22:45:59 -0700 Subject: [PATCH] Create leveldb server via Thrift. Summary: First draft. Unit tests pass. Test Plan: unit tests attached Reviewers: heyongqiang Reviewed By: heyongqiang Differential Revision: https://reviews.facebook.net/D3969 --- Makefile | 16 +- build_detect_platform | 17 +- fbcode.sh | 9 +- hdfs/README | 2 +- snappy/README | 2 +- thrift/README | 18 + thrift/folly/Likely.h | 39 + thrift/folly/experimental/io/IOBuf.h | 1050 +++++ thrift/folly/experimental/io/IOBufQueue.h | 201 + thrift/folly/experimental/io/check.h | 2 + thrift/gen-cpp/DB.cpp | 4161 +++++++++++++++++ thrift/gen-cpp/DB.h | 2225 +++++++++ thrift/gen-cpp/leveldb_constants.cpp | 17 + thrift/gen-cpp/leveldb_constants.h | 24 + thrift/gen-cpp/leveldb_types.cpp | 1656 +++++++ thrift/gen-cpp/leveldb_types.h | 867 ++++ thrift/gen-cpp/reflection_types.h | 247 + thrift/leveldb.thrift | 180 + thrift/lib/cpp/ClientUtil.h | 229 + thrift/lib/cpp/EventHandlerBase.h | 311 ++ thrift/lib/cpp/README | 67 + thrift/lib/cpp/Reflection.h | 40 + thrift/lib/cpp/TARGETS | 46 + thrift/lib/cpp/TApplicationException.h | 108 + thrift/lib/cpp/TDispatchProcessor.h | 145 + thrift/lib/cpp/TLogging.h | 193 + thrift/lib/cpp/TProcessor.h | 120 + thrift/lib/cpp/TReflectionLocal.h | 96 + thrift/lib/cpp/Thrift.h | 261 ++ thrift/lib/cpp/async/SimpleCallback.h | 98 + thrift/lib/cpp/async/TARGETS | 103 + thrift/lib/cpp/async/TAsyncChannel.h | 113 + .../lib/cpp/async/TAsyncDispatchProcessor.h | 175 + thrift/lib/cpp/async/TAsyncEventChannel.h | 95 + thrift/lib/cpp/async/TAsyncProcessor.h | 90 + thrift/lib/cpp/async/TAsyncSSLServerSocket.h | 171 + thrift/lib/cpp/async/TAsyncSSLSocket.h | 357 ++ thrift/lib/cpp/async/TAsyncServerSocket.h | 513 ++ thrift/lib/cpp/async/TAsyncSignalHandler.h | 100 + thrift/lib/cpp/async/TAsyncSocket.h | 551 +++ thrift/lib/cpp/async/TAsyncTimeout.h | 144 + thrift/lib/cpp/async/TAsyncTimeoutSet.h | 198 + thrift/lib/cpp/async/TAsyncTransport.h | 428 ++ thrift/lib/cpp/async/TBinaryAsyncChannel.h | 146 + thrift/lib/cpp/async/TDelayedDestruction.h | 178 + thrift/lib/cpp/async/TEventBase.h | 397 ++ thrift/lib/cpp/async/TEventBaseManager.h | 121 + thrift/lib/cpp/async/TEventConnection.h | 239 + thrift/lib/cpp/async/TEventHandler.h | 182 + thrift/lib/cpp/async/TEventServer.h | 1010 ++++ thrift/lib/cpp/async/TEventTask.h | 49 + thrift/lib/cpp/async/TEventUtil.h | 43 + thrift/lib/cpp/async/TEventWorker.h | 264 ++ thrift/lib/cpp/async/TFramedAsyncChannel.h | 204 + thrift/lib/cpp/async/THeaderAsyncChannel.h | 137 + thrift/lib/cpp/async/THttpAsyncChannel.h | 204 + thrift/lib/cpp/async/TNotificationPipe.h | 337 ++ thrift/lib/cpp/async/TNotificationQueue.h | 592 +++ thrift/lib/cpp/async/TQueuingAsyncProcessor.h | 70 + thrift/lib/cpp/async/TStreamAsyncChannel.h | 441 ++ thrift/lib/cpp/async/TStreamAsyncChannel.tcc | 488 ++ thrift/lib/cpp/async/TSyncToAsyncProcessor.h | 42 + thrift/lib/cpp/async/TUndelayedDestruction.h | 117 + thrift/lib/cpp/async/TUnframedAsyncChannel.h | 168 + .../lib/cpp/async/TUnframedAsyncChannel.tcc | 126 + thrift/lib/cpp/async/TZlibAsyncChannel.h | 164 + thrift/lib/cpp/concurrency/Exception.h | 64 + thrift/lib/cpp/concurrency/FunctionRunner.h | 131 + thrift/lib/cpp/concurrency/Monitor.h | 124 + thrift/lib/cpp/concurrency/Mutex.h | 276 ++ .../lib/cpp/concurrency/PosixThreadFactory.h | 137 + thrift/lib/cpp/concurrency/TARGETS | 17 + thrift/lib/cpp/concurrency/Thread.h | 125 + thrift/lib/cpp/concurrency/ThreadLocal.h | 202 + thrift/lib/cpp/concurrency/ThreadManager.h | 216 + thrift/lib/cpp/concurrency/TimerManager.h | 122 + thrift/lib/cpp/concurrency/Util.h | 167 + thrift/lib/cpp/concurrency/test/TARGETS | 45 + .../cpp/concurrency/test/ThreadFactoryTests.h | 354 ++ .../cpp/concurrency/test/TimerManagerTests.h | 142 + thrift/lib/cpp/config.h | 316 ++ thrift/lib/cpp/processor/PeekProcessor.h | 82 + thrift/lib/cpp/processor/StatsProcessor.h | 266 ++ thrift/lib/cpp/processor/TARGETS | 9 + thrift/lib/cpp/processor/test/EventLog.h | 94 + thrift/lib/cpp/processor/test/Handlers.h | 311 ++ thrift/lib/cpp/processor/test/ServerThread.h | 143 + thrift/lib/cpp/processor/test/TARGETS | 24 + thrift/lib/cpp/processor/test/proc.thrift | 22 + thrift/lib/cpp/protocol/TARGETS | 18 + thrift/lib/cpp/protocol/TBase64Utils.h | 42 + thrift/lib/cpp/protocol/TBinaryProtocol.h | 302 ++ thrift/lib/cpp/protocol/TBinaryProtocol.tcc | 441 ++ thrift/lib/cpp/protocol/TCompactProtocol.h | 313 ++ thrift/lib/cpp/protocol/TCompactProtocol.tcc | 820 ++++ thrift/lib/cpp/protocol/TDebugProtocol.h | 360 ++ thrift/lib/cpp/protocol/THeaderProtocol.h | 370 ++ thrift/lib/cpp/protocol/TJSONProtocol.h | 348 ++ thrift/lib/cpp/protocol/TNeutroniumProtocol.h | 287 ++ .../lib/cpp/protocol/TPhpSerializeProtocol.h | 115 + thrift/lib/cpp/protocol/TProtocol.h | 753 +++ thrift/lib/cpp/protocol/TProtocolException.h | 104 + thrift/lib/cpp/protocol/TProtocolTap.h | 188 + thrift/lib/cpp/protocol/TProtocolTypes.h | 34 + thrift/lib/cpp/protocol/TSimpleJSONProtocol.h | 82 + thrift/lib/cpp/protocol/TVirtualProtocol.h | 523 +++ .../lib/cpp/protocol/neutronium/Decoder-inl.h | 438 ++ thrift/lib/cpp/protocol/neutronium/Decoder.h | 239 + .../lib/cpp/protocol/neutronium/Encoder-inl.h | 357 ++ thrift/lib/cpp/protocol/neutronium/Encoder.h | 180 + .../lib/cpp/protocol/neutronium/InternTable.h | 102 + thrift/lib/cpp/protocol/neutronium/README | 67 + .../lib/cpp/protocol/neutronium/Schema-inl.h | 32 + thrift/lib/cpp/protocol/neutronium/Schema.h | 111 + thrift/lib/cpp/protocol/neutronium/TARGETS | 24 + thrift/lib/cpp/protocol/neutronium/Utils.h | 110 + .../protocol/neutronium/intern_table.thrift | 6 + .../lib/cpp/protocol/neutronium/test/TARGETS | 41 + .../neutronium/test/neutronium_test.thrift | 80 + thrift/lib/cpp/server/TARGETS | 30 + thrift/lib/cpp/server/TConnectionContext.h | 96 + thrift/lib/cpp/server/TNonblockingServer.h | 1165 +++++ .../cpp/server/TNonblockingServerObserver.h | 54 + thrift/lib/cpp/server/TRpcTransportContext.h | 69 + thrift/lib/cpp/server/TServer.h | 324 ++ thrift/lib/cpp/server/TSimpleServer.h | 99 + thrift/lib/cpp/server/TThreadPoolServer.h | 130 + thrift/lib/cpp/server/TThreadedServer.h | 223 + .../lib/cpp/server/test/AggregatorTest.thrift | 41 + .../lib/cpp/server/test/AggregatorUtilTest.h | 143 + thrift/lib/cpp/server/test/TARGETS | 60 + thrift/lib/cpp/server/test/connctx.thrift | 7 + thrift/lib/cpp/test/DebugProtocolTest.thrift | 27 + thrift/lib/cpp/test/NetworkUtil.h | 39 + .../lib/cpp/test/STAR.newdev.facebook.com.crt | 34 + .../lib/cpp/test/STAR.newdev.facebook.com.key | 27 + thrift/lib/cpp/test/ScopedEventBaseThread.h | 66 + thrift/lib/cpp/test/SocketPair.h | 59 + thrift/lib/cpp/test/TARGETS | 167 + thrift/lib/cpp/test/TimeUtil.h | 113 + thrift/lib/cpp/test/checks.h | 107 + thrift/lib/cpp/test/loadgen/Controller.h | 70 + thrift/lib/cpp/test/loadgen/IntervalTimer.h | 137 + thrift/lib/cpp/test/loadgen/LatencyMonitor.h | 129 + .../lib/cpp/test/loadgen/LatencyScoreBoard.h | 98 + thrift/lib/cpp/test/loadgen/LoadConfig.h | 67 + thrift/lib/cpp/test/loadgen/Monitor.h | 71 + thrift/lib/cpp/test/loadgen/OpEnabledState.h | 77 + thrift/lib/cpp/test/loadgen/QpsMonitor.h | 73 + thrift/lib/cpp/test/loadgen/QpsScoreBoard.h | 80 + thrift/lib/cpp/test/loadgen/RNG.h | 140 + thrift/lib/cpp/test/loadgen/ScoreBoard.h | 59 + .../lib/cpp/test/loadgen/ScoreBoardOpVector.h | 127 + thrift/lib/cpp/test/loadgen/TARGETS | 32 + thrift/lib/cpp/test/loadgen/TerminalMonitor.h | 85 + .../lib/cpp/test/loadgen/WeightedLoadConfig.h | 60 + thrift/lib/cpp/test/loadgen/Worker.h | 275 ++ thrift/lib/cpp/test/loadgen/WorkerIf.h | 70 + thrift/lib/cpp/test/loadgen/loadgen.h | 84 + thrift/lib/cpp/thrift_config.h | 61 + thrift/lib/cpp/transport/HDFS.h | 61 + thrift/lib/cpp/transport/TARGETS | 88 + thrift/lib/cpp/transport/TBufferTransports.h | 927 ++++ thrift/lib/cpp/transport/TFDTransport.h | 73 + thrift/lib/cpp/transport/TFileTransport.h | 462 ++ thrift/lib/cpp/transport/THDFSFileTransport.h | 52 + thrift/lib/cpp/transport/THeader.h | 286 ++ thrift/lib/cpp/transport/THeaderTransport.h | 185 + thrift/lib/cpp/transport/THttpClient.h | 75 + thrift/lib/cpp/transport/THttpServer.h | 64 + thrift/lib/cpp/transport/THttpTransport.h | 107 + thrift/lib/cpp/transport/TMemPagedFactory.h | 85 + thrift/lib/cpp/transport/TMemPagedTransport.h | 147 + .../lib/cpp/transport/TMemPagedTransport.tcc | 349 ++ thrift/lib/cpp/transport/TRpcTransport.h | 47 + thrift/lib/cpp/transport/TSSLServerSocket.h | 46 + thrift/lib/cpp/transport/TSSLSocket.h | 365 ++ thrift/lib/cpp/transport/TServerSocket.h | 84 + thrift/lib/cpp/transport/TServerTransport.h | 103 + .../lib/cpp/transport/TShortReadTransport.h | 97 + .../lib/cpp/transport/TSimpleFileTransport.h | 45 + thrift/lib/cpp/transport/TSocket.h | 382 ++ thrift/lib/cpp/transport/TSocketAddress.h | 646 +++ thrift/lib/cpp/transport/TSocketPool.h | 218 + thrift/lib/cpp/transport/TTransport.h | 353 ++ .../lib/cpp/transport/TTransportException.h | 140 + thrift/lib/cpp/transport/TTransportUtils.h | 330 ++ thrift/lib/cpp/transport/TVirtualTransport.h | 134 + thrift/lib/cpp/transport/TZlibTransport.h | 292 ++ thrift/lib/cpp/transport/test/TARGETS | 69 + thrift/lib/cpp/util/AsyncClientUtil.h | 68 + thrift/lib/cpp/util/BitwiseCast.h | 64 + thrift/lib/cpp/util/EnumUtils.h | 55 + thrift/lib/cpp/util/ScopedServerThread.h | 103 + thrift/lib/cpp/util/ServerCreator.h | 50 + thrift/lib/cpp/util/ServerCreatorBase.h | 181 + thrift/lib/cpp/util/SyncServerCreator.h | 171 + thrift/lib/cpp/util/TARGETS | 74 + thrift/lib/cpp/util/TEventServerCreator.h | 195 + thrift/lib/cpp/util/THttpParser.h | 130 + .../lib/cpp/util/TNonblockingServerCreator.h | 275 ++ thrift/lib/cpp/util/TSimpleServerCreator.h | 60 + .../lib/cpp/util/TThreadPoolServerCreator.h | 134 + thrift/lib/cpp/util/TThreadedServerCreator.h | 71 + thrift/lib/cpp/util/ThriftSerializer-inl.h | 191 + thrift/lib/cpp/util/ThriftSerializer.h | 224 + .../lib/cpp/util/ThriftSerializerTest.thrift | 29 + thrift/lib/cpp/util/VarintUtils.h | 64 + thrift/lib/cpp/util/VarintUtils.tcc | 63 + thrift/lib/cpp/util/shared_ptr_util.h | 75 + thrift/lib/cpp/util/test/TARGETS | 17 + thrift/openhandles.h | 227 + thrift/server.cpp | 46 + thrift/server_utils.cpp | 400 ++ thrift/test/simpletest.cpp | 212 + util/options.cc | 1 - 216 files changed, 45837 insertions(+), 10 deletions(-) create mode 100644 thrift/README create mode 100644 thrift/folly/Likely.h create mode 100644 thrift/folly/experimental/io/IOBuf.h create mode 100644 thrift/folly/experimental/io/IOBufQueue.h create mode 100644 thrift/folly/experimental/io/check.h create mode 100644 thrift/gen-cpp/DB.cpp create mode 100644 thrift/gen-cpp/DB.h create mode 100644 thrift/gen-cpp/leveldb_constants.cpp create mode 100644 thrift/gen-cpp/leveldb_constants.h create mode 100644 thrift/gen-cpp/leveldb_types.cpp create mode 100644 thrift/gen-cpp/leveldb_types.h create mode 100644 thrift/gen-cpp/reflection_types.h create mode 100644 thrift/leveldb.thrift create mode 100644 thrift/lib/cpp/ClientUtil.h create mode 100644 thrift/lib/cpp/EventHandlerBase.h create mode 100644 thrift/lib/cpp/README create mode 100644 thrift/lib/cpp/Reflection.h create mode 100644 thrift/lib/cpp/TARGETS create mode 100644 thrift/lib/cpp/TApplicationException.h create mode 100644 thrift/lib/cpp/TDispatchProcessor.h create mode 100644 thrift/lib/cpp/TLogging.h create mode 100644 thrift/lib/cpp/TProcessor.h create mode 100644 thrift/lib/cpp/TReflectionLocal.h create mode 100644 thrift/lib/cpp/Thrift.h create mode 100644 thrift/lib/cpp/async/SimpleCallback.h create mode 100644 thrift/lib/cpp/async/TARGETS create mode 100644 thrift/lib/cpp/async/TAsyncChannel.h create mode 100644 thrift/lib/cpp/async/TAsyncDispatchProcessor.h create mode 100644 thrift/lib/cpp/async/TAsyncEventChannel.h create mode 100644 thrift/lib/cpp/async/TAsyncProcessor.h create mode 100644 thrift/lib/cpp/async/TAsyncSSLServerSocket.h create mode 100644 thrift/lib/cpp/async/TAsyncSSLSocket.h create mode 100644 thrift/lib/cpp/async/TAsyncServerSocket.h create mode 100644 thrift/lib/cpp/async/TAsyncSignalHandler.h create mode 100644 thrift/lib/cpp/async/TAsyncSocket.h create mode 100644 thrift/lib/cpp/async/TAsyncTimeout.h create mode 100644 thrift/lib/cpp/async/TAsyncTimeoutSet.h create mode 100644 thrift/lib/cpp/async/TAsyncTransport.h create mode 100644 thrift/lib/cpp/async/TBinaryAsyncChannel.h create mode 100644 thrift/lib/cpp/async/TDelayedDestruction.h create mode 100644 thrift/lib/cpp/async/TEventBase.h create mode 100644 thrift/lib/cpp/async/TEventBaseManager.h create mode 100644 thrift/lib/cpp/async/TEventConnection.h create mode 100644 thrift/lib/cpp/async/TEventHandler.h create mode 100644 thrift/lib/cpp/async/TEventServer.h create mode 100644 thrift/lib/cpp/async/TEventTask.h create mode 100644 thrift/lib/cpp/async/TEventUtil.h create mode 100644 thrift/lib/cpp/async/TEventWorker.h create mode 100644 thrift/lib/cpp/async/TFramedAsyncChannel.h create mode 100644 thrift/lib/cpp/async/THeaderAsyncChannel.h create mode 100644 thrift/lib/cpp/async/THttpAsyncChannel.h create mode 100644 thrift/lib/cpp/async/TNotificationPipe.h create mode 100644 thrift/lib/cpp/async/TNotificationQueue.h create mode 100644 thrift/lib/cpp/async/TQueuingAsyncProcessor.h create mode 100644 thrift/lib/cpp/async/TStreamAsyncChannel.h create mode 100644 thrift/lib/cpp/async/TStreamAsyncChannel.tcc create mode 100644 thrift/lib/cpp/async/TSyncToAsyncProcessor.h create mode 100644 thrift/lib/cpp/async/TUndelayedDestruction.h create mode 100644 thrift/lib/cpp/async/TUnframedAsyncChannel.h create mode 100644 thrift/lib/cpp/async/TUnframedAsyncChannel.tcc create mode 100644 thrift/lib/cpp/async/TZlibAsyncChannel.h create mode 100644 thrift/lib/cpp/concurrency/Exception.h create mode 100644 thrift/lib/cpp/concurrency/FunctionRunner.h create mode 100644 thrift/lib/cpp/concurrency/Monitor.h create mode 100644 thrift/lib/cpp/concurrency/Mutex.h create mode 100644 thrift/lib/cpp/concurrency/PosixThreadFactory.h create mode 100644 thrift/lib/cpp/concurrency/TARGETS create mode 100644 thrift/lib/cpp/concurrency/Thread.h create mode 100644 thrift/lib/cpp/concurrency/ThreadLocal.h create mode 100644 thrift/lib/cpp/concurrency/ThreadManager.h create mode 100644 thrift/lib/cpp/concurrency/TimerManager.h create mode 100644 thrift/lib/cpp/concurrency/Util.h create mode 100644 thrift/lib/cpp/concurrency/test/TARGETS create mode 100644 thrift/lib/cpp/concurrency/test/ThreadFactoryTests.h create mode 100644 thrift/lib/cpp/concurrency/test/TimerManagerTests.h create mode 100644 thrift/lib/cpp/config.h create mode 100644 thrift/lib/cpp/processor/PeekProcessor.h create mode 100644 thrift/lib/cpp/processor/StatsProcessor.h create mode 100644 thrift/lib/cpp/processor/TARGETS create mode 100644 thrift/lib/cpp/processor/test/EventLog.h create mode 100644 thrift/lib/cpp/processor/test/Handlers.h create mode 100644 thrift/lib/cpp/processor/test/ServerThread.h create mode 100644 thrift/lib/cpp/processor/test/TARGETS create mode 100644 thrift/lib/cpp/processor/test/proc.thrift create mode 100644 thrift/lib/cpp/protocol/TARGETS create mode 100644 thrift/lib/cpp/protocol/TBase64Utils.h create mode 100644 thrift/lib/cpp/protocol/TBinaryProtocol.h create mode 100644 thrift/lib/cpp/protocol/TBinaryProtocol.tcc create mode 100644 thrift/lib/cpp/protocol/TCompactProtocol.h create mode 100644 thrift/lib/cpp/protocol/TCompactProtocol.tcc create mode 100644 thrift/lib/cpp/protocol/TDebugProtocol.h create mode 100644 thrift/lib/cpp/protocol/THeaderProtocol.h create mode 100644 thrift/lib/cpp/protocol/TJSONProtocol.h create mode 100644 thrift/lib/cpp/protocol/TNeutroniumProtocol.h create mode 100644 thrift/lib/cpp/protocol/TPhpSerializeProtocol.h create mode 100644 thrift/lib/cpp/protocol/TProtocol.h create mode 100644 thrift/lib/cpp/protocol/TProtocolException.h create mode 100644 thrift/lib/cpp/protocol/TProtocolTap.h create mode 100644 thrift/lib/cpp/protocol/TProtocolTypes.h create mode 100644 thrift/lib/cpp/protocol/TSimpleJSONProtocol.h create mode 100644 thrift/lib/cpp/protocol/TVirtualProtocol.h create mode 100644 thrift/lib/cpp/protocol/neutronium/Decoder-inl.h create mode 100644 thrift/lib/cpp/protocol/neutronium/Decoder.h create mode 100644 thrift/lib/cpp/protocol/neutronium/Encoder-inl.h create mode 100644 thrift/lib/cpp/protocol/neutronium/Encoder.h create mode 100644 thrift/lib/cpp/protocol/neutronium/InternTable.h create mode 100644 thrift/lib/cpp/protocol/neutronium/README create mode 100644 thrift/lib/cpp/protocol/neutronium/Schema-inl.h create mode 100644 thrift/lib/cpp/protocol/neutronium/Schema.h create mode 100644 thrift/lib/cpp/protocol/neutronium/TARGETS create mode 100644 thrift/lib/cpp/protocol/neutronium/Utils.h create mode 100644 thrift/lib/cpp/protocol/neutronium/intern_table.thrift create mode 100644 thrift/lib/cpp/protocol/neutronium/test/TARGETS create mode 100644 thrift/lib/cpp/protocol/neutronium/test/neutronium_test.thrift create mode 100644 thrift/lib/cpp/server/TARGETS create mode 100644 thrift/lib/cpp/server/TConnectionContext.h create mode 100644 thrift/lib/cpp/server/TNonblockingServer.h create mode 100644 thrift/lib/cpp/server/TNonblockingServerObserver.h create mode 100644 thrift/lib/cpp/server/TRpcTransportContext.h create mode 100644 thrift/lib/cpp/server/TServer.h create mode 100644 thrift/lib/cpp/server/TSimpleServer.h create mode 100644 thrift/lib/cpp/server/TThreadPoolServer.h create mode 100644 thrift/lib/cpp/server/TThreadedServer.h create mode 100644 thrift/lib/cpp/server/test/AggregatorTest.thrift create mode 100644 thrift/lib/cpp/server/test/AggregatorUtilTest.h create mode 100644 thrift/lib/cpp/server/test/TARGETS create mode 100644 thrift/lib/cpp/server/test/connctx.thrift create mode 100644 thrift/lib/cpp/test/DebugProtocolTest.thrift create mode 100644 thrift/lib/cpp/test/NetworkUtil.h create mode 100644 thrift/lib/cpp/test/STAR.newdev.facebook.com.crt create mode 100644 thrift/lib/cpp/test/STAR.newdev.facebook.com.key create mode 100644 thrift/lib/cpp/test/ScopedEventBaseThread.h create mode 100644 thrift/lib/cpp/test/SocketPair.h create mode 100644 thrift/lib/cpp/test/TARGETS create mode 100644 thrift/lib/cpp/test/TimeUtil.h create mode 100644 thrift/lib/cpp/test/checks.h create mode 100644 thrift/lib/cpp/test/loadgen/Controller.h create mode 100644 thrift/lib/cpp/test/loadgen/IntervalTimer.h create mode 100644 thrift/lib/cpp/test/loadgen/LatencyMonitor.h create mode 100644 thrift/lib/cpp/test/loadgen/LatencyScoreBoard.h create mode 100644 thrift/lib/cpp/test/loadgen/LoadConfig.h create mode 100644 thrift/lib/cpp/test/loadgen/Monitor.h create mode 100644 thrift/lib/cpp/test/loadgen/OpEnabledState.h create mode 100644 thrift/lib/cpp/test/loadgen/QpsMonitor.h create mode 100644 thrift/lib/cpp/test/loadgen/QpsScoreBoard.h create mode 100644 thrift/lib/cpp/test/loadgen/RNG.h create mode 100644 thrift/lib/cpp/test/loadgen/ScoreBoard.h create mode 100644 thrift/lib/cpp/test/loadgen/ScoreBoardOpVector.h create mode 100644 thrift/lib/cpp/test/loadgen/TARGETS create mode 100644 thrift/lib/cpp/test/loadgen/TerminalMonitor.h create mode 100644 thrift/lib/cpp/test/loadgen/WeightedLoadConfig.h create mode 100644 thrift/lib/cpp/test/loadgen/Worker.h create mode 100644 thrift/lib/cpp/test/loadgen/WorkerIf.h create mode 100644 thrift/lib/cpp/test/loadgen/loadgen.h create mode 100644 thrift/lib/cpp/thrift_config.h create mode 100644 thrift/lib/cpp/transport/HDFS.h create mode 100644 thrift/lib/cpp/transport/TARGETS create mode 100644 thrift/lib/cpp/transport/TBufferTransports.h create mode 100644 thrift/lib/cpp/transport/TFDTransport.h create mode 100644 thrift/lib/cpp/transport/TFileTransport.h create mode 100644 thrift/lib/cpp/transport/THDFSFileTransport.h create mode 100644 thrift/lib/cpp/transport/THeader.h create mode 100644 thrift/lib/cpp/transport/THeaderTransport.h create mode 100644 thrift/lib/cpp/transport/THttpClient.h create mode 100644 thrift/lib/cpp/transport/THttpServer.h create mode 100644 thrift/lib/cpp/transport/THttpTransport.h create mode 100644 thrift/lib/cpp/transport/TMemPagedFactory.h create mode 100644 thrift/lib/cpp/transport/TMemPagedTransport.h create mode 100644 thrift/lib/cpp/transport/TMemPagedTransport.tcc create mode 100644 thrift/lib/cpp/transport/TRpcTransport.h create mode 100644 thrift/lib/cpp/transport/TSSLServerSocket.h create mode 100644 thrift/lib/cpp/transport/TSSLSocket.h create mode 100644 thrift/lib/cpp/transport/TServerSocket.h create mode 100644 thrift/lib/cpp/transport/TServerTransport.h create mode 100644 thrift/lib/cpp/transport/TShortReadTransport.h create mode 100644 thrift/lib/cpp/transport/TSimpleFileTransport.h create mode 100644 thrift/lib/cpp/transport/TSocket.h create mode 100644 thrift/lib/cpp/transport/TSocketAddress.h create mode 100644 thrift/lib/cpp/transport/TSocketPool.h create mode 100644 thrift/lib/cpp/transport/TTransport.h create mode 100644 thrift/lib/cpp/transport/TTransportException.h create mode 100644 thrift/lib/cpp/transport/TTransportUtils.h create mode 100644 thrift/lib/cpp/transport/TVirtualTransport.h create mode 100644 thrift/lib/cpp/transport/TZlibTransport.h create mode 100644 thrift/lib/cpp/transport/test/TARGETS create mode 100644 thrift/lib/cpp/util/AsyncClientUtil.h create mode 100644 thrift/lib/cpp/util/BitwiseCast.h create mode 100644 thrift/lib/cpp/util/EnumUtils.h create mode 100644 thrift/lib/cpp/util/ScopedServerThread.h create mode 100644 thrift/lib/cpp/util/ServerCreator.h create mode 100644 thrift/lib/cpp/util/ServerCreatorBase.h create mode 100644 thrift/lib/cpp/util/SyncServerCreator.h create mode 100644 thrift/lib/cpp/util/TARGETS create mode 100644 thrift/lib/cpp/util/TEventServerCreator.h create mode 100644 thrift/lib/cpp/util/THttpParser.h create mode 100644 thrift/lib/cpp/util/TNonblockingServerCreator.h create mode 100644 thrift/lib/cpp/util/TSimpleServerCreator.h create mode 100644 thrift/lib/cpp/util/TThreadPoolServerCreator.h create mode 100644 thrift/lib/cpp/util/TThreadedServerCreator.h create mode 100644 thrift/lib/cpp/util/ThriftSerializer-inl.h create mode 100644 thrift/lib/cpp/util/ThriftSerializer.h create mode 100644 thrift/lib/cpp/util/ThriftSerializerTest.thrift create mode 100644 thrift/lib/cpp/util/VarintUtils.h create mode 100644 thrift/lib/cpp/util/VarintUtils.tcc create mode 100644 thrift/lib/cpp/util/shared_ptr_util.h create mode 100644 thrift/lib/cpp/util/test/TARGETS create mode 100644 thrift/openhandles.h create mode 100644 thrift/server.cpp create mode 100644 thrift/server_utils.cpp create mode 100644 thrift/test/simpletest.cpp diff --git a/Makefile b/Makefile index f40c3d1fd5..0876340243 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,7 @@ CXXFLAGS += -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT) LDFLAGS += $(PLATFORM_LDFLAGS) LIBOBJECTS = $(SOURCES:.cc=.o) +LIBOBJECTS += $(SOURCESCPP:.cpp=.o) MEMENVOBJECTS = $(MEMENV_SOURCES:.cc=.o) TESTUTIL = ./util/testutil.o @@ -51,7 +52,8 @@ TESTS = \ table_test \ version_edit_test \ version_set_test \ - write_batch_test + write_batch_test \ + leveldb_server_test PROGRAMS = db_bench $(TESTS) BENCHMARKS = db_bench_sqlite3 db_bench_tree_db @@ -71,20 +73,20 @@ SHARED2 = $(SHARED1).$(SHARED_MAJOR) SHARED3 = $(SHARED1).$(SHARED_MAJOR).$(SHARED_MINOR) SHARED = $(SHARED1) $(SHARED2) $(SHARED3) $(SHARED3): - $(CXX) $(LDFLAGS) $(PLATFORM_SHARED_LDFLAGS)$(INSTALL_PATH)/$(SHARED2) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(SOURCES) -o $(SHARED3) $(EXEC_LDFLAGS_SHARED) + $(CXX) $(LDFLAGS) $(PLATFORM_SHARED_LDFLAGS)$(INSTALL_PATH)/$(SHARED2) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(SOURCES) $(SOURCESCPP) -o $(SHARED3) $(EXEC_LDFLAGS_SHARED) $(SHARED2): $(SHARED3) ln -fs $(SHARED3) $(SHARED2) $(SHARED1): $(SHARED3) ln -fs $(SHARED3) $(SHARED1) endif -all: $(SHARED) $(LIBRARY) +all: $(SHARED) $(LIBRARY) $(THRIFTSERVER) check: all $(PROGRAMS) $(TESTS) for t in $(TESTS); do echo "***** Running $$t"; ./$$t || exit 1; done clean: - -rm -f $(PROGRAMS) $(BENCHMARKS) $(LIBRARY) $(SHARED) $(MEMENVLIBRARY) */*.o */*/*.o ios-x86/*/*.o ios-arm/*/*.o build_config.mk + -rm -f $(PROGRAMS) $(BENCHMARKS) $(LIBRARY) $(SHARED) $(MEMENVLIBRARY) $(THRIFTSERVER) */*.o */*/*.o ios-x86/*/*.o ios-arm/*/*.o build_config.mk -rm -rf ios-x86/* ios-arm/* $(LIBRARY): $(LIBOBJECTS) @@ -161,6 +163,12 @@ $(MEMENVLIBRARY) : $(MEMENVOBJECTS) memenv_test : helpers/memenv/memenv_test.o $(MEMENVLIBRARY) $(LIBRARY) $(TESTHARNESS) $(CXX) helpers/memenv/memenv_test.o $(MEMENVLIBRARY) $(LIBRARY) $(TESTHARNESS) -o $@ $(LDFLAGS) +leveldb_server: thrift/server.o $(LIBRARY) + $(CXX) thrift/server.o $(LIBRARY) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) + +leveldb_server_test: thrift/test/simpletest.o $(LIBRARY) + $(CXX) thrift/test/simpletest.o $(LIBRARY) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) + ifeq ($(PLATFORM), IOS) # For iOS, create universal object files to be used on both the simulator and # a device. diff --git a/build_detect_platform b/build_detect_platform index 6854c9c8e7..a40d78c7a0 100755 --- a/build_detect_platform +++ b/build_detect_platform @@ -101,15 +101,21 @@ esac # of all files matching either rule, so we need to append -print to make the # prune take effect. DIRS="util db table" +if test "$USE_THRIFT"; then + DIRS+=" thrift/gen-cpp thrift/server_utils.cpp" + THRIFTSERVER=leveldb_server +fi set -f # temporarily disable globbing so that our patterns aren't expanded PRUNE_TEST="-name *test*.cc -prune" PRUNE_BENCH="-name *_bench.cc -prune" PORTABLE_FILES=`find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o -name '*.cc' -print | sort | tr "\n" " "` +PORTABLE_CPP=`find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o -name '*.cpp' -print | sort | tr "\n" " "` set +f # re-enable globbing # The sources consist of the portable files, plus the platform-specific port # file. echo "SOURCES=$PORTABLE_FILES $PORT_FILE" >> $OUTPUT +echo "SOURCESCPP=$PORTABLE_CPP" >> $OUTPUT echo "MEMENV_SOURCES=helpers/memenv/memenv.cc" >> $OUTPUT if [ "$PLATFORM" = "OS_ANDROID_CROSSCOMPILE" ]; then @@ -136,7 +142,7 @@ EOF EOF if [ "$?" = 0 ]; then COMMON_FLAGS="$COMMON_FLAGS -DSNAPPY" - PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lsnappy" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lsnappy -L./snappy/libs" fi # Test whether zlib library is installed @@ -183,6 +189,14 @@ if test "$USE_HDFS"; then PLATFORM_LDFLAGS+=$HDFS_LDFLAGS fi +# shall we build thrift server +if test "$USE_THRIFT"; then + THRIFT_CCFLAGS=" -I./thrift -I./thrift/gen-cpp -I./thrift/lib/cpp -I/usr/include -std=gnu++0x" + THRIFT_LDFLAGS=" -lserver -lthrift_base -ltransport -lthrift_exception -lutil -L./thrift/libs " + COMMON_FLAGS+=$THRIFT_CCFLAGS + PLATFORM_LDFLAGS+=$THRIFT_LDFLAGS +fi + PLATFORM_CCFLAGS="$PLATFORM_CCFLAGS $COMMON_FLAGS" PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS $COMMON_FLAGS" @@ -193,3 +207,4 @@ echo "PLATFORM_CXXFLAGS=$PLATFORM_CXXFLAGS" >> $OUTPUT echo "PLATFORM_SHARED_CFLAGS=$PLATFORM_SHARED_CFLAGS" >> $OUTPUT echo "PLATFORM_SHARED_EXT=$PLATFORM_SHARED_EXT" >> $OUTPUT echo "PLATFORM_SHARED_LDFLAGS=$PLATFORM_SHARED_LDFLAGS" >> $OUTPUT +echo "THRIFTSERVER=$THRIFTSERVER" >> $OUTPUT diff --git a/fbcode.sh b/fbcode.sh index 1f1a2f0eb6..0f8cc3853f 100644 --- a/fbcode.sh +++ b/fbcode.sh @@ -22,8 +22,13 @@ fi SNAPPY_INCLUDE=" -I ./snappy" SNAPPY_LIBS=" -L./snappy/libs" +# location of boost headers and libraries +THRIFT_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/boost/default/eed002c/include -std=gnu++0x" +THRIFT_INCLUDE+=" -I./thrift -I./thrift/gen-cpp -I./thrift/lib/cpp" +THRIFT_LIBS=" -L $TOOLCHAIN_LIB_BASE/boost/default/eed002c/lib" + CC="$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.6.2-glibc-2.13/bin/gcc" -CXX="$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.6.2-glibc-2.13/bin/g++ $JINCLUDE $SNAPPY_INCLUDE" +CXX="$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.6.2-glibc-2.13/bin/g++ $JINCLUDE $SNAPPY_INCLUDE $THRIFT_INCLUDE" AR=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ar RANLIB=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ranlib @@ -32,7 +37,7 @@ CFLAGS+=" -I $TOOLCHAIN_LIB_BASE/jemalloc/jemalloc-2.2.5/96de4f9/include -DHAVE_ EXEC_LDFLAGS=" -Wl,--whole-archive $TOOLCHAIN_LIB_BASE/jemalloc/jemalloc-2.2.4/96de4f9/lib/libjemalloc.a " EXEC_LDFLAGS+="-Wl,--no-whole-archive $TOOLCHAIN_LIB_BASE/libunwind/libunwind-20100810/4bc2c16/lib/libunwind.a" -EXEC_LDFLAGS+="$HDFSLIB $SNAPPY_LIBS" +EXEC_LDFLAGS+="$HDFSLIB $SNAPPY_LIBS $THRIFT_LIBS " EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $TOOLCHAIN_LIB_BASE/jemalloc/jemalloc-2.2.4/96de4f9/lib/libjemalloc.so" export CC CXX AR RANLIB CFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED diff --git a/hdfs/README b/hdfs/README index d5085ef11d..2d4da29dca 100644 --- a/hdfs/README +++ b/hdfs/README @@ -16,7 +16,7 @@ If you want to compile leveldb with hdfs support, please set the following enviroment variables appropriately: USE_HDFS=1 JAVA_HOME=/usr/local/jdk-6u22-64 - LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$JAVA_HOME/jre/lib/amd64/server:$JAVA_HOME/jre/lib/amd64/:./snappy/libs + LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/jdk-6u22-64/jre/lib/amd64/server:/usr/local/jdk-6u22-64/jre/lib/amd64/:./snappy/libs make clean all db_bench To run dbbench, diff --git a/snappy/README b/snappy/README index 649d3ea1ed..756c97d306 100644 --- a/snappy/README +++ b/snappy/README @@ -2,5 +2,5 @@ This is a pre-compiled version of snappy 1.0.5 from http://code.google.com/p/snappy/downloads/detail?name=snappy-1.0.5.tar.gz This is compiled using gcc-4.6.2-glibc-2.13 on centos5.2 -and uses jemalloc. This is here so that one can compie leveldb to use +and uses jemalloc. This is here so that one can compile leveldb to use snappy easily. diff --git a/thrift/README b/thrift/README new file mode 100644 index 0000000000..756f742c4c --- /dev/null +++ b/thrift/README @@ -0,0 +1,18 @@ +This directory has the thrift server code that exposes leveldb apis. + +The thrift header files are in ./thrift/lib. These are part of +Apache Thrift code base and are needed for compilation of the leveldb +thrift server. The thrift libraries are copied into ./thrift/libs. +If you want to use a different version of thrift, please update these +directories with the corresponding thrift header files and the +compiled thrift libraries. + +If you want to compile leveldb with thrift-server support, please set the following +enviroment variables appropriately: + USE_THRIFT=1 + LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./thrift/libs + make clean leveldb_server leveldb_server_test + +You can run the leveldb server unit tests by + ./leveldb_server_test + diff --git a/thrift/folly/Likely.h b/thrift/folly/Likely.h new file mode 100644 index 0000000000..c535e8aefa --- /dev/null +++ b/thrift/folly/Likely.h @@ -0,0 +1,39 @@ +/* + * Copyright 2012 Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Compiler hints to indicate the fast path of an "if" branch: whether + * the if condition is likely to be true or false. + * + * @author Tudor Bosman (tudorb@fb.com) + */ + +#ifndef FOLLY_BASE_LIKELY_H_ +#define FOLLY_BASE_LIKELY_H_ + +#undef LIKELY +#undef UNLIKELY + +#if defined(__GNUC__) && __GNUC__ >= 4 +#define LIKELY(x) (__builtin_expect((x), 1)) +#define UNLIKELY(x) (__builtin_expect((x), 0)) +#else +#define LIKELY(x) (x) +#define UNLIKELY(x) (x) +#endif + +#endif /* FOLLY_BASE_LIKELY_H_ */ + diff --git a/thrift/folly/experimental/io/IOBuf.h b/thrift/folly/experimental/io/IOBuf.h new file mode 100644 index 0000000000..1a416e69b8 --- /dev/null +++ b/thrift/folly/experimental/io/IOBuf.h @@ -0,0 +1,1050 @@ +/* + * Copyright 2012 Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FOLLY_IO_IOBUF_H_ +#define FOLLY_IO_IOBUF_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace folly { + +/** + * An IOBuf is a pointer to a buffer of data. + * + * IOBuf objects are intended to be used primarily for networking code, and are + * modelled somewhat after FreeBSD's mbuf data structure, and Linux's sk_buff + * structure. + * + * IOBuf objects facilitate zero-copy network programming, by allowing multiple + * IOBuf objects to point to the same underlying buffer of data, using a + * reference count to track when the buffer is no longer needed and can be + * freed. + * + * + * Data Layout + * ----------- + * + * The IOBuf itself is a small object containing a pointer to the buffer and + * information about which segment of the buffer contains valid data. + * + * The data layout looks like this: + * + * +-------+ + * | IOBuf | + * +-------+ + * / + * | + * v + * +------------+--------------------+-----------+ + * | headroom | data | tailroom | + * +------------+--------------------+-----------+ + * ^ ^ ^ ^ + * buffer() data() tail() bufferEnd() + * + * The length() method returns the length of the valid data; capacity() + * returns the entire capacity of the buffer (from buffer() to bufferEnd()). + * The headroom() and tailroom() methods return the amount of unused capacity + * available before and after the data. + * + * + * Buffer Sharing + * -------------- + * + * The buffer itself is reference counted, and multiple IOBuf objects may point + * to the same buffer. Each IOBuf may point to a different section of valid + * data within the underlying buffer. For example, if multiple protocol + * requests are read from the network into a single buffer, a separate IOBuf + * may be created for each request, all sharing the same underlying buffer. + * + * In other words, when multiple IOBufs share the same underlying buffer, the + * data() and tail() methods on each IOBuf may point to a different segment of + * the data. However, the buffer() and bufferEnd() methods will point to the + * same location for all IOBufs sharing the same underlying buffer. + * + * +-----------+ +---------+ + * | IOBuf 1 | | IOBuf 2 | + * +-----------+ +---------+ + * | | _____/ | + * data | tail |/ data | tail + * v v v + * +-------------------------------------+ + * | | | | | + * +-------------------------------------+ + * + * If you only read data from an IOBuf, you don't need to worry about other + * IOBuf objects possibly sharing the same underlying buffer. However, if you + * ever write to the buffer you need to first ensure that no other IOBufs point + * to the same buffer. The unshare() method may be used to ensure that you + * have an unshared buffer. + * + * + * IOBuf Chains + * ------------ + * + * IOBuf objects also contain pointers to next and previous IOBuf objects. + * This can be used to represent a single logical piece of data that its stored + * in non-contiguous chunks in separate buffers. + * + * A single IOBuf object can only belong to one chain at a time. + * + * IOBuf chains are always circular. The "prev" pointer in the head of the + * chain points to the tail of the chain. However, it is up to the user to + * decide which IOBuf is the head. Internally the IOBuf code does not care + * which element is the head. + * + * The lifetime of all IOBufs in the chain are linked: when one element in the + * chain is deleted, all other chained elements are also deleted. Conceptually + * it is simplest to treat this as if the head of the chain owns all other + * IOBufs in the chain. When you delete the head of the chain, it will delete + * the other elements as well. For this reason, prependChain() and + * appendChain() take ownership of of the new elements being added to this + * chain. + * + * When the coalesce() method is used to coalesce an entire IOBuf chain into a + * single IOBuf, all other IOBufs in the chain are eliminated and automatically + * deleted. The unshare() method may coalesce the chain; if it does it will + * similarly delete all IOBufs eliminated from the chain. + * + * As discussed in the following section, it is up to the user to maintain a + * lock around the entire IOBuf chain if multiple threads need to access the + * chain. IOBuf does not provide any internal locking. + * + * + * Synchronization + * --------------- + * + * When used in multithread programs, a single IOBuf object should only be used + * in a single thread at a time. If a caller uses a single IOBuf across + * multiple threads the caller is responsible for using an external lock to + * synchronize access to the IOBuf. + * + * Two separate IOBuf objects may be accessed concurrently in separate threads + * without locking, even if they point to the same underlying buffer. The + * buffer reference count is always accessed atomically, and no other + * operations should affect other IOBufs that point to the same data segment. + * The caller is responsible for using unshare() to ensure that the data buffer + * is not shared by other IOBufs before writing to it, and this ensures that + * the data itself is not modified in one thread while also being accessed from + * another thread. + * + * For IOBuf chains, no two IOBufs in the same chain should be accessed + * simultaneously in separate threads. The caller must maintain a lock around + * the entire chain if the chain, or individual IOBufs in the chain, may be + * accessed by multiple threads. + * + * + * IOBuf Object Allocation/Sharing + * ------------------------------- + * + * IOBuf objects themselves are always allocated on the heap. The IOBuf + * constructors are private, so IOBuf objects may not be created on the stack. + * In part this is done since some IOBuf objects use small-buffer optimization + * and contain the buffer data immediately after the IOBuf object itself. The + * coalesce() and unshare() methods also expect to be able to delete subsequent + * IOBuf objects in the chain if they are no longer needed due to coalescing. + * + * The IOBuf structure also does not provide room for an intrusive refcount on + * the IOBuf object itself, only the underlying data buffer is reference + * counted. If users want to share the same IOBuf object between multiple + * parts of the code, they are responsible for managing this sharing on their + * own. (For example, by using a shared_ptr. Alternatively, users always have + * the option of using clone() to create a second IOBuf that points to the same + * underlying buffer.) + * + * With jemalloc, allocating small objects like IOBuf objects should be + * relatively fast, and the cost of allocating IOBuf objects on the heap and + * cloning new IOBufs should be relatively cheap. + */ +namespace detail { +// Is T a unique_ptr<> to a standard-layout type? +template struct IsUniquePtrToSL + : public std::false_type { }; +template +struct IsUniquePtrToSL< + std::unique_ptr, + typename std::enable_if::value>::type> + : public std::true_type { }; +} // namespace detail + +class IOBuf { + public: + typedef void (*FreeFunction)(void* buf, void* userData); + + /** + * Allocate a new IOBuf object with the requested capacity. + * + * Returns a new IOBuf object that must be (eventually) deleted by the + * caller. The returned IOBuf may actually have slightly more capacity than + * requested. + * + * The data pointer will initially point to the start of the newly allocated + * buffer, and will have a data length of 0. + * + * Throws std::bad_alloc on error. + */ + static std::unique_ptr create(uint32_t capacity); + + /** + * Create a new IOBuf pointing to an existing data buffer. + * + * The new IOBuffer will assume ownership of the buffer, and free it by + * calling the specified FreeFunction when the last IOBuf pointing to this + * buffer is destroyed. The function will be called with a pointer to the + * buffer as the first argument, and the supplied userData value as the + * second argument. The free function must never throw exceptions. + * + * If no FreeFunction is specified, the buffer will be freed using free(). + * + * The IOBuf data pointer will initially point to the start of the buffer, + * and the length will be the full capacity of the buffer. + * + * On error, std::bad_alloc will be thrown. If freeOnError is true (the + * default) the buffer will be freed before throwing the error. + */ + static std::unique_ptr takeOwnership(void* buf, uint32_t capacity, + FreeFunction freeFn = NULL, + void* userData = NULL, + bool freeOnError = true); + + /** + * Create a new IOBuf pointing to an existing data buffer made up of + * count objects of a given standard-layout type. + * + * This is dangerous -- it is essentially equivalent to doing + * reinterpret_cast on your data -- but it's often useful + * for serialization / deserialization. + * + * The new IOBuffer will assume ownership of the buffer, and free it + * appropriately (by calling the UniquePtr's custom deleter, or by calling + * delete or delete[] appropriately if there is no custom deleter) + * when the buffer is destroyed. The custom deleter, if any, must never + * throw exceptions. + * + * The IOBuf data pointer will initially point to the start of the buffer, + * and the length will be the full capacity of the buffer (count * + * sizeof(T)). + * + * On error, std::bad_alloc will be thrown, and the buffer will be freed + * before throwing the error. + */ + template + static typename std::enable_if::value, + std::unique_ptr>::type + takeOwnership(UniquePtr&& buf, size_t count=1); + + /** + * Create a new IOBuf object that points to an existing user-owned buffer. + * + * This should only be used when the caller knows the lifetime of the IOBuf + * object ahead of time and can ensure that all IOBuf objects that will point + * to this buffer will be destroyed before the buffer itself is destroyed. + * + * This buffer will not be freed automatically when the last IOBuf + * referencing it is destroyed. It is the caller's responsibility to free + * the buffer after the last IOBuf has been destroyed. + * + * The IOBuf data pointer will initially point to the start of the buffer, + * and the length will be the full capacity of the buffer. + * + * An IOBuf created using wrapBuffer() will always be reported as shared. + * unshare() may be used to create a writable copy of the buffer. + * + * On error, std::bad_alloc will be thrown. + */ + static std::unique_ptr wrapBuffer(const void* buf, uint32_t capacity); + + /** + * Convenience function to create a new IOBuf object that copies data from a + * user-supplied buffer, optionally allocating a given amount of + * headroom and tailroom. + */ + static std::unique_ptr copyBuffer(const void* buf, uint32_t size, + uint32_t headroom=0, + uint32_t minTailroom=0); + + /** + * Convenience function to free a chain of IOBufs held by a unique_ptr. + */ + static void destroy(std::unique_ptr&& data) { + auto destroyer = std::move(data); + } + + /** + * Destroy this IOBuf. + * + * Deleting an IOBuf will automatically destroy all IOBufs in the chain. + * (See the comments above regarding the ownership model of IOBuf chains. + * All subsequent IOBufs in the chain are considered to be owned by the head + * of the chain. Users should only explicitly delete the head of a chain.) + * + * When each individual IOBuf is destroyed, it will release its reference + * count on the underlying buffer. If it was the last user of the buffer, + * the buffer will be freed. + */ + ~IOBuf(); + + /** + * Check whether the chain is empty (i.e., whether the IOBufs in the + * chain have a total data length of zero). + * + * This method is semantically equivalent to + * i->computeChainDataLength()==0 + * but may run faster because it can short-circuit as soon as it + * encounters a buffer with length()!=0 + */ + bool empty() const; + + /** + * Get the pointer to the start of the data. + */ + const uint8_t* data() const { + return data_; + } + + /** + * Get a writable pointer to the start of the data. + * + * The caller is responsible for calling unshare() first to ensure that it is + * actually safe to write to the buffer. + */ + uint8_t* writableData() { + return data_; + } + + /** + * Get the pointer to the end of the data. + */ + const uint8_t* tail() const { + return data_ + length_; + } + + /** + * Get a writable pointer to the end of the data. + * + * The caller is responsible for calling unshare() first to ensure that it is + * actually safe to write to the buffer. + */ + uint8_t* writableTail() { + return data_ + length_; + } + + /** + * Get the data length. + */ + uint32_t length() const { + return length_; + } + + /** + * Get the amount of head room. + * + * Returns the number of bytes in the buffer before the start of the data. + */ + uint32_t headroom() const { + return data_ - buffer(); + } + + /** + * Get the amount of tail room. + * + * Returns the number of bytes in the buffer after the end of the data. + */ + uint32_t tailroom() const { + return bufferEnd() - tail(); + } + + /** + * Get the pointer to the start of the buffer. + * + * Note that this is the pointer to the very beginning of the usable buffer, + * not the start of valid data within the buffer. Use the data() method to + * get a pointer to the start of the data within the buffer. + */ + const uint8_t* buffer() const { + return (flags_ & kFlagExt) ? ext_.buf : int_.buf; + } + + /** + * Get a writable pointer to the start of the buffer. + * + * The caller is responsible for calling unshare() first to ensure that it is + * actually safe to write to the buffer. + */ + uint8_t* writableBuffer() { + return (flags_ & kFlagExt) ? ext_.buf : int_.buf; + } + + /** + * Get the pointer to the end of the buffer. + * + * Note that this is the pointer to the very end of the usable buffer, + * not the end of valid data within the buffer. Use the tail() method to + * get a pointer to the end of the data within the buffer. + */ + const uint8_t* bufferEnd() const { + return (flags_ & kFlagExt) ? + ext_.buf + ext_.capacity : + int_.buf + kMaxInternalDataSize; + } + + /** + * Get the total size of the buffer. + * + * This returns the total usable length of the buffer. Use the length() + * method to get the length of the actual valid data in this IOBuf. + */ + uint32_t capacity() const { + return (flags_ & kFlagExt) ? ext_.capacity : kMaxInternalDataSize; + } + + /** + * Get a pointer to the next IOBuf in this chain. + */ + IOBuf* next() { + return next_; + } + const IOBuf* next() const { + return next_; + } + + /** + * Get a pointer to the previous IOBuf in this chain. + */ + IOBuf* prev() { + return prev_; + } + const IOBuf* prev() const { + return prev_; + } + + /** + * Shift the data forwards in the buffer. + * + * This shifts the data pointer forwards in the buffer to increase the + * headroom. This is commonly used to increase the headroom in a newly + * allocated buffer. + * + * The caller is responsible for ensuring that there is sufficient + * tailroom in the buffer before calling advance(). + * + * If there is a non-zero data length, advance() will use memmove() to shift + * the data forwards in the buffer. In this case, the caller is responsible + * for making sure the buffer is unshared, so it will not affect other IOBufs + * that may be sharing the same underlying buffer. + */ + void advance(uint32_t amount) { + // In debug builds, assert if there is a problem. + assert(amount <= tailroom()); + + if (length_ > 0) { + memmove(data_ + amount, data_, length_); + } + data_ += amount; + } + + /** + * Shift the data backwards in the buffer. + * + * The caller is responsible for ensuring that there is sufficient headroom + * in the buffer before calling retreat(). + * + * If there is a non-zero data length, retreat() will use memmove() to shift + * the data backwards in the buffer. In this case, the caller is responsible + * for making sure the buffer is unshared, so it will not affect other IOBufs + * that may be sharing the same underlying buffer. + */ + void retreat(uint32_t amount) { + // In debug builds, assert if there is a problem. + assert(amount <= headroom()); + + if (length_ > 0) { + memmove(data_ - amount, data_, length_); + } + data_ -= amount; + } + + /** + * Adjust the data pointer to include more valid data at the beginning. + * + * This moves the data pointer backwards to include more of the available + * buffer. The caller is responsible for ensuring that there is sufficient + * headroom for the new data. The caller is also responsible for populating + * this section with valid data. + * + * This does not modify any actual data in the buffer. + */ + void prepend(uint32_t amount) { + CHECK(amount <= headroom()); + data_ -= amount; + } + + /** + * Adjust the tail pointer to include more valid data at the end. + * + * This moves the tail pointer forwards to include more of the available + * buffer. The caller is responsible for ensuring that there is sufficient + * tailroom for the new data. The caller is also responsible for populating + * this section with valid data. + * + * This does not modify any actual data in the buffer. + */ + void append(uint32_t amount) { + CHECK(amount <= tailroom()); + length_ += amount; + } + + /** + * Adjust the data pointer forwards to include less valid data. + * + * This moves the data pointer forwards so that the first amount bytes are no + * longer considered valid data. The caller is responsible for ensuring that + * amount is less than or equal to the actual data length. + * + * This does not modify any actual data in the buffer. + */ + void trimStart(uint32_t amount) { + CHECK(amount <= length_); + data_ += amount; + length_ -= amount; + } + + /** + * Adjust the tail pointer backwards to include less valid data. + * + * This moves the tail pointer backwards so that the last amount bytes are no + * longer considered valid data. The caller is responsible for ensuring that + * amount is less than or equal to the actual data length. + * + * This does not modify any actual data in the buffer. + */ + void trimEnd(uint32_t amount) { + CHECK(amount <= length_); + length_ -= amount; + } + + /** + * Clear the buffer. + * + * Postcondition: headroom() == 0, length() == 0, tailroom() == capacity() + */ + void clear() { + data_ = writableBuffer(); + length_ = 0; + } + + /** + * Ensure that this buffer has at least minHeadroom headroom bytes and at + * least minTailroom tailroom bytes. The buffer must be writable + * (you must call unshare() before this, if necessary). + * + * Postcondition: headroom() >= minHeadroom, tailroom() >= minTailroom, + * the data (between data() and data() + length()) is preserved. + */ + void reserve(uint32_t minHeadroom, uint32_t minTailroom) { + // Maybe we don't need to do anything. + if (headroom() >= minHeadroom && tailroom() >= minTailroom) { + return; + } + // If the buffer is empty but we have enough total room (head + tail), + // move the data_ pointer around. + if (length() == 0 && + headroom() + tailroom() >= minHeadroom + minTailroom) { + data_ = writableBuffer() + minHeadroom; + return; + } + // Bah, we have to do actual work. + reserveSlow(minHeadroom, minTailroom); + } + + /** + * Return true if this IOBuf is part of a chain of multiple IOBufs, or false + * if this is the only IOBuf in its chain. + */ + bool isChained() const { + assert((next_ == this) == (prev_ == this)); + return next_ != this; + } + + /** + * Get the number of IOBufs in this chain. + * + * Beware that this method has to walk the entire chain. + * Use isChained() if you just want to check if this IOBuf is part of a chain + * or not. + */ + uint32_t countChainElements() const; + + /** + * Get the length of all the data in this IOBuf chain. + * + * Beware that this method has to walk the entire chain. + */ + uint64_t computeChainDataLength() const; + + /** + * Insert another IOBuf chain immediately before this IOBuf. + * + * For example, if there are two IOBuf chains (A, B, C) and (D, E, F), + * and B->prependChain(D) is called, the (D, E, F) chain will be subsumed + * and become part of the chain starting at A, which will now look like + * (A, D, E, F, B, C) + * + * Note that since IOBuf chains are circular, head->prependChain(other) can + * be used to append the other chain at the very end of the chain pointed to + * by head. For example, if there are two IOBuf chains (A, B, C) and + * (D, E, F), and A->prependChain(D) is called, the chain starting at A will + * now consist of (A, B, C, D, E, F) + * + * The elements in the specified IOBuf chain will become part of this chain, + * and will be owned by the head of this chain. When this chain is + * destroyed, all elements in the supplied chain will also be destroyed. + * + * For this reason, appendChain() only accepts an rvalue-reference to a + * unique_ptr(), to make it clear that it is taking ownership of the supplied + * chain. If you have a raw pointer, you can pass in a new temporary + * unique_ptr around the raw pointer. If you have an existing, + * non-temporary unique_ptr, you must call std::move(ptr) to make it clear + * that you are destroying the original pointer. + */ + void prependChain(std::unique_ptr&& iobuf); + + /** + * Append another IOBuf chain immediately after this IOBuf. + * + * For example, if there are two IOBuf chains (A, B, C) and (D, E, F), + * and B->appendChain(D) is called, the (D, E, F) chain will be subsumed + * and become part of the chain starting at A, which will now look like + * (A, B, D, E, F, C) + * + * The elements in the specified IOBuf chain will become part of this chain, + * and will be owned by the head of this chain. When this chain is + * destroyed, all elements in the supplied chain will also be destroyed. + * + * For this reason, appendChain() only accepts an rvalue-reference to a + * unique_ptr(), to make it clear that it is taking ownership of the supplied + * chain. If you have a raw pointer, you can pass in a new temporary + * unique_ptr around the raw pointer. If you have an existing, + * non-temporary unique_ptr, you must call std::move(ptr) to make it clear + * that you are destroying the original pointer. + */ + void appendChain(std::unique_ptr&& iobuf) { + // Just use prependChain() on the next element in our chain + next_->prependChain(std::move(iobuf)); + } + + /** + * Remove this IOBuf from its current chain. + * + * Since ownership of all elements an IOBuf chain is normally maintained by + * the head of the chain, unlink() transfers ownership of this IOBuf from the + * chain and gives it to the caller. A new unique_ptr to the IOBuf is + * returned to the caller. The caller must store the returned unique_ptr (or + * call release() on it) to take ownership, otherwise the IOBuf will be + * immediately destroyed. + * + * Since unlink transfers ownership of the IOBuf to the caller, be careful + * not to call unlink() on the head of a chain if you already maintain + * ownership on the head of the chain via other means. The pop() method + * is a better choice for that situation. + */ + std::unique_ptr unlink() { + next_->prev_ = prev_; + prev_->next_ = next_; + prev_ = this; + next_ = this; + return std::unique_ptr(this); + } + + /** + * Remove this IOBuf from its current chain and return a unique_ptr to + * the IOBuf that formerly followed it in the chain. + */ + std::unique_ptr pop() { + IOBuf *next = next_; + next_->prev_ = prev_; + prev_->next_ = next_; + prev_ = this; + next_ = this; + return std::unique_ptr((next == this) ? NULL : next); + } + + /** + * Remove a subchain from this chain. + * + * Remove the subchain starting at head and ending at tail from this chain. + * + * Returns a unique_ptr pointing to head. (In other words, ownership of the + * head of the subchain is transferred to the caller.) If the caller ignores + * the return value and lets the unique_ptr be destroyed, the subchain will + * be immediately destroyed. + * + * The subchain referenced by the specified head and tail must be part of the + * same chain as the current IOBuf, but must not contain the current IOBuf. + * However, the specified head and tail may be equal to each other (i.e., + * they may be a subchain of length 1). + */ + std::unique_ptr separateChain(IOBuf* head, IOBuf* tail) { + assert(head != this); + assert(tail != this); + + head->prev_->next_ = tail->next_; + tail->next_->prev_ = head->prev_; + + head->prev_ = tail; + tail->next_ = head; + + return std::unique_ptr(head); + } + + /** + * Return true if at least one of the IOBufs in this chain are shared, + * or false if all of the IOBufs point to unique buffers. + * + * Use isSharedOne() to only check this IOBuf rather than the entire chain. + */ + bool isShared() const { + const IOBuf* current = this; + while (true) { + if (current->isSharedOne()) { + return true; + } + current = current->next_; + if (current == this) { + return false; + } + } + } + + /** + * Return true if other IOBufs are also pointing to the buffer used by this + * IOBuf, and false otherwise. + * + * If this IOBuf points at a buffer owned by another (non-IOBuf) part of the + * code (i.e., if the IOBuf was created using wrapBuffer(), or was cloned + * from such an IOBuf), it is always considered shared. + * + * This only checks the current IOBuf, and not other IOBufs in the chain. + */ + bool isSharedOne() const { + // If this is a user-owned buffer, it is always considered shared + if (flags_ & kFlagUserOwned) { + return true; + } + + if (flags_ & kFlagExt) { + return ext_.sharedInfo->refcount.load(std::memory_order_acquire) > 1; + } else { + return false; + } + } + + /** + * Ensure that this IOBuf has a unique buffer that is not shared by other + * IOBufs. + * + * unshare() operates on an entire chain of IOBuf objects. If the chain is + * shared, it may also coalesce the chain when making it unique. If the + * chain is coalesced, subsequent IOBuf objects in the current chain will be + * automatically deleted. + * + * Note that buffers owned by other (non-IOBuf) users are automatically + * considered shared. + * + * Throws std::bad_alloc on error. On error the IOBuf chain will be + * unmodified. + * + * Currently unshare may also throw std::overflow_error if it tries to + * coalesce. (TODO: In the future it would be nice if unshare() were smart + * enough not to coalesce the entire buffer if the data is too large. + * However, in practice this seems unlikely to become an issue.) + */ + void unshare() { + if (isChained()) { + unshareChained(); + } else { + unshareOne(); + } + } + + /** + * Ensure that this IOBuf has a unique buffer that is not shared by other + * IOBufs. + * + * unshareOne() operates on a single IOBuf object. This IOBuf will have a + * unique buffer after unshareOne() returns, but other IOBufs in the chain + * may still be shared after unshareOne() returns. + * + * Throws std::bad_alloc on error. On error the IOBuf will be unmodified. + */ + void unshareOne() { + if (isSharedOne()) { + unshareOneSlow(); + } + } + + /** + * Coalesce this IOBuf chain into a single buffer. + * + * This method moves all of the data in this IOBuf chain into a single + * contiguous buffer, if it is not already in one buffer. After coalesce() + * returns, this IOBuf will be a chain of length one. Other IOBufs in the + * chain will be automatically deleted. + * + * After coalescing, the IOBuf will have at least as much headroom as the + * first IOBuf in the chain, and at least as much tailroom as the last IOBuf + * in the chain. + * + * Throws std::bad_alloc on error. On error the IOBuf chain will be + * unmodified. Throws std::overflow_error if the length of the entire chain + * larger than can be described by a uint32_t capacity. + */ + void coalesce() { + if (!isChained()) { + return; + } + coalesceSlow(); + } + + /** + * Ensure that this chain has at least maxLength bytes available as a + * contiguous memory range. + * + * This method coalesces whole buffers in the chain into this buffer as + * necessary until this buffer's length() is at least maxLength. + * + * After coalescing, the IOBuf will have at least as much headroom as the + * first IOBuf in the chain, and at least as much tailroom as the last IOBuf + * that was coalesced. + * + * Throws std::bad_alloc on error. On error the IOBuf chain will be + * unmodified. Throws std::overflow_error if the length of the coalesced + * portion of the chain is larger than can be described by a uint32_t + * capacity. (Although maxLength is uint32_t, gather() doesn't split + * buffers, so coalescing whole buffers may result in a capacity that can't + * be described in uint32_t. + * + * Upon return, either enough of the chain was coalesced into a contiguous + * region, or the entire chain was coalesced. That is, + * length() >= maxLength || !isChained() is true. + */ + void gather(uint32_t maxLength) { + if (!isChained() || length_ >= maxLength) { + return; + } + coalesceSlow(maxLength); + } + + /** + * Return a new IOBuf chain sharing the same data as this chain. + * + * The new IOBuf chain will normally point to the same underlying data + * buffers as the original chain. (The one exception to this is if some of + * the IOBufs in this chain contain small internal data buffers which cannot + * be shared.) + */ + std::unique_ptr clone() const; + + /** + * Return a new IOBuf with the same data as this IOBuf. + * + * The new IOBuf returned will not be part of a chain (even if this IOBuf is + * part of a larger chain). + */ + std::unique_ptr cloneOne() const; + + // Overridden operator new and delete. + // These directly use malloc() and free() to allocate the space for IOBuf + // objects. This is needed since IOBuf::create() manually uses malloc when + // allocating IOBuf objects with an internal buffer. + void* operator new(size_t size); + void* operator new(size_t size, void* ptr); + void operator delete(void* ptr); + + private: + enum FlagsEnum { + kFlagExt = 0x1, + kFlagUserOwned = 0x2, + kFlagFreeSharedInfo = 0x4, + }; + + // Values for the ExternalBuf type field. + // We currently don't really use this for anything, other than to have it + // around for debugging purposes. We store it at the moment just because we + // have the 4 extra bytes in the ExternalBuf struct that would just be + // padding otherwise. + enum ExtBufTypeEnum { + kExtAllocated = 0, + kExtUserSupplied = 1, + kExtUserOwned = 2, + }; + + struct SharedInfo { + SharedInfo(); + SharedInfo(FreeFunction fn, void* arg); + + // A pointer to a function to call to free the buffer when the refcount + // hits 0. If this is NULL, free() will be used instead. + FreeFunction freeFn; + void* userData; + std::atomic refcount; + }; + struct ExternalBuf { + uint32_t capacity; + uint32_t type; + uint8_t* buf; + // SharedInfo may be NULL if kFlagUserOwned is set. It is non-NULL + // in all other cases. + SharedInfo* sharedInfo; + }; + struct InternalBuf { + uint8_t buf[] __attribute__((aligned)); + }; + + // The maximum size for an IOBuf object, including any internal data buffer + static const uint32_t kMaxIOBufSize = 256; + static const uint32_t kMaxInternalDataSize; + + // Forbidden copy constructor and assignment opererator + IOBuf(IOBuf const &); + IOBuf& operator=(IOBuf const &); + + /** + * Create a new IOBuf with internal data. + * + * end is a pointer to the end of the IOBuf's internal data buffer. + */ + explicit IOBuf(uint8_t* end); + + /** + * Create a new IOBuf pointing to an external buffer. + * + * The caller is responsible for holding a reference count for this new + * IOBuf. The IOBuf constructor does not automatically increment the + * reference count. + */ + IOBuf(ExtBufTypeEnum type, uint32_t flags, + uint8_t* buf, uint32_t capacity, + uint8_t* data, uint32_t length, + SharedInfo* sharedInfo); + + void unshareOneSlow(); + void unshareChained(); + void coalesceSlow(size_t maxLength=std::numeric_limits::max()); + void decrementRefcount(); + void reserveSlow(uint32_t minHeadroom, uint32_t minTailroom); + + static size_t goodExtBufferSize(uint32_t minCapacity); + static void initExtBuffer(uint8_t* buf, size_t mallocSize, + SharedInfo** infoReturn, + uint32_t* capacityReturn); + static void allocExtBuffer(uint32_t minCapacity, + uint8_t** bufReturn, + SharedInfo** infoReturn, + uint32_t* capacityReturn); + + /* + * Member variables + */ + + /* + * Links to the next and the previous IOBuf in this chain. + * + * The chain is circularly linked (the last element in the chain points back + * at the head), and next_ and prev_ can never be NULL. If this IOBuf is the + * only element in the chain, next_ and prev_ will both point to this. + */ + IOBuf* next_; + IOBuf* prev_; + + /* + * A pointer to the start of the data referenced by this IOBuf, and the + * length of the data. + * + * This may refer to any subsection of the actual buffer capacity. + */ + uint8_t* data_; + uint32_t length_; + uint32_t flags_; + + union { + ExternalBuf ext_; + InternalBuf int_; + }; + + struct DeleterBase { + virtual ~DeleterBase() { } + virtual void dispose(void* p) = 0; + }; + + template + struct UniquePtrDeleter : public DeleterBase { + typedef typename UniquePtr::pointer Pointer; + typedef typename UniquePtr::deleter_type Deleter; + + explicit UniquePtrDeleter(Deleter deleter) : deleter_(std::move(deleter)){ } + void dispose(void* p) { + try { + deleter_(static_cast(p)); + delete this; + } catch (...) { + abort(); + } + } + + private: + Deleter deleter_; + }; + + static void freeUniquePtrBuffer(void* ptr, void* userData) { + static_cast(userData)->dispose(ptr); + } +}; + +template +typename std::enable_if::value, + std::unique_ptr>::type +IOBuf::takeOwnership(UniquePtr&& buf, size_t count) { + size_t size = count * sizeof(typename UniquePtr::element_type); + CHECK_LT(size, size_t(std::numeric_limits::max())); + auto deleter = new UniquePtrDeleter(buf.get_deleter()); + return takeOwnership(buf.release(), + size, + &IOBuf::freeUniquePtrBuffer, + deleter); +} + +inline std::unique_ptr IOBuf::copyBuffer( + const void* data, uint32_t size, uint32_t headroom, + uint32_t minTailroom) { + uint32_t capacity = headroom + size + minTailroom; + std::unique_ptr buf = create(capacity); + buf->advance(headroom); + memcpy(buf->writableData(), data, size); + buf->append(size); + return buf; +} + +} // folly + +#endif // FOLLY_IO_IOBUF_H_ diff --git a/thrift/folly/experimental/io/IOBufQueue.h b/thrift/folly/experimental/io/IOBufQueue.h new file mode 100644 index 0000000000..283cd3ac5d --- /dev/null +++ b/thrift/folly/experimental/io/IOBufQueue.h @@ -0,0 +1,201 @@ +/* + * Copyright 2012 Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FOLLY_IO_IOBUF_QUEUE_H +#define FOLLY_IO_IOBUF_QUEUE_H + +#include "folly/experimental/io/IOBuf.h" + +#include +#include + +namespace folly { + +/** + * An IOBufQueue encapsulates a chain of IOBufs and provides + * convenience functions to append data to the back of the chain + * and remove data from the front. + */ +class IOBufQueue { + public: + struct Options { + Options() : cacheChainLength(false) { } + bool cacheChainLength; + }; + + explicit IOBufQueue(const Options& options = Options()); + + /** + * Add a buffer or buffer chain to the end of this queue. The + * queue takes ownership of buf. + */ + void append(std::unique_ptr&& buf); + + /** + * Add a queue to the end of this queue. The queue takes ownership of + * all buffers from the other queue. + */ + void append(IOBufQueue& other); + void append(IOBufQueue&& other) { + append(other); // call lvalue reference overload, above + } + + /** + * Copy len bytes, starting at buf, to the end of this queue. + * The caller retains ownership of the source data. + */ + void append(const void* buf, size_t len); + + /** + * Copy a string to the end of this queue. + * The caller retains ownership of the source data. + */ + void append(const std::string& buf) { + append(buf.data(), buf.length()); + } + + /** + * Append a chain of IOBuf objects that point to consecutive regions + * within buf. + * + * Just like IOBuf::wrapBuffer, this should only be used when the caller + * knows ahead of time and can ensure that all IOBuf objects that will point + * to this buffer will be destroyed before the buffer itself is destroyed; + * all other caveats from wrapBuffer also apply. + * + * Every buffer except for the last will wrap exactly blockSize bytes. + * Importantly, this method may be used to wrap buffers larger than 4GB. + */ + void wrapBuffer(const void* buf, size_t len, + uint32_t blockSize=(1U << 31)); // default block size: 2GB + + /** + * Obtain a writable block of contiguous bytes at the end of this + * queue, allocating more space if necessary. The amount of space + * reserved will be between min and max, inclusive; the IOBufQueue + * implementation may pick a value in that range that makes efficient + * use of already-allocated internal space. + * + * If the caller subsequently writes anything into the returned space, + * it must call the postallocate() method. + * + * @return The starting address of the block and the length in bytes. + * + * @note The point of the preallocate()/postallocate() mechanism is + * to support I/O APIs such as Thrift's TAsyncSocket::ReadCallback + * that request a buffer from the application and then, in a later + * callback, tell the application how much of the buffer they've + * filled with data. + */ + std::pair preallocate(uint32_t min, uint32_t max); + + /** + * Tell the queue that the caller has written data into the first n + * bytes provided by the previous preallocate() call. + * + * @note n should be less than or equal to the size returned by + * preallocate(). If n is zero, the caller may skip the call + * to postallocate(). If n is nonzero, the caller must not + * invoke any other non-const methods on this IOBufQueue between + * the call to preallocate and the call to postallocate(). + */ + void postallocate(uint32_t n); + + /** + * Obtain a writable block of n contiguous bytes, allocating more space + * if necessary, and mark it as used. The caller can fill it later. + */ + void* allocate(uint32_t n) { + void* p = preallocate(n, n).first; + postallocate(n); + return p; + } + + /** + * Split off the first n bytes of the queue into a separate IOBuf chain, + * and transfer ownership of the new chain to the caller. The IOBufQueue + * retains ownership of everything after the split point. + * + * @warning If the split point lies in the middle of some IOBuf within + * the chain, this function may, as an implementation detail, + * clone that IOBuf. + * + * @throws std::underflow_error if n exceeds the number of bytes + * in the queue. + */ + std::unique_ptr split(size_t n); + + /** + * Similar to IOBuf::trimStart, but works on the whole queue. Will + * pop off buffers that have been completely trimmed. + */ + void trimStart(size_t amount); + + /** + * Similar to IOBuf::trimEnd, but works on the whole queue. Will + * pop off buffers that have been completely trimmed. + */ + void trimEnd(size_t amount); + + /** + * Transfer ownership of the queue's entire IOBuf chain to the caller. + */ + std::unique_ptr&& move() { + chainLength_ = 0; + return std::move(head_); + } + + /** + * Access + */ + const folly::IOBuf* front() const { + return head_.get(); + } + + /** + * Total chain length, only valid if cacheLength was specified in the + * constructor. + */ + size_t chainLength() const { + if (!options_.cacheChainLength) { + throw std::invalid_argument("IOBufQueue: chain length not cached"); + } + return chainLength_; + } + + const Options& options() const { + return options_; + } + + /** Movable */ + IOBufQueue(IOBufQueue&&); + IOBufQueue& operator=(IOBufQueue&&); + + private: + static const size_t kChainLengthNotCached = (size_t)-1; + /** Not copyable */ + IOBufQueue(const IOBufQueue&) = delete; + IOBufQueue& operator=(const IOBufQueue&) = delete; + + Options options_; + size_t chainLength_; + /** Everything that has been appended but not yet discarded or moved out */ + std::unique_ptr head_; +}; + +} // folly + +#endif // FOLLY_IO_IOBUF_QUEUE_H diff --git a/thrift/folly/experimental/io/check.h b/thrift/folly/experimental/io/check.h new file mode 100644 index 0000000000..52324a39d0 --- /dev/null +++ b/thrift/folly/experimental/io/check.h @@ -0,0 +1,2 @@ +#define CHECK(a) +#define CHECK_LT(a, b) diff --git a/thrift/gen-cpp/DB.cpp b/thrift/gen-cpp/DB.cpp new file mode 100644 index 0000000000..183f43d3fa --- /dev/null +++ b/thrift/gen-cpp/DB.cpp @@ -0,0 +1,4161 @@ +/** + * Autogenerated by Thrift + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +#include "DB.h" + +namespace Tleveldb { + +uint32_t DB_Open_args::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == apache::thrift::protocol::T_STRING) { + xfer += iprot->readBinary(this->dbname); + this->__isset.dbname = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->dboptions.read(iprot); + this->__isset.dboptions = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t DB_Open_args::write(apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("DB_Open_args"); + xfer += oprot->writeFieldBegin("dbname", apache::thrift::protocol::T_STRING, 1); + xfer += oprot->writeBinary(this->dbname); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("dboptions", apache::thrift::protocol::T_STRUCT, 2); + xfer += this->dboptions.write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +uint32_t DB_Open_pargs::write(apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("DB_Open_pargs"); + xfer += oprot->writeFieldBegin("dbname", apache::thrift::protocol::T_STRING, 1); + xfer += oprot->writeBinary((*(this->dbname))); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("dboptions", apache::thrift::protocol::T_STRUCT, 2); + xfer += (*(this->dboptions)).write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +uint32_t DB_Open_result::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 0: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->success.read(iprot); + this->__isset.success = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 1: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->se.read(iprot); + this->__isset.se = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t DB_Open_result::write(apache::thrift::protocol::TProtocol* oprot) const { + + uint32_t xfer = 0; + + xfer += oprot->writeStructBegin("DB_Open_result"); + + if (this->__isset.success) { + xfer += oprot->writeFieldBegin("success", apache::thrift::protocol::T_STRUCT, 0); + xfer += this->success.write(oprot); + xfer += oprot->writeFieldEnd(); + } else if (this->__isset.se) { + xfer += oprot->writeFieldBegin("se", apache::thrift::protocol::T_STRUCT, 1); + xfer += this->se.write(oprot); + xfer += oprot->writeFieldEnd(); + } + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +uint32_t DB_Open_presult::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 0: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += (*(this->success)).read(iprot); + this->__isset.success = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 1: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->se.read(iprot); + this->__isset.se = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t DB_Close_args::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->dbhandle.read(iprot); + this->__isset.dbhandle = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == apache::thrift::protocol::T_STRING) { + xfer += iprot->readBinary(this->dbname); + this->__isset.dbname = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t DB_Close_args::write(apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("DB_Close_args"); + xfer += oprot->writeFieldBegin("dbhandle", apache::thrift::protocol::T_STRUCT, 1); + xfer += this->dbhandle.write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("dbname", apache::thrift::protocol::T_STRING, 2); + xfer += oprot->writeBinary(this->dbname); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +uint32_t DB_Close_pargs::write(apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("DB_Close_pargs"); + xfer += oprot->writeFieldBegin("dbhandle", apache::thrift::protocol::T_STRUCT, 1); + xfer += (*(this->dbhandle)).write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("dbname", apache::thrift::protocol::T_STRING, 2); + xfer += oprot->writeBinary((*(this->dbname))); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +uint32_t DB_Close_result::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 0: + if (ftype == apache::thrift::protocol::T_I32) { + int32_t ecast6; + xfer += iprot->readI32(ecast6); + this->success = (Code)ecast6; + this->__isset.success = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t DB_Close_result::write(apache::thrift::protocol::TProtocol* oprot) const { + + uint32_t xfer = 0; + + xfer += oprot->writeStructBegin("DB_Close_result"); + + if (this->__isset.success) { + xfer += oprot->writeFieldBegin("success", apache::thrift::protocol::T_I32, 0); + xfer += oprot->writeI32((int32_t)this->success); + xfer += oprot->writeFieldEnd(); + } + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +uint32_t DB_Close_presult::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 0: + if (ftype == apache::thrift::protocol::T_I32) { + int32_t ecast7; + xfer += iprot->readI32(ecast7); + (*(this->success)) = (Code)ecast7; + this->__isset.success = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t DB_Put_args::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->dbhandle.read(iprot); + this->__isset.dbhandle = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->keyvalue.read(iprot); + this->__isset.keyvalue = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 3: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->options.read(iprot); + this->__isset.options = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t DB_Put_args::write(apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("DB_Put_args"); + xfer += oprot->writeFieldBegin("dbhandle", apache::thrift::protocol::T_STRUCT, 1); + xfer += this->dbhandle.write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("keyvalue", apache::thrift::protocol::T_STRUCT, 2); + xfer += this->keyvalue.write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("options", apache::thrift::protocol::T_STRUCT, 3); + xfer += this->options.write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +uint32_t DB_Put_pargs::write(apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("DB_Put_pargs"); + xfer += oprot->writeFieldBegin("dbhandle", apache::thrift::protocol::T_STRUCT, 1); + xfer += (*(this->dbhandle)).write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("keyvalue", apache::thrift::protocol::T_STRUCT, 2); + xfer += (*(this->keyvalue)).write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("options", apache::thrift::protocol::T_STRUCT, 3); + xfer += (*(this->options)).write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +uint32_t DB_Put_result::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 0: + if (ftype == apache::thrift::protocol::T_I32) { + int32_t ecast8; + xfer += iprot->readI32(ecast8); + this->success = (Code)ecast8; + this->__isset.success = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t DB_Put_result::write(apache::thrift::protocol::TProtocol* oprot) const { + + uint32_t xfer = 0; + + xfer += oprot->writeStructBegin("DB_Put_result"); + + if (this->__isset.success) { + xfer += oprot->writeFieldBegin("success", apache::thrift::protocol::T_I32, 0); + xfer += oprot->writeI32((int32_t)this->success); + xfer += oprot->writeFieldEnd(); + } + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +uint32_t DB_Put_presult::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 0: + if (ftype == apache::thrift::protocol::T_I32) { + int32_t ecast9; + xfer += iprot->readI32(ecast9); + (*(this->success)) = (Code)ecast9; + this->__isset.success = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t DB_Delete_args::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->dbhandle.read(iprot); + this->__isset.dbhandle = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->key.read(iprot); + this->__isset.key = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 3: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->options.read(iprot); + this->__isset.options = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t DB_Delete_args::write(apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("DB_Delete_args"); + xfer += oprot->writeFieldBegin("dbhandle", apache::thrift::protocol::T_STRUCT, 1); + xfer += this->dbhandle.write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("key", apache::thrift::protocol::T_STRUCT, 2); + xfer += this->key.write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("options", apache::thrift::protocol::T_STRUCT, 3); + xfer += this->options.write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +uint32_t DB_Delete_pargs::write(apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("DB_Delete_pargs"); + xfer += oprot->writeFieldBegin("dbhandle", apache::thrift::protocol::T_STRUCT, 1); + xfer += (*(this->dbhandle)).write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("key", apache::thrift::protocol::T_STRUCT, 2); + xfer += (*(this->key)).write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("options", apache::thrift::protocol::T_STRUCT, 3); + xfer += (*(this->options)).write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +uint32_t DB_Delete_result::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 0: + if (ftype == apache::thrift::protocol::T_I32) { + int32_t ecast10; + xfer += iprot->readI32(ecast10); + this->success = (Code)ecast10; + this->__isset.success = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t DB_Delete_result::write(apache::thrift::protocol::TProtocol* oprot) const { + + uint32_t xfer = 0; + + xfer += oprot->writeStructBegin("DB_Delete_result"); + + if (this->__isset.success) { + xfer += oprot->writeFieldBegin("success", apache::thrift::protocol::T_I32, 0); + xfer += oprot->writeI32((int32_t)this->success); + xfer += oprot->writeFieldEnd(); + } + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +uint32_t DB_Delete_presult::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 0: + if (ftype == apache::thrift::protocol::T_I32) { + int32_t ecast11; + xfer += iprot->readI32(ecast11); + (*(this->success)) = (Code)ecast11; + this->__isset.success = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t DB_Write_args::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->dbhandle.read(iprot); + this->__isset.dbhandle = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == apache::thrift::protocol::T_LIST) { + { + this->batch.clear(); + uint32_t _size12; + apache::thrift::protocol::TType _etype15; + xfer += iprot->readListBegin(_etype15, _size12); + this->batch.resize(_size12); + uint32_t _i16; + for (_i16 = 0; _i16 < _size12; ++_i16) + { + xfer += this->batch[_i16].read(iprot); + } + xfer += iprot->readListEnd(); + } + this->__isset.batch = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 3: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->options.read(iprot); + this->__isset.options = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t DB_Write_args::write(apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("DB_Write_args"); + xfer += oprot->writeFieldBegin("dbhandle", apache::thrift::protocol::T_STRUCT, 1); + xfer += this->dbhandle.write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("batch", apache::thrift::protocol::T_LIST, 2); + { + xfer += oprot->writeListBegin(apache::thrift::protocol::T_STRUCT, this->batch.size()); + std::vector ::const_iterator _iter17; + for (_iter17 = this->batch.begin(); _iter17 != this->batch.end(); ++_iter17) + { + xfer += (*_iter17).write(oprot); + } + xfer += oprot->writeListEnd(); + } + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("options", apache::thrift::protocol::T_STRUCT, 3); + xfer += this->options.write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +uint32_t DB_Write_pargs::write(apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("DB_Write_pargs"); + xfer += oprot->writeFieldBegin("dbhandle", apache::thrift::protocol::T_STRUCT, 1); + xfer += (*(this->dbhandle)).write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("batch", apache::thrift::protocol::T_LIST, 2); + { + xfer += oprot->writeListBegin(apache::thrift::protocol::T_STRUCT, (*(this->batch)).size()); + std::vector ::const_iterator _iter18; + for (_iter18 = (*(this->batch)).begin(); _iter18 != (*(this->batch)).end(); ++_iter18) + { + xfer += (*_iter18).write(oprot); + } + xfer += oprot->writeListEnd(); + } + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("options", apache::thrift::protocol::T_STRUCT, 3); + xfer += (*(this->options)).write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +uint32_t DB_Write_result::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 0: + if (ftype == apache::thrift::protocol::T_I32) { + int32_t ecast19; + xfer += iprot->readI32(ecast19); + this->success = (Code)ecast19; + this->__isset.success = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t DB_Write_result::write(apache::thrift::protocol::TProtocol* oprot) const { + + uint32_t xfer = 0; + + xfer += oprot->writeStructBegin("DB_Write_result"); + + if (this->__isset.success) { + xfer += oprot->writeFieldBegin("success", apache::thrift::protocol::T_I32, 0); + xfer += oprot->writeI32((int32_t)this->success); + xfer += oprot->writeFieldEnd(); + } + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +uint32_t DB_Write_presult::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 0: + if (ftype == apache::thrift::protocol::T_I32) { + int32_t ecast20; + xfer += iprot->readI32(ecast20); + (*(this->success)) = (Code)ecast20; + this->__isset.success = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t DB_Get_args::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->dbhandle.read(iprot); + this->__isset.dbhandle = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->inputkey.read(iprot); + this->__isset.inputkey = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 3: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->options.read(iprot); + this->__isset.options = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t DB_Get_args::write(apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("DB_Get_args"); + xfer += oprot->writeFieldBegin("dbhandle", apache::thrift::protocol::T_STRUCT, 1); + xfer += this->dbhandle.write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("inputkey", apache::thrift::protocol::T_STRUCT, 2); + xfer += this->inputkey.write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("options", apache::thrift::protocol::T_STRUCT, 3); + xfer += this->options.write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +uint32_t DB_Get_pargs::write(apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("DB_Get_pargs"); + xfer += oprot->writeFieldBegin("dbhandle", apache::thrift::protocol::T_STRUCT, 1); + xfer += (*(this->dbhandle)).write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("inputkey", apache::thrift::protocol::T_STRUCT, 2); + xfer += (*(this->inputkey)).write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("options", apache::thrift::protocol::T_STRUCT, 3); + xfer += (*(this->options)).write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +uint32_t DB_Get_result::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 0: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->success.read(iprot); + this->__isset.success = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t DB_Get_result::write(apache::thrift::protocol::TProtocol* oprot) const { + + uint32_t xfer = 0; + + xfer += oprot->writeStructBegin("DB_Get_result"); + + if (this->__isset.success) { + xfer += oprot->writeFieldBegin("success", apache::thrift::protocol::T_STRUCT, 0); + xfer += this->success.write(oprot); + xfer += oprot->writeFieldEnd(); + } + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +uint32_t DB_Get_presult::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 0: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += (*(this->success)).read(iprot); + this->__isset.success = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t DB_NewIterator_args::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->dbhandle.read(iprot); + this->__isset.dbhandle = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->options.read(iprot); + this->__isset.options = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 3: + if (ftype == apache::thrift::protocol::T_I32) { + int32_t ecast21; + xfer += iprot->readI32(ecast21); + this->iteratorType = (IteratorType)ecast21; + this->__isset.iteratorType = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 4: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->target.read(iprot); + this->__isset.target = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t DB_NewIterator_args::write(apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("DB_NewIterator_args"); + xfer += oprot->writeFieldBegin("dbhandle", apache::thrift::protocol::T_STRUCT, 1); + xfer += this->dbhandle.write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("options", apache::thrift::protocol::T_STRUCT, 2); + xfer += this->options.write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("iteratorType", apache::thrift::protocol::T_I32, 3); + xfer += oprot->writeI32((int32_t)this->iteratorType); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("target", apache::thrift::protocol::T_STRUCT, 4); + xfer += this->target.write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +uint32_t DB_NewIterator_pargs::write(apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("DB_NewIterator_pargs"); + xfer += oprot->writeFieldBegin("dbhandle", apache::thrift::protocol::T_STRUCT, 1); + xfer += (*(this->dbhandle)).write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("options", apache::thrift::protocol::T_STRUCT, 2); + xfer += (*(this->options)).write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("iteratorType", apache::thrift::protocol::T_I32, 3); + xfer += oprot->writeI32((int32_t)(*(this->iteratorType))); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("target", apache::thrift::protocol::T_STRUCT, 4); + xfer += (*(this->target)).write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +uint32_t DB_NewIterator_result::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 0: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->success.read(iprot); + this->__isset.success = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t DB_NewIterator_result::write(apache::thrift::protocol::TProtocol* oprot) const { + + uint32_t xfer = 0; + + xfer += oprot->writeStructBegin("DB_NewIterator_result"); + + if (this->__isset.success) { + xfer += oprot->writeFieldBegin("success", apache::thrift::protocol::T_STRUCT, 0); + xfer += this->success.write(oprot); + xfer += oprot->writeFieldEnd(); + } + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +uint32_t DB_NewIterator_presult::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 0: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += (*(this->success)).read(iprot); + this->__isset.success = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t DB_DeleteIterator_args::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->dbhandle.read(iprot); + this->__isset.dbhandle = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->iterator.read(iprot); + this->__isset.iterator = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t DB_DeleteIterator_args::write(apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("DB_DeleteIterator_args"); + xfer += oprot->writeFieldBegin("dbhandle", apache::thrift::protocol::T_STRUCT, 1); + xfer += this->dbhandle.write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("iterator", apache::thrift::protocol::T_STRUCT, 2); + xfer += this->iterator.write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +uint32_t DB_DeleteIterator_pargs::write(apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("DB_DeleteIterator_pargs"); + xfer += oprot->writeFieldBegin("dbhandle", apache::thrift::protocol::T_STRUCT, 1); + xfer += (*(this->dbhandle)).write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("iterator", apache::thrift::protocol::T_STRUCT, 2); + xfer += (*(this->iterator)).write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +uint32_t DB_DeleteIterator_result::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 0: + if (ftype == apache::thrift::protocol::T_I32) { + int32_t ecast22; + xfer += iprot->readI32(ecast22); + this->success = (Code)ecast22; + this->__isset.success = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t DB_DeleteIterator_result::write(apache::thrift::protocol::TProtocol* oprot) const { + + uint32_t xfer = 0; + + xfer += oprot->writeStructBegin("DB_DeleteIterator_result"); + + if (this->__isset.success) { + xfer += oprot->writeFieldBegin("success", apache::thrift::protocol::T_I32, 0); + xfer += oprot->writeI32((int32_t)this->success); + xfer += oprot->writeFieldEnd(); + } + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +uint32_t DB_DeleteIterator_presult::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 0: + if (ftype == apache::thrift::protocol::T_I32) { + int32_t ecast23; + xfer += iprot->readI32(ecast23); + (*(this->success)) = (Code)ecast23; + this->__isset.success = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t DB_GetNext_args::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->dbhandle.read(iprot); + this->__isset.dbhandle = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->iterator.read(iprot); + this->__isset.iterator = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t DB_GetNext_args::write(apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("DB_GetNext_args"); + xfer += oprot->writeFieldBegin("dbhandle", apache::thrift::protocol::T_STRUCT, 1); + xfer += this->dbhandle.write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("iterator", apache::thrift::protocol::T_STRUCT, 2); + xfer += this->iterator.write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +uint32_t DB_GetNext_pargs::write(apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("DB_GetNext_pargs"); + xfer += oprot->writeFieldBegin("dbhandle", apache::thrift::protocol::T_STRUCT, 1); + xfer += (*(this->dbhandle)).write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("iterator", apache::thrift::protocol::T_STRUCT, 2); + xfer += (*(this->iterator)).write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +uint32_t DB_GetNext_result::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 0: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->success.read(iprot); + this->__isset.success = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t DB_GetNext_result::write(apache::thrift::protocol::TProtocol* oprot) const { + + uint32_t xfer = 0; + + xfer += oprot->writeStructBegin("DB_GetNext_result"); + + if (this->__isset.success) { + xfer += oprot->writeFieldBegin("success", apache::thrift::protocol::T_STRUCT, 0); + xfer += this->success.write(oprot); + xfer += oprot->writeFieldEnd(); + } + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +uint32_t DB_GetNext_presult::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 0: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += (*(this->success)).read(iprot); + this->__isset.success = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t DB_GetPrev_args::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->dbhandle.read(iprot); + this->__isset.dbhandle = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->iterator.read(iprot); + this->__isset.iterator = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t DB_GetPrev_args::write(apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("DB_GetPrev_args"); + xfer += oprot->writeFieldBegin("dbhandle", apache::thrift::protocol::T_STRUCT, 1); + xfer += this->dbhandle.write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("iterator", apache::thrift::protocol::T_STRUCT, 2); + xfer += this->iterator.write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +uint32_t DB_GetPrev_pargs::write(apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("DB_GetPrev_pargs"); + xfer += oprot->writeFieldBegin("dbhandle", apache::thrift::protocol::T_STRUCT, 1); + xfer += (*(this->dbhandle)).write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("iterator", apache::thrift::protocol::T_STRUCT, 2); + xfer += (*(this->iterator)).write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +uint32_t DB_GetPrev_result::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 0: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->success.read(iprot); + this->__isset.success = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t DB_GetPrev_result::write(apache::thrift::protocol::TProtocol* oprot) const { + + uint32_t xfer = 0; + + xfer += oprot->writeStructBegin("DB_GetPrev_result"); + + if (this->__isset.success) { + xfer += oprot->writeFieldBegin("success", apache::thrift::protocol::T_STRUCT, 0); + xfer += this->success.write(oprot); + xfer += oprot->writeFieldEnd(); + } + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +uint32_t DB_GetPrev_presult::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 0: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += (*(this->success)).read(iprot); + this->__isset.success = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t DB_GetSnapshot_args::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->dbhandle.read(iprot); + this->__isset.dbhandle = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t DB_GetSnapshot_args::write(apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("DB_GetSnapshot_args"); + xfer += oprot->writeFieldBegin("dbhandle", apache::thrift::protocol::T_STRUCT, 1); + xfer += this->dbhandle.write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +uint32_t DB_GetSnapshot_pargs::write(apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("DB_GetSnapshot_pargs"); + xfer += oprot->writeFieldBegin("dbhandle", apache::thrift::protocol::T_STRUCT, 1); + xfer += (*(this->dbhandle)).write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +uint32_t DB_GetSnapshot_result::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 0: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->success.read(iprot); + this->__isset.success = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t DB_GetSnapshot_result::write(apache::thrift::protocol::TProtocol* oprot) const { + + uint32_t xfer = 0; + + xfer += oprot->writeStructBegin("DB_GetSnapshot_result"); + + if (this->__isset.success) { + xfer += oprot->writeFieldBegin("success", apache::thrift::protocol::T_STRUCT, 0); + xfer += this->success.write(oprot); + xfer += oprot->writeFieldEnd(); + } + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +uint32_t DB_GetSnapshot_presult::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 0: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += (*(this->success)).read(iprot); + this->__isset.success = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t DB_ReleaseSnapshot_args::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->dbhandle.read(iprot); + this->__isset.dbhandle = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->snapshot.read(iprot); + this->__isset.snapshot = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t DB_ReleaseSnapshot_args::write(apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("DB_ReleaseSnapshot_args"); + xfer += oprot->writeFieldBegin("dbhandle", apache::thrift::protocol::T_STRUCT, 1); + xfer += this->dbhandle.write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("snapshot", apache::thrift::protocol::T_STRUCT, 2); + xfer += this->snapshot.write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +uint32_t DB_ReleaseSnapshot_pargs::write(apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("DB_ReleaseSnapshot_pargs"); + xfer += oprot->writeFieldBegin("dbhandle", apache::thrift::protocol::T_STRUCT, 1); + xfer += (*(this->dbhandle)).write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("snapshot", apache::thrift::protocol::T_STRUCT, 2); + xfer += (*(this->snapshot)).write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +uint32_t DB_ReleaseSnapshot_result::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 0: + if (ftype == apache::thrift::protocol::T_I32) { + int32_t ecast24; + xfer += iprot->readI32(ecast24); + this->success = (Code)ecast24; + this->__isset.success = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t DB_ReleaseSnapshot_result::write(apache::thrift::protocol::TProtocol* oprot) const { + + uint32_t xfer = 0; + + xfer += oprot->writeStructBegin("DB_ReleaseSnapshot_result"); + + if (this->__isset.success) { + xfer += oprot->writeFieldBegin("success", apache::thrift::protocol::T_I32, 0); + xfer += oprot->writeI32((int32_t)this->success); + xfer += oprot->writeFieldEnd(); + } + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +uint32_t DB_ReleaseSnapshot_presult::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 0: + if (ftype == apache::thrift::protocol::T_I32) { + int32_t ecast25; + xfer += iprot->readI32(ecast25); + (*(this->success)) = (Code)ecast25; + this->__isset.success = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t DB_CompactRange_args::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->dbhandle.read(iprot); + this->__isset.dbhandle = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->begin.read(iprot); + this->__isset.begin = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 3: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->end.read(iprot); + this->__isset.end = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t DB_CompactRange_args::write(apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("DB_CompactRange_args"); + xfer += oprot->writeFieldBegin("dbhandle", apache::thrift::protocol::T_STRUCT, 1); + xfer += this->dbhandle.write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("begin", apache::thrift::protocol::T_STRUCT, 2); + xfer += this->begin.write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("end", apache::thrift::protocol::T_STRUCT, 3); + xfer += this->end.write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +uint32_t DB_CompactRange_pargs::write(apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("DB_CompactRange_pargs"); + xfer += oprot->writeFieldBegin("dbhandle", apache::thrift::protocol::T_STRUCT, 1); + xfer += (*(this->dbhandle)).write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("begin", apache::thrift::protocol::T_STRUCT, 2); + xfer += (*(this->begin)).write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("end", apache::thrift::protocol::T_STRUCT, 3); + xfer += (*(this->end)).write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +uint32_t DB_CompactRange_result::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 0: + if (ftype == apache::thrift::protocol::T_I32) { + int32_t ecast26; + xfer += iprot->readI32(ecast26); + this->success = (Code)ecast26; + this->__isset.success = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t DB_CompactRange_result::write(apache::thrift::protocol::TProtocol* oprot) const { + + uint32_t xfer = 0; + + xfer += oprot->writeStructBegin("DB_CompactRange_result"); + + if (this->__isset.success) { + xfer += oprot->writeFieldBegin("success", apache::thrift::protocol::T_I32, 0); + xfer += oprot->writeI32((int32_t)this->success); + xfer += oprot->writeFieldEnd(); + } + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +uint32_t DB_CompactRange_presult::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 0: + if (ftype == apache::thrift::protocol::T_I32) { + int32_t ecast27; + xfer += iprot->readI32(ecast27); + (*(this->success)) = (Code)ecast27; + this->__isset.success = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +int32_t DBClient::getNextSendSequenceId() +{ + return nextSendSequenceId_++; +} + +int32_t DBClient::getNextRecvSequenceId() +{ + return nextRecvSequenceId_++; +} + +void DBClient::Open(DBHandle& _return, const Text& dbname, const DBOptions& dboptions) +{ + std::unique_ptr ctx(this->getContextStack("DB.Open", NULL)); + + try { + this->setContextStack(ctx.get()); + send_Open(dbname, dboptions); + recv_Open(_return); + } catch(apache::thrift::transport::TTransportException& ex) { + this->handlerError(ctx.get(), "DB.Open"); + iprot_->getTransport()->close(); + oprot_->getTransport()->close(); + throw; + } catch(apache::thrift::TApplicationException& ex) { + if (ex.getType() == apache::thrift::TApplicationException::BAD_SEQUENCE_ID) { + this->handlerError(ctx.get(), "DB.Open"); + iprot_->getTransport()->close(); + oprot_->getTransport()->close(); + } + throw; + } +} + +void DBClient::send_Open(const Text& dbname, const DBOptions& dboptions) +{ + apache::thrift::ContextStack* ctx = this->getContextStack(); + this->preWrite(ctx, "DB.Open"); + oprot_->writeMessageBegin("Open", apache::thrift::protocol::T_CALL, getNextSendSequenceId()); + + DB_Open_pargs args; + args.dbname = &dbname; + args.dboptions = &dboptions; + args.write(oprot_); + + oprot_->writeMessageEnd(); + uint32_t _bytes28 = oprot_->getTransport()->writeEnd(); + oprot_->getTransport()->flush(); + this->postWrite(ctx, "DB.Open", _bytes28); + return; +} + +void DBClient::recv_Open(DBHandle& _return) +{ + apache::thrift::ContextStack* ctx = this->getContextStack(); + uint32_t bytes; + int32_t rseqid = 0; + int32_t eseqid = getNextRecvSequenceId(); + std::string fname; + apache::thrift::protocol::TMessageType mtype; + this->preRead(ctx, "DB.Open"); + + iprot_->readMessageBegin(fname, mtype, rseqid); + if (this->checkSeqid_ && rseqid != eseqid) { + iprot_->skip(apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::BAD_SEQUENCE_ID); + } + if (mtype == apache::thrift::protocol::T_EXCEPTION) { + apache::thrift::TApplicationException x; + x.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw x; + } + if (mtype != apache::thrift::protocol::T_REPLY) { + iprot_->skip(apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::INVALID_MESSAGE_TYPE); + } + if (fname.compare("Open") != 0) { + iprot_->skip(apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::WRONG_METHOD_NAME); + } + DB_Open_presult result; + result.success = &_return; + result.read(iprot_); + iprot_->readMessageEnd(); + bytes = iprot_->getTransport()->readEnd(); + this->postRead(ctx, "DB.Open", bytes); + + if (result.__isset.success) { + // _return pointer has now been filled + return; + } + if (result.__isset.se) { + throw result.se; + } + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::MISSING_RESULT, "Open failed: unknown result"); +} + +Code DBClient::Close(const DBHandle& dbhandle, const Text& dbname) +{ + std::unique_ptr ctx(this->getContextStack("DB.Close", NULL)); + + try { + this->setContextStack(ctx.get()); + send_Close(dbhandle, dbname); + return recv_Close(); + } catch(apache::thrift::transport::TTransportException& ex) { + this->handlerError(ctx.get(), "DB.Close"); + iprot_->getTransport()->close(); + oprot_->getTransport()->close(); + throw; + } catch(apache::thrift::TApplicationException& ex) { + if (ex.getType() == apache::thrift::TApplicationException::BAD_SEQUENCE_ID) { + this->handlerError(ctx.get(), "DB.Close"); + iprot_->getTransport()->close(); + oprot_->getTransport()->close(); + } + throw; + } +} + +void DBClient::send_Close(const DBHandle& dbhandle, const Text& dbname) +{ + apache::thrift::ContextStack* ctx = this->getContextStack(); + this->preWrite(ctx, "DB.Close"); + oprot_->writeMessageBegin("Close", apache::thrift::protocol::T_CALL, getNextSendSequenceId()); + + DB_Close_pargs args; + args.dbhandle = &dbhandle; + args.dbname = &dbname; + args.write(oprot_); + + oprot_->writeMessageEnd(); + uint32_t _bytes29 = oprot_->getTransport()->writeEnd(); + oprot_->getTransport()->flush(); + this->postWrite(ctx, "DB.Close", _bytes29); + return; +} + +Code DBClient::recv_Close() +{ + apache::thrift::ContextStack* ctx = this->getContextStack(); + uint32_t bytes; + int32_t rseqid = 0; + int32_t eseqid = getNextRecvSequenceId(); + std::string fname; + apache::thrift::protocol::TMessageType mtype; + this->preRead(ctx, "DB.Close"); + + iprot_->readMessageBegin(fname, mtype, rseqid); + if (this->checkSeqid_ && rseqid != eseqid) { + iprot_->skip(apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::BAD_SEQUENCE_ID); + } + if (mtype == apache::thrift::protocol::T_EXCEPTION) { + apache::thrift::TApplicationException x; + x.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw x; + } + if (mtype != apache::thrift::protocol::T_REPLY) { + iprot_->skip(apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::INVALID_MESSAGE_TYPE); + } + if (fname.compare("Close") != 0) { + iprot_->skip(apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::WRONG_METHOD_NAME); + } + Code _return; + DB_Close_presult result; + result.success = &_return; + result.read(iprot_); + iprot_->readMessageEnd(); + bytes = iprot_->getTransport()->readEnd(); + this->postRead(ctx, "DB.Close", bytes); + + if (result.__isset.success) { + return _return; + } + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::MISSING_RESULT, "Close failed: unknown result"); +} + +Code DBClient::Put(const DBHandle& dbhandle, const kv& keyvalue, const WriteOptions& options) +{ + std::unique_ptr ctx(this->getContextStack("DB.Put", NULL)); + + try { + this->setContextStack(ctx.get()); + send_Put(dbhandle, keyvalue, options); + return recv_Put(); + } catch(apache::thrift::transport::TTransportException& ex) { + this->handlerError(ctx.get(), "DB.Put"); + iprot_->getTransport()->close(); + oprot_->getTransport()->close(); + throw; + } catch(apache::thrift::TApplicationException& ex) { + if (ex.getType() == apache::thrift::TApplicationException::BAD_SEQUENCE_ID) { + this->handlerError(ctx.get(), "DB.Put"); + iprot_->getTransport()->close(); + oprot_->getTransport()->close(); + } + throw; + } +} + +void DBClient::send_Put(const DBHandle& dbhandle, const kv& keyvalue, const WriteOptions& options) +{ + apache::thrift::ContextStack* ctx = this->getContextStack(); + this->preWrite(ctx, "DB.Put"); + oprot_->writeMessageBegin("Put", apache::thrift::protocol::T_CALL, getNextSendSequenceId()); + + DB_Put_pargs args; + args.dbhandle = &dbhandle; + args.keyvalue = &keyvalue; + args.options = &options; + args.write(oprot_); + + oprot_->writeMessageEnd(); + uint32_t _bytes30 = oprot_->getTransport()->writeEnd(); + oprot_->getTransport()->flush(); + this->postWrite(ctx, "DB.Put", _bytes30); + return; +} + +Code DBClient::recv_Put() +{ + apache::thrift::ContextStack* ctx = this->getContextStack(); + uint32_t bytes; + int32_t rseqid = 0; + int32_t eseqid = getNextRecvSequenceId(); + std::string fname; + apache::thrift::protocol::TMessageType mtype; + this->preRead(ctx, "DB.Put"); + + iprot_->readMessageBegin(fname, mtype, rseqid); + if (this->checkSeqid_ && rseqid != eseqid) { + iprot_->skip(apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::BAD_SEQUENCE_ID); + } + if (mtype == apache::thrift::protocol::T_EXCEPTION) { + apache::thrift::TApplicationException x; + x.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw x; + } + if (mtype != apache::thrift::protocol::T_REPLY) { + iprot_->skip(apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::INVALID_MESSAGE_TYPE); + } + if (fname.compare("Put") != 0) { + iprot_->skip(apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::WRONG_METHOD_NAME); + } + Code _return; + DB_Put_presult result; + result.success = &_return; + result.read(iprot_); + iprot_->readMessageEnd(); + bytes = iprot_->getTransport()->readEnd(); + this->postRead(ctx, "DB.Put", bytes); + + if (result.__isset.success) { + return _return; + } + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::MISSING_RESULT, "Put failed: unknown result"); +} + +Code DBClient::Delete(const DBHandle& dbhandle, const Slice& key, const WriteOptions& options) +{ + std::unique_ptr ctx(this->getContextStack("DB.Delete", NULL)); + + try { + this->setContextStack(ctx.get()); + send_Delete(dbhandle, key, options); + return recv_Delete(); + } catch(apache::thrift::transport::TTransportException& ex) { + this->handlerError(ctx.get(), "DB.Delete"); + iprot_->getTransport()->close(); + oprot_->getTransport()->close(); + throw; + } catch(apache::thrift::TApplicationException& ex) { + if (ex.getType() == apache::thrift::TApplicationException::BAD_SEQUENCE_ID) { + this->handlerError(ctx.get(), "DB.Delete"); + iprot_->getTransport()->close(); + oprot_->getTransport()->close(); + } + throw; + } +} + +void DBClient::send_Delete(const DBHandle& dbhandle, const Slice& key, const WriteOptions& options) +{ + apache::thrift::ContextStack* ctx = this->getContextStack(); + this->preWrite(ctx, "DB.Delete"); + oprot_->writeMessageBegin("Delete", apache::thrift::protocol::T_CALL, getNextSendSequenceId()); + + DB_Delete_pargs args; + args.dbhandle = &dbhandle; + args.key = &key; + args.options = &options; + args.write(oprot_); + + oprot_->writeMessageEnd(); + uint32_t _bytes31 = oprot_->getTransport()->writeEnd(); + oprot_->getTransport()->flush(); + this->postWrite(ctx, "DB.Delete", _bytes31); + return; +} + +Code DBClient::recv_Delete() +{ + apache::thrift::ContextStack* ctx = this->getContextStack(); + uint32_t bytes; + int32_t rseqid = 0; + int32_t eseqid = getNextRecvSequenceId(); + std::string fname; + apache::thrift::protocol::TMessageType mtype; + this->preRead(ctx, "DB.Delete"); + + iprot_->readMessageBegin(fname, mtype, rseqid); + if (this->checkSeqid_ && rseqid != eseqid) { + iprot_->skip(apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::BAD_SEQUENCE_ID); + } + if (mtype == apache::thrift::protocol::T_EXCEPTION) { + apache::thrift::TApplicationException x; + x.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw x; + } + if (mtype != apache::thrift::protocol::T_REPLY) { + iprot_->skip(apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::INVALID_MESSAGE_TYPE); + } + if (fname.compare("Delete") != 0) { + iprot_->skip(apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::WRONG_METHOD_NAME); + } + Code _return; + DB_Delete_presult result; + result.success = &_return; + result.read(iprot_); + iprot_->readMessageEnd(); + bytes = iprot_->getTransport()->readEnd(); + this->postRead(ctx, "DB.Delete", bytes); + + if (result.__isset.success) { + return _return; + } + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::MISSING_RESULT, "Delete failed: unknown result"); +} + +Code DBClient::Write(const DBHandle& dbhandle, const std::vector & batch, const WriteOptions& options) +{ + std::unique_ptr ctx(this->getContextStack("DB.Write", NULL)); + + try { + this->setContextStack(ctx.get()); + send_Write(dbhandle, batch, options); + return recv_Write(); + } catch(apache::thrift::transport::TTransportException& ex) { + this->handlerError(ctx.get(), "DB.Write"); + iprot_->getTransport()->close(); + oprot_->getTransport()->close(); + throw; + } catch(apache::thrift::TApplicationException& ex) { + if (ex.getType() == apache::thrift::TApplicationException::BAD_SEQUENCE_ID) { + this->handlerError(ctx.get(), "DB.Write"); + iprot_->getTransport()->close(); + oprot_->getTransport()->close(); + } + throw; + } +} + +void DBClient::send_Write(const DBHandle& dbhandle, const std::vector & batch, const WriteOptions& options) +{ + apache::thrift::ContextStack* ctx = this->getContextStack(); + this->preWrite(ctx, "DB.Write"); + oprot_->writeMessageBegin("Write", apache::thrift::protocol::T_CALL, getNextSendSequenceId()); + + DB_Write_pargs args; + args.dbhandle = &dbhandle; + args.batch = &batch; + args.options = &options; + args.write(oprot_); + + oprot_->writeMessageEnd(); + uint32_t _bytes32 = oprot_->getTransport()->writeEnd(); + oprot_->getTransport()->flush(); + this->postWrite(ctx, "DB.Write", _bytes32); + return; +} + +Code DBClient::recv_Write() +{ + apache::thrift::ContextStack* ctx = this->getContextStack(); + uint32_t bytes; + int32_t rseqid = 0; + int32_t eseqid = getNextRecvSequenceId(); + std::string fname; + apache::thrift::protocol::TMessageType mtype; + this->preRead(ctx, "DB.Write"); + + iprot_->readMessageBegin(fname, mtype, rseqid); + if (this->checkSeqid_ && rseqid != eseqid) { + iprot_->skip(apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::BAD_SEQUENCE_ID); + } + if (mtype == apache::thrift::protocol::T_EXCEPTION) { + apache::thrift::TApplicationException x; + x.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw x; + } + if (mtype != apache::thrift::protocol::T_REPLY) { + iprot_->skip(apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::INVALID_MESSAGE_TYPE); + } + if (fname.compare("Write") != 0) { + iprot_->skip(apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::WRONG_METHOD_NAME); + } + Code _return; + DB_Write_presult result; + result.success = &_return; + result.read(iprot_); + iprot_->readMessageEnd(); + bytes = iprot_->getTransport()->readEnd(); + this->postRead(ctx, "DB.Write", bytes); + + if (result.__isset.success) { + return _return; + } + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::MISSING_RESULT, "Write failed: unknown result"); +} + +void DBClient::Get(ResultItem& _return, const DBHandle& dbhandle, const Slice& inputkey, const ReadOptions& options) +{ + std::unique_ptr ctx(this->getContextStack("DB.Get", NULL)); + + try { + this->setContextStack(ctx.get()); + send_Get(dbhandle, inputkey, options); + recv_Get(_return); + } catch(apache::thrift::transport::TTransportException& ex) { + this->handlerError(ctx.get(), "DB.Get"); + iprot_->getTransport()->close(); + oprot_->getTransport()->close(); + throw; + } catch(apache::thrift::TApplicationException& ex) { + if (ex.getType() == apache::thrift::TApplicationException::BAD_SEQUENCE_ID) { + this->handlerError(ctx.get(), "DB.Get"); + iprot_->getTransport()->close(); + oprot_->getTransport()->close(); + } + throw; + } +} + +void DBClient::send_Get(const DBHandle& dbhandle, const Slice& inputkey, const ReadOptions& options) +{ + apache::thrift::ContextStack* ctx = this->getContextStack(); + this->preWrite(ctx, "DB.Get"); + oprot_->writeMessageBegin("Get", apache::thrift::protocol::T_CALL, getNextSendSequenceId()); + + DB_Get_pargs args; + args.dbhandle = &dbhandle; + args.inputkey = &inputkey; + args.options = &options; + args.write(oprot_); + + oprot_->writeMessageEnd(); + uint32_t _bytes33 = oprot_->getTransport()->writeEnd(); + oprot_->getTransport()->flush(); + this->postWrite(ctx, "DB.Get", _bytes33); + return; +} + +void DBClient::recv_Get(ResultItem& _return) +{ + apache::thrift::ContextStack* ctx = this->getContextStack(); + uint32_t bytes; + int32_t rseqid = 0; + int32_t eseqid = getNextRecvSequenceId(); + std::string fname; + apache::thrift::protocol::TMessageType mtype; + this->preRead(ctx, "DB.Get"); + + iprot_->readMessageBegin(fname, mtype, rseqid); + if (this->checkSeqid_ && rseqid != eseqid) { + iprot_->skip(apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::BAD_SEQUENCE_ID); + } + if (mtype == apache::thrift::protocol::T_EXCEPTION) { + apache::thrift::TApplicationException x; + x.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw x; + } + if (mtype != apache::thrift::protocol::T_REPLY) { + iprot_->skip(apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::INVALID_MESSAGE_TYPE); + } + if (fname.compare("Get") != 0) { + iprot_->skip(apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::WRONG_METHOD_NAME); + } + DB_Get_presult result; + result.success = &_return; + result.read(iprot_); + iprot_->readMessageEnd(); + bytes = iprot_->getTransport()->readEnd(); + this->postRead(ctx, "DB.Get", bytes); + + if (result.__isset.success) { + // _return pointer has now been filled + return; + } + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::MISSING_RESULT, "Get failed: unknown result"); +} + +void DBClient::NewIterator(ResultIterator& _return, const DBHandle& dbhandle, const ReadOptions& options, IteratorType iteratorType, const Slice& target) +{ + std::unique_ptr ctx(this->getContextStack("DB.NewIterator", NULL)); + + try { + this->setContextStack(ctx.get()); + send_NewIterator(dbhandle, options, iteratorType, target); + recv_NewIterator(_return); + } catch(apache::thrift::transport::TTransportException& ex) { + this->handlerError(ctx.get(), "DB.NewIterator"); + iprot_->getTransport()->close(); + oprot_->getTransport()->close(); + throw; + } catch(apache::thrift::TApplicationException& ex) { + if (ex.getType() == apache::thrift::TApplicationException::BAD_SEQUENCE_ID) { + this->handlerError(ctx.get(), "DB.NewIterator"); + iprot_->getTransport()->close(); + oprot_->getTransport()->close(); + } + throw; + } +} + +void DBClient::send_NewIterator(const DBHandle& dbhandle, const ReadOptions& options, IteratorType iteratorType, const Slice& target) +{ + apache::thrift::ContextStack* ctx = this->getContextStack(); + this->preWrite(ctx, "DB.NewIterator"); + oprot_->writeMessageBegin("NewIterator", apache::thrift::protocol::T_CALL, getNextSendSequenceId()); + + DB_NewIterator_pargs args; + args.dbhandle = &dbhandle; + args.options = &options; + args.iteratorType = &iteratorType; + args.target = ⌖ + args.write(oprot_); + + oprot_->writeMessageEnd(); + uint32_t _bytes34 = oprot_->getTransport()->writeEnd(); + oprot_->getTransport()->flush(); + this->postWrite(ctx, "DB.NewIterator", _bytes34); + return; +} + +void DBClient::recv_NewIterator(ResultIterator& _return) +{ + apache::thrift::ContextStack* ctx = this->getContextStack(); + uint32_t bytes; + int32_t rseqid = 0; + int32_t eseqid = getNextRecvSequenceId(); + std::string fname; + apache::thrift::protocol::TMessageType mtype; + this->preRead(ctx, "DB.NewIterator"); + + iprot_->readMessageBegin(fname, mtype, rseqid); + if (this->checkSeqid_ && rseqid != eseqid) { + iprot_->skip(apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::BAD_SEQUENCE_ID); + } + if (mtype == apache::thrift::protocol::T_EXCEPTION) { + apache::thrift::TApplicationException x; + x.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw x; + } + if (mtype != apache::thrift::protocol::T_REPLY) { + iprot_->skip(apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::INVALID_MESSAGE_TYPE); + } + if (fname.compare("NewIterator") != 0) { + iprot_->skip(apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::WRONG_METHOD_NAME); + } + DB_NewIterator_presult result; + result.success = &_return; + result.read(iprot_); + iprot_->readMessageEnd(); + bytes = iprot_->getTransport()->readEnd(); + this->postRead(ctx, "DB.NewIterator", bytes); + + if (result.__isset.success) { + // _return pointer has now been filled + return; + } + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::MISSING_RESULT, "NewIterator failed: unknown result"); +} + +Code DBClient::DeleteIterator(const DBHandle& dbhandle, const Iterator& iterator) +{ + std::unique_ptr ctx(this->getContextStack("DB.DeleteIterator", NULL)); + + try { + this->setContextStack(ctx.get()); + send_DeleteIterator(dbhandle, iterator); + return recv_DeleteIterator(); + } catch(apache::thrift::transport::TTransportException& ex) { + this->handlerError(ctx.get(), "DB.DeleteIterator"); + iprot_->getTransport()->close(); + oprot_->getTransport()->close(); + throw; + } catch(apache::thrift::TApplicationException& ex) { + if (ex.getType() == apache::thrift::TApplicationException::BAD_SEQUENCE_ID) { + this->handlerError(ctx.get(), "DB.DeleteIterator"); + iprot_->getTransport()->close(); + oprot_->getTransport()->close(); + } + throw; + } +} + +void DBClient::send_DeleteIterator(const DBHandle& dbhandle, const Iterator& iterator) +{ + apache::thrift::ContextStack* ctx = this->getContextStack(); + this->preWrite(ctx, "DB.DeleteIterator"); + oprot_->writeMessageBegin("DeleteIterator", apache::thrift::protocol::T_CALL, getNextSendSequenceId()); + + DB_DeleteIterator_pargs args; + args.dbhandle = &dbhandle; + args.iterator = &iterator; + args.write(oprot_); + + oprot_->writeMessageEnd(); + uint32_t _bytes35 = oprot_->getTransport()->writeEnd(); + oprot_->getTransport()->flush(); + this->postWrite(ctx, "DB.DeleteIterator", _bytes35); + return; +} + +Code DBClient::recv_DeleteIterator() +{ + apache::thrift::ContextStack* ctx = this->getContextStack(); + uint32_t bytes; + int32_t rseqid = 0; + int32_t eseqid = getNextRecvSequenceId(); + std::string fname; + apache::thrift::protocol::TMessageType mtype; + this->preRead(ctx, "DB.DeleteIterator"); + + iprot_->readMessageBegin(fname, mtype, rseqid); + if (this->checkSeqid_ && rseqid != eseqid) { + iprot_->skip(apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::BAD_SEQUENCE_ID); + } + if (mtype == apache::thrift::protocol::T_EXCEPTION) { + apache::thrift::TApplicationException x; + x.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw x; + } + if (mtype != apache::thrift::protocol::T_REPLY) { + iprot_->skip(apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::INVALID_MESSAGE_TYPE); + } + if (fname.compare("DeleteIterator") != 0) { + iprot_->skip(apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::WRONG_METHOD_NAME); + } + Code _return; + DB_DeleteIterator_presult result; + result.success = &_return; + result.read(iprot_); + iprot_->readMessageEnd(); + bytes = iprot_->getTransport()->readEnd(); + this->postRead(ctx, "DB.DeleteIterator", bytes); + + if (result.__isset.success) { + return _return; + } + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::MISSING_RESULT, "DeleteIterator failed: unknown result"); +} + +void DBClient::GetNext(ResultPair& _return, const DBHandle& dbhandle, const Iterator& iterator) +{ + std::unique_ptr ctx(this->getContextStack("DB.GetNext", NULL)); + + try { + this->setContextStack(ctx.get()); + send_GetNext(dbhandle, iterator); + recv_GetNext(_return); + } catch(apache::thrift::transport::TTransportException& ex) { + this->handlerError(ctx.get(), "DB.GetNext"); + iprot_->getTransport()->close(); + oprot_->getTransport()->close(); + throw; + } catch(apache::thrift::TApplicationException& ex) { + if (ex.getType() == apache::thrift::TApplicationException::BAD_SEQUENCE_ID) { + this->handlerError(ctx.get(), "DB.GetNext"); + iprot_->getTransport()->close(); + oprot_->getTransport()->close(); + } + throw; + } +} + +void DBClient::send_GetNext(const DBHandle& dbhandle, const Iterator& iterator) +{ + apache::thrift::ContextStack* ctx = this->getContextStack(); + this->preWrite(ctx, "DB.GetNext"); + oprot_->writeMessageBegin("GetNext", apache::thrift::protocol::T_CALL, getNextSendSequenceId()); + + DB_GetNext_pargs args; + args.dbhandle = &dbhandle; + args.iterator = &iterator; + args.write(oprot_); + + oprot_->writeMessageEnd(); + uint32_t _bytes36 = oprot_->getTransport()->writeEnd(); + oprot_->getTransport()->flush(); + this->postWrite(ctx, "DB.GetNext", _bytes36); + return; +} + +void DBClient::recv_GetNext(ResultPair& _return) +{ + apache::thrift::ContextStack* ctx = this->getContextStack(); + uint32_t bytes; + int32_t rseqid = 0; + int32_t eseqid = getNextRecvSequenceId(); + std::string fname; + apache::thrift::protocol::TMessageType mtype; + this->preRead(ctx, "DB.GetNext"); + + iprot_->readMessageBegin(fname, mtype, rseqid); + if (this->checkSeqid_ && rseqid != eseqid) { + iprot_->skip(apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::BAD_SEQUENCE_ID); + } + if (mtype == apache::thrift::protocol::T_EXCEPTION) { + apache::thrift::TApplicationException x; + x.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw x; + } + if (mtype != apache::thrift::protocol::T_REPLY) { + iprot_->skip(apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::INVALID_MESSAGE_TYPE); + } + if (fname.compare("GetNext") != 0) { + iprot_->skip(apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::WRONG_METHOD_NAME); + } + DB_GetNext_presult result; + result.success = &_return; + result.read(iprot_); + iprot_->readMessageEnd(); + bytes = iprot_->getTransport()->readEnd(); + this->postRead(ctx, "DB.GetNext", bytes); + + if (result.__isset.success) { + // _return pointer has now been filled + return; + } + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::MISSING_RESULT, "GetNext failed: unknown result"); +} + +void DBClient::GetPrev(ResultPair& _return, const DBHandle& dbhandle, const Iterator& iterator) +{ + std::unique_ptr ctx(this->getContextStack("DB.GetPrev", NULL)); + + try { + this->setContextStack(ctx.get()); + send_GetPrev(dbhandle, iterator); + recv_GetPrev(_return); + } catch(apache::thrift::transport::TTransportException& ex) { + this->handlerError(ctx.get(), "DB.GetPrev"); + iprot_->getTransport()->close(); + oprot_->getTransport()->close(); + throw; + } catch(apache::thrift::TApplicationException& ex) { + if (ex.getType() == apache::thrift::TApplicationException::BAD_SEQUENCE_ID) { + this->handlerError(ctx.get(), "DB.GetPrev"); + iprot_->getTransport()->close(); + oprot_->getTransport()->close(); + } + throw; + } +} + +void DBClient::send_GetPrev(const DBHandle& dbhandle, const Iterator& iterator) +{ + apache::thrift::ContextStack* ctx = this->getContextStack(); + this->preWrite(ctx, "DB.GetPrev"); + oprot_->writeMessageBegin("GetPrev", apache::thrift::protocol::T_CALL, getNextSendSequenceId()); + + DB_GetPrev_pargs args; + args.dbhandle = &dbhandle; + args.iterator = &iterator; + args.write(oprot_); + + oprot_->writeMessageEnd(); + uint32_t _bytes37 = oprot_->getTransport()->writeEnd(); + oprot_->getTransport()->flush(); + this->postWrite(ctx, "DB.GetPrev", _bytes37); + return; +} + +void DBClient::recv_GetPrev(ResultPair& _return) +{ + apache::thrift::ContextStack* ctx = this->getContextStack(); + uint32_t bytes; + int32_t rseqid = 0; + int32_t eseqid = getNextRecvSequenceId(); + std::string fname; + apache::thrift::protocol::TMessageType mtype; + this->preRead(ctx, "DB.GetPrev"); + + iprot_->readMessageBegin(fname, mtype, rseqid); + if (this->checkSeqid_ && rseqid != eseqid) { + iprot_->skip(apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::BAD_SEQUENCE_ID); + } + if (mtype == apache::thrift::protocol::T_EXCEPTION) { + apache::thrift::TApplicationException x; + x.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw x; + } + if (mtype != apache::thrift::protocol::T_REPLY) { + iprot_->skip(apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::INVALID_MESSAGE_TYPE); + } + if (fname.compare("GetPrev") != 0) { + iprot_->skip(apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::WRONG_METHOD_NAME); + } + DB_GetPrev_presult result; + result.success = &_return; + result.read(iprot_); + iprot_->readMessageEnd(); + bytes = iprot_->getTransport()->readEnd(); + this->postRead(ctx, "DB.GetPrev", bytes); + + if (result.__isset.success) { + // _return pointer has now been filled + return; + } + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::MISSING_RESULT, "GetPrev failed: unknown result"); +} + +void DBClient::GetSnapshot(ResultSnapshot& _return, const DBHandle& dbhandle) +{ + std::unique_ptr ctx(this->getContextStack("DB.GetSnapshot", NULL)); + + try { + this->setContextStack(ctx.get()); + send_GetSnapshot(dbhandle); + recv_GetSnapshot(_return); + } catch(apache::thrift::transport::TTransportException& ex) { + this->handlerError(ctx.get(), "DB.GetSnapshot"); + iprot_->getTransport()->close(); + oprot_->getTransport()->close(); + throw; + } catch(apache::thrift::TApplicationException& ex) { + if (ex.getType() == apache::thrift::TApplicationException::BAD_SEQUENCE_ID) { + this->handlerError(ctx.get(), "DB.GetSnapshot"); + iprot_->getTransport()->close(); + oprot_->getTransport()->close(); + } + throw; + } +} + +void DBClient::send_GetSnapshot(const DBHandle& dbhandle) +{ + apache::thrift::ContextStack* ctx = this->getContextStack(); + this->preWrite(ctx, "DB.GetSnapshot"); + oprot_->writeMessageBegin("GetSnapshot", apache::thrift::protocol::T_CALL, getNextSendSequenceId()); + + DB_GetSnapshot_pargs args; + args.dbhandle = &dbhandle; + args.write(oprot_); + + oprot_->writeMessageEnd(); + uint32_t _bytes38 = oprot_->getTransport()->writeEnd(); + oprot_->getTransport()->flush(); + this->postWrite(ctx, "DB.GetSnapshot", _bytes38); + return; +} + +void DBClient::recv_GetSnapshot(ResultSnapshot& _return) +{ + apache::thrift::ContextStack* ctx = this->getContextStack(); + uint32_t bytes; + int32_t rseqid = 0; + int32_t eseqid = getNextRecvSequenceId(); + std::string fname; + apache::thrift::protocol::TMessageType mtype; + this->preRead(ctx, "DB.GetSnapshot"); + + iprot_->readMessageBegin(fname, mtype, rseqid); + if (this->checkSeqid_ && rseqid != eseqid) { + iprot_->skip(apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::BAD_SEQUENCE_ID); + } + if (mtype == apache::thrift::protocol::T_EXCEPTION) { + apache::thrift::TApplicationException x; + x.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw x; + } + if (mtype != apache::thrift::protocol::T_REPLY) { + iprot_->skip(apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::INVALID_MESSAGE_TYPE); + } + if (fname.compare("GetSnapshot") != 0) { + iprot_->skip(apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::WRONG_METHOD_NAME); + } + DB_GetSnapshot_presult result; + result.success = &_return; + result.read(iprot_); + iprot_->readMessageEnd(); + bytes = iprot_->getTransport()->readEnd(); + this->postRead(ctx, "DB.GetSnapshot", bytes); + + if (result.__isset.success) { + // _return pointer has now been filled + return; + } + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::MISSING_RESULT, "GetSnapshot failed: unknown result"); +} + +Code DBClient::ReleaseSnapshot(const DBHandle& dbhandle, const Snapshot& snapshot) +{ + std::unique_ptr ctx(this->getContextStack("DB.ReleaseSnapshot", NULL)); + + try { + this->setContextStack(ctx.get()); + send_ReleaseSnapshot(dbhandle, snapshot); + return recv_ReleaseSnapshot(); + } catch(apache::thrift::transport::TTransportException& ex) { + this->handlerError(ctx.get(), "DB.ReleaseSnapshot"); + iprot_->getTransport()->close(); + oprot_->getTransport()->close(); + throw; + } catch(apache::thrift::TApplicationException& ex) { + if (ex.getType() == apache::thrift::TApplicationException::BAD_SEQUENCE_ID) { + this->handlerError(ctx.get(), "DB.ReleaseSnapshot"); + iprot_->getTransport()->close(); + oprot_->getTransport()->close(); + } + throw; + } +} + +void DBClient::send_ReleaseSnapshot(const DBHandle& dbhandle, const Snapshot& snapshot) +{ + apache::thrift::ContextStack* ctx = this->getContextStack(); + this->preWrite(ctx, "DB.ReleaseSnapshot"); + oprot_->writeMessageBegin("ReleaseSnapshot", apache::thrift::protocol::T_CALL, getNextSendSequenceId()); + + DB_ReleaseSnapshot_pargs args; + args.dbhandle = &dbhandle; + args.snapshot = &snapshot; + args.write(oprot_); + + oprot_->writeMessageEnd(); + uint32_t _bytes39 = oprot_->getTransport()->writeEnd(); + oprot_->getTransport()->flush(); + this->postWrite(ctx, "DB.ReleaseSnapshot", _bytes39); + return; +} + +Code DBClient::recv_ReleaseSnapshot() +{ + apache::thrift::ContextStack* ctx = this->getContextStack(); + uint32_t bytes; + int32_t rseqid = 0; + int32_t eseqid = getNextRecvSequenceId(); + std::string fname; + apache::thrift::protocol::TMessageType mtype; + this->preRead(ctx, "DB.ReleaseSnapshot"); + + iprot_->readMessageBegin(fname, mtype, rseqid); + if (this->checkSeqid_ && rseqid != eseqid) { + iprot_->skip(apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::BAD_SEQUENCE_ID); + } + if (mtype == apache::thrift::protocol::T_EXCEPTION) { + apache::thrift::TApplicationException x; + x.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw x; + } + if (mtype != apache::thrift::protocol::T_REPLY) { + iprot_->skip(apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::INVALID_MESSAGE_TYPE); + } + if (fname.compare("ReleaseSnapshot") != 0) { + iprot_->skip(apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::WRONG_METHOD_NAME); + } + Code _return; + DB_ReleaseSnapshot_presult result; + result.success = &_return; + result.read(iprot_); + iprot_->readMessageEnd(); + bytes = iprot_->getTransport()->readEnd(); + this->postRead(ctx, "DB.ReleaseSnapshot", bytes); + + if (result.__isset.success) { + return _return; + } + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::MISSING_RESULT, "ReleaseSnapshot failed: unknown result"); +} + +Code DBClient::CompactRange(const DBHandle& dbhandle, const Slice& begin, const Slice& end) +{ + std::unique_ptr ctx(this->getContextStack("DB.CompactRange", NULL)); + + try { + this->setContextStack(ctx.get()); + send_CompactRange(dbhandle, begin, end); + return recv_CompactRange(); + } catch(apache::thrift::transport::TTransportException& ex) { + this->handlerError(ctx.get(), "DB.CompactRange"); + iprot_->getTransport()->close(); + oprot_->getTransport()->close(); + throw; + } catch(apache::thrift::TApplicationException& ex) { + if (ex.getType() == apache::thrift::TApplicationException::BAD_SEQUENCE_ID) { + this->handlerError(ctx.get(), "DB.CompactRange"); + iprot_->getTransport()->close(); + oprot_->getTransport()->close(); + } + throw; + } +} + +void DBClient::send_CompactRange(const DBHandle& dbhandle, const Slice& begin, const Slice& end) +{ + apache::thrift::ContextStack* ctx = this->getContextStack(); + this->preWrite(ctx, "DB.CompactRange"); + oprot_->writeMessageBegin("CompactRange", apache::thrift::protocol::T_CALL, getNextSendSequenceId()); + + DB_CompactRange_pargs args; + args.dbhandle = &dbhandle; + args.begin = &begin; + args.end = &end; + args.write(oprot_); + + oprot_->writeMessageEnd(); + uint32_t _bytes40 = oprot_->getTransport()->writeEnd(); + oprot_->getTransport()->flush(); + this->postWrite(ctx, "DB.CompactRange", _bytes40); + return; +} + +Code DBClient::recv_CompactRange() +{ + apache::thrift::ContextStack* ctx = this->getContextStack(); + uint32_t bytes; + int32_t rseqid = 0; + int32_t eseqid = getNextRecvSequenceId(); + std::string fname; + apache::thrift::protocol::TMessageType mtype; + this->preRead(ctx, "DB.CompactRange"); + + iprot_->readMessageBegin(fname, mtype, rseqid); + if (this->checkSeqid_ && rseqid != eseqid) { + iprot_->skip(apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::BAD_SEQUENCE_ID); + } + if (mtype == apache::thrift::protocol::T_EXCEPTION) { + apache::thrift::TApplicationException x; + x.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw x; + } + if (mtype != apache::thrift::protocol::T_REPLY) { + iprot_->skip(apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::INVALID_MESSAGE_TYPE); + } + if (fname.compare("CompactRange") != 0) { + iprot_->skip(apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::WRONG_METHOD_NAME); + } + Code _return; + DB_CompactRange_presult result; + result.success = &_return; + result.read(iprot_); + iprot_->readMessageEnd(); + bytes = iprot_->getTransport()->readEnd(); + this->postRead(ctx, "DB.CompactRange", bytes); + + if (result.__isset.success) { + return _return; + } + throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::MISSING_RESULT, "CompactRange failed: unknown result"); +} + +bool DBProcessor::dispatchCall(::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, const std::string& fname, int32_t seqid, apache::thrift::server::TConnectionContext* connectionContext) { + ProcessMap::iterator pfn; + pfn = processMap_.find(fname); + if (pfn == processMap_.end()) { + iprot->skip(apache::thrift::protocol::T_STRUCT); + iprot->readMessageEnd(); + iprot->getTransport()->readEnd(); + apache::thrift::TApplicationException x(apache::thrift::TApplicationException::UNKNOWN_METHOD, "Invalid method name: '"+fname+"'"); + oprot->writeMessageBegin(fname, apache::thrift::protocol::T_EXCEPTION, seqid); + x.write(oprot); + oprot->writeMessageEnd(); + oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + return true; + } + const ProcessFunction& pf = pfn->second; + (this->*pf)(seqid, iprot, oprot, connectionContext); + return true; +} + +void DBProcessor::process_Open(int32_t seqid, apache::thrift::protocol::TProtocol* iprot, apache::thrift::protocol::TProtocol* oprot, apache::thrift::server::TConnectionContext* connectionContext) +{ + std::unique_ptr ctx(this->getContextStack("DB.Open", connectionContext)); + + this->preRead(ctx.get(), "DB.Open"); + DB_Open_args args; + args.read(iprot); + iprot->readMessageEnd(); + uint32_t bytes = iprot->getTransport()->readEnd(); + + this->postRead(ctx.get(), "DB.Open", bytes); + + DB_Open_result result; + try { + iface_->Open(result.success, args.dbname, args.dboptions); + result.__isset.success = true; + } catch (LeveldbException &se) { + result.se = se; + result.__isset.se = true; + } catch (const std::exception& e) { + this->handlerError(ctx.get(), "DB.Open"); + + + apache::thrift::TApplicationException x(e.what()); + oprot->writeMessageBegin("Open", apache::thrift::protocol::T_EXCEPTION, seqid); + x.write(oprot); + oprot->writeMessageEnd(); + oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + return; + } + + this->preWrite(ctx.get(), "DB.Open"); + oprot->writeMessageBegin("Open", apache::thrift::protocol::T_REPLY, seqid); + result.write(oprot); + oprot->writeMessageEnd(); + bytes = oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + + this->postWrite(ctx.get(), "DB.Open", bytes); + +} + +void DBProcessor::process_Close(int32_t seqid, apache::thrift::protocol::TProtocol* iprot, apache::thrift::protocol::TProtocol* oprot, apache::thrift::server::TConnectionContext* connectionContext) +{ + std::unique_ptr ctx(this->getContextStack("DB.Close", connectionContext)); + + this->preRead(ctx.get(), "DB.Close"); + DB_Close_args args; + args.read(iprot); + iprot->readMessageEnd(); + uint32_t bytes = iprot->getTransport()->readEnd(); + + this->postRead(ctx.get(), "DB.Close", bytes); + + DB_Close_result result; + try { + result.success = iface_->Close(args.dbhandle, args.dbname); + result.__isset.success = true; + } catch (const std::exception& e) { + this->handlerError(ctx.get(), "DB.Close"); + + + apache::thrift::TApplicationException x(e.what()); + oprot->writeMessageBegin("Close", apache::thrift::protocol::T_EXCEPTION, seqid); + x.write(oprot); + oprot->writeMessageEnd(); + oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + return; + } + + this->preWrite(ctx.get(), "DB.Close"); + oprot->writeMessageBegin("Close", apache::thrift::protocol::T_REPLY, seqid); + result.write(oprot); + oprot->writeMessageEnd(); + bytes = oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + + this->postWrite(ctx.get(), "DB.Close", bytes); + +} + +void DBProcessor::process_Put(int32_t seqid, apache::thrift::protocol::TProtocol* iprot, apache::thrift::protocol::TProtocol* oprot, apache::thrift::server::TConnectionContext* connectionContext) +{ + std::unique_ptr ctx(this->getContextStack("DB.Put", connectionContext)); + + this->preRead(ctx.get(), "DB.Put"); + DB_Put_args args; + args.read(iprot); + iprot->readMessageEnd(); + uint32_t bytes = iprot->getTransport()->readEnd(); + + this->postRead(ctx.get(), "DB.Put", bytes); + + DB_Put_result result; + try { + result.success = iface_->Put(args.dbhandle, args.keyvalue, args.options); + result.__isset.success = true; + } catch (const std::exception& e) { + this->handlerError(ctx.get(), "DB.Put"); + + + apache::thrift::TApplicationException x(e.what()); + oprot->writeMessageBegin("Put", apache::thrift::protocol::T_EXCEPTION, seqid); + x.write(oprot); + oprot->writeMessageEnd(); + oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + return; + } + + this->preWrite(ctx.get(), "DB.Put"); + oprot->writeMessageBegin("Put", apache::thrift::protocol::T_REPLY, seqid); + result.write(oprot); + oprot->writeMessageEnd(); + bytes = oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + + this->postWrite(ctx.get(), "DB.Put", bytes); + +} + +void DBProcessor::process_Delete(int32_t seqid, apache::thrift::protocol::TProtocol* iprot, apache::thrift::protocol::TProtocol* oprot, apache::thrift::server::TConnectionContext* connectionContext) +{ + std::unique_ptr ctx(this->getContextStack("DB.Delete", connectionContext)); + + this->preRead(ctx.get(), "DB.Delete"); + DB_Delete_args args; + args.read(iprot); + iprot->readMessageEnd(); + uint32_t bytes = iprot->getTransport()->readEnd(); + + this->postRead(ctx.get(), "DB.Delete", bytes); + + DB_Delete_result result; + try { + result.success = iface_->Delete(args.dbhandle, args.key, args.options); + result.__isset.success = true; + } catch (const std::exception& e) { + this->handlerError(ctx.get(), "DB.Delete"); + + + apache::thrift::TApplicationException x(e.what()); + oprot->writeMessageBegin("Delete", apache::thrift::protocol::T_EXCEPTION, seqid); + x.write(oprot); + oprot->writeMessageEnd(); + oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + return; + } + + this->preWrite(ctx.get(), "DB.Delete"); + oprot->writeMessageBegin("Delete", apache::thrift::protocol::T_REPLY, seqid); + result.write(oprot); + oprot->writeMessageEnd(); + bytes = oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + + this->postWrite(ctx.get(), "DB.Delete", bytes); + +} + +void DBProcessor::process_Write(int32_t seqid, apache::thrift::protocol::TProtocol* iprot, apache::thrift::protocol::TProtocol* oprot, apache::thrift::server::TConnectionContext* connectionContext) +{ + std::unique_ptr ctx(this->getContextStack("DB.Write", connectionContext)); + + this->preRead(ctx.get(), "DB.Write"); + DB_Write_args args; + args.read(iprot); + iprot->readMessageEnd(); + uint32_t bytes = iprot->getTransport()->readEnd(); + + this->postRead(ctx.get(), "DB.Write", bytes); + + DB_Write_result result; + try { + result.success = iface_->Write(args.dbhandle, args.batch, args.options); + result.__isset.success = true; + } catch (const std::exception& e) { + this->handlerError(ctx.get(), "DB.Write"); + + + apache::thrift::TApplicationException x(e.what()); + oprot->writeMessageBegin("Write", apache::thrift::protocol::T_EXCEPTION, seqid); + x.write(oprot); + oprot->writeMessageEnd(); + oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + return; + } + + this->preWrite(ctx.get(), "DB.Write"); + oprot->writeMessageBegin("Write", apache::thrift::protocol::T_REPLY, seqid); + result.write(oprot); + oprot->writeMessageEnd(); + bytes = oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + + this->postWrite(ctx.get(), "DB.Write", bytes); + +} + +void DBProcessor::process_Get(int32_t seqid, apache::thrift::protocol::TProtocol* iprot, apache::thrift::protocol::TProtocol* oprot, apache::thrift::server::TConnectionContext* connectionContext) +{ + std::unique_ptr ctx(this->getContextStack("DB.Get", connectionContext)); + + this->preRead(ctx.get(), "DB.Get"); + DB_Get_args args; + args.read(iprot); + iprot->readMessageEnd(); + uint32_t bytes = iprot->getTransport()->readEnd(); + + this->postRead(ctx.get(), "DB.Get", bytes); + + DB_Get_result result; + try { + iface_->Get(result.success, args.dbhandle, args.inputkey, args.options); + result.__isset.success = true; + } catch (const std::exception& e) { + this->handlerError(ctx.get(), "DB.Get"); + + + apache::thrift::TApplicationException x(e.what()); + oprot->writeMessageBegin("Get", apache::thrift::protocol::T_EXCEPTION, seqid); + x.write(oprot); + oprot->writeMessageEnd(); + oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + return; + } + + this->preWrite(ctx.get(), "DB.Get"); + oprot->writeMessageBegin("Get", apache::thrift::protocol::T_REPLY, seqid); + result.write(oprot); + oprot->writeMessageEnd(); + bytes = oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + + this->postWrite(ctx.get(), "DB.Get", bytes); + +} + +void DBProcessor::process_NewIterator(int32_t seqid, apache::thrift::protocol::TProtocol* iprot, apache::thrift::protocol::TProtocol* oprot, apache::thrift::server::TConnectionContext* connectionContext) +{ + std::unique_ptr ctx(this->getContextStack("DB.NewIterator", connectionContext)); + + this->preRead(ctx.get(), "DB.NewIterator"); + DB_NewIterator_args args; + args.read(iprot); + iprot->readMessageEnd(); + uint32_t bytes = iprot->getTransport()->readEnd(); + + this->postRead(ctx.get(), "DB.NewIterator", bytes); + + DB_NewIterator_result result; + try { + iface_->NewIterator(result.success, args.dbhandle, args.options, args.iteratorType, args.target); + result.__isset.success = true; + } catch (const std::exception& e) { + this->handlerError(ctx.get(), "DB.NewIterator"); + + + apache::thrift::TApplicationException x(e.what()); + oprot->writeMessageBegin("NewIterator", apache::thrift::protocol::T_EXCEPTION, seqid); + x.write(oprot); + oprot->writeMessageEnd(); + oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + return; + } + + this->preWrite(ctx.get(), "DB.NewIterator"); + oprot->writeMessageBegin("NewIterator", apache::thrift::protocol::T_REPLY, seqid); + result.write(oprot); + oprot->writeMessageEnd(); + bytes = oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + + this->postWrite(ctx.get(), "DB.NewIterator", bytes); + +} + +void DBProcessor::process_DeleteIterator(int32_t seqid, apache::thrift::protocol::TProtocol* iprot, apache::thrift::protocol::TProtocol* oprot, apache::thrift::server::TConnectionContext* connectionContext) +{ + std::unique_ptr ctx(this->getContextStack("DB.DeleteIterator", connectionContext)); + + this->preRead(ctx.get(), "DB.DeleteIterator"); + DB_DeleteIterator_args args; + args.read(iprot); + iprot->readMessageEnd(); + uint32_t bytes = iprot->getTransport()->readEnd(); + + this->postRead(ctx.get(), "DB.DeleteIterator", bytes); + + DB_DeleteIterator_result result; + try { + result.success = iface_->DeleteIterator(args.dbhandle, args.iterator); + result.__isset.success = true; + } catch (const std::exception& e) { + this->handlerError(ctx.get(), "DB.DeleteIterator"); + + + apache::thrift::TApplicationException x(e.what()); + oprot->writeMessageBegin("DeleteIterator", apache::thrift::protocol::T_EXCEPTION, seqid); + x.write(oprot); + oprot->writeMessageEnd(); + oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + return; + } + + this->preWrite(ctx.get(), "DB.DeleteIterator"); + oprot->writeMessageBegin("DeleteIterator", apache::thrift::protocol::T_REPLY, seqid); + result.write(oprot); + oprot->writeMessageEnd(); + bytes = oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + + this->postWrite(ctx.get(), "DB.DeleteIterator", bytes); + +} + +void DBProcessor::process_GetNext(int32_t seqid, apache::thrift::protocol::TProtocol* iprot, apache::thrift::protocol::TProtocol* oprot, apache::thrift::server::TConnectionContext* connectionContext) +{ + std::unique_ptr ctx(this->getContextStack("DB.GetNext", connectionContext)); + + this->preRead(ctx.get(), "DB.GetNext"); + DB_GetNext_args args; + args.read(iprot); + iprot->readMessageEnd(); + uint32_t bytes = iprot->getTransport()->readEnd(); + + this->postRead(ctx.get(), "DB.GetNext", bytes); + + DB_GetNext_result result; + try { + iface_->GetNext(result.success, args.dbhandle, args.iterator); + result.__isset.success = true; + } catch (const std::exception& e) { + this->handlerError(ctx.get(), "DB.GetNext"); + + + apache::thrift::TApplicationException x(e.what()); + oprot->writeMessageBegin("GetNext", apache::thrift::protocol::T_EXCEPTION, seqid); + x.write(oprot); + oprot->writeMessageEnd(); + oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + return; + } + + this->preWrite(ctx.get(), "DB.GetNext"); + oprot->writeMessageBegin("GetNext", apache::thrift::protocol::T_REPLY, seqid); + result.write(oprot); + oprot->writeMessageEnd(); + bytes = oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + + this->postWrite(ctx.get(), "DB.GetNext", bytes); + +} + +void DBProcessor::process_GetPrev(int32_t seqid, apache::thrift::protocol::TProtocol* iprot, apache::thrift::protocol::TProtocol* oprot, apache::thrift::server::TConnectionContext* connectionContext) +{ + std::unique_ptr ctx(this->getContextStack("DB.GetPrev", connectionContext)); + + this->preRead(ctx.get(), "DB.GetPrev"); + DB_GetPrev_args args; + args.read(iprot); + iprot->readMessageEnd(); + uint32_t bytes = iprot->getTransport()->readEnd(); + + this->postRead(ctx.get(), "DB.GetPrev", bytes); + + DB_GetPrev_result result; + try { + iface_->GetPrev(result.success, args.dbhandle, args.iterator); + result.__isset.success = true; + } catch (const std::exception& e) { + this->handlerError(ctx.get(), "DB.GetPrev"); + + + apache::thrift::TApplicationException x(e.what()); + oprot->writeMessageBegin("GetPrev", apache::thrift::protocol::T_EXCEPTION, seqid); + x.write(oprot); + oprot->writeMessageEnd(); + oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + return; + } + + this->preWrite(ctx.get(), "DB.GetPrev"); + oprot->writeMessageBegin("GetPrev", apache::thrift::protocol::T_REPLY, seqid); + result.write(oprot); + oprot->writeMessageEnd(); + bytes = oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + + this->postWrite(ctx.get(), "DB.GetPrev", bytes); + +} + +void DBProcessor::process_GetSnapshot(int32_t seqid, apache::thrift::protocol::TProtocol* iprot, apache::thrift::protocol::TProtocol* oprot, apache::thrift::server::TConnectionContext* connectionContext) +{ + std::unique_ptr ctx(this->getContextStack("DB.GetSnapshot", connectionContext)); + + this->preRead(ctx.get(), "DB.GetSnapshot"); + DB_GetSnapshot_args args; + args.read(iprot); + iprot->readMessageEnd(); + uint32_t bytes = iprot->getTransport()->readEnd(); + + this->postRead(ctx.get(), "DB.GetSnapshot", bytes); + + DB_GetSnapshot_result result; + try { + iface_->GetSnapshot(result.success, args.dbhandle); + result.__isset.success = true; + } catch (const std::exception& e) { + this->handlerError(ctx.get(), "DB.GetSnapshot"); + + + apache::thrift::TApplicationException x(e.what()); + oprot->writeMessageBegin("GetSnapshot", apache::thrift::protocol::T_EXCEPTION, seqid); + x.write(oprot); + oprot->writeMessageEnd(); + oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + return; + } + + this->preWrite(ctx.get(), "DB.GetSnapshot"); + oprot->writeMessageBegin("GetSnapshot", apache::thrift::protocol::T_REPLY, seqid); + result.write(oprot); + oprot->writeMessageEnd(); + bytes = oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + + this->postWrite(ctx.get(), "DB.GetSnapshot", bytes); + +} + +void DBProcessor::process_ReleaseSnapshot(int32_t seqid, apache::thrift::protocol::TProtocol* iprot, apache::thrift::protocol::TProtocol* oprot, apache::thrift::server::TConnectionContext* connectionContext) +{ + std::unique_ptr ctx(this->getContextStack("DB.ReleaseSnapshot", connectionContext)); + + this->preRead(ctx.get(), "DB.ReleaseSnapshot"); + DB_ReleaseSnapshot_args args; + args.read(iprot); + iprot->readMessageEnd(); + uint32_t bytes = iprot->getTransport()->readEnd(); + + this->postRead(ctx.get(), "DB.ReleaseSnapshot", bytes); + + DB_ReleaseSnapshot_result result; + try { + result.success = iface_->ReleaseSnapshot(args.dbhandle, args.snapshot); + result.__isset.success = true; + } catch (const std::exception& e) { + this->handlerError(ctx.get(), "DB.ReleaseSnapshot"); + + + apache::thrift::TApplicationException x(e.what()); + oprot->writeMessageBegin("ReleaseSnapshot", apache::thrift::protocol::T_EXCEPTION, seqid); + x.write(oprot); + oprot->writeMessageEnd(); + oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + return; + } + + this->preWrite(ctx.get(), "DB.ReleaseSnapshot"); + oprot->writeMessageBegin("ReleaseSnapshot", apache::thrift::protocol::T_REPLY, seqid); + result.write(oprot); + oprot->writeMessageEnd(); + bytes = oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + + this->postWrite(ctx.get(), "DB.ReleaseSnapshot", bytes); + +} + +void DBProcessor::process_CompactRange(int32_t seqid, apache::thrift::protocol::TProtocol* iprot, apache::thrift::protocol::TProtocol* oprot, apache::thrift::server::TConnectionContext* connectionContext) +{ + std::unique_ptr ctx(this->getContextStack("DB.CompactRange", connectionContext)); + + this->preRead(ctx.get(), "DB.CompactRange"); + DB_CompactRange_args args; + args.read(iprot); + iprot->readMessageEnd(); + uint32_t bytes = iprot->getTransport()->readEnd(); + + this->postRead(ctx.get(), "DB.CompactRange", bytes); + + DB_CompactRange_result result; + try { + result.success = iface_->CompactRange(args.dbhandle, args.begin, args.end); + result.__isset.success = true; + } catch (const std::exception& e) { + this->handlerError(ctx.get(), "DB.CompactRange"); + + + apache::thrift::TApplicationException x(e.what()); + oprot->writeMessageBegin("CompactRange", apache::thrift::protocol::T_EXCEPTION, seqid); + x.write(oprot); + oprot->writeMessageEnd(); + oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + return; + } + + this->preWrite(ctx.get(), "DB.CompactRange"); + oprot->writeMessageBegin("CompactRange", apache::thrift::protocol::T_REPLY, seqid); + result.write(oprot); + oprot->writeMessageEnd(); + bytes = oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + + this->postWrite(ctx.get(), "DB.CompactRange", bytes); + +} + +::boost::shared_ptr< ::apache::thrift::TProcessor > DBProcessorFactory::getProcessor(::apache::thrift::server::TConnectionContext* ctx) { + ::apache::thrift::ReleaseHandler< DBIfFactory > cleanup(handlerFactory_); + ::boost::shared_ptr< DBIf > handler(handlerFactory_->getHandler(ctx), cleanup); + ::boost::shared_ptr< ::apache::thrift::TProcessor > processor(new DBProcessor(handler)); + return processor; +} +} // namespace + diff --git a/thrift/gen-cpp/DB.h b/thrift/gen-cpp/DB.h new file mode 100644 index 0000000000..7e47be9555 --- /dev/null +++ b/thrift/gen-cpp/DB.h @@ -0,0 +1,2225 @@ +/** + * Autogenerated by Thrift + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +#ifndef _Tleveldb_DB_H +#define _Tleveldb_DB_H + +#include +#include "leveldb_types.h" + +namespace Tleveldb { + +class DBIf { + public: + virtual ~DBIf() {} + virtual void Open(DBHandle& _return, const Text& dbname, const DBOptions& dboptions) = 0; + virtual Code Close(const DBHandle& dbhandle, const Text& dbname) = 0; + virtual Code Put(const DBHandle& dbhandle, const kv& keyvalue, const WriteOptions& options) = 0; + virtual Code Delete(const DBHandle& dbhandle, const Slice& key, const WriteOptions& options) = 0; + virtual Code Write(const DBHandle& dbhandle, const std::vector & batch, const WriteOptions& options) = 0; + virtual void Get(ResultItem& _return, const DBHandle& dbhandle, const Slice& inputkey, const ReadOptions& options) = 0; + virtual void NewIterator(ResultIterator& _return, const DBHandle& dbhandle, const ReadOptions& options, IteratorType iteratorType, const Slice& target) = 0; + virtual Code DeleteIterator(const DBHandle& dbhandle, const Iterator& iterator) = 0; + virtual void GetNext(ResultPair& _return, const DBHandle& dbhandle, const Iterator& iterator) = 0; + virtual void GetPrev(ResultPair& _return, const DBHandle& dbhandle, const Iterator& iterator) = 0; + virtual void GetSnapshot(ResultSnapshot& _return, const DBHandle& dbhandle) = 0; + virtual Code ReleaseSnapshot(const DBHandle& dbhandle, const Snapshot& snapshot) = 0; + virtual Code CompactRange(const DBHandle& dbhandle, const Slice& begin, const Slice& end) = 0; +}; + +class DBIfFactory { + public: + typedef DBIf Handler; + + virtual ~DBIfFactory() {} + + virtual DBIf* getHandler(::apache::thrift::server::TConnectionContext* ctx) = 0; + virtual void releaseHandler(DBIf* handler) = 0; +}; + +class DBIfSingletonFactory : virtual public DBIfFactory { + public: + DBIfSingletonFactory(const boost::shared_ptr& iface) : iface_(iface) {} + virtual ~DBIfSingletonFactory() {} + + virtual DBIf* getHandler(::apache::thrift::server::TConnectionContext*) { + return iface_.get(); + } + virtual void releaseHandler(DBIf* handler) {} + + protected: + boost::shared_ptr iface_; +}; + +class DBNull : virtual public DBIf { + public: + virtual ~DBNull() {} + void Open(DBHandle& /* _return */, const Text& /* dbname */, const DBOptions& /* dboptions */) { + return; + } + Code Close(const DBHandle& /* dbhandle */, const Text& /* dbname */) { + Code _return = (Code)0; + return _return; + } + Code Put(const DBHandle& /* dbhandle */, const kv& /* keyvalue */, const WriteOptions& /* options */) { + Code _return = (Code)0; + return _return; + } + Code Delete(const DBHandle& /* dbhandle */, const Slice& /* key */, const WriteOptions& /* options */) { + Code _return = (Code)0; + return _return; + } + Code Write(const DBHandle& /* dbhandle */, const std::vector & /* batch */, const WriteOptions& /* options */) { + Code _return = (Code)0; + return _return; + } + void Get(ResultItem& /* _return */, const DBHandle& /* dbhandle */, const Slice& /* inputkey */, const ReadOptions& /* options */) { + return; + } + void NewIterator(ResultIterator& /* _return */, const DBHandle& /* dbhandle */, const ReadOptions& /* options */, IteratorType /* iteratorType */, const Slice& /* target */) { + return; + } + Code DeleteIterator(const DBHandle& /* dbhandle */, const Iterator& /* iterator */) { + Code _return = (Code)0; + return _return; + } + void GetNext(ResultPair& /* _return */, const DBHandle& /* dbhandle */, const Iterator& /* iterator */) { + return; + } + void GetPrev(ResultPair& /* _return */, const DBHandle& /* dbhandle */, const Iterator& /* iterator */) { + return; + } + void GetSnapshot(ResultSnapshot& /* _return */, const DBHandle& /* dbhandle */) { + return; + } + Code ReleaseSnapshot(const DBHandle& /* dbhandle */, const Snapshot& /* snapshot */) { + Code _return = (Code)0; + return _return; + } + Code CompactRange(const DBHandle& /* dbhandle */, const Slice& /* begin */, const Slice& /* end */) { + Code _return = (Code)0; + return _return; + } +}; + +class DB_Open_args { + public: + + static const uint64_t _reflection_id = 5143148560683113100U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + DB_Open_args() : dbname("") { + } + + DB_Open_args(const DB_Open_args&) = default; + DB_Open_args& operator=(const DB_Open_args&) = default; + DB_Open_args(DB_Open_args&&) = default; + DB_Open_args& operator=(DB_Open_args&&) = default; + + void __clear() { + dbname = ""; + dboptions.__clear(); + __isset.__clear(); + } + + virtual ~DB_Open_args() throw() {} + + Text dbname; + DBOptions dboptions; + + struct __isset { + __isset() { __clear(); } + void __clear() { + dbname = false; + dboptions = false; + } + bool dbname; + bool dboptions; + } __isset; + + bool operator == (const DB_Open_args & rhs) const + { + if (!(this->dbname == rhs.dbname)) + return false; + if (!(this->dboptions == rhs.dboptions)) + return false; + return true; + } + bool operator != (const DB_Open_args &rhs) const { + return !(*this == rhs); + } + + bool operator < (const DB_Open_args & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class DB_Open_pargs { + public: + + static const uint64_t _reflection_id = 5558263615018901836U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + + virtual ~DB_Open_pargs() throw() {} + + const Text* dbname; + const DBOptions* dboptions; + + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class DB_Open_result { + public: + + static const uint64_t _reflection_id = 585101419942616428U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + DB_Open_result() { + } + + DB_Open_result(const DB_Open_result&) = default; + DB_Open_result& operator=(const DB_Open_result&) = default; + DB_Open_result(DB_Open_result&&) = default; + DB_Open_result& operator=(DB_Open_result&&) = default; + + void __clear() { + success.__clear(); + se.__clear(); + __isset.__clear(); + } + + virtual ~DB_Open_result() throw() {} + + DBHandle success; + LeveldbException se; + + struct __isset { + __isset() { __clear(); } + void __clear() { + success = false; + se = false; + } + bool success; + bool se; + } __isset; + + bool operator == (const DB_Open_result & rhs) const + { + if (!(this->success == rhs.success)) + return false; + if (!(this->se == rhs.se)) + return false; + return true; + } + bool operator != (const DB_Open_result &rhs) const { + return !(*this == rhs); + } + + bool operator < (const DB_Open_result & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class DB_Open_presult { + public: + + static const uint64_t _reflection_id = 1768872265754142060U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + + virtual ~DB_Open_presult() throw() {} + + DBHandle* success; + LeveldbException se; + + struct __isset { + __isset() { __clear(); } + void __clear() { + success = false; + se = false; + } + bool success; + bool se; + } __isset; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + +}; + +class DB_Close_args { + public: + + static const uint64_t _reflection_id = 3653575979806469484U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + DB_Close_args() : dbname("") { + } + + DB_Close_args(const DB_Close_args&) = default; + DB_Close_args& operator=(const DB_Close_args&) = default; + DB_Close_args(DB_Close_args&&) = default; + DB_Close_args& operator=(DB_Close_args&&) = default; + + void __clear() { + dbhandle.__clear(); + dbname = ""; + __isset.__clear(); + } + + virtual ~DB_Close_args() throw() {} + + DBHandle dbhandle; + Text dbname; + + struct __isset { + __isset() { __clear(); } + void __clear() { + dbhandle = false; + dbname = false; + } + bool dbhandle; + bool dbname; + } __isset; + + bool operator == (const DB_Close_args & rhs) const + { + if (!(this->dbhandle == rhs.dbhandle)) + return false; + if (!(this->dbname == rhs.dbname)) + return false; + return true; + } + bool operator != (const DB_Close_args &rhs) const { + return !(*this == rhs); + } + + bool operator < (const DB_Close_args & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class DB_Close_pargs { + public: + + static const uint64_t _reflection_id = 2939164562981598508U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + + virtual ~DB_Close_pargs() throw() {} + + const DBHandle* dbhandle; + const Text* dbname; + + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class DB_Close_result { + public: + + static const uint64_t _reflection_id = 11420986846439017932U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + DB_Close_result() : success(static_cast(0)) { + } + + DB_Close_result(const DB_Close_result&) = default; + DB_Close_result& operator=(const DB_Close_result&) = default; + DB_Close_result(DB_Close_result&&) = default; + DB_Close_result& operator=(DB_Close_result&&) = default; + + void __clear() { + success = static_cast(0); + __isset.__clear(); + } + + virtual ~DB_Close_result() throw() {} + + Code success; + + struct __isset { + __isset() { __clear(); } + void __clear() { + success = false; + } + bool success; + } __isset; + + bool operator == (const DB_Close_result & rhs) const + { + if (!(this->success == rhs.success)) + return false; + return true; + } + bool operator != (const DB_Close_result &rhs) const { + return !(*this == rhs); + } + + bool operator < (const DB_Close_result & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class DB_Close_presult { + public: + + static const uint64_t _reflection_id = 6223865401759091820U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + + virtual ~DB_Close_presult() throw() {} + + Code* success; + + struct __isset { + __isset() { __clear(); } + void __clear() { + success = false; + } + bool success; + } __isset; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + +}; + +class DB_Put_args { + public: + + static const uint64_t _reflection_id = 15095251014297097356U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + DB_Put_args() { + } + + DB_Put_args(const DB_Put_args&) = default; + DB_Put_args& operator=(const DB_Put_args&) = default; + DB_Put_args(DB_Put_args&&) = default; + DB_Put_args& operator=(DB_Put_args&&) = default; + + void __clear() { + dbhandle.__clear(); + keyvalue.__clear(); + options.__clear(); + __isset.__clear(); + } + + virtual ~DB_Put_args() throw() {} + + DBHandle dbhandle; + kv keyvalue; + WriteOptions options; + + struct __isset { + __isset() { __clear(); } + void __clear() { + dbhandle = false; + keyvalue = false; + options = false; + } + bool dbhandle; + bool keyvalue; + bool options; + } __isset; + + bool operator == (const DB_Put_args & rhs) const + { + if (!(this->dbhandle == rhs.dbhandle)) + return false; + if (!(this->keyvalue == rhs.keyvalue)) + return false; + if (!(this->options == rhs.options)) + return false; + return true; + } + bool operator != (const DB_Put_args &rhs) const { + return !(*this == rhs); + } + + bool operator < (const DB_Put_args & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class DB_Put_pargs { + public: + + static const uint64_t _reflection_id = 5565059585805949260U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + + virtual ~DB_Put_pargs() throw() {} + + const DBHandle* dbhandle; + const kv* keyvalue; + const WriteOptions* options; + + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class DB_Put_result { + public: + + static const uint64_t _reflection_id = 16570405853514888204U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + DB_Put_result() : success(static_cast(0)) { + } + + DB_Put_result(const DB_Put_result&) = default; + DB_Put_result& operator=(const DB_Put_result&) = default; + DB_Put_result(DB_Put_result&&) = default; + DB_Put_result& operator=(DB_Put_result&&) = default; + + void __clear() { + success = static_cast(0); + __isset.__clear(); + } + + virtual ~DB_Put_result() throw() {} + + Code success; + + struct __isset { + __isset() { __clear(); } + void __clear() { + success = false; + } + bool success; + } __isset; + + bool operator == (const DB_Put_result & rhs) const + { + if (!(this->success == rhs.success)) + return false; + return true; + } + bool operator != (const DB_Put_result &rhs) const { + return !(*this == rhs); + } + + bool operator < (const DB_Put_result & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class DB_Put_presult { + public: + + static const uint64_t _reflection_id = 10104675682455674284U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + + virtual ~DB_Put_presult() throw() {} + + Code* success; + + struct __isset { + __isset() { __clear(); } + void __clear() { + success = false; + } + bool success; + } __isset; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + +}; + +class DB_Delete_args { + public: + + static const uint64_t _reflection_id = 14071755674720794316U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + DB_Delete_args() { + } + + DB_Delete_args(const DB_Delete_args&) = default; + DB_Delete_args& operator=(const DB_Delete_args&) = default; + DB_Delete_args(DB_Delete_args&&) = default; + DB_Delete_args& operator=(DB_Delete_args&&) = default; + + void __clear() { + dbhandle.__clear(); + key.__clear(); + options.__clear(); + __isset.__clear(); + } + + virtual ~DB_Delete_args() throw() {} + + DBHandle dbhandle; + Slice key; + WriteOptions options; + + struct __isset { + __isset() { __clear(); } + void __clear() { + dbhandle = false; + key = false; + options = false; + } + bool dbhandle; + bool key; + bool options; + } __isset; + + bool operator == (const DB_Delete_args & rhs) const + { + if (!(this->dbhandle == rhs.dbhandle)) + return false; + if (!(this->key == rhs.key)) + return false; + if (!(this->options == rhs.options)) + return false; + return true; + } + bool operator != (const DB_Delete_args &rhs) const { + return !(*this == rhs); + } + + bool operator < (const DB_Delete_args & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class DB_Delete_pargs { + public: + + static const uint64_t _reflection_id = 10963630852202029196U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + + virtual ~DB_Delete_pargs() throw() {} + + const DBHandle* dbhandle; + const Slice* key; + const WriteOptions* options; + + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class DB_Delete_result { + public: + + static const uint64_t _reflection_id = 13146432606937654604U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + DB_Delete_result() : success(static_cast(0)) { + } + + DB_Delete_result(const DB_Delete_result&) = default; + DB_Delete_result& operator=(const DB_Delete_result&) = default; + DB_Delete_result(DB_Delete_result&&) = default; + DB_Delete_result& operator=(DB_Delete_result&&) = default; + + void __clear() { + success = static_cast(0); + __isset.__clear(); + } + + virtual ~DB_Delete_result() throw() {} + + Code success; + + struct __isset { + __isset() { __clear(); } + void __clear() { + success = false; + } + bool success; + } __isset; + + bool operator == (const DB_Delete_result & rhs) const + { + if (!(this->success == rhs.success)) + return false; + return true; + } + bool operator != (const DB_Delete_result &rhs) const { + return !(*this == rhs); + } + + bool operator < (const DB_Delete_result & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class DB_Delete_presult { + public: + + static const uint64_t _reflection_id = 17322073978832327340U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + + virtual ~DB_Delete_presult() throw() {} + + Code* success; + + struct __isset { + __isset() { __clear(); } + void __clear() { + success = false; + } + bool success; + } __isset; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + +}; + +class DB_Write_args { + public: + + static const uint64_t _reflection_id = 12580138032621479660U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + DB_Write_args() { + } + + DB_Write_args(const DB_Write_args&) = default; + DB_Write_args& operator=(const DB_Write_args&) = default; + DB_Write_args(DB_Write_args&&) = default; + DB_Write_args& operator=(DB_Write_args&&) = default; + + void __clear() { + dbhandle.__clear(); + batch.clear(); + options.__clear(); + __isset.__clear(); + } + + virtual ~DB_Write_args() throw() {} + + DBHandle dbhandle; + std::vector batch; + WriteOptions options; + + struct __isset { + __isset() { __clear(); } + void __clear() { + dbhandle = false; + batch = false; + options = false; + } + bool dbhandle; + bool batch; + bool options; + } __isset; + + bool operator == (const DB_Write_args & rhs) const + { + if (!(this->dbhandle == rhs.dbhandle)) + return false; + if (!(this->batch == rhs.batch)) + return false; + if (!(this->options == rhs.options)) + return false; + return true; + } + bool operator != (const DB_Write_args &rhs) const { + return !(*this == rhs); + } + + bool operator < (const DB_Write_args & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class DB_Write_pargs { + public: + + static const uint64_t _reflection_id = 3898152651480297548U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + + virtual ~DB_Write_pargs() throw() {} + + const DBHandle* dbhandle; + const std::vector * batch; + const WriteOptions* options; + + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class DB_Write_result { + public: + + static const uint64_t _reflection_id = 18306458689824631724U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + DB_Write_result() : success(static_cast(0)) { + } + + DB_Write_result(const DB_Write_result&) = default; + DB_Write_result& operator=(const DB_Write_result&) = default; + DB_Write_result(DB_Write_result&&) = default; + DB_Write_result& operator=(DB_Write_result&&) = default; + + void __clear() { + success = static_cast(0); + __isset.__clear(); + } + + virtual ~DB_Write_result() throw() {} + + Code success; + + struct __isset { + __isset() { __clear(); } + void __clear() { + success = false; + } + bool success; + } __isset; + + bool operator == (const DB_Write_result & rhs) const + { + if (!(this->success == rhs.success)) + return false; + return true; + } + bool operator != (const DB_Write_result &rhs) const { + return !(*this == rhs); + } + + bool operator < (const DB_Write_result & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class DB_Write_presult { + public: + + static const uint64_t _reflection_id = 11189139329514764236U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + + virtual ~DB_Write_presult() throw() {} + + Code* success; + + struct __isset { + __isset() { __clear(); } + void __clear() { + success = false; + } + bool success; + } __isset; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + +}; + +class DB_Get_args { + public: + + static const uint64_t _reflection_id = 5802112698229680972U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + DB_Get_args() { + } + + DB_Get_args(const DB_Get_args&) = default; + DB_Get_args& operator=(const DB_Get_args&) = default; + DB_Get_args(DB_Get_args&&) = default; + DB_Get_args& operator=(DB_Get_args&&) = default; + + void __clear() { + dbhandle.__clear(); + inputkey.__clear(); + options.__clear(); + __isset.__clear(); + } + + virtual ~DB_Get_args() throw() {} + + DBHandle dbhandle; + Slice inputkey; + ReadOptions options; + + struct __isset { + __isset() { __clear(); } + void __clear() { + dbhandle = false; + inputkey = false; + options = false; + } + bool dbhandle; + bool inputkey; + bool options; + } __isset; + + bool operator == (const DB_Get_args & rhs) const + { + if (!(this->dbhandle == rhs.dbhandle)) + return false; + if (!(this->inputkey == rhs.inputkey)) + return false; + if (!(this->options == rhs.options)) + return false; + return true; + } + bool operator != (const DB_Get_args &rhs) const { + return !(*this == rhs); + } + + bool operator < (const DB_Get_args & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class DB_Get_pargs { + public: + + static const uint64_t _reflection_id = 775586916617672204U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + + virtual ~DB_Get_pargs() throw() {} + + const DBHandle* dbhandle; + const Slice* inputkey; + const ReadOptions* options; + + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class DB_Get_result { + public: + + static const uint64_t _reflection_id = 471316346092127948U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + DB_Get_result() { + } + + DB_Get_result(const DB_Get_result&) = default; + DB_Get_result& operator=(const DB_Get_result&) = default; + DB_Get_result(DB_Get_result&&) = default; + DB_Get_result& operator=(DB_Get_result&&) = default; + + void __clear() { + success.__clear(); + __isset.__clear(); + } + + virtual ~DB_Get_result() throw() {} + + ResultItem success; + + struct __isset { + __isset() { __clear(); } + void __clear() { + success = false; + } + bool success; + } __isset; + + bool operator == (const DB_Get_result & rhs) const + { + if (!(this->success == rhs.success)) + return false; + return true; + } + bool operator != (const DB_Get_result &rhs) const { + return !(*this == rhs); + } + + bool operator < (const DB_Get_result & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class DB_Get_presult { + public: + + static const uint64_t _reflection_id = 15884536389555503916U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + + virtual ~DB_Get_presult() throw() {} + + ResultItem* success; + + struct __isset { + __isset() { __clear(); } + void __clear() { + success = false; + } + bool success; + } __isset; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + +}; + +class DB_NewIterator_args { + public: + + static const uint64_t _reflection_id = 6645485240268124940U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + DB_NewIterator_args() : iteratorType(static_cast(0)) { + } + + DB_NewIterator_args(const DB_NewIterator_args&) = default; + DB_NewIterator_args& operator=(const DB_NewIterator_args&) = default; + DB_NewIterator_args(DB_NewIterator_args&&) = default; + DB_NewIterator_args& operator=(DB_NewIterator_args&&) = default; + + void __clear() { + dbhandle.__clear(); + options.__clear(); + iteratorType = static_cast(0); + target.__clear(); + __isset.__clear(); + } + + virtual ~DB_NewIterator_args() throw() {} + + DBHandle dbhandle; + ReadOptions options; + IteratorType iteratorType; + Slice target; + + struct __isset { + __isset() { __clear(); } + void __clear() { + dbhandle = false; + options = false; + iteratorType = false; + target = false; + } + bool dbhandle; + bool options; + bool iteratorType; + bool target; + } __isset; + + bool operator == (const DB_NewIterator_args & rhs) const + { + if (!(this->dbhandle == rhs.dbhandle)) + return false; + if (!(this->options == rhs.options)) + return false; + if (!(this->iteratorType == rhs.iteratorType)) + return false; + if (!(this->target == rhs.target)) + return false; + return true; + } + bool operator != (const DB_NewIterator_args &rhs) const { + return !(*this == rhs); + } + + bool operator < (const DB_NewIterator_args & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class DB_NewIterator_pargs { + public: + + static const uint64_t _reflection_id = 5353644888786218764U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + + virtual ~DB_NewIterator_pargs() throw() {} + + const DBHandle* dbhandle; + const ReadOptions* options; + const IteratorType* iteratorType; + const Slice* target; + + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class DB_NewIterator_result { + public: + + static const uint64_t _reflection_id = 16747725486624219692U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + DB_NewIterator_result() { + } + + DB_NewIterator_result(const DB_NewIterator_result&) = default; + DB_NewIterator_result& operator=(const DB_NewIterator_result&) = default; + DB_NewIterator_result(DB_NewIterator_result&&) = default; + DB_NewIterator_result& operator=(DB_NewIterator_result&&) = default; + + void __clear() { + success.__clear(); + __isset.__clear(); + } + + virtual ~DB_NewIterator_result() throw() {} + + ResultIterator success; + + struct __isset { + __isset() { __clear(); } + void __clear() { + success = false; + } + bool success; + } __isset; + + bool operator == (const DB_NewIterator_result & rhs) const + { + if (!(this->success == rhs.success)) + return false; + return true; + } + bool operator != (const DB_NewIterator_result &rhs) const { + return !(*this == rhs); + } + + bool operator < (const DB_NewIterator_result & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class DB_NewIterator_presult { + public: + + static const uint64_t _reflection_id = 30225981924653164U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + + virtual ~DB_NewIterator_presult() throw() {} + + ResultIterator* success; + + struct __isset { + __isset() { __clear(); } + void __clear() { + success = false; + } + bool success; + } __isset; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + +}; + +class DB_DeleteIterator_args { + public: + + static const uint64_t _reflection_id = 15751789239200963564U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + DB_DeleteIterator_args() { + } + + DB_DeleteIterator_args(const DB_DeleteIterator_args&) = default; + DB_DeleteIterator_args& operator=(const DB_DeleteIterator_args&) = default; + DB_DeleteIterator_args(DB_DeleteIterator_args&&) = default; + DB_DeleteIterator_args& operator=(DB_DeleteIterator_args&&) = default; + + void __clear() { + dbhandle.__clear(); + iterator.__clear(); + __isset.__clear(); + } + + virtual ~DB_DeleteIterator_args() throw() {} + + DBHandle dbhandle; + Iterator iterator; + + struct __isset { + __isset() { __clear(); } + void __clear() { + dbhandle = false; + iterator = false; + } + bool dbhandle; + bool iterator; + } __isset; + + bool operator == (const DB_DeleteIterator_args & rhs) const + { + if (!(this->dbhandle == rhs.dbhandle)) + return false; + if (!(this->iterator == rhs.iterator)) + return false; + return true; + } + bool operator != (const DB_DeleteIterator_args &rhs) const { + return !(*this == rhs); + } + + bool operator < (const DB_DeleteIterator_args & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class DB_DeleteIterator_pargs { + public: + + static const uint64_t _reflection_id = 6515929643750099916U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + + virtual ~DB_DeleteIterator_pargs() throw() {} + + const DBHandle* dbhandle; + const Iterator* iterator; + + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class DB_DeleteIterator_result { + public: + + static const uint64_t _reflection_id = 567840673627283948U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + DB_DeleteIterator_result() : success(static_cast(0)) { + } + + DB_DeleteIterator_result(const DB_DeleteIterator_result&) = default; + DB_DeleteIterator_result& operator=(const DB_DeleteIterator_result&) = default; + DB_DeleteIterator_result(DB_DeleteIterator_result&&) = default; + DB_DeleteIterator_result& operator=(DB_DeleteIterator_result&&) = default; + + void __clear() { + success = static_cast(0); + __isset.__clear(); + } + + virtual ~DB_DeleteIterator_result() throw() {} + + Code success; + + struct __isset { + __isset() { __clear(); } + void __clear() { + success = false; + } + bool success; + } __isset; + + bool operator == (const DB_DeleteIterator_result & rhs) const + { + if (!(this->success == rhs.success)) + return false; + return true; + } + bool operator != (const DB_DeleteIterator_result &rhs) const { + return !(*this == rhs); + } + + bool operator < (const DB_DeleteIterator_result & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class DB_DeleteIterator_presult { + public: + + static const uint64_t _reflection_id = 16445220168564239308U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + + virtual ~DB_DeleteIterator_presult() throw() {} + + Code* success; + + struct __isset { + __isset() { __clear(); } + void __clear() { + success = false; + } + bool success; + } __isset; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + +}; + +class DB_GetNext_args { + public: + + static const uint64_t _reflection_id = 6184101100378654860U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + DB_GetNext_args() { + } + + DB_GetNext_args(const DB_GetNext_args&) = default; + DB_GetNext_args& operator=(const DB_GetNext_args&) = default; + DB_GetNext_args(DB_GetNext_args&&) = default; + DB_GetNext_args& operator=(DB_GetNext_args&&) = default; + + void __clear() { + dbhandle.__clear(); + iterator.__clear(); + __isset.__clear(); + } + + virtual ~DB_GetNext_args() throw() {} + + DBHandle dbhandle; + Iterator iterator; + + struct __isset { + __isset() { __clear(); } + void __clear() { + dbhandle = false; + iterator = false; + } + bool dbhandle; + bool iterator; + } __isset; + + bool operator == (const DB_GetNext_args & rhs) const + { + if (!(this->dbhandle == rhs.dbhandle)) + return false; + if (!(this->iterator == rhs.iterator)) + return false; + return true; + } + bool operator != (const DB_GetNext_args &rhs) const { + return !(*this == rhs); + } + + bool operator < (const DB_GetNext_args & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class DB_GetNext_pargs { + public: + + static const uint64_t _reflection_id = 14242404761042565548U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + + virtual ~DB_GetNext_pargs() throw() {} + + const DBHandle* dbhandle; + const Iterator* iterator; + + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class DB_GetNext_result { + public: + + static const uint64_t _reflection_id = 824363650095435948U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + DB_GetNext_result() { + } + + DB_GetNext_result(const DB_GetNext_result&) = default; + DB_GetNext_result& operator=(const DB_GetNext_result&) = default; + DB_GetNext_result(DB_GetNext_result&&) = default; + DB_GetNext_result& operator=(DB_GetNext_result&&) = default; + + void __clear() { + success.__clear(); + __isset.__clear(); + } + + virtual ~DB_GetNext_result() throw() {} + + ResultPair success; + + struct __isset { + __isset() { __clear(); } + void __clear() { + success = false; + } + bool success; + } __isset; + + bool operator == (const DB_GetNext_result & rhs) const + { + if (!(this->success == rhs.success)) + return false; + return true; + } + bool operator != (const DB_GetNext_result &rhs) const { + return !(*this == rhs); + } + + bool operator < (const DB_GetNext_result & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class DB_GetNext_presult { + public: + + static const uint64_t _reflection_id = 17974802380941600588U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + + virtual ~DB_GetNext_presult() throw() {} + + ResultPair* success; + + struct __isset { + __isset() { __clear(); } + void __clear() { + success = false; + } + bool success; + } __isset; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + +}; + +class DB_GetPrev_args { + public: + + static const uint64_t _reflection_id = 8870271973715332140U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + DB_GetPrev_args() { + } + + DB_GetPrev_args(const DB_GetPrev_args&) = default; + DB_GetPrev_args& operator=(const DB_GetPrev_args&) = default; + DB_GetPrev_args(DB_GetPrev_args&&) = default; + DB_GetPrev_args& operator=(DB_GetPrev_args&&) = default; + + void __clear() { + dbhandle.__clear(); + iterator.__clear(); + __isset.__clear(); + } + + virtual ~DB_GetPrev_args() throw() {} + + DBHandle dbhandle; + Iterator iterator; + + struct __isset { + __isset() { __clear(); } + void __clear() { + dbhandle = false; + iterator = false; + } + bool dbhandle; + bool iterator; + } __isset; + + bool operator == (const DB_GetPrev_args & rhs) const + { + if (!(this->dbhandle == rhs.dbhandle)) + return false; + if (!(this->iterator == rhs.iterator)) + return false; + return true; + } + bool operator != (const DB_GetPrev_args &rhs) const { + return !(*this == rhs); + } + + bool operator < (const DB_GetPrev_args & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class DB_GetPrev_pargs { + public: + + static const uint64_t _reflection_id = 16552208169457784972U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + + virtual ~DB_GetPrev_pargs() throw() {} + + const DBHandle* dbhandle; + const Iterator* iterator; + + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class DB_GetPrev_result { + public: + + static const uint64_t _reflection_id = 17005837545637039820U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + DB_GetPrev_result() { + } + + DB_GetPrev_result(const DB_GetPrev_result&) = default; + DB_GetPrev_result& operator=(const DB_GetPrev_result&) = default; + DB_GetPrev_result(DB_GetPrev_result&&) = default; + DB_GetPrev_result& operator=(DB_GetPrev_result&&) = default; + + void __clear() { + success.__clear(); + __isset.__clear(); + } + + virtual ~DB_GetPrev_result() throw() {} + + ResultPair success; + + struct __isset { + __isset() { __clear(); } + void __clear() { + success = false; + } + bool success; + } __isset; + + bool operator == (const DB_GetPrev_result & rhs) const + { + if (!(this->success == rhs.success)) + return false; + return true; + } + bool operator != (const DB_GetPrev_result &rhs) const { + return !(*this == rhs); + } + + bool operator < (const DB_GetPrev_result & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class DB_GetPrev_presult { + public: + + static const uint64_t _reflection_id = 12518588444439336556U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + + virtual ~DB_GetPrev_presult() throw() {} + + ResultPair* success; + + struct __isset { + __isset() { __clear(); } + void __clear() { + success = false; + } + bool success; + } __isset; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + +}; + +class DB_GetSnapshot_args { + public: + + static const uint64_t _reflection_id = 14984968825961323724U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + DB_GetSnapshot_args() { + } + + DB_GetSnapshot_args(const DB_GetSnapshot_args&) = default; + DB_GetSnapshot_args& operator=(const DB_GetSnapshot_args&) = default; + DB_GetSnapshot_args(DB_GetSnapshot_args&&) = default; + DB_GetSnapshot_args& operator=(DB_GetSnapshot_args&&) = default; + + void __clear() { + dbhandle.__clear(); + __isset.__clear(); + } + + virtual ~DB_GetSnapshot_args() throw() {} + + DBHandle dbhandle; + + struct __isset { + __isset() { __clear(); } + void __clear() { + dbhandle = false; + } + bool dbhandle; + } __isset; + + bool operator == (const DB_GetSnapshot_args & rhs) const + { + if (!(this->dbhandle == rhs.dbhandle)) + return false; + return true; + } + bool operator != (const DB_GetSnapshot_args &rhs) const { + return !(*this == rhs); + } + + bool operator < (const DB_GetSnapshot_args & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class DB_GetSnapshot_pargs { + public: + + static const uint64_t _reflection_id = 7188867685881890956U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + + virtual ~DB_GetSnapshot_pargs() throw() {} + + const DBHandle* dbhandle; + + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class DB_GetSnapshot_result { + public: + + static const uint64_t _reflection_id = 18029040421792131212U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + DB_GetSnapshot_result() { + } + + DB_GetSnapshot_result(const DB_GetSnapshot_result&) = default; + DB_GetSnapshot_result& operator=(const DB_GetSnapshot_result&) = default; + DB_GetSnapshot_result(DB_GetSnapshot_result&&) = default; + DB_GetSnapshot_result& operator=(DB_GetSnapshot_result&&) = default; + + void __clear() { + success.__clear(); + __isset.__clear(); + } + + virtual ~DB_GetSnapshot_result() throw() {} + + ResultSnapshot success; + + struct __isset { + __isset() { __clear(); } + void __clear() { + success = false; + } + bool success; + } __isset; + + bool operator == (const DB_GetSnapshot_result & rhs) const + { + if (!(this->success == rhs.success)) + return false; + return true; + } + bool operator != (const DB_GetSnapshot_result &rhs) const { + return !(*this == rhs); + } + + bool operator < (const DB_GetSnapshot_result & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class DB_GetSnapshot_presult { + public: + + static const uint64_t _reflection_id = 10451056179121017996U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + + virtual ~DB_GetSnapshot_presult() throw() {} + + ResultSnapshot* success; + + struct __isset { + __isset() { __clear(); } + void __clear() { + success = false; + } + bool success; + } __isset; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + +}; + +class DB_ReleaseSnapshot_args { + public: + + static const uint64_t _reflection_id = 14945627971981075500U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + DB_ReleaseSnapshot_args() { + } + + DB_ReleaseSnapshot_args(const DB_ReleaseSnapshot_args&) = default; + DB_ReleaseSnapshot_args& operator=(const DB_ReleaseSnapshot_args&) = default; + DB_ReleaseSnapshot_args(DB_ReleaseSnapshot_args&&) = default; + DB_ReleaseSnapshot_args& operator=(DB_ReleaseSnapshot_args&&) = default; + + void __clear() { + dbhandle.__clear(); + snapshot.__clear(); + __isset.__clear(); + } + + virtual ~DB_ReleaseSnapshot_args() throw() {} + + DBHandle dbhandle; + Snapshot snapshot; + + struct __isset { + __isset() { __clear(); } + void __clear() { + dbhandle = false; + snapshot = false; + } + bool dbhandle; + bool snapshot; + } __isset; + + bool operator == (const DB_ReleaseSnapshot_args & rhs) const + { + if (!(this->dbhandle == rhs.dbhandle)) + return false; + if (!(this->snapshot == rhs.snapshot)) + return false; + return true; + } + bool operator != (const DB_ReleaseSnapshot_args &rhs) const { + return !(*this == rhs); + } + + bool operator < (const DB_ReleaseSnapshot_args & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class DB_ReleaseSnapshot_pargs { + public: + + static const uint64_t _reflection_id = 8283387247692679788U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + + virtual ~DB_ReleaseSnapshot_pargs() throw() {} + + const DBHandle* dbhandle; + const Snapshot* snapshot; + + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class DB_ReleaseSnapshot_result { + public: + + static const uint64_t _reflection_id = 15486363403695239660U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + DB_ReleaseSnapshot_result() : success(static_cast(0)) { + } + + DB_ReleaseSnapshot_result(const DB_ReleaseSnapshot_result&) = default; + DB_ReleaseSnapshot_result& operator=(const DB_ReleaseSnapshot_result&) = default; + DB_ReleaseSnapshot_result(DB_ReleaseSnapshot_result&&) = default; + DB_ReleaseSnapshot_result& operator=(DB_ReleaseSnapshot_result&&) = default; + + void __clear() { + success = static_cast(0); + __isset.__clear(); + } + + virtual ~DB_ReleaseSnapshot_result() throw() {} + + Code success; + + struct __isset { + __isset() { __clear(); } + void __clear() { + success = false; + } + bool success; + } __isset; + + bool operator == (const DB_ReleaseSnapshot_result & rhs) const + { + if (!(this->success == rhs.success)) + return false; + return true; + } + bool operator != (const DB_ReleaseSnapshot_result &rhs) const { + return !(*this == rhs); + } + + bool operator < (const DB_ReleaseSnapshot_result & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class DB_ReleaseSnapshot_presult { + public: + + static const uint64_t _reflection_id = 13925350785061977420U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + + virtual ~DB_ReleaseSnapshot_presult() throw() {} + + Code* success; + + struct __isset { + __isset() { __clear(); } + void __clear() { + success = false; + } + bool success; + } __isset; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + +}; + +class DB_CompactRange_args { + public: + + static const uint64_t _reflection_id = 6075170995358567596U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + DB_CompactRange_args() { + } + + DB_CompactRange_args(const DB_CompactRange_args&) = default; + DB_CompactRange_args& operator=(const DB_CompactRange_args&) = default; + DB_CompactRange_args(DB_CompactRange_args&&) = default; + DB_CompactRange_args& operator=(DB_CompactRange_args&&) = default; + + void __clear() { + dbhandle.__clear(); + begin.__clear(); + end.__clear(); + __isset.__clear(); + } + + virtual ~DB_CompactRange_args() throw() {} + + DBHandle dbhandle; + Slice begin; + Slice end; + + struct __isset { + __isset() { __clear(); } + void __clear() { + dbhandle = false; + begin = false; + end = false; + } + bool dbhandle; + bool begin; + bool end; + } __isset; + + bool operator == (const DB_CompactRange_args & rhs) const + { + if (!(this->dbhandle == rhs.dbhandle)) + return false; + if (!(this->begin == rhs.begin)) + return false; + if (!(this->end == rhs.end)) + return false; + return true; + } + bool operator != (const DB_CompactRange_args &rhs) const { + return !(*this == rhs); + } + + bool operator < (const DB_CompactRange_args & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class DB_CompactRange_pargs { + public: + + static const uint64_t _reflection_id = 11177200356381642316U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + + virtual ~DB_CompactRange_pargs() throw() {} + + const DBHandle* dbhandle; + const Slice* begin; + const Slice* end; + + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class DB_CompactRange_result { + public: + + static const uint64_t _reflection_id = 12835198829390966636U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + DB_CompactRange_result() : success(static_cast(0)) { + } + + DB_CompactRange_result(const DB_CompactRange_result&) = default; + DB_CompactRange_result& operator=(const DB_CompactRange_result&) = default; + DB_CompactRange_result(DB_CompactRange_result&&) = default; + DB_CompactRange_result& operator=(DB_CompactRange_result&&) = default; + + void __clear() { + success = static_cast(0); + __isset.__clear(); + } + + virtual ~DB_CompactRange_result() throw() {} + + Code success; + + struct __isset { + __isset() { __clear(); } + void __clear() { + success = false; + } + bool success; + } __isset; + + bool operator == (const DB_CompactRange_result & rhs) const + { + if (!(this->success == rhs.success)) + return false; + return true; + } + bool operator != (const DB_CompactRange_result &rhs) const { + return !(*this == rhs); + } + + bool operator < (const DB_CompactRange_result & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class DB_CompactRange_presult { + public: + + static const uint64_t _reflection_id = 14927478234826947852U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + + virtual ~DB_CompactRange_presult() throw() {} + + Code* success; + + struct __isset { + __isset() { __clear(); } + void __clear() { + success = false; + } + bool success; + } __isset; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + +}; + +class DBClient : virtual public DBIf, virtual public apache::thrift::TClientBase { + public: + DBClient(boost::shared_ptr prot) : + checkSeqid_(true), + nextSendSequenceId_(1), + nextRecvSequenceId_(1), + piprot_(prot), + poprot_(prot) { + iprot_ = prot.get(); + oprot_ = prot.get(); + } + DBClient(boost::shared_ptr iprot, boost::shared_ptr oprot) : + checkSeqid_(true), + nextSendSequenceId_(1), + nextRecvSequenceId_(1), + piprot_(iprot), + poprot_(oprot) { + iprot_ = iprot.get(); + oprot_ = oprot.get(); + } + boost::shared_ptr getInputProtocol() { + return piprot_; + } + boost::shared_ptr getOutputProtocol() { + return poprot_; + } + void Open(DBHandle& _return, const Text& dbname, const DBOptions& dboptions); + void send_Open(const Text& dbname, const DBOptions& dboptions); + void recv_Open(DBHandle& _return); + Code Close(const DBHandle& dbhandle, const Text& dbname); + void send_Close(const DBHandle& dbhandle, const Text& dbname); + Code recv_Close(); + Code Put(const DBHandle& dbhandle, const kv& keyvalue, const WriteOptions& options); + void send_Put(const DBHandle& dbhandle, const kv& keyvalue, const WriteOptions& options); + Code recv_Put(); + Code Delete(const DBHandle& dbhandle, const Slice& key, const WriteOptions& options); + void send_Delete(const DBHandle& dbhandle, const Slice& key, const WriteOptions& options); + Code recv_Delete(); + Code Write(const DBHandle& dbhandle, const std::vector & batch, const WriteOptions& options); + void send_Write(const DBHandle& dbhandle, const std::vector & batch, const WriteOptions& options); + Code recv_Write(); + void Get(ResultItem& _return, const DBHandle& dbhandle, const Slice& inputkey, const ReadOptions& options); + void send_Get(const DBHandle& dbhandle, const Slice& inputkey, const ReadOptions& options); + void recv_Get(ResultItem& _return); + void NewIterator(ResultIterator& _return, const DBHandle& dbhandle, const ReadOptions& options, IteratorType iteratorType, const Slice& target); + void send_NewIterator(const DBHandle& dbhandle, const ReadOptions& options, IteratorType iteratorType, const Slice& target); + void recv_NewIterator(ResultIterator& _return); + Code DeleteIterator(const DBHandle& dbhandle, const Iterator& iterator); + void send_DeleteIterator(const DBHandle& dbhandle, const Iterator& iterator); + Code recv_DeleteIterator(); + void GetNext(ResultPair& _return, const DBHandle& dbhandle, const Iterator& iterator); + void send_GetNext(const DBHandle& dbhandle, const Iterator& iterator); + void recv_GetNext(ResultPair& _return); + void GetPrev(ResultPair& _return, const DBHandle& dbhandle, const Iterator& iterator); + void send_GetPrev(const DBHandle& dbhandle, const Iterator& iterator); + void recv_GetPrev(ResultPair& _return); + void GetSnapshot(ResultSnapshot& _return, const DBHandle& dbhandle); + void send_GetSnapshot(const DBHandle& dbhandle); + void recv_GetSnapshot(ResultSnapshot& _return); + Code ReleaseSnapshot(const DBHandle& dbhandle, const Snapshot& snapshot); + void send_ReleaseSnapshot(const DBHandle& dbhandle, const Snapshot& snapshot); + Code recv_ReleaseSnapshot(); + Code CompactRange(const DBHandle& dbhandle, const Slice& begin, const Slice& end); + void send_CompactRange(const DBHandle& dbhandle, const Slice& begin, const Slice& end); + Code recv_CompactRange(); + + /** + * Disable checking the seqid field in server responses. + * + * This should only be used with broken servers that return incorrect seqid values. + */ + void _disableSequenceIdChecks() { + checkSeqid_ = false; + } + + protected: + bool checkSeqid_; + int32_t nextSendSequenceId_; + int32_t nextRecvSequenceId_; + int32_t getNextSendSequenceId(); + int32_t getNextRecvSequenceId(); + boost::shared_ptr piprot_; + boost::shared_ptr poprot_; + apache::thrift::protocol::TProtocol* iprot_; + apache::thrift::protocol::TProtocol* oprot_; +}; + +class DBProcessor : public ::apache::thrift::TDispatchProcessor { + protected: + boost::shared_ptr iface_; + virtual bool dispatchCall(apache::thrift::protocol::TProtocol* iprot, apache::thrift::protocol::TProtocol* oprot, const std::string& fname, int32_t seqid, apache::thrift::server::TConnectionContext* connectionContext); + private: + typedef void (DBProcessor::*ProcessFunction)(int32_t, apache::thrift::protocol::TProtocol*, apache::thrift::protocol::TProtocol*, apache::thrift::server::TConnectionContext*); + typedef std::map ProcessMap; + ProcessMap processMap_; + void process_Open(int32_t seqid, apache::thrift::protocol::TProtocol* iprot, apache::thrift::protocol::TProtocol* oprot, apache::thrift::server::TConnectionContext* connectionContext); + void process_Close(int32_t seqid, apache::thrift::protocol::TProtocol* iprot, apache::thrift::protocol::TProtocol* oprot, apache::thrift::server::TConnectionContext* connectionContext); + void process_Put(int32_t seqid, apache::thrift::protocol::TProtocol* iprot, apache::thrift::protocol::TProtocol* oprot, apache::thrift::server::TConnectionContext* connectionContext); + void process_Delete(int32_t seqid, apache::thrift::protocol::TProtocol* iprot, apache::thrift::protocol::TProtocol* oprot, apache::thrift::server::TConnectionContext* connectionContext); + void process_Write(int32_t seqid, apache::thrift::protocol::TProtocol* iprot, apache::thrift::protocol::TProtocol* oprot, apache::thrift::server::TConnectionContext* connectionContext); + void process_Get(int32_t seqid, apache::thrift::protocol::TProtocol* iprot, apache::thrift::protocol::TProtocol* oprot, apache::thrift::server::TConnectionContext* connectionContext); + void process_NewIterator(int32_t seqid, apache::thrift::protocol::TProtocol* iprot, apache::thrift::protocol::TProtocol* oprot, apache::thrift::server::TConnectionContext* connectionContext); + void process_DeleteIterator(int32_t seqid, apache::thrift::protocol::TProtocol* iprot, apache::thrift::protocol::TProtocol* oprot, apache::thrift::server::TConnectionContext* connectionContext); + void process_GetNext(int32_t seqid, apache::thrift::protocol::TProtocol* iprot, apache::thrift::protocol::TProtocol* oprot, apache::thrift::server::TConnectionContext* connectionContext); + void process_GetPrev(int32_t seqid, apache::thrift::protocol::TProtocol* iprot, apache::thrift::protocol::TProtocol* oprot, apache::thrift::server::TConnectionContext* connectionContext); + void process_GetSnapshot(int32_t seqid, apache::thrift::protocol::TProtocol* iprot, apache::thrift::protocol::TProtocol* oprot, apache::thrift::server::TConnectionContext* connectionContext); + void process_ReleaseSnapshot(int32_t seqid, apache::thrift::protocol::TProtocol* iprot, apache::thrift::protocol::TProtocol* oprot, apache::thrift::server::TConnectionContext* connectionContext); + void process_CompactRange(int32_t seqid, apache::thrift::protocol::TProtocol* iprot, apache::thrift::protocol::TProtocol* oprot, apache::thrift::server::TConnectionContext* connectionContext); + public: + DBProcessor(boost::shared_ptr iface) : + iface_(iface) { + processMap_["Open"] = &DBProcessor::process_Open; + processMap_["Close"] = &DBProcessor::process_Close; + processMap_["Put"] = &DBProcessor::process_Put; + processMap_["Delete"] = &DBProcessor::process_Delete; + processMap_["Write"] = &DBProcessor::process_Write; + processMap_["Get"] = &DBProcessor::process_Get; + processMap_["NewIterator"] = &DBProcessor::process_NewIterator; + processMap_["DeleteIterator"] = &DBProcessor::process_DeleteIterator; + processMap_["GetNext"] = &DBProcessor::process_GetNext; + processMap_["GetPrev"] = &DBProcessor::process_GetPrev; + processMap_["GetSnapshot"] = &DBProcessor::process_GetSnapshot; + processMap_["ReleaseSnapshot"] = &DBProcessor::process_ReleaseSnapshot; + processMap_["CompactRange"] = &DBProcessor::process_CompactRange; + } + + virtual ~DBProcessor() {} + + boost::shared_ptr > getProcessFunctions() { + boost::shared_ptr > rSet(new std::set()); + rSet->insert("DB.Open"); + rSet->insert("DB.Close"); + rSet->insert("DB.Put"); + rSet->insert("DB.Delete"); + rSet->insert("DB.Write"); + rSet->insert("DB.Get"); + rSet->insert("DB.NewIterator"); + rSet->insert("DB.DeleteIterator"); + rSet->insert("DB.GetNext"); + rSet->insert("DB.GetPrev"); + rSet->insert("DB.GetSnapshot"); + rSet->insert("DB.ReleaseSnapshot"); + rSet->insert("DB.CompactRange"); + return rSet; + } +}; + +class DBProcessorFactory : public ::apache::thrift::TProcessorFactory { + public: + DBProcessorFactory(const ::boost::shared_ptr< DBIfFactory >& handlerFactory) : + handlerFactory_(handlerFactory) {} + + ::boost::shared_ptr< ::apache::thrift::TProcessor > getProcessor(::apache::thrift::server::TConnectionContext* ctx); + + protected: + ::boost::shared_ptr< DBIfFactory > handlerFactory_; +}; + +class DBMultiface : virtual public DBIf { + public: + DBMultiface(std::vector >& ifaces) : ifaces_(ifaces) { + } + virtual ~DBMultiface() {} + protected: + std::vector > ifaces_; + DBMultiface() {} + void add(boost::shared_ptr iface) { + ifaces_.push_back(iface); + } + public: + void Open(DBHandle& _return, const Text& dbname, const DBOptions& dboptions) { + uint32_t i; + uint32_t sz = ifaces_.size(); + for (i = 0; i < sz; ++i) { + ifaces_[i]->Open(_return, dbname, dboptions); + } + } + + Code Close(const DBHandle& dbhandle, const Text& dbname) { + uint32_t i; + uint32_t sz = ifaces_.size(); + for (i = 0; i < sz - 1; ++i) { + ifaces_[i]->Close(dbhandle, dbname); + } + return ifaces_[i]->Close(dbhandle, dbname); + } + + Code Put(const DBHandle& dbhandle, const kv& keyvalue, const WriteOptions& options) { + uint32_t i; + uint32_t sz = ifaces_.size(); + for (i = 0; i < sz - 1; ++i) { + ifaces_[i]->Put(dbhandle, keyvalue, options); + } + return ifaces_[i]->Put(dbhandle, keyvalue, options); + } + + Code Delete(const DBHandle& dbhandle, const Slice& key, const WriteOptions& options) { + uint32_t i; + uint32_t sz = ifaces_.size(); + for (i = 0; i < sz - 1; ++i) { + ifaces_[i]->Delete(dbhandle, key, options); + } + return ifaces_[i]->Delete(dbhandle, key, options); + } + + Code Write(const DBHandle& dbhandle, const std::vector & batch, const WriteOptions& options) { + uint32_t i; + uint32_t sz = ifaces_.size(); + for (i = 0; i < sz - 1; ++i) { + ifaces_[i]->Write(dbhandle, batch, options); + } + return ifaces_[i]->Write(dbhandle, batch, options); + } + + void Get(ResultItem& _return, const DBHandle& dbhandle, const Slice& inputkey, const ReadOptions& options) { + uint32_t i; + uint32_t sz = ifaces_.size(); + for (i = 0; i < sz; ++i) { + ifaces_[i]->Get(_return, dbhandle, inputkey, options); + } + } + + void NewIterator(ResultIterator& _return, const DBHandle& dbhandle, const ReadOptions& options, IteratorType iteratorType, const Slice& target) { + uint32_t i; + uint32_t sz = ifaces_.size(); + for (i = 0; i < sz; ++i) { + ifaces_[i]->NewIterator(_return, dbhandle, options, iteratorType, target); + } + } + + Code DeleteIterator(const DBHandle& dbhandle, const Iterator& iterator) { + uint32_t i; + uint32_t sz = ifaces_.size(); + for (i = 0; i < sz - 1; ++i) { + ifaces_[i]->DeleteIterator(dbhandle, iterator); + } + return ifaces_[i]->DeleteIterator(dbhandle, iterator); + } + + void GetNext(ResultPair& _return, const DBHandle& dbhandle, const Iterator& iterator) { + uint32_t i; + uint32_t sz = ifaces_.size(); + for (i = 0; i < sz; ++i) { + ifaces_[i]->GetNext(_return, dbhandle, iterator); + } + } + + void GetPrev(ResultPair& _return, const DBHandle& dbhandle, const Iterator& iterator) { + uint32_t i; + uint32_t sz = ifaces_.size(); + for (i = 0; i < sz; ++i) { + ifaces_[i]->GetPrev(_return, dbhandle, iterator); + } + } + + void GetSnapshot(ResultSnapshot& _return, const DBHandle& dbhandle) { + uint32_t i; + uint32_t sz = ifaces_.size(); + for (i = 0; i < sz; ++i) { + ifaces_[i]->GetSnapshot(_return, dbhandle); + } + } + + Code ReleaseSnapshot(const DBHandle& dbhandle, const Snapshot& snapshot) { + uint32_t i; + uint32_t sz = ifaces_.size(); + for (i = 0; i < sz - 1; ++i) { + ifaces_[i]->ReleaseSnapshot(dbhandle, snapshot); + } + return ifaces_[i]->ReleaseSnapshot(dbhandle, snapshot); + } + + Code CompactRange(const DBHandle& dbhandle, const Slice& begin, const Slice& end) { + uint32_t i; + uint32_t sz = ifaces_.size(); + for (i = 0; i < sz - 1; ++i) { + ifaces_[i]->CompactRange(dbhandle, begin, end); + } + return ifaces_[i]->CompactRange(dbhandle, begin, end); + } + +}; + +} // namespace + +#endif diff --git a/thrift/gen-cpp/leveldb_constants.cpp b/thrift/gen-cpp/leveldb_constants.cpp new file mode 100644 index 0000000000..383e58a48f --- /dev/null +++ b/thrift/gen-cpp/leveldb_constants.cpp @@ -0,0 +1,17 @@ +/** + * Autogenerated by Thrift + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +#include "leveldb_constants.h" + +namespace Tleveldb { + +const leveldbConstants g_leveldb_constants; + +leveldbConstants::leveldbConstants() { +} + +} // namespace + diff --git a/thrift/gen-cpp/leveldb_constants.h b/thrift/gen-cpp/leveldb_constants.h new file mode 100644 index 0000000000..d219f26398 --- /dev/null +++ b/thrift/gen-cpp/leveldb_constants.h @@ -0,0 +1,24 @@ +/** + * Autogenerated by Thrift + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +#ifndef leveldb_CONSTANTS_H +#define leveldb_CONSTANTS_H + +#include "leveldb_types.h" + +namespace Tleveldb { + +class leveldbConstants { + public: + leveldbConstants(); + +}; + +extern const leveldbConstants g_leveldb_constants; + +} // namespace + +#endif diff --git a/thrift/gen-cpp/leveldb_types.cpp b/thrift/gen-cpp/leveldb_types.cpp new file mode 100644 index 0000000000..e8f3979479 --- /dev/null +++ b/thrift/gen-cpp/leveldb_types.cpp @@ -0,0 +1,1656 @@ +/** + * Autogenerated by Thrift + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +#include "leveldb_types.h" + +#include +#include +#include + +namespace Tleveldb { + +int _kCompressionTypeValues[] = { + kNoCompression, + kSnappyCompression +}; + +const char* _kCompressionTypeNames[] = { + "kNoCompression", + "kSnappyCompression" +}; + +const std::map _CompressionType_VALUES_TO_NAMES(apache::thrift::TEnumIterator(2, _kCompressionTypeValues, _kCompressionTypeNames), apache::thrift::TEnumIterator(-1, NULL, NULL)); + +const std::map _CompressionType_NAMES_TO_VALUES(apache::thrift::TEnumInverseIterator(2, _kCompressionTypeValues, _kCompressionTypeNames), apache::thrift::TEnumInverseIterator(-1, NULL, NULL)); + +} // namespace +namespace apache { namespace thrift { +template<> +const char* TEnumTraits< ::Tleveldb::CompressionType>::findName( ::Tleveldb::CompressionType value) { +return findName( ::Tleveldb::_CompressionType_VALUES_TO_NAMES, value); +} + +template<> +bool TEnumTraits< ::Tleveldb::CompressionType>::findValue(const char* name, ::Tleveldb::CompressionType* out) { +return findValue( ::Tleveldb::_CompressionType_NAMES_TO_VALUES, name, out); +} +}} // apache::thrift + +namespace Tleveldb { +int _kCodeValues[] = { + kOk, + kNotFound, + kCorruption, + kNotSupported, + kInvalidArgument, + kIOError, + kEnd +}; + +const char* _kCodeNames[] = { + "kOk", + "kNotFound", + "kCorruption", + "kNotSupported", + "kInvalidArgument", + "kIOError", + "kEnd" +}; + +const std::map _Code_VALUES_TO_NAMES(apache::thrift::TEnumIterator(7, _kCodeValues, _kCodeNames), apache::thrift::TEnumIterator(-1, NULL, NULL)); + +const std::map _Code_NAMES_TO_VALUES(apache::thrift::TEnumInverseIterator(7, _kCodeValues, _kCodeNames), apache::thrift::TEnumInverseIterator(-1, NULL, NULL)); + +} // namespace +namespace apache { namespace thrift { +template<> +const char* TEnumTraits< ::Tleveldb::Code>::findName( ::Tleveldb::Code value) { +return findName( ::Tleveldb::_Code_VALUES_TO_NAMES, value); +} + +template<> +bool TEnumTraits< ::Tleveldb::Code>::findValue(const char* name, ::Tleveldb::Code* out) { +return findValue( ::Tleveldb::_Code_NAMES_TO_VALUES, name, out); +} +}} // apache::thrift + +namespace Tleveldb { +int _kIteratorTypeValues[] = { + seekToFirst, + seekToLast, + seekToKey +}; + +const char* _kIteratorTypeNames[] = { + "seekToFirst", + "seekToLast", + "seekToKey" +}; + +const std::map _IteratorType_VALUES_TO_NAMES(apache::thrift::TEnumIterator(3, _kIteratorTypeValues, _kIteratorTypeNames), apache::thrift::TEnumIterator(-1, NULL, NULL)); + +const std::map _IteratorType_NAMES_TO_VALUES(apache::thrift::TEnumInverseIterator(3, _kIteratorTypeValues, _kIteratorTypeNames), apache::thrift::TEnumInverseIterator(-1, NULL, NULL)); + +} // namespace +namespace apache { namespace thrift { +template<> +const char* TEnumTraits< ::Tleveldb::IteratorType>::findName( ::Tleveldb::IteratorType value) { +return findName( ::Tleveldb::_IteratorType_VALUES_TO_NAMES, value); +} + +template<> +bool TEnumTraits< ::Tleveldb::IteratorType>::findValue(const char* name, ::Tleveldb::IteratorType* out) { +return findValue( ::Tleveldb::_IteratorType_NAMES_TO_VALUES, name, out); +} +}} // apache::thrift + +namespace Tleveldb { +// Reflection initializer for struct leveldb.Slice +namespace { +void reflectionInitializer_2199896239461470156(::apache::thrift::reflection::Schema& schema) { + const uint64_t id = 2199896239461470156U; + if (schema.dataTypes.count(id)) return; + ::apache::thrift::reflection::DataType dt; + dt.name = "struct leveldb.Slice"; + dt.__isset.fields = true; + { + ::apache::thrift::reflection::StructField f; + f.isRequired = true; + f.type = 1U; + f.name = "data"; + dt.fields[1] = f; + } + { + ::apache::thrift::reflection::StructField f; + f.isRequired = true; + f.type = 5U; + f.name = "size"; + dt.fields[2] = f; + } + schema.dataTypes[id] = dt; + schema.names[dt.name] = id; +} +} // namespace + +const uint64_t Slice::_reflection_id; +void Slice::_reflection_register(::apache::thrift::reflection::Schema& schema) { + reflectionInitializer_2199896239461470156(schema); +} +uint32_t Slice::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == apache::thrift::protocol::T_STRING) { + xfer += iprot->readBinary(this->data); + this->__isset.data = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == apache::thrift::protocol::T_I32) { + xfer += iprot->readI32(this->size); + this->__isset.size = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t Slice::write(apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("Slice"); + xfer += oprot->writeFieldBegin("data", apache::thrift::protocol::T_STRING, 1); + xfer += oprot->writeBinary(this->data); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("size", apache::thrift::protocol::T_I32, 2); + xfer += oprot->writeI32(this->size); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +void swap(Slice &a, Slice &b) { + using ::std::swap; + (void)a; + (void)b; + swap(a.data, b.data); + swap(a.size, b.size); + swap(a.__isset, b.__isset); +} + +// Reflection initializer for struct leveldb.Range +namespace { +void reflectionInitializer_6170219570187881516(::apache::thrift::reflection::Schema& schema) { + const uint64_t id = 6170219570187881516U; + if (schema.dataTypes.count(id)) return; + reflectionInitializer_2199896239461470156(schema); // struct leveldb.Slice + ::apache::thrift::reflection::DataType dt; + dt.name = "struct leveldb.Range"; + dt.__isset.fields = true; + { + ::apache::thrift::reflection::StructField f; + f.isRequired = true; + f.type = 2199896239461470156U; + f.name = "start"; + dt.fields[1] = f; + } + { + ::apache::thrift::reflection::StructField f; + f.isRequired = true; + f.type = 2199896239461470156U; + f.name = "limit"; + dt.fields[2] = f; + } + schema.dataTypes[id] = dt; + schema.names[dt.name] = id; +} +} // namespace + +const uint64_t Range::_reflection_id; +void Range::_reflection_register(::apache::thrift::reflection::Schema& schema) { + reflectionInitializer_6170219570187881516(schema); +} +uint32_t Range::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->start.read(iprot); + this->__isset.start = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->limit.read(iprot); + this->__isset.limit = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t Range::write(apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("Range"); + xfer += oprot->writeFieldBegin("start", apache::thrift::protocol::T_STRUCT, 1); + xfer += this->start.write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("limit", apache::thrift::protocol::T_STRUCT, 2); + xfer += this->limit.write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +void swap(Range &a, Range &b) { + using ::std::swap; + (void)a; + (void)b; + swap(a.start, b.start); + swap(a.limit, b.limit); + swap(a.__isset, b.__isset); +} + +// Reflection initializer for enum leveldb.CompressionType +namespace { +void reflectionInitializer_13050266898380890824(::apache::thrift::reflection::Schema& schema) { + const uint64_t id = 13050266898380890824U; + if (schema.dataTypes.count(id)) return; + ::apache::thrift::reflection::DataType dt; + dt.name = "enum leveldb.CompressionType"; + dt.__isset.enumValues = true; + dt.enumValues["kNoCompression"] = 0; + dt.enumValues["kSnappyCompression"] = 1; + schema.dataTypes[id] = dt; + schema.names[dt.name] = id; +} +} // namespace + +// Reflection initializer for struct leveldb.DBOptions +namespace { +void reflectionInitializer_6731746507948871532(::apache::thrift::reflection::Schema& schema) { + const uint64_t id = 6731746507948871532U; + if (schema.dataTypes.count(id)) return; + reflectionInitializer_13050266898380890824(schema); // enum leveldb.CompressionType + ::apache::thrift::reflection::DataType dt; + dt.name = "struct leveldb.DBOptions"; + dt.__isset.fields = true; + { + ::apache::thrift::reflection::StructField f; + f.isRequired = true; + f.type = 2U; + f.name = "create_if_missing"; + dt.fields[1] = f; + } + { + ::apache::thrift::reflection::StructField f; + f.isRequired = true; + f.type = 2U; + f.name = "error_if_exists"; + dt.fields[2] = f; + } + { + ::apache::thrift::reflection::StructField f; + f.isRequired = true; + f.type = 5U; + f.name = "write_buffer_size"; + dt.fields[3] = f; + } + { + ::apache::thrift::reflection::StructField f; + f.isRequired = true; + f.type = 5U; + f.name = "max_open_files"; + dt.fields[4] = f; + } + { + ::apache::thrift::reflection::StructField f; + f.isRequired = true; + f.type = 5U; + f.name = "block_size"; + dt.fields[5] = f; + } + { + ::apache::thrift::reflection::StructField f; + f.isRequired = true; + f.type = 5U; + f.name = "block_restart_interval"; + dt.fields[6] = f; + } + { + ::apache::thrift::reflection::StructField f; + f.isRequired = true; + f.type = 13050266898380890824U; + f.name = "compression"; + dt.fields[7] = f; + } + schema.dataTypes[id] = dt; + schema.names[dt.name] = id; +} +} // namespace + +const uint64_t DBOptions::_reflection_id; +void DBOptions::_reflection_register(::apache::thrift::reflection::Schema& schema) { + reflectionInitializer_6731746507948871532(schema); +} +uint32_t DBOptions::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == apache::thrift::protocol::T_BOOL) { + xfer += iprot->readBool(this->create_if_missing); + this->__isset.create_if_missing = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == apache::thrift::protocol::T_BOOL) { + xfer += iprot->readBool(this->error_if_exists); + this->__isset.error_if_exists = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 3: + if (ftype == apache::thrift::protocol::T_I32) { + xfer += iprot->readI32(this->write_buffer_size); + this->__isset.write_buffer_size = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 4: + if (ftype == apache::thrift::protocol::T_I32) { + xfer += iprot->readI32(this->max_open_files); + this->__isset.max_open_files = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 5: + if (ftype == apache::thrift::protocol::T_I32) { + xfer += iprot->readI32(this->block_size); + this->__isset.block_size = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 6: + if (ftype == apache::thrift::protocol::T_I32) { + xfer += iprot->readI32(this->block_restart_interval); + this->__isset.block_restart_interval = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 7: + if (ftype == apache::thrift::protocol::T_I32) { + int32_t ecast0; + xfer += iprot->readI32(ecast0); + this->compression = (CompressionType)ecast0; + this->__isset.compression = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t DBOptions::write(apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("DBOptions"); + xfer += oprot->writeFieldBegin("create_if_missing", apache::thrift::protocol::T_BOOL, 1); + xfer += oprot->writeBool(this->create_if_missing); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("error_if_exists", apache::thrift::protocol::T_BOOL, 2); + xfer += oprot->writeBool(this->error_if_exists); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("write_buffer_size", apache::thrift::protocol::T_I32, 3); + xfer += oprot->writeI32(this->write_buffer_size); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("max_open_files", apache::thrift::protocol::T_I32, 4); + xfer += oprot->writeI32(this->max_open_files); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("block_size", apache::thrift::protocol::T_I32, 5); + xfer += oprot->writeI32(this->block_size); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("block_restart_interval", apache::thrift::protocol::T_I32, 6); + xfer += oprot->writeI32(this->block_restart_interval); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("compression", apache::thrift::protocol::T_I32, 7); + xfer += oprot->writeI32((int32_t)this->compression); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +void swap(DBOptions &a, DBOptions &b) { + using ::std::swap; + (void)a; + (void)b; + swap(a.create_if_missing, b.create_if_missing); + swap(a.error_if_exists, b.error_if_exists); + swap(a.write_buffer_size, b.write_buffer_size); + swap(a.max_open_files, b.max_open_files); + swap(a.block_size, b.block_size); + swap(a.block_restart_interval, b.block_restart_interval); + swap(a.compression, b.compression); + swap(a.__isset, b.__isset); +} + +// Reflection initializer for struct leveldb.WriteOptions +namespace { +void reflectionInitializer_8830325115029814540(::apache::thrift::reflection::Schema& schema) { + const uint64_t id = 8830325115029814540U; + if (schema.dataTypes.count(id)) return; + ::apache::thrift::reflection::DataType dt; + dt.name = "struct leveldb.WriteOptions"; + dt.__isset.fields = true; + { + ::apache::thrift::reflection::StructField f; + f.isRequired = true; + f.type = 2U; + f.name = "sync"; + dt.fields[1] = f; + } + schema.dataTypes[id] = dt; + schema.names[dt.name] = id; +} +} // namespace + +const uint64_t WriteOptions::_reflection_id; +void WriteOptions::_reflection_register(::apache::thrift::reflection::Schema& schema) { + reflectionInitializer_8830325115029814540(schema); +} +uint32_t WriteOptions::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == apache::thrift::protocol::T_BOOL) { + xfer += iprot->readBool(this->sync); + this->__isset.sync = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t WriteOptions::write(apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("WriteOptions"); + xfer += oprot->writeFieldBegin("sync", apache::thrift::protocol::T_BOOL, 1); + xfer += oprot->writeBool(this->sync); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +void swap(WriteOptions &a, WriteOptions &b) { + using ::std::swap; + (void)a; + (void)b; + swap(a.sync, b.sync); + swap(a.__isset, b.__isset); +} + +// Reflection initializer for struct leveldb.Snapshot +namespace { +void reflectionInitializer_15670548806217660204(::apache::thrift::reflection::Schema& schema) { + const uint64_t id = 15670548806217660204U; + if (schema.dataTypes.count(id)) return; + ::apache::thrift::reflection::DataType dt; + dt.name = "struct leveldb.Snapshot"; + dt.__isset.fields = true; + { + ::apache::thrift::reflection::StructField f; + f.isRequired = true; + f.type = 6U; + f.name = "snapshotid"; + dt.fields[1] = f; + } + schema.dataTypes[id] = dt; + schema.names[dt.name] = id; +} +} // namespace + +const uint64_t Snapshot::_reflection_id; +void Snapshot::_reflection_register(::apache::thrift::reflection::Schema& schema) { + reflectionInitializer_15670548806217660204(schema); +} +uint32_t Snapshot::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == apache::thrift::protocol::T_I64) { + xfer += iprot->readI64(this->snapshotid); + this->__isset.snapshotid = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t Snapshot::write(apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("Snapshot"); + xfer += oprot->writeFieldBegin("snapshotid", apache::thrift::protocol::T_I64, 1); + xfer += oprot->writeI64(this->snapshotid); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +void swap(Snapshot &a, Snapshot &b) { + using ::std::swap; + (void)a; + (void)b; + swap(a.snapshotid, b.snapshotid); + swap(a.__isset, b.__isset); +} + +// Reflection initializer for struct leveldb.ReadOptions +namespace { +void reflectionInitializer_1092669993626789804(::apache::thrift::reflection::Schema& schema) { + const uint64_t id = 1092669993626789804U; + if (schema.dataTypes.count(id)) return; + reflectionInitializer_15670548806217660204(schema); // struct leveldb.Snapshot + ::apache::thrift::reflection::DataType dt; + dt.name = "struct leveldb.ReadOptions"; + dt.__isset.fields = true; + { + ::apache::thrift::reflection::StructField f; + f.isRequired = true; + f.type = 2U; + f.name = "verify_checksums"; + dt.fields[1] = f; + } + { + ::apache::thrift::reflection::StructField f; + f.isRequired = true; + f.type = 2U; + f.name = "fill_cache"; + dt.fields[2] = f; + } + { + ::apache::thrift::reflection::StructField f; + f.isRequired = true; + f.type = 15670548806217660204U; + f.name = "snapshot"; + dt.fields[3] = f; + } + schema.dataTypes[id] = dt; + schema.names[dt.name] = id; +} +} // namespace + +const uint64_t ReadOptions::_reflection_id; +void ReadOptions::_reflection_register(::apache::thrift::reflection::Schema& schema) { + reflectionInitializer_1092669993626789804(schema); +} +uint32_t ReadOptions::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == apache::thrift::protocol::T_BOOL) { + xfer += iprot->readBool(this->verify_checksums); + this->__isset.verify_checksums = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == apache::thrift::protocol::T_BOOL) { + xfer += iprot->readBool(this->fill_cache); + this->__isset.fill_cache = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 3: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->snapshot.read(iprot); + this->__isset.snapshot = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t ReadOptions::write(apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("ReadOptions"); + xfer += oprot->writeFieldBegin("verify_checksums", apache::thrift::protocol::T_BOOL, 1); + xfer += oprot->writeBool(this->verify_checksums); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("fill_cache", apache::thrift::protocol::T_BOOL, 2); + xfer += oprot->writeBool(this->fill_cache); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("snapshot", apache::thrift::protocol::T_STRUCT, 3); + xfer += this->snapshot.write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +void swap(ReadOptions &a, ReadOptions &b) { + using ::std::swap; + (void)a; + (void)b; + swap(a.verify_checksums, b.verify_checksums); + swap(a.fill_cache, b.fill_cache); + swap(a.snapshot, b.snapshot); + swap(a.__isset, b.__isset); +} + +// Reflection initializer for struct leveldb.DBHandle +namespace { +void reflectionInitializer_8973827971994157004(::apache::thrift::reflection::Schema& schema) { + const uint64_t id = 8973827971994157004U; + if (schema.dataTypes.count(id)) return; + ::apache::thrift::reflection::DataType dt; + dt.name = "struct leveldb.DBHandle"; + dt.__isset.fields = true; + { + ::apache::thrift::reflection::StructField f; + f.isRequired = true; + f.type = 1U; + f.name = "dbname"; + dt.fields[1] = f; + } + { + ::apache::thrift::reflection::StructField f; + f.isRequired = true; + f.type = 6U; + f.name = "handleid"; + dt.fields[2] = f; + } + schema.dataTypes[id] = dt; + schema.names[dt.name] = id; +} +} // namespace + +const uint64_t DBHandle::_reflection_id; +void DBHandle::_reflection_register(::apache::thrift::reflection::Schema& schema) { + reflectionInitializer_8973827971994157004(schema); +} +uint32_t DBHandle::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == apache::thrift::protocol::T_STRING) { + xfer += iprot->readBinary(this->dbname); + this->__isset.dbname = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == apache::thrift::protocol::T_I64) { + xfer += iprot->readI64(this->handleid); + this->__isset.handleid = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t DBHandle::write(apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("DBHandle"); + xfer += oprot->writeFieldBegin("dbname", apache::thrift::protocol::T_STRING, 1); + xfer += oprot->writeBinary(this->dbname); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("handleid", apache::thrift::protocol::T_I64, 2); + xfer += oprot->writeI64(this->handleid); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +void swap(DBHandle &a, DBHandle &b) { + using ::std::swap; + (void)a; + (void)b; + swap(a.dbname, b.dbname); + swap(a.handleid, b.handleid); + swap(a.__isset, b.__isset); +} + +// Reflection initializer for struct leveldb.Iterator +namespace { +void reflectionInitializer_11184146435197093740(::apache::thrift::reflection::Schema& schema) { + const uint64_t id = 11184146435197093740U; + if (schema.dataTypes.count(id)) return; + ::apache::thrift::reflection::DataType dt; + dt.name = "struct leveldb.Iterator"; + dt.__isset.fields = true; + { + ::apache::thrift::reflection::StructField f; + f.isRequired = true; + f.type = 6U; + f.name = "iteratorid"; + dt.fields[1] = f; + } + schema.dataTypes[id] = dt; + schema.names[dt.name] = id; +} +} // namespace + +const uint64_t Iterator::_reflection_id; +void Iterator::_reflection_register(::apache::thrift::reflection::Schema& schema) { + reflectionInitializer_11184146435197093740(schema); +} +uint32_t Iterator::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == apache::thrift::protocol::T_I64) { + xfer += iprot->readI64(this->iteratorid); + this->__isset.iteratorid = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t Iterator::write(apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("Iterator"); + xfer += oprot->writeFieldBegin("iteratorid", apache::thrift::protocol::T_I64, 1); + xfer += oprot->writeI64(this->iteratorid); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +void swap(Iterator &a, Iterator &b) { + using ::std::swap; + (void)a; + (void)b; + swap(a.iteratorid, b.iteratorid); + swap(a.__isset, b.__isset); +} + +// Reflection initializer for struct leveldb.kv +namespace { +void reflectionInitializer_16082992224095104076(::apache::thrift::reflection::Schema& schema) { + const uint64_t id = 16082992224095104076U; + if (schema.dataTypes.count(id)) return; + reflectionInitializer_2199896239461470156(schema); // struct leveldb.Slice + ::apache::thrift::reflection::DataType dt; + dt.name = "struct leveldb.kv"; + dt.__isset.fields = true; + { + ::apache::thrift::reflection::StructField f; + f.isRequired = true; + f.type = 2199896239461470156U; + f.name = "key"; + dt.fields[1] = f; + } + { + ::apache::thrift::reflection::StructField f; + f.isRequired = true; + f.type = 2199896239461470156U; + f.name = "value"; + dt.fields[2] = f; + } + schema.dataTypes[id] = dt; + schema.names[dt.name] = id; +} +} // namespace + +const uint64_t kv::_reflection_id; +void kv::_reflection_register(::apache::thrift::reflection::Schema& schema) { + reflectionInitializer_16082992224095104076(schema); +} +uint32_t kv::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->key.read(iprot); + this->__isset.key = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->value.read(iprot); + this->__isset.value = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t kv::write(apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("kv"); + xfer += oprot->writeFieldBegin("key", apache::thrift::protocol::T_STRUCT, 1); + xfer += this->key.write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("value", apache::thrift::protocol::T_STRUCT, 2); + xfer += this->value.write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +void swap(kv &a, kv &b) { + using ::std::swap; + (void)a; + (void)b; + swap(a.key, b.key); + swap(a.value, b.value); + swap(a.__isset, b.__isset); +} + +// Reflection initializer for enum leveldb.Code +namespace { +void reflectionInitializer_13230961887206242376(::apache::thrift::reflection::Schema& schema) { + const uint64_t id = 13230961887206242376U; + if (schema.dataTypes.count(id)) return; + ::apache::thrift::reflection::DataType dt; + dt.name = "enum leveldb.Code"; + dt.__isset.enumValues = true; + dt.enumValues["kCorruption"] = 2; + dt.enumValues["kEnd"] = 6; + dt.enumValues["kIOError"] = 5; + dt.enumValues["kInvalidArgument"] = 4; + dt.enumValues["kNotFound"] = 1; + dt.enumValues["kNotSupported"] = 3; + dt.enumValues["kOk"] = 0; + schema.dataTypes[id] = dt; + schema.names[dt.name] = id; +} +} // namespace + +// Reflection initializer for struct leveldb.ResultItem +namespace { +void reflectionInitializer_13211316281207238796(::apache::thrift::reflection::Schema& schema) { + const uint64_t id = 13211316281207238796U; + if (schema.dataTypes.count(id)) return; + reflectionInitializer_13230961887206242376(schema); // enum leveldb.Code + reflectionInitializer_2199896239461470156(schema); // struct leveldb.Slice + ::apache::thrift::reflection::DataType dt; + dt.name = "struct leveldb.ResultItem"; + dt.__isset.fields = true; + { + ::apache::thrift::reflection::StructField f; + f.isRequired = true; + f.type = 13230961887206242376U; + f.name = "status"; + dt.fields[1] = f; + } + { + ::apache::thrift::reflection::StructField f; + f.isRequired = true; + f.type = 2199896239461470156U; + f.name = "value"; + dt.fields[2] = f; + } + schema.dataTypes[id] = dt; + schema.names[dt.name] = id; +} +} // namespace + +const uint64_t ResultItem::_reflection_id; +void ResultItem::_reflection_register(::apache::thrift::reflection::Schema& schema) { + reflectionInitializer_13211316281207238796(schema); +} +uint32_t ResultItem::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == apache::thrift::protocol::T_I32) { + int32_t ecast1; + xfer += iprot->readI32(ecast1); + this->status = (Code)ecast1; + this->__isset.status = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->value.read(iprot); + this->__isset.value = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t ResultItem::write(apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("ResultItem"); + xfer += oprot->writeFieldBegin("status", apache::thrift::protocol::T_I32, 1); + xfer += oprot->writeI32((int32_t)this->status); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("value", apache::thrift::protocol::T_STRUCT, 2); + xfer += this->value.write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +void swap(ResultItem &a, ResultItem &b) { + using ::std::swap; + (void)a; + (void)b; + swap(a.status, b.status); + swap(a.value, b.value); + swap(a.__isset, b.__isset); +} + +// Reflection initializer for struct leveldb.ResultPair +namespace { +void reflectionInitializer_14875242256166808460(::apache::thrift::reflection::Schema& schema) { + const uint64_t id = 14875242256166808460U; + if (schema.dataTypes.count(id)) return; + reflectionInitializer_13230961887206242376(schema); // enum leveldb.Code + reflectionInitializer_16082992224095104076(schema); // struct leveldb.kv + ::apache::thrift::reflection::DataType dt; + dt.name = "struct leveldb.ResultPair"; + dt.__isset.fields = true; + { + ::apache::thrift::reflection::StructField f; + f.isRequired = true; + f.type = 13230961887206242376U; + f.name = "status"; + dt.fields[1] = f; + } + { + ::apache::thrift::reflection::StructField f; + f.isRequired = true; + f.type = 16082992224095104076U; + f.name = "keyvalue"; + dt.fields[2] = f; + } + schema.dataTypes[id] = dt; + schema.names[dt.name] = id; +} +} // namespace + +const uint64_t ResultPair::_reflection_id; +void ResultPair::_reflection_register(::apache::thrift::reflection::Schema& schema) { + reflectionInitializer_14875242256166808460(schema); +} +uint32_t ResultPair::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == apache::thrift::protocol::T_I32) { + int32_t ecast2; + xfer += iprot->readI32(ecast2); + this->status = (Code)ecast2; + this->__isset.status = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->keyvalue.read(iprot); + this->__isset.keyvalue = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t ResultPair::write(apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("ResultPair"); + xfer += oprot->writeFieldBegin("status", apache::thrift::protocol::T_I32, 1); + xfer += oprot->writeI32((int32_t)this->status); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("keyvalue", apache::thrift::protocol::T_STRUCT, 2); + xfer += this->keyvalue.write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +void swap(ResultPair &a, ResultPair &b) { + using ::std::swap; + (void)a; + (void)b; + swap(a.status, b.status); + swap(a.keyvalue, b.keyvalue); + swap(a.__isset, b.__isset); +} + +// Reflection initializer for struct leveldb.ResultSnapshot +namespace { +void reflectionInitializer_16627180600575569004(::apache::thrift::reflection::Schema& schema) { + const uint64_t id = 16627180600575569004U; + if (schema.dataTypes.count(id)) return; + reflectionInitializer_13230961887206242376(schema); // enum leveldb.Code + reflectionInitializer_15670548806217660204(schema); // struct leveldb.Snapshot + ::apache::thrift::reflection::DataType dt; + dt.name = "struct leveldb.ResultSnapshot"; + dt.__isset.fields = true; + { + ::apache::thrift::reflection::StructField f; + f.isRequired = true; + f.type = 13230961887206242376U; + f.name = "status"; + dt.fields[1] = f; + } + { + ::apache::thrift::reflection::StructField f; + f.isRequired = true; + f.type = 15670548806217660204U; + f.name = "snapshot"; + dt.fields[2] = f; + } + schema.dataTypes[id] = dt; + schema.names[dt.name] = id; +} +} // namespace + +const uint64_t ResultSnapshot::_reflection_id; +void ResultSnapshot::_reflection_register(::apache::thrift::reflection::Schema& schema) { + reflectionInitializer_16627180600575569004(schema); +} +uint32_t ResultSnapshot::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == apache::thrift::protocol::T_I32) { + int32_t ecast3; + xfer += iprot->readI32(ecast3); + this->status = (Code)ecast3; + this->__isset.status = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->snapshot.read(iprot); + this->__isset.snapshot = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t ResultSnapshot::write(apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("ResultSnapshot"); + xfer += oprot->writeFieldBegin("status", apache::thrift::protocol::T_I32, 1); + xfer += oprot->writeI32((int32_t)this->status); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("snapshot", apache::thrift::protocol::T_STRUCT, 2); + xfer += this->snapshot.write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +void swap(ResultSnapshot &a, ResultSnapshot &b) { + using ::std::swap; + (void)a; + (void)b; + swap(a.status, b.status); + swap(a.snapshot, b.snapshot); + swap(a.__isset, b.__isset); +} + +// Reflection initializer for struct leveldb.ResultIterator +namespace { +void reflectionInitializer_595886977232564460(::apache::thrift::reflection::Schema& schema) { + const uint64_t id = 595886977232564460U; + if (schema.dataTypes.count(id)) return; + reflectionInitializer_11184146435197093740(schema); // struct leveldb.Iterator + reflectionInitializer_13230961887206242376(schema); // enum leveldb.Code + ::apache::thrift::reflection::DataType dt; + dt.name = "struct leveldb.ResultIterator"; + dt.__isset.fields = true; + { + ::apache::thrift::reflection::StructField f; + f.isRequired = true; + f.type = 13230961887206242376U; + f.name = "status"; + dt.fields[1] = f; + } + { + ::apache::thrift::reflection::StructField f; + f.isRequired = true; + f.type = 11184146435197093740U; + f.name = "iterator"; + dt.fields[2] = f; + } + schema.dataTypes[id] = dt; + schema.names[dt.name] = id; +} +} // namespace + +const uint64_t ResultIterator::_reflection_id; +void ResultIterator::_reflection_register(::apache::thrift::reflection::Schema& schema) { + reflectionInitializer_595886977232564460(schema); +} +uint32_t ResultIterator::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == apache::thrift::protocol::T_I32) { + int32_t ecast4; + xfer += iprot->readI32(ecast4); + this->status = (Code)ecast4; + this->__isset.status = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == apache::thrift::protocol::T_STRUCT) { + xfer += this->iterator.read(iprot); + this->__isset.iterator = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t ResultIterator::write(apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("ResultIterator"); + xfer += oprot->writeFieldBegin("status", apache::thrift::protocol::T_I32, 1); + xfer += oprot->writeI32((int32_t)this->status); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("iterator", apache::thrift::protocol::T_STRUCT, 2); + xfer += this->iterator.write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +void swap(ResultIterator &a, ResultIterator &b) { + using ::std::swap; + (void)a; + (void)b; + swap(a.status, b.status); + swap(a.iterator, b.iterator); + swap(a.__isset, b.__isset); +} + +// Reflection initializer for struct leveldb.LeveldbException +namespace { +void reflectionInitializer_2551220192341843436(::apache::thrift::reflection::Schema& schema) { + const uint64_t id = 2551220192341843436U; + if (schema.dataTypes.count(id)) return; + reflectionInitializer_13230961887206242376(schema); // enum leveldb.Code + ::apache::thrift::reflection::DataType dt; + dt.name = "struct leveldb.LeveldbException"; + dt.__isset.fields = true; + { + ::apache::thrift::reflection::StructField f; + f.isRequired = true; + f.type = 1U; + f.name = "message"; + dt.fields[1] = f; + } + { + ::apache::thrift::reflection::StructField f; + f.isRequired = true; + f.type = 13230961887206242376U; + f.name = "errorCode"; + dt.fields[2] = f; + } + schema.dataTypes[id] = dt; + schema.names[dt.name] = id; +} +} // namespace + +const uint64_t LeveldbException::_reflection_id; +void LeveldbException::_reflection_register(::apache::thrift::reflection::Schema& schema) { + reflectionInitializer_2551220192341843436(schema); +} +uint32_t LeveldbException::read(apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == apache::thrift::protocol::T_STRING) { + xfer += iprot->readBinary(this->message); + this->__isset.message = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == apache::thrift::protocol::T_I32) { + int32_t ecast5; + xfer += iprot->readI32(ecast5); + this->errorCode = (Code)ecast5; + this->__isset.errorCode = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t LeveldbException::write(apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("LeveldbException"); + xfer += oprot->writeFieldBegin("message", apache::thrift::protocol::T_STRING, 1); + xfer += oprot->writeBinary(this->message); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("errorCode", apache::thrift::protocol::T_I32, 2); + xfer += oprot->writeI32((int32_t)this->errorCode); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +void swap(LeveldbException &a, LeveldbException &b) { + using ::std::swap; + (void)a; + (void)b; + swap(a.message, b.message); + swap(a.errorCode, b.errorCode); + swap(a.__isset, b.__isset); +} + +} // namespace diff --git a/thrift/gen-cpp/leveldb_types.h b/thrift/gen-cpp/leveldb_types.h new file mode 100644 index 0000000000..1756bcb08f --- /dev/null +++ b/thrift/gen-cpp/leveldb_types.h @@ -0,0 +1,867 @@ +/** + * Autogenerated by Thrift + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +#ifndef leveldb_TYPES_H +#define leveldb_TYPES_H + +#include +#include +#include +#include + +namespace apache { namespace thrift { namespace reflection { +class Schema; +}}} + + +namespace Tleveldb { + +enum CompressionType { + kNoCompression = 0, + kSnappyCompression = 1 +}; + +extern const std::map _CompressionType_VALUES_TO_NAMES; + +extern const std::map _CompressionType_NAMES_TO_VALUES; + +enum Code { + kOk = 0, + kNotFound = 1, + kCorruption = 2, + kNotSupported = 3, + kInvalidArgument = 4, + kIOError = 5, + kEnd = 6 +}; + +extern const std::map _Code_VALUES_TO_NAMES; + +extern const std::map _Code_NAMES_TO_VALUES; + +enum IteratorType { + seekToFirst = 0, + seekToLast = 1, + seekToKey = 2 +}; + +extern const std::map _IteratorType_VALUES_TO_NAMES; + +extern const std::map _IteratorType_NAMES_TO_VALUES; + +typedef std::string Text; + +typedef std::string Bytes; + +class Slice { + public: + + static const uint64_t _reflection_id = 2199896239461470156U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + Slice() : data(""), size(0) { + } + + Slice(const Slice&) = default; + Slice& operator=(const Slice&) = default; + Slice(Slice&&) = default; + Slice& operator=(Slice&&) = default; + + void __clear() { + data = ""; + size = 0; + __isset.__clear(); + } + + virtual ~Slice() throw() {} + + Text data; + int32_t size; + + struct __isset { + __isset() { __clear(); } + void __clear() { + data = false; + size = false; + } + bool data; + bool size; + } __isset; + + bool operator == (const Slice & rhs) const + { + if (!(this->data == rhs.data)) + return false; + if (!(this->size == rhs.size)) + return false; + return true; + } + bool operator != (const Slice &rhs) const { + return !(*this == rhs); + } + + bool operator < (const Slice & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class Slice; +void swap(Slice &a, Slice &b); + +class Range { + public: + + static const uint64_t _reflection_id = 6170219570187881516U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + Range() { + } + + Range(const Range&) = default; + Range& operator=(const Range&) = default; + Range(Range&&) = default; + Range& operator=(Range&&) = default; + + void __clear() { + start.__clear(); + limit.__clear(); + __isset.__clear(); + } + + virtual ~Range() throw() {} + + Slice start; + Slice limit; + + struct __isset { + __isset() { __clear(); } + void __clear() { + start = false; + limit = false; + } + bool start; + bool limit; + } __isset; + + bool operator == (const Range & rhs) const + { + if (!(this->start == rhs.start)) + return false; + if (!(this->limit == rhs.limit)) + return false; + return true; + } + bool operator != (const Range &rhs) const { + return !(*this == rhs); + } + + bool operator < (const Range & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class Range; +void swap(Range &a, Range &b); + +class DBOptions { + public: + + static const uint64_t _reflection_id = 6731746507948871532U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + DBOptions() : create_if_missing(0), error_if_exists(0), write_buffer_size(0), max_open_files(0), block_size(0), block_restart_interval(0), compression(static_cast(0)) { + } + + DBOptions(const DBOptions&) = default; + DBOptions& operator=(const DBOptions&) = default; + DBOptions(DBOptions&&) = default; + DBOptions& operator=(DBOptions&&) = default; + + void __clear() { + create_if_missing = 0; + error_if_exists = 0; + write_buffer_size = 0; + max_open_files = 0; + block_size = 0; + block_restart_interval = 0; + compression = static_cast(0); + __isset.__clear(); + } + + virtual ~DBOptions() throw() {} + + bool create_if_missing; + bool error_if_exists; + int32_t write_buffer_size; + int32_t max_open_files; + int32_t block_size; + int32_t block_restart_interval; + CompressionType compression; + + struct __isset { + __isset() { __clear(); } + void __clear() { + create_if_missing = false; + error_if_exists = false; + write_buffer_size = false; + max_open_files = false; + block_size = false; + block_restart_interval = false; + compression = false; + } + bool create_if_missing; + bool error_if_exists; + bool write_buffer_size; + bool max_open_files; + bool block_size; + bool block_restart_interval; + bool compression; + } __isset; + + bool operator == (const DBOptions & rhs) const + { + if (!(this->create_if_missing == rhs.create_if_missing)) + return false; + if (!(this->error_if_exists == rhs.error_if_exists)) + return false; + if (!(this->write_buffer_size == rhs.write_buffer_size)) + return false; + if (!(this->max_open_files == rhs.max_open_files)) + return false; + if (!(this->block_size == rhs.block_size)) + return false; + if (!(this->block_restart_interval == rhs.block_restart_interval)) + return false; + if (!(this->compression == rhs.compression)) + return false; + return true; + } + bool operator != (const DBOptions &rhs) const { + return !(*this == rhs); + } + + bool operator < (const DBOptions & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class DBOptions; +void swap(DBOptions &a, DBOptions &b); + +class WriteOptions { + public: + + static const uint64_t _reflection_id = 8830325115029814540U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + WriteOptions() : sync(0) { + } + + WriteOptions(const WriteOptions&) = default; + WriteOptions& operator=(const WriteOptions&) = default; + WriteOptions(WriteOptions&&) = default; + WriteOptions& operator=(WriteOptions&&) = default; + + void __clear() { + sync = 0; + __isset.__clear(); + } + + virtual ~WriteOptions() throw() {} + + bool sync; + + struct __isset { + __isset() { __clear(); } + void __clear() { + sync = false; + } + bool sync; + } __isset; + + bool operator == (const WriteOptions & rhs) const + { + if (!(this->sync == rhs.sync)) + return false; + return true; + } + bool operator != (const WriteOptions &rhs) const { + return !(*this == rhs); + } + + bool operator < (const WriteOptions & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class WriteOptions; +void swap(WriteOptions &a, WriteOptions &b); + +class Snapshot { + public: + + static const uint64_t _reflection_id = 15670548806217660204U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + Snapshot() : snapshotid(0) { + } + + Snapshot(const Snapshot&) = default; + Snapshot& operator=(const Snapshot&) = default; + Snapshot(Snapshot&&) = default; + Snapshot& operator=(Snapshot&&) = default; + + void __clear() { + snapshotid = 0; + __isset.__clear(); + } + + virtual ~Snapshot() throw() {} + + int64_t snapshotid; + + struct __isset { + __isset() { __clear(); } + void __clear() { + snapshotid = false; + } + bool snapshotid; + } __isset; + + bool operator == (const Snapshot & rhs) const + { + if (!(this->snapshotid == rhs.snapshotid)) + return false; + return true; + } + bool operator != (const Snapshot &rhs) const { + return !(*this == rhs); + } + + bool operator < (const Snapshot & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class Snapshot; +void swap(Snapshot &a, Snapshot &b); + +class ReadOptions { + public: + + static const uint64_t _reflection_id = 1092669993626789804U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + ReadOptions() : verify_checksums(0), fill_cache(0) { + } + + ReadOptions(const ReadOptions&) = default; + ReadOptions& operator=(const ReadOptions&) = default; + ReadOptions(ReadOptions&&) = default; + ReadOptions& operator=(ReadOptions&&) = default; + + void __clear() { + verify_checksums = 0; + fill_cache = 0; + snapshot.__clear(); + __isset.__clear(); + } + + virtual ~ReadOptions() throw() {} + + bool verify_checksums; + bool fill_cache; + Snapshot snapshot; + + struct __isset { + __isset() { __clear(); } + void __clear() { + verify_checksums = false; + fill_cache = false; + snapshot = false; + } + bool verify_checksums; + bool fill_cache; + bool snapshot; + } __isset; + + bool operator == (const ReadOptions & rhs) const + { + if (!(this->verify_checksums == rhs.verify_checksums)) + return false; + if (!(this->fill_cache == rhs.fill_cache)) + return false; + if (!(this->snapshot == rhs.snapshot)) + return false; + return true; + } + bool operator != (const ReadOptions &rhs) const { + return !(*this == rhs); + } + + bool operator < (const ReadOptions & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class ReadOptions; +void swap(ReadOptions &a, ReadOptions &b); + +class DBHandle { + public: + + static const uint64_t _reflection_id = 8973827971994157004U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + DBHandle() : dbname(""), handleid(0) { + } + + DBHandle(const DBHandle&) = default; + DBHandle& operator=(const DBHandle&) = default; + DBHandle(DBHandle&&) = default; + DBHandle& operator=(DBHandle&&) = default; + + void __clear() { + dbname = ""; + handleid = 0; + __isset.__clear(); + } + + virtual ~DBHandle() throw() {} + + Text dbname; + int64_t handleid; + + struct __isset { + __isset() { __clear(); } + void __clear() { + dbname = false; + handleid = false; + } + bool dbname; + bool handleid; + } __isset; + + bool operator == (const DBHandle & rhs) const + { + if (!(this->dbname == rhs.dbname)) + return false; + if (!(this->handleid == rhs.handleid)) + return false; + return true; + } + bool operator != (const DBHandle &rhs) const { + return !(*this == rhs); + } + + bool operator < (const DBHandle & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class DBHandle; +void swap(DBHandle &a, DBHandle &b); + +class Iterator { + public: + + static const uint64_t _reflection_id = 11184146435197093740U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + Iterator() : iteratorid(0) { + } + + Iterator(const Iterator&) = default; + Iterator& operator=(const Iterator&) = default; + Iterator(Iterator&&) = default; + Iterator& operator=(Iterator&&) = default; + + void __clear() { + iteratorid = 0; + __isset.__clear(); + } + + virtual ~Iterator() throw() {} + + int64_t iteratorid; + + struct __isset { + __isset() { __clear(); } + void __clear() { + iteratorid = false; + } + bool iteratorid; + } __isset; + + bool operator == (const Iterator & rhs) const + { + if (!(this->iteratorid == rhs.iteratorid)) + return false; + return true; + } + bool operator != (const Iterator &rhs) const { + return !(*this == rhs); + } + + bool operator < (const Iterator & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class Iterator; +void swap(Iterator &a, Iterator &b); + +class kv { + public: + + static const uint64_t _reflection_id = 16082992224095104076U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + kv() { + } + + kv(const kv&) = default; + kv& operator=(const kv&) = default; + kv(kv&&) = default; + kv& operator=(kv&&) = default; + + void __clear() { + key.__clear(); + value.__clear(); + __isset.__clear(); + } + + virtual ~kv() throw() {} + + Slice key; + Slice value; + + struct __isset { + __isset() { __clear(); } + void __clear() { + key = false; + value = false; + } + bool key; + bool value; + } __isset; + + bool operator == (const kv & rhs) const + { + if (!(this->key == rhs.key)) + return false; + if (!(this->value == rhs.value)) + return false; + return true; + } + bool operator != (const kv &rhs) const { + return !(*this == rhs); + } + + bool operator < (const kv & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class kv; +void swap(kv &a, kv &b); + +class ResultItem { + public: + + static const uint64_t _reflection_id = 13211316281207238796U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + ResultItem() : status(static_cast(0)) { + } + + ResultItem(const ResultItem&) = default; + ResultItem& operator=(const ResultItem&) = default; + ResultItem(ResultItem&&) = default; + ResultItem& operator=(ResultItem&&) = default; + + void __clear() { + status = static_cast(0); + value.__clear(); + __isset.__clear(); + } + + virtual ~ResultItem() throw() {} + + Code status; + Slice value; + + struct __isset { + __isset() { __clear(); } + void __clear() { + status = false; + value = false; + } + bool status; + bool value; + } __isset; + + bool operator == (const ResultItem & rhs) const + { + if (!(this->status == rhs.status)) + return false; + if (!(this->value == rhs.value)) + return false; + return true; + } + bool operator != (const ResultItem &rhs) const { + return !(*this == rhs); + } + + bool operator < (const ResultItem & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class ResultItem; +void swap(ResultItem &a, ResultItem &b); + +class ResultPair { + public: + + static const uint64_t _reflection_id = 14875242256166808460U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + ResultPair() : status(static_cast(0)) { + } + + ResultPair(const ResultPair&) = default; + ResultPair& operator=(const ResultPair&) = default; + ResultPair(ResultPair&&) = default; + ResultPair& operator=(ResultPair&&) = default; + + void __clear() { + status = static_cast(0); + keyvalue.__clear(); + __isset.__clear(); + } + + virtual ~ResultPair() throw() {} + + Code status; + kv keyvalue; + + struct __isset { + __isset() { __clear(); } + void __clear() { + status = false; + keyvalue = false; + } + bool status; + bool keyvalue; + } __isset; + + bool operator == (const ResultPair & rhs) const + { + if (!(this->status == rhs.status)) + return false; + if (!(this->keyvalue == rhs.keyvalue)) + return false; + return true; + } + bool operator != (const ResultPair &rhs) const { + return !(*this == rhs); + } + + bool operator < (const ResultPair & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class ResultPair; +void swap(ResultPair &a, ResultPair &b); + +class ResultSnapshot { + public: + + static const uint64_t _reflection_id = 16627180600575569004U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + ResultSnapshot() : status(static_cast(0)) { + } + + ResultSnapshot(const ResultSnapshot&) = default; + ResultSnapshot& operator=(const ResultSnapshot&) = default; + ResultSnapshot(ResultSnapshot&&) = default; + ResultSnapshot& operator=(ResultSnapshot&&) = default; + + void __clear() { + status = static_cast(0); + snapshot.__clear(); + __isset.__clear(); + } + + virtual ~ResultSnapshot() throw() {} + + Code status; + Snapshot snapshot; + + struct __isset { + __isset() { __clear(); } + void __clear() { + status = false; + snapshot = false; + } + bool status; + bool snapshot; + } __isset; + + bool operator == (const ResultSnapshot & rhs) const + { + if (!(this->status == rhs.status)) + return false; + if (!(this->snapshot == rhs.snapshot)) + return false; + return true; + } + bool operator != (const ResultSnapshot &rhs) const { + return !(*this == rhs); + } + + bool operator < (const ResultSnapshot & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class ResultSnapshot; +void swap(ResultSnapshot &a, ResultSnapshot &b); + +class ResultIterator { + public: + + static const uint64_t _reflection_id = 595886977232564460U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + ResultIterator() : status(static_cast(0)) { + } + + ResultIterator(const ResultIterator&) = default; + ResultIterator& operator=(const ResultIterator&) = default; + ResultIterator(ResultIterator&&) = default; + ResultIterator& operator=(ResultIterator&&) = default; + + void __clear() { + status = static_cast(0); + iterator.__clear(); + __isset.__clear(); + } + + virtual ~ResultIterator() throw() {} + + Code status; + Iterator iterator; + + struct __isset { + __isset() { __clear(); } + void __clear() { + status = false; + iterator = false; + } + bool status; + bool iterator; + } __isset; + + bool operator == (const ResultIterator & rhs) const + { + if (!(this->status == rhs.status)) + return false; + if (!(this->iterator == rhs.iterator)) + return false; + return true; + } + bool operator != (const ResultIterator &rhs) const { + return !(*this == rhs); + } + + bool operator < (const ResultIterator & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class ResultIterator; +void swap(ResultIterator &a, ResultIterator &b); + +class LeveldbException : public apache::thrift::TException { + public: + + static const uint64_t _reflection_id = 2551220192341843436U; + static void _reflection_register(::apache::thrift::reflection::Schema&); + LeveldbException() : message(""), errorCode(static_cast(0)) { + } + + LeveldbException(const LeveldbException&) = default; + LeveldbException& operator=(const LeveldbException&) = default; + LeveldbException(LeveldbException&&) = default; + LeveldbException& operator=(LeveldbException&&) = default; + + void __clear() { + message = ""; + errorCode = static_cast(0); + __isset.__clear(); + } + + virtual ~LeveldbException() throw() {} + + Text message; + Code errorCode; + + struct __isset { + __isset() { __clear(); } + void __clear() { + message = false; + errorCode = false; + } + bool message; + bool errorCode; + } __isset; + + bool operator == (const LeveldbException & rhs) const + { + if (!(this->message == rhs.message)) + return false; + if (!(this->errorCode == rhs.errorCode)) + return false; + return true; + } + bool operator != (const LeveldbException &rhs) const { + return !(*this == rhs); + } + + bool operator < (const LeveldbException & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + + virtual const char* what() const throw() { + return "LeveldbException"; + } + +}; + +class LeveldbException; +void swap(LeveldbException &a, LeveldbException &b); + +} // namespace + +#endif diff --git a/thrift/gen-cpp/reflection_types.h b/thrift/gen-cpp/reflection_types.h new file mode 100644 index 0000000000..b2ce3492d1 --- /dev/null +++ b/thrift/gen-cpp/reflection_types.h @@ -0,0 +1,247 @@ +/** + * Autogenerated by Thrift + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated SignedSource<> + */ +#ifndef reflection_TYPES_H +#define reflection_TYPES_H + +#include +#include +#include +#include + + +#include +#include + +namespace apache { namespace thrift { namespace reflection { + +enum Type { + TYPE_VOID = 0, + TYPE_STRING = 1, + TYPE_BOOL = 2, + TYPE_BYTE = 3, + TYPE_I16 = 4, + TYPE_I32 = 5, + TYPE_I64 = 6, + TYPE_DOUBLE = 7, + TYPE_ENUM = 8, + TYPE_LIST = 9, + TYPE_SET = 10, + TYPE_MAP = 11, + TYPE_STRUCT = 12, + TYPE_SERVICE = 13, + TYPE_PROGRAM = 14 +}; + +extern const std::map _Type_VALUES_TO_NAMES; + +extern const std::map _Type_NAMES_TO_VALUES; + +class StructField { + public: + + StructField() : isRequired(0), type(0), name("") { + } + + StructField(const StructField&) = default; + StructField& operator=(const StructField&) = default; + StructField(StructField&&) = default; + StructField& operator=(StructField&&) = default; + + void __clear() { + isRequired = 0; + type = 0; + name = ""; + annotations.clear(); + __isset.__clear(); + } + + virtual ~StructField() throw() {} + + bool isRequired; + int64_t type; + std::string name; + boost::container::flat_map annotations; + + struct __isset { + __isset() { __clear(); } + void __clear() { + isRequired = false; + type = false; + name = false; + annotations = false; + } + bool isRequired; + bool type; + bool name; + bool annotations; + } __isset; + + bool operator == (const StructField & rhs) const + { + if (!(this->isRequired == rhs.isRequired)) + return false; + if (!(this->type == rhs.type)) + return false; + if (!(this->name == rhs.name)) + return false; + if (__isset.annotations != rhs.__isset.annotations) + return false; + else if (__isset.annotations && !(annotations == rhs.annotations)) + return false; + return true; + } + bool operator != (const StructField &rhs) const { + return !(*this == rhs); + } + + bool operator < (const StructField & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class StructField; +void swap(StructField &a, StructField &b); + +class DataType { + public: + + DataType() : name(""), mapKeyType(0), valueType(0) { + } + + DataType(const DataType&) = default; + DataType& operator=(const DataType&) = default; + DataType(DataType&&) = default; + DataType& operator=(DataType&&) = default; + + void __clear() { + name = ""; + fields.clear(); + mapKeyType = 0; + valueType = 0; + enumValues.clear(); + __isset.__clear(); + } + + virtual ~DataType() throw() {} + + std::string name; + boost::container::flat_map fields; + int64_t mapKeyType; + int64_t valueType; + boost::container::flat_map enumValues; + + struct __isset { + __isset() { __clear(); } + void __clear() { + name = false; + fields = false; + mapKeyType = false; + valueType = false; + enumValues = false; + } + bool name; + bool fields; + bool mapKeyType; + bool valueType; + bool enumValues; + } __isset; + + bool operator == (const DataType & rhs) const + { + if (!(this->name == rhs.name)) + return false; + if (__isset.fields != rhs.__isset.fields) + return false; + else if (__isset.fields && !(fields == rhs.fields)) + return false; + if (__isset.mapKeyType != rhs.__isset.mapKeyType) + return false; + else if (__isset.mapKeyType && !(mapKeyType == rhs.mapKeyType)) + return false; + if (__isset.valueType != rhs.__isset.valueType) + return false; + else if (__isset.valueType && !(valueType == rhs.valueType)) + return false; + if (__isset.enumValues != rhs.__isset.enumValues) + return false; + else if (__isset.enumValues && !(enumValues == rhs.enumValues)) + return false; + return true; + } + bool operator != (const DataType &rhs) const { + return !(*this == rhs); + } + + bool operator < (const DataType & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class DataType; +void swap(DataType &a, DataType &b); + +class Schema { + public: + + Schema() { + } + + Schema(const Schema&) = default; + Schema& operator=(const Schema&) = default; + Schema(Schema&&) = default; + Schema& operator=(Schema&&) = default; + + void __clear() { + dataTypes.clear(); + names.clear(); + __isset.__clear(); + } + + virtual ~Schema() throw() {} + + std::unordered_map dataTypes; + std::unordered_map names; + + struct __isset { + __isset() { __clear(); } + void __clear() { + dataTypes = false; + names = false; + } + bool dataTypes; + bool names; + } __isset; + + bool operator == (const Schema & rhs) const + { + if (!(this->dataTypes == rhs.dataTypes)) + return false; + if (!(this->names == rhs.names)) + return false; + return true; + } + bool operator != (const Schema &rhs) const { + return !(*this == rhs); + } + + bool operator < (const Schema & ) const; + + uint32_t read(apache::thrift::protocol::TProtocol* iprot); + uint32_t write(apache::thrift::protocol::TProtocol* oprot) const; + +}; + +class Schema; +void swap(Schema &a, Schema &b); + +}}} // namespace + +#endif diff --git a/thrift/leveldb.thrift b/thrift/leveldb.thrift new file mode 100644 index 0000000000..b0de6326ef --- /dev/null +++ b/thrift/leveldb.thrift @@ -0,0 +1,180 @@ +#!/usr/local/bin/thrift --gen cpp + +namespace java Tleveldb +namespace cpp Tleveldb +namespace rb Tleveldb +namespace py Tleveldb +namespace perl Tleveldb + +// Types +typedef binary Text +typedef binary Bytes + +// A basic object needed for storing keys and values +struct Slice { + 1:Text data; + 2:i32 size +} + +// Different compression types supported +enum CompressionType { + kNoCompression = 0x0, + kSnappyCompression = 0x1 +} + +// Error codes +enum Code { + kOk = 0, + kNotFound = 1, + kCorruption = 2, + kNotSupported = 3, + kInvalidArgument = 4, + kIOError = 5, + kEnd = 6 +} + +// A range object +struct Range { + 1:Slice start; // Included in the range + 2:Slice limit // Not included in the range +} + +// Options to creating a database +struct DBOptions { + 1:bool create_if_missing; + 2:bool error_if_exists; + 3:i32 write_buffer_size; + 4:i32 max_open_files; + 5:i32 block_size; + 6:i32 block_restart_interval; + 7:CompressionType compression +} + +// Options for writing +struct WriteOptions { + 1:bool sync +} + +struct Snapshot { + 1:i64 snapshotid // server generated +} + +// Options for reading. If you do not have a +// snapshot, set snapshot.snapshotid = 0 +struct ReadOptions { + 1:bool verify_checksums; + 2:bool fill_cache, + 3:Snapshot snapshot +} + +// Represents a open database object +struct DBHandle { + 1:Text dbname; //name of the database + 2:i64 handleid // server generated +} + +struct Iterator { + 1:i64 iteratorid // server generated +} + +// flags for the iterator +enum IteratorType { + seekToFirst = 0, + seekToLast = 1, + seekToKey = 2 +} + +struct kv { + 1:Slice key; + 2:Slice value +} + +// Return a single value from the Get call +struct ResultItem { + 1:Code status; + 2:Slice value +} + +// Return a key,value from a Scan call +struct ResultPair { + 1:Code status; + 2:kv keyvalue +} + +// Snapshot result +struct ResultSnapshot { + 1:Code status; + 2:Snapshot snapshot +} + +// Iterator result +struct ResultIterator { + 1:Code status; + 2:Iterator iterator +} + +exception LeveldbException { + 1:Text message, + 2:Code errorCode +} + +// The Database service +service DB { + + // opens the database + DBHandle Open(1:Text dbname, 2:DBOptions dboptions) + throws (1:LeveldbException se), + + // closes the database + Code Close(1:DBHandle dbhandle, 2:Text dbname), + + // puts a key in the database + Code Put(1:DBHandle dbhandle, 2:kv keyvalue, 3:WriteOptions options), + + // deletes a key from the database + Code Delete(1:DBHandle dbhandle, 2:Slice key, 3:WriteOptions options), + + // writes batch of keys into the database + Code Write(1:DBHandle dbhandle, 2:list batch, 3:WriteOptions options), + + // fetch a key from the DB. + // ResultItem.status == kNotFound means key is non existant + // ResultItem.status == kOk means key is found + ResultItem Get(1:DBHandle dbhandle, 2:Slice inputkey, + 3:ReadOptions options), + + // start iteration over a set of keys. If iteratorType.seekToFirst + // is set, then position the iterator at the first key in the source. + // If iteratorType.seekToLast is set, then position at the last key in the + // source. If iteratorType.seekToKey is set, then position at the first + // key in the source that is at or past target. + // If any two of iteratorType.seekToFirst & iteratorType.seekToLast + // and iteratorType.seekToKey are set, then error. + // If either iteratorType.seekToFirst or iteratorType.seekToLast is set, + // then target is not used. + ResultIterator NewIterator(1:DBHandle dbhandle, 2:ReadOptions options, + 3:IteratorType iteratorType, + 4:Slice target), + + // Release resources associated with an iterator allocated previously + // via a call to NewIterator. The call to this method may be skipped + // if the iterator had already traversed all the keys in the specified + // range. If the application wants to terminate a scan before processing + // all the resulting keys, then it is essential to invoke this method. + Code DeleteIterator(1:DBHandle dbhandle, 2:Iterator iterator), + + // Return the previous/next from this iteration + ResultPair GetNext(1:DBHandle dbhandle, 2:Iterator iterator), + ResultPair GetPrev(1:DBHandle dbhandle, 2:Iterator iterator), + + // Create snapshot. + ResultSnapshot GetSnapshot(1:DBHandle dbhandle), + + // Release snapshots + Code ReleaseSnapshot(1:DBHandle dbhandle, 2:Snapshot snapshot), + + // compact a range of keys + // begin.size == 0 to start at a range earlier than the first existing key + // end.size == 0 to end at a range later than the last existing key + Code CompactRange(1:DBHandle dbhandle, 2:Slice begin, 3:Slice end), +} diff --git a/thrift/lib/cpp/ClientUtil.h b/thrift/lib/cpp/ClientUtil.h new file mode 100644 index 0000000000..5772ce3796 --- /dev/null +++ b/thrift/lib/cpp/ClientUtil.h @@ -0,0 +1,229 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_CLIENTUTIL_H_ +#define THRIFT_CLIENTUTIL_H_ 1 + +#include "thrift/lib/cpp/transport/TBufferTransports.h" +#include "thrift/lib/cpp/transport/TSocket.h" +#include "thrift/lib/cpp/protocol/TBinaryProtocol.h" + +namespace apache { namespace thrift { namespace util { + +/* + * Versions that accept a host and port + */ + +template +ClientT* createClient(const std::string& host, uint16_t port) { + boost::shared_ptr socket( + new transport::TSocket(host, port)); + // We could specialize this to not create a wrapper transport when + // TransportT is TTransport or TSocket. However, everyone should always + // use a TFramedTransport or TBufferedTransport wrapper for performance + // reasons. + boost::shared_ptr transport(new TransportT(socket)); + boost::shared_ptr protocol( new ProtocolT(transport)); + transport->open(); + + return new ClientT(protocol); +} + +template +boost::shared_ptr createClientPtr(const std::string& host, + uint16_t port) { + return boost::shared_ptr( + createClient(host, port)); +} + +template +ClientT* createClient(const std::string& host, + uint16_t port, + bool useFramed = true) { + if (useFramed) { + return createClient( + host, port); + } else { + return createClient( + host, port); + } +} + +template +ClientT* createClient(const std::string& host, + uint16_t port, + bool useFramed = true) { + return createClient >( + host, port, useFramed); +} + +template +boost::shared_ptr createClientPtr(const std::string& host, + uint16_t port, + bool useFramed = true) { + return boost::shared_ptr( + createClient(host, port, useFramed)); +} + +template +boost::shared_ptr createClientPtr(const std::string& host, + uint16_t port, + bool useFramed = true) { + return boost::shared_ptr( + createClient(host, port, useFramed)); +} + +/* + * Versions that accept TSocketAddress + */ + +template +ClientT* createClient(const transport::TSocketAddress* address) { + boost::shared_ptr socket( + new transport::TSocket(address)); + // We could specialize this to not create a wrapper transport when + // TransportT is TTransport or TSocket. However, everyone should always + // use a TFramedTransport or TBufferedTransport wrapper for performance + // reasons. + boost::shared_ptr transport(new TransportT(socket)); + boost::shared_ptr protocol( new ProtocolT(transport)); + transport->open(); + + return new ClientT(protocol); +} + +template +boost::shared_ptr createClientPtr( + const transport::TSocketAddress* address) { + return boost::shared_ptr( + createClient(address)); +} + +template +ClientT* createClient(const transport::TSocketAddress* address, + bool useFramed = true) { + if (useFramed) { + return createClient( + address); + } else { + return createClient( + address); + } +} + +template +ClientT* createClient(const transport::TSocketAddress* address, + bool useFramed = true) { + return createClient >( + address, useFramed); +} + +template +boost::shared_ptr createClientPtr( + const transport::TSocketAddress* address, + bool useFramed = true) { + return boost::shared_ptr( + createClient(address, useFramed)); +} + +template +boost::shared_ptr createClientPtr( + const transport::TSocketAddress* address, + bool useFramed = true) { + return boost::shared_ptr( + createClient(address, useFramed)); +} + +/* + * Versions that accept TSocketAddress and socket options + */ + +template +ClientT* createClient(const transport::TSocketAddress* address, + const transport::TSocket::Options& options) { + boost::shared_ptr socket( + new transport::TSocket(address)); + socket->setSocketOptions(options); + + // We could specialize this to not create a wrapper transport when + // TransportT is TTransport or TSocket. However, everyone should always + // use a TFramedTransport or TBufferedTransport wrapper for performance + // reasons. + boost::shared_ptr transport(new TransportT(socket)); + boost::shared_ptr protocol( new ProtocolT(transport)); + transport->open(); + + return new ClientT(protocol); +} + +template +boost::shared_ptr createClientPtr( + const transport::TSocketAddress* address, + const transport::TSocket::Options& options) { + + return boost::shared_ptr( + createClient(address), options); +} + +template +ClientT* createClient(const transport::TSocketAddress* address, + const transport::TSocket::Options& options, + bool useFramed = true) { + if (useFramed) { + return createClient( + address, options); + } else { + return createClient( + address, options); + } +} + +template +ClientT* createClient(const transport::TSocketAddress* address, + const transport::TSocket::Options& options, + bool useFramed = true + ) { + return createClient >( + address, options, useFramed); +} + +template +boost::shared_ptr createClientPtr( + const transport::TSocketAddress* address, + const transport::TSocket::Options& options, + bool useFramed = true) { + return boost::shared_ptr( + createClient(address, options, useFramed)); +} + +template +boost::shared_ptr createClientPtr( + const transport::TSocketAddress* address, + const transport::TSocket::Options& options, + bool useFramed = true) { + return boost::shared_ptr( + createClient(address, options, useFramed)); +} + + +}}} // apache::thrift::util + +#endif // THRIFT_CLIENTUTIL_H_ diff --git a/thrift/lib/cpp/EventHandlerBase.h b/thrift/lib/cpp/EventHandlerBase.h new file mode 100644 index 0000000000..0915e03664 --- /dev/null +++ b/thrift/lib/cpp/EventHandlerBase.h @@ -0,0 +1,311 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef THRIFT_EVENTHANDLERBASE_H_ +#define THRIFT_EVENTHANDLERBASE_H_ 1 + +#include +#include +#include +#include "thrift/lib/cpp/server/TConnectionContext.h" +#include + +namespace apache { namespace thrift { + +using server::TConnectionContext; + +/** + * Virtual interface class that can handle events from the processor. To + * use this you should subclass it and implement the methods that you care + * about. Your subclass can also store local data that you may care about, + * such as additional "arguments" to these methods (stored in the object + * instance's state). + */ +class TProcessorEventHandler { + public: + + virtual ~TProcessorEventHandler() {} + + /** + * Called before calling other callback methods. + * Expected to return some sort of context object. + * The return value is passed to all other callbacks + * for that function invocation. + */ + virtual void* getContext(const char* fn_name, + TConnectionContext* connectionContext) { + return NULL; + } + + /** + * Expected to free resources associated with a context. + */ + virtual void freeContext(void* ctx, const char* fn_name) { } + + /** + * Called before reading arguments. + */ + virtual void preRead(void* ctx, const char* fn_name) {} + + /** + * Called between reading arguments and calling the handler. + */ + virtual void postRead(void* ctx, const char* fn_name, uint32_t bytes) {} + + /** + * Called between calling the handler and writing the response. + */ + virtual void preWrite(void* ctx, const char* fn_name) {} + + /** + * Called after writing the response. + */ + virtual void postWrite(void* ctx, const char* fn_name, uint32_t bytes) {} + + /** + * Called when an async function call completes successfully. + */ + virtual void asyncComplete(void* ctx, const char* fn_name) {} + + /** + * Called if the handler throws an undeclared exception. + */ + virtual void handlerError(void* ctx, const char* fn_name) {} + + protected: + TProcessorEventHandler() {} +}; + +/** + * A helper class used by the generated code to free each context. + */ +class TProcessorContextFreer { + public: + TProcessorContextFreer(boost::shared_ptr handler, + void* context, const char* method) : + handler_(handler), context_(context), method_(method) {} + ~TProcessorContextFreer() { + if (handler_ != NULL) { + handler_->freeContext(context_, method_); + } + } + + void unregister() { handler_.reset(); } + + private: + boost::shared_ptr handler_; + void* context_; + const char* method_; +}; + +class ContextStack { + friend class EventHandlerBase; + + public: + ContextStack( + const std::vector>& handlers, + const char* method, + TConnectionContext* connectionContext) + : handlers_(handlers) + , method_(method) { + for (auto handler: handlers) { + ctxs.push_back(handler->getContext(method, connectionContext)); + } + } + + ~ContextStack() { + for (size_t i = 0; i < handlers_.size(); i++) { + handlers_[i]->freeContext(ctxs[i], method_); + } + } + + private: + std::vector ctxs; + std::vector> handlers_; + const char* method_; +}; + +class EventHandlerBase { + private: + int setEventHandlerPos_; + ContextStack* s_; + + public: + EventHandlerBase() + : setEventHandlerPos_(-1) + , s_(NULL) + {} + + void addEventHandler( + const boost::shared_ptr& handler) { + handlers_.push_back(handler); + } + + void clearEventHandlers() { + handlers_.clear(); + setEventHandlerPos_ = -1; + if (eventHandler_) { + setEventHandler(eventHandler_); + } + } + + boost::shared_ptr getEventHandler() { + return eventHandler_; + } + + void setEventHandler(boost::shared_ptr eventHandler) { + eventHandler_ = eventHandler; + if (setEventHandlerPos_ > 0) { + handlers_.erase(handlers_.begin() + setEventHandlerPos_); + } + setEventHandlerPos_ = handlers_.size(); + handlers_.push_back(eventHandler); + } + + /** + * These functions are only used in the client handler + * implementation. The server process functions maintain + * ContextStack on the stack and binds ctx in to the async calls. + * + * Clients are not thread safe, so using a member variable is okay. + * Client send_ and recv_ functions contain parameters based off of + * the function call, and adding a parameter there would change the + * function signature enough that other thrift users might break. + * + * The generated code should be the ONLY user of s_. All other functions + * should just use the ContextStack parameter. + */ + ContextStack* getContextStack() { + return s_; + } + + // Context only freed by freer, this is only used across function calls. + void setContextStack(ContextStack* s) { + s_ = s; + } + + protected: + std::unique_ptr getContextStack( + const char* fn_name, + TConnectionContext* connectionContext) { + std::unique_ptr ctx( + new ContextStack(handlers_, fn_name, connectionContext)); + return ctx; + } + + void preWrite(ContextStack* s, const char* fn_name) { + if (s) { + for (size_t i = 0; i < handlers_.size(); i++) { + handlers_[i]->preWrite(s->ctxs[i], fn_name); + } + } + } + + void postWrite(ContextStack* s, const char* fn_name, + uint32_t bytes) { + if (s) { + for (size_t i = 0; i < handlers_.size(); i++) { + handlers_[i]->postWrite(s->ctxs[i], fn_name, bytes); + } + } + } + + void preRead(ContextStack* s, const char* fn_name) { + if (s) { + for (size_t i = 0; i < handlers_.size(); i++) { + handlers_[i]->preRead(s->ctxs[i], fn_name); + } + } + } + + void postRead(ContextStack* s, const char* fn_name, + uint32_t bytes) { + if (s) { + for (size_t i = 0; i < handlers_.size(); i++) { + handlers_[i]->postRead(s->ctxs[i], fn_name, bytes); + } + } + } + + void handlerError(ContextStack* s, const char* fn_name) { + if (s) { + for (size_t i = 0; i < handlers_.size(); i++) { + handlers_[i]->handlerError(s->ctxs[i], fn_name); + } + } + } + + void asyncComplete(ContextStack* s, const char* fn_name) { + if (s) { + for (size_t i = 0; i < handlers_.size(); i++) { + handlers_[i]->asyncComplete(s->ctxs[i], fn_name); + } + } + } + + std::vector> handlers_; + boost::shared_ptr eventHandler_; +}; + +class TProcessorEventHandlerFactory { + public: + virtual boost::shared_ptr getEventHandler() = 0; +}; + +/** + * Base class for all thrift processors. Used to automatically attach event + * handlers to processors at creation time. + */ +class TProcessorBase : public EventHandlerBase { + public: + TProcessorBase(); + + static void addProcessorEventHandlerFactory( + boost::shared_ptr factory); + + static void removeProcessorEventHandlerFactory( + boost::shared_ptr factory); + + private: + static std::vector> + registeredHandlerFactories_; +}; + +/** + * Base class for all thrift clients. Used to automatically attach event + * handlers to clients at creation time. + */ +class TClientBase : public EventHandlerBase { + public: + TClientBase(); + + static void addClientEventHandlerFactory( + boost::shared_ptr factory); + + static void removeClientEventHandlerFactory( + boost::shared_ptr factory); + + private: + static std::vector> + registeredHandlerFactories_; +}; + +}} // apache::thrift + +#endif // #ifndef THRIFT_EVENTHANDLERBASE_H_ diff --git a/thrift/lib/cpp/README b/thrift/lib/cpp/README new file mode 100644 index 0000000000..576d017027 --- /dev/null +++ b/thrift/lib/cpp/README @@ -0,0 +1,67 @@ +Thrift C++ Software Library + +License +======= + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. + +Using Thrift with C++ +===================== + +The Thrift C++ libraries are built using the GNU tools. Follow the instructions +in the top-level README, or run bootstrap.sh in this folder to generate the +Makefiles. + +In case you do not want to open another README file, do this: + ./bootstrap.sh + ./configure (--with-boost=/usr/local) + make + sudo make install + +Thrift is divided into two libraries. + +libthrift + The core Thrift library contains all the core Thrift code. It requires + boost shared pointers, pthreads, and librt. + +libthriftnb + This library contains the Thrift nonblocking server, which uses libevent. + To link this library you will also need to link libevent. + +Linking Against Thrift +====================== + +After you build and install Thrift the libraries are installed to +/usr/local/lib by default. Make sure this is in your LDPATH. + +On Linux, the best way to do this is to ensure that /usr/local/lib is in +your /etc/ld.so.conf and then run /sbin/ldconfig. + +Depending upon whether you are linking dynamically or statically and how +your build environment it set up, you may need to include additional +libraries when linking against thrift, such as librt and/or libpthread. If +you are using libthriftnb you will also need libevent. + +Dependencies +============ + +boost shared pointers +http://www.boost.org/libs/smart_ptr/smart_ptr.htm + +libevent (for libthriftnb only) +http://monkey.org/~provos/libevent/ diff --git a/thrift/lib/cpp/Reflection.h b/thrift/lib/cpp/Reflection.h new file mode 100644 index 0000000000..bbe04095a0 --- /dev/null +++ b/thrift/lib/cpp/Reflection.h @@ -0,0 +1,40 @@ +/** + * Copyright 2012 Facebook + * @author Tudor Bosman (tudorb@fb.com) + */ + +#ifndef THRIFT_LIB_CPP_REFLECTION_H_ +#define THRIFT_LIB_CPP_REFLECTION_H_ + +#include +#include + +#include "reflection_types.h" + +namespace apache { +namespace thrift { +namespace reflection { + +namespace detail { +const size_t kTypeBits = 5; +const uint64_t kTypeMask = (1ULL << kTypeBits) - 1; +} // namespace detail + +inline int64_t makeTypeId(Type type, uint64_t hash) { + return static_cast((hash & ~detail::kTypeMask) | type); +} + +inline Type getType(int64_t typeId) { + return static_cast(typeId & detail::kTypeMask); +} + +inline bool isBaseType(Type type) { + return type <= TYPE_DOUBLE; +} + +} // namespace reflection +} // namespace thrift +} // namespace apache + +#endif /* THRIFT_LIB_CPP_REFLECTION_H_ */ + diff --git a/thrift/lib/cpp/TARGETS b/thrift/lib/cpp/TARGETS new file mode 100644 index 0000000000..8e0b3bfa5e --- /dev/null +++ b/thrift/lib/cpp/TARGETS @@ -0,0 +1,46 @@ +# TARGETS file for thrift/lib/cpp + +cpp_library ( + name = "thrift_exception", + srcs = [ + "TApplicationException.cpp", + ], + external_deps = [ + ('boost', None), + ], +) + +cpp_library ( + name = "thrift", + srcs = [ + "VirtualProfiling.cpp", + ], + deps = [ + "@/thrift/lib/cpp:thrift_base", + "@/thrift/lib/cpp/concurrency", + "@/thrift/lib/cpp/processor", + "@/thrift/lib/cpp/protocol", + "@/thrift/lib/cpp/server", + "@/thrift/lib/cpp/transport", + "@/thrift/lib/cpp:thrift_exception", + ] +) + +cpp_library ( + name = "thrift_base", + srcs = [ + "Thrift.cpp", + "EventHandlerBase.cpp", + ], + external_deps = [ + ('boost', None), + ], +) + +cpp_library( + name = "reflection", + srcs = [], + deps = [ + "@/thrift/lib/thrift:reflection-cpp", + ], +) diff --git a/thrift/lib/cpp/TApplicationException.h b/thrift/lib/cpp/TApplicationException.h new file mode 100644 index 0000000000..8350471e0e --- /dev/null +++ b/thrift/lib/cpp/TApplicationException.h @@ -0,0 +1,108 @@ +// Copyright (c) 2006- Facebook +// Distributed under the Thrift Software License +// +// See accompanying file LICENSE or visit the Thrift site at: +// http://developers.facebook.com/thrift/ + +#ifndef _THRIFT_TAPPLICATIONEXCEPTION_H_ +#define _THRIFT_TAPPLICATIONEXCEPTION_H_ 1 + +#include "thrift/lib/cpp/Thrift.h" + + +namespace apache { namespace thrift { + +namespace protocol { + class TProtocol; +} + +/** + * This class is thrown when some high-level communication errors with + * the remote peer occur, and also when a server throws an unexpected + * exception from a handler method. Because of the latter case, this + * class can be serialized. + */ +class TApplicationException : public TException { + public: + + /** + * Error codes for the various types of exceptions. + */ + enum TApplicationExceptionType + { UNKNOWN = 0 + , UNKNOWN_METHOD = 1 + , INVALID_MESSAGE_TYPE = 2 + , WRONG_METHOD_NAME = 3 + , BAD_SEQUENCE_ID = 4 + , MISSING_RESULT = 5 + , INVALID_TRANSFORM = 6 + , INVALID_PROTOCOL = 7 + , UNSUPPORTED_CLIENT_TYPE = 8 + }; + + + TApplicationException() : + type_(UNKNOWN) {} + + TApplicationException(TApplicationExceptionType type) : + type_(type) {} + + TApplicationException(const std::string& message) : + message_(message), + type_(UNKNOWN) {} + + TApplicationException(TApplicationExceptionType type, + const std::string& message) : + message_(message), + type_(type) {} + + virtual ~TApplicationException() throw() {} + + /** + * Returns an error code that provides information about the type of error + * that has occurred. + * + * @return Error code + */ + TApplicationExceptionType getType() { + return type_; + } + + virtual const char* what() const throw() { + if (message_.empty()) { + switch (type_) { + case UNKNOWN : return "TApplicationException: Unknown application exception"; + case UNKNOWN_METHOD : return "TApplicationException: Unknown method"; + case INVALID_MESSAGE_TYPE : return "TApplicationException: Invalid message type"; + case WRONG_METHOD_NAME : return "TApplicationException: Wrong method name"; + case BAD_SEQUENCE_ID : return "TApplicationException: Bad sequence identifier"; + case MISSING_RESULT : return "TApplicationException: Missing result"; + case INVALID_TRANSFORM : + return "TApplicationException: Invalid transform"; + case INVALID_PROTOCOL : + return "TApplicationException: Invalid protocol"; + case UNSUPPORTED_CLIENT_TYPE: + return "TApplicationException: Unsupported client type"; + default : return "TApplicationException: (Invalid exception type)"; + }; + } else { + return message_.c_str(); + } + } + + uint32_t read(protocol::TProtocol* iprot); + uint32_t write(protocol::TProtocol* oprot) const; + + protected: + std::string message_; + + /** + * Error code + */ + TApplicationExceptionType type_; + +}; + +}} // apache::thrift + +#endif // #ifndef _THRIFT_TAPPLICATIONEXCEPTION_H_ diff --git a/thrift/lib/cpp/TDispatchProcessor.h b/thrift/lib/cpp/TDispatchProcessor.h new file mode 100644 index 0000000000..22c54b259c --- /dev/null +++ b/thrift/lib/cpp/TDispatchProcessor.h @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef _THRIFT_TDISPATCHPROCESSOR_H_ +#define _THRIFT_TDISPATCHPROCESSOR_H_ 1 + +#include "thrift/lib/cpp/TProcessor.h" + +namespace apache { namespace thrift { + +using server::TConnectionContext; + +/** + * TDispatchProcessor is a helper class to parse the message header then call + * another function to dispatch based on the function name. + * + * Subclasses must implement dispatchCall() to dispatch on the function name. + */ +template +class TDispatchProcessorT : public TProcessor { + public: + virtual bool process(boost::shared_ptr in, + boost::shared_ptr out, + TConnectionContext* connectionContext) { + protocol::TProtocol* inRaw = in.get(); + protocol::TProtocol* outRaw = out.get(); + + // Try to dynamic cast to the template protocol type + Protocol_* specificIn = dynamic_cast(inRaw); + Protocol_* specificOut = dynamic_cast(outRaw); + if (specificIn && specificOut) { + return processFast(specificIn, specificOut, connectionContext); + } + + // Log the fact that we have to use the slow path + T_GENERIC_PROTOCOL(this, inRaw, specificIn); + T_GENERIC_PROTOCOL(this, outRaw, specificOut); + + std::string fname; + protocol::TMessageType mtype; + int32_t seqid; + inRaw->readMessageBegin(fname, mtype, seqid); + + // If this doesn't look like a valid call, log an error and return false so + // that the server will close the connection. + // + // (The old generated processor code used to try to skip a T_STRUCT and + // continue. However, that seems unsafe.) + if (mtype != protocol::T_CALL && mtype != protocol::T_ONEWAY) { + GlobalOutput.printf("received invalid message type %d from client", + mtype); + return false; + } + + return this->dispatchCall(inRaw, outRaw, fname, seqid, connectionContext); + } + + protected: + bool processFast(Protocol_* in, Protocol_* out, + TConnectionContext* connectionContext) { + std::string fname; + protocol::TMessageType mtype; + int32_t seqid; + in->readMessageBegin(fname, mtype, seqid); + + if (mtype != protocol::T_CALL && mtype != protocol::T_ONEWAY) { + GlobalOutput.printf("received invalid message type %d from client", + mtype); + return false; + } + + return this->dispatchCallTemplated(in, out, fname, + seqid, connectionContext); + } + + /** + * dispatchCall() methods must be implemented by subclasses + */ + virtual bool dispatchCall(apache::thrift::protocol::TProtocol* in, + apache::thrift::protocol::TProtocol* out, + const std::string& fname, int32_t seqid, + TConnectionContext* connectionContext) = 0; + + virtual bool dispatchCallTemplated(Protocol_* in, Protocol_* out, + const std::string& fname, int32_t seqid, + TConnectionContext* connectionContext) = 0; +}; + +/** + * Non-templatized version of TDispatchProcessor, that doesn't bother trying to + * perform a dynamic_cast. + */ +class TDispatchProcessor : public TProcessor { + public: + virtual bool process(boost::shared_ptr in, + boost::shared_ptr out, + TConnectionContext* connectionContext) { + std::string fname; + protocol::TMessageType mtype; + int32_t seqid; + in->readMessageBegin(fname, mtype, seqid); + + if (mtype != protocol::T_CALL && mtype != protocol::T_ONEWAY) { + GlobalOutput.printf("received invalid message type %d from client", + mtype); + return false; + } + + return dispatchCall(in.get(), out.get(), fname, seqid, connectionContext); + } + + protected: + virtual bool dispatchCall(apache::thrift::protocol::TProtocol* in, + apache::thrift::protocol::TProtocol* out, + const std::string& fname, int32_t seqid, + TConnectionContext* connectionContext) = 0; +}; + +// Specialize TDispatchProcessorT for TProtocol and TDummyProtocol just to use +// the generic TDispatchProcessor. +template <> +class TDispatchProcessorT : + public TDispatchProcessor {}; +template <> +class TDispatchProcessorT : + public TDispatchProcessor {}; + +}} // apache::thrift + +#endif // _THRIFT_TDISPATCHPROCESSOR_H_ diff --git a/thrift/lib/cpp/TLogging.h b/thrift/lib/cpp/TLogging.h new file mode 100644 index 0000000000..3f591cbb63 --- /dev/null +++ b/thrift/lib/cpp/TLogging.h @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef THRIFT_TLOGGING_H +#define THRIFT_TLOGGING_H 1 + +#include "thrift/lib/cpp/thrift_config.h" +#include "thrift/lib/cpp/concurrency/Util.h" + +#include + +/** + * Contains utility macros for debugging and logging. + * + */ + +#ifdef THRIFT_HAVE_CLOCK_GETTIME +#include +#endif +#include + +#ifdef THRIFT_HAVE_STDINT_H +#include +#endif + +/** + * T_GLOBAL_DEBUGGING_LEVEL = 0: all debugging turned off, debug macros undefined + * T_GLOBAL_DEBUGGING_LEVEL = 1: all debugging turned on + */ +#ifndef T_GLOBAL_DEBUGGING_LEVEL +#define T_GLOBAL_DEBUGGING_LEVEL 0 +#endif + +/** + * T_GLOBAL_LOGGING_LEVEL = 0: all logging turned off, logging macros undefined + * T_GLOBAL_LOGGING_LEVEL = 1: all logging turned on + */ +#define T_GLOBAL_LOGGING_LEVEL 1 + +/** + * Standard wrapper around fprintf what will prefix the file name and line + * number to the line. Uses T_GLOBAL_DEBUGGING_LEVEL to control whether it is + * turned on or off. + * + * @param format_string + */ +#define T_DEBUG(format_string,...) \ + T_DEBUG_L(0, format_string, ##__VA_ARGS__) + +#define COMPUTE_TIME \ + int64_t nowMs = apache::thrift::concurrency::Util::currentTime(); \ + time_t nowSec = (time_t) (nowMs / 1000); \ + nowMs -= nowSec * 1000; \ + int ms = (int)nowMs; \ + char dbgtime[26]; \ + ctime_r(&nowSec, dbgtime); \ + dbgtime[24] = '\0'; + +/** + * analogous to T_DEBUG but also prints the time + * + * @param string format_string input: printf style format string + */ +#define T_DEBUG_T(format_string,...) \ + do { \ + if (T_GLOBAL_DEBUGGING_LEVEL > 0) { \ + COMPUTE_TIME \ + fprintf(stderr, "[%s,%d] [%s, %d ms] " format_string " \n", \ + __FILE__, __LINE__, dbgtime, ms, \ + ##__VA_ARGS__); \ + } \ + } while(0) + + +/** + * analogous to T_DEBUG but uses input level to determine whether or not the string + * should be logged. + * + * @param int level: specified debug level + * @param string format_string input: format string + */ +#define T_DEBUG_L(level, format_string, ...) \ + do { \ + if (T_GLOBAL_DEBUGGING_LEVEL > (level)) { \ + COMPUTE_TIME \ + fprintf(stderr, "[%s,%d] [%s, %d ms] " format_string " \n", \ + __FILE__, __LINE__, dbgtime, ms, ##__VA_ARGS__); \ + } \ + } while (0) + + +/** + * Explicit error logging. Prints time, file name and line number + * + * @param string format_string input: printf style format string + */ +#define T_ERROR(format_string,...) \ + { \ + COMPUTE_TIME \ + fprintf(stderr,"[%s,%d] [%s, %d ms] ERROR: " format_string " \n", \ + __FILE__, __LINE__,dbgtime, ms, \ + ##__VA_ARGS__); \ + } + + +/** + * Analogous to T_ERROR, additionally aborting the process. + * WARNING: macro calls abort(), ending program execution + * + * @param string format_string input: printf style format string + */ +#define T_ERROR_ABORT(format_string,...) \ + { \ + COMPUTE_TIME \ + fprintf(stderr,"[%s,%d] [%s, %d ms] ERROR: Going to abort " \ + format_string " \n", \ + __FILE__, __LINE__,dbgtime, ms, \ + ##__VA_ARGS__); \ + exit(1); \ + } + + +/** + * Log input message + * + * @param string format_string input: printf style format string + */ +#if T_GLOBAL_LOGGING_LEVEL > 0 + #define T_LOG_OPER(format_string,...) \ + { \ + if (T_GLOBAL_LOGGING_LEVEL > 0) { \ + COMPUTE_TIME \ + fprintf(stderr,"[%s, %d ms] " format_string " \n", \ + dbgtime, ms, ##__VA_ARGS__); \ + } \ + } +#else + #define T_LOG_OPER(format_string,...) +#endif + + +/** + * T_GLOBAL_DEBUG_VIRTUAL = 0 or unset: normal operation, + * virtual call debug messages disabled + * T_GLOBAL_DEBUG_VIRTUAL = 1: log a debug messages whenever an + * avoidable virtual call is made + * T_GLOBAL_DEBUG_VIRTUAL = 2: record detailed info that can be + * printed by calling + * apache::thrift::profile_print_info() + */ +#if T_GLOBAL_DEBUG_VIRTUAL > 1 + #define T_VIRTUAL_CALL() \ + ::apache::thrift::profile_virtual_call(typeid(*this)) + #define T_GENERIC_PROTOCOL(template_class, generic_prot, specific_prot) \ + do { \ + if (!(specific_prot)) { \ + ::apache::thrift::profile_generic_protocol( \ + typeid(*template_class), typeid(*generic_prot)); \ + } \ + } while (0) +#elif T_GLOBAL_DEBUG_VIRTUAL == 1 + #define T_VIRTUAL_CALL() \ + fprintf(stderr,"[%s,%d] virtual call\n", __FILE__, __LINE__) + #define T_GENERIC_PROTOCOL(template_class, generic_prot, specific_prot) \ + do { \ + if (!(specific_prot)) { \ + fprintf(stderr, \ + "[%s,%d] failed to cast to specific protocol type\n", \ + __FILE__, __LINE__); \ + } \ + } while (0) +#else + #define T_VIRTUAL_CALL() + #define T_GENERIC_PROTOCOL(template_class, generic_prot, specific_prot) +#endif + +#endif // #ifndef THRIFT_TLOGGING_H diff --git a/thrift/lib/cpp/TProcessor.h b/thrift/lib/cpp/TProcessor.h new file mode 100644 index 0000000000..f54cbea61f --- /dev/null +++ b/thrift/lib/cpp/TProcessor.h @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef THRIFT_TPROCESSOR_H_ +#define THRIFT_TPROCESSOR_H_ 1 + +#include +#include +#include "thrift/lib/cpp/EventHandlerBase.h" +#include "thrift/lib/cpp/server/TConnectionContext.h" +#include "thrift/lib/cpp/protocol/TProtocol.h" +#include + +namespace apache { namespace thrift { + +/** + * A processor is a generic object that acts upon two streams of data, one + * an input and the other an output. The definition of this object is loose, + * though the typical case is for some sort of server that either generates + * responses to an input stream or forwards data from one pipe onto another. + * + */ +class TProcessor : public TProcessorBase { + public: + virtual ~TProcessor() {} + + virtual bool process(boost::shared_ptr in, + boost::shared_ptr out, + TConnectionContext* connectionContext) = 0; + + bool process(boost::shared_ptr io, + TConnectionContext* connectionContext) { + return process(io, io, connectionContext); + } + + protected: + TProcessor() {} +}; + +class TProcessorFactory { + public: + virtual ~TProcessorFactory() {} + + /** + * Get the TProcessor to use for a particular connection. + * + * This method is always invoked in the same thread that the connection was + * accepted on. This generally means that this call does not need to be + * thread safe, as it will always be invoked from a single thread. + */ + virtual boost::shared_ptr getProcessor( + server::TConnectionContext* ctx) = 0; +}; + +class TSingletonProcessorFactory : public TProcessorFactory { + public: + explicit TSingletonProcessorFactory( + const boost::shared_ptr& processor) : + processor_(processor) {} + + boost::shared_ptr getProcessor(server::TConnectionContext*) { + return processor_; + } + + boost::shared_ptr getProcessor() { + return processor_; + } + + private: + boost::shared_ptr processor_; +}; + +/** + * This is a helper class to allow boost::shared_ptr to be used with handler + * pointers returned by the generated handler factories. + * + * The handler factory classes generated by the thrift compiler return raw + * pointers, and factory->releaseHandler() must be called when the handler is + * no longer needed. + * + * A ReleaseHandler object can be instantiated and passed as the second + * parameter to a shared_ptr, so that factory->releaseHandler() will be called + * when the object is no longer needed, instead of deleting the pointer. + */ +template +class ReleaseHandler { + public: + explicit ReleaseHandler( + const boost::shared_ptr& handlerFactory) : + handlerFactory_(handlerFactory) {} + + void operator()(typename HandlerFactory_::Handler* handler) { + if (handler) { + handlerFactory_->releaseHandler(handler); + } + } + + private: + boost::shared_ptr handlerFactory_; +}; + +}} // apache::thrift + +#endif // #ifndef THRIFT_TPROCESSOR_H_ diff --git a/thrift/lib/cpp/TReflectionLocal.h b/thrift/lib/cpp/TReflectionLocal.h new file mode 100644 index 0000000000..d621f5c9e4 --- /dev/null +++ b/thrift/lib/cpp/TReflectionLocal.h @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_TREFLECTIONLOCAL_H_ +#define _THRIFT_TREFLECTIONLOCAL_H_ 1 + +#include +#include +#include "thrift/lib/cpp/protocol/TProtocol.h" + +/** + * Local Reflection is a blanket term referring to the the structure + * and generation of this particular representation of Thrift types. + * (It is called local because it cannot be serialized by Thrift). + * + */ + +namespace apache { namespace thrift { namespace reflection { namespace local { + +using apache::thrift::protocol::TType; + +// We include this many bytes of the structure's fingerprint when serializing +// a top-level structure. Long enough to make collisions unlikely, short +// enough to not significantly affect the amount of memory used. +const int FP_PREFIX_LEN = 4; + +struct FieldMeta { + int16_t tag; + bool is_optional; +}; + +struct TypeSpec { + TType ttype; + uint8_t fp_prefix[FP_PREFIX_LEN]; + + // Use an anonymous union here so we can fit two TypeSpecs in one cache line. + union { + struct { + // Use parallel arrays here for denser packing (of the arrays). + FieldMeta* metas; + TypeSpec** specs; + } tstruct; + struct { + TypeSpec *subtype1; + TypeSpec *subtype2; + } tcontainer; + }; + + // Static initialization of unions isn't really possible, + // so take the plunge and use constructors. + // Hopefully they'll be evaluated at compile time. + + TypeSpec(TType ttype) : ttype(ttype) { + std::memset(fp_prefix, 0, FP_PREFIX_LEN); + } + + TypeSpec(TType ttype, + const uint8_t* fingerprint, + FieldMeta* metas, + TypeSpec** specs) : + ttype(ttype) + { + std::memcpy(fp_prefix, fingerprint, FP_PREFIX_LEN); + tstruct.metas = metas; + tstruct.specs = specs; + } + + TypeSpec(TType ttype, TypeSpec* subtype1, TypeSpec* subtype2) : + ttype(ttype) + { + std::memset(fp_prefix, 0, FP_PREFIX_LEN); + tcontainer.subtype1 = subtype1; + tcontainer.subtype2 = subtype2; + } + +}; + +}}}} // apache::thrift::reflection::local + +#endif // #ifndef _THRIFT_TREFLECTIONLOCAL_H_ diff --git a/thrift/lib/cpp/Thrift.h b/thrift/lib/cpp/Thrift.h new file mode 100644 index 0000000000..b739c5376e --- /dev/null +++ b/thrift/lib/cpp/Thrift.h @@ -0,0 +1,261 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef THRIFT_THRIFT_H_ +#define THRIFT_THRIFT_H_ + +#include "thrift/lib/cpp/thrift_config.h" +#include +#include + +#include +#include +#ifdef THRIFT_HAVE_INTTYPES_H +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include "thrift/lib/cpp/TLogging.h" + +namespace apache { namespace thrift { + +struct ltstr { + bool operator()(const char* s1, const char* s2) const { + return strcmp(s1, s2) < 0; + } +}; + +/** + * Helper template class for enum<->string conversion. + */ +template +struct TEnumTraits { + /** + * Finds the name of a given enum value, returning it or NULL on failure. + * Specialized implementations will be emitted as part of enum codegen. + * + * Example specialization: + * template<> + * const char* TEnumTraits::findName(MyEnum value) { + * return findName(_MyEnum_VALUES_TO_NAMES, value); + * } + * Note the use of helper function 'findName(...)', below. + */ + static const char* findName(T value); + /** + * Attempts to find a value for a given name. + * Specialized implementations will be emitted as part of enum codegen. + * + * Example implementation: + * template<> + * bool TEnumTraits::findValue(const char* name, + * MyEnum* outValue) { + * return findValue(_MyEnum_NAMES_TO_VALUES, name, outValue); + * } + * Note the use of helper function 'findValue(...)', below. + */ + static bool findValue(const char* name, T* outValue); + private: + /** + * Helper method used by codegen implementation of findName, Supports + * use with strict and non-strict enums by way of template parameter + * 'ValueType'. + */ + template + static const char* findName(const std::map& map, + T value) { + auto found = map.find(value); + if (found == map.end()) { + return NULL; + } else { + return found->second; + } + } + + /** + * Helper method used by codegen implementation of findValue, Supports + * use with strict and non-strict enums by way of template parameter + * 'ValueType'. + */ + template + static bool findValue(const std::map& map, + const char* name, T* out) { + auto found = map.find(name); + if (found == map.end()) { + return false; + } else { + *out = static_cast(found->second); + return true; + } + } +}; + +template +class TEnumIterator : public std::map::iterator { + public: + TEnumIterator(int n, + T* enums, + const char** names) : + ii_(0), n_(n), enums_(enums), names_(names) { + } + + int operator ++() { + return ++ii_; + } + + bool operator !=(const TEnumIterator& end) { + assert(end.n_ == -1); + return (ii_ != n_); + } + + std::pair operator*() const { + return std::make_pair(enums_[ii_], names_[ii_]); + } + + private: + int ii_; + const int n_; + T* enums_; + const char** names_; +}; + +template +class TEnumInverseIterator : public std::map::iterator { + public: + TEnumInverseIterator(int n, + T* enums, + const char** names) : + ii_(0), n_(n), enums_(enums), names_(names) { + } + + int operator ++() { + return ++ii_; + } + + bool operator !=(const TEnumInverseIterator& end) { + assert(end.n_ == -1); + return (ii_ != n_); + } + + std::pair operator*() const { + return std::make_pair(names_[ii_], enums_[ii_]); + } + + private: + int ii_; + const int n_; + T* enums_; + const char** names_; +}; + +class TOutput { + public: + TOutput() : f_(&errorTimeWrapper) {} + + inline void setOutputFunction(void (*function)(const char *)){ + f_ = function; + } + + inline void operator()(const char *message){ + f_(message); + } + + // It is important to have a const char* overload here instead of + // just the string version, otherwise errno could be corrupted + // if there is some problem allocating memory when constructing + // the string. + void perror(const char *message, int errno_copy); + inline void perror(const std::string &message, int errno_copy) { + perror(message.c_str(), errno_copy); + } + + void printf(const char *message, ...); + + inline static void errorTimeWrapper(const char* msg) { + time_t now; + char dbgtime[26]; + time(&now); + ctime_r(&now, dbgtime); + dbgtime[24] = 0; + fprintf(stderr, "Thrift: %s %s\n", dbgtime, msg); + } + + /** Just like strerror_r but returns a C++ string object. */ + static std::string strerror_s(int errno_copy); + + private: + void (*f_)(const char *); +}; + +extern TOutput GlobalOutput; + +/** + * Base class for all Thrift exceptions. + * Should never be instantiated, only caught. + */ +class TException : public std::exception { +}; + +/** + * Base class for exceptions from the Thrift library, and occasionally + * from the generated code. This class should not be thrown by user code. + * Instances of this class are not meant to be serialized. + */ +class TLibraryException : public TException { + public: + TLibraryException() {} + + explicit TLibraryException(const std::string& message) : + message_(message) {} + + TLibraryException(const char* message, int errnoValue); + + virtual ~TLibraryException() throw() {} + + virtual const char* what() const throw() { + if (message_.empty()) { + return "Default TLibraryException."; + } else { + return message_.c_str(); + } + } + + protected: + std::string message_; + +}; + +#if T_GLOBAL_DEBUG_VIRTUAL > 1 +void profile_virtual_call(const std::type_info& info); +void profile_generic_protocol(const std::type_info& template_type, + const std::type_info& prot_type); +void profile_print_info(FILE *f); +void profile_print_info(); +void profile_write_pprof(FILE* gen_calls_f, FILE* virtual_calls_f); +#endif + +}} // apache::thrift + +#endif // #ifndef THRIFT_THRIFT_H_ diff --git a/thrift/lib/cpp/async/SimpleCallback.h b/thrift/lib/cpp/async/SimpleCallback.h new file mode 100644 index 0000000000..00551e2469 --- /dev/null +++ b/thrift/lib/cpp/async/SimpleCallback.h @@ -0,0 +1,98 @@ +#ifndef _THRIFT_ASYNC_SIMPLECALLBACK_H_ +#define _THRIFT_ASYNC_SIMPLECALLBACK_H_ 1 + +#include "thrift/lib/cpp/Thrift.h" +namespace apache { namespace thrift { + +/** + * A template class for forming simple method callbacks with either an empty + * argument list or one argument of known type. + * + * For more efficiency where tr1::function is overkill. + */ + +template ///< type of return value +class SimpleCallback { + typedef R (C::*cfptr_t)(A); ///< pointer-to-member-function type + cfptr_t fptr_; ///< the embedded function pointer + C* obj_; ///< object whose function we're wrapping + public: + /** + * Constructor for empty callback object. + */ + SimpleCallback() : + fptr_(NULL), obj_(NULL) {} + /** + * Construct callback wrapper for member function. + * + * @param fptr pointer-to-member-function + * @param "this" for object associated with callback + */ + SimpleCallback(cfptr_t fptr, const C* obj) : + fptr_(fptr), obj_(const_cast(obj)) + {} + + /** + * Make a call to the member function we've wrapped. + * + * @param i argument for the wrapped member function + * @return value from that function + */ + R operator()(A i) const { + (obj_->*fptr_)(i); + } + + operator bool() const { + return obj_ != NULL && fptr_ != NULL; + } + + ~SimpleCallback() {} +}; + +/** + * Specialization of SimpleCallback for empty argument list. + */ +template ///< type of return value +class SimpleCallback { + typedef R (C::*cfptr_t)(); ///< pointer-to-member-function type + cfptr_t fptr_; ///< the embedded function pointer + C* obj_; ///< object whose function we're wrapping + public: + /** + * Constructor for empty callback object. + */ + SimpleCallback() : + fptr_(NULL), obj_(NULL) {} + + /** + * Construct callback wrapper for member function. + * + * @param fptr pointer-to-member-function + * @param obj "this" for object associated with callback + */ + SimpleCallback(cfptr_t fptr, const C* obj) : + fptr_(fptr), obj_(const_cast(obj)) + {} + + /** + * Make a call to the member function we've wrapped. + * + * @return value from that function + */ + R operator()() const { + (obj_->*fptr_)(); + } + + operator bool() const { + return obj_ != NULL && fptr_ != NULL; + } + + ~SimpleCallback() {} +}; + +}} // apache::thrift + +#endif /* !_THRIFT_ASYNC_SIMPLECALLBACK_H_ */ diff --git a/thrift/lib/cpp/async/TARGETS b/thrift/lib/cpp/async/TARGETS new file mode 100644 index 0000000000..6c6411b271 --- /dev/null +++ b/thrift/lib/cpp/async/TARGETS @@ -0,0 +1,103 @@ +# The base async library +cpp_library( + name = 'async_base', + srcs = [ + 'TAsyncServerSocket.cpp', + 'TAsyncSignalHandler.cpp', + 'TAsyncSocket.cpp', + 'TAsyncTimeout.cpp', + 'TBinaryAsyncChannel.cpp', + 'THeaderAsyncChannel.cpp', + 'TEventBase.cpp', + 'TEventBaseManager.cpp', + 'TEventHandler.cpp', + 'TFramedAsyncChannel.cpp', + 'TNotificationPipe.cpp', + 'TUnframedAsyncChannel.cpp', + 'THttpAsyncChannel.cpp', + ], + deps = [ + '@/folly/experimental/io', + '@/thrift/lib/cpp:thrift_base', + '@/thrift/lib/cpp/transport', + '@/thrift/lib/cpp/transport:header', + '@/thrift/lib/cpp/protocol', + '@/thrift/lib/cpp/server', + '@/thrift/lib/cpp/util:httpparser', + ], + external_deps = [ ('libevent', None) ], +) + +# TEventServer library (async name is used all over), now depends on ssl +cpp_library( + name = 'async', + srcs = [ + 'TEventConnection.cpp', + 'TEventServer.cpp', + 'TEventTask.cpp', + 'TEventWorker.cpp', + ], + deps = [ + ':async_ssl', + ], +) + +cpp_library ( + name = "async_ssl", + srcs = [ + 'TAsyncSSLServerSocket.cpp', + 'TAsyncSSLSocket.cpp', + ], + deps = [ + ":async_base", + "@/thrift/lib/cpp/transport:transport_ssl", + ], + external_deps = [ ("openssl", None, "ssl"), + ("openssl", None, "crypto") ], +) + +# This library is the same as async_ssl, except that it has the debug +# logging level set to 4, whereas async_ssl has debug logging disabled. +cpp_library ( + name = "async_ssl_log", + srcs = [ + 'TAsyncSSLServerSocket.cpp', + 'TAsyncSSLSocket.cpp', + ], + deps = [ + ":async_base", + "@/thrift/lib/cpp/transport:transport_ssl", + ], + external_deps = [ ("openssl", None, "ssl"), + ("openssl", None, "crypto") ], + preprocessor_flags = [ + "-DT_GLOBAL_DEBUGGING_LEVEL=4" + ], + output_subdir = "async_ssl_log", +) + +cpp_library( + name = 'zlib', + srcs = [ + 'TZlibAsyncChannel.cpp', + ], + deps = [ + ':async', + '@/thrift/lib/cpp/transport:zlib', + ], +) + +# Parts of the async library that depend on C++11 features. +# This is separate from the main async library for now so that users who cannot +# depend on C++11 features yet will still be able to use the bulk of the async +# library. +cpp_library( + name = 'async_cxx11', + srcs = [ + 'TAsyncTimeoutSet.cpp', + ], + deps = [ + ':async_base', + '@/folly', + ], +) diff --git a/thrift/lib/cpp/async/TAsyncChannel.h b/thrift/lib/cpp/async/TAsyncChannel.h new file mode 100644 index 0000000000..ebf9e6e8a0 --- /dev/null +++ b/thrift/lib/cpp/async/TAsyncChannel.h @@ -0,0 +1,113 @@ +// Copyright (c) 2006- Facebook +// Distributed under the Thrift Software License +// +// See accompanying file LICENSE or visit the Thrift site at: +// http://developers.facebook.com/thrift/ + +#ifndef THRIFT_ASYNC_TASYNCCHANNEL_H_ +#define THRIFT_ASYNC_TASYNCCHANNEL_H_ 1 + +#include +#include "thrift/lib/cpp/Thrift.h" +#include "thrift/lib/cpp/transport/TTransportUtils.h" + +namespace apache { namespace thrift { namespace async { + +class TAsyncTransport; + +/** + * TAsyncChannel defines an asynchronous API for message-based I/O. + */ +class TAsyncChannel { + public: + typedef std::tr1::function VoidCallback; + + virtual ~TAsyncChannel() {} + + // is the channel readable (possibly closed by the remote site)? + virtual bool readable() const = 0; + // is the channel in a good state? + virtual bool good() const = 0; + virtual bool error() const = 0; + virtual bool timedOut() const = 0; + + /** + * Send a message over the channel. + * + * @return call "cob" on success, "errorCob" on fail. Caller must be ready + * for either cob to be called before return. Only one cob will be + * called and it will be called exactly once per invocation. + */ + virtual void sendMessage( + const VoidCallback& cob, + const VoidCallback& errorCob, + apache::thrift::transport::TMemoryBuffer* message) = 0; + + /** + * Receive a message from the channel. + * + * @return call "cob" on success, "errorCob" on fail. Caller must be ready + * for either cob to be called before return. Only one cob will be + * called and it will be called exactly once per invocation. + */ + virtual void recvMessage( + const VoidCallback& cob, + const VoidCallback& errorCob, + apache::thrift::transport::TMemoryBuffer* message) = 0; + + /** + * Send a message over the channel and receive a response. + * + * @return call "cob" on success, "errorCob" on fail. Caller must be ready + * for either cob to be called before return. Only one cob will be + * called and it will be called exactly once per invocation. + */ + virtual void sendAndRecvMessage( + const VoidCallback& cob, + const VoidCallback& errorCob, + apache::thrift::transport::TMemoryBuffer* sendBuf, + apache::thrift::transport::TMemoryBuffer* recvBuf) = 0; + + /** + * Send a message over the channel, single cob version. (See above.) + * + * @return call "cob" on success or fail; channel status must be queried + * by the cob. + */ + void sendMessage(const VoidCallback& cob, + apache::thrift::transport::TMemoryBuffer* message) { + return sendMessage(cob, cob, message); + } + + /** + * Receive a message from the channel, single cob version. (See above.) + * + * @return call "cob" on success or fail; channel status must be queried + * by the cob. + */ + void recvMessage(const VoidCallback& cob, + apache::thrift::transport::TMemoryBuffer* message) { + return recvMessage(cob, cob, message); + } + + /** + * Send a message over the channel and receive response, single cob version. + * (See above.) + * + * @return call "cob" on success or fail; channel status must be queried + * by the cob. + */ + void sendAndRecvMessage(const VoidCallback& cob, + apache::thrift::transport::TMemoryBuffer* sendBuf, + apache::thrift::transport::TMemoryBuffer* recvBuf) { + return sendAndRecvMessage(cob, cob, sendBuf, recvBuf); + } + + // TODO(dreiss): Make this nonvirtual when TFramedSocketAsyncChannel gets + // renamed to TFramedAsyncChannel. + virtual boost::shared_ptr getTransport() = 0; +}; + +}}} // apache::thrift::async + +#endif // #ifndef THRIFT_ASYNC_TASYNCCHANNEL_H_ diff --git a/thrift/lib/cpp/async/TAsyncDispatchProcessor.h b/thrift/lib/cpp/async/TAsyncDispatchProcessor.h new file mode 100644 index 0000000000..c6cdf51042 --- /dev/null +++ b/thrift/lib/cpp/async/TAsyncDispatchProcessor.h @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_ASYNC_TASYNCDISPATCHPROCESSOR_H_ +#define THRIFT_ASYNC_TASYNCDISPATCHPROCESSOR_H_ 1 + +#include "thrift/lib/cpp/async/TAsyncProcessor.h" + +namespace apache { namespace thrift { namespace async { + +/** + * TAsyncDispatchProcessor is a helper class to parse the message header then + * call another function to dispatch based on the function name. + * + * Subclasses must implement dispatchCall() to dispatch on the function name. + */ +template +class TAsyncDispatchProcessorT : public TAsyncProcessor { + public: + virtual void process(std::tr1::function _return, + boost::shared_ptr in, + boost::shared_ptr out, + TConnectionContext* context) { + protocol::TProtocol* inRaw = in.get(); + protocol::TProtocol* outRaw = out.get(); + + // Try to dynamic cast to the template protocol type + Protocol_* specificIn = dynamic_cast(inRaw); + Protocol_* specificOut = dynamic_cast(outRaw); + if (specificIn && specificOut) { + return processFast(_return, specificIn, specificOut, context); + } + + // Log the fact that we have to use the slow path + T_GENERIC_PROTOCOL(this, inRaw, specificIn); + T_GENERIC_PROTOCOL(this, outRaw, specificOut); + + std::string fname; + protocol::TMessageType mtype; + int32_t seqid; + try { + inRaw->readMessageBegin(fname, mtype, seqid); + } catch (const TException &ex) { + GlobalOutput.printf("received invalid message from client: %s", + ex.what()); + _return(false); + return; + } + + // If this doesn't look like a valid call, log an error and return false so + // that the server will close the connection. + // + // (The old generated processor code used to try to skip a T_STRUCT and + // continue. However, that seems unsafe.) + if (mtype != protocol::T_CALL && mtype != protocol::T_ONEWAY) { + GlobalOutput.printf("received invalid message type %d from client", + mtype); + _return(false); + return; + } + + return this->dispatchCall(_return, inRaw, outRaw, fname, seqid, context); + } + + void processFast(std::tr1::function _return, + Protocol_* in, Protocol_* out, + TConnectionContext* context) { + std::string fname; + protocol::TMessageType mtype; + int32_t seqid; + try { + in->readMessageBegin(fname, mtype, seqid); + } catch (const TException &ex) { + GlobalOutput.printf("received invalid message from client: %s", + ex.what()); + _return(false); + return; + } + + if (mtype != protocol::T_CALL && mtype != protocol::T_ONEWAY) { + GlobalOutput.printf("received invalid message type %d from client", + mtype); + _return(false); + return; + } + + return this->dispatchCallTemplated(_return, in, out, fname, seqid, context); + } + + virtual void dispatchCall(std::tr1::function _return, + apache::thrift::protocol::TProtocol* in, + apache::thrift::protocol::TProtocol* out, + const std::string& fname, int32_t seqid, + TConnectionContext* context) = 0; + + virtual void dispatchCallTemplated(std::tr1::function _return, + Protocol_* in, Protocol_* out, + const std::string& fname, int32_t seqid, + TConnectionContext* context) = 0; +}; + +/** + * Non-templatized version of TAsyncDispatchProcessor, + * that doesn't bother trying to perform a dynamic_cast. + */ +class TAsyncDispatchProcessor : public TAsyncProcessor { + public: + virtual void process(std::tr1::function _return, + boost::shared_ptr in, + boost::shared_ptr out, + TConnectionContext* context) { + protocol::TProtocol* inRaw = in.get(); + protocol::TProtocol* outRaw = out.get(); + + std::string fname; + protocol::TMessageType mtype; + int32_t seqid; + try { + inRaw->readMessageBegin(fname, mtype, seqid); + } catch (const TException &ex) { + GlobalOutput.printf("received invalid message from client: %s", + ex.what()); + _return(false); + return; + } + + // If this doesn't look like a valid call, log an error and return false so + // that the server will close the connection. + // + // (The old generated processor code used to try to skip a T_STRUCT and + // continue. However, that seems unsafe.) + if (mtype != protocol::T_CALL && mtype != protocol::T_ONEWAY) { + GlobalOutput.printf("received invalid message type %d from client", + mtype); + _return(false); + return; + } + + return dispatchCall(_return, inRaw, outRaw, fname, seqid, context); + } + + virtual void dispatchCall(std::tr1::function _return, + apache::thrift::protocol::TProtocol* in, + apache::thrift::protocol::TProtocol* out, + const std::string& fname, int32_t seqid, + TConnectionContext* context) = 0; +}; + +// Specialize TAsyncDispatchProcessorT for TProtocol and TDummyProtocol just to +// use the generic TDispatchProcessor. +template <> +class TAsyncDispatchProcessorT : + public TAsyncDispatchProcessor {}; +template <> +class TAsyncDispatchProcessorT : + public TAsyncDispatchProcessor {}; + +}}} // apache::thrift::async + +#endif // THRIFT_ASYNC_TASYNCDISPATCHPROCESSOR_H_ diff --git a/thrift/lib/cpp/async/TAsyncEventChannel.h b/thrift/lib/cpp/async/TAsyncEventChannel.h new file mode 100644 index 0000000000..b573b3f36b --- /dev/null +++ b/thrift/lib/cpp/async/TAsyncEventChannel.h @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_ASYNC_TASYNCEVENTCHANNEL_H_ +#define THRIFT_ASYNC_TASYNCEVENTCHANNEL_H_ 1 + +#include "thrift/lib/cpp/async/TAsyncChannel.h" +#include "thrift/lib/cpp/async/TDelayedDestruction.h" + +namespace apache { namespace thrift { namespace async { + +class TEventBase; + +/** + * TAsyncEventChannel defines an API for TAsyncChannel objects that are driven + * by TEventBase. + */ +class TAsyncEventChannel : public TAsyncChannel, + public TDelayedDestruction { + public: + + /** + * Determine if this channel is idle (i.e., has no outstanding reads or + * writes). + */ + virtual bool isIdle() const = 0; + + /** + * Attach the channel to a TEventBase. + * + * This may only be called if the channel is not currently attached to a + * TEventBase (by an earlier call to detachEventBase()). + * + * This method must be invoked in the TEventBase's thread. + */ + virtual void attachEventBase(TEventBase* eventBase) = 0; + + /** + * Detach the channel from its TEventBase. + * + * This may only be called when the channel is idle and has no reads or + * writes pending. Once detached, the channel may not be used again until it + * is re-attached to a TEventBase by calling attachEventBase(). + * + * This method must be called from the current TEventBase's thread. + */ + virtual void detachEventBase() = 0; + + /** + * Get the receive timeout. + * + * @return Returns the current receive timeout, in milliseconds. A return + * value of 0 indicates that no timeout is set. + */ + virtual uint32_t getRecvTimeout() const = 0; + + /** + * Set the timeout for receiving messages. + * + * When set to a non-zero value, the entire message must be received within + * the specified number of milliseconds, or the receive will fail and the + * channel will be closed. + */ + virtual void setRecvTimeout(uint32_t milliseconds) = 0; + + protected: + /** + * Protected destructor. + * + * Users of TAsyncEventChannel must never delete it directly. Instead, invoke + * destroy() instead. (See the documentation in TDelayedDestruction.h for + * more details.) + */ + + virtual ~TAsyncEventChannel() { } +}; + +}}} // apache::thrift::async + +#endif // THRIFT_ASYNC_TASYNCEVENTCHANNEL_H_ diff --git a/thrift/lib/cpp/async/TAsyncProcessor.h b/thrift/lib/cpp/async/TAsyncProcessor.h new file mode 100644 index 0000000000..591e9877b5 --- /dev/null +++ b/thrift/lib/cpp/async/TAsyncProcessor.h @@ -0,0 +1,90 @@ +// Copyright (c) 2006- Facebook +// Distributed under the Thrift Software License +// +// See accompanying file LICENSE or visit the Thrift site at: +// http://developers.facebook.com/thrift/ + +#ifndef THRIFT_TASYNCPROCESSOR_H +#define THRIFT_TASYNCPROCESSOR_H 1 + +#include +#include +#include "thrift/lib/cpp/TProcessor.h" +#include "thrift/lib/cpp/server/TConnectionContext.h" +#include "thrift/lib/cpp/protocol/TProtocol.h" + +using apache::thrift::server::TConnectionContext; + +namespace apache { namespace thrift { namespace async { + +/** + * Async version of a TProcessor. It is not expected to complete by the time + * the call to process returns. Instead, it calls a cob to signal completion. + * + * @author David Reiss + */ + +class TEventServer; // forward declaration + +class TAsyncProcessor : public TProcessorBase { + public: + virtual ~TAsyncProcessor() {} + + virtual void process(std::tr1::function _return, + boost::shared_ptr in, + boost::shared_ptr out, + TConnectionContext* context = NULL) = 0; + + void process(std::tr1::function _return, + boost::shared_ptr io) { + return process(_return, io, io); + } + + const TEventServer* getAsyncServer() { + return asyncServer_; + } + protected: + TAsyncProcessor() {} + + const TEventServer* asyncServer_; + private: + friend class TEventServer; + void setAsyncServer(const TEventServer* server) { + asyncServer_ = server; + } +}; + +class TAsyncProcessorFactory { + public: + virtual ~TAsyncProcessorFactory() {} + + /** + * Get the TAsyncProcessor to use for a particular connection. + */ + virtual boost::shared_ptr getProcessor( + server::TConnectionContext* ctx) = 0; +}; + +class TAsyncSingletonProcessorFactory : public TAsyncProcessorFactory { + public: + explicit TAsyncSingletonProcessorFactory( + const boost::shared_ptr& processor) : + processor_(processor) {} + + boost::shared_ptr getProcessor(server::TConnectionContext*) { + return processor_; + } + + private: + boost::shared_ptr processor_; +}; + + +}}} // apache::thrift::async + +// XXX I'm lazy for now +namespace apache { namespace thrift { +using apache::thrift::async::TAsyncProcessor; +}} + +#endif // #ifndef THRIFT_TASYNCPROCESSOR_H diff --git a/thrift/lib/cpp/async/TAsyncSSLServerSocket.h b/thrift/lib/cpp/async/TAsyncSSLServerSocket.h new file mode 100644 index 0000000000..d4e2c9aa7a --- /dev/null +++ b/thrift/lib/cpp/async/TAsyncSSLServerSocket.h @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_ASYNC_TASYNCSSLSERVERSOCKET_H_ +#define THRIFT_ASYNC_TASYNCSSLSERVERSOCKET_H_ 1 + +#include "thrift/lib/cpp/async/TAsyncServerSocket.h" +#include "thrift/lib/cpp/transport/TSSLSocket.h" + +namespace apache { namespace thrift { + +namespace transport { +class TSocketAddress; +} + +namespace async { + +class TAsyncSSLSocket; + +class TAsyncSSLServerSocket : public TDelayedDestruction, + private TAsyncServerSocket::AcceptCallback { + public: + class SSLAcceptCallback { + public: + virtual ~SSLAcceptCallback() {} + + /** + * connectionAccepted() is called whenever a new client connection is + * received. + * + * The SSLAcceptCallback will remain installed after connectionAccepted() + * returns. + * + * @param sock The newly accepted client socket. The + * SSLAcceptCallback + * assumes ownership of this socket, and is responsible + * for closing it when done. + */ + virtual void connectionAccepted( + const boost::shared_ptr &sock) + THRIFT_NOEXCEPT = 0; + + /** + * acceptError() is called if an error occurs while accepting. + * + * The SSLAcceptCallback will remain installed even after an accept error. + * If the callback wants to uninstall itself and stop trying to accept new + * connections, it must explicit call setAcceptCallback(NULL). + * + * @param ex An exception representing the error. + */ + virtual void acceptError(const std::exception& ex) THRIFT_NOEXCEPT = 0; + }; + + /** + * Create a new TAsyncSSLServerSocket with the specified TEventBase. + * + * @param eventBase The TEventBase to use for driving the asynchronous I/O. + * If this parameter is NULL, attachEventBase() must be + * called before this socket can begin accepting + * connections. All TAsyncSSLSocket objects accepted by + * this server socket will be attached to this TEventBase + * when they are created. + */ + explicit TAsyncSSLServerSocket( + const boost::shared_ptr& ctx, + TEventBase* eventBase = NULL); + + /** + * Destroy the socket. + * + * destroy() must be called to destroy the socket. The normal destructor is + * private, and should not be invoked directly. This prevents callers from + * deleting a TAsyncSSLServerSocket while it is invoking a callback. + */ + virtual void destroy(); + + void bind(const transport::TSocketAddress& address) { + serverSocket_->bind(address); + } + void bind(uint16_t port) { + serverSocket_->bind(port); + } + void getAddress(transport::TSocketAddress* addressReturn) { + serverSocket_->getAddress(addressReturn); + } + void listen(int backlog) { + serverSocket_->listen(backlog); + } + + /** + * Helper function to create a shared_ptr. + * + * This passes in the correct destructor object, since TAsyncSSLServerSocket's + * destructor is protected and cannot be invoked directly. + */ + static boost::shared_ptr newSocket( + const boost::shared_ptr& ctx, + TEventBase* evb) { + return boost::shared_ptr( + new TAsyncSSLServerSocket(ctx, evb), + Destructor()); + } + + /** + * Set the accept callback. + * + * This method may only be invoked from the TEventBase's loop thread. + * + * @param callback The callback to invoke when a new socket + * connection is accepted and a new TAsyncSSLSocket is + * created. + * + * Throws TTransportException on error. + */ + void setSSLAcceptCallback(SSLAcceptCallback* callback); + + SSLAcceptCallback *getSSLAcceptCallback() const { + return sslCallback_; + } + + void attachEventBase(TEventBase* eventBase); + void detachEventBase(); + + /** + * Returns the TEventBase that the handler is currently attached to. + */ + TEventBase* getEventBase() const { + return eventBase_; + } + + protected: + /** + * Protected destructor. + * + * Invoke destroy() instead to destroy the TAsyncSSLServerSocket. + */ + virtual ~TAsyncSSLServerSocket(); + + private: + virtual void connectionAccepted(int fd, + const transport::TSocketAddress& clientAddr) + THRIFT_NOEXCEPT; + virtual void acceptError(const std::exception& ex) THRIFT_NOEXCEPT; + + TEventBase* eventBase_; + TAsyncServerSocket* serverSocket_; + // SSL context + boost::shared_ptr ctx_; + // The accept callback + SSLAcceptCallback* sslCallback_; +}; + +}}} // apache::thrift::async + +#endif // THRIFT_ASYNC_TASYNCSSLSERVERSOCKET_H_ diff --git a/thrift/lib/cpp/async/TAsyncSSLSocket.h b/thrift/lib/cpp/async/TAsyncSSLSocket.h new file mode 100644 index 0000000000..f5b312f102 --- /dev/null +++ b/thrift/lib/cpp/async/TAsyncSSLSocket.h @@ -0,0 +1,357 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_ASYNC_TASYNCSSLSOCKET_H_ +#define THRIFT_ASYNC_TASYNCSSLSOCKET_H_ 1 + +#include "thrift/lib/cpp/async/TAsyncSocket.h" +#include "thrift/lib/cpp/async/TAsyncTimeout.h" +#include "thrift/lib/cpp/transport/TSSLSocket.h" +#include "thrift/lib/cpp/transport/TTransportException.h" + +namespace apache { namespace thrift { + +namespace async { + +class TSSLException: public apache::thrift::transport::TTransportException { + public: + TSSLException(int sslError, int errno_copy); + + int getSSLError() const { return error_; } + + protected: + int error_; +}; + +/** + * A class for performing asynchronous I/O on an SSL connection. + * + * TAsyncSSLSocket allows users to asynchronously wait for data on an + * SSL connection, and to asynchronously send data. + * + * The APIs for reading and writing are intentionally asymmetric. + * Waiting for data to read is a persistent API: a callback is + * installed, and is notified whenever new data is available. It + * continues to be notified of new events until it is uninstalled. + * + * TAsyncSSLSocket does not provide read timeout functionality, + * because it typically cannot determine when the timeout should be + * active. Generally, a timeout should only be enabled when + * processing is blocked waiting on data from the remote endpoint. + * For server connections, the timeout should not be active if the + * server is currently processing one or more outstanding requests for + * this connection. For client connections, the timeout should not be + * active if there are no requests pending on the connection. + * Additionally, if a client has multiple pending requests, it will + * ususally want a separate timeout for each request, rather than a + * single read timeout. + * + * The write API is fairly intuitive: a user can request to send a + * block of data, and a callback will be informed once the entire + * block has been transferred to the kernel, or on error. + * TAsyncSSLSocket does provide a send timeout, since most callers + * want to give up if the remote end stops responding and no further + * progress can be made sending the data. + */ +class TAsyncSSLSocket : public TAsyncSocket { + public: + +#if THRIFT_HAVE_UNIQUE_PTR + typedef std::unique_ptr UniquePtr; +#endif + + class HandshakeCallback { + public: + virtual ~HandshakeCallback() {} + + /** + * handshakeSuccess() is called when a new SSL connection is + * established, i.e., after SSL_accept/connect() returns successfully. + * + * The HandshakeCallback will be uninstalled before handshakeSuccess() + * is called. + * + * @param sock SSL socket on which the handshake was initiated + */ + virtual void handshakeSuccess(TAsyncSSLSocket *sock) THRIFT_NOEXCEPT = 0; + + /** + * handshakeError() is called if an error occurs while + * establishing the SSL connection. + * + * The HandshakeCallback will be uninstalled before handshakeError() + * is called. + * + * @param sock SSL socket on which the handshake was initiated + * @param ex An exception representing the error. + */ + virtual void handshakeError( + TAsyncSSLSocket *sock, + const apache::thrift::transport::TTransportException& ex) + THRIFT_NOEXCEPT = 0; + }; + + class HandshakeTimeout : public TAsyncTimeout { + public: + HandshakeTimeout(TAsyncSSLSocket* sslSocket, TEventBase* eventBase) + : TAsyncTimeout(eventBase) + , sslSocket_(sslSocket) {} + + virtual void timeoutExpired() THRIFT_NOEXCEPT { + sslSocket_->timeoutExpired(); + } + + private: + TAsyncSSLSocket* sslSocket_; + }; + + /** + * Create a client TAsyncSSLSocket + */ + TAsyncSSLSocket(const boost::shared_ptr &ctx, + TEventBase* evb) : + TAsyncSocket(evb), + corked_(false), + server_(false), + sslState_(STATE_UNINIT), + ctx_(ctx), + handshakeCallback_(NULL), + ssl_(NULL), + sslSession_(NULL), + handshakeTimeout_(this, evb) { + } + + /** + * Create a TAsyncSSLSocket from an already connected socket file descriptor. + * + * Note that while TAsyncSSLSocket enables TCP_NODELAY for sockets it creates + * when connecting, it does not change the socket options when given an + * existing file descriptor. If callers want TCP_NODELAY enabled when using + * this version of the constructor, they need to explicitly call + * setNoDelay(true) after the constructor returns. + * + * @param ctx SSL context for this connection. + * @param evb EventBase that will manage this socket. + * @param fd File descriptor to take over (should be a connected socket). + */ + TAsyncSSLSocket(const boost::shared_ptr& ctx, + TEventBase* evb, int fd, bool server = true); + + /** + * Helper function to create a shared_ptr. + */ + static boost::shared_ptr newSocket( + const boost::shared_ptr& ctx, + TEventBase* evb, int fd, bool server=true) { + return boost::shared_ptr( + new TAsyncSSLSocket(ctx, evb, fd, server), + Destructor()); + } + + /** + * Helper function to create a shared_ptr. + */ + static boost::shared_ptr newSocket( + const boost::shared_ptr& ctx, + TEventBase* evb) { + return boost::shared_ptr( + new TAsyncSSLSocket(ctx, evb), + Destructor()); + } + + /** + * TODO: implement support for SSL renegosiation. + * + * This involves proper handling of the SSL_ERROR_WANT_READ/WRITE + * code as a result of SSL_write/read(), instead of returning an + * error. In that case, the READ/WRITE event should be registered, + * and a flag (e.g., writeBlockedOnRead) should be set to indiciate + * the condition. In the next invocation of read/write callback, if + * the flag is on, performWrite()/performRead() should be called in + * addition to the normal call to performRead()/performWrite(), and + * the flag should be reset. + */ + + // Inherit TAsyncTransport methods from TAsyncSocket except the + // following. + // See the documentation in TAsyncTransport.h + // TODO: implement graceful shutdown in close() + // TODO: implement detachSSL() that returns the SSL connection + virtual void closeNow(); + virtual void shutdownWrite(); + virtual void shutdownWriteNow(); + virtual bool good() const; + virtual bool connecting() const; + + /** + * Accept an SSL connection on the socket. + * + * The callback will be invoked and uninstalled when an SSL + * connection has been established on the underlying socket. + * + * @param callback callback object to invoke on success/failure + * @param timeout timeout for this function in milliseconds, or 0 for no + * timeout + */ + void sslAccept(HandshakeCallback* callback, uint32_t timeout = 0); + + /** + * Invoke SSL accept following an asynchronous session cache lookup + */ + void restartSSLAccept(); + + /** + * Connect to the given address, invoking callback when complete or on error + * + * Note timeout applies to TCP + SSL connection time + */ + void connect(ConnectCallback* callback, + const transport::TSocketAddress& address, + int timeout = 0, + const OptionList &options = emptyOptionList) THRIFT_NOEXCEPT; + + using TAsyncSocket::connect; + + /** + * Initiate an SSL connection on the socket + * THe callback will be invoked and uninstalled when an SSL connection + * has been establshed on the underlying socket. + * + * @param callback callback object to invoke on success/failure + * @param timeout timeout for this function in milliseconds, or 0 for no + * timeout + */ + void sslConnect(HandshakeCallback *callback, uint64_t timeout = 0); + + enum SSLStateEnum { + STATE_UNINIT, + STATE_ACCEPTING, + STATE_CACHE_LOOKUP, + STATE_CONNECTING, + STATE_ESTABLISHED, + STATE_REMOTE_CLOSED, /// remote end closed; we can still write + STATE_CLOSING, ///< close() called, but waiting on writes to complete + /// close() called with pending writes, before connect() has completed + STATE_CONNECTING_CLOSING, + STATE_CLOSED, + STATE_ERROR + }; + + SSLStateEnum getSSLState() const { return sslState_;} + + /** + * Get a handle to the negotiated SSL session. This increments the session + * refcount and must be deallocated by the caller. + */ + SSL_SESSION *getSSLSession(); + + /** + * Set the SSL session to be used during sslConnect. TAsyncSSLSocket will + * hold a reference to the session until it is destroyed or released by the + * underlying SSL structure. + * + * @param takeOwnership if true, TAsyncSSLSocket will assume the caller's + * reference count to session. + */ + void setSSLSession(SSL_SESSION *session, bool takeOwnership = false); + +#ifdef OPENSSL_NPN_NEGOTIATED + /** + * Get the name of the protocol selected by the client during + * Next Protocol Negotiation (NPN) + * + * @param protoName Name of the protocol (not guaranteed to be + * null terminated); will be set to NULL if + * the client did not negotiate a protocol. + * Note: the TAsyncSSLSocket retains ownership + * of this string. + * @param protoNameLen Length of the name. + */ + void getSelectedNextProtocol(const unsigned char** protoName, + unsigned* protoLen); +#endif // OPENSSL_NPN_NEGOTIATED + + /** + * Determine if the session specified during setSSLSession was reused + * or if the server rejected it and issued a new session. + */ + bool getSSLSessionReused() const; + + virtual void attachEventBase(TEventBase* eventBase) { + TAsyncSocket::attachEventBase(eventBase); + handshakeTimeout_.attachEventBase(eventBase); + } + + virtual void detachEventBase() { + TAsyncSocket::detachEventBase(); + handshakeTimeout_.detachEventBase(); + } + + void timeoutExpired() THRIFT_NOEXCEPT; + + protected: + + /** + * Protected destructor. + * + * Users of TAsyncSSLSocket must never delete it directly. Instead, invoke + * destroy() instead. (See the documentation in TDelayedDestruction.h for + * more details.) + */ + ~TAsyncSSLSocket(); + + // Inherit event notification methods from TAsyncSocket except + // the following. + + void handleRead() THRIFT_NOEXCEPT; + void handleWrite() THRIFT_NOEXCEPT; + void handleAccept() THRIFT_NOEXCEPT; + void handleConnect() THRIFT_NOEXCEPT; + + void invalidState(HandshakeCallback* callback); + bool willBlock(int ret, int *errorOut) THRIFT_NOEXCEPT; + + // TAsyncSocket calls this at the wrong time for SSL + void handleInitialReadWrite() THRIFT_NOEXCEPT {} + + ssize_t performRead(void* buf, size_t buflen); + ssize_t performWrite(const iovec* vec, uint32_t count, bool haveMore, + uint32_t* countWritten, uint32_t* partialWritten); + + // Inherit error handling methods from TAsyncSocket, plus the following. + void failHandshake(const char* fn, const transport::TTransportException& ex); + + void invokeHandshakeCallback(); + + // Whether we've applied the TCP_CORK option to the socket + bool corked_; + // SSL related members. + bool server_; + SSLStateEnum sslState_; + boost::shared_ptr ctx_; + // Callback for SSL_accept() or SSL_connect() + HandshakeCallback* handshakeCallback_; + SSL* ssl_; + SSL_SESSION *sslSession_; + HandshakeTimeout handshakeTimeout_; +}; + +}}} // apache::thrift::async + +#endif // #ifndef THRIFT_ASYNC_TASYNCSSLSOCKET_H_ diff --git a/thrift/lib/cpp/async/TAsyncServerSocket.h b/thrift/lib/cpp/async/TAsyncServerSocket.h new file mode 100644 index 0000000000..5c2a46bff3 --- /dev/null +++ b/thrift/lib/cpp/async/TAsyncServerSocket.h @@ -0,0 +1,513 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_ASYNC_TASYNCSERVERSOCKET_H_ +#define THRIFT_ASYNC_TASYNCSERVERSOCKET_H_ 1 + +#include "thrift/lib/cpp/thrift_config.h" +#include "thrift/lib/cpp/async/TDelayedDestruction.h" +#include "thrift/lib/cpp/async/TEventHandler.h" +#include +#include +#include +#include +#include +#include +#include + +namespace apache { namespace thrift { + +namespace transport { +class TSocketAddress; +} + +namespace async { + +class TNotificationPipe; + +/** + * A listening socket that asynchronously informs a callback whenever a new + * connection has been accepted. + * + * Unlike most async interfaces that always invoke their callback in the same + * TEventBase thread, TAsyncServerSocket is unusual in that it can distribute + * the callbacks across multiple TEventBase threads. + * + * This supports a common use case for network servers to distribute incoming + * connections across a number of TEventBase threads. (Servers typically run + * with one TEventBase thread per CPU.) + * + * Despite being able to invoke callbacks in multiple TEventBase threads, + * TAsyncServerSocket still has one "primary" TEventBase. Operations that + * modify the TAsyncServerSocket state may only be performed from the primary + * TEventBase thread. + */ +class TAsyncServerSocket : public TDelayedDestruction, + private TEventHandler { + public: +#if THRIFT_HAVE_UNIQUE_PTR + typedef std::unique_ptr UniquePtr; +#endif + + class AcceptCallback { + public: + virtual ~AcceptCallback() {} + + /** + * connectionAccepted() is called whenever a new client connection is + * received. + * + * The AcceptCallback will remain installed after connectionAccepted() + * returns. + * + * @param fd The newly accepted client socket. The AcceptCallback + * assumes ownership of this socket, and is responsible + * for closing it when done. The newly accepted file + * descriptor will have already been put into + * non-blocking mode. + * @param clientAddr A reference to a TSocketAddress struct containing the + * client's address. This struct is only guaranteed to + * remain valid until connectionAccepted() returns. + */ + virtual void connectionAccepted(int fd, + const transport::TSocketAddress& clientAddr) + THRIFT_NOEXCEPT = 0; + + /** + * acceptError() is called if an error occurs while accepting. + * + * The AcceptCallback will remain installed even after an accept error, + * as the errors are typically somewhat transient, such as being out of + * file descriptors. The server socket must be explicitly stopped if you + * wish to stop accepting after an error. + * + * @param ex An exception representing the error. + */ + virtual void acceptError(const std::exception& ex) THRIFT_NOEXCEPT = 0; + + /** + * acceptStarted() will be called in the callback's TEventBase thread + * after this callback has been added to the TAsyncServerSocket. + * + * acceptStarted() will be called before any calls to connectionAccepted() + * or acceptError() are made on this callback. + * + * acceptStarted() makes it easier for callbacks to perform initialization + * inside the callback thread. (The call to addAcceptCallback() must + * always be made from the TAsyncServerSocket's primary TEventBase thread. + * acceptStarted() provides a hook that will always be invoked in the + * callback's thread.) + * + * Note that the call to acceptStarted() is made once the callback is + * added, regardless of whether or not the TAsyncServerSocket is actually + * accepting at the moment. acceptStarted() will be called even if the + * TAsyncServerSocket is paused when the callback is added (including if + * the initial call to startAccepting() on the TAsyncServerSocket has not + * been made yet). + */ + virtual void acceptStarted() THRIFT_NOEXCEPT {} + + /** + * acceptStopped() will be called when this AcceptCallback is removed from + * the TAsyncServerSocket, or when the TAsyncServerSocket is destroyed, + * whichever occurs first. + * + * No more calls to connectionAccepted() or acceptError() will be made + * after acceptStopped() is invoked. + */ + virtual void acceptStopped() THRIFT_NOEXCEPT {} + }; + + static const uint32_t kDefaultMaxAcceptAtOnce = 30; + static const uint32_t kDefaultCallbackAcceptAtOnce = 5; + static const uint32_t kDefaultMaxMessagesInPipe = 0xffffffff; + /** + * Create a new TAsyncServerSocket with the specified TEventBase. + * + * @param eventBase The TEventBase to use for driving the asynchronous I/O. + * If this parameter is NULL, attachEventBase() must be + * called before this socket can begin accepting + * connections. + */ + explicit TAsyncServerSocket(TEventBase* eventBase = NULL); + + /** + * Helper function to create a shared_ptr. + * + * This passes in the correct destructor object, since TAsyncServerSocket's + * destructor is protected and cannot be invoked directly. + */ + static boost::shared_ptr newSocket(TEventBase* evb) { + return boost::shared_ptr(new TAsyncServerSocket(evb), + Destructor()); + } + + /** + * Destroy the socket. + * + * TAsyncServerSocket::destroy() must be called to destroy the socket. + * The normal destructor is private, and should not be invoked directly. + * This prevents callers from deleting a TAsyncServerSocket while it is + * invoking a callback. + * + * destroy() must be invoked from the socket's primary TEventBase thread. + * + * If there are AcceptCallbacks still installed when destroy() is called, + * acceptStopped() will be called on these callbacks to notify them that + * accepting has stopped. Accept callbacks being driven by other TEventBase + * threads may continue to receive new accept callbacks for a brief period of + * time after destroy() returns. They will not receive any more callback + * invocations once acceptStopped() is invoked. + */ + virtual void destroy(); + + /** + * Attach this TAsyncServerSocket to its primary TEventBase. + * + * This may only be called if the TAsyncServerSocket is not already attached + * to a TEventBase. The TAsyncServerSocket must be attached to a TEventBase + * before it can begin accepting connections. + */ + void attachEventBase(TEventBase *eventBase); + + /** + * Detach the TAsyncServerSocket from its primary TEventBase. + * + * detachEventBase() may only be called if the TAsyncServerSocket is not + * currently accepting connections. + */ + void detachEventBase(); + + /** + * Get the TEventBase used by this socket. + */ + TEventBase* getEventBase() const { + return eventBase_; + } + + /** + * Create a TAsyncServerSocket from an existing socket file descriptor. + * + * useExistingSocket() will cause the TAsyncServerSocket to take ownership of + * the specified file descriptor, and use it to listen for new connections. + * The TAsyncServerSocket will close the file descriptor when it is + * destroyed. + * + * useExistingSocket() must be called before bind() or listen(). + * + * The supplied file descriptor will automatically be put into non-blocking + * mode. The caller may have already directly called bind() and possibly + * listen on the file descriptor. If so the caller should skip calling the + * corresponding TAsyncServerSocket::bind() and listen() methods. + * + * On error a TTransportException will be thrown and the caller will retain + * ownership of the file descriptor. + */ + void useExistingSocket(int fd); + + /** + * Return the underlying file descriptor + */ + int getSocket() const { + return socket_; + } + + /** + * Bind to the specified address. + * + * This must be called from the primary TEventBase thread. + * + * Throws TTransportException on error. + */ + void bind(const transport::TSocketAddress& address); + + /** + * Bind to the specified port. + * + * This must be called from the primary TEventBase thread. + * + * Throws TTransportException on error. + */ + void bind(uint16_t port); + + /** + * Get the local address to which the socket is bound. + * + * Throws TTransportException on error. + */ + void getAddress(transport::TSocketAddress* addressReturn); + + /** + * Begin listening for connections. + * + * This calls ::listen() with the specified backlog. + * + * Once listen() is invoked the socket will actually be open so that remote + * clients may establish connections. (Clients that attempt to connect + * before listen() is called will receive a connection refused error.) + * + * At least one callback must be set and startAccepting() must be called to + * actually begin notifying the accept callbacks of newly accepted + * connections. The backlog parameter controls how many connections the + * kernel will accept and buffer internally while the accept callbacks are + * paused (or if accepting is enabled but the callbacks cannot keep up). + * + * bind() must be called before calling listen(). + * listen() must be called from the primary TEventBase thread. + * + * Throws TTransportException on error. + */ + void listen(int backlog); + + /** + * Add an AcceptCallback. + * + * When a new socket is accepted, one of the AcceptCallbacks will be invoked + * with the new socket. The AcceptCallbacks are invoked in a round-robin + * fashion. This allows the accepted sockets to distributed among a pool of + * threads, each running its own TEventBase object. This is a common model, + * since most asynchronous-style servers typically run one TEventBase thread + * per CPU. + * + * The TEventBase object associated with each AcceptCallback must be running + * its loop. If the TEventBase loop is not running, sockets will still be + * scheduled for the callback, but the callback cannot actually get invoked + * until the loop runs. + * + * This method must be invoked from the TAsyncServerSocket's primary + * TEventBase thread. + * + * Note that startAccepting() must be called on the TAsyncServerSocket to + * cause it to actually start accepting sockets once callbacks have been + * installed. + * + * @param callback The callback to invoke. + * @param eventBase The TEventBase to use to invoke the callback. This + * parameter may be NULL, in which case the callback will be invoked in + * the TAsyncServerSocket's primary TEventBase. + * @param maxAtOnce The maximum number of connections to accept in this + * callback on a single iteration of the event base loop. + * This only takes effect when eventBase is non-NULL. When + * using a NULL eventBase for the callback, the + * setMaxAcceptAtOnce() method controls how many + * connections the main event base will accept at once. + */ + void addAcceptCallback(AcceptCallback *callback, TEventBase *eventBase, + uint32_t maxAtOnce = kDefaultCallbackAcceptAtOnce); + + /** + * Remove an AcceptCallback. + * + * This allows a single AcceptCallback to be removed from the round-robin + * pool. + * + * This method must be invoked from the TAsyncServerSocket's primary + * TEventBase thread. Use TEventBase::runInEventBaseThread() to schedule the + * operation in the correct TEventBase if your code is not in the server + * socket's primary TEventBase. + * + * Given that the accept callback is being driven by a different TEventBase, + * the AcceptCallback may continue to be invoked for a short period of time + * after removeAcceptCallback() returns in this thread. Once the other + * TEventBase thread receives the notification to stop, it will call + * acceptStopped() on the callback to inform it that it is fully stopped and + * will not receive any new sockets. + * + * If the last accept callback is removed while the socket is accepting, + * the socket will implicitly pause accepting. If a callback is later added, + * it will resume accepting immediately, without requiring startAccepting() + * to be invoked. + * + * @param callback The callback to uninstall. + * @param eventBase The TEventBase associated with this callback. This must + * be the same TEventBase that was used when the callback was installed + * with addAcceptCallback(). + */ + void removeAcceptCallback(AcceptCallback *callback, TEventBase *eventBase); + + /** + * Begin accepting connctions on this socket. + * + * bind() and listen() must be called before calling startAccepting(). + * + * When a TAsyncServerSocket is initially created, it will not begin + * accepting connections until at least one callback has been added and + * startAccepting() has been called. startAccepting() can also be used to + * resume accepting connections after a call to pauseAccepting(). + * + * If startAccepting() is called when there are no accept callbacks + * installed, the socket will not actually begin accepting until an accept + * callback is added. + * + * This method may only be called from the primary TEventBase thread. + */ + void startAccepting(); + + /** + * Pause accepting connections. + * + * startAccepting() may be called to resume accepting. + * + * This method may only be called from the primary TEventBase thread. + * If there are AcceptCallbacks being driven by other TEventBase threads they + * may continue to receive callbacks for a short period of time after + * pauseAccepting() returns. + * + * Unlike removeAcceptCallback() or destroy(), acceptStopped() will not be + * called on the AcceptCallback objects simply due to a temporary pause. If + * the server socket is later destroyed while paused, acceptStopped() will be + * called all of the installed AcceptCallbacks. + */ + void pauseAccepting(); + + /** + * Get the maximum number of connections that will be accepted each time + * around the event loop. + */ + uint32_t getMaxAcceptAtOnce() const { + return maxAcceptAtOnce_; + } + + /** + * Set the maximum number of connections that will be accepted each time + * around the event loop. + * + * This provides a very coarse-grained way of controlling how fast the + * TAsyncServerSocket will accept connections. If you find that when your + * server is overloaded TAsyncServerSocket accepts connections more quickly + * than your code can process them, you can try lowering this number so that + * fewer connections will be accepted each event loop iteration. + * + * For more explicit control over the accept rate, you can also use + * pauseAccepting() to temporarily pause accepting when your server is + * overloaded, and then use startAccepting() later to resume accepting. + */ + void setMaxAcceptAtOnce(uint32_t numConns) { + maxAcceptAtOnce_ = numConns; + } + + /** + * Get the maximum number of unprocessed messages which a NotificationPipe + * can hold. + */ + uint32_t getMaxNumMessagesInPipe() const { + return maxNumMsgsInPipe_; + } + + /** + * Set the maximum number of unprocessed messages in NotificationPipe. + * No new message will be sent to that NotificationPipe if there are more + * than such number of unprocessed messages in that pipe. + */ + void setMaxNumMessagesInPipe(uint32_t num) { + maxNumMsgsInPipe_ = num; + } + + /** + * Get the speed of adjusting connection accept rate. + */ + double getAcceptRateAdjustSpeed() const { + return acceptRateAdjustSpeed_; + } + + /** + * Set the speed of adjusting connection accept rate. + */ + void setAcceptRateAdjustSpeed(double speed) { + acceptRateAdjustSpeed_ = speed; + } + + /** + * Get the number of connections dropped by the TAsyncServerSocket + */ + double getNumDroppedConnections() const { + return numDroppedConnections_; + } + + protected: + /** + * Protected destructor. + * + * Invoke destroy() instead to destroy the TAsyncServerSocket. + */ + virtual ~TAsyncServerSocket(); + + private: + /** + * A struct to keep track of the callbacks associated with this server + * socket. + */ + struct CallbackInfo { + CallbackInfo(AcceptCallback *callback, TEventBase *eventBase) + : callback(callback), + eventBase(eventBase), + pipe(NULL) {} + + AcceptCallback *callback; + TEventBase *eventBase; + // Note that the TNotificationPipe is actually owned by the RemoteAcceptor. + // The RemoteAcceptor will destroy the TNotificationPipe (and itself) + // once the pipe is closed by the TAsyncServerSocket. + TNotificationPipe *pipe; + }; + class RemoteAcceptor; + enum MessageType { + MSG_NEW_CONN = 0, + MSG_ERROR = 1 + }; + class BackoffTimeout; + + // Inherited from TEventHandler + virtual void handlerReady(uint16_t events) THRIFT_NOEXCEPT; + + int createSocket(int family); + void setupSocket(int fd); + void dispatchSocket(int socket, const transport::TSocketAddress& address); + void dispatchError(const char *msg, int errnoValue); + void enterBackoff(); + void backoffTimeoutExpired(); + + CallbackInfo* nextCallback() { + CallbackInfo* info = &callbacks_[callbackIndex_]; + + ++callbackIndex_; + if (callbackIndex_ >= callbacks_.size()) { + callbackIndex_ = 0; + } + + return info; + } + + TEventBase *eventBase_; + sa_family_t addressFamily_; + int socket_; + bool accepting_; + uint32_t maxAcceptAtOnce_; + uint32_t maxNumMsgsInPipe_; + double acceptRateAdjustSpeed_; //0 to disable auto adjust + double acceptRate_; + int64_t lastAccepTimestamp_; // milliseconds + int64_t numDroppedConnections_; + uint32_t callbackIndex_; + BackoffTimeout *backoffTimeout_; + std::vector callbacks_; +}; + +}}} // apache::thrift::async + +#endif // THRIFT_ASYNC_TASYNCSERVERSOCKET_H_ diff --git a/thrift/lib/cpp/async/TAsyncSignalHandler.h b/thrift/lib/cpp/async/TAsyncSignalHandler.h new file mode 100644 index 0000000000..7a80be21b2 --- /dev/null +++ b/thrift/lib/cpp/async/TAsyncSignalHandler.h @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_ASYNC_TASYNCSIGNALHANDLER_H_ +#define THRIFT_ASYNC_TASYNCSIGNALHANDLER_H_ 1 + +#include "thrift/lib/cpp/thrift_config.h" +#include +#include + +namespace apache { namespace thrift { namespace async { + +class TEventBase; + +/** + * A handler to receive notification about POSIX signals. + * + * TAsyncSignalHandler allows code to process signals from within a TEventBase + * loop. Standard signal handlers interrupt execution of the main thread, and + * are run while the main thread is paused. As a result, great care must be + * taken to avoid race conditions if the signal handler has to access or modify + * any data used by the main thread. + * + * TAsyncSignalHandler solves this problem by running the TAsyncSignalHandler + * callback in normal thread of execution, as a TEventBase callback. + * + * TAsyncSignalHandler may only be used in a single thread. It will only + * process signals received by the thread where the TAsyncSignalHandler is + * registered. It is the user's responsibility to ensure that signals are + * delivered to the desired thread in multi-threaded programs. + */ +class TAsyncSignalHandler { + public: + /** + * Create a new TAsyncSignalHandler. + */ + TAsyncSignalHandler(TEventBase* eventBase); + virtual ~TAsyncSignalHandler(); + + /** + * Register to receive callbacks about the specified signal. + * + * Once the handler has been registered for a particular signal, + * signalReceived() will be called each time this thread receives this + * signal. + * + * Throws a TException if an error occurs, or if this handler is already + * registered for this signal. + */ + void registerSignalHandler(int signum); + + /** + * Unregister for callbacks about the specified signal. + * + * Throws a TException if an error occurs, or if this signal was not + * registered. + */ + void unregisterSignalHandler(int signum); + + /** + * signalReceived() will called to indicate that the specified signal has + * been received. + * + * signalReceived() will always be invoked from the TEventBase loop (i.e., + * after the main POSIX signal handler has returned control to the TEventBase + * thread). + */ + virtual void signalReceived(int signum) THRIFT_NOEXCEPT = 0; + + private: + typedef std::map SignalEventMap; + + // Forbidden copy constructor and assignment operator + TAsyncSignalHandler(TAsyncSignalHandler const &); + TAsyncSignalHandler& operator=(TAsyncSignalHandler const &); + + static void libeventCallback(int signum, short events, void* arg); + + TEventBase* eventBase_; + SignalEventMap signalEvents_; +}; + +}}} // apache::thrift::async + +#endif // THRIFT_ASYNC_TASYNCSIGNALHANDLER_H_ diff --git a/thrift/lib/cpp/async/TAsyncSocket.h b/thrift/lib/cpp/async/TAsyncSocket.h new file mode 100644 index 0000000000..ea18a81af4 --- /dev/null +++ b/thrift/lib/cpp/async/TAsyncSocket.h @@ -0,0 +1,551 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_ASYNC_TASYNCSOCKET_H_ +#define THRIFT_ASYNC_TASYNCSOCKET_H_ 1 + +#include +#include +#include "thrift/lib/cpp/async/TAsyncTimeout.h" +#include "thrift/lib/cpp/async/TAsyncTransport.h" +#include "thrift/lib/cpp/async/TDelayedDestruction.h" +#include "thrift/lib/cpp/async/TEventHandler.h" + +#include +#include + +namespace apache { namespace thrift { namespace async { + +/** + * A class for performing asynchronous I/O on a socket. + * + * TAsyncSocket allows users to asynchronously wait for data on a socket, and + * to asynchronously send data. + * + * The APIs for reading and writing are intentionally asymmetric. Waiting for + * data to read is a persistent API: a callback is installed, and is notified + * whenever new data is available. It continues to be notified of new events + * until it is uninstalled. + * + * TAsyncSocket does not provide read timeout functionality, because it + * typically cannot determine when the timeout should be active. Generally, a + * timeout should only be enabled when processing is blocked waiting on data + * from the remote endpoint. For server sockets, the timeout should not be + * active if the server is currently processing one or more outstanding + * requests for this socket. For client sockets, the timeout should not be + * active if there are no requests pending on the socket. Additionally, if a + * client has multiple pending requests, it will ususally want a separate + * timeout for each request, rather than a single read timeout. + * + * The write API is fairly intuitive: a user can request to send a block of + * data, and a callback will be informed once the entire block has been + * transferred to the kernel, or on error. TAsyncSocket does provide a send + * timeout, since most callers want to give up if the remote end stops + * responding and no further progress can be made sending the data. + */ +class TAsyncSocket : public TAsyncTransport, + public TDelayedDestruction { + public: +#if THRIFT_HAVE_UNIQUE_PTR + typedef std::unique_ptr UniquePtr; +#endif + + class ConnectCallback { + public: + virtual ~ConnectCallback() {} + + /** + * connectSuccess() will be invoked when the connection has been + * successfully established. + */ + virtual void connectSuccess() THRIFT_NOEXCEPT = 0; + + /** + * connectError() will be invoked if the connection attempt fails. + * + * @param ex An exception describing the error that occurred. + */ + virtual void connectError(const transport::TTransportException& ex) + THRIFT_NOEXCEPT = 0; + }; + + /** + * Create a new unconnected TAsyncSocket. + * + * connect() must later be called on this socket to establish a connection. + */ + explicit TAsyncSocket(TEventBase* evb); + + /** + * Create a new TAsyncSocket and begin the connection process. + * + * @param evb EventBase that will manage this socket. + * @param address The address to connect to. + * @param connectTimeout Optional timeout in milliseconds for the connection + * attempt. + */ + TAsyncSocket(TEventBase* evb, + const transport::TSocketAddress& address, + uint32_t connectTimeout = 0); + + /** + * Create a new TAsyncSocket and begin the connection process. + * + * @param evb EventBase that will manage this socket. + * @param ip IP address to connect to (dotted-quad). + * @param port Destination port in host byte order. + * @param connectTimeout Optional timeout in milliseconds for the connection + * attempt. + */ + TAsyncSocket(TEventBase* evb, + const std::string& ip, + uint16_t port, + uint32_t connectTimeout = 0); + + /** + * Create a TAsyncSocket from an already connected socket file descriptor. + * + * Note that while TAsyncSocket enables TCP_NODELAY for sockets it creates + * when connecting, it does not change the socket options when given an + * existing file descriptor. If callers want TCP_NODELAY enabled when using + * this version of the constructor, they need to explicitly call + * setNoDelay(true) after the constructor returns. + * + * @param evb EventBase that will manage this socket. + * @param fd File descriptor to take over (should be a connected socket). + */ + TAsyncSocket(TEventBase* evb, int fd); + + /** + * Helper function to create a shared_ptr. + * + * This passes in the correct destructor object, since TAsyncSocket's + * destructor is protected and cannot be invoked directly. + */ + static boost::shared_ptr newSocket(TEventBase* evb) { + return boost::shared_ptr(new TAsyncSocket(evb), + Destructor()); + } + + /** + * Helper function to create a shared_ptr. + */ + static boost::shared_ptr newSocket( + TEventBase* evb, + const transport::TSocketAddress& address, + uint32_t connectTimeout = 0) { + return boost::shared_ptr( + new TAsyncSocket(evb, address, connectTimeout), + Destructor()); + } + + /** + * Helper function to create a shared_ptr. + */ + static boost::shared_ptr newSocket( + TEventBase* evb, + const std::string& ip, + uint16_t port, + uint32_t connectTimeout = 0) { + return boost::shared_ptr( + new TAsyncSocket(evb, ip, port, connectTimeout), + Destructor()); + } + + /** + * Helper function to create a shared_ptr. + */ + static boost::shared_ptr newSocket(TEventBase* evb, int fd) { + return boost::shared_ptr(new TAsyncSocket(evb, fd), + Destructor()); + } + + /** + * Destroy the socket. + * + * TAsyncSocket::destroy() must be called to destroy the socket. + * The normal destructor is private, and should not be invoked directly. + * This prevents callers from deleting a TAsyncSocket while it is invoking a + * callback. + */ + virtual void destroy(); + + /** + * Get the TEventBase used by this socket. + */ + virtual TEventBase* getEventBase() const { + return eventBase_; + } + + /** + * Get the file descriptor used by the TAsyncSocket. + */ + int getFd() const { + return fd_; + } + + /** + * Extract the file descriptor from the TAsyncSocket. + * + * This will immediately cause any installed callbacks to be invoked with an + * error. The TAsyncSocket may no longer be used after the file descriptor + * has been extracted. + * + * Returns the file descriptor. The caller assumes ownership of the + * descriptor, and it will not be closed when the TAsyncSocket is destroyed. + */ + int detachFd(); + + /** + * Class that consists of the input parameters for setsockopt(). + * + * The memory referenced by optval should be valid throughout the + * life cycle of the SocketOption object. + */ + class SocketOption { + public: + SocketOption(): level_(0), optname_(0), optval_(NULL), size_(0) {} + + template + SocketOption(int level, int optname, const T* optval): + level_(level), optname_(optname), optval_(optval), size_(sizeof(T)) {} + + int apply(int fd) const { + return setsockopt(fd, level_, optname_, optval_, size_); + } + + protected: + int level_; + int optname_; + const void *optval_; + size_t size_; + }; + + typedef std::list OptionList; + + static OptionList emptyOptionList; + + /** + * Initiate a connection. + * + * @param callback The callback to inform when the connection attempt + * completes. + * @param address The address to connect to. + * @param timeout A timeout value, in milliseconds. If the connection + * does not succeed within this period, + * callback->connectError() will be invoked. + */ + virtual void connect(ConnectCallback* callback, + const transport::TSocketAddress& address, + int timeout = 0, + const OptionList &options = emptyOptionList) THRIFT_NOEXCEPT; + void connect(ConnectCallback* callback, const std::string& ip, uint16_t port, + int timeout = 00, + const OptionList &options = emptyOptionList) THRIFT_NOEXCEPT; + + /** + * Set the send timeout. + * + * If write requests do not make any progress for more than the specified + * number of milliseconds, fail all pending writes and close the socket. + * + * If write requests are currently pending when setSendTimeout() is called, + * the timeout interval is immediately restarted using the new value. + * + * (See the comments for TAsyncSocket for an explanation of why TAsyncSocket + * provides setSendTimeout() but not setRecvTimeout().) + * + * @param milliseconds The timeout duration, in milliseconds. If 0, no + * timeout will be used. + */ + void setSendTimeout(uint32_t milliseconds); + + /** + * Get the send timeout. + * + * @return Returns the current send timeout, in milliseconds. A return value + * of 0 indicates that no timeout is set. + */ + uint32_t getSendTimeout() const { + return sendTimeout_; + } + + // Methods inherited from TAsyncTransport + // See the documentation in TAsyncTransport.h + virtual void setReadCallback(ReadCallback* callback); + virtual ReadCallback* getReadCallback() const; + + virtual void write(WriteCallback* callback, const void* buf, size_t bytes); + virtual void writev(WriteCallback* callback, const iovec* vec, size_t count); + virtual void writeChain(WriteCallback* callback, + std::unique_ptr&& buf, + bool cork = false); + + virtual void close(); + virtual void closeNow(); + virtual void shutdownWrite(); + virtual void shutdownWriteNow(); + + virtual bool readable() const; + virtual bool good() const; + virtual bool error() const; + virtual void attachEventBase(TEventBase* eventBase); + virtual void detachEventBase(); + + virtual void getLocalAddress(transport::TSocketAddress* address) const; + virtual void getPeerAddress(transport::TSocketAddress* address) const; + + virtual bool connecting() const { + return (state_ == STATE_CONNECTING); + } + + // Methods controlling socket options + + /** + * Force writes to be transmitted immediately. + * + * This controls the TCP_NODELAY socket option. When enabled, TCP segments + * are sent as soon as possible, even if it is not a full frame of data. + * When disabled, the data may be buffered briefly to try and wait for a full + * frame of data. + * + * By default, TCP_NODELAY is enabled for TAsyncSocket objects. + * + * This method will fail if the socket is not currently open. + * + * @return Returns 0 if the TCP_NODELAY flag was successfully updated, + * or a non-zero errno value on error. + */ + int setNoDelay(bool noDelay); + + /** + * Generic API for reading a socket option. + * + * @param level same as the "level" parameter in getsockopt(). + * @param optname same as the "optname" parameter in getsockopt(). + * @param optval pointer to the variable in which the option value should + * be returned. + * @return same as the return value of getsockopt(). + */ + template + int getSockOpt(int level, int optname, T *optval) { + return getsockopt(fd_, level, optname, optval, sizeof(T)); + } + + /** + * Generic API for setting a socket option. + * + * @param level same as the "level" parameter in getsockopt(). + * @param optname same as the "optname" parameter in getsockopt(). + * @param optval the option value to set. + * @return same as the return value of setsockopt(). + */ + template + int setSockOpt(int level, int optname, const T *optval) { + return setsockopt(fd_, level, optname, optval, sizeof(T)); + } + + protected: + enum ReadResultEnum { + READ_EOF = 0, + READ_ERROR = -1, + READ_BLOCKING = -2, + }; + + /** + * Protected destructor. + * + * Users of TAsyncSocket must never delete it directly. Instead, invoke + * destroy() instead. (See the documentation in TDelayedDestruction.h for + * more details.) + */ + ~TAsyncSocket(); + + enum StateEnum { + STATE_UNINIT, + STATE_CONNECTING, + STATE_ESTABLISHED, + STATE_CLOSED, + STATE_ERROR + }; + enum ShutdownFlags { + /// shutdownWrite() called, but we are still waiting on writes to drain + SHUT_WRITE_PENDING = 0x01, + /// writes have been completely shut down + SHUT_WRITE = 0x02, + /** + * Reads have been shutdown. + * + * At the moment we don't distinguish between remote read shutdown + * (received EOF from the remote end) and local read shutdown. We can + * only receive EOF when a read callback is set, and we immediately inform + * it of the EOF. Therefore there doesn't seem to be any reason to have a + * separate state of "received EOF but the local side may still want to + * read". + * + * We also don't currently provide any API for only shutting down the read + * side of a socket. (This is a no-op as far as TCP is concerned, anyway.) + */ + SHUT_READ = 0x04, + }; + + class WriteRequest; + + class WriteTimeout : public TAsyncTimeout { + public: + WriteTimeout(TAsyncSocket* socket, TEventBase* eventBase) + : TAsyncTimeout(eventBase) + , socket_(socket) {} + + virtual void timeoutExpired() THRIFT_NOEXCEPT { + socket_->timeoutExpired(); + } + + private: + TAsyncSocket* socket_; + }; + + class IoHandler : public TEventHandler { + public: + IoHandler(TAsyncSocket* socket, TEventBase* eventBase) + : TEventHandler(eventBase, -1) + , socket_(socket) {} + IoHandler(TAsyncSocket* socket, TEventBase* eventBase, int fd) + : TEventHandler(eventBase, fd) + , socket_(socket) {} + + virtual void handlerReady(uint16_t events) THRIFT_NOEXCEPT { + socket_->ioReady(events); + } + + private: + TAsyncSocket* socket_; + }; + + void init(); + + // event notification methods + void ioReady(uint16_t events) THRIFT_NOEXCEPT; + virtual void handleInitialReadWrite() THRIFT_NOEXCEPT; + virtual void handleRead() THRIFT_NOEXCEPT; + virtual void handleWrite() THRIFT_NOEXCEPT; + virtual void handleConnect() THRIFT_NOEXCEPT; + void timeoutExpired() THRIFT_NOEXCEPT; + + /** + * Attempt to read from the socket. + * + * @param buf The buffer to read data into. + * @param buflen The length of the buffer. + * + * @return Returns the number of bytes read, or READ_EOF on EOF, or + * READ_ERROR on error, or READ_BLOCKING if the operation will + * block. + */ + virtual ssize_t performRead(void* buf, size_t buflen); + + /** + * Write as much data as possible to the socket without blocking, + * and queue up any leftover data to send when the socket can + * handle writes again. + * + * @param callback The callback to invoke when the write is completed. + * @param vec Array of buffers to write; this method will make a + * copy of the vector (but not the buffers themselves) + * if the write has to be completed asynchronously. + * @param count Number of elements in vec. + * @param buf The IOBuf that manages the buffers referenced by + * vec, or a pointer to NULL if the buffers are not + * associated with an IOBuf. Note that ownership of + * the IOBuf is transferred here; upon completion of + * the write, the TAsyncSocket deletes the IOBuf. + * @param cork Whether to delay the write until the next non-corked + * write operation. (Note: may not be supported in all + * subclasses or on all platforms.) + */ + void writeImpl(WriteCallback* callback, const iovec* vec, size_t count, + std::unique_ptr&& buf, bool cork = false); + + /** + * Attempt to write to the socket. + * + * @param vec The iovec array pointing to the buffers to write. + * @param count The length of the iovec array. + * @param haveMore This flag is inherited from TAsyncSocket but is + * not handled here. + * @param countWritten On return, the value pointed to by this parameter + * will contain the number of iovec entries that were + * fully written. + * @param partialWritten On return, the value pointed to by this parameter + * will contain the number of bytes written in the + * partially written iovec entry. + * + * @return Returns the total number of bytes written, or -1 on error. If no + * data can be written immediately, 0 is returned. + */ + virtual ssize_t performWrite(const iovec* vec, uint32_t count, + bool haveMore, uint32_t* countWritten, + uint32_t* partialWritten); + + bool updateEventRegistration(); + + /** + * Update event registration. + * + * @param enable Flags of events to enable. Set it to 0 if no events + * need to be enabled in this call. + * @param disable Flags of events + * to disable. Set it to 0 if no events need to be disabled in this + * call. + * + * @return true iff the update is successful. + */ + bool updateEventRegistration(uint16_t enable, uint16_t disable); + + // error handling methods + void startFail(); + void finishFail(); + void fail(const char* fn, const transport::TTransportException& ex); + void failConnect(const char* fn, const transport::TTransportException& ex); + void failRead(const char* fn, const transport::TTransportException& ex); + void failWrite(const char* fn, WriteCallback* callback, size_t bytesWritten, + const transport::TTransportException& ex); + void failWrite(const char* fn, const transport::TTransportException& ex); + void failAllWrites(const transport::TTransportException& ex); + void invalidState(ConnectCallback* callback); + void invalidState(ReadCallback* callback); + void invalidState(WriteCallback* callback); + + uint8_t state_; ///< StateEnum describing current state + uint8_t shutdownFlags_; ///< Shutdown state (ShutdownFlags) + uint16_t eventFlags_; ///< TEventBase::HandlerFlags settings + int fd_; ///< The socket file descriptor + uint32_t sendTimeout_; ///< The send timeout, in milliseconds + TEventBase* eventBase_; ///< The TEventBase + WriteTimeout writeTimeout_; ///< A timeout for connect and write + IoHandler ioHandler_; ///< A TEventHandler to monitor the fd + + ConnectCallback* connectCallback_; ///< ConnectCallback + ReadCallback* readCallback_; ///< ReadCallback + WriteRequest* writeReqHead_; ///< Chain of WriteRequests + WriteRequest* writeReqTail_; ///< End of WriteRequest chain +}; + +}}} // apache::thrift::async + +#endif // #ifndef THRIFT_ASYNC_TASYNCSOCKET_H_ diff --git a/thrift/lib/cpp/async/TAsyncTimeout.h b/thrift/lib/cpp/async/TAsyncTimeout.h new file mode 100644 index 0000000000..fb7678870b --- /dev/null +++ b/thrift/lib/cpp/async/TAsyncTimeout.h @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_ASYNC_TASYNCTIMEOUT_H_ +#define THRIFT_ASYNC_TASYNCTIMEOUT_H_ 1 + +#include "thrift/lib/cpp/thrift_config.h" +#include "thrift/lib/cpp/async/TEventUtil.h" +#include + +namespace apache { namespace thrift { namespace async { + +class TEventBase; + +/** + * TAsyncTimeout is used to asynchronously wait for a timeout to occur. + */ +class TAsyncTimeout : private boost::noncopyable { + public: + enum InternalEnum { + INTERNAL, + NORMAL + }; + + /** + * Create a new TAsyncTimeout object, driven by the specified TEventBase. + */ + explicit TAsyncTimeout(TEventBase* eventBase); + + /** + * Create a new internal TAsyncTimeout object. + * + * Internal timeouts are like regular timeouts, but will not stop the + * TEventBase loop from exiting if the only remaining events are internal + * timeouts. + * + * This is useful for implementing fallback timeouts to abort the TEventBase + * loop if the other events have not been processed within a specified time + * period: if the event loop takes too long the timeout will fire and can + * stop the event loop. However, if all other events complete, the event + * loop will exit even though the internal timeout is still installed. + */ + TAsyncTimeout(TEventBase* eventBase, InternalEnum internal); + + /** + * Create a new TAsyncTimeout object, not yet assigned to a TEventBase. + * + * attachEventBase() must be called prior to scheduling the timeout. + */ + TAsyncTimeout(); + + /** + * TAsyncTimeout destructor. + * + * The timeout will be automatically cancelled if it is running. + */ + virtual ~TAsyncTimeout(); + + /** + * timeoutExpired() is invoked when the timeout period has expired. + */ + virtual void timeoutExpired() THRIFT_NOEXCEPT = 0; + + /** + * Schedule the timeout to fire in the specified number of milliseconds. + * + * After the specified number of milliseconds has elapsed, timeoutExpired() + * will be invoked by the TEventBase's main loop. + * + * If the timeout is already running, it will be rescheduled with the + * new timeout value. + * + * @param milliseconds The timeout duration, in milliseconds. + * + * @return Returns true if the timeout was successfully scheduled, + * and false if an error occurred. After an error, the timeout is + * always unscheduled, even if scheduleTimeout() was just + * rescheduling an existing timeout. + */ + bool scheduleTimeout(uint32_t milliseconds); + + /** + * Cancel the timeout, if it is running. + */ + void cancelTimeout(); + + /** + * Returns true if the timeout is currently scheduled. + */ + bool isScheduled() { + return TEventUtil::isEventRegistered(&event_); + } + + /** + * Attach the timeout to a TEventBase. + * + * This may only be called if the timeout is not currently attached to a + * TEventBase (either by using the default constructor, or by calling + * detachEventBase()). + * + * This method must be invoked in the TEventBase's thread. + * + * The internal parameter specifies if this timeout should be treated as an + * internal event. TEventBase::loop() will return when there are no more + * non-internal events remaining. + */ + void attachEventBase(TEventBase* eventBase, + InternalEnum internal = NORMAL); + + /** + * Detach the timeout from its TEventBase. + * + * This may only be called when the timeout is not running. + * Once detached, the timeout may not be scheduled again until it is + * re-attached to a TEventBase by calling attachEventBase(). + * + * This method must be called from the current TEventBase's thread. + */ + void detachEventBase(); + + private: + static void libeventCallback(int fd, short events, void* arg); + + struct event event_; +}; + +}}} // apache::thrift::async + +#endif // THRIFT_ASYNC_TASYNCTIMEOUT_H_ diff --git a/thrift/lib/cpp/async/TAsyncTimeoutSet.h b/thrift/lib/cpp/async/TAsyncTimeoutSet.h new file mode 100644 index 0000000000..df5962ba38 --- /dev/null +++ b/thrift/lib/cpp/async/TAsyncTimeoutSet.h @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_ASYNC_TASYNCTIMEOUTSET_H_ +#define THRIFT_ASYNC_TASYNCTIMEOUTSET_H_ 1 + +#include "thrift/lib/cpp/async/TAsyncTimeout.h" +#include "thrift/lib/cpp/async/TDelayedDestruction.h" + +#include +#include +#include + +namespace apache { namespace thrift { namespace async { + +/** + * TAsyncTimeoutSet exists for efficiently managing a group of timeouts events + * that always have the same timeout interval. + * + * TAsyncTimeoutSet takes advantage of the fact that the timeouts are always + * scheduled in sorted order. (Since each timeout has the same interval, when + * a new timeout is scheduled it will always be the last timeout in the set.) + * This avoids the need to perform any additional sorting of the timeouts + * within a single TAsyncTimeoutSet. + * + * TAsyncTimeoutSet is useful whenever you have a large group of objects that + * each need their own timeout, but with the same interval for each object. + * For example, managing idle timeouts for thousands of connection, or + * scheduling health checks for a large group of servers. + */ +class TAsyncTimeoutSet : private TAsyncTimeout, public TDelayedDestruction { + public: + typedef std::unique_ptr UniquePtr; + + /** + * A callback to be notified when a timeout has expired. + * + * TAsyncTimeoutSet::Callback is very similar to TAsyncTimeout. The primary + * distinction is that TAsyncTimeout can choose its timeout interval each + * time it is scheduled. On the other hand, TAsyncTimeoutSet::Callback + * always uses the timeout interval defined by the TAsyncTimeoutSet where it + * is scheduled. + */ + class Callback { + public: + Callback() + : timeoutSet_(NULL), + expiration_(0), + prev_(NULL), + next_(NULL) {} + + virtual ~Callback(); + + /** + * timeoutExpired() is invoked when the timeout has expired. + */ + virtual void timeoutExpired() THRIFT_NOEXCEPT = 0; + + /** + * Cancel the timeout, if it is running. + * + * If the timeout is not scheduled, cancelTimeout() does nothing. + */ + void cancelTimeout() { + if (timeoutSet_ == NULL) { + // We're not scheduled, so there's nothing to do. + return; + } + cancelTimeoutImpl(); + } + + /** + * Return true if this timeout is currently scheduled, and false otherwise. + */ + bool isScheduled() const { + return timeoutSet_ != NULL; + } + + private: + // Get the time remaining until this timeout expires + std::chrono::milliseconds getTimeRemaining( + std::chrono::milliseconds now) const { + if (now >= expiration_) { + return std::chrono::milliseconds(0); + } + return expiration_ - now; + } + + void setScheduled(TAsyncTimeoutSet* timeoutSet, Callback* prev); + void cancelTimeoutImpl(); + + TAsyncTimeoutSet* timeoutSet_; + std::chrono::milliseconds expiration_; + Callback* prev_; + Callback* next_; + + // Give TAsyncTimeoutSet direct access to our members so it can take care + // of scheduling/cancelling. + friend class TAsyncTimeoutSet; + }; + + /** + * Create a new TAsyncTimeoutSet with the specified interval. + */ + TAsyncTimeoutSet(TEventBase* eventBase, + std::chrono::milliseconds intervalMS, + std::chrono::milliseconds atMostEveryN = + std::chrono::milliseconds(0)); + + /** + * Destroy the TAsyncTimeoutSet. + * + * Normally a TAsyncTimeoutSet should only be destroyed when there are no + * more callbacks pending in the set. If there are timeout callbacks pending + * for this set, destroying the TAsyncTimeoutSet will automatically cancel + * them. If you destroy a TAsyncTimeoutSet with callbacks pending, your + * callback code needs to be aware that the callbacks will never be invoked. + */ + virtual void destroy(); + + /** + * Get the interval for this TAsyncTimeoutSet. + * + * Returns the timeout interval in milliseconds. All callbacks scheduled + * with scheduleTimeout() will be invoked after this amount of time has + * passed since the call to scheduleTimeout(). + */ + std::chrono::milliseconds getInterval() const { + return interval_; + } + + /** + * Schedule the specified Callback to be invoked after the TAsyncTimeoutSet's + * specified timeout interval. + * + * If the callback is already scheduled, this cancels the existing timeout + * before scheduling the new timeout. + */ + void scheduleTimeout(Callback* callback); + + /** + * Limit how frequently this TAsyncTimeoutSet will fire. + */ + void fireAtMostEvery(const std::chrono::milliseconds& ms) { + atMostEveryN_ = ms; + } + + /** + * Get a pointer to the next Callback scheduled to be invoked (may be null). + */ + Callback* front() { return head_; } + const Callback* front() const { return head_; } + + protected: + /** + * Protected destructor. + * + * Use destroy() instead. See the comments in TDelayedDestruction for more + * details. + */ + virtual ~TAsyncTimeoutSet(); + + private: + // Forbidden copy constructor and assignment operator + TAsyncTimeoutSet(TAsyncTimeoutSet const &) = delete; + TAsyncTimeoutSet& operator=(TAsyncTimeoutSet const &) = delete; + + // Private methods to be invoked by TAsyncTimeoutSet::Callback + void headChanged(); + + // Methods inherited from TAsyncTimeout + virtual void timeoutExpired() THRIFT_NOEXCEPT; + + std::chrono::milliseconds interval_; + std::chrono::milliseconds atMostEveryN_; + bool inTimeoutExpired_; + Callback* head_; + Callback* tail_; +}; + +}}} // apache::thrift::async + +#endif // THRIFT_ASYNC_TASYNCTIMEOUTSET_H_ diff --git a/thrift/lib/cpp/async/TAsyncTransport.h b/thrift/lib/cpp/async/TAsyncTransport.h new file mode 100644 index 0000000000..76ab7c15b9 --- /dev/null +++ b/thrift/lib/cpp/async/TAsyncTransport.h @@ -0,0 +1,428 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_ASYNC_TASYNCTRANSPORT_H_ +#define THRIFT_ASYNC_TASYNCTRANSPORT_H_ 1 + +#include "thrift/lib/cpp/thrift_config.h" +#include +#include +#include + +namespace folly { +class IOBuf; +} + +namespace apache { namespace thrift { + +namespace transport { +class TSocketAddress; +class TTransportException; +} + +namespace async { + +class TEventBase; + +/** + * TAsyncTransport defines an asynchronous API for streaming I/O. + * + * This class provides an API to for asynchronously waiting for data + * on a streaming transport, and for asynchronously sending data. + * + * The APIs for reading and writing are intentionally asymmetric. Waiting for + * data to read is a persistent API: a callback is installed, and is notified + * whenever new data is available. It continues to be notified of new events + * until it is uninstalled. + * + * TAsyncTransport does not provide read timeout functionality, because it + * typically cannot determine when the timeout should be active. Generally, a + * timeout should only be enabled when processing is blocked waiting on data + * from the remote endpoint. For server-side applications, the timeout should + * not be active if the server is currently processing one or more outstanding + * requests on this transport. For client-side applications, the timeout + * should not be active if there are no requests pending on the transport. + * Additionally, if a client has multiple pending requests, it will ususally + * want a separate timeout for each request, rather than a single read timeout. + * + * The write API is fairly intuitive: a user can request to send a block of + * data, and a callback will be informed once the entire block has been + * transferred to the kernel, or on error. TAsyncTransport does provide a send + * timeout, since most callers want to give up if the remote end stops + * responding and no further progress can be made sending the data. + */ +class TAsyncTransport { + public: + class ReadCallback { + public: + virtual ~ReadCallback() {} + + /** + * When data becomes available, getReadBuffer() will be invoked to get the + * buffer into which data should be read. + * + * This method allows the ReadCallback to delay buffer allocation until + * data becomes available. This allows applications to manage large + * numbers of idle connections, without having to maintain a separate read + * buffer for each idle connection. + * + * It is possible that in some cases, getReadBuffer() may be called + * multiple times before readDataAvailable() is invoked. In this case, the + * data will be written to the buffer returned from the most recent call to + * readDataAvailable(). If the previous calls to readDataAvailable() + * returned different buffers, the ReadCallback is responsible for ensuring + * that they are not leaked. + * + * If getReadBuffer() throws an exception, returns a NULL buffer, or + * returns a 0 length, the ReadCallback will be uninstalled and its + * readError() method will be invoked. + * + * getReadBuffer() is not allowed to change the transport state before it + * returns. (For example, it should never uninstall the read callback, or + * set a different read callback.) + * + * @param bufReturn getReadBuffer() should update *bufReturn to contain the + * address of the read buffer. This parameter will never + * be NULL. + * @param lenReturn getReadBuffer() should update *lenReturn to contain the + * maximum number of bytes that may be written to the read + * buffer. This parameter will never be NULL. + */ + virtual void getReadBuffer(void** bufReturn, size_t* lenReturn) = 0; + + /** + * readDataAvailable() will be invoked when data has been successfully read + * into the buffer returned by the last call to getReadBuffer(). + * + * The read callback remains installed after readDataAvailable() returns. + * It must be explicitly uninstalled to stop receiving read events. + * getReadBuffer() will be called at least once before each call to + * readDataAvailable(). getReadBuffer() will also be called before any + * call to readEOF(). + * + * @param len The number of bytes placed in the buffer. + */ + virtual void readDataAvailable(size_t len) THRIFT_NOEXCEPT = 0; + + /** + * readEOF() will be invoked when the transport is closed. + * + * The read callback will be automatically uninstalled immediately before + * readEOF() is invoked. + */ + virtual void readEOF() THRIFT_NOEXCEPT = 0; + + /** + * readError() will be invoked if an error occurs reading from the + * transport. + * + * The read callback will be automatically uninstalled immediately before + * readError() is invoked. + * + * @param ex An exception describing the error that occurred. + */ + virtual void readError(const transport::TTransportException& ex) + THRIFT_NOEXCEPT = 0; + }; + + class WriteCallback { + public: + virtual ~WriteCallback() {} + + /** + * writeSuccess() will be invoked when all of the data has been + * successfully written. + * + * Note that this mainly signals that the buffer containing the data to + * write is no longer needed and may be freed or re-used. It does not + * guarantee that the data has been fully transmitted to the remote + * endpoint. For example, on socket-based transports, writeSuccess() only + * indicates that the data has been given to the kernel for eventual + * transmission. + */ + virtual void writeSuccess() THRIFT_NOEXCEPT = 0; + + /** + * writeError() will be invoked if an error occurs writing the data. + * + * @param bytesWritten The number of bytes that were successfull + * @param ex An exception describing the error that occurred. + */ + virtual void writeError(size_t bytesWritten, + const transport::TTransportException& ex) + THRIFT_NOEXCEPT = 0; + }; + + virtual ~TAsyncTransport() {} + + /** + * Set the read callback. + * + * See the documentation for ReadCallback above for a description of how the + * callback will be invoked. Note that the callback remains installed until + * it is explicitly uninstalled, or until an error occurs. + * + * If a ReadCallback is already installed, it is replaced with the new + * callback. + * + * @param callback The callback to invoke when data is available. + * This parameter may be NULL to uninstall the current + * read callback. + */ + virtual void setReadCallback(ReadCallback* callback) = 0; + + /** + * Get the currently installed read callback. + * + * @return Returns a pointer to the installed ReadCallback, or NULL if no + * ReadCallback is installed. + */ + virtual ReadCallback* getReadCallback() const = 0; + + /** + * Write data to the transport. + * + * write() will always return immediately. The WriteCallback will later be + * invoked from the main TEventBase loop when the write has completed. + * + * Additional write attempts may be started before the first write completes. + * The subsequent write requests will be queued, and processed in the order + * in which they were called. + * + * @param callback The callback to invoke when the data has been written. + * The callback may not be NULL. + * @param buf The buffer containing the data to write. The caller is + * responsible for ensuring that this buffer remains valid + * until the callback is invoked. This parameter may not + * be NULL. + * @param bytes The number of bytes to write. + */ + virtual void write(WriteCallback* callback, + const void* buf, size_t bytes) = 0; + + /** + * Write non-contiguous data to the transport. + * + * writev() will always return immediately. The WriteCallback will later be + * invoked from the main TEventBase loop when the write has completed. + * + * Additional write attempts may be started before the first write completes. + * The subsequent write requests will be queued, and processed in the order + * in which they were called. + * + * @param callback The callback to invoke when the data has been written. + * The callback may not be NULL. + * @param vec A pointer to an array of iovec objects. The caller is + * responsible for ensuring that the buffers remain valid + * until the callback is invoked. This parameter may not + * be NULL. + * @param count The number of iovec objects in the vec array. + */ + virtual void writev(WriteCallback* callback, + const iovec* vec, size_t count) = 0; + + /** + * Write a chain of IOBufs to the transport. + * + * writeChain() will always return immediately. The WriteCallback will + * later be invoked from the main TEventBase loop when the write has + * completed. + * + * Additional write attempts may be started before the first write completes. + * The subsequent write requests will be queued, and processed in the order + * in which they were called. + * + * @param callback The callback to invoke when the data has been written. + * The callback may not be NULL. + * @param iob The head of an IOBuf chain. The TAsyncTransport + * will take ownership of this chain and delete it + * after writing. + * @param cork Whether to delay the write until the next non-corked + * write operation. (Note: may not be supported in all + * subclasses or on all platforms.) + */ + virtual void writeChain(WriteCallback* callback, + std::unique_ptr&& iob, + bool cork = false) = 0; + + /** + * Close the transport. + * + * This gracefully closes the transport, waiting for all pending write + * requests to complete before actually closing the underlying transport. + * + * If a read callback is set, readEOF() will be called immediately. If there + * are outstanding write requests, the close will be delayed until all + * remaining writes have completed. No new writes may be started after + * close() has been called. + */ + virtual void close() = 0; + + /** + * Close the transport immediately. + * + * This closes the transport immediately, dropping any outstanding data + * waiting to be written. + * + * If a read callback is set, readEOF() will be called immediately. + * If there are outstanding write requests, these requests will be aborted + * and writeError() will be invoked immediately on all outstanding write + * callbacks. + */ + virtual void closeNow() = 0; + + /** + * Perform a half-shutdown of the write side of the transport. + * + * The caller should not make any more calls to write() or writev() after + * shutdownWrite() is called. Any future write attempts will fail + * immediately. + * + * Not all transport types support half-shutdown. If the underlying + * transport does not support half-shutdown, it will fully shutdown both the + * read and write sides of the transport. (Fully shutting down the socket is + * better than doing nothing at all, since the caller may rely on the + * shutdownWrite() call to notify the other end of the connection that no + * more data can be read.) + * + * If there is pending data still waiting to be written on the transport, + * the actual shutdown will be delayed until the pending data has been + * written. + * + * Note: There is no corresponding shutdownRead() equivalent. Simply + * uninstall the read callback if you wish to stop reading. (On TCP sockets + * at least, shutting down the read side of the socket is a no-op anyway.) + */ + virtual void shutdownWrite() = 0; + + /** + * Perform a half-shutdown of the write side of the transport. + * + * shutdownWriteNow() is identical to shutdownWrite(), except that it + * immediately performs the shutdown, rather than waiting for pending writes + * to complete. Any pending write requests will be immediately failed when + * shutdownWriteNow() is called. + */ + virtual void shutdownWriteNow() = 0; + + /** + * Determine if transport is open and ready to read or write. + * + * Note that this function returns false on EOF; you must also call error() + * to distinguish between an EOF and an error. + * + * @return true iff the transport is open and ready, false otherwise. + */ + virtual bool good() const = 0; + + /** + * Determine if the transport is readable or not. + * + * @return true iff the transport is readable, false otherwise. + */ + virtual bool readable() const = 0; + + /** + * Determine if transport is connected to the endpoint + * + * @return false iff the transport is connected, otherwise true + */ + virtual bool connecting() const = 0; + + /** + * Determine if an error has occurred with this transport. + * + * @return true iff an error has occurred (not EOF). + */ + virtual bool error() const = 0; + + /** + * Attach the transport to a TEventBase. + * + * This may only be called if the transport is not currently attached to a + * TEventBase (by an earlier call to detachEventBase()). + * + * This method must be invoked in the TEventBase's thread. + */ + virtual void attachEventBase(TEventBase* eventBase) = 0; + + /** + * Detach the transport from its TEventBase. + * + * This may only be called when the transport is idle and has no reads or + * writes pending. Once detached, the transport may not be used again until + * it is re-attached to a TEventBase by calling attachEventBase(). + * + * This method must be called from the current TEventBase's thread. + */ + virtual void detachEventBase() = 0; + + /** + * Get the TEventBase used by this transport. + * + * Returns NULL if this transport is not currently attached to a TEventBase. + */ + virtual TEventBase* getEventBase() const = 0; + + /** + * Set the send timeout. + * + * If write requests do not make any progress for more than the specified + * number of milliseconds, fail all pending writes and close the transport. + * + * If write requests are currently pending when setSendTimeout() is called, + * the timeout interval is immediately restarted using the new value. + * + * @param milliseconds The timeout duration, in milliseconds. If 0, no + * timeout will be used. + */ + virtual void setSendTimeout(uint32_t milliseconds) = 0; + + /** + * Get the send timeout. + * + * @return Returns the current send timeout, in milliseconds. A return value + * of 0 indicates that no timeout is set. + */ + virtual uint32_t getSendTimeout() const = 0; + + /** + * Get the address of the local endpoint of this transport. + * + * This function may throw TTransportException on error. + * + * @param address The local address will be stored in the specified + * TSocketAddress. + */ + virtual void getLocalAddress(transport::TSocketAddress* address) const = 0; + + /** + * Get the address of the remote endpoint to which this transport is + * connected. + * + * This function may throw TTransportException on error. + * + * @param address The remote endpoint's address will be stored in the + * specified TSocketAddress. + */ + virtual void getPeerAddress(transport::TSocketAddress* address) const = 0; +}; + +}}} // apache::thrift::async + +#endif // #ifndef THRIFT_ASYNC_TASYNCTRANSPORT_H_ diff --git a/thrift/lib/cpp/async/TBinaryAsyncChannel.h b/thrift/lib/cpp/async/TBinaryAsyncChannel.h new file mode 100644 index 0000000000..2b8b2f5e41 --- /dev/null +++ b/thrift/lib/cpp/async/TBinaryAsyncChannel.h @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_ASYNC_TBINARYASYNCCHANNEL_H_ +#define THRIFT_ASYNC_TBINARYASYNCCHANNEL_H_ 1 + +#include "thrift/lib/cpp/async/TUnframedAsyncChannel.h" + +namespace apache { namespace thrift { namespace async { + +namespace detail { + +/** + * A class to determine the end of a raw TBinaryProtocol message. + */ +class TBinaryACProtocolTraits { + public: + TBinaryACProtocolTraits() : strictRead_(true) {} + + // Methods required by TUnframedACReadState + bool getMessageLength(uint8_t* buffer, + uint32_t bufferLength, + uint32_t* messageLength); + + // Methods specific to TBinaryAsyncChannel + + void setStrictRead(bool strictRead) { + strictRead_ = strictRead; + } + bool getStrictRead() const { + return strictRead_; + } + + private: + bool strictRead_; +}; + +} // namespace detail + +/** + * TBinaryAsyncChannel + * + * This is a TAsyncChannel implementation that reads and writes raw (unframed) + * messages encoded using TBinaryProtocol. + */ +class TBinaryAsyncChannel : + public TUnframedAsyncChannel { + private: + typedef TUnframedAsyncChannel Parent; + + public: + TBinaryAsyncChannel(const boost::shared_ptr& transport) + : Parent(transport) {} + + /** + * Helper function to create a shared_ptr. + * + * This passes in the correct destructor object, since TBinaryAsyncChannel's + * destructor is protected and cannot be invoked directly. + */ + static boost::shared_ptr newChannel( + const boost::shared_ptr& transport) { + return boost::shared_ptr( + new TBinaryAsyncChannel(transport), Destructor()); + } + + // Note that we inherit setMaxMessageSize() and getMaxMessageSize() + // from TUnframedAsyncChannel. + + void setStrictRead(bool strictRead) { + readState_.getProtocolTraits()->setStrictRead(strictRead); + } + bool getStrictRead() const { + return readState_.getProtocolTraits()->getStrictRead(); + } + + protected: + /** + * Protected destructor. + * + * Users of TBinaryAsyncChannel must never delete it directly. Instead, + * invoke destroy(). + */ + virtual ~TBinaryAsyncChannel() { } +}; + +class TBinaryAsyncChannelFactory : public TStreamAsyncChannelFactory { + public: + TBinaryAsyncChannelFactory() + : maxMessageSize_(0x7fffffff) + , recvTimeout_(0) + , sendTimeout_(0) + , strictRead_(true) {} + + void setMaxMessageSize(uint32_t bytes) { + maxMessageSize_ = bytes; + } + + void setRecvTimeout(uint32_t milliseconds) { + recvTimeout_ = milliseconds; + } + + void setSendTimeout(uint32_t milliseconds) { + sendTimeout_ = milliseconds; + } + + void setStrictRead(bool strict) { + strictRead_ = strict; + } + + virtual boost::shared_ptr newChannel( + const boost::shared_ptr& transport) { + boost::shared_ptr channel( + TBinaryAsyncChannel::newChannel(transport)); + transport->setSendTimeout(sendTimeout_); + channel->setMaxMessageSize(maxMessageSize_); + channel->setRecvTimeout(recvTimeout_); + channel->setStrictRead(strictRead_); + return channel; + } + + private: + uint32_t maxMessageSize_; + uint32_t recvTimeout_; + uint32_t sendTimeout_; + bool strictRead_; +}; + +}}} // apache::thrift::async + +#endif // THRIFT_ASYNC_TBINARYASYNCCHANNEL_H_ diff --git a/thrift/lib/cpp/async/TDelayedDestruction.h b/thrift/lib/cpp/async/TDelayedDestruction.h new file mode 100644 index 0000000000..2e484765c7 --- /dev/null +++ b/thrift/lib/cpp/async/TDelayedDestruction.h @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_ASYNC_TDELAYEDDESTRUCTION_H_ +#define THRIFT_ASYNC_TDELAYEDDESTRUCTION_H_ 1 + +#include +#include +#include + +namespace apache { namespace thrift { namespace async { + +/** + * TDelayedDestruction is a helper class to ensure objects are not deleted + * while they still have functions executing in a higher stack frame. + * + * This is useful for objects that invoke callback functions, to ensure that a + * callback does not destroy the calling object. + * + * Classes needing this functionality should: + * - derive from TDelayedDestruction + * - make their destructor private or protected, so it cannot be called + * directly + * - create a DestructorGuard object on the stack in each public method that + * may invoke a callback + * + * TDelayedDestruction does not perform any locking. It is intended to be used + * only from a single thread. + */ +class TDelayedDestruction : private boost::noncopyable { + public: + /** + * Helper class to allow TDelayedDestruction classes to be used with + * boost::shared_ptr. + * + * This class can be specified as the destructor argument when creating the + * shared_ptr, and it will destroy the guarded class properly when all + * shared_ptr references are released. + */ + class Destructor { + public: + void operator()(TDelayedDestruction* dd) const { + dd->destroy(); + } + }; + + /** + * destroy() requests destruction of the object. + * + * This method will destroy the object after it has no more functions running + * higher up on the stack. (i.e., No more DestructorGuard objects exist for + * this object.) This method must be used instead of the destructor. + */ + virtual void destroy() { + // If guardCount_ is not 0, just set destroyPending_ to delay + // actual destruction. + if (guardCount_ != 0) { + destroyPending_ = true; + } else { + destroyNow(false); + } + } + + /** + * Classes should create a DestructorGuard object on the stack in any + * function that may invoke callback functions. + * + * The DestructorGuard prevents the guarded class from being destroyed while + * it exists. Without this, the callback function could delete the guarded + * object, causing problems when the callback function returns and the + * guarded object's method resumes execution. + */ + class DestructorGuard { + public: + explicit DestructorGuard(TDelayedDestruction* dd) : dd_(dd) { + ++dd_->guardCount_; + assert(dd_->guardCount_ > 0); // check for wrapping + } + + ~DestructorGuard() { + assert(dd_->guardCount_ > 0); + --dd_->guardCount_; + if (dd_->guardCount_ == 0 && dd_->destroyPending_) { + dd_->destroyNow(true); + } + } + + private: + TDelayedDestruction* dd_; + }; + + protected: + /** + * destroyNow() is invoked to actually destroy the object, after destroy() + * has been called and no more DestructorGuard objects exist. By default it + * calls "delete this", but subclasses may override this behavior. + * + * @param delayed This parameter is true if destruction was delayed because + * of a DestructorGuard object, or false if destroyNow() is + * being called directly from destroy(). + */ + virtual void destroyNow(bool delayed) { + delete this; + (void)delayed; // prevent unused variable warnings + } + + TDelayedDestruction() + : guardCount_(0) + , destroyPending_(false) {} + + /** + * Protected destructor. + * + * Making this protected ensures that users cannot delete TDelayedDestruction + * objects directly, and that everyone must use destroy() instead. + * Subclasses of TDelayedDestruction must also define their destructors as + * protected or private in order for this to work. + * + * This also means that TDelayedDestruction objects cannot be created + * directly on the stack; they must always be dynamically allocated on the + * heap. + * + * In order to use a TDelayedDestruction object with a shared_ptr, create the + * shared_ptr using a TDelayedDestruction::Destructor as the second argument + * to the shared_ptr constructor. + */ + virtual ~TDelayedDestruction() {} + + /** + * Get the number of DestructorGuards currently protecting this object. + * + * This is primarily intended for debugging purposes, such as asserting + * that an object has at least 1 guard. + */ + uint32_t getDestructorGuardCount() const { + return guardCount_; + } + + private: + /** + * guardCount_ is incremented by DestructorGuard, to indicate that one of + * the TDelayedDestruction object's methods is currently running. + * + * If destroy() is called while guardCount_ is non-zero, destruction will + * be delayed until guardCount_ drops to 0. This allows TDelayedDestruction + * objects to invoke callbacks without having to worry about being deleted + * before the callback returns. + */ + uint32_t guardCount_; + + /** + * destroyPending_ is set to true if destoy() is called while guardCount_ is + * non-zero. + * + * If destroyPending_ is true, the object will be destroyed the next time + * guardCount_ drops to 0. + */ + bool destroyPending_; +}; + +}}} // apache::thrift::async + +#endif // THRIFT_ASYNC_TDELAYEDDESTRUCTION_H_ diff --git a/thrift/lib/cpp/async/TEventBase.h b/thrift/lib/cpp/async/TEventBase.h new file mode 100644 index 0000000000..42009a756d --- /dev/null +++ b/thrift/lib/cpp/async/TEventBase.h @@ -0,0 +1,397 @@ +// Copyright (c) 2006- Facebook +// Distributed under the Thrift Software License +// +// See accompanying file LICENSE or visit the Thrift site at: +// http://developers.facebook.com/thrift/ + +#ifndef THRIFT_ASYNC_TEVENTBASE_H_ +#define THRIFT_ASYNC_TEVENTBASE_H_ 1 + +#include "thrift/lib/cpp/Thrift.h" +#include "thrift/lib/cpp/async/TAsyncTimeout.h" +#include "thrift/lib/cpp/server/TServer.h" +#include "thrift/lib/cpp/transport/TTransportUtils.h" +#include "thrift/lib/cpp/concurrency/ThreadManager.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include // libevent +#include +#include + +namespace apache { namespace thrift { namespace async { + +typedef std::tr1::function Cob; +template +class TNotificationQueue; + +/** + * This class is a wrapper for all asynchronous I/O processing functionality + * used in thrift. + * + * TEventBase provides a main loop that notifies TEventHandler callback objects + * when I/O is ready on a file descriptor, and notifies TAsyncTimeout objects + * when a specified timeout has expired. More complex, higher-level callback + * mechanisms can then be built on top of TEventHandler and TAsyncTimeout. + * + * A TEventBase object can only drive an event loop for a single thread. To + * take advantage of multiple CPU cores, most asynchronous I/O servers have one + * thread per CPU, and use a separate TEventBase for each thread. + * + * In general, most TEventBase methods may only be called from the thread + * running the TEventBase's loop. There are a few exceptions to this rule, for + * methods that are explicitly intended to allow communication with a + * TEventBase from other threads. When it is safe to call a method from + * another thread it is explicitly listed in the method comments. + */ +class TEventBase : private boost::noncopyable { + public: + /** + * A callback interface to use with runInLoop() + * + * Derive from this class if you need to delay some code execution until the + * next iteration of the event loop. This allows you to schedule code to be + * invoked from the top-level of the loop, after your immediate callers have + * returned. + * + * If a LoopCallback object is destroyed while it is scheduled to be run in + * the next loop iteration, it will automatically be cancelled. + */ + class LoopCallback { + public: + virtual ~LoopCallback() {} + + virtual void runLoopCallback() THRIFT_NOEXCEPT = 0; + void cancelLoopCallback() { + hook_.unlink(); + } + + bool isLoopCallbackScheduled() const { + return hook_.is_linked(); + } + + private: + typedef boost::intrusive::list_member_hook< + boost::intrusive::link_mode > ListHook; + + ListHook hook_; + + typedef boost::intrusive::list< + LoopCallback, + boost::intrusive::member_hook, + boost::intrusive::constant_time_size > List; + + // TEventBase needs access to LoopCallbackList (and therefore to hook_) + friend class TEventBase; + }; + + /** + * Create a new TEventBase object. + */ + TEventBase(); + + /** + * Create a new TEventBase object that will use the specified libevent + * event_base object to drive the event loop. + * + * The TEventBase will take ownership of this event_base, and will call + * event_base_free(evb) when the TEventBase is destroyed. + */ + explicit TEventBase(event_base* evb); + ~TEventBase(); + + /** + * Runs the event loop. + * + * loop() will loop waiting for I/O or timeouts and invoking TEventHandler + * and TAsyncTimeout callbacks as their events become ready. loop() will + * only return when there are no more events remaining to process, or after + * terminateLoopSoon() has been called. + * + * loop() may be called again to restart event processing after a previous + * call to loop() or loopForever() has returned. + * + * Returns true if the loop completed normally (if it processed all + * outstanding requests, or if terminateLoopSoon() was called). If an error + * occurs waiting for events, false will be returned. + */ + bool loop(); + + /** + * Runs the event loop. + * + * loopForever() behaves like loop(), except that it keeps running even if + * when there are no more user-supplied TEventHandlers or TAsyncTimeouts + * registered. It will only return after terminateLoopSoon() has been + * called. + * + * This is useful for callers that want to wait for other threads to call + * runInEventBaseThread(), even when there are no other scheduled events. + * + * loopForever() may be called again to restart event processing after a + * previous call to loop() or loopForever() has returned. + * + * Throws a TLibraryException if an error occurs. + */ + void loopForever(); + + /** + * Causes the event loop to exit soon. + * + * This will cause an existing call to loop() or loopForever() to stop event + * processing and return, even if there are still events remaining to be + * processed. + * + * It is safe to call terminateLoopSoon() from another thread to cause loop() + * to wake up and return in the TEventBase loop thread. terminateLoopSoon() + * may also be called from the loop thread itself (for example, a + * TEventHandler or TAsyncTimeout callback may call terminateLoopSoon() to + * cause the loop to exit after the callback returns.) + * + * Note that the caller is responsible for ensuring that cleanup of all event + * callbacks occurs properly. Since terminateLoopSoon() causes the loop to + * exit even when there are pending events present, there may be remaining + * callbacks present waiting to be invoked. If the loop is later restarted + * pending events will continue to be processed normally, however if the + * TEventBase is destroyed after calling terminateLoopSoon() it is the + * caller's responsibility to ensure that cleanup happens properly even if + * some outstanding events are never processed. + */ + void terminateLoopSoon(); + + /** + * Adds the given callback to a queue of things run after the current pass + * through the event loop completes. Note that if this callback calls + * runInLoop() the new callback won't be called until the main event loop + * has gone through a cycle. + * + * This method may only be called from the TEventBase's thread. This + * essentially allows an event handler to schedule an additional callback to + * be invoked after it returns. + * + * Use runInEventBaseThread() to schedule functions from another thread. + */ + void runInLoop(LoopCallback* callback); + + /** + * Convenience function to call runInLoop() with a tr1::function. + * + * This creates a LoopCallback object to wrap the tr1::function, and invoke + * the tr1::function when the loop callback fires. This is slightly more + * expensive than defining your own LoopCallback, but more convenient in + * areas that aren't performance sensitive where you just want to use + * tr1::bind. (tr1::bind is fairly slow on even by itself.) + * + * This method may only be called from the TEventBase's thread. This + * essentially allows an event handler to schedule an additional callback to + * be invoked after it returns. + * + * Use runInEventBaseThread() to schedule functions from another thread. + */ + void runInLoop(const Cob& c); + + /** + * Run the specified function in the TEventBase's thread. + * + * This method is thread-safe, and may be called from another thread. + * + * If runInEventBaseThread() is called when the TEventBase loop is not + * running, the function call will be delayed until the next time the loop is + * started. + * + * If runInEventBaseThread() returns true the function has successfully been + * scheduled to run in the loop thread. However, if the loop is terminated + * (and never later restarted) before it has a chance to run the requested + * function, the function may never be run at all. The caller is responsible + * for handling this situation correctly if they may terminate the loop with + * outstanding runInEventBaseThread() calls pending. + * + * If two calls to runInEventBaseThread() are made from the same thread, the + * functions will always be run in the order that they were scheduled. + * Ordering between functions scheduled from separate threads is not + * guaranteed. + * + * @param fn The function to run. The function must not throw any + * exceptions. + * @param arg An argument to pass to the function. + * + * @return Returns true if the function was successfully scheduled, or false + * if there was an error scheduling the function. + */ + template + bool runInEventBaseThread(void (*fn)(T*), T* arg) { + return runInEventBaseThread(reinterpret_cast(fn), + reinterpret_cast(arg)); + } + + bool runInEventBaseThread(void (*fn)(void*), void* arg); + + /** + * Run the specified function in the TEventBase's thread + * + * This version of runInEventBaseThread() takes a tr1::function object. + * Note that this is less efficient than the version that takes a plain + * function pointer and void* argument, as it has to allocate memory to copy + * the tr1::function object. + * + * If the TEventBase loop is terminated before it has a chance to run this + * function, the allocated memory will be leaked. The caller is responsible + * for ensuring that the TEventBase loop is not terminated before this + * function can run. + * + * The function must not throw any exceptions. + */ + bool runInEventBaseThread(const std::tr1::function& fn); + + /** + * Runs the given Cob at some time after the specified number of + * milliseconds. (No guarantees exactly when.) + * + * @return true iff the cob was successfully registered. + */ + bool runAfterDelay(const Cob& c, int milliseconds); + + /** + * Set the maximum desired latency in us and provide a callback which will be + * called when that latency is exceeded. + */ + void setMaxLatency(int64_t maxLatency, const Cob& maxLatencyCob) { + maxLatency_ = maxLatency; + maxLatencyCob_ = maxLatencyCob; + } + /** + * Set smoothing coefficient for loop load average; # of milliseconds + * for exp(-1) (1/2.71828...) decay. + */ + void setLoadAvgMsec(uint32_t ms); + + /** + * Get the average loop time in microseconds (an exponentially-smoothed ave) + */ + double getAvgLoopTime() const { + return avgLoopTime_; + } + + /** + * Verify that current thread is the TEventBase thread, if the TEventBase is + * running. + * + * This is primarily intended for debugging, to assert that functions that + * register or unregister events are only ever invoked in the TEventBase's + * thread. + */ + bool isInEventBaseThread() const { + return !running_ || pthread_equal(loopThread_, pthread_self()); + } + + // --------- interface to underlying libevent base ------------ + // Avoid using these functions if possible. These functions are not + // guaranteed to always be present if we ever provide alternative TEventBase + // implementations that do not use libevent internally. + event_base* getLibeventBase() const { return evb_; } + static const char* getLibeventVersion() { return event_get_version(); } + static const char* getLibeventMethod() { return event_get_method(); } + + private: + // --------- libevent callbacks (not for client use) ------------ + + /** + * Called after a delay to break out of an idle event loop. We need to + * use this instead of event_base_loopexit() since the latter installs + * an event within libevent which is queued until it expires. Installing + * our own timed event lets us delete it when another event causes the + * loop to exit earlier.. + */ + static void loopTimerCallback(int fd, short which, void* arg); + + static void runTr1FunctionPtr(std::tr1::function* fn); + + // small object used as a callback arg with enough info to execute the + // appropriate client-provided Cob + class CobTimeout : public TAsyncTimeout { + public: + CobTimeout(TEventBase* b, const Cob& c) : TAsyncTimeout(b), cob_(c) {} + + virtual void timeoutExpired() THRIFT_NOEXCEPT; + + private: + Cob cob_; + + public: + typedef boost::intrusive::list_member_hook< + boost::intrusive::link_mode > ListHook; + + ListHook hook; + + typedef boost::intrusive::list< + CobTimeout, + boost::intrusive::member_hook, + boost::intrusive::constant_time_size > List; + }; + + typedef LoopCallback::List LoopCallbackList; + class FunctionRunner; + + // executes any callbacks queued by runInLoop() + void runLoopCallbacks(); + + void initNotificationQueue(); + + CobTimeout::List pendingCobTimeouts_; + + LoopCallbackList loopCallbacks_; + + // stop_ is set by terminateLoopSoon() and is used by the main loop + // to determine if it should exit + bool stop_; + // running_ is set to true while loop() is running + bool running_; + // The ID of the thread running the main loop. + // Only valid while running_ is true. + pthread_t loopThread_; + + // pointer to underlying event_base class doing the heavy lifting + event_base* evb_; + + // A notification queue for runInEventBaseThread() to use + // to send function requests to the TEventBase thread. + std::unique_ptr>> queue_; + std::unique_ptr fnRunner_; + + // limit for latency in microseconds (0 disables) + int64_t maxLatency_; + + // smoothed loop time used to invoke latency callbacks; differs from + // avgLoopTime_ in that it's scaled down after triggering a callback + // to reduce spamminess + double maxLatencyLoopTime_; + + // exponentially-smoothed average loop time for latency-limiting + double avgLoopTime_; + + // set to true if the event_base_loop(EVLOOP_ONCE) returned because + // the loop timeout fired, rather than because it found events to process + bool loopTimedOut_; + + // factor used for exponential smoothing of load average + double expCoeff_; + + // callback called when latency limit is exceeded + Cob maxLatencyCob_; + + // we'll wait this long before running deferred callbacks if the event + // loop is idle. + static const int kDEFAULT_IDLE_WAIT_USEC = 20000; // 20ms +}; + +}}} // apache::thrift::async + +#endif // #ifndef THRIFT_ASYNC_TEVENTBASE_H_ diff --git a/thrift/lib/cpp/async/TEventBaseManager.h b/thrift/lib/cpp/async/TEventBaseManager.h new file mode 100644 index 0000000000..d6d1ae0101 --- /dev/null +++ b/thrift/lib/cpp/async/TEventBaseManager.h @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_ASYNC_TEVENTBASEMANAGER_H +#define THRIFT_ASYNC_TEVENTBASEMANAGER_H 1 + +#include "thrift/lib/cpp/concurrency/ThreadLocal.h" + +namespace apache { namespace thrift { namespace async { + +class TEventBase; + +/** + * Manager for per-thread TEventBase objects. + * This class will find or create a TEventBase for the current + * thread, associated with thread-specific storage for that thread. + * Although a typical application will generally only have one + * TEventBaseManager, there is no restriction on multiple instances; + * the TEventBases belong to one instance are isolated from those of + * another. + */ +class TEventBaseManager { + public: + TEventBaseManager() {} + ~TEventBaseManager() {} + + /** + * Get the TEventBase for this thread, or create one if none exists yet. + * + * If no TEventBase exists for this thread yet, a new one will be created and + * returned. May throw std::bad_alloc if allocation fails. + */ + TEventBase* getEventBase() const { + // localStore_.get() will never return NULL. + // InfoManager::allocate() will throw an exception instead if it cannot + // allocate a new EventBaseInfo or TEventBase. + return localStore_.get()->eventBase; + } + + /** + * Get the TEventBase for this thread. + * + * Returns NULL if no TEventBase has been created for this thread yet. + */ + TEventBase* getExistingEventBase() const { + EventBaseInfo* info = localStore_.getNoAlloc(); + if (info == NULL) { + return NULL; + } + return info->eventBase; + } + + /** + * Set the TEventBase to be used by this thread. + * + * This may only be called if no TEventBase has been defined for this thread + * yet. If a TEventBase is already defined for this thread, a + * TLibraryException is thrown. std::bad_alloc may also be thrown if + * allocation fails while setting the TEventBase. + * + * This should typically be invoked by the code that will call loop() on the + * TEventBase, to make sure the TEventBaseManager points to the correct + * TEventBase that is actually running in this thread. + */ + void setEventBase(TEventBase *eventBase, bool takeOwnership); + + /** + * Clear the TEventBase for this thread. + * + * This can be used if the code driving the TEventBase loop() has finished + * the loop and new events should no longer be added to the TEventBase. + */ + void clearEventBase(); + + private: + struct EventBaseInfo { + EventBaseInfo(TEventBase *evb, bool owned) + : eventBase(evb), + owned(owned) {} + + TEventBase *eventBase; + bool owned; + }; + + class InfoManager { + public: + EventBaseInfo* allocate(); + void destroy(EventBaseInfo* info); + + void replace(EventBaseInfo* oldInfo, EventBaseInfo* newInfo) { + if (oldInfo != newInfo) { + destroy(oldInfo); + } + } + }; + + // Forbidden copy constructor and assignment opererator + TEventBaseManager(TEventBaseManager const &); + TEventBaseManager& operator=(TEventBaseManager const &); + + concurrency::ThreadLocal localStore_; +}; + +}}} // apache::thrift::async + +#endif // THRIFT_ASYNC_TEVENTBASEMANAGER_H diff --git a/thrift/lib/cpp/async/TEventConnection.h b/thrift/lib/cpp/async/TEventConnection.h new file mode 100644 index 0000000000..c46c749ba9 --- /dev/null +++ b/thrift/lib/cpp/async/TEventConnection.h @@ -0,0 +1,239 @@ +// Copyright (c) 2006- Facebook +// Distributed under the Thrift Software License +// +// See accompanying file LICENSE or visit the Thrift site at: +// http://developers.facebook.com/thrift/ + +#ifndef THRIFT_ASYNC_TEVENTCONNECTION_H_ +#define THRIFT_ASYNC_TEVENTCONNECTION_H_ 1 + +#include "thrift/lib/cpp/server/TConnectionContext.h" +#include "thrift/lib/cpp/transport/TSocketAddress.h" +#include "thrift/lib/cpp/async/TEventServer.h" +#include +#include + +namespace apache { namespace thrift { + +class TProcessor; + +namespace protocol { +class TProtocol; +} + +namespace server { +class TServerEventHandler; +} + +namespace transport { +class TMemoryBuffer; +} + +namespace async { + +class TAsyncEventChannel; +class TAsyncProcessor; +class TEventWorker; +class TAsyncSocket; +class TaskCompletionMessage; + +/** + * Represents a connection that is handled via libevent. This connection + * essentially encapsulates a socket that has some associated libevent state. + */ +class TEventConnection : private boost::noncopyable, + public TEventBase::LoopCallback { + public: + + /** + * Constructor for TEventConnection. + * + * @param asyncSocket shared pointer to the async socket + * @param address the peer address of this connection + * @param worker the worker instance that is handling this connection + */ + TEventConnection(boost::shared_ptr asyncSocket, + const transport::TSocketAddress* address, + TEventWorker* worker, TEventServer::TransportType transport); + + /** + * (Re-)Initialize a TEventConnection. We break this out from the + * constructor to allow for pooling. + * + * @param asyncSocket shared pointer to the async socket + * @param address the peer address of this connection + * @param worker the worker instance that is handling this connection + */ + void init(boost::shared_ptr asyncSocket, + const transport::TSocketAddress* address, + TEventWorker* worker, TEventServer::TransportType transport); + + /// First cause -- starts i/o on connection + void start(); + + /// Shut down the connection even if it's OK; used for load reduction. + void stop() { + shutdown_ = true; + } + + /// Return a pointer to the worker that owns us + TEventWorker* getWorker() const { + return worker_; + } + + /// cause the notification callback to occur within the appropriate context + bool notifyCompletion(TaskCompletionMessage &&msg); + + /// Run scheduled read when there are too many reads on the stack + void runLoopCallback() THRIFT_NOEXCEPT; + + boost::shared_ptr getProcessor() const { + return processor_; + } + + boost::shared_ptr + getInputProtocol() const { + return inputProtocol_; + } + + boost::shared_ptr + getOutputProtocol() const { + return outputProtocol_; + } + + /// Get the per-server event handler set for this server, if any + boost::shared_ptr + getServerEventHandler() const { + return serverEventHandler_; + } + + /// Get the TConnectionContext for this connection + server::TConnectionContext* getConnectionContext() { + return &context_; + } + + /// Destructor -- close down the connection. + ~TEventConnection(); + + /** + * Check the size of our memory buffers and resize if needed. Do not call + * when a call is in progress. + */ + void checkBufferMemoryUsage(); + + private: + class ConnContext : public server::TConnectionContext { + public: + void init(const transport::TSocketAddress* address, + boost::shared_ptr inputProtocol, + boost::shared_ptr outputProtocol) { + address_ = *address; + inputProtocol_ = inputProtocol; + outputProtocol_ = outputProtocol; + } + + virtual const transport::TSocketAddress* getPeerAddress() const { + return &address_; + } + + void reset() { + address_.reset(); + cleanupUserData(); + } + + // TODO(dsanduleac): implement the virtual getInputProtocol() & such + + virtual boost::shared_ptr getInputProtocol() const { + // from TEventConnection + return inputProtocol_; + } + + virtual boost::shared_ptr getOutputProtocol() const { + return outputProtocol_; + } + + private: + transport::TSocketAddress address_; + boost::shared_ptr inputProtocol_; + boost::shared_ptr outputProtocol_; + }; + + void readNextRequest(); + void handleReadSuccess(); + void handleReadFailure(); + void handleAsyncTaskComplete(bool success); + void handleSendSuccess(); + void handleSendFailure(); + + void handleEOF(); + void handleFailure(const char* msg); + void cleanup(); + + //! The worker instance handling this connection. + TEventWorker* worker_; + + //! This connection's socket. + boost::shared_ptr asyncSocket_; + + //! Transport that the processor reads from. + boost::shared_ptr inputTransport_; + + //! Transport that the processor writes to. + boost::shared_ptr outputTransport_; + + /// Largest size of read buffer seen since buffer was constructed + size_t largestReadBufferSize_; + + /// Largest size of write buffer seen since buffer was constructed + size_t largestWriteBufferSize_; + + /// Count of the number of calls for use with getResizeBufferEveryN(). + int32_t callsForResize_; + + //! Protocol decoder. + boost::shared_ptr inputProtocol_; + + //! Protocol encoder. + boost::shared_ptr outputProtocol_; + + //! Channel that actually performs the socket I/O and callbacks. + TAsyncEventChannel* asyncChannel_; + + /// Count of outstanding processor callbacks (generally 0 or 1). + int32_t processorActive_; + + //! Count of the number of handleReadSuccess frames on the stack + int32_t readersActive_; + + /// Sync processor if we're in queuing mode + boost::shared_ptr processor_; + + /// Flag used to shut down connection (used for load-shedding mechanism). + bool shutdown_; + + /// Flag indicating that we have deferred closing down (processor was active) + bool closing_; + + /// The per-server event handler set for this erver, if any + boost::shared_ptr + serverEventHandler_; + + /// per-connection context + ConnContext context_; + + /// Our processor + boost::shared_ptr asyncProcessor_; + + /// So that TEventWorker can call handleAsyncTaskComplete(); + friend class TEventWorker; + + /// Make the server a friend so it can manage tasks when overloaded + friend class TEventServer; + + /// Make an async task a friend so it can communicate a cleanup() to us. + friend class TEventTask; +}; + +}}} // apache::thrift::async + +#endif // #ifndef THRIFT_ASYNC_TEVENTCONNECTION_H_ diff --git a/thrift/lib/cpp/async/TEventHandler.h b/thrift/lib/cpp/async/TEventHandler.h new file mode 100644 index 0000000000..0d720ed311 --- /dev/null +++ b/thrift/lib/cpp/async/TEventHandler.h @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_ASYNC_TEVENTHANDLER_H_ +#define THRIFT_ASYNC_TEVENTHANDLER_H_ 1 + +#include "thrift/lib/cpp/thrift_config.h" +#include "thrift/lib/cpp/async/TEventUtil.h" +#include +#include + +namespace apache { namespace thrift { namespace async { + +class TEventBase; + +/** + * The TEventHandler class is used to asynchronously wait for events on a file + * descriptor. + * + * Users that wish to wait on I/O events should derive from TEventHandler and + * implement the handlerReady() method. + */ +class TEventHandler : private boost::noncopyable { + public: + enum EventFlags { + NONE = 0, + READ = EV_READ, + WRITE = EV_WRITE, + READ_WRITE = (READ | WRITE), + PERSIST = EV_PERSIST + }; + + /** + * Create a new TEventHandler object. + * + * @param eventBase The TEventBase to use to drive this event handler. + * This may be NULL, in which case the TEventBase must be + * set separately using initHandler() or attachEventBase() + * before the handler can be registered. + * @param fd The file descriptor that this TEventHandler will + * monitor. This may be -1, in which case the file + * descriptor must be set separately using initHandler() or + * changeHandlerFD() before the handler can be registered. + */ + explicit TEventHandler(TEventBase* eventBase = NULL, int fd = -1); + + /** + * TEventHandler destructor. + * + * The event will be automatically unregistered if it is still registered. + */ + virtual ~TEventHandler(); + + /** + * handlerReady() is invoked when the handler is ready. + * + * @param events A bitset indicating the events that are ready. + */ + virtual void handlerReady(uint16_t events) THRIFT_NOEXCEPT = 0; + + /** + * Register the handler. + * + * If the handler is already registered, the registration will be updated + * to wait on the new set of events. + * + * @param events A bitset specifying the events to monitor. + * If the PERSIST bit is set, the handler will remain + * registered even after handlerReady() is called. + * + * @return Returns true if the handler was successfully registered, + * or false if an error occurred. After an error, the handler is + * always unregistered, even if it was already registered prior to + * this call to registerHandler(). + */ + bool registerHandler(uint16_t events) { + return registerImpl(events, false); + } + + /** + * Unregister the handler, if it is registered. + */ + void unregisterHandler(); + + /** + * Returns true if the handler is currently registered. + */ + bool isHandlerRegistered() const { + return TEventUtil::isEventRegistered(&event_); + } + + /** + * Attach the handler to a TEventBase. + * + * This may only be called if the handler is not currently attached to a + * TEventBase (either by using the default constructor, or by calling + * detachEventBase()). + * + * This method must be invoked in the TEventBase's thread. + */ + void attachEventBase(TEventBase* eventBase); + + /** + * Detach the handler from its TEventBase. + * + * This may only be called when the handler is not currently registered. + * Once detached, the handler may not be registered again until it is + * re-attached to a TEventBase by calling attachEventBase(). + * + * This method must be called from the current TEventBase's thread. + */ + void detachEventBase(); + + /** + * Change the file descriptor that this handler is associated with. + * + * This may only be called when the handler is not currently registered. + */ + void changeHandlerFD(int fd); + + /** + * Attach the handler to a TEventBase, and change the file descriptor. + * + * This method may only be called if the handler is not currently attached to + * a TEventBase. This is primarily intended to be used to initialize + * TEventHandler objects created using the default constructor. + */ + void initHandler(TEventBase* eventBase, int fd); + + /** + * Return the set of events that we're currently registered for. + */ + uint16_t getRegisteredEvents() const { + return (isHandlerRegistered()) ? + event_.ev_events : 0; + } + + /** + * Register the handler as an internal event. + * + * This event will not count as an active event for determining if the + * TEventBase loop has more events to process. The TEventBase loop runs + * only as long as there are active TEventHandlers, however "internal" event + * handlers are not counted. Therefore this event handler will not prevent + * TEventBase loop from exiting with no more work to do if there are no other + * non-internal event handlers registered. + * + * This is intended to be used only in very rare cases by the internal + * TEventBase code. This API is not guaranteed to remain stable or portable + * in the future. + */ + bool registerInternalHandler(uint16_t events) { + return registerImpl(events, true); + } + + private: + bool registerImpl(uint16_t events, bool internal); + void ensureNotRegistered(const char* fn); + + static void libeventCallback(int fd, short events, void* arg); + + struct event event_; +}; + +}}} // apache::thrift::async + +#endif // THRIFT_ASYNC_TEVENTHANDLER_H_ diff --git a/thrift/lib/cpp/async/TEventServer.h b/thrift/lib/cpp/async/TEventServer.h new file mode 100644 index 0000000000..296d3e5e18 --- /dev/null +++ b/thrift/lib/cpp/async/TEventServer.h @@ -0,0 +1,1010 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_ASYNC_TEVENTSERVER_H_ +#define THRIFT_ASYNC_TEVENTSERVER_H_ 1 + +#include "thrift/lib/cpp/Thrift.h" +#include "thrift/lib/cpp/server/TServer.h" +#include "thrift/lib/cpp/async/TAsyncProcessor.h" +#include "thrift/lib/cpp/transport/TTransportUtils.h" +#include "thrift/lib/cpp/transport/TSSLSocket.h" +#include "thrift/lib/cpp/protocol/THeaderProtocol.h" +#include "thrift/lib/cpp/concurrency/Mutex.h" +#include "thrift/lib/cpp/concurrency/ThreadLocal.h" +#include "thrift/lib/cpp/async/TEventBase.h" +#include "thrift/lib/cpp/async/TEventBaseManager.h" +#include +#include +#include +#include + +namespace apache { namespace thrift { + +namespace concurrency { +class ThreadFactory; +class ThreadManager; +} + +namespace async { + +using apache::thrift::protocol::TDualProtocolFactory; + +// Forward declaration of classes +class TAsyncServerSocket; +class TEventConnection; +class TEventWorker; + +/** + This is a non-blocking event-based server for high performance that + operates an I/O thread for each cpu core and uses callbacks for + notification of processing or I/O operations. + + It does not use the TServerTransport framework, but rather has socket + operations hardcoded for use with libevent and implements framing + compatible with TFramedTransport. A single "listener" thread accepts + connections and sends them to the server threads via a single socketpair(); + the server threads are each responsible for allocating and pooling actual + connection objects, avoiding the need for the locks required by a common + pool. + + The original Thrift server was by Mark Slee . + A non-blocking variant was produced which allowed for a large number + of processing threads but restricted network I/O to a single thread -- + suboptimal on a multi-core CPU. David Reiss + and Mark Rabkin refactored this server into a + callback-driven event-based configuration. Ed Hall + elaborated on this to support network I/O on multiple threads (ideally + one per CPU core). + + @author Mark Slee + @author David Reiss + @author Mark Rabkin + @author Ed Hall + */ + +class TEventServer : public apache::thrift::server::TServer { + + public: + enum TransportType { + FRAMED = 0, + HEADER = 1, + + /*********** Deprecation Warning ******************* + * * + * The unframed transports are deprecated ! * + * They should be used for legancy services only * + * Also note: they only works with TBinaryProtocol * + ***************************************************/ + + UNFRAMED_BINARY = 2 + }; + + protected: + //! Default max size of per-worker connection pool. + static const uint32_t T_ASYNC_MAX_CONNECTION_POOL_SIZE = 64; + + /// Starting size of a TEventConnection's read buffer + static const int T_ASYNC_READ_BUFFER_DEFAULT_SIZE = 1024; + + /// Starting size of a TEventConnection's write buffer + static const int T_ASYNC_WRITE_BUFFER_DEFAULT_SIZE = 1024; + + /// Maximum size of read buffer allocated to idle connection (0 = unlimited) + static const int T_ASYNC_IDLE_READ_BUFFER_LIMIT = 0; + + /// Maximum size of write buffer allocated to idle connection (0 = unlimited) + static const int T_ASYNC_IDLE_WRITE_BUFFER_LIMIT = 0; + + /// # of calls before resizing oversized buffers (0 = check only on close) + static const int T_ASYNC_RESIZE_BUFFER_EVERY_N = 0; + + //! Default number of worker threads (should be # of processor cores). + static const int T_ASYNC_DEFAULT_WORKER_THREADS = 4; + + //! Maximum size of a frame we'll accept (default = 64MB) + static const int T_ASYNC_DEFAULT_MAX_FRAME_SIZE = 67108864; + + static const uint32_t T_MAX_NUM_MESSAGES_IN_PIPE = 0xffffffff; + + /// Listen backlog + static const int T_LISTEN_BACKLOG = 1024; + + //! Transport type + static const TransportType T_ASYNC_DEFAULT_TRANSPORT_TYPE = FRAMED; + + private: + struct WorkerInfo { + boost::shared_ptr worker; + boost::shared_ptr thread; + }; + + //! Max size of per-worker connection pool (may be set). + uint32_t maxConnectionPoolSize_; + + //! SSL context + boost::shared_ptr sslContext_; + + //! Factory that creates connection processor objects. + boost::shared_ptr asyncProcessorFactory_; + + //! Port to listen on + uint16_t port_; + + //! Listen socket + TAsyncServerSocket* socket_; + + //! The TEventBase currently driving serve(). NULL when not serving. + TEventBase* serveEventBase_; + + //! Number of worker threads (may be set) (should be # of CPU cores) + int nWorkers_; + + //! Milliseconds we'll wait for data to appear (0 = infinity) + int timeout_; + + //! Manager of per-thread TEventBase objects. + TEventBaseManager eventBaseManager_; + + //! Last worker chosen -- used to select workers in round-robin sequence. + uint32_t workerChoice_; + + //! List of workers. + typedef std::vector WorkerVector; + WorkerVector workers_; + + //! Maximum number of bytes accepted in a frame. + uint32_t maxFrameSize_; + + /// We initialize (and reinitialize) TEventConnection's read buffer to + /// this size. + size_t readBufferDefaultSize_; + + /// We initialize (and reinitialize) TEventConnection's write buffer to + /// this size. + size_t writeBufferDefaultSize_; + + /** + * Max read buffer size for an idle TConnection. When we place an idle + * TConnection into TEventWorker::connectionStack_ or on every + * resizeBufferEveryN_ calls, we insure that its read buffer is <= to + * this size; otherwise we replace it with a new one to insure that idle + * connections don't hog memory. 0 disables this check. + */ + size_t idleReadBufferLimit_; + + /** + * Max write buffer size for an idle connection. When we place an idle + * TConnection into TEventWorker::connectionStack_ or on every + * resizeBufferEveryN_ calls, we insure that its write buffer is <= to + * this size; otherwise we replace it with a new one to insure that idle + * connections don't hog memory. 0 disables this check. + */ + size_t idleWriteBufferLimit_; + + /** + * Every N calls we check the buffer size limits on a connected + * TEventConnection. 0 disables (i.e. the checks are only done when a + * connection closes). + */ + int32_t resizeBufferEveryN_; + + /** + * Call timeout in ms. When nonzero, limits the amount of time we allow + * between the start of a call and the actual invokation of its processor. + * The connection closes if it is exceeded. + */ + int32_t callTimeout_; + + /** + * The thread manager used when we're in queuing mode. + */ + boost::shared_ptr threadManager_; + + /** + * Thread local storage to track the current connection being processed + */ + concurrency::ThreadLocal > + currentConnection_; + + /** + * The time in milliseconds before an unperformed task expires -- + * queuing mode only. (0 == infinite) + */ + uint64_t taskExpireTime_; + + /** + * Set true if we are in queuing mode, false if not. + */ + bool queuingMode_; + + /** + * The speed for adjusting connection accept rate. + * 0 for disabling auto adjusting connection accept rate. + */ + double acceptRateAdjustSpeed_; + + /** + * The maximum number of unprocessed messages which a NotificationPipe + * can hold. + */ + uint32_t maxNumMsgsInPipe_; + + /** + * The transport type to use + */ + TransportType transportType_; + + void addWorker(concurrency::ThreadFactory* threadFactory); + + /** + * No-op signal handler (for SIGPIPE) + */ + static void sigNoOp(int signo) { + (void)signo; + } + + /** + * Set the current connection + */ + void setCurrentConnection(TEventConnection* conn) { + assert(currentConnection_.get() == NULL); + currentConnection_.set(conn); + } + + /** + * Clear the current connection + */ + void clearCurrentConnection() { + currentConnection_.clear(); + } + // Allow TEventConnection and TEventTask to access setCurrentConnection() + // and clearCurrentConnection(). Only these two private + // methods are meant to be used by TEventConnection and TEventTask. + friend class TEventConnection; + friend class TEventTask; + + public: + /** Construct an async Thrift server. + You need to compile your thrift configuration with thrift_cpp_options = + "cob_style" to get the required TAsyncProcessor class; this differs + from the usual TProcessor object by adding a completion callback. + TBinaryProtocol is assumed for both input and output with this + constructor. + @param processor the TAsyncProcessor object for this service + @param port the TCP port number for this service + @param nWorkers the number of worker threads -- should be the same + as the number of CPU cores, though if a process has + more than one TEventServer the cores can be split + between them. + */ + template + TEventServer(boost::shared_ptr processor, + int port, + int nWorkers = T_ASYNC_DEFAULT_WORKER_THREADS, + THRIFT_OVERLOAD_IF(AsyncProcessor, TAsyncProcessor)) : + apache::thrift::server::TServer(boost::shared_ptr()), + maxConnectionPoolSize_(T_ASYNC_MAX_CONNECTION_POOL_SIZE), + asyncProcessorFactory_(new TAsyncSingletonProcessorFactory(processor)), + port_(port), + socket_(NULL), + serveEventBase_(NULL), + nWorkers_(nWorkers), + timeout_(0), + eventBaseManager_(), + workerChoice_(0), + maxFrameSize_(T_ASYNC_DEFAULT_MAX_FRAME_SIZE), + readBufferDefaultSize_(T_ASYNC_READ_BUFFER_DEFAULT_SIZE), + writeBufferDefaultSize_(T_ASYNC_WRITE_BUFFER_DEFAULT_SIZE), + idleReadBufferLimit_(T_ASYNC_IDLE_READ_BUFFER_LIMIT), + idleWriteBufferLimit_(T_ASYNC_IDLE_WRITE_BUFFER_LIMIT), + resizeBufferEveryN_(T_ASYNC_RESIZE_BUFFER_EVERY_N), + callTimeout_(0), + taskExpireTime_(0), + queuingMode_(false), + acceptRateAdjustSpeed_(0), + maxNumMsgsInPipe_(T_MAX_NUM_MESSAGES_IN_PIPE), + transportType_(T_ASYNC_DEFAULT_TRANSPORT_TYPE) { + processor->setAsyncServer(this); + } + + /** Construct an async Thrift server for a particular TProtocol. + See above; adds a "protocolFactory" parameter to replace the + default TBinaryProtocol. + @param processor the TAsyncProcessor object for this service + @param protocolFactory the TProtocolFactory to use for input & output + @param port the TCP port number for this service + @param nWorkers the number of worker threads + */ + template + TEventServer(boost::shared_ptr processor, + boost::shared_ptr + protocolFactory, + int port, + int nWorkers = T_ASYNC_DEFAULT_WORKER_THREADS, + THRIFT_OVERLOAD_IF(AsyncProcessor, TAsyncProcessor)) : + apache::thrift::server::TServer(boost::shared_ptr()), + maxConnectionPoolSize_(T_ASYNC_MAX_CONNECTION_POOL_SIZE), + asyncProcessorFactory_(new TAsyncSingletonProcessorFactory(processor)), + port_(port), + socket_(NULL), + serveEventBase_(NULL), + nWorkers_(nWorkers), + timeout_(0), + eventBaseManager_(), + workerChoice_(0), + maxFrameSize_(T_ASYNC_DEFAULT_MAX_FRAME_SIZE), + readBufferDefaultSize_(T_ASYNC_READ_BUFFER_DEFAULT_SIZE), + writeBufferDefaultSize_(T_ASYNC_WRITE_BUFFER_DEFAULT_SIZE), + idleReadBufferLimit_(T_ASYNC_IDLE_READ_BUFFER_LIMIT), + idleWriteBufferLimit_(T_ASYNC_IDLE_WRITE_BUFFER_LIMIT), + resizeBufferEveryN_(T_ASYNC_RESIZE_BUFFER_EVERY_N), + callTimeout_(0), + taskExpireTime_(0), + queuingMode_(false), + acceptRateAdjustSpeed_(0), + maxNumMsgsInPipe_(T_MAX_NUM_MESSAGES_IN_PIPE), + transportType_(T_ASYNC_DEFAULT_TRANSPORT_TYPE) { + processor->setAsyncServer(this); + + setProtocolFactory(protocolFactory); + } + + /** Construct an async Thrift server with different input & output TProtocol. + See above; adds "inputProtocolFactory" and "outputProtocolFactory" + parameters. + @param processor the TAsyncProcessor object for this service + @param inputProtocolFactory the TProtocolFactory to use for input + @param outputProtocolFactory the TProtocolFactory to use for output + @param port the TCP port number for this service + @param nWorkers the number of worker threads + + @deprecated use TDuplex* ctor below + */ + template + TEventServer(boost::shared_ptr processor, + boost::shared_ptr + inputProtocolFactory, + boost::shared_ptr + outputProtocolFactory, + int port, + int nWorkers = T_ASYNC_DEFAULT_WORKER_THREADS, + THRIFT_OVERLOAD_IF(AsyncProcessor, TAsyncProcessor)) : + apache::thrift::server::TServer(boost::shared_ptr()), + maxConnectionPoolSize_(T_ASYNC_MAX_CONNECTION_POOL_SIZE), + asyncProcessorFactory_(new TAsyncSingletonProcessorFactory(processor)), + port_(port), + socket_(NULL), + serveEventBase_(NULL), + nWorkers_(nWorkers), + timeout_(0), + eventBaseManager_(), + workerChoice_(0), + maxFrameSize_(T_ASYNC_DEFAULT_MAX_FRAME_SIZE), + readBufferDefaultSize_(T_ASYNC_READ_BUFFER_DEFAULT_SIZE), + writeBufferDefaultSize_(T_ASYNC_WRITE_BUFFER_DEFAULT_SIZE), + idleReadBufferLimit_(T_ASYNC_IDLE_READ_BUFFER_LIMIT), + idleWriteBufferLimit_(T_ASYNC_IDLE_WRITE_BUFFER_LIMIT), + resizeBufferEveryN_(T_ASYNC_RESIZE_BUFFER_EVERY_N), + callTimeout_(0), + taskExpireTime_(0), + queuingMode_(false), + acceptRateAdjustSpeed_(0), + maxNumMsgsInPipe_(T_MAX_NUM_MESSAGES_IN_PIPE), + transportType_(T_ASYNC_DEFAULT_TRANSPORT_TYPE) { + processor->setAsyncServer(this); + + setDuplexProtocolFactory( + boost::shared_ptr( + new TDualProtocolFactory(inputProtocolFactory, outputProtocolFactory))); + } + + + /** Construct an async Thrift server with custom input & output TProtocol. + See above; Replaces protocolFactory with duplexProtocolFactory + parameters. + @param processor the TAsyncProcessor object for this service + @param duplexProtocolFactory the TProtocolFactory to use for input/output + @param port the TCP port number for this service + @param nWorkers the number of worker threads + */ + template + TEventServer(boost::shared_ptr processor, + boost::shared_ptr + duplexProtocolFactory, + int port, + int nWorkers = T_ASYNC_DEFAULT_WORKER_THREADS, + THRIFT_OVERLOAD_IF(AsyncProcessor, TAsyncProcessor)): + apache::thrift::server::TServer(boost::shared_ptr()), + maxConnectionPoolSize_(T_ASYNC_MAX_CONNECTION_POOL_SIZE), + asyncProcessorFactory_(new TAsyncSingletonProcessorFactory(processor)), + port_(port), + socket_(NULL), + serveEventBase_(NULL), + nWorkers_(nWorkers), + timeout_(0), + eventBaseManager_(), + workerChoice_(0), + maxFrameSize_(T_ASYNC_DEFAULT_MAX_FRAME_SIZE), + readBufferDefaultSize_(T_ASYNC_READ_BUFFER_DEFAULT_SIZE), + writeBufferDefaultSize_(T_ASYNC_WRITE_BUFFER_DEFAULT_SIZE), + idleReadBufferLimit_(T_ASYNC_IDLE_READ_BUFFER_LIMIT), + idleWriteBufferLimit_(T_ASYNC_IDLE_WRITE_BUFFER_LIMIT), + resizeBufferEveryN_(T_ASYNC_RESIZE_BUFFER_EVERY_N), + callTimeout_(0), + taskExpireTime_(0), + queuingMode_(false), + acceptRateAdjustSpeed_(0), + maxNumMsgsInPipe_(T_MAX_NUM_MESSAGES_IN_PIPE), + transportType_(T_ASYNC_DEFAULT_TRANSPORT_TYPE) { + processor->setAsyncServer(this); + + setDuplexProtocolFactory(duplexProtocolFactory); + } + + + /** Construct a task-queuing Thrift server for a particular TProtocol. + Largely compatible with TNonblockingServer. + + @param processor the TProcessor object for this service + @param protocolFactory the TProtocolFactory to use for input & output + @param port the TCP port number for this service + @param threadManager the thread manager we use for task queuing + @param nWorkers the number of worker threads + */ + template + TEventServer( + boost::shared_ptr& processor, + boost::shared_ptr& + protocolFactory, + int port, + boost::shared_ptr const& threadManager = + boost::shared_ptr(), + int nWorkers = T_ASYNC_DEFAULT_WORKER_THREADS, + THRIFT_OVERLOAD_IF(Processor, TProcessor)) : + apache::thrift::server::TServer(processor), + maxConnectionPoolSize_(T_ASYNC_MAX_CONNECTION_POOL_SIZE), + port_(port), + socket_(NULL), + serveEventBase_(NULL), + nWorkers_(nWorkers), + timeout_(0), + eventBaseManager_(), + workerChoice_(0), + maxFrameSize_(T_ASYNC_DEFAULT_MAX_FRAME_SIZE), + readBufferDefaultSize_(T_ASYNC_READ_BUFFER_DEFAULT_SIZE), + writeBufferDefaultSize_(T_ASYNC_WRITE_BUFFER_DEFAULT_SIZE), + idleReadBufferLimit_(T_ASYNC_IDLE_READ_BUFFER_LIMIT), + idleWriteBufferLimit_(T_ASYNC_IDLE_WRITE_BUFFER_LIMIT), + resizeBufferEveryN_(T_ASYNC_RESIZE_BUFFER_EVERY_N), + callTimeout_(0), + taskExpireTime_(0), + queuingMode_(true), + acceptRateAdjustSpeed_(0), + maxNumMsgsInPipe_(T_MAX_NUM_MESSAGES_IN_PIPE), + transportType_(T_ASYNC_DEFAULT_TRANSPORT_TYPE) { + setProtocolFactory(protocolFactory); + setThreadManager(threadManager); + } + + /** Construct a task-queuing Thrift server for a particular TProtocol. + Largely compatible with TNonblockingServer. + + @param processor the TProcessor object for this service + @param protocolFactory the TProtocolFactory to use for input & output + @param port the TCP port number for this service + @param threadManager the thread manager we use for task queuing + @param nWorkers the number of worker threads + */ + template + TEventServer( + boost::shared_ptr& processor, + boost::shared_ptr& + duplexProtocolFactory, + int port, + boost::shared_ptr const& threadManager = + boost::shared_ptr(), + int nWorkers = T_ASYNC_DEFAULT_WORKER_THREADS, + THRIFT_OVERLOAD_IF(Processor, TProcessor)) : + apache::thrift::server::TServer(processor), + maxConnectionPoolSize_(T_ASYNC_MAX_CONNECTION_POOL_SIZE), + port_(port), + socket_(NULL), + serveEventBase_(NULL), + nWorkers_(nWorkers), + timeout_(0), + eventBaseManager_(), + workerChoice_(0), + maxFrameSize_(T_ASYNC_DEFAULT_MAX_FRAME_SIZE), + readBufferDefaultSize_(T_ASYNC_READ_BUFFER_DEFAULT_SIZE), + writeBufferDefaultSize_(T_ASYNC_WRITE_BUFFER_DEFAULT_SIZE), + idleReadBufferLimit_(T_ASYNC_IDLE_READ_BUFFER_LIMIT), + idleWriteBufferLimit_(T_ASYNC_IDLE_WRITE_BUFFER_LIMIT), + resizeBufferEveryN_(T_ASYNC_RESIZE_BUFFER_EVERY_N), + callTimeout_(0), + taskExpireTime_(0), + queuingMode_(true), + acceptRateAdjustSpeed_(0), + maxNumMsgsInPipe_(T_MAX_NUM_MESSAGES_IN_PIPE), + transportType_(T_ASYNC_DEFAULT_TRANSPORT_TYPE) { + setDuplexProtocolFactory(duplexProtocolFactory); + setThreadManager(threadManager); + } + + virtual ~TEventServer() { + } + + /** + * + */ + void setSSLContext(boost::shared_ptr context) { + sslContext_ = context; + } + + boost::shared_ptr getSSLContext() const { + return sslContext_; + } + + /** + * Use the provided socket rather than binding to port_. The caller must + * call ::bind on this socket, but should not call ::listen. + * + * NOTE: TEventServe takes ownership of this 'socket' so if binding fails + * we destroy this socket, while cleaning itself up. So, 'accept' better + * work the first time :) + */ + void useExistingSocket(int socket); + + /** + * Return the file descriptor associated with the listening socket + */ + int getListenSocket() const; + + /** + * Get the TAsyncProcessorFactory object used by this server. + * + * @return a pointer to the processorFactory. + */ + boost::shared_ptr getAsyncProcessorFactory() const { + return asyncProcessorFactory_; + } + + /** + * Set the TAsyncProcessor object used by this server. + */ + void setAsyncProcessorFactory(boost::shared_ptr pf) { + asyncProcessorFactory_ = pf; + } + + /** + * Get the TEventBase used by the current thread. + * This will be different between each worker and the listener. Use this + * for any event monitoring within a processor and be careful NOT to + * cache between connections (since they may be executed by different + * workers). + * + * @return a pointer to the TEventBase. + */ + TEventBase* getEventBase() const { + return eventBaseManager_.getEventBase(); + } + + /** + * Get the TEventServer's main event base. + * + * @return a pointer to the TEventBase. + */ + TEventBase* getServeEventBase() const { + return serveEventBase_; + } + + /** + * Get the TEventBaseManager used by this server. + * This can be used to find or create the TEventBase associated with + * any given thread, including any new threads created by clients. + * + * @return a pointer to the TEventBaseManager. + */ + TEventBaseManager* getEventBaseManager() { + return &eventBaseManager_; + } + const TEventBaseManager* getEventBaseManager() const { + return &eventBaseManager_; + } + + /** + * Set the port to serve + */ + void setPort(uint16_t port) { + port_ = port ; + } + + /** + *Set the maximum number of inactive connection objects pooled. + * Since these objects consume memory, we need to limit how many we keep. + * You can disable pooling altogether by setting this to zero. Note that + * the actual maximum is nWorkers*size since each worker thread maintains + * its own pool (to avoid the need for locks). + * + * @param size the maximum number of inactive connections to pool. + */ + void setMaxConnectionPoolSize(uint32_t size) { + maxConnectionPoolSize_ = size; + } + + /** Get the maximum number of inactive connection objects pooled. + @return the maximum pool size. + */ + uint32_t getMaxConnectionPoolSize() const { + return maxConnectionPoolSize_; + } + + /** + * Get the maximum number of unprocessed messages which a NotificationPipe + * can hold. + */ + uint32_t getMaxNumMessagesInPipe() const { + return maxNumMsgsInPipe_; + } + /** + * Set the maximum number of unprocessed messages in NotificationPipe. + * No new message will be sent to that NotificationPipe if there are more + * than such number of unprocessed messages in that pipe. + */ + void setMaxNumMessagesInPipe(uint32_t num) { + maxNumMsgsInPipe_ = num; + } + + /** + * Get the speed of adjusting connection accept rate. + */ + double getAcceptRateAdjustSpeed() const { + return acceptRateAdjustSpeed_; + } + + /** + * Set the speed of adjusting connection accept rate. + */ + void setAcceptRateAdjustSpeed(double speed) { + acceptRateAdjustSpeed_ = speed; + } + + /** + * Get the number of connections dropped by the TAsyncServerSocket + */ + void getNumDroppedConnections() const; + + /** Reset the maximum number of inactive connection objects to the default. + */ + void resetMaxConnectionPoolSize() { + setMaxConnectionPoolSize(T_ASYNC_MAX_CONNECTION_POOL_SIZE); + } + + /** Get maximum number of milliseconds we'll wait for data (0 = infinity). + * + * @return number of milliseconds, or 0 if no timeout set. + */ + int getRecvTimeout() const { + return timeout_; + } + + /** Set maximum number of milliseconds we'll wait for data (0 = infinity). + * Note: existing connections are unaffected by this call. + * + * @param timeout number of milliseconds, or 0 to disable timeouts. + */ + void setRecvTimeout(int timeout) { + timeout_ = timeout; + } + + /** Set the maximum frame size server will accept. + * + * @param size the maximum size in bytes of a frame we'll accept. + */ + void setMaxFrameSize(uint32_t size) { + maxFrameSize_ = size; + } + + /** Get the maximum frame size server will accept. + * + * @return the maximum pool size. + */ + uint32_t getMaxFrameSize() const { + return maxFrameSize_; + } + + /** + * Get the starting size of a TEventConnection object's read buffer. + * + * @return # bytes we init a TEventConnection object's read buffer to. + */ + size_t getReadBufferDefaultSize() const { + return readBufferDefaultSize_; + } + + /** + * Set the starting size of a TEventConnection object's read buffer. + * + * @param size # bytes we init a TEventConnection object's read buffer to. + */ + void setReadBufferDefaultSize(size_t size) { + readBufferDefaultSize_ = size; + } + + /** + * Get the starting size of a TEventConnection object's write buffer. + * + * @return # bytes we init a TEventConnection object's write buffer to. + */ + size_t getWriteBufferDefaultSize() const { + return writeBufferDefaultSize_; + } + + /** + * Set the starting size of a TEventConnection object's write buffer. + * + * @param size # bytes we init a TEventConnection object's write buffer to. + */ + void setWriteBufferDefaultSize(size_t size) { + writeBufferDefaultSize_ = size; + } + + /** + * Get the maximum size of read buffer allocated to idle TConnection objects. + * + * @return # bytes beyond which we will shrink buffers when idle. + */ + size_t getIdleReadBufferLimit() const { + return idleReadBufferLimit_; + } + + /** + * Set the maximum size read buffer allocated to idle TEventConnection + * objects. If a TEventConnection object is found (either on connection + * close or between calls when resizeBufferEveryN_ is set) with more than + * this much memory + * allocated to its read buffer, we shrink it to this value. + * + * @param limit of bytes beyond which we will shrink buffers when checked. + */ + void setIdleReadBufferLimit(size_t limit) { + idleReadBufferLimit_ = limit; + } + + /** + * Get the maximum size of write buffer allocated to idle TEventConnection + * objects. + * + * @return # bytes beyond which we will reallocate buffers when checked. + */ + size_t getIdleWriteBufferLimit() const { + return idleWriteBufferLimit_; + } + + /** + * Set the maximum size write buffer allocated to idle TEventConnection + * objects. If a TEventConnection object is found (either on connection + * close or between calls when resizeBufferEveryN_ is set) with more than + * this much memory allocated to its write buffer, we destroy and construct + * that buffer. + * + * @param limit of bytes beyond which we will shrink buffers when idle. + */ + void setIdleWriteBufferLimit(size_t limit) { + idleWriteBufferLimit_ = limit; + } + + /** + * Get # of calls made between buffer size checks. 0 means disabled. + * + * @return # of calls between buffer size checks. + */ + int32_t getResizeBufferEveryN() const { + return resizeBufferEveryN_; + } + + /** + * Check buffer sizes every "count" calls. This allows buffer limits + * to be enforced for persistant connections with a controllable degree + * of overhead. 0 disables checks except at connection close. + * + * @param count the number of calls between checks, or 0 to disable + */ + void setResizeBufferEveryN(int32_t count) { + resizeBufferEveryN_ = count; + } + + /** + * Set a call timeout in milliseconds. + * + * When a worker's TEventBase starts taking longer than this amount of time + * to process a single loop, start dropping connections to reduce loadj + * + * TODO: This should be renamed something other than "call timeout" + * + * @param milliseconds the call timeout (0 inhibits) + */ + void setCallTimeout(int32_t milliseconds) { + callTimeout_ = milliseconds; + } + + /** + * Get the call timeout in milliseconds. 0 (default) disables. + * + * @return the call timeout in milliseconds + */ + int32_t getCallTimeout() const { + return callTimeout_; + } + + /** + * Set Thread Manager (for queuing mode). + * + * @param threadManager a shared pointer to the thread manager + */ + void setThreadManager(boost::shared_ptr + threadManager) { + threadManager_ = threadManager; + } + + /** + * Get Thread Manager (for queuing mode). + * + * @return a shared pointer to the thread manager + */ + boost::shared_ptr getThreadManager() { + return threadManager_; + } + + /** + * Get the task expire time (for queuing mode). + * + * @return task expire time + */ + int64_t getTaskExpireTime() const { + return taskExpireTime_; + } + + /** + * Return whether we are in queuing mode or not. + * + * @return true if we are in queuing mode, false if not. + */ + bool queuingMode() const { + return queuingMode_; + } + + /** + * Set the transport type to use + * + * @param transportType transport type + */ + void setTransportType(TransportType transportType) { + + /*********** Deprecation Warning ******************* + * * + * The unframed transports are deprecated ! * + * They should be used for legancy services only * + * Also note: they only works with TBinaryProtocol * + ***************************************************/ + + if (transportType == UNFRAMED_BINARY && + !dynamic_cast( + getDuplexProtocolFactory()->getInputProtocolFactory().get())) { + throw TLibraryException( + "UnFramedTransport can only be used with TBinaryProtocol"); + } else if (transportType == HEADER && + !dynamic_cast( + getDuplexProtocolFactory()->getInputProtocolFactory().get())) { + throw TLibraryException( + "HEADER transport can only be used with THeaderProtocol"); + } + + transportType_ = transportType; + } + + /** + * Get the transport type to use + * + * @return transport type + */ + TransportType getTransportType() { + return transportType_; + } + + + /** + * Call this to complete initialization + */ + void setup(); + + /** + * Kill the workers and wait for listeners to quit + */ + void cleanUp(); + + /** + * One stop solution: + * + * starts worker threads, enters accept loop; when + * the accept loop exits, shuts down and joins workers. + */ + void serve(); + + /** + * Call this to stop the server, if started by serve() + * + * This causes the main serve() function to stop listening for new + * connections, closes existing connections, shut down the worker threads, + * and then return. + */ + void stop(); + + /** + * Call this to stop listening on the server port. + * + * This causes the main serve() function to stop listening for new + * connections while still allows the worker threads to process + * existing connections. stop() still needs to be called to clear + * up the worker threads. + */ + void stopListening(); + + /** + * Terminate a given pending task. Callable by the thread manager or + * from the server context. + */ + void expireClose( + boost::shared_ptr task); + + /** + * In task queue mode, drop a task from the head of the queue and shut + * down the associated connection. + */ + bool drainPendingTask(); + + /** + * Get the TConnectionContext for the connection currently being processed. + * + * This is intended to be invoked from within the TAsyncProcessor (or the + * handler used by the TProcessor). + * + * @return Return a pointer to the TConnectionContext for the current + * connection, or NULL if invoked outside of a call to + * TAsyncProcessor::process(). The returned TConnectionContext + * object is guaranteed to remain valid until the + * TAsyncProcessor invokes its success or error callback. However, + * getConnectionContext() may return NULL when invoked after + * process() has returned. + * + * In other words, async handlers may continue using the + * TConnectionContext object for the duration of the handler + * processing. However, they must retrieve the TConnectionContext + * inside the call to process() and cache it for later use if + * they need it later. + */ + virtual server::TConnectionContext* getConnectionContext() const; + + // We use this to get the processor when in task queue mode + using TServer::getProcessor; +}; + +}}} // apache::thrift::async + +#endif // #ifndef THRIFT_ASYNC_TEVENTSERVER_H_ diff --git a/thrift/lib/cpp/async/TEventTask.h b/thrift/lib/cpp/async/TEventTask.h new file mode 100644 index 0000000000..ab21081ca6 --- /dev/null +++ b/thrift/lib/cpp/async/TEventTask.h @@ -0,0 +1,49 @@ +#ifndef _THRIFT_TEVENTTASK_H_ +#define _THRIFT_TEVENTTASK_H_ 1 + +#include "thrift/lib/cpp/Thrift.h" +#include "thrift/lib/cpp/server/TServer.h" +#include "thrift/lib/cpp/transport/TBufferTransports.h" +#include "thrift/lib/cpp/concurrency/ThreadManager.h" +#include "thrift/lib/cpp/async/TEventConnection.h" +#include +#include +#include "thrift/lib/cpp/TProcessor.h" +#include "thrift/lib/cpp/protocol/TProtocol.h" + +namespace apache { namespace thrift { namespace async { + +class TEventTask : public apache::thrift::concurrency::Runnable { + public: + explicit TEventTask(TEventConnection* connection); + + void run(); + + TEventConnection* getConnection() const { + return connection_; + } + + private: + boost::shared_ptr processor_; + boost::shared_ptr input_; + boost::shared_ptr output_; + TEventConnection* connection_; + TConnectionContext* connectionContext_; +}; + +class TaskCompletionMessage { + public: + explicit TaskCompletionMessage(TEventConnection *inConnection) + : connection(inConnection) {} + + TaskCompletionMessage(TaskCompletionMessage &&msg) + : connection(msg.connection) { + msg.connection = NULL; + } + + TEventConnection *connection; +}; + +} } } // namespace apache::thrift::async + +#endif // !_THRIFT_TEVENTTASK_H_ diff --git a/thrift/lib/cpp/async/TEventUtil.h b/thrift/lib/cpp/async/TEventUtil.h new file mode 100644 index 0000000000..4d0b62b68f --- /dev/null +++ b/thrift/lib/cpp/async/TEventUtil.h @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_ASYNC_TEVENTUTIL_H_ +#define THRIFT_ASYNC_TEVENTUTIL_H_ 1 + +#include // libevent + +namespace apache { namespace thrift { namespace async { + +/** + * low-level libevent utility functions + */ +class TEventUtil { + public: + static bool isEventRegistered(const struct event* ev) { + // If any of these flags are set, the event is registered. + enum { + EVLIST_REGISTERED = (EVLIST_INSERTED | EVLIST_ACTIVE | + EVLIST_TIMEOUT | EVLIST_SIGNAL) + }; + return (ev->ev_flags & EVLIST_REGISTERED); + } +}; + +}}} // apache::thrift::async + +#endif // THRIFT_ASYNC_TEVENTUTIL_H_ diff --git a/thrift/lib/cpp/async/TEventWorker.h b/thrift/lib/cpp/async/TEventWorker.h new file mode 100644 index 0000000000..ef57271a11 --- /dev/null +++ b/thrift/lib/cpp/async/TEventWorker.h @@ -0,0 +1,264 @@ +// Copyright (c) 2006- Facebook +// Distributed under the Thrift Software License +// +// See accompanying file LICENSE or visit the Thrift site at: +// http://developers.facebook.com/thrift/ + +#ifndef THRIFT_SERVER_TEVENTWORKER_H_ +#define THRIFT_SERVER_TEVENTWORKER_H_ 1 + +#include "thrift/lib/cpp/async/TAsyncServerSocket.h" +#include "thrift/lib/cpp/async/TAsyncSSLSocket.h" +#include "thrift/lib/cpp/async/TEventServer.h" +#include "thrift/lib/cpp/async/TEventBase.h" +#include "thrift/lib/cpp/async/TEventHandler.h" +#include "thrift/lib/cpp/async/TNotificationQueue.h" +#include "thrift/lib/cpp/server/TServer.h" +#include +#include +#include + +namespace apache { namespace thrift { namespace async { + +// Forward declaration of classes +class TAsyncProcessorFactory; +class TEventConnection; +class TEventServer; +class TaskCompletionMessage; +/** + * TEventWorker drives the actual I/O for TEventServer connections. + * + * The TEventServer itself accepts incoming connections, then hands off each + * connection to a TEventWorker running in another thread. There should + * typically be around one TEventWorker thread per core. + */ +class TEventWorker : + public apache::thrift::server::TServer, + public TAsyncServerSocket::AcceptCallback, + public TAsyncSSLSocket::HandshakeCallback, + public TNotificationQueue::Consumer { + private: + /// Object that processes requests. + boost::shared_ptr asyncProcessorFactory_; + + /// The mother ship. + TEventServer* server_; + + /// An instance's TEventBase for I/O. + TEventBase eventBase_; + + /// Our ID in [0:nWorkers). + uint32_t workerID_; + + /// Pipe that task completion notifications are sent over + TNotificationQueue notificationQueue_; + + /** + * A stack of idle TEventConnection objects for reuse. + * When we close a connection, we place it on this stack so that the + * object can be reused later, rather than freeing the memory and + * reallocating a new object later. + */ + std::stack connectionStack_; + + /// Transport type to use + TEventServer::TransportType transportType_; + + /** + * Called when the connection is fully accepted (after SSL accept if needed) + */ + void finishConnectionAccepted(TAsyncSocket *asyncSock); + + /** + * Create or reuse a TEventConnection initialized for the given socket FD. + * + * @param socket the FD of a freshly-connected socket. + * @param address the peer address of the socket. + * @return pointer to a TConenction object for that socket. + */ + TEventConnection* createConnection( + boost::shared_ptr asyncSocket, + const transport::TSocketAddress* address); + + /** + * Handler called when a new connection may be available. + */ + void acceptConnections(); + + void makeCompletionCallback(); + + /** + * Initialize our TEventBase to generate incoming connection events. + * Note that this is called once before the main loop is executed and + * sets up a READ event on the output of the listener's socktpair. + */ + void registerEvents(); + + /** + * Callback used when loop latency exceeds the requested threshold. + */ + void maxLatencyCob(); + + typedef std::list ConnectionList; + + // pointer hash functor + static const uint64_t kPtrHashMult = 5700357409661599291LL; + static const uint64_t kPtrHashShift = 3; + template + struct hash { }; + template + struct hash { + size_t operator()(T* p) const { + return ((size_t)p ^ ((size_t)p >> kPtrHashShift)) * kPtrHashMult; + } + size_t operator()(const T* p) const { + return ((size_t)p ^ ((size_t)p >> kPtrHashShift)) * kPtrHashMult; + } + }; + typedef __gnu_cxx::hash_map > ConnectionMap; + + /** + * The list of active connections (used to allow the oldest connections + * to be shed during overload). + */ + ConnectionList activeConnectionList_; + /** + * A hash map used to map connections to their place in the connection list + */ + ConnectionMap activeConnectionMap_; + + public: + + /** + * TEventWorker is the actual server object for existing connections. + * One or more of these should be created by TEventServer (one per + * CPU core is recommended). + * + * @param processorFactory a TAsyncProcessorFactory object as + * obtained from the generated Thrift code (the user service + * is integrated through this). + * @param inputProtocolFactory the TProtocolFactory class supporting + * inbound Thrift requests. + * @param outputProtocolFactory the TProtocolFactory class supporting + * responses (if any) after processing completes. + * @param server the TEventServer which created us. + * @param workerID the ID assigned to this worker + */ + TEventWorker(boost::shared_ptr processorFactory, + boost::shared_ptr + protocolFactory, + TEventServer* server, + uint32_t workerID) : + TServer(boost::shared_ptr()), + asyncProcessorFactory_(processorFactory), + server_(server), + eventBase_(), + workerID_(workerID), + transportType_(TEventServer::FRAMED) { + + setDuplexProtocolFactory(protocolFactory); + transportType_ = server->getTransportType(); + if (transportType_ == TEventServer::FRAMED) { + // do the dynamic cast once rather than per connection + if (dynamic_cast( + protocolFactory.get())) { + transportType_ = TEventServer::HEADER; + } + } + } + + /** + * Destroy a TEventWorker. We've use boost::scoped_ptr<> to take care + * of freeing up memory, so nothing to be done here but release the + * connection stack. + */ + virtual ~TEventWorker(); + + /** + * Get my TAsyncProcessorFactory object. + * + * @returns pointer to my TAsyncProcessorFactory object. + */ + boost::shared_ptr getAsyncProcessorFactory() { + return asyncProcessorFactory_; + } + + /** + * Get my TEventBase object. + * + * @returns pointer to my TEventBase object. + */ + TEventBase* getEventBase() { + return &eventBase_; + } + + /** + * Get underlying server. + * + * @returns pointer to TEventServer + */ + TEventServer* getServer() const { + return server_; + } + + /** + * Get my numeric worker ID (for diagnostics). + * + * @return integer ID of this worker + */ + int32_t getID() { + return workerID_; + } + + /** + * Dispose of a TEventConnection object. + * Will add to a pool of these objects or destroy as necessary. + * + * @param connection a now-idle connection. + */ + void returnConnection(TEventConnection* connection); + + /** + * Cause a completion callback for the requested connection to occur + * within that connection's context. + * + * @param msg task completion message + * @return true if notification was sent, false if it failed. + */ + bool notifyCompletion(TaskCompletionMessage &&msg); + + /** + * Task completed (called in this worker's thread) + */ + void messageAvailable(TaskCompletionMessage &&msg); + + virtual void connectionAccepted(int fd, + const transport::TSocketAddress& clientAddr) + THRIFT_NOEXCEPT; + virtual void acceptError(const std::exception& ex) THRIFT_NOEXCEPT; + virtual void acceptStopped() THRIFT_NOEXCEPT; + + /** + * TAsyncSSLSocket::HandshakeCallback interface + */ + void handshakeSuccess(TAsyncSSLSocket *sock) THRIFT_NOEXCEPT; + void handshakeError(TAsyncSSLSocket *sock, + const transport::TTransportException& ex) THRIFT_NOEXCEPT; + + + /** + * Enter event loop and serve. + */ + void serve(); + + /** + * Exit event loop. + */ + void shutdown(); +}; + +}}} // apache::thrift::async + +#endif // #ifndef THRIFT_SERVER_TEVENTWORKER_H_ diff --git a/thrift/lib/cpp/async/TFramedAsyncChannel.h b/thrift/lib/cpp/async/TFramedAsyncChannel.h new file mode 100644 index 0000000000..40a6819e24 --- /dev/null +++ b/thrift/lib/cpp/async/TFramedAsyncChannel.h @@ -0,0 +1,204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_ASYNC_TFRAMEDASYNCCHANNEL_H_ +#define THRIFT_ASYNC_TFRAMEDASYNCCHANNEL_H_ 1 + +#include "thrift/lib/cpp/async/TStreamAsyncChannel.h" + +namespace apache { namespace thrift { namespace async { + +namespace detail { + +/** + * Encapsulation of one outstanding write request on a TFramedAsyncChannel. + */ +class TFramedACWriteRequest : + public TAsyncChannelWriteRequestBase { + public: + TFramedACWriteRequest(const VoidCallback& callback, + const VoidCallback& errorCallback, + transport::TMemoryBuffer* message, + TAsyncEventChannel* channel); + + void write(TAsyncTransport* transport, + TAsyncTransport::WriteCallback* callback) THRIFT_NOEXCEPT; + + void writeSuccess() THRIFT_NOEXCEPT; + void writeError(size_t bytesWritten, + const transport::TTransportException& ex) THRIFT_NOEXCEPT; + + private: + union { + uint32_t frameSize_; + char frameSizeBuf_[sizeof(uint32_t)]; + }; +}; + +/** + * Read state for TFramedAsyncChannel + */ +class TFramedACReadState { + public: + typedef std::tr1::function VoidCallback; + + TFramedACReadState(); + + // Methods required by TStreamAsyncChannel + + void setCallbackBuffer(transport::TMemoryBuffer* buffer) { + buffer_ = buffer; + bytesRead_ = 0; + } + void unsetCallbackBuffer() { + buffer_ = NULL; + } + + bool hasReadAheadData() { + assert(bytesRead_ == 0); + return false; + } + bool hasPartialMessage() { + return (bytesRead_ > 0); + } + + void getReadBuffer(void** bufReturn, size_t* lenReturn); + bool readDataAvailable(size_t len); + + // Other methods specifict to TFramedAsyncChannel + + void setMaxFrameSize(uint32_t size) { + maxFrameSize_ = size; + } + + uint32_t getMaxFrameSize() const { + return maxFrameSize_; + } + + private: + /// maximum frame size accepted + uint32_t maxFrameSize_; + + union { + uint32_t frameSize_; + char frameSizeBuf_[sizeof(uint32_t)]; + }; + + /** + * The number of bytes read. + * + * This includes the bytes in the frame size. When bytesRead_ is less than + * sizeof(uint32_t), we are still reading the frame size. Otherwise, we have + * read bytesRead_ - sizeof(uint32_t) bytes of the body. + */ + uint32_t bytesRead_; + apache::thrift::transport::TMemoryBuffer* buffer_; +}; + +} // namespace detail + +/** + * TFramedAsyncChannel + * + * This is a TAsyncChannel implementation that reads and writes messages + * prefixed with a 4-byte frame length. + * + * Its messages are compatible with TFramedTransport. + */ +class TFramedAsyncChannel : + public TStreamAsyncChannel { + private: + typedef TStreamAsyncChannel Parent; + + public: + explicit TFramedAsyncChannel( + const boost::shared_ptr& transport + ) + : Parent(transport) {} + + /** + * Helper function to create a shared_ptr. + * + * This passes in the correct destructor object, since TFramedAsyncChannel's + * destructor is protected and cannot be invoked directly. + */ + static boost::shared_ptr newChannel( + const boost::shared_ptr& transport) { + return boost::shared_ptr( + new TFramedAsyncChannel(transport), Destructor()); + } + + /// size in bytes beyond which we'll reject a given frame size. + void setMaxFrameSize(uint32_t size) { + readState_.setMaxFrameSize(size); + } + + uint32_t getMaxFrameSize() const { + return readState_.getMaxFrameSize(); + } + + protected: + /** + * Protected destructor. + * + * Users of TFramedAsyncChannel must never delete it directly. Instead, + * invoke destroy(). + */ + virtual ~TFramedAsyncChannel() { } +}; + +class TFramedAsyncChannelFactory : public TStreamAsyncChannelFactory { + public: + TFramedAsyncChannelFactory() + : maxFrameSize_(0x7fffffff) + , recvTimeout_(0) + , sendTimeout_(0) {} + + void setMaxFrameSize(uint32_t bytes) { + maxFrameSize_ = bytes; + } + + void setRecvTimeout(uint32_t milliseconds) { + recvTimeout_ = milliseconds; + } + + void setSendTimeout(uint32_t milliseconds) { + sendTimeout_ = milliseconds; + } + + virtual boost::shared_ptr newChannel( + const boost::shared_ptr& transport) { + boost::shared_ptr channel( + TFramedAsyncChannel::newChannel(transport)); + transport->setSendTimeout(sendTimeout_); + channel->setMaxFrameSize(maxFrameSize_); + channel->setRecvTimeout(recvTimeout_); + return channel; + } + + private: + uint32_t maxFrameSize_; + uint32_t recvTimeout_; + uint32_t sendTimeout_; +}; + +}}} // apache::thrift::async + +#endif // THRIFT_ASYNC_TFRAMEDASYNCCHANNEL_H_ diff --git a/thrift/lib/cpp/async/THeaderAsyncChannel.h b/thrift/lib/cpp/async/THeaderAsyncChannel.h new file mode 100644 index 0000000000..ecb77c3ff7 --- /dev/null +++ b/thrift/lib/cpp/async/THeaderAsyncChannel.h @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_ASYNC_THEADERASYNCCHANNEL_H_ +#define THRIFT_ASYNC_THEADERASYNCCHANNEL_H_ 1 + +#include "thrift/lib/cpp/async/TUnframedAsyncChannel.h" + +namespace apache { namespace thrift { namespace async { + +namespace detail { + +/** + * A class to determine the end of a THeaderProtocol message. This is not as + * sophisticated as the logic in THeaderTransport, so it cannot yet handle any + * unframed transports, just THeader and TFramed. However, the previous + * implementation used TFramedAsyncChannel, so the limitation is not new. + */ +class THeaderACProtocolTraits { + public: + + THeaderACProtocolTraits() + : maxMessageSize_(0x7ffffff) {} + + // Methods required by TUnframedACReadState + bool getMessageLength(uint8_t* buffer, + uint32_t bufferLength, + uint32_t* messageLength); + + void setMaxMessageSize(uint32_t maxMessageSize) { + maxMessageSize_ = maxMessageSize; + } + + private: + uint32_t maxMessageSize_; +}; + +} // namespace detail + +/** + * THeaderAsyncChannel + * + * This is a TAsyncChannel implementation that reads and writes + * messages encoded using THeaderProtocol. + */ +class THeaderAsyncChannel : + public TUnframedAsyncChannel { + private: + typedef TUnframedAsyncChannel Parent; + + public: + explicit THeaderAsyncChannel( + const boost::shared_ptr& transport) + : Parent(transport) {} + + /** + * Helper function to create a shared_ptr. + * + * This passes in the correct destructor object, since THeaderAsyncChannel's + * destructor is protected and cannot be invoked directly. + */ + static boost::shared_ptr newChannel( + const boost::shared_ptr& transport) { + return boost::shared_ptr( + new THeaderAsyncChannel(transport), Destructor()); + } + + void setMaxMessageSize(uint32_t size) { + Parent::setMaxMessageSize(size); + readState_.getProtocolTraits()->setMaxMessageSize(size); + } + + // Note that we inherit getMaxMessageSize() from TUnframedAsyncChannel. + + protected: + /** + * Protected destructor. + * + * Users of THeaderAsyncChannel must never delete it directly. Instead, + * invoke destroy(). + */ + virtual ~THeaderAsyncChannel() { } +}; + +class THeaderAsyncChannelFactory : public TStreamAsyncChannelFactory { + public: + THeaderAsyncChannelFactory() + : maxMessageSize_(0x7fffffff) + , recvTimeout_(0) + , sendTimeout_(0) {} + + void setMaxMessageSize(uint32_t bytes) { + maxMessageSize_ = bytes; + } + + void setRecvTimeout(uint32_t milliseconds) { + recvTimeout_ = milliseconds; + } + + void setSendTimeout(uint32_t milliseconds) { + sendTimeout_ = milliseconds; + } + + virtual boost::shared_ptr newChannel( + const boost::shared_ptr& transport) { + boost::shared_ptr channel( + THeaderAsyncChannel::newChannel(transport)); + transport->setSendTimeout(sendTimeout_); + channel->setMaxMessageSize(maxMessageSize_); + channel->setRecvTimeout(recvTimeout_); + return channel; + } + + private: + uint32_t maxMessageSize_; + uint32_t recvTimeout_; + uint32_t sendTimeout_; +}; + +}}} // apache::thrift::async + +#endif // THRIFT_ASYNC_THEADERASYNCCHANNEL_H_ diff --git a/thrift/lib/cpp/async/THttpAsyncChannel.h b/thrift/lib/cpp/async/THttpAsyncChannel.h new file mode 100644 index 0000000000..9eafc3a819 --- /dev/null +++ b/thrift/lib/cpp/async/THttpAsyncChannel.h @@ -0,0 +1,204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_ASYNC_THTTPASYNCCHANNEL_H_ +#define THRIFT_ASYNC_THTTPASYNCCHANNEL_H_ 1 + +#include "thrift/lib/cpp/async/TStreamAsyncChannel.h" +#include "thrift/lib/cpp/util/THttpParser.h" + +namespace apache { namespace thrift { namespace async { + +class THttpAsyncChannel; + +namespace detail { + +/** + * Encapsulation of one outstanding write request on a THttpAsyncChannel. + */ +class THttpACWriteRequest : + public TAsyncChannelWriteRequestBase { + public: + typedef std::tr1::function VoidCallback; + + THttpACWriteRequest(const VoidCallback& callback, + const VoidCallback& errorCallback, + transport::TMemoryBuffer* message, + TAsyncEventChannel* channel); + + void write(TAsyncTransport* transport, + TAsyncTransport::WriteCallback* callback) THRIFT_NOEXCEPT; + + void writeSuccess() THRIFT_NOEXCEPT; + void writeError(size_t bytesWritten, + const transport::TTransportException& ex) THRIFT_NOEXCEPT; + + private: + char lengthBuf_[64]; + THttpAsyncChannel* channel_; +}; + +/** + * Read state for THttpAsyncChannel + */ +class THttpACReadState { + public: + typedef std::tr1::function VoidCallback; + + THttpACReadState() { + } + + // Methods required by TStreamAsyncChannel + + void setCallbackBuffer(transport::TMemoryBuffer* buffer) { + parser_->setDataBuffer(buffer); + } + void unsetCallbackBuffer() { + parser_->unsetDataBuffer(); + } + + bool hasReadAheadData() { + return parser_->hasReadAheadData(); + } + bool hasPartialMessage() { + return parser_->hasPartialMessage(); + } + + void getReadBuffer(void** bufReturn, size_t* lenReturn); + bool readDataAvailable(size_t len); + + // Other methods specific to THttpAsyncChannel + void setParser(boost::shared_ptr parser) { + parser_ = parser; + } + private: + boost::shared_ptr parser_; +}; + +} // namespace detail + +/** + * THttpAsyncChannel + * + * This is a TAsyncChannel implementation that reads and writes messages + * encapuated in HTTP. + * + * Its messages are compatible with THttpTransport. + */ +class THttpAsyncChannel : + public TStreamAsyncChannel { + private: + typedef TStreamAsyncChannel Parent; + boost::shared_ptr parser_; + + public: + explicit THttpAsyncChannel( + const boost::shared_ptr& transport) + : Parent(transport) { + } + + /** + * Helper function to create a shared_ptr. + * + * This passes in the correct destructor object, since THttpAsyncChannel's + * destructor is protected and cannot be invoked directly. + */ + static boost::shared_ptr newChannel( + const boost::shared_ptr& transport) { + return boost::shared_ptr( + new THttpAsyncChannel(transport), Destructor()); + } + + /// size in bytes beyond which we'll reject a given http size. + void setMaxHttpSize(uint32_t size) { + parser_->setMaxSize(size); + } + + uint32_t getMaxHttpSize() const { + return parser_->getMaxSize(); + } + + void setParser(boost::shared_ptr parser) { + parser_ = parser; + readState_.setParser(parser); + } + + boost::shared_ptr getParser() const { + return parser_; + } + + int constructHeader(iovec* ops, + int opsLen, + int contentLength, + char* contentLengthBuf) { + return parser_->constructHeader(ops, + opsLen, + contentLength, + contentLengthBuf); + } + + protected: + /** + * Protected destructor. + * + * Users of THttpAsyncChannel must never delete it directly. Instead, + * invoke destroy(). + */ + virtual ~THttpAsyncChannel() { } +}; + +class THttpAsyncChannelFactory : public TStreamAsyncChannelFactory { + public: + THttpAsyncChannelFactory() + : maxHttpSize_(0x7fffffff) + , recvTimeout_(0) + , sendTimeout_(0) {} + + void setMaxHttpSize(uint32_t bytes) { + maxHttpSize_ = bytes; + } + + void setRecvTimeout(uint32_t milliseconds) { + recvTimeout_ = milliseconds; + } + + void setSendTimeout(uint32_t milliseconds) { + sendTimeout_ = milliseconds; + } + + virtual boost::shared_ptr newChannel( + const boost::shared_ptr& transport) { + boost::shared_ptr channel( + THttpAsyncChannel::newChannel(transport)); + transport->setSendTimeout(sendTimeout_); + channel->setMaxHttpSize(maxHttpSize_); + channel->setRecvTimeout(recvTimeout_); + return channel; + } + + private: + uint32_t maxHttpSize_; + uint32_t recvTimeout_; + uint32_t sendTimeout_; +}; + +}}} // apache::thrift::async + +#endif // THRIFT_ASYNC_THTTPASYNCCHANNEL_H_ diff --git a/thrift/lib/cpp/async/TNotificationPipe.h b/thrift/lib/cpp/async/TNotificationPipe.h new file mode 100644 index 0000000000..40b2f51d52 --- /dev/null +++ b/thrift/lib/cpp/async/TNotificationPipe.h @@ -0,0 +1,337 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_ASYNC_TNOTIFICATIONPIPE_H +#define THRIFT_ASYNC_TNOTIFICATIONPIPE_H 1 + +#include "thrift/lib/cpp/async/TDelayedDestruction.h" +#include "thrift/lib/cpp/async/TEventBase.h" +#include "thrift/lib/cpp/async/TEventHandler.h" +#include "thrift/lib/cpp/concurrency/Mutex.h" +#include +#include +#include + +namespace apache { namespace thrift { namespace async { + +/** + * A simple notification pipe for sending messages to a TEventBase thread. + * + * TNotificationPipe is a unidirectional pipe for sending small, atomic + * messages. + * + * TNotificationPipe cannot be send messages larger than a fixed size. + * TNotificationPipe::kMaxMessageSize defines the maximum message size + * supported. If you need to pass larger amounts of data between threads, + * consider just passing a pointer to the data over the pipe, and using some + * external mechanism to synchronize management of the memory. + * + * + * TNotificationPipe provides two parallel APIs for writing and closing the + * pipe: a thread-safe version and a non-thread-safe version. Which version to + * use depends on how the caller uses the pipe: + * + * - If there is only a single writer thread, you can use the non-thread-safe + * versions of trySendMessage() and close(). This guarantees close() is + * never called by one thread while another thread is attempting to send a + * message. + * + * - If there are multiple writers, but the pipe is never closed by the + * writers, you can use the non-thread-safe version of trySendMessage(). + * Multiple simultaneous trySendMessage() calls will not interfere with each + * other. Since none of the writer threads call close, a call to close() + * cannot be running simultaneously with a write attempt. (With this model, + * the TNotificationPipe is never closed until it is destroyed. It is up to + * the caller to ensure the TNotificationPipe is not destroyed while write + * threads still have a pointer or reference to it.) + * + * In other circumstances (if one thread may call close while another thread is + * simultaneously trying to write), the thread-safe versions + * trySendMessageSync() and closeSync() must be used. + */ +class TNotificationPipe : public TDelayedDestruction, + private TEventHandler, + private TEventBase::LoopCallback { + public: + /** + * A callback interface for receiving notification of messages from the pipe. + */ + class Callback { + public: + virtual ~Callback() {} + + /** + * notificationMessage() will be invoked whenever a new + * message is available from the pipe. + */ + virtual void notificationMessage(const void *msg, uint32_t msgSize) = 0; + + /** + * notificationPipeError() will be invoked if an error occurs while reading + * from the pipe. Before notificationPipeError() is invoked, the read + * callback will automatically be uninstalled and the pipe will be closed. + */ + virtual void notificationPipeError(const std::exception& ex) = 0; + + /** + * notificationPipeClosed() is invoked in the read thread after the write + * end of the pipe is closed. + */ + virtual void notificationPipeClosed() = 0; + }; + + /** + * Helper function to create a new shared_ptr. + * + * This simply sets the correct destructor to call destroy() instead of + * directly deleting the TNotificationPipe. + */ + static boost::shared_ptr newPipe(TEventBase *base) { + return boost::shared_ptr(new TNotificationPipe(base), + Destructor()); + } + + /** + * Create a new TNotificationPipe. + * + * @param eventBase The TEventBase to use for receiving read notifications + * from this pipe. All read events will be processed in this + * TEventBase's thread. trySendMessage() may be called from any thread. + */ + TNotificationPipe(TEventBase *eventBase); + + /** + * Destroy this TNotificationPipe. + * + * This method may only be called from the read thread. + * + * This will automatically close the pipe if it is not already closed. + */ + virtual void destroy(); + + /** + * Close the pipe. + * + * This version of close() is not thread-safe. It should only be used if the + * caller is sure no other thread is attempting to write a message at the + * same time. + * + * Use closeSync() if other threads may be attempting to send a message + * simultaneously. The other threads must use also use the thread-safe + * trySendMessageSync() or trySendFrameSync() calls. + */ + void close(); + + /** + * A thread-safe version of close(). + */ + void closeSync(); + + /** + * Send a message over the pipe. + * + * trySendMessage() is best-effort. It will either immediately succeed to + * send the message, or it will fail immediately if the pipe reader is too + * busy and it's backlog of unread messages is too large. + * + * trySendMessage() also does not support arbitrarily large messages. + * It will also fail immediately if msgSize is larger than (PIPE_BUF - 4). + * + * If trySendMessage() succeeds, the message is guaranteed to be delivered to + * the pipe reader, except in the case where the pipe reader explicitly stops + * reading and destroys the pipe before processing all of its messages. + * + * On failure a TTransportException is thrown. The error code will be + * TTransportException::BAD_ARGS if the message is too large, + * TTransportException::TIMED_OUT if the message cannot be sent right now + * because the pipe is full, or TTransportException::NOT_OPEN if the pipe has + * already been closed. + * + * This method is thread safe with other simultaneous trySendMessage() calls, + * but not with close() calls. Use trySendMessageSync() and closeSync() if a + * close may occur simultaneously on another thread. + */ + void trySendMessage(const void *msg, uint32_t msgSize); + + /** + * A thread-safe version of trySendMessage(). + * + * This may be called simultaneously with closeSync(). + */ + void trySendMessageSync(const void *msg, uint32_t msgSize); + + /** + * Send a message over the pipe. + * + * This is identical to trySendMessage(), except that the caller must provide + * 4 bytes at the beginning of the message where we can write a frame length. + * This allows us to avoid copying the message into a new buffer. + * (trySendMessage() always has to make a copy of the message.) + * + * @param frame A pointer to the frame buffer. trySendFrame() will + * overwrite the first 4 bytes of this buffer. When the read callback + * receives the message, it will not see these first 4 bytes. + * @param frameSize The full size of the frame buffer. This must be at + * least 4 bytes long. The actual message size that will be sent is + * frameSize - 4. + */ + void trySendFrame(void *frame, uint32_t frameSize); + + /** + * A thread-safe version of trySendFrame(). + * + * This may be called simultaneously with closeSync(). + */ + void trySendFrameSync(void *frame, uint32_t frameSize); + + /** + * Get the number of messages which haven't been processed. + */ + int64_t getNumNotProcessed() const { + return numInputs_ - numOutputs_; + } + + /** + * Set the callback to receive read notifications from this pipe. + * + * This method must be invoked from the pipe's read thread. + * + * May throw TLibraryException on error. The callback will always be unset + * (NULL) after an error. + */ + void setReadCallback(Callback *callback); + + /** + * Mark the pipe read event handler as an "internal" event handler. + * + * This causes the notification pipe not to be counted when determining if + * the TEventBase has any more active events to wait on. This is intended to + * be used only be internal TEventBase code. This API is not guaranteed to + * remain stable or portable in the future. + * + * May throw TLibraryException if it fails to re-register its event handler + * with the correct flags. + */ + void setInternal(bool internal); + + /** + * Get the maximum number of messages that will be read on a single iteration + * of the event loop. + */ + uint32_t getMaxReadAtOnce() const { + return maxReadAtOnce_; + } + + /** + * Set the maximum number of messages to read each iteration of the event + * loop. + * + * If messages are being received faster than they can be processed, this + * helps limit the rate at which they will be read. This can be used to + * prevent the notification pipe reader from starving other users of the + * event loop. + */ + void setMaxReadAtOnce(uint32_t numMessages) { + maxReadAtOnce_ = numMessages; + } + + /** + * The maximum message size that can be sent over a TNotificationPipe. + * + * This restriction ensures that trySendMessage() can send all messages + * atomically. This is (PIPE_BUF - 4) bytes. (On Linux, this is 4092 + * bytes.) + */ + static const uint32_t kMaxMessageSize = PIPE_BUF - 4; + + /** + * The default maximum number of messages that will be read each time around + * the event loop. + * + * This value used for each TNotificationPipe can be changed using the + * setMaxReadAtOnce() method. + */ + static const uint32_t kDefaultMaxReadAtOnce = 10; + + private: + enum ReadAction { + kDoNothing, + kContinue, + kWaitForRead, + kRunInNextLoop, + }; + + // Forbidden copy constructor and assignment opererator + TNotificationPipe(TNotificationPipe const &); + TNotificationPipe& operator=(TNotificationPipe const &); + + // TEventHandler methods + virtual void handlerReady(uint16_t events) THRIFT_NOEXCEPT; + + // TEventBase::LoopCallback methods + virtual void runLoopCallback() THRIFT_NOEXCEPT; + + void initPipe(); + void registerPipeEvent(); + void readMessages(ReadAction action); + ReadAction performRead(); + ReadAction processReadData(uint32_t* messagesProcessed); + ReadAction handleError(const char* fmt, ...) + __attribute__((format(printf, 2, 3))); + void checkMessage(uint32_t msgSize); + void writeFrame(const void *frame, uint32_t frameSize); + + TEventBase *eventBase_; + Callback *readCallback_; + int readPipe_; + int writePipe_; + bool internal_; + uint32_t maxReadAtOnce_; + int64_t numInputs_; + int64_t numOutputs_; + + /** + * Mutex for guarding numInputs_ + */ + concurrency::Mutex numInputsMutex_; + + /** + * A mutex that guards writePipe_. + * + * This is used by closeSync(), trySendMessageSync(), and trySendFrameSync(), + * since trySendMessageSync() and trySendFrameSync() read writePipe_ + * and closeSync() resets it to -1. + */ + concurrency::NoStarveReadWriteMutex writePipeMutex_; + + /** + * A pointer to the end of valid read data in the read buffer. + */ + uint8_t *readPtr_; + /** + * An internal read buffer + * + * This is large enough to contain the maximum possible message plus the + * mssage length. + */ + uint8_t readBuffer_[kMaxMessageSize + 4]; +}; + +}}} // apache::thrift::async + +#endif // THRIFT_ASYNC_TNOTIFICATIONPIPE_H diff --git a/thrift/lib/cpp/async/TNotificationQueue.h b/thrift/lib/cpp/async/TNotificationQueue.h new file mode 100644 index 0000000000..3817a85fc1 --- /dev/null +++ b/thrift/lib/cpp/async/TNotificationQueue.h @@ -0,0 +1,592 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_ASYNC_TNOTIFICATIONQUEUE_H_ +#define THRIFT_ASYNC_TNOTIFICATIONQUEUE_H_ 1 + +#include "thrift/lib/cpp/Thrift.h" +#include "thrift/lib/cpp/async/TEventBase.h" +#include "thrift/lib/cpp/async/TEventHandler.h" + +#include "external/google_base/spinlock.h" +#include "external/glog/logging.h" +#include +#include "folly/eventfd.h" + +namespace apache { namespace thrift { namespace async { + +/** + * An exception class to be thrown when a TNotificationQueue is full. + */ +class TQueueFullException : public TLibraryException { + public: + TQueueFullException() : + TLibraryException("unable to add message to TNotificationQueue: " + "queue is full") {} +}; + +/** + * A producer-consumer queue for passing messages between TEventBase threads. + * + * Messages can be added to the queue from any thread. Multiple consumers may + * listen to the queue from multiple TEventBase threads. + * + * A TNotificationQueue may not be destroyed while there are still consumers + * registered to receive events from the queue. It is the user's + * responsibility to ensure that all consumers are unregistered before the + * queue is destroyed. + * + * MessageT should be MoveConstructible (i.e., must support either a move + * constructor or a copy constructor, or both). Ideally it's move constructor + * (or copy constructor if no move constructor is provided) should never throw + * exceptions. If the constructor may throw, the consumers could end up + * spinning trying to move a message off the queue and failing, and then + * retrying. + */ +template +class TNotificationQueue { + public: + /** + * A callback interface for consuming messages from the queue as they arrive. + */ + class Consumer : private TEventHandler { + public: + enum : uint16_t { kDefaultMaxReadAtOnce = 10 }; + + Consumer() + : queue_(NULL), + destroyedFlagPtr_(NULL), + maxReadAtOnce_(kDefaultMaxReadAtOnce) {} + + virtual ~Consumer(); + + /** + * messageAvailable() will be invoked whenever a new + * message is available from the pipe. + */ + virtual void messageAvailable(MessageT&& message) = 0; + + /** + * Begin consuming messages from the specified queue. + * + * messageAvailable() will be called whenever a message is available. This + * consumer will continue to consume messages until stopConsuming() is + * called. + * + * A Consumer may only consume messages from a single TNotificationQueue at + * a time. startConsuming() should not be called if this consumer is + * already consuming. + */ + void startConsuming(TEventBase* eventBase, TNotificationQueue* queue) { + init(eventBase, queue); + registerHandler(READ | PERSIST); + } + + /** + * Same as above but registers this event handler as internal so that it + * doesn't count towards the pending reader count for the IOLoop. + */ + void startConsumingInternal( + TEventBase* eventBase, TNotificationQueue* queue) { + init(eventBase, queue); + registerInternalHandler(READ | PERSIST); + } + + /** + * Stop consuming messages. + * + * startConsuming() may be called again to resume consumption of messages + * at a later point in time. + */ + void stopConsuming(); + + /** + * Get the TNotificationQueue that this consumer is currently consuming + * messages from. Returns NULL if the consumer is not currently consuming + * events from any queue. + */ + TNotificationQueue* getCurrentQueue() const { + return queue_; + } + + /** + * Set a limit on how many messages this consumer will read each iteration + * around the event loop. + * + * This helps rate-limit how much work the Consumer will do each event loop + * iteration, to prevent it from starving other event handlers. + * + * A limit of 0 means no limit will be enforced. If unset, the limit + * defaults to kDefaultMaxReadAtOnce (defined to 10 above). + */ + void setMaxReadAtOnce(uint32_t maxAtOnce) { + maxReadAtOnce_ = maxAtOnce; + } + uint32_t getMaxReadAtOnce() const { + return maxReadAtOnce_; + } + + private: + void init(TEventBase* eventBase, TNotificationQueue* queue); + + virtual void handlerReady(uint16_t events) THRIFT_NOEXCEPT; + + TNotificationQueue* queue_; + bool* destroyedFlagPtr_; + uint32_t maxReadAtOnce_; + }; + + enum class FdType { + EVENTFD, + PIPE + }; + + /** + * Create a new TNotificationQueue. + * + * If the maxSize parameter is specified, this sets the maximum queue size + * that will be enforced by tryPutMessage(). (This size is advisory, and may + * be exceeded if producers explicitly use putMessage() instead of + * tryPutMessage().) + * + * The fdType parameter determines the type of file descriptor used + * internally to signal message availability. The default (eventfd) is + * preferable for performance and because it won't fail when the queue gets + * too long. It is not available on on older and non-linux kernels, however. + * In this case the code will fall back to using a pipe, the parameter is + * mostly for testing purposes. + */ + explicit TNotificationQueue(uint32_t maxSize = 0, + FdType fdType = FdType::EVENTFD) + : spinlock_(), + eventfd_(-1), + pipeFds_({-1, -1}), + advisoryMaxQueueSize_(maxSize), + queue_() { + if (fdType == FdType::EVENTFD) { + eventfd_ = folly::eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK | EFD_SEMAPHORE); + if (eventfd_ == -1) { + if (errno == ENOSYS || errno == EINVAL) { + // eventfd not availalble + T_ERROR("failed to create eventfd for TNotificationQueue: %d, " + "falling back to pipe mode", errno); + fdType = FdType::PIPE; + } else { + // some other error + throw TLibraryException("Failed to create eventfd for " + "TNotificationQueue", errno); + } + } + } + if (fdType == FdType::PIPE) { + if (pipe(pipeFds_)) { + throw TLibraryException("Failed to create pipe for TNotificationQueue", + errno); + } + try { + // put both ends of the pipe into non-blocking mode + if (fcntl(pipeFds_[0], F_SETFL, O_RDONLY | O_NONBLOCK) != 0) { + throw TLibraryException("failed to put TNotificationQueue pipe read " + "endpoint into non-blocking mode", errno); + } + if (fcntl(pipeFds_[1], F_SETFL, O_WRONLY | O_NONBLOCK) != 0) { + throw TLibraryException("failed to put TNotificationQueue pipe write " + "endpoint into non-blocking mode", errno); + } + } catch (...) { + ::close(pipeFds_[0]); + ::close(pipeFds_[1]); + throw; + } + } + } + + ~TNotificationQueue() { + if (eventfd_ >= 0) { + ::close(eventfd_); + eventfd_ = -1; + } + if (pipeFds_[0] >= 0) { + ::close(pipeFds_[0]); + pipeFds_[0] = -1; + } + if (pipeFds_[1] >= 0) { + ::close(pipeFds_[1]); + pipeFds_[1] = -1; + } + } + + /** + * Set the advisory maximum queue size. + * + * This maximum queue size affects calls to tryPutMessage(). Message + * producers can still use the putMessage() call to unconditionally put a + * message on the queue, ignoring the configured maximum queue size. This + * can cause the queue size to exceed the configured maximum. + */ + void setMaxQueueSize(uint32_t max) { + advisoryMaxQueueSize_ = max; + } + + /** + * Attempt to put a message on the queue if the queue is not already full. + * + * If the queue is full, a TQueueFullException will be thrown. The + * setMaxQueueSize() function controls the maximum queue size. + * + * This method may contend briefly on a spinlock if many threads are + * concurrently accessing the queue, but for all intents and purposes it will + * immediately place the message on the queue and return. + * + * tryPutMessage() may throw std::bad_alloc if memory allocation fails, and + * may throw any other exception thrown by the MessageT move/copy + * constructor. + */ + void tryPutMessage(MessageT&& message) { + putMessageImpl(std::move(message), advisoryMaxQueueSize_); + } + void tryPutMessage(const MessageT& message) { + putMessageImpl(message, advisoryMaxQueueSize_); + } + + /** + * Unconditionally put a message on the queue. + * + * This method is like tryPutMessage(), but ignores the maximum queue size + * and always puts the message on the queue, even if the maximum queue size + * would be exceeded. + * + * putMessage() may throw std::bad_alloc if memory allocation fails, and may + * throw any other exception thrown by the MessageT move/copy constructor. + */ + void putMessage(MessageT&& message) { + putMessageImpl(std::move(message), 0); + } + void putMessage(const MessageT& message) { + putMessageImpl(message, 0); + } + + /** + * Put several messages on the queue. + */ + template + void putMessages(InputIteratorT first, InputIteratorT last) { + typedef typename std::iterator_traits::iterator_category + IterCategory; + putMessagesImpl(first, last, IterCategory()); + } + + /** + * Try to immediately pull a message off of the queue, without blocking. + * + * If a message is immediately available, the result parameter will be + * updated to contain the message contents and true will be returned. + * + * If no message is available, false will be returned and result will be left + * unmodified. + */ + bool tryConsume(MessageT& result) { + if (!tryConsumeEvent()) { + return false; + } + + try { + facebook::SpinLockHolder guard(&spinlock_); + result = std::move(queue_.front()); + queue_.pop_front(); + } catch (...) { + // Handle an exception if the assignment operator happens to throw. + // We consumed an event but weren't able to pop the message off the + // queue. Signal the event again since the message is still in the + // queue. + signalEvent(1); + throw; + } + + return true; + } + + private: + // Forbidden copy constructor and assignment operator + TNotificationQueue(TNotificationQueue const &) = delete; + TNotificationQueue& operator=(TNotificationQueue const &) = delete; + + inline void checkQueueSize(size_t maxSize) const { + assert(spinlock_.IsHeld()); + if (maxSize > 0 && queue_.size() >= maxSize) { + throw TQueueFullException(); + } + } + + inline void signalEvent(uint64_t numAdded = 1) const { + static const uint8_t kPipeMessage[] = { + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 + }; + + ssize_t bytes_written = 0; + ssize_t bytes_expected = 0; + if (eventfd_ >= 0) { + bytes_expected = static_cast(sizeof(numAdded)); + bytes_written = ::write(eventfd_, &numAdded, sizeof(numAdded)); + } else { + // pipe semantics, add one message for each numAdded + bytes_expected = numAdded; + do { + size_t messageSize = std::min(numAdded, sizeof(kPipeMessage)); + ssize_t rc = ::write(pipeFds_[1], kPipeMessage, messageSize); + if (rc < 0) { + // TODO: if the pipe is full, write will fail with EAGAIN. + // See task #1044651 for how this could be handled + break; + } + numAdded -= rc; + bytes_written += rc; + } while (numAdded > 0); + } + if (bytes_written != bytes_expected) { + throw TLibraryException("failed to signal TNotificationQueue after " + "write", errno); + } + } + + bool tryConsumeEvent() { + uint64_t value = 0; + ssize_t rc = -1; + if (eventfd_ >= 0) { + rc = ::read(eventfd_, &value, sizeof(value)); + } else { + uint8_t value8; + rc = ::read(pipeFds_[0], &value8, sizeof(value8)); + value = value8; + } + if (rc < 0) { + // EAGAIN should pretty much be the only error we can ever get. + // This means someone else already processed the only available message. + assert(errno == EAGAIN); + return false; + } + assert(value == 1); + return true; + } + + void putMessageImpl(MessageT&& message, size_t maxSize) { + { + facebook::SpinLockHolder guard(&spinlock_); + checkQueueSize(maxSize); + queue_.push_back(std::move(message)); + } + signalEvent(); + } + + void putMessageImpl(const MessageT& message, size_t maxSize) { + { + facebook::SpinLockHolder guard(&spinlock_); + checkQueueSize(maxSize); + queue_.push_back(message); + } + signalEvent(); + } + + template + void putMessagesImpl(InputIteratorT first, InputIteratorT last, + std::input_iterator_tag) { + uint64_t numAdded = 0; + { + facebook::SpinLockHolder guard(&spinlock_); + while (first != last) { + queue_.push_back(*first); + ++first; + ++numAdded; + } + } + signalEvent(numAdded); + } + + template + void putMessagesImpl(InputIteratorT first, InputIteratorT last, + std::forward_iterator_tag) { + uint64_t numAdded = std::distance(first, last); + { + facebook::SpinLockHolder guard(&spinlock_); + queue_.insert(queue_.end(), first, last); + } + signalEvent(numAdded); + } + + facebook::SpinLock spinlock_; + int eventfd_; + int pipeFds_[2]; // to fallback to on older/non-linux systems + uint32_t advisoryMaxQueueSize_; + std::deque queue_; +}; + +template +TNotificationQueue::Consumer::~Consumer() { + // If we are in the middle of a call to handlerReady(), destroyedFlagPtr_ + // will be non-NULL. Mark the value that it points to, so that + // handlerReady() will know the callback is destroyed, and that it cannot + // access any member variables anymore. + if (destroyedFlagPtr_) { + *destroyedFlagPtr_ = true; + } +} + +template +void TNotificationQueue::Consumer::handlerReady(uint16_t events) + THRIFT_NOEXCEPT { + uint32_t numProcessed = 0; + while (true) { + // Try to decrement the eventfd. + // + // We decrement the eventfd before checking the queue, and only pop a + // message off the queue if we read from the eventfd. + // + // Reading the eventfd first allows us to not have to hold the spinlock + // while accessing the eventfd. If we popped from the queue first, we + // would have to hold the lock while reading from or writing to the + // eventfd. (Multiple consumers may be woken up from a single eventfd + // notification. If we popped from the queue first, we could end up + // popping a message from the queue before the eventfd has been notified by + // the producer, unless the consumer and producer both held the spinlock + // around the entire operation.) + if (!queue_->tryConsumeEvent()) { + // no message available right now + return; + } + + // Now pop the message off of the queue. + // We successfully consumed the eventfd notification. + // There should be a message available for us to consume. + // + // We have to manually acquire and release the spinlock here, rather than + // using SpinLockHolder since the MessageT has to be constructed while + // holding the spinlock and available after we release it. SpinLockHolder + // unfortunately doesn't provide a release() method. (We can't construct + // MessageT first since we have no guarantee that MessageT has a default + // constructor. + queue_->spinlock_.Lock(); + bool locked = true; + + try { + // The eventfd is incremented once for every message, and only + // decremented when a message is popped off. There should always be a + // message here to read. + CHECK(!queue_->queue_.empty()); + + // Pull a message off the queue. + MessageT msg(std::move(queue_->queue_.front())); + queue_->queue_.pop_front(); + + // Check to see if the queue is empty now. + // We use this as an optimization to see if we should bother trying to + // loop again and read another message after invoking this callback. + bool wasEmpty = queue_->queue_.empty(); + + // Now unlock the spinlock before we invoke the callback. + queue_->spinlock_.Unlock(); + locked = false; + + // Call the callback + bool callbackDestroyed = false; + CHECK(destroyedFlagPtr_ == NULL); + destroyedFlagPtr_ = &callbackDestroyed; + messageAvailable(std::move(msg)); + + // If the callback was destroyed before it returned, we are done + if (callbackDestroyed) { + return; + } + destroyedFlagPtr_ = NULL; + + // If the callback is no longer installed, we are done. + if (queue_ == NULL) { + return; + } + + // If we have hit maxReadAtOnce_, we are done. + ++numProcessed; + if (maxReadAtOnce_ > 0 && numProcessed >= maxReadAtOnce_) { + return; + } + + // If the queue was empty before we invoked the callback, it's probable + // that it is still empty now. Just go ahead and return, rather than + // looping again and trying to re-read from the eventfd. (If a new + // message had in fact arrived while we were invoking the callback, we + // will simply be woken up the next time around the event loop and will + // process the message then.) + if (wasEmpty) { + return; + } + } catch (const std::exception& ex) { + // This catch block is really just to handle the case where the MessageT + // constructor throws. The messageAvailable() callback itself is + // declared as noexcept and should never throw. + // + // If the MessageT constructor does throw we try to handle it as best as + // we can, but we can't work miracles. We will just ignore the error for + // now and return. The next time around the event loop we will end up + // trying to read the message again. If MessageT continues to throw we + // will never make forward progress and will keep trying each time around + // the event loop. + if (locked) { + // Unlock the spinlock. + queue_->spinlock_.Unlock(); + + // Push a notification back on the eventfd since we didn't actually + // read the message off of the queue. + queue_->signalEvent(1); + } + + return; + } + } +} + +template +void TNotificationQueue::Consumer::init( + TEventBase* eventBase, + TNotificationQueue* queue) { + assert(eventBase->isInEventBaseThread()); + assert(queue_ == NULL); + assert(!isHandlerRegistered()); + + queue_ = queue; + if (queue_->eventfd_ >= 0) { + initHandler(eventBase, queue_->eventfd_); + } else { + initHandler(eventBase, queue_->pipeFds_[0]); + } +} + +template +void TNotificationQueue::Consumer::stopConsuming() { + if (queue_ == NULL) { + assert(!isHandlerRegistered()); + return; + } + + assert(isHandlerRegistered()); + unregisterHandler(); + detachEventBase(); + queue_ = NULL; +} + +}}} // apache::thrift::async + +#endif // THRIFT_ASYNC_TNOTIFICATIONQUEUE_H_ diff --git a/thrift/lib/cpp/async/TQueuingAsyncProcessor.h b/thrift/lib/cpp/async/TQueuingAsyncProcessor.h new file mode 100644 index 0000000000..5f2c649823 --- /dev/null +++ b/thrift/lib/cpp/async/TQueuingAsyncProcessor.h @@ -0,0 +1,70 @@ +#ifndef _THRIFT_TQUEUINGASYNCPROCESSOR_H_ +#define _THRIFT_TQUEUINGASYNCPROCESSOR_H_ 1 + +#include +#include +#include "thrift/lib/cpp/TProcessor.h" +#include "thrift/lib/cpp/async/TAsyncProcessor.h" +#include "thrift/lib/cpp/async/TEventTask.h" +#include "thrift/lib/cpp/concurrency/Exception.h" + +namespace apache { namespace thrift { namespace async { + +/** + * Adapter to allow a TProcessor to be used as a TAsyncProcessor. + * + * Note: this is not intended for use outside of TEventConnection since the + * callback mechanism used in TEventTask will invoke handleAsyncTaskComplete() + * regardless of what is passed in as the cob. + * + * Uses a per-server task queue for all calls. + */ +class TQueuingAsyncProcessor : public TAsyncProcessor { + public: + TQueuingAsyncProcessor( + boost::shared_ptr processor, + boost::shared_ptr threadManager, + int64_t taskExpireTime, + TEventConnection* connection) + : processor_(processor) + , threadManager_(threadManager) + , taskExpireTime_(taskExpireTime) + , connection_(connection) + {} + + virtual void process( + std::tr1::function cob, + boost::shared_ptr in, + boost::shared_ptr out, + TConnectionContext* context) { + + boost::shared_ptr task = + boost::shared_ptr( + new TEventTask(connection_)); + + try { + threadManager_->add(task, 0LL, taskExpireTime_); + } catch (apache::thrift::concurrency::IllegalStateException & ise) { + T_ERROR("IllegalStateException: TQueuingAsyncProcessor::process() %s", + ise.what()); + // no task will be making a callback + return cob(false); + } + } + + private: + boost::shared_ptr processor_; + + /// For processing via thread pool + boost::shared_ptr threadManager_; + + /// Time in milliseconds before an unperformed task expires (0 == infinite). + int64_t taskExpireTime_; + + /// The worker that started us + TEventConnection* connection_; +}; + +}}} // apache::thrift::async + +#endif // #ifndef _THRIFT_TQUEUINGASYNCPROCESSOR_H_ diff --git a/thrift/lib/cpp/async/TStreamAsyncChannel.h b/thrift/lib/cpp/async/TStreamAsyncChannel.h new file mode 100644 index 0000000000..3f9985ce12 --- /dev/null +++ b/thrift/lib/cpp/async/TStreamAsyncChannel.h @@ -0,0 +1,441 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_ASYNC_TSTREAMASYNCCHANNEL_H_ +#define THRIFT_ASYNC_TSTREAMASYNCCHANNEL_H_ 1 + +#include "thrift/lib/cpp/async/TAsyncEventChannel.h" +#include "thrift/lib/cpp/async/TAsyncTransport.h" +#include "thrift/lib/cpp/async/TAsyncTimeout.h" + +#include + +namespace apache { namespace thrift { namespace async { + +class TAsyncTransport; + +template +class TAsyncChannelWriteRequestBase { + public: + typedef std::tr1::function VoidCallback; + + TAsyncChannelWriteRequestBase(const VoidCallback& callback, + const VoidCallback& errorCallback, + transport::TMemoryBuffer* message) + : buffer_(message), + next_(NULL), + callback_(callback), + errorCallback_(errorCallback) { + + // The WriteRequest's buffer consumes all of the data in message, + // so we don't attempt to resend data; yet is also an observer + // which prevents consumed data from being overwritten while it's pending + // for the transport + uint32_t len = message->available_read(); + message->borrow(NULL, &len); + message->consume(len); + } + + virtual ~TAsyncChannelWriteRequestBase() { + } + + void setNext(Subclass_* next) { + assert(next_ == NULL); + next_ = next; + } + + Subclass_* getNext() const { + return next_; + } + + protected: + apache::thrift::transport::TMemoryBuffer buffer_; + + void invokeCallback() { + // unlink the buffer before invoking the callback, since we are + // now done with it. Not strictly required but faster. + buffer_.unlink(); + callback_(); + } + + void invokeErrorCallback() { + // unlink the buffer before invoking the callback, since we are + // now done with it. Not strictly required but faster. + buffer_.unlink(); + errorCallback_(); + } + + private: + TAsyncChannelWriteRequestBase(); + + Subclass_* next_; + + VoidCallback callback_; + VoidCallback errorCallback_; +}; + + +/** + * TStreamAsyncChannel is a helper class for channel implementations that use + * TAsyncTransport underneath. + * + * TStreamAsyncChannel provides the basic functionality for implementing a + * message-based asynchronous channel on top of a streaming TAsyncTransport. + * + * It requires two template arguments that control how the stream is broken up + * into messagess: + * + * WriteRequest_: + * + * This template parameter controls how messages are written to the + * underlying stream. It must implement the following methods: + * + * - WriteRequest_(const VoidCallback& callback, + * const VoidCallback& errorCallback, + * transport::TMemoryBuffer* message); + * + * The WriteRequest_ constructor accepts the success and error callbacks, + * and the TMemoryBuffer containing the data to send. The WriteRequest_ + * may consume data from the message, but does not own the TMemoryBuffer + * (i.e., it should not delete the TMemoryBuffer.) + * + * - void setNext(WriteRequest_* next); + * - WriteRequest_* getNext() const; + * + * These two methods support chaining together a list of WriteRequest_ + * objects. This is used when multiple write requests are pending on the + * channel. + * + * - void write(TAsyncTransport* transport, + * TAsyncTransport::WriteCallback* callback) THRIFT_NOEXCEPT; + * + * This method will be called to schedule the write. The WriteRequest_ + * should invoke the transport's write() or writev() method with the data + * to send, and set the specified callback as the transport callback. + * + * Note that this API requires the WriteRequest_ to write the entire + * message with a single write() or writev() call. This allows the code + * to let the TAsyncTransport perform the write queuing when multiple + * messages are pending. (If needed we could rewrite this API in the + * future to relax this restriction.) + * + * - void writeSuccess() THRIFT_NOEXCEPT; + * - void writeError(size_t bytesWritten, + * const TTransportException& ex) THRIFT_NOEXCEPT; + * + * Either writeSuccess() or writeError() will be invoked once the message + * write has completed. + * + * ReadState_: + * + * This template parameter controls how the incoming stream is broken up into + * individual messages. It must implement the following methods: + * + * - ReadState_(); + * + * The ReadState_ constructor takes no arguments. + * + * - void setCallbackBuffer(transport::TMemoryBuffer* buffer); + * + * When a new read is started, setCallbackBuffer() is called to set the + * buffer into which the message data should be placed. + * + * - void unsetCallbackBuffer(); + * + * unsetCallbackBuffer() is called to clear the callback buffer when after + * a full message has been read. + * + * - bool hasReadAheadData(); + * + * Some ReadState_ implementations may perform read-ahead, and read past + * the end of the message when reading from the underlying transport. + * hasReadAheadData() is called when a new read starts, to see if the + * ReadState_ has pending data for a new message that has already been read + * from the transport. + * + * If hasReadAheadData() returns true, readDataAvailable(0) will be called + * immediately, rather than waiting for new data from the transport. + * + * - bool hasPartialMessage(); + * + * When EOF is read from the underlying transport, hasPartialMessage() is + * called to see if the EOF should be treated as an error or a normal + * close. (It is an error if hasPartialMessage() returns true.) + * + * - void getReadBuffer(void** bufReturn, size_t* lenReturn); + * + * When data becomes available on the underlying transport, getReadBuffer() + * is called to get the buffer where the data should be placed. + * + * - bool readDataAvailable(size_t len); + * + * readDataAvailable() is called when new data has been read from the + * underlying transport. The data will have been placed in the buffer + * returned by the previous getReadBuffer() call. + */ +template +class TStreamAsyncChannel : public TAsyncEventChannel, + protected TAsyncTransport::ReadCallback, + protected TAsyncTransport::WriteCallback, + protected TAsyncTimeout { + public: + TStreamAsyncChannel(const boost::shared_ptr& transport); + + /** + * Helper function to create a shared_ptr. + * + * This passes in the correct destructor object, since TStreamAsyncChannel's + * destructor is protected and cannot be invoked directly. + */ + static boost::shared_ptr newChannel( + const boost::shared_ptr& transport) { + return boost::shared_ptr( + new TStreamAsyncChannel(transport), Destructor()); + } + + /** + * Destroy the channel. + * + * destroy() must be called to destroy the channel. The normal destructor + * is private, and should not be invoked directly. This prevents callers + * from deleting a TStreamAsyncChannel while it is invoking a callback. + */ + virtual void destroy(); + + // Methods inherited from TAsyncEventChannel + virtual bool readable() const; + virtual bool good() const; + virtual bool error() const; + virtual bool timedOut() const; + + /** + * Send a message to the channel; note that "errorCob" will be called + * after a partial write as well as other errors. We will call "errorCob" + * immediately (before return) if the channel is unusable for some reason, + * and "cob" immediately if we're able to perform the write without delay. + */ + virtual void sendMessage(const VoidCallback& cob, + const VoidCallback& errorCob, + apache::thrift::transport::TMemoryBuffer* message); + + /** + * Receive a message from the channel; note that "errorCob" will be called + * after a partial read as well as other errors. We will call "errorCob" + * immediately (before return) if the channel is unusable for some reason, + * and "cob" immediately if we're able to perform the read without delay. + * + * Note that an EOF is considered normal, so "cob" will be called although + * "good()" will be false. + */ + virtual void recvMessage(const VoidCallback& cob, + const VoidCallback& errorCob, + apache::thrift::transport::TMemoryBuffer* message); + + /** + * Send a message to the channel and receive the response; note that the + * "errorCob: will be called after a write error and no receive is attempted. + * Also, a partial write or read will result in errorCob being called. + * We call "errorCob" before return if the channel is unusable for some + * reason. It is conceivable that "cob" will be called before return if data + * is somehow available in the channel when a read is first attempted. + */ + virtual void sendAndRecvMessage( + const VoidCallback& cob, + const VoidCallback& errorCob, + transport::TMemoryBuffer* sendBuf, + transport::TMemoryBuffer* recvBuf); + + /** + * Close this channel. + * + * This gracefully closes the channel, waiting for all pending send + * requests to complete before actually closing the underlying transport. + * + * If a recvMessage() call is pending, it will be immediately failed. + */ + void close(); + + /** + * Close the channel immediately. + * + * This closes the channel immediately, dropping any outstanding messages + * waiting to be sent. + * + * If a recvMessage() call is pending, it will be immediately failed. + */ + void closeNow(); + + /** + * Attach the channel to a TEventBase. + * + * This may only be called if the channel is not currently attached to a + * TEventBase (by an earlier call to detachEventBase()). + * + * This method must be invoked in the TEventBase's thread. + */ + void attachEventBase(TEventBase* eventBase); + + /** + * Detach the channel from its TEventBase. + * + * This may only be called when the channel is idle and has no reads or + * writes pending. Once detached, the channel may not be used again until it + * is re-attached to a TEventBase by calling attachEventBase(). + * + * This method must be called from the current TEventBase's thread. + */ + void detachEventBase(); + + /** + * Get the TEventBase used by this channel. + */ + TEventBase* getEventBase() const; + + /** + * Set the timeout for receiving messages. + * + * When set to a non-zero value, the entire message must be received within + * the specified number of milliseconds, or the receive will fail and the + * channel will be closed. + * + * If setRecvTimeout() is invoked while a recvMessage() call is currently in + * progress, the timeout will be restarted using the new value. + */ + void setRecvTimeout(uint32_t milliseconds); + + /** + * Get the receive timeout. + * + * @return Returns the current receive timeout, in milliseconds. A return + * value of 0 indicates that no timeout is set. + */ + uint32_t getRecvTimeout() const { + return recvTimeout_; + } + + /** + * Get the TAsyncTransport used by this channel. + */ + virtual boost::shared_ptr getTransport() { + return transport_; + } + + /** + * Determine if this channel is idle (i.e., has no outstanding reads or + * writes). + */ + bool isIdle() const { + return (writeReqHead_ == NULL) && (!readCallback_) && + !transport_->connecting(); + } + + protected: + struct ReadQueueEntry { + ReadQueueEntry(const VoidCallback& cob, + const VoidCallback& errorCob, + apache::thrift::transport::TMemoryBuffer* message) { + readCallback = cob; + readErrorCallback = errorCob; + readBuffer = message; + } + VoidCallback readCallback; + VoidCallback readErrorCallback; + transport::TMemoryBuffer *readBuffer; + int64_t startTime; + }; + + /** + * Protected destructor. + * + * Users of TStreamAsyncChannel must never delete it directly. Instead, + * invoke destroy(). + */ + virtual ~TStreamAsyncChannel() {} + + // callbacks from TAsyncTransport + void getReadBuffer(void** bufReturn, size_t* lenReturn); + void readDataAvailable(size_t len) THRIFT_NOEXCEPT; + void readEOF() THRIFT_NOEXCEPT; + void readError(const transport::TTransportException& ex) THRIFT_NOEXCEPT; + + void writeSuccess() THRIFT_NOEXCEPT; + void writeError(size_t bytesWritten, + const transport::TTransportException& ex) THRIFT_NOEXCEPT; + + // callback from TAsyncTimeout + void timeoutExpired() THRIFT_NOEXCEPT; + + bool invokeReadDataAvailable(size_t len) THRIFT_NOEXCEPT; + void processReadEOF() THRIFT_NOEXCEPT; + void invokeReadCallback(VoidCallback cb, + char const* callbackName) THRIFT_NOEXCEPT; + + void pushWriteRequest(WriteRequest_* req) { + if (writeReqTail_ == NULL) { + assert(writeReqHead_ == NULL); + writeReqHead_ = req; + } else { + writeReqTail_->setNext(req); + } + writeReqTail_ = req; + } + + WriteRequest_* popWriteRequest() { + assert(writeReqHead_ != NULL); + + WriteRequest_* req = writeReqHead_; + writeReqHead_ = req->getNext(); + if (writeReqHead_ == NULL) { + assert(writeReqTail_ == req); + writeReqTail_ = NULL; + } + return req; + } + + void failAllReads(); + + boost::shared_ptr transport_; + WriteRequest_* writeReqHead_; + WriteRequest_* writeReqTail_; + + ReadState_ readState_; + VoidCallback readCallback_; + VoidCallback readErrorCallback_; + std::list readCallbackQ_; + + uint32_t recvTimeout_; + // true if a timeout has occurred + bool timedOut_; + + private: + // Forbidden copy constructor and assignment opererator + TStreamAsyncChannel(TStreamAsyncChannel const &); + TStreamAsyncChannel& operator=(TStreamAsyncChannel const &); +}; + +class TStreamAsyncChannelFactory { + public: + virtual ~TStreamAsyncChannelFactory() {} + + virtual boost::shared_ptr newChannel( + const boost::shared_ptr& transport) = 0; +}; + +}}} // apache::thrift::async + +#endif // THRIFT_ASYNC_TSTREAMASYNCCHANNEL_H_ diff --git a/thrift/lib/cpp/async/TStreamAsyncChannel.tcc b/thrift/lib/cpp/async/TStreamAsyncChannel.tcc new file mode 100644 index 0000000000..f149e0f2ea --- /dev/null +++ b/thrift/lib/cpp/async/TStreamAsyncChannel.tcc @@ -0,0 +1,488 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_ASYNC_TSTREAMASYNCCHANNEL_TCC_ +#define THRIFT_ASYNC_TSTREAMASYNCCHANNEL_TCC_ 1 + +#include "thrift/lib/cpp/async/TStreamAsyncChannel.h" +#include "thrift/lib/cpp/transport/TSocketAddress.h" + +namespace apache { namespace thrift { namespace async { + +template +TStreamAsyncChannel::TStreamAsyncChannel( + const boost::shared_ptr& transport) + : TAsyncTimeout(transport->getEventBase()) + , transport_(transport) + , writeReqHead_(NULL) + , writeReqTail_(NULL) + , readState_() + , readCallback_() + , readErrorCallback_() + , recvTimeout_(0) + , timedOut_(false) { +} + +template +void TStreamAsyncChannel::destroy() { + // When destroy is called, close the channel immediately + closeNow(); + + // Then call TDelayedDestruction::destroy() to take care of + // whether or not we need immediate or delayed destruction + TDelayedDestruction::destroy(); +} + +template +bool TStreamAsyncChannel::readable() const { + return transport_->readable(); +} + +template +bool TStreamAsyncChannel::good() const { + return transport_->good(); +} + +template +bool TStreamAsyncChannel::error() const { + return (timedOut_ || transport_->error()); +} + +template +bool TStreamAsyncChannel::timedOut() const { + return timedOut_; +} + +template +void TStreamAsyncChannel::sendMessage( + const VoidCallback& cob, + const VoidCallback& errorCob, + transport::TMemoryBuffer* message) { + assert(message); + DestructorGuard dg(this); + + if (!good()) { + T_DEBUG_T("sendMessage: transport went bad, bailing out."); + return errorCob(); + } + + if (message->available_read() == 0) { + T_ERROR("sendMessage: buffer is empty"); + return errorCob(); + } + + WriteRequest_* writeReq; + try { + writeReq = new WriteRequest_(cob, errorCob, message, this); + } catch (const std::exception& ex) { + T_ERROR("sendMessage: failed to allocate new write request object"); + errorCob(); + return; + } + + pushWriteRequest(writeReq); + writeReq->write(transport_.get(), this); +} + +template +void TStreamAsyncChannel::recvMessage( + const VoidCallback& cob, + const VoidCallback& errorCob, + transport::TMemoryBuffer* message) { + assert(message); + DestructorGuard dg(this); + + if (!good()) { + T_DEBUG_T("recvMessage: transport went bad, bailing out."); + return errorCob(); + } + + if (message->available_read() != 0) { + T_ERROR("recvMessage: buffer is not empty."); + return errorCob(); + } + + if (readCallbackQ_.empty() && readCallback_ == NULL) { + readState_.setCallbackBuffer(message); + readCallback_ = cob; + readErrorCallback_ = errorCob; + } else { + readCallbackQ_.push_back(ReadQueueEntry(cob, errorCob, message)); + return; + } + + // Some ReadState implementations perform read-ahead, + // and they may already have data waiting to be processed. + // If so, we need to invoke readDataAvailable() immediately, rather than + // waiting for new data from the transport. + if (readState_.hasReadAheadData()) { + if (invokeReadDataAvailable(0)) { + // We already invoked the callback + return; + } + } + + // start the read timeout + if (recvTimeout_ > 0) { + scheduleTimeout(recvTimeout_); + } + + // start reading from the transport + // Note that setReadCallback() may invoke our read callback methods + // immediately, so the read may complete before setReadCallback() returns. + transport_->setReadCallback(this); +} + +template +void TStreamAsyncChannel::sendAndRecvMessage( + const VoidCallback& cob, + const VoidCallback& errorCob, + transport::TMemoryBuffer* sendBuf, + transport::TMemoryBuffer* recvBuf) { + // TODO: it would be better to perform this bind once, rather than + // each time sendAndRecvMessage() is called. + const VoidCallback& send_done = + std::tr1::bind(&TStreamAsyncChannel::recvMessage, this, cob, errorCob, + recvBuf); + + return sendMessage(send_done, errorCob, sendBuf); +} + +template +void TStreamAsyncChannel::close() { + DestructorGuard dg(this); // transport::close can invoke callbacks + + transport_->setReadCallback(NULL); + transport_->close(); + + if (readCallback_) { + processReadEOF(); + } + + // no need to free the write-queue here. The underlying transport will + // drain the writes first +} + +template +void TStreamAsyncChannel::closeNow() { + DestructorGuard dg(this); // transport::closeNow can invoke callbacks + + transport_->setReadCallback(NULL); + transport_->closeNow(); + + if (readCallback_) { + processReadEOF(); + } + + // no need to free the write-queue here. The underlying transport will + // fail pending writes first +} + +template +void TStreamAsyncChannel::attachEventBase( + TEventBase* eventBase) { + TAsyncTimeout::attachEventBase(eventBase); + transport_->attachEventBase(eventBase); +} + +template +void TStreamAsyncChannel::detachEventBase() { + // detachEventBase() may not be called while in the middle of reading or + // writing a message. Make sure there are no read callbacks + assert(!readCallback_ && readCallbackQ_.empty()); + // Even though readCallback_ is unset, the read timeout might still be + // installed. This happens when detachEventBase() is invoked by the + // recvMessage() callback, because invokeReadDataAvailable() optimizes and + // leaves the timeout and transport read callback installed while invoking + // the recvMessage() callback. Make sure we cancel the read timeout before + // detaching from the event base. + if (transport_->getReadCallback() == this) { + cancelTimeout(); + transport_->setReadCallback(NULL); + } + + TAsyncTimeout::detachEventBase(); + transport_->detachEventBase(); +} + +template +TEventBase* +TStreamAsyncChannel::getEventBase() const { + return transport_->getEventBase(); +} + +template +void TStreamAsyncChannel::setRecvTimeout( + uint32_t milliseconds) { + recvTimeout_ = milliseconds; + // If we are currently reading, update the timeout + if (transport_->getReadCallback() == this) { + if (milliseconds > 0) { + scheduleTimeout(milliseconds); + } else { + cancelTimeout(); + } + } +} + +template +void TStreamAsyncChannel::getReadBuffer( + void** bufReturn, size_t* lenReturn) { + readState_.getReadBuffer(bufReturn, lenReturn); +} + +template +void TStreamAsyncChannel::readDataAvailable( + size_t len) THRIFT_NOEXCEPT { + invokeReadDataAvailable(len); +} + +template +void TStreamAsyncChannel::readEOF() THRIFT_NOEXCEPT { + // readCallback_ may be NULL if readEOF() is invoked while the read callback + // is already running inside invokeReadDataAvailable(), since + // invokeReadDataAvailable() leaves the transport read callback installed + // while calling the channel read callback. + if (readCallback_) { + processReadEOF(); + } +} + +template +void TStreamAsyncChannel::readError( + const transport::TTransportException& ex) THRIFT_NOEXCEPT { + // readCallback_ may be NULL if readEOF() is invoked while the read callback + // is already running inside invokeReadDataAvailable(), since + // invokeReadDataAvailable() leaves the transport read callback installed + // while calling the channel read callback. + if (!readCallback_) { + return; + } + + DestructorGuard dg(this); + + cancelTimeout(); + failAllReads(); +} + +template +void TStreamAsyncChannel::writeSuccess() + THRIFT_NOEXCEPT { + DestructorGuard dg(this); + + WriteRequest_* req = popWriteRequest(); + req->writeSuccess(); + delete req; +} + +template +void TStreamAsyncChannel::writeError( + size_t bytesWritten, + const transport::TTransportException& ex) THRIFT_NOEXCEPT { + DestructorGuard dg(this); + + if (ex.getType() == transport::TTransportException::TIMED_OUT) { + timedOut_ = true; + } + + WriteRequest_* req = popWriteRequest(); + req->writeError(bytesWritten, ex); + delete req; +} + +template +void TStreamAsyncChannel::timeoutExpired() + THRIFT_NOEXCEPT { + DestructorGuard dg(this); + + timedOut_ = true; + + // Close the transport. It isn't usable anymore, since we are leaving + // it in a state with a partial message outstanding. + transport_->setReadCallback(NULL); + transport_->close(); + + // TODO: It would be nice not to have to always log an error message here; + // ideally the callback should decide if this is worth logging or not. + // Unfortunately the TAsyncChannel API doesn't allow us to pass any error + // info back to the callback. + T_ERROR("TStreamAsyncChannel: read timeout"); + + failAllReads(); +} + +template +bool TStreamAsyncChannel::invokeReadDataAvailable( + size_t len) THRIFT_NOEXCEPT { + DestructorGuard dg(this); + assert(readCallback_); + + bool readDone; + try { + readDone = readState_.readDataAvailable(len); + } catch (const std::exception& ex) { + // The channel is in an unknown state after an error processing read data. + // Close the channel to ensure that callers cannot try to read from this + // channel again. + // + // Make sure we do this after clearing our callbacks, so that the + // channel won't call our readEOF() method. + cancelTimeout(); + transport_->setReadCallback(NULL); + + std::string addressStr; + try { + transport::TSocketAddress addr; + transport_->getPeerAddress(&addr); + addressStr = addr.describe(); + } catch (const std::exception& e) { + addressStr = "unknown"; + } + + T_ERROR("error reading message from %s: %s", addressStr.c_str(), ex.what()); + failAllReads(); + return true; + } + + if (!readDone) { + // We read some data, but didn't finish reading a full message. + if (recvTimeout_ > 0) { + // Reset the timeout whenever we receive any data. + // TODO: This matches the old TAsyncChannel behavior, but it seems like + // it would make more sense to have the timeout apply to the entire + // message as a whole. Eventually we should remove this code that resets + // the timeout. + scheduleTimeout(recvTimeout_); + } + return false; + } + + TEventBase* ourEventBase = transport_->getEventBase(); + + // We read a full message. Invoke the read callback. + invokeReadCallback(readCallback_, "read callback"); + + // Note that we cleared readCallback_ and readErrorCallback_ before invoking + // the callback, but left ourself installed as the TAsyncTransport read + // callback. + // + // This allows us to avoid changing the TAsyncTransport read callback if the + // channel read callback immediately called recvMessage() again. This is + // fairly common, and we avoid 2 unnecessary epoll_ctl() calls by not + // changing the transport read callback. This results in a noticeable + // performance improvement. + // + // If readCallback_ is set again after the callback returns, we're still + // reading. recvMessage() will have taken care of reseting the receive + // timeout, so we have nothing else to do. + // + // If readCallback_ is unset, recvMessage() wasn't called again and we need + // to stop reading. If our TEventBase has changed, detachEventBase() will + // have already stopped reading. (Note that if the TEventBase has changed, + // it's possible that readCallback_ has already been set again to start + // reading in the other thread.) + if (transport_->getEventBase() == ourEventBase && !readCallback_) { + if (readCallbackQ_.empty()) { + cancelTimeout(); + transport_->setReadCallback(NULL); + } else { + // There are queued readers, pop one. This block should have the same + // effect as if recvMessage were called + const ReadQueueEntry &qentry = readCallbackQ_.front(); + readCallback_ = qentry.readCallback; + readErrorCallback_ = qentry.readErrorCallback; + readState_.setCallbackBuffer(qentry.readBuffer); + readCallbackQ_.pop_front(); + + if (readState_.hasReadAheadData()) { + return invokeReadDataAvailable(0); + } else if (recvTimeout_ > 0) { + scheduleTimeout(recvTimeout_); + } + } + } + return true; +} + +template +void TStreamAsyncChannel::failAllReads() { + invokeReadCallback(readErrorCallback_, "read error callback"); + + while (!readCallbackQ_.empty()) { + const ReadQueueEntry &qentry = readCallbackQ_.front(); + invokeReadCallback(qentry.readErrorCallback, "read error callback"); + readCallbackQ_.pop_front(); + } +} + +template +void TStreamAsyncChannel::processReadEOF() + THRIFT_NOEXCEPT { + DestructorGuard dg(this); + assert(readCallback_); + + VoidCallback cb; + const char* cbName; + if (readState_.hasPartialMessage()) { + cb = readErrorCallback_; + cbName = "read error callback"; + } else { + // We call the normal (non-error) callback if no data has been received yet + // when EOF occurs. + // + // TODO: It would be nicer to have a mechanism to indicate to the caller + // that EOF was received, instead of treating this just like 0-sized + // message. + cb = readCallback_; + cbName = "read callback"; + } + + cancelTimeout(); + invokeReadCallback(cb, cbName); + + // Any queued reads should be notified like the else case above as only + // the first reader can have partial data. + while (!readCallbackQ_.empty()) { + const ReadQueueEntry &qentry = readCallbackQ_.front(); + invokeReadCallback(qentry.readCallback, cbName); + readCallbackQ_.pop_front(); + } +} + +template +void TStreamAsyncChannel::invokeReadCallback( + VoidCallback cb, char const* callbackName) THRIFT_NOEXCEPT { + readState_.unsetCallbackBuffer(); + readCallback_ = VoidCallback(); + readErrorCallback_ = VoidCallback(); + + try { + cb(); + } catch (const std::exception& ex) { + T_ERROR("TAsyncChannel: %s threw %s exception: %s", + callbackName, typeid(ex).name(), ex.what()); + abort(); + } catch (...) { + T_ERROR("TAsyncChannel: %s threw exception", callbackName); + abort(); + } +} + +}}} // apache::thrift::async + +#endif // THRIFT_ASYNC_TSTREAMASYNCCHANNEL_TCC_ diff --git a/thrift/lib/cpp/async/TSyncToAsyncProcessor.h b/thrift/lib/cpp/async/TSyncToAsyncProcessor.h new file mode 100644 index 0000000000..4099a9c022 --- /dev/null +++ b/thrift/lib/cpp/async/TSyncToAsyncProcessor.h @@ -0,0 +1,42 @@ +// Copyright (c) 2006- Facebook +// Distributed under the Thrift Software License +// +// See accompanying file LICENSE or visit the Thrift site at: +// http://developers.facebook.com/thrift/ + +#ifndef _THRIFT_TSYNCTOASYNCPROCESSOR_H_ +#define _THRIFT_TSYNCTOASYNCPROCESSOR_H_ 1 + +#include +#include +#include "thrift/lib/cpp/TProcessor.h" +#include "thrift/lib/cpp/async/TAsyncProcessor.h" + +namespace apache { namespace thrift { namespace async { + +/** + * Adapter to allow a TProcessor to be used as a TAsyncProcessor. + * + * Note that this should only be used for handlers that return quickly without + * blocking, since async servers can be stalled by a single blocking operation. + */ +class TSyncToAsyncProcessor : public TAsyncProcessor { + public: + TSyncToAsyncProcessor(boost::shared_ptr processor) + : processor_(processor) + {} + + virtual void process(std::tr1::function _return, + boost::shared_ptr in, + boost::shared_ptr out, + TConnectionContext* context) { + return _return(processor_->process(in, out, context)); + } + + private: + boost::shared_ptr processor_; +}; + +}}} // apache::thrift::async + +#endif // #ifndef _THRIFT_TSYNCTOASYNCPROCESSOR_H_ diff --git a/thrift/lib/cpp/async/TUndelayedDestruction.h b/thrift/lib/cpp/async/TUndelayedDestruction.h new file mode 100644 index 0000000000..fcc9a5cb9c --- /dev/null +++ b/thrift/lib/cpp/async/TUndelayedDestruction.h @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_ASYNC_TUNDELAYEDDESTRUCTION_H_ +#define THRIFT_ASYNC_TUNDELAYEDDESTRUCTION_H_ 1 + +#include +#include +#include +#include + +namespace apache { namespace thrift { namespace async { + +/** + * A helper class to allow a TDelayedDestruction object to be instantiated on + * the stack. + * + * This class derives from an existing TDelayedDestruction type and makes the + * destructor public again. This allows objects of this type to be declared on + * the stack or directly inside another class. Normally TDelayedDestruction + * objects must be dynamically allocated on the heap. + * + * However, the trade-off is that you lose some of the protections provided by + * TDelayedDestruction::destroy(). TDelayedDestruction::destroy() will + * automatically delay destruction of the object until it is safe to do so. + * If you use TUndelayedDestruction, you become responsible for ensuring that + * you only destroy the object where it is safe to do so. Attempting to + * destroy a TUndelayedDestruction object while it has a non-zero destructor + * guard count will abort the program. + */ +template +class TUndelayedDestruction : public TDD { + public: + // We want to expose all constructors provided by the parent class. + // C++11 adds constructor inheritance to support this. Unfortunately gcc + // does not implement constructor inheritance yet, so we have to fake it with + // variadic templates. +#if THRIFT_HAVE_CONSTRUCTOR_INHERITANCE + using TDD::TDD; +#else + // We unfortunately can't simulate constructor inheritance as well as I'd + // like. + // + // Ideally we would use std::enable_if<> and std::is_constructible<> to + // provide only constructor methods that are valid for our parent class. + // Unfortunately std::is_constructible<> doesn't work for types that aren't + // destructible. In gcc-4.6 it results in a compiler error. In the latest + // gcc code it looks like it has been fixed to return false. (The language + // in the standard seems to indicate that returning false is the correct + // behavior for non-destructible types, which is unfortunate.) + template + explicit TUndelayedDestruction(Args&& ...args) + : TDD(std::forward(args)...) {} +#endif + + /** + * Public destructor. + * + * The caller is responsible for ensuring that the object is only destroyed + * where it is safe to do so. (i.e., when the destructor guard count is 0). + * + * The exact conditions for meeting this may be dependant upon your class + * semantics. Typically you are only guaranteed that it is safe to destroy + * the object directly from the event loop (e.g., directly from a + * TEventBase::LoopCallback), or when the event loop is stopped. + */ + virtual ~TUndelayedDestruction() { + // Crash if the caller is destroying us with outstanding destructor guards. + if (this->getDestructorGuardCount() != 0) { + abort(); + } + // Invoke destroy. This is necessary since our base class may have + // implemented custom behavior in destroy(). + this->destroy(); + } + + protected: + /** + * Override our parent's destroy() method to make it protected. + * Callers should use the normal destructor instead of destroy + */ + virtual void destroy() { + this->TDD::destroy(); + } + + virtual void destroyNow(bool delayed) { + // Do nothing. This will always be invoked from the call to destroy inside + // our destructor. + assert(!delayed); + // prevent unused variable warnings when asserts are compiled out. + (void)delayed; + } + + private: + // Forbidden copy constructor and assignment operator + TUndelayedDestruction(TUndelayedDestruction const &) = delete; + TUndelayedDestruction& operator=(TUndelayedDestruction const &) = delete; +}; + +}}} // apache::thrift::async + +#endif // THRIFT_ASYNC_TUNDELAYEDDESTRUCTION_H_ diff --git a/thrift/lib/cpp/async/TUnframedAsyncChannel.h b/thrift/lib/cpp/async/TUnframedAsyncChannel.h new file mode 100644 index 0000000000..9dbffde392 --- /dev/null +++ b/thrift/lib/cpp/async/TUnframedAsyncChannel.h @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_ASYNC_TUNFRAMEDASYNCCHANNEL_H_ +#define THRIFT_ASYNC_TUNFRAMEDASYNCCHANNEL_H_ 1 + +#include "thrift/lib/cpp/async/TStreamAsyncChannel.h" + +namespace apache { namespace thrift { namespace async { + +namespace detail { + +/** + * Encapsulation of one outstanding write request on a TUnframedAsyncChannel. + */ +class TUnframedACWriteRequest : + public TAsyncChannelWriteRequestBase { + public: + typedef std::tr1::function VoidCallback; + + TUnframedACWriteRequest(const VoidCallback& callback, + const VoidCallback& errorCallback, + transport::TMemoryBuffer* message, + TAsyncEventChannel* channel); + + void write(TAsyncTransport* transport, + TAsyncTransport::WriteCallback* callback) THRIFT_NOEXCEPT; + + void writeSuccess() THRIFT_NOEXCEPT; + void writeError(size_t bytesWritten, + const transport::TTransportException& ex) THRIFT_NOEXCEPT; +}; + +/** + * Read state for TUnframedAsyncChannel + */ +template +class TUnframedACReadState { + public: + typedef std::tr1::function VoidCallback; + typedef ProtocolTraits_ ProtocolTraits; + + TUnframedACReadState(); + ~TUnframedACReadState(); + + // Methods required by TStreamAsyncChannel + + void setCallbackBuffer(transport::TMemoryBuffer* buffer) { + callbackBuffer_ = buffer; + } + void unsetCallbackBuffer() { + callbackBuffer_ = NULL; + } + + bool hasReadAheadData() { + return (memBuffer_.available_read() > 0); + } + bool hasPartialMessage() { + return (memBuffer_.available_read() > 0); + } + + void getReadBuffer(void** bufReturn, size_t* lenReturn); + bool readDataAvailable(size_t len); + + // Methods specific to TUnframedACReadState + + void setMaxMessageSize(uint32_t size) { + maxMessageSize_ = size; + } + + uint32_t getMaxMessageSize() const { + return maxMessageSize_; + } + + ProtocolTraits_* getProtocolTraits() { + return &protocolTraits_; + } + const ProtocolTraits_* getProtocolTraits() const { + return &protocolTraits_; + } + + private: + bool getMessageLength(uint8_t* buffer, + uint32_t bufferLength, + uint32_t* messageLength); + + /// maximum frame size accepted + uint32_t maxMessageSize_; + + apache::thrift::transport::TMemoryBuffer memBuffer_; + apache::thrift::transport::TMemoryBuffer* callbackBuffer_; + ProtocolTraits_ protocolTraits_; +}; + +} // namespace detail + +/** + * TUnframedAsyncChannel + * + * This is a TAsyncChannel implementation that reads and writes raw (unframed) + * messages. When reading messages, ProtocolTraits_ is used to determine the + * end of a message. + */ +template +class TUnframedAsyncChannel : + public TStreamAsyncChannel > { + private: + typedef TStreamAsyncChannel > + Parent; + typedef TUnframedAsyncChannel Self; + + public: + explicit TUnframedAsyncChannel( + const boost::shared_ptr& transport + ) + : Parent(transport) {} + + /** + * Helper function to create a shared_ptr. + * + * This passes in the correct destructor object, since TUnframedAsyncChannel's + * destructor is protected and cannot be invoked directly. + */ + static boost::shared_ptr newChannel( + const boost::shared_ptr& transport) { + return boost::shared_ptr(new Self(transport), + typename Self::Destructor()); + } + + /// size in bytes beyond which we'll reject a given message. + void setMaxMessageSize(uint32_t size) { + this->readState_.setMaxMessageSize(size); + } + + uint32_t getMaxMessageSize() const { + return this->readState_.getMaxMessageSize(); + } + + protected: + /** + * Protected destructor. + * + * Users of TUnframedAsyncChannel must never delete it directly. Instead, + * invoke destroy(). + */ + virtual ~TUnframedAsyncChannel() { } +}; + +}}} // apache::thrift::async + +#endif // THRIFT_ASYNC_TUNFRAMEDASYNCCHANNEL_H_ diff --git a/thrift/lib/cpp/async/TUnframedAsyncChannel.tcc b/thrift/lib/cpp/async/TUnframedAsyncChannel.tcc new file mode 100644 index 0000000000..b23d2461b3 --- /dev/null +++ b/thrift/lib/cpp/async/TUnframedAsyncChannel.tcc @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_ASYNC_TUNFRAMEDASYNCCHANNEL_TCC_ +#define THRIFT_ASYNC_TUNFRAMEDASYNCCHANNEL_TCC_ 1 + +#include "thrift/lib/cpp/async/TUnframedAsyncChannel.h" + +#include "thrift/lib/cpp/transport/TBufferTransports.h" + +namespace { +const uint32_t kInitialBufferSize = 4096; +} + +namespace apache { namespace thrift { namespace async { namespace detail { + +template +TUnframedACReadState::TUnframedACReadState() + : maxMessageSize_(0x7fffffff) + , memBuffer_(kInitialBufferSize) + , callbackBuffer_(NULL) + , protocolTraits_() { +} + +template +TUnframedACReadState::~TUnframedACReadState() { +} + +template +void TUnframedACReadState::getReadBuffer(void** bufReturn, + size_t* lenReturn) { + uint32_t bytesAvailable = memBuffer_.available_write(); + if (bytesAvailable > 0) { + // If there is room available in the buffer, just return it. + *lenReturn = bytesAvailable; + *bufReturn = memBuffer_.getWritePtr(bytesAvailable); + return; + } + + uint32_t bufferSize = memBuffer_.getBufferSize(); + uint32_t available_read = memBuffer_.available_read(); + // we get this much without growing the buffer capacity + uint32_t additionalSpace = bufferSize - available_read; + if (additionalSpace == 0) { + // We need more room. memBuffer_ will at least double it's capacity when + // asked for even a single byte. + additionalSpace = kInitialBufferSize; + } + + // Don't allow more than maxMessageSize_. + // Be careful not to over- or underflow uint32_t when checking. + // + // readDataAvailable() fails the read when we've already read maxMessageSize_ + // bytes, so available_read should always be less than maxMessageSize_ here. + // (Unless maxMessageSize_ is 0, but that's a programmer bug.) + assert(available_read < maxMessageSize_); + if (available_read > maxMessageSize_ - additionalSpace) { + // Don't ask for more than maxMessageSize_ total (but we might get more) + additionalSpace = maxMessageSize_ - available_read; + } + + try { + uint8_t* newBuffer = memBuffer_.getWritePtr(additionalSpace); + *lenReturn = memBuffer_.available_write(); + *bufReturn = newBuffer; + } catch (std::exception &ex) { + T_ERROR("TUnframedAsyncChannel: failed to allocate larger read buffer: %s", + ex.what()); + *lenReturn = 0; + *bufReturn = NULL; + } +} + +template +bool TUnframedACReadState::readDataAvailable(size_t len) { + assert(memBuffer_.available_read() + len <= memBuffer_.getBufferSize()); + memBuffer_.wroteBytes(len); + + uint32_t messageLength = 0; + uint32_t bytesRead = memBuffer_.available_read(); + uint8_t *buffer = (uint8_t *)memBuffer_.borrow(NULL, &bytesRead); + if (!protocolTraits_.getMessageLength(buffer, bytesRead, &messageLength)) { + // We're not at the end of the message yet. + // + // If we've hit maxMessageSize_ already, fail now instead of waiting until + // getReadBuffer() is called again. + if (bytesRead >= maxMessageSize_) { + throw transport::TTransportException( + transport::TTransportException::CORRUPTED_DATA, + "TUnframedAsyncChannel: max message size exceeded"); + } + return false; + } + + // We've read a full message. + // Swap the data into the callback's buffer. + // Note that we may have actually read more than one message, + // so we have to make sure to save any remaining data after the end of the + // message. + assert(messageLength <= bytesRead); + + callbackBuffer_->link(&memBuffer_, messageLength); + memBuffer_.consume(messageLength); + + // We've put a new message in callbackBuffer_ + return true; +} + +}}}} // apache::thrift::async::detail + +#endif // THRIFT_ASYNC_TUNFRAMEDASYNCCHANNEL_TCC_ diff --git a/thrift/lib/cpp/async/TZlibAsyncChannel.h b/thrift/lib/cpp/async/TZlibAsyncChannel.h new file mode 100644 index 0000000000..a56770d95f --- /dev/null +++ b/thrift/lib/cpp/async/TZlibAsyncChannel.h @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_ASYNC_TZLIBASYNCCHANNEL_H_ +#define THRIFT_ASYNC_TZLIBASYNCCHANNEL_H_ 1 + +#include "thrift/lib/cpp/async/TAsyncEventChannel.h" +#include "thrift/lib/cpp/transport/TZlibTransport.h" + +namespace apache { namespace thrift { namespace async { + +class TZlibAsyncChannel : public TAsyncEventChannel { + public: + TZlibAsyncChannel(const boost::shared_ptr& channel); + + /** + * Helper function to create a shared_ptr. + * + * This passes in the correct destructor object, since TZlibAsyncChannel's + * destructor is protected and cannot be invoked directly. + */ + static boost::shared_ptr newChannel( + const boost::shared_ptr& channel) { + return boost::shared_ptr( + new TZlibAsyncChannel(channel), Destructor()); + } + virtual bool readable() const { + return channel_->readable(); + } + virtual bool good() const { + return channel_->good(); + } + virtual bool error() const { + return channel_->error(); + } + virtual bool timedOut() const { + return channel_->timedOut(); + } + virtual bool isIdle() const { + return channel_->isIdle(); + } + + virtual void sendMessage(const VoidCallback& cob, + const VoidCallback& errorCob, + transport::TMemoryBuffer* message); + virtual void recvMessage(const VoidCallback& cob, + const VoidCallback& errorCob, + transport::TMemoryBuffer* message); + virtual void sendAndRecvMessage(const VoidCallback& cob, + const VoidCallback& errorCob, + transport::TMemoryBuffer* sendBuf, + transport::TMemoryBuffer* recvBuf); + + virtual boost::shared_ptr getTransport() { + return channel_->getTransport(); + } + + virtual void attachEventBase(TEventBase* eventBase) { + channel_->attachEventBase(eventBase); + } + virtual void detachEventBase() { + channel_->detachEventBase(); + } + + virtual uint32_t getRecvTimeout() const { + return channel_->getRecvTimeout(); + } + + virtual void setRecvTimeout(uint32_t milliseconds) { + channel_->setRecvTimeout(milliseconds); + } + + protected: + /** + * Protected destructor. + * + * Users of TZlibAsyncChannel must never delete it directly. Instead, + * invoke destroy(). + */ + virtual ~TZlibAsyncChannel() { } + + private: + class SendRequest { + public: + SendRequest(); + + bool isSet() const { + return static_cast(callback_); + } + + void set(const VoidCallback& callback, + const VoidCallback& errorCallback, + transport::TMemoryBuffer* message); + + void send(TAsyncEventChannel* channel); + + private: + void invokeCallback(VoidCallback callback); + void sendSuccess(); + void sendError(); + + boost::shared_ptr compressedBuffer_; + transport::TZlibTransport zlibTransport_; + VoidCallback sendSuccess_; + VoidCallback sendError_; + + VoidCallback callback_; + VoidCallback errorCallback_; + }; + + class RecvRequest { + public: + RecvRequest(); + + bool isSet() const { + return static_cast(callback_); + } + + void set(const VoidCallback& callback, + const VoidCallback& errorCallback, + transport::TMemoryBuffer* message); + + void recv(TAsyncEventChannel* channel); + + private: + void invokeCallback(VoidCallback callback); + void recvSuccess(); + void recvError(); + + boost::shared_ptr compressedBuffer_; + transport::TZlibTransport zlibTransport_; + VoidCallback recvSuccess_; + VoidCallback recvError_; + + VoidCallback callback_; + VoidCallback errorCallback_; + transport::TMemoryBuffer *callbackBuffer_; + }; + + boost::shared_ptr channel_; + + // TODO: support multiple pending send requests + SendRequest sendRequest_; + RecvRequest recvRequest_; +}; + +}}} // apache::thrift::async + +#endif // THRIFT_ASYNC_TZLIBASYNCCHANNEL_H_ diff --git a/thrift/lib/cpp/concurrency/Exception.h b/thrift/lib/cpp/concurrency/Exception.h new file mode 100644 index 0000000000..bef1eb2f9c --- /dev/null +++ b/thrift/lib/cpp/concurrency/Exception.h @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_CONCURRENCY_EXCEPTION_H_ +#define _THRIFT_CONCURRENCY_EXCEPTION_H_ 1 + +#include +#include "thrift/lib/cpp/Thrift.h" + +namespace apache { namespace thrift { namespace concurrency { + +class NoSuchTaskException : public apache::thrift::TLibraryException {}; + +class UncancellableTaskException : public apache::thrift::TLibraryException {}; + +class InvalidArgumentException : public apache::thrift::TLibraryException {}; + +class IllegalStateException : public apache::thrift::TLibraryException { +public: + IllegalStateException() {} + IllegalStateException(const std::string& message) : TLibraryException(message) {} +}; + +class TimedOutException : public apache::thrift::TLibraryException { +public: + TimedOutException():TLibraryException("TimedOutException"){}; + TimedOutException(const std::string& message ) : + TLibraryException(message) {} +}; + +class TooManyPendingTasksException : public apache::thrift::TLibraryException { +public: + TooManyPendingTasksException():TLibraryException("TooManyPendingTasksException"){}; + TooManyPendingTasksException(const std::string& message ) : + TLibraryException(message) {} +}; + +class SystemResourceException : public apache::thrift::TLibraryException { +public: + SystemResourceException() {} + + SystemResourceException(const std::string& message) : + TLibraryException(message) {} +}; + +}}} // apache::thrift::concurrency + +#endif // #ifndef _THRIFT_CONCURRENCY_EXCEPTION_H_ diff --git a/thrift/lib/cpp/concurrency/FunctionRunner.h b/thrift/lib/cpp/concurrency/FunctionRunner.h new file mode 100644 index 0000000000..322d0c3136 --- /dev/null +++ b/thrift/lib/cpp/concurrency/FunctionRunner.h @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_CONCURRENCY_FUNCTION_RUNNER_H +#define _THRIFT_CONCURRENCY_FUNCTION_RUNNER_H 1 + +#include +#include "thrift/lib/cpp/concurrency/Thread.h" + +namespace apache { namespace thrift { namespace concurrency { + +/** + * Convenient implementation of Runnable that will execute arbitrary callbacks. + * Interfaces are provided to accept both a generic 'void(void)' callback, and + * a 'void* (void*)' pthread_create-style callback. + * + * Example use: + * void* my_thread_main(void* arg); + * shared_ptr factory = ...; + * // To create a thread that executes my_thread_main once: + * shared_ptr thread = factory->newThread( + * FunctionRunner::create(my_thread_main, some_argument)); + * thread->start(); + * + * bool A::foo(); + * A* a = new A(); + * // To create a thread that executes a.foo() every 100 milliseconds: + * factory->newThread(FunctionRunner::create( + * std::tr1::bind(&A::foo, a), 100))->start(); + * + */ + +class FunctionRunner : public Runnable { + public: + // This is the type of callback 'pthread_create()' expects. + typedef void* (*PthreadFuncPtr)(void *arg); + // This a fully-generic void(void) callback for custom bindings. + typedef std::tr1::function VoidFunc; + + typedef std::tr1::function BoolFunc; + + /** + * Syntactic sugar to make it easier to create new FunctionRunner + * objects wrapped in shared_ptr. + */ + static boost::shared_ptr create(const VoidFunc& cob) { + return boost::shared_ptr(new FunctionRunner(cob)); + } + + static boost::shared_ptr create(PthreadFuncPtr func, + void* arg) { + return boost::shared_ptr(new FunctionRunner(func, arg)); + } + + static boost::shared_ptr create(const BoolFunc& cob, + int intervalMs) { + return boost::shared_ptr(new FunctionRunner(cob, + intervalMs)); + } + + /** + * Given a 'pthread_create' style callback, this FunctionRunner will + * execute the given callback. Note that the 'void*' return value is ignored. + */ + FunctionRunner(PthreadFuncPtr func, void* arg) + : func_(std::tr1::bind(func, arg)), repFunc_(0), initFunc_(0) + { } + + /** + * Given a generic callback, this FunctionRunner will execute it. + */ + FunctionRunner(const VoidFunc& cob) + : func_(cob), repFunc_(0), initFunc_(0) + { } + + /** + * Given a bool foo(...) type callback, FunctionRunner will execute + * the callback repeatedly with 'intervalMs' milliseconds between the calls, + * until it returns false. Note that the actual interval between calls will + * be intervalMs plus execution time of the callback. + */ + FunctionRunner(const BoolFunc& cob, int intervalMs) + : func_(0), repFunc_(cob), intervalMs_(intervalMs), initFunc_(0) + { } + + /** + * Set a callback to be called when the thread is started. + */ + void setInitFunc(const VoidFunc& initFunc) { + initFunc_ = initFunc; + } + + void run() { + if (initFunc_) { + initFunc_(); + } + if (repFunc_) { + while(repFunc_()) { + usleep(intervalMs_*1000); + } + } else { + func_(); + } + } + + private: + VoidFunc func_; + BoolFunc repFunc_; + int intervalMs_; + VoidFunc initFunc_; +}; + +}}} // apache::thrift::concurrency + +#endif // #ifndef _THRIFT_CONCURRENCY_FUNCTION_RUNNER_H diff --git a/thrift/lib/cpp/concurrency/Monitor.h b/thrift/lib/cpp/concurrency/Monitor.h new file mode 100644 index 0000000000..c5164ff917 --- /dev/null +++ b/thrift/lib/cpp/concurrency/Monitor.h @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_CONCURRENCY_MONITOR_H_ +#define _THRIFT_CONCURRENCY_MONITOR_H_ 1 + +#include "thrift/lib/cpp/concurrency/Exception.h" +#include "thrift/lib/cpp/concurrency/Mutex.h" + +#include + + +namespace apache { namespace thrift { namespace concurrency { + +/** + * A monitor is a combination mutex and condition-event. Waiting and + * notifying condition events requires that the caller own the mutex. Mutex + * lock and unlock operations can be performed independently of condition + * events. This is more or less analogous to java.lang.Object multi-thread + * operations. + * + * Note the Monitor can create a new, internal mutex; alternatively, a + * separate Mutex can be passed in and the Monitor will re-use it without + * taking ownership. It's the user's responsibility to make sure that the + * Mutex is not deallocated before the Monitor. + * + * Note that all methods are const. Monitors implement logical constness, not + * bit constness. This allows const methods to call monitor methods without + * needing to cast away constness or change to non-const signatures. + * + * @version $Id:$ + */ +class Monitor : boost::noncopyable { + public: + /** Creates a new mutex, and takes ownership of it. */ + Monitor(); + + /** Uses the provided mutex without taking ownership. */ + explicit Monitor(Mutex* mutex); + + /** Uses the mutex inside the provided Monitor without taking ownership. */ + explicit Monitor(Monitor* monitor); + + /** Deallocates the mutex only if we own it. */ + virtual ~Monitor(); + + Mutex& mutex() const; + + virtual void lock() const; + + virtual void unlock() const; + + /** + * Waits a maximum of the specified timeout in milliseconds for the condition + * to occur, or waits forever if timeout_ms == 0. + * + * Returns 0 if condition occurs, ETIMEDOUT on timeout, or an error code. + */ + int waitForTimeRelative(int64_t timeout_ms) const; + + /** + * Waits until the absolute time specified using struct timespec. + * Returns 0 if condition occurs, ETIMEDOUT on timeout, or an error code. + */ + int waitForTime(const timespec* abstime) const; + + /** + * Waits forever until the condition occurs. + * Returns 0 if condition occurs, or an error code otherwise. + */ + int waitForever() const; + + /** + * Exception-throwing version of waitForTimeRelative(), called simply + * wait(int64) for historical reasons. Timeout is in milliseconds. + * + * If the condition occurs, this function returns cleanly; on timeout or + * error an exception is thrown. + */ + void wait(int64_t timeout_ms = 0LL) const; + + + /** Wakes up one thread waiting on this monitor. */ + virtual void notify() const; + + /** Wakes up all waiting threads on this monitor. */ + virtual void notifyAll() const; + + private: + + class Impl; + + Impl* impl_; +}; + +class Synchronized { + public: + Synchronized(const Monitor* monitor) : g(monitor->mutex()) { } + Synchronized(const Monitor& monitor) : g(monitor.mutex()) { } + + private: + Guard g; +}; + + +}}} // apache::thrift::concurrency + +#endif // #ifndef _THRIFT_CONCURRENCY_MONITOR_H_ diff --git a/thrift/lib/cpp/concurrency/Mutex.h b/thrift/lib/cpp/concurrency/Mutex.h new file mode 100644 index 0000000000..a0e5ca56ec --- /dev/null +++ b/thrift/lib/cpp/concurrency/Mutex.h @@ -0,0 +1,276 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef THRIFT_CONCURRENCY_MUTEX_H_ +#define THRIFT_CONCURRENCY_MUTEX_H_ 1 + +#include +#include + +namespace apache { namespace thrift { namespace concurrency { + +#ifndef THRIFT_NO_CONTENTION_PROFILING + +/** + * Determines if the Thrift Mutex and ReadWriteMutex classes will attempt to + * profile their blocking acquire methods. If this value is set to non-zero, + * Thrift will attempt to invoke the callback once every profilingSampleRate + * times. However, as the sampling is not synchronized the rate is not + * guaranteed, and could be subject to big bursts and swings. Please ensure + * your sampling callback is as performant as your application requires. + * + * The callback will get called with the wait time taken to lock the mutex in + * usec and a (void*) that uniquely identifies the Mutex (or ReadWriteMutex) + * being locked. + * + * The enableMutexProfiling() function is unsynchronized; calling this function + * while profiling is already enabled may result in race conditions. On + * architectures where a pointer assignment is atomic, this is safe but there + * is no guarantee threads will agree on a single callback within any + * particular time period. + */ +typedef void (*MutexWaitCallback)(const void* id, int64_t waitTimeMicros); +void enableMutexProfiling(int32_t profilingSampleRate, + MutexWaitCallback callback); + +#endif + +/** + * A simple mutex class + * + * @version $Id:$ + */ +class Mutex { + public: + typedef void (*Initializer)(void*); + + // Specifying the type of the mutex with one of the static Initializer + // methods defined in this class. + explicit Mutex(Initializer init = DEFAULT_INITIALIZER); + + // Specifying the type of the mutex with an integer. The value has + // to be supported by the underlying implementation, currently + // pthread_mutex. So the possible values are PTHREAD_MUTEX_NORMAL, + // PTHREAD_MUTEX_ERRORCHECK, PTHREAD_MUTEX_RECURSIVE and + // PTHREAD_MUTEX_DEFAULT. + explicit Mutex(int type); + + virtual ~Mutex() {} + virtual void lock() const; + virtual bool trylock() const; + virtual bool timedlock(int64_t milliseconds) const; + virtual void unlock() const; + + /** + * Determine if the mutex is locked. + * + * This is intended to be used primarily as a debugging aid, and is not + * guaranteed to be a fast operation. For example, a common use case is to + * assert(mutex.isLocked()) in functions that may only be called with a + * particular mutex already locked. + * + * TODO: This method currently always returns false for recursive mutexes. + * Avoid calling this method on recursive mutexes. + */ + virtual bool isLocked() const; + + void* getUnderlyingImpl() const; + + static void DEFAULT_INITIALIZER(void*); + static void ADAPTIVE_INITIALIZER(void*); + static void RECURSIVE_INITIALIZER(void*); + + private: + + class impl; + boost::shared_ptr impl_; +}; + +class ReadWriteMutex { +public: + ReadWriteMutex(); + virtual ~ReadWriteMutex() {} + + // these get the lock and block until it is done successfully + virtual void acquireRead() const; + virtual void acquireWrite() const; + + // these get the lock and block until it is done successfully + // or run out of time + virtual bool timedRead(int64_t milliseconds) const; + virtual bool timedWrite(int64_t milliseconds) const; + + // these attempt to get the lock, returning false immediately if they fail + virtual bool attemptRead() const; + virtual bool attemptWrite() const; + + // this releases both read and write locks + virtual void release() const; + +private: + + class impl; + boost::shared_ptr impl_; +}; + +/** + * A ReadWriteMutex that guarantees writers will not be starved by readers: + * When a writer attempts to acquire the mutex, all new readers will be + * blocked from acquiring the mutex until the writer has acquired and + * released it. In some operating systems, this may already be guaranteed + * by a regular ReadWriteMutex. + */ +class NoStarveReadWriteMutex : public ReadWriteMutex { +public: + NoStarveReadWriteMutex(); + + virtual void acquireRead() const; + virtual void acquireWrite() const; + + // these get the lock and block until it is done successfully + // or run out of time + virtual bool timedRead(int64_t milliseconds) const; + virtual bool timedWrite(int64_t milliseconds) const; + +private: + Mutex mutex_; + mutable volatile bool writerWaiting_; +}; + +class Guard : boost::noncopyable { + public: + explicit Guard(const Mutex& value, int64_t timeout = 0) : mutex_(&value) { + if (timeout == 0) { + value.lock(); + } else if (timeout < 0) { + if (!value.trylock()) { + mutex_ = NULL; + } + } else { + if (!value.timedlock(timeout)) { + mutex_ = NULL; + } + } + } + ~Guard() { + if (mutex_) { + mutex_->unlock(); + } + } + + /* + * This is really operator bool. However, implementing it to return + * bool is actually harmful. See + * www.artima.com/cppsource/safebool.html for the details; in brief, + * converting to bool allows a lot of nonsensical operations in + * addition to simple testing. To avoid that, we return a pointer to + * member which can only be used for testing. + */ + typedef const Mutex*const Guard::*const pBoolMember; + inline operator pBoolMember() const { + return mutex_ != NULL ? &Guard::mutex_ : NULL; + } + + private: + const Mutex* mutex_; +}; + +// Can be used as second argument to RWGuard to make code more readable +// as to whether we're doing acquireRead() or acquireWrite(). +enum RWGuardType { + RW_READ = 0, + RW_WRITE = 1, +}; + + +class RWGuard : boost::noncopyable { + public: + explicit RWGuard(const ReadWriteMutex& value, bool write = false, + int64_t timeout=0) + : rw_mutex_(value), locked_(true) { + if (write) { + if (timeout) { + locked_ = rw_mutex_.timedWrite(timeout); + } else { + rw_mutex_.acquireWrite(); + } + } else { + if (timeout) { + locked_ = rw_mutex_.timedRead(timeout); + } else { + rw_mutex_.acquireRead(); + } + } + } + + RWGuard(const ReadWriteMutex& value, RWGuardType type, int64_t timeout = 0) + : rw_mutex_(value), locked_(true) { + if (type == RW_WRITE) { + if (timeout) { + locked_ = rw_mutex_.timedWrite(timeout); + } else { + rw_mutex_.acquireWrite(); + } + } else { + if (timeout) { + locked_ = rw_mutex_.timedRead(timeout); + } else { + rw_mutex_.acquireRead(); + } + } + } + ~RWGuard() { + if (locked_) { + rw_mutex_.release(); + } + } + + typedef const bool RWGuard::*const pBoolMember; + operator pBoolMember() const { + return locked_ ? &RWGuard::locked_ : NULL; + } + + bool operator!() const { + return !locked_; + } + + bool release() { + if (!locked_) return false; + rw_mutex_.release(); + locked_ = false; + return true; + } + + private: + const ReadWriteMutex& rw_mutex_; + mutable bool locked_; +}; + + +// A little hack to prevent someone from trying to do "Guard(m);" +// Such a use is invalid because the temporary Guard object is +// destroyed at the end of the line, releasing the lock. +// Sorry for polluting the global namespace, but I think it's worth it. +#define Guard(m) incorrect_use_of_Guard(m) +#define RWGuard(m) incorrect_use_of_RWGuard(m) + + +}}} // apache::thrift::concurrency + +#endif // #ifndef THRIFT_CONCURRENCY_MUTEX_H_ diff --git a/thrift/lib/cpp/concurrency/PosixThreadFactory.h b/thrift/lib/cpp/concurrency/PosixThreadFactory.h new file mode 100644 index 0000000000..1055a04c6b --- /dev/null +++ b/thrift/lib/cpp/concurrency/PosixThreadFactory.h @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + + */ +#ifndef THRIFT_CONCURRENCY_POSIXTHREADFACTORY_H_ +#define THRIFT_CONCURRENCY_POSIXTHREADFACTORY_H_ 1 + +#include + +#include "thrift/lib/cpp/concurrency/Thread.h" + +#include + +namespace apache { namespace thrift { namespace concurrency { + +void getLiveThreadIds(std::set* tids); + +/** + * A thread factory to create posix threads + * + * @version $Id:$ + */ +class PosixThreadFactory : public ThreadFactory { + + public: + + /** + * POSIX Thread scheduler policies + */ + enum POLICY { + OTHER, + FIFO, + ROUND_ROBIN + }; + + /** + * POSIX Thread scheduler relative priorities, + * + * Absolute priority is determined by scheduler policy and OS. This + * enumeration specifies relative priorities such that one can specify a + * priority within a giving scheduler policy without knowing the absolute + * value of the priority. + */ + enum PRIORITY { + LOWEST = 0, + LOWER = 1, + LOW = 2, + NORMAL = 3, + HIGH = 4, + HIGHER = 5, + HIGHEST = 6, + INCREMENT = 7, + DECREMENT = 8 + }; + + /** + * Posix thread (pthread) factory. All threads created by a factory are reference-counted + * via boost::shared_ptr and boost::weak_ptr. The factory guarantees that threads and + * the Runnable tasks they host will be properly cleaned up once the last strong reference + * to both is given up. + * + * Threads are created with the specified policy, priority, stack-size and detachable-mode + * detached means the thread is free-running and will release all system resources the + * when it completes. A detachable thread is not joinable. The join method + * of a detachable thread will return immediately with no error. + * + * By default threads are not joinable. + */ + + explicit PosixThreadFactory(POLICY policy=ROUND_ROBIN, + PRIORITY priority=NORMAL, + int stackSize=1, + bool detached=true); + + // From ThreadFactory; + boost::shared_ptr newThread(boost::shared_ptr runnable) const; + + // From ThreadFactory; + Thread::id_t getCurrentThreadId() const; + + /** + * Gets stack size for created threads + * + * @return int size in megabytes + */ + virtual int getStackSize() const; + + /** + * Sets stack size for created threads + * + * @param value size in megabytes + */ + virtual void setStackSize(int value); + + /** + * Gets priority relative to current policy + */ + virtual PRIORITY getPriority() const; + + /** + * Sets priority relative to current policy + */ + virtual void setPriority(PRIORITY priority); + + /** + * Sets detached mode of threads + */ + virtual void setDetached(bool detached); + + /** + * Gets current detached mode + */ + virtual bool isDetached() const; + + private: + class Impl; + boost::shared_ptr impl_; +}; + +}}} // apache::thrift::concurrency + +#endif // #ifndef _THRIFT_CONCURRENCY_POSIXTHREADFACTORY_H_ diff --git a/thrift/lib/cpp/concurrency/TARGETS b/thrift/lib/cpp/concurrency/TARGETS new file mode 100644 index 0000000000..2663ab449e --- /dev/null +++ b/thrift/lib/cpp/concurrency/TARGETS @@ -0,0 +1,17 @@ +# TARGETS file for thrift/lib/cpp/concurrency + +cpp_library ( + name = "concurrency", + srcs = [ + "Util.cpp", + "Monitor.cpp", + "Mutex.cpp", + "PosixThreadFactory.cpp", + "ThreadManager.cpp", + "TimerManager.cpp" + ], + deps = [ '@/common/base:profiler', + '@/thrift/lib/cpp:thrift_base' ], +) + + diff --git a/thrift/lib/cpp/concurrency/Thread.h b/thrift/lib/cpp/concurrency/Thread.h new file mode 100644 index 0000000000..198223a18d --- /dev/null +++ b/thrift/lib/cpp/concurrency/Thread.h @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_CONCURRENCY_THREAD_H_ +#define _THRIFT_CONCURRENCY_THREAD_H_ 1 + +#include +#include +#include + +namespace apache { namespace thrift { namespace concurrency { + +class Thread; + +/** + * Minimal runnable class. More or less analogous to java.lang.Runnable. + * + * @version $Id:$ + */ +class Runnable { + + public: + virtual ~Runnable() {}; + virtual void run() = 0; + + /** + * Gets the thread object that is hosting this runnable object - can return + * an empty boost::shared pointer if no references remain on the thread object + */ + virtual boost::shared_ptr thread() { return thread_.lock(); } + + /** + * Sets the thread that is executing this object. This is only meant for + * use by concrete implementations of Thread. + */ + virtual void thread(boost::shared_ptr value) { thread_ = value; } + + private: + boost::weak_ptr thread_; +}; + +/** + * Minimal thread class. Returned by thread factory bound to a Runnable object + * and ready to start execution. More or less analogous to java.lang.Thread + * (minus all the thread group, priority, mode and other baggage, since that + * is difficult to abstract across platforms and is left for platform-specific + * ThreadFactory implementations to deal with + * + * @see apache::thrift::concurrency::ThreadFactory) + */ +class Thread { + + public: + + typedef uint64_t id_t; + + virtual ~Thread() {}; + + /** + * Starts the thread. Does platform specific thread creation and + * configuration then invokes the run method of the Runnable object bound + * to this thread. + */ + virtual void start() = 0; + + /** + * Join this thread. Current thread blocks until this target thread + * completes. + */ + virtual void join() = 0; + + /** + * Gets the thread's platform-specific ID + */ + virtual id_t getId() = 0; + + /** + * Gets the runnable object this thread is hosting + */ + virtual boost::shared_ptr runnable() const { return _runnable; } + + protected: + virtual void runnable(boost::shared_ptr value) { _runnable = value; } + + private: + boost::shared_ptr _runnable; + +}; + +/** + * Factory to create platform-specific thread object and bind them to Runnable + * object for execution + */ +class ThreadFactory { + + public: + virtual ~ThreadFactory() {} + virtual boost::shared_ptr newThread(boost::shared_ptr runnable) const = 0; + + /** Gets the current thread id or unknown_thread_id if the current thread is not a thrift thread */ + + static const Thread::id_t unknown_thread_id; + + virtual Thread::id_t getCurrentThreadId() const = 0; +}; + +}}} // apache::thrift::concurrency + +#endif // #ifndef _THRIFT_CONCURRENCY_THREAD_H_ diff --git a/thrift/lib/cpp/concurrency/ThreadLocal.h b/thrift/lib/cpp/concurrency/ThreadLocal.h new file mode 100644 index 0000000000..826f33c657 --- /dev/null +++ b/thrift/lib/cpp/concurrency/ThreadLocal.h @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_CONCURRENCY_THREADLOCAL_H_ +#define THRIFT_CONCURRENCY_THREADLOCAL_H_ 1 + +#include "thrift/lib/cpp/Thrift.h" +#include + +namespace apache { namespace thrift { namespace concurrency { + +template +class DefaultThreadLocalManager; + +/** + * ThreadLocal manages thread-local storage for a particular object type. + * + * Each ThreadLocal object contains a separate instance of an object for each + * thread that accesses the ThreadLocal object. + * + * Note that you should avoid creating too many ThreadLocal objects (e.g., such + * as keeping a ThreadLocal member variable in commonly allocated objects). + * The number of ThreadLocal objects cannot be larger than the value of + * PTHREAD_KEYS_MAX, which is 1024 on many systems. + * + * The ManagerT template parameter controls how object allocation and + * deallocation should be performed. When get() is called from a thread that + * does not already have an instance of the object, Manager::allocate() is + * called. When a thread exits, Manager::destroy() is called if the thread has + * an instance of this object. + */ +template > +class ThreadLocal { + public: + typedef T DataType; + typedef ManagerT Manager; + + /** + * Create a new ThreadLocal object. + */ + ThreadLocal() { + int ret = pthread_key_create(&key_, &ThreadLocal::onThreadExit); + if (ret != 0) { + throw TLibraryException("failed to allocate new thread-local key", ret); + } + } + + /** + * Access this thread's local instance of the object. + * + * If there is no instance of the object in this thread, Manager::allocate() + * will be called to allocate a new instance. (Though some Manager + * implementations may return NULL, if each thread's instance must be + * expilcitly initialized.) + */ + T *get() const { + T *obj = getNoAlloc(); + if (obj == NULL) { + Manager m; + obj = m.allocate(); + if (obj != NULL) { + setImpl(obj); + } + } + return obj; + } + + /** + * Access this thread's local instance of the object. + * + * If there is no instance of the object in this thread, NULL will be + * returned. Manager::allocate() will never be called. + */ + T *getNoAlloc() const { + return static_cast(pthread_getspecific(key_)); + } + + /** + * Operator overload to perform get() + */ + T *operator->() const { + return get(); + } + + /** + * Operator overload to perform get() + */ + T &operator*() const { + return *get(); + } + + /** + * Set the instance of the object to be used by this thread. + */ + void set(T* obj) { + T *old = getNoAlloc(); + Manager m; + m.replace(old, obj); + setImpl(obj); + } + + /** + * Clear the instance of the object used by this thread. + * + * If this thread had a non-NULL object, Manager::destroy() will be called. + */ + void clear() { + T *obj = getNoAlloc(); + if (obj != NULL) { + Manager m; + m.destroy(obj); + setImpl(NULL); + } + } + + private: + void setImpl(T* obj) const { + int ret = pthread_setspecific(key_, obj); + if (ret != 0) { + throw TLibraryException("failed to update thread-local key", ret); + } + } + + static void onThreadExit(void* arg) { + T *obj = static_cast(arg); + if (obj != NULL) { + Manager m; + m.destroy(obj); + } + } + + pthread_key_t key_; +}; + +template +class DefaultThreadLocalManager { + public: + T* allocate() { + return new T; + } + + void destroy(T* t) { + delete t; + } + + void replace(T* oldObj, T* newObj) { + if (oldObj != newObj) { + delete oldObj; + } + } +}; + +template +class DestroyOnlyThreadLocalManager { + public: + T* allocate() { + return NULL; + } + + void destroy(T* t) { + delete t; + } + + void replace(T* oldObj, T* newObj) { + if (oldObj != newObj) { + delete oldObj; + } + } +}; + +template +class NoopThreadLocalManager { + public: + T* allocate() { + return NULL; + } + + void destroy(T*) { + } + + void replace(T*, T*) { + } +}; + +}}} // apache::thrift::concurrency + +#endif // THRIFT_CONCURRENCY_THREADLOCAL_H_ diff --git a/thrift/lib/cpp/concurrency/ThreadManager.h b/thrift/lib/cpp/concurrency/ThreadManager.h new file mode 100644 index 0000000000..bc23f90d35 --- /dev/null +++ b/thrift/lib/cpp/concurrency/ThreadManager.h @@ -0,0 +1,216 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_CONCURRENCY_THREADMANAGER_H_ +#define _THRIFT_CONCURRENCY_THREADMANAGER_H_ 1 + +#include +#include +#include + +namespace apache { namespace thrift { namespace concurrency { + +class Runnable; +class ThreadFactory; + +/** + * ThreadManager class + * + * This class manages a pool of threads. It uses a ThreadFactory to create + * threads. It never actually creates or destroys worker threads, rather + * It maintains statistics on number of idle threads, number of active threads, + * task backlog, and average wait and service times and informs the PoolPolicy + * object bound to instances of this manager of interesting transitions. It is + * then up the PoolPolicy object to decide if the thread pool size needs to be + * adjusted and call this object addWorker and removeWorker methods to make + * changes. + * + * This design allows different policy implementations to used this code to + * handle basic worker thread management and worker task execution and focus on + * policy issues. The simplest policy, StaticPolicy, does nothing other than + * create a fixed number of threads. + */ +class ThreadManager { + + protected: + ThreadManager() {} + + public: + class Task; + typedef std::tr1::function)> ExpireCallback; + typedef std::tr1::function InitCallback; + + virtual ~ThreadManager() {} + + /** + * Starts the thread manager. Verifies all attributes have been properly + * initialized, then allocates necessary resources to begin operation + */ + virtual void start() = 0; + + /** + * Stops the thread manager. Aborts all remaining unprocessed task, shuts + * down all created worker threads, and releases all allocated resources. + * This method blocks for all worker threads to complete, thus it can + * potentially block forever if a worker thread is running a task that + * won't terminate. + */ + virtual void stop() = 0; + + /** + * Joins the thread manager. This is the same as stop, except that it will + * wait until all the tasks have finished, rather than aborting the tasks. + */ + virtual void join() = 0; + + enum STATE { + UNINITIALIZED, + STARTING, + STARTED, + JOINING, + STOPPING, + STOPPED + }; + + virtual const STATE state() const = 0; + + virtual boost::shared_ptr threadFactory() const = 0; + + virtual void threadFactory(boost::shared_ptr value) = 0; + + virtual void addWorker(size_t value=1) = 0; + + virtual void removeWorker(size_t value=1) = 0; + + /** + * Gets the current number of idle worker threads + */ + virtual size_t idleWorkerCount() const = 0; + + /** + * Gets the current number of total worker threads + */ + virtual size_t workerCount() const = 0; + + /** + * Gets the current number of pending tasks + */ + virtual size_t pendingTaskCount() const = 0; + + /** + * Gets the current number of pending and executing tasks + */ + virtual size_t totalTaskCount() const = 0; + + /** + * Gets the maximum pending task count. 0 indicates no maximum + */ + virtual size_t pendingTaskCountMax() const = 0; + + /** + * Gets the number of tasks which have been expired without being run. + */ + virtual size_t expiredTaskCount() = 0; + + /** + * Adds a task to be executed at some time in the future by a worker thread. + * + * This method will block if pendingTaskCountMax() in not zero and + * pendingTaskCount() is greater than or equal to pendingTaskCountMax(). If + * this method is called in the context of a ThreadManager worker thread it + * will throw a TooManyPendingTasksException + * + * @param task The task to queue for execution + * + * @param timeout Time to wait in milliseconds to add a task when a + * pending-task-count is specified. Specific cases: + * timeout = 0 : Wait forever to queue task. + * timeout = -1 : Return immediately if pending task count exceeds specified + * max + * @param expiration when nonzero, the number of milliseconds the task is + * valid to be run; if exceeded, the task will be dropped off the queue and + * not run. + * + * @throws TooManyPendingTasksException Pending task count exceeds max + * pending task count + */ + virtual void add(boost::shared_ptrtask, + int64_t timeout=0LL, + int64_t expiration=0LL) = 0; + + /** + * Removes a pending task + */ + virtual void remove(boost::shared_ptr task) = 0; + + /** + * Remove the next pending task which would be run. + * + * @return the task removed. + */ + virtual boost::shared_ptr removeNextPending() = 0; + + /** + * Set a callback to be called when a task is expired and not run. + * + * @param expireCallback a function called with the shared_ptr for + * the expired task. + */ + virtual void setExpireCallback(ExpireCallback expireCallback) = 0; + + /** + * Set a callback to be called when a worker thread is created. + */ + virtual void setThreadInitCallback(InitCallback initCallback) = 0; + + static boost::shared_ptr newThreadManager(); + + /** + * Creates a simple thread manager the uses count number of worker threads + * and has a pendingTaskCountMax maximum pending tasks. The default, 0, + * specified no limit on pending tasks + */ + static boost::shared_ptr + newSimpleThreadManager(size_t count = 4, + size_t pendingTaskCountMax = 0, + bool enableTaskStats = false); + + /** + * Get an internal statistics. + * + * @param waitTimeUs - average time (us) task spent in a queue + * @param runTimeUs - average time (us) task spent running + * @param maxItems - max items collected for stats + */ + virtual void getStats(int64_t& waitTimeUs, int64_t& runTimeUs, + int64_t maxItems) { + waitTimeUs = 0; + runTimeUs = 0; + } + + class Task; + + class Worker; + + class Impl; +}; + +}}} // apache::thrift::concurrency + +#endif // #ifndef _THRIFT_CONCURRENCY_THREADMANAGER_H_ diff --git a/thrift/lib/cpp/concurrency/TimerManager.h b/thrift/lib/cpp/concurrency/TimerManager.h new file mode 100644 index 0000000000..b90a718722 --- /dev/null +++ b/thrift/lib/cpp/concurrency/TimerManager.h @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_CONCURRENCY_TIMERMANAGER_H_ +#define _THRIFT_CONCURRENCY_TIMERMANAGER_H_ 1 + +#include "thrift/lib/cpp/concurrency/Exception.h" +#include "thrift/lib/cpp/concurrency/Monitor.h" +#include "thrift/lib/cpp/concurrency/Thread.h" + +#include +#include +#include + +namespace apache { namespace thrift { namespace concurrency { + +/** + * Timer Manager + * + * This class dispatches timer tasks when they fall due. + * + * @version $Id:$ + */ +class TimerManager { + + public: + + TimerManager(); + + virtual ~TimerManager(); + + virtual boost::shared_ptr threadFactory() const; + + virtual void threadFactory(boost::shared_ptr value); + + /** + * Starts the timer manager service + * + * @throws IllegalArgumentException Missing thread factory attribute + */ + virtual void start(); + + /** + * Stops the timer manager service + */ + virtual void stop(); + + virtual size_t taskCount() const ; + + /** + * Adds a task to be executed at some time in the future by a worker thread. + * + * @param task The task to execute + * @param timeout Time in milliseconds to delay before executing task + */ + virtual void add(boost::shared_ptr task, int64_t timeout); + + /** + * Adds a task to be executed at some time in the future by a worker thread. + * + * @param task The task to execute + * @param timeout Absolute time in the future to execute task. + */ + virtual void add(boost::shared_ptr task, const struct timespec& timeout); + + /** + * Removes a pending task + * + * @throws NoSuchTaskException Specified task doesn't exist. It was either + * processed already or this call was made for a + * task that was never added to this timer + * + * @throws UncancellableTaskException Specified task is already being + * executed or has completed execution. + */ + virtual void remove(boost::shared_ptr task); + + enum STATE { + UNINITIALIZED, + STARTING, + STARTED, + STOPPING, + STOPPED + }; + + virtual const STATE state() const; + + private: + boost::shared_ptr threadFactory_; + class Task; + friend class Task; + std::multimap > taskMap_; + size_t taskCount_; + Monitor monitor_; + STATE state_; + class Dispatcher; + friend class Dispatcher; + boost::shared_ptr dispatcher_; + boost::shared_ptr dispatcherThread_; + typedef std::multimap >::iterator task_iterator; + typedef std::pair task_range; +}; + +}}} // apache::thrift::concurrency + +#endif // #ifndef _THRIFT_CONCURRENCY_TIMERMANAGER_H_ diff --git a/thrift/lib/cpp/concurrency/Util.h b/thrift/lib/cpp/concurrency/Util.h new file mode 100644 index 0000000000..58fc9cd6b7 --- /dev/null +++ b/thrift/lib/cpp/concurrency/Util.h @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_CONCURRENCY_UTIL_H_ +#define _THRIFT_CONCURRENCY_UTIL_H_ 1 + +#include +#include +#include +#include +#include + +namespace apache { namespace thrift { namespace concurrency { + +/** + * Utility methods + * + * This class contains basic utility methods for converting time formats, + * and other common platform-dependent concurrency operations. + * It should not be included in API headers for other concurrency library + * headers, since it will, by definition, pull in all sorts of horrid + * platform dependent crap. Rather it should be included directly in + * concurrency library implementation source. + * + * @version $Id:$ + */ +class Util { + public: + + static const int64_t NS_PER_S = 1000000000LL; + static const int64_t US_PER_S = 1000000LL; + static const int64_t MS_PER_S = 1000LL; + + static const int64_t NS_PER_MS = NS_PER_S / MS_PER_S; + static const int64_t NS_PER_US = NS_PER_S / US_PER_S; + static const int64_t US_PER_MS = US_PER_S / MS_PER_S; + + /** + * Converts millisecond timestamp into a timespec struct + * + * @param struct timespec& result + * @param time or duration in milliseconds + */ + static void toTimespec(struct timespec& result, int64_t value) { + result.tv_sec = value / MS_PER_S; // ms to s + result.tv_nsec = (value % MS_PER_S) * NS_PER_MS; // ms to ns + } + + static void toTimeval(struct timeval& result, int64_t value) { + result.tv_sec = value / MS_PER_S; // ms to s + result.tv_usec = (value % MS_PER_S) * US_PER_MS; // ms to us + } + + static const void toTicks(int64_t& result, int64_t secs, int64_t oldTicks, + int64_t oldTicksPerSec, int64_t newTicksPerSec) { + result = secs * newTicksPerSec; + result += oldTicks * newTicksPerSec / oldTicksPerSec; + + int64_t oldPerNew = oldTicksPerSec / newTicksPerSec; + if (oldPerNew && ((oldTicks % oldPerNew) >= (oldPerNew / 2))) { + ++result; + } + } + /** + * Converts struct timespec to arbitrary-sized ticks since epoch + */ + static const void toTicks(int64_t& result, + const struct timespec& value, + int64_t ticksPerSec) { + return toTicks(result, value.tv_sec, value.tv_nsec, NS_PER_S, ticksPerSec); + } + + /** + * Converts struct timeval to arbitrary-sized ticks since epoch + */ + static const void toTicks(int64_t& result, + const struct timeval& value, + int64_t ticksPerSec) { + return toTicks(result, value.tv_sec, value.tv_usec, US_PER_S, ticksPerSec); + } + + /** + * Converts struct timespec to milliseconds + */ + static const void toMilliseconds(int64_t& result, + const struct timespec& value) { + return toTicks(result, value, MS_PER_S); + } + + /** + * Converts struct timeval to milliseconds + */ + static const void toMilliseconds(int64_t& result, + const struct timeval& value) { + return toTicks(result, value, MS_PER_S); + } + + /** + * Converts struct timespec to microseconds + */ + static const void toUsec(int64_t& result, const struct timespec& value) { + return toTicks(result, value, US_PER_S); + } + + /** + * Converts struct timeval to microseconds + */ + static const void toUsec(int64_t& result, const struct timeval& value) { + return toTicks(result, value, US_PER_S); + } + + /** + * Get current time as a number of arbitrary-size ticks from epoch + */ + static const int64_t currentTimeTicks(int64_t ticksPerSec); + + /** + * Get current time as milliseconds from epoch + */ + static const int64_t currentTime() { return currentTimeTicks(MS_PER_S); } + + /** + * Get current time as micros from epoch + */ + static const int64_t currentTimeUsec() { return currentTimeTicks(US_PER_S); } + + /** + * Get monotonic time as a number of arbitrary-size ticks from some + * unspecified starting point. + * + * This may fall back to the current time (potentially non-monotonic) on + * systems that do not support monotonic time. + */ + static const int64_t monotonicTimeTicks(int64_t ticksPerSec); + + /** + * Get monotonic time as milliseconds. + */ + static const int64_t monotonicTime() { return monotonicTimeTicks(MS_PER_S); } + + /** + * Get current time as micros from epoch + */ + static const int64_t monotonicTimeUsec() { + return monotonicTimeTicks(US_PER_S); + } +}; + +}}} // apache::thrift::concurrency + +#endif // #ifndef _THRIFT_CONCURRENCY_UTIL_H_ diff --git a/thrift/lib/cpp/concurrency/test/TARGETS b/thrift/lib/cpp/concurrency/test/TARGETS new file mode 100644 index 0000000000..3d1b19a231 --- /dev/null +++ b/thrift/lib/cpp/concurrency/test/TARGETS @@ -0,0 +1,45 @@ +# TARGETS file for thrift/lib/cpp/concurrency/test + +cpp_binary ( + name = "test", + srcs = [ + "Tests.cpp" + ], + deps = [ + "@/thrift/lib/cpp/concurrency" + ] +) + +cpp_unittest ( + name = "test_mutex", + srcs = [ + "RWMutexTest.cpp" + ], + deps = [ + "@/thrift/lib/cpp/concurrency", + "@/thrift/lib/cpp:thrift", + ], + external_deps = [ + ('boost', '>= 1.37', 'boost_unit_test_framework'), + ], + type = 'boost', + owner = 'putivsky', + emails = ['adback-dev@lists.facebook.com', + 'thrift-team@lists.facebook.com'], + +) + +cpp_unittest( + name = 'ThreadManagerTests', + srcs = [ + 'ThreadManagerTests.cpp', + ], + deps = [ + '@/thrift/lib/cpp/concurrency', + ], + external_deps = [ + ('boost', '>= 1.37', 'boost_unit_test_framework'), + ], + type = 'boost', + emails = ['thrift-team@lists.facebook.com'], +) diff --git a/thrift/lib/cpp/concurrency/test/ThreadFactoryTests.h b/thrift/lib/cpp/concurrency/test/ThreadFactoryTests.h new file mode 100644 index 0000000000..9f6f75e872 --- /dev/null +++ b/thrift/lib/cpp/concurrency/test/ThreadFactoryTests.h @@ -0,0 +1,354 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "thrift/lib/cpp/config.h" +#include "thrift/lib/cpp/concurrency/Thread.h" +#include "thrift/lib/cpp/concurrency/PosixThreadFactory.h" +#include "thrift/lib/cpp/concurrency/Monitor.h" +#include "thrift/lib/cpp/concurrency/Util.h" + +#include +#include +#include +#include + +namespace apache { namespace thrift { namespace concurrency { namespace test { + +using boost::shared_ptr; +using namespace apache::thrift::concurrency; + +/** + * ThreadManagerTests class + * + * @version $Id:$ + */ +class ThreadFactoryTests { + +public: + + static const double ERROR; + + class Task: public Runnable { + + public: + + Task() {} + + void run() { + std::cout << "\t\t\tHello World" << std::endl; + } + }; + + /** + * Hello world test + */ + bool helloWorldTest() { + + PosixThreadFactory threadFactory = PosixThreadFactory(); + + shared_ptr task = shared_ptr(new ThreadFactoryTests::Task()); + + shared_ptr thread = threadFactory.newThread(task); + + thread->start(); + + thread->join(); + + std::cout << "\t\t\tSuccess!" << std::endl; + + return true; + } + + /** + * Reap N threads + */ + class ReapNTask: public Runnable { + + public: + + ReapNTask(Monitor& monitor, int& activeCount) : + _monitor(monitor), + _count(activeCount) {} + + void run() { + Synchronized s(_monitor); + + _count--; + + //std::cout << "\t\t\tthread count: " << _count << std::endl; + + if (_count == 0) { + _monitor.notify(); + } + } + + Monitor& _monitor; + + int& _count; + }; + + bool reapNThreads(int loop=1, int count=10) { + + PosixThreadFactory threadFactory = PosixThreadFactory(); + + Monitor* monitor = new Monitor(); + + for(int lix = 0; lix < loop; lix++) { + + int* activeCount = new int(count); + + std::set > threads; + + int tix; + + for (tix = 0; tix < count; tix++) { + try { + threads.insert(threadFactory.newThread(shared_ptr(new ReapNTask(*monitor, *activeCount)))); + } catch(SystemResourceException& e) { + std::cout << "\t\t\tfailed to create " << lix * count + tix << " thread " << e.what() << std::endl; + throw e; + } + } + + tix = 0; + for (std::set >::const_iterator thread = threads.begin(); thread != threads.end(); tix++, ++thread) { + + try { + (*thread)->start(); + } catch(SystemResourceException& e) { + std::cout << "\t\t\tfailed to start " << lix * count + tix << " thread " << e.what() << std::endl; + throw e; + } + } + + { + Synchronized s(*monitor); + while (*activeCount > 0) { + monitor->wait(1000); + } + } + + std::cout << "\t\t\treaped " << lix * count << " threads" << std::endl; + } + + std::cout << "\t\t\tSuccess!" << std::endl; + + return true; + } + + class SynchStartTask: public Runnable { + + public: + + enum STATE { + UNINITIALIZED, + STARTING, + STARTED, + STOPPING, + STOPPED + }; + + SynchStartTask(Monitor& monitor, volatile STATE& state) : + _monitor(monitor), + _state(state) {} + + void run() { + { + Synchronized s(_monitor); + if (_state == SynchStartTask::STARTING) { + _state = SynchStartTask::STARTED; + _monitor.notify(); + } + } + + { + Synchronized s(_monitor); + while (_state == SynchStartTask::STARTED) { + _monitor.wait(); + } + + if (_state == SynchStartTask::STOPPING) { + _state = SynchStartTask::STOPPED; + _monitor.notifyAll(); + } + } + } + + private: + Monitor& _monitor; + volatile STATE& _state; + }; + + bool synchStartTest() { + + Monitor monitor; + + SynchStartTask::STATE state = SynchStartTask::UNINITIALIZED; + + shared_ptr task = shared_ptr(new SynchStartTask(monitor, state)); + + PosixThreadFactory threadFactory = PosixThreadFactory(); + + shared_ptr thread = threadFactory.newThread(task); + + if (state == SynchStartTask::UNINITIALIZED) { + + state = SynchStartTask::STARTING; + + thread->start(); + } + + { + Synchronized s(monitor); + while (state == SynchStartTask::STARTING) { + monitor.wait(); + } + } + + assert(state != SynchStartTask::STARTING); + + { + Synchronized s(monitor); + + try { + monitor.wait(100); + } catch(TimedOutException& e) { + } + + if (state == SynchStartTask::STARTED) { + + state = SynchStartTask::STOPPING; + + monitor.notify(); + } + + while (state == SynchStartTask::STOPPING) { + monitor.wait(); + } + } + + assert(state == SynchStartTask::STOPPED); + + bool success = true; + + std::cout << "\t\t\t" << (success ? "Success" : "Failure") << "!" << std::endl; + + return true; + } + + /** See how accurate monitor timeout is. */ + + bool monitorTimeoutTest(size_t count=1000, int64_t timeout=10) { + + Monitor monitor; + + int64_t startTime = Util::currentTime(); + + for (size_t ix = 0; ix < count; ix++) { + { + Synchronized s(monitor); + try { + monitor.wait(timeout); + } catch(TimedOutException& e) { + } + } + } + + int64_t endTime = Util::currentTime(); + + double error = ((endTime - startTime) - (count * timeout)) / (double)(count * timeout); + + if (error < 0.0) { + + error *= 1.0; + } + + bool success = error < ThreadFactoryTests::ERROR; + + std::cout << "\t\t\t" << (success ? "Success" : "Failure") << "! expected time: " << count * timeout << "ms elapsed time: "<< endTime - startTime << "ms error%: " << error * 100.0 << std::endl; + + return success; + } + + + class FloodTask : public Runnable { + public: + + FloodTask(const size_t id) :_id(id) {} + ~FloodTask(){ + if(_id % 1000 == 0) { + std::cout << "\t\tthread " << _id << " done" << std::endl; + } + } + + void run(){ + if(_id % 1000 == 0) { + std::cout << "\t\tthread " << _id << " started" << std::endl; + } + + usleep(1); + } + const size_t _id; + }; + + void foo(PosixThreadFactory *tf) { + } + + bool floodNTest(size_t loop=1, size_t count=100000) { + + bool success = false; + + for(size_t lix = 0; lix < loop; lix++) { + + PosixThreadFactory threadFactory = PosixThreadFactory(); + threadFactory.setDetached(true); + + for(size_t tix = 0; tix < count; tix++) { + + try { + + shared_ptr task(new FloodTask(lix * count + tix )); + + shared_ptr thread = threadFactory.newThread(task); + + thread->start(); + + usleep(1); + + } catch (TException& e) { + + std::cout << "\t\t\tfailed to start " << lix * count + tix << " thread " << e.what() << std::endl; + + return success; + } + } + + std::cout << "\t\t\tflooded " << (lix + 1) * count << " threads" << std::endl; + + success = true; + } + + return success; + } +}; + +const double ThreadFactoryTests::ERROR = .20; + +}}}} // apache::thrift::concurrency::test + diff --git a/thrift/lib/cpp/concurrency/test/TimerManagerTests.h b/thrift/lib/cpp/concurrency/test/TimerManagerTests.h new file mode 100644 index 0000000000..964afbca52 --- /dev/null +++ b/thrift/lib/cpp/concurrency/test/TimerManagerTests.h @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "thrift/lib/cpp/concurrency/TimerManager.h" +#include "thrift/lib/cpp/concurrency/PosixThreadFactory.h" +#include "thrift/lib/cpp/concurrency/Monitor.h" +#include "thrift/lib/cpp/concurrency/Util.h" + +#include +#include + +namespace apache { namespace thrift { namespace concurrency { namespace test { + +using namespace apache::thrift::concurrency; + +/** + * ThreadManagerTests class + * + * @version $Id:$ + */ +class TimerManagerTests { + + public: + + static const double ERROR; + + class Task: public Runnable { + public: + + Task(Monitor& monitor, int64_t timeout) : + _timeout(timeout), + _startTime(Util::currentTime()), + _monitor(monitor), + _success(false), + _done(false) {} + + ~Task() { std::cerr << this << std::endl; } + + void run() { + + _endTime = Util::currentTime(); + + // Figure out error percentage + + int64_t delta = _endTime - _startTime; + + + delta = delta > _timeout ? delta - _timeout : _timeout - delta; + + float error = delta / _timeout; + + if(error < ERROR) { + _success = true; + } + + _done = true; + + std::cout << "\t\t\tTimerManagerTests::Task[" << this << "] done" << std::endl; //debug + + {Synchronized s(_monitor); + _monitor.notifyAll(); + } + } + + int64_t _timeout; + int64_t _startTime; + int64_t _endTime; + Monitor& _monitor; + bool _success; + bool _done; + }; + + /** + * This test creates two tasks and waits for the first to expire within 10% + * of the expected expiration time. It then verifies that the timer manager + * properly clean up itself and the remaining orphaned timeout task when the + * manager goes out of scope and its destructor is called. + */ + bool test00(int64_t timeout=1000LL) { + + shared_ptr orphanTask = shared_ptr(new TimerManagerTests::Task(_monitor, 10 * timeout)); + + { + + TimerManager timerManager; + + timerManager.threadFactory(shared_ptr(new PosixThreadFactory())); + + timerManager.start(); + + assert(timerManager.state() == TimerManager::STARTED); + + shared_ptr task = shared_ptr(new TimerManagerTests::Task(_monitor, timeout)); + + { + Synchronized s(_monitor); + + timerManager.add(orphanTask, 10 * timeout); + + timerManager.add(task, timeout); + + _monitor.wait(); + } + + assert(task->_done); + + + std::cout << "\t\t\t" << (task->_success ? "Success" : "Failure") << "!" << std::endl; + } + + // timerManager.stop(); This is where it happens via destructor + + assert(!orphanTask->_done); + + return true; + } + + friend class TestTask; + + Monitor _monitor; +}; + +const double TimerManagerTests::ERROR = .20; + +}}}} // apache::thrift::concurrency + diff --git a/thrift/lib/cpp/config.h b/thrift/lib/cpp/config.h new file mode 100644 index 0000000000..786524271c --- /dev/null +++ b/thrift/lib/cpp/config.h @@ -0,0 +1,316 @@ +/* config.h. Generated by configure. */ +/* + * Generated by simpkins on 2009-11-18, + * on an x86_64 CentOS 5.2 system. + */ +/* config.hin. Generated from configure.ac by autoheader. */ + +/* Define if the AI_ADDRCONFIG symbol is unavailable */ +/* #undef AI_ADDRCONFIG */ + +/* Possible value for SIGNED_RIGHT_SHIFT_IS */ +#define ARITHMETIC_RIGHT_SHIFT 1 + +/* Define to one of `_getb67', `GETB67', `getb67' for Cray-2 and Cray-YMP + systems. This function is required for `alloca.c' support on those systems. + */ +/* #undef CRAY_STACKSEG_END */ + +/* Define to 1 if using `alloca.c'. */ +/* #undef C_ALLOCA */ + +/* Define to 1 if you have `alloca', as a function or macro. */ +#define HAVE_ALLOCA 1 + +/* Define to 1 if you have and it should be used (not on Ultrix). + */ +#define HAVE_ALLOCA_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_INET_H 1 + +/* define if the Boost library is available */ +#define HAVE_BOOST + +/* Define to 1 if you have the `bzero' function. */ +#define HAVE_BZERO 1 + +/* Define to 1 if you have the `clock_gettime' function. */ +#define HAVE_CLOCK_GETTIME 1 + +/* Define to 1 if you have the declaration of `strerror_r', and to 0 if you + don't. */ +#define HAVE_DECL_STRERROR_R 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_DLFCN_H 1 + +/* Define to 1 if you don't have `vprintf' but do have `_doprnt.' */ +/* #undef HAVE_DOPRNT */ + +/* Define to 1 if you have the header file. */ +#define HAVE_ENDIAN_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_FCNTL_H 1 + +/* Define to 1 if you have the `ftruncate' function. */ +#define HAVE_FTRUNCATE 1 + +/* Define to 1 if you have the `gethostbyname' function. */ +#define HAVE_GETHOSTBYNAME 1 + +/* Define to 1 if you have the `gettimeofday' function. */ +#define HAVE_GETTIMEOFDAY 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_INTTYPES_H 1 + +/* define if libevent is available */ +#define HAVE_LIBEVENT + +/* Define to 1 if you have the header file. */ +#define HAVE_LIBINTL_H 1 + +/* Define to 1 if you have the `pthread' library (-lpthread). */ +#define HAVE_LIBPTHREAD 1 + +/* Define to 1 if you have the `rt' library (-lrt). */ +#define HAVE_LIBRT 1 + +/* Define to 1 if you have the `socket' library (-lsocket). */ +/* #undef HAVE_LIBSOCKET */ + +/* Define to 1 if you have the header file. */ +#define HAVE_LIMITS_H 1 + +/* Define to 1 if your system has a GNU libc compatible `malloc' function, and + to 0 otherwise. */ +#define HAVE_MALLOC 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_MALLOC_H 1 + +/* Define to 1 if you have the `memmove' function. */ +#define HAVE_MEMMOVE 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_MEMORY_H 1 + +/* Define to 1 if you have the `memset' function. */ +#define HAVE_MEMSET 1 + +/* Define to 1 if you have the `mkdir' function. */ +#define HAVE_MKDIR 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_NETDB_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_NETINET_IN_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_PTHREAD_H 1 + +/* Define to 1 if the system has the type `ptrdiff_t'. */ +#define HAVE_PTRDIFF_T 1 + +/* Define to 1 if your system has a GNU libc compatible `realloc' function, + and to 0 otherwise. */ +#define HAVE_REALLOC 1 + +/* Define to 1 if you have the `realpath' function. */ +#define HAVE_REALPATH 1 + +/* Define to 1 if you have the `sched_get_priority_max' function. */ +#define HAVE_SCHED_GET_PRIORITY_MAX 1 + +/* Define to 1 if you have the `sched_get_priority_min' function. */ +#define HAVE_SCHED_GET_PRIORITY_MIN 1 + +/* Define to 1 if you have the `select' function. */ +#define HAVE_SELECT 1 + +/* Define to 1 if you have the `socket' function. */ +#define HAVE_SOCKET 1 + +/* Define to 1 if you have the `sqrt' function. */ +#define HAVE_SQRT 1 + +/* Define to 1 if `stat' has the bug that it succeeds when given the + zero-length file name argument. */ +/* #undef HAVE_STAT_EMPTY_STRING_BUG */ + +/* Define to 1 if stdbool.h conforms to C99. */ +#define HAVE_STDBOOL_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDDEF_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDINT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDLIB_H 1 + +/* Define to 1 if you have the `strchr' function. */ +#define HAVE_STRCHR 1 + +/* Define to 1 if you have the `strdup' function. */ +#define HAVE_STRDUP 1 + +/* Define to 1 if you have the `strerror' function. */ +#define HAVE_STRERROR 1 + +/* Define to 1 if you have the `strerror_r' function. */ +#define HAVE_STRERROR_R 1 + +/* Define to 1 if you have the `strftime' function. */ +#define HAVE_STRFTIME 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRINGS_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRING_H 1 + +/* Define to 1 if you have the `strstr' function. */ +#define HAVE_STRSTR 1 + +/* Define to 1 if you have the `strtol' function. */ +#define HAVE_STRTOL 1 + +/* Define to 1 if you have the `strtoul' function. */ +#define HAVE_STRTOUL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SELECT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SOCKET_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_STAT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TIME_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TYPES_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_UNISTD_H 1 + +/* Define to 1 if you have the `vprintf' function. */ +#define HAVE_VPRINTF 1 + +/* define if zlib is available */ +#define HAVE_ZLIB + +/* Define to 1 if the system has the type `_Bool'. */ +/* #undef HAVE__BOOL */ + +/* Possible value for SIGNED_RIGHT_SHIFT_IS */ +#define LOGICAL_RIGHT_SHIFT 2 + +/* Define to 1 if `lstat' dereferences a symlink specified with a trailing + slash. */ +#define LSTAT_FOLLOWS_SLASHED_SYMLINK 1 + +/* Name of package */ +#define PACKAGE "thrift" + +/* Define to the address where bug reports for this package should be sent. */ +#define PACKAGE_BUGREPORT "" + +/* Define to the full name of this package. */ +#define PACKAGE_NAME "thrift" + +/* Define to the full name and version of this package. */ +#define PACKAGE_STRING "thrift 20080411" + +/* Define to the one symbol short name of this package. */ +#define PACKAGE_TARNAME "thrift" + +/* Define to the version of this package. */ +#define PACKAGE_VERSION "20080411" + +/* Define to the type of arg 1 for `select'. */ +#define SELECT_TYPE_ARG1 int + +/* Define to the type of args 2, 3 and 4 for `select'. */ +#define SELECT_TYPE_ARG234 (fd_set *) + +/* Define to the type of arg 5 for `select'. */ +#define SELECT_TYPE_ARG5 (struct timeval *) + +/* Indicates the effect of the right shift operator on negative signed + integers */ +#define SIGNED_RIGHT_SHIFT_IS 1 + +/* If using the C implementation of alloca, define if you know the + direction of stack growth for your system; otherwise it will be + automatically deduced at run-time. + STACK_DIRECTION > 0 => grows toward higher addresses + STACK_DIRECTION < 0 => grows toward lower addresses + STACK_DIRECTION = 0 => direction of growth unknown */ +/* #undef STACK_DIRECTION */ + +/* Define to 1 if you have the ANSI C header files. */ +#define STDC_HEADERS 1 + +/* Define to 1 if strerror_r returns char *. */ +#define STRERROR_R_CHAR_P 1 + +/* Define to 1 if you can safely include both and . */ +#define TIME_WITH_SYS_TIME 1 + +/* Define to 1 if your declares `struct tm'. */ +/* #undef TM_IN_SYS_TIME */ + +/* Possible value for SIGNED_RIGHT_SHIFT_IS */ +#define UNKNOWN_RIGHT_SHIFT 3 + +/* Version number of package */ +#define VERSION "20080411" + +/* Define to 1 if `lex' declares `yytext' as a `char *' by default, not a + `char[]'. */ +#define YYTEXT_POINTER 1 + +/* Calls Google Perftools ProfilerRegisterThread() on pthread creation + + Set define to 0. It is exposing a linux 6.12 posix bug when an + application receives a SIGKILL, which causes the machine to crash. + */ +#ifndef GOOGLE_PERFTOOLS_REGISTER_THREAD +# define GOOGLE_PERFTOOLS_REGISTER_THREAD 0 +#endif + +/* Define to empty if `const' does not conform to ANSI C. */ +/* #undef const */ + +/* Define to `__inline__' or `__inline' if that's what the C compiler + calls it, or to nothing if 'inline' is not supported under any name. */ +#ifndef __cplusplus +/* #undef inline */ +#endif + +/* Define to rpl_malloc if the replacement function should be used. */ +/* #undef malloc */ + +/* Define to `int' if does not define. */ +/* #undef mode_t */ + +/* Define to `long' if does not define. */ +/* #undef off_t */ + +/* Define to rpl_realloc if the replacement function should be used. */ +/* #undef realloc */ + +/* Define to `unsigned' if does not define. */ +/* #undef size_t */ + +/* Define to empty if the keyword `volatile' does not work. Warning: valid + code using `volatile' can become incorrect without. Disable with care. */ +/* #undef volatile */ diff --git a/thrift/lib/cpp/processor/PeekProcessor.h b/thrift/lib/cpp/processor/PeekProcessor.h new file mode 100644 index 0000000000..9058252e47 --- /dev/null +++ b/thrift/lib/cpp/processor/PeekProcessor.h @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef PEEKPROCESSOR_H +#define PEEKPROCESSOR_H + +#include +#include "thrift/lib/cpp/TProcessor.h" +#include "thrift/lib/cpp/transport/TTransport.h" +#include "thrift/lib/cpp/transport/TTransportUtils.h" +#include "thrift/lib/cpp/transport/TBufferTransports.h" +#include + +namespace apache { namespace thrift { namespace processor { + +namespace server { +class TConnectionContext; +} + +/* + * Class for peeking at the raw data that is being processed by another processor + * and gives the derived class a chance to change behavior accordingly + * + */ +class PeekProcessor : public apache::thrift::TProcessor { + + public: + PeekProcessor(); + virtual ~PeekProcessor(); + + // Input here: actualProcessor - the underlying processor + // protocolFactory - the protocol factory used to wrap the memory buffer + // transportFactory - this TPipedTransportFactory is used to wrap the source transport + // via a call to getPipedTransport + void initialize(boost::shared_ptr actualProcessor, + boost::shared_ptr protocolFactory, + boost::shared_ptr transportFactory); + + boost::shared_ptr getPipedTransport(boost::shared_ptr in); + + void setTargetTransport(boost::shared_ptr targetTransport); + + virtual bool process(boost::shared_ptr in, + boost::shared_ptr out, + TConnectionContext* connectionContext); + + // The following three functions can be overloaded by child classes to + // achieve desired peeking behavior + virtual void peekName(const std::string& fname); + virtual void peekBuffer(uint8_t* buffer, uint32_t size); + virtual void peek(boost::shared_ptr in, + apache::thrift::protocol::TType ftype, + int16_t fid); + virtual void peekEnd(); + + private: + boost::shared_ptr actualProcessor_; + boost::shared_ptr pipedProtocol_; + boost::shared_ptr transportFactory_; + boost::shared_ptr memoryBuffer_; + boost::shared_ptr targetTransport_; +}; + +}}} // apache::thrift::processor + +#endif diff --git a/thrift/lib/cpp/processor/StatsProcessor.h b/thrift/lib/cpp/processor/StatsProcessor.h new file mode 100644 index 0000000000..6197adf6ec --- /dev/null +++ b/thrift/lib/cpp/processor/StatsProcessor.h @@ -0,0 +1,266 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef STATSPROCESSOR_H +#define STATSPROCESSOR_H + +#include +#include "thrift/lib/cpp/transport/TTransport.h" +#include "thrift/lib/cpp/protocol/TProtocol.h" +#include "thrift/lib/cpp/TProcessor.h" + +namespace apache { namespace thrift { namespace processor { + +/* + * Class for keeping track of function call statistics and printing them if desired + * + */ +class StatsProcessor : public apache::thrift::TProcessor { +public: + StatsProcessor(bool print, bool frequency) + : print_(print), + frequency_(frequency) + {} + virtual ~StatsProcessor() {}; + + virtual bool process(boost::shared_ptr piprot, + boost::shared_ptr poprot, + void* serverContext) { + + piprot_ = piprot; + + std::string fname; + apache::thrift::protocol::TMessageType mtype; + int32_t seqid; + + piprot_->readMessageBegin(fname, mtype, seqid); + if (mtype != apache::thrift::protocol::T_CALL) { + if (print_) { + printf("Unknown message type\n"); + } + throw apache::thrift::TLibraryException("Unexpected message type"); + } + if (print_) { + printf("%s (", fname.c_str()); + } + if (frequency_) { + if (frequency_map_.find(fname) != frequency_map_.end()) { + frequency_map_[fname]++; + } else { + frequency_map_[fname] = 1; + } + } + + apache::thrift::protocol::TType ftype; + int16_t fid; + + while (true) { + piprot_->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + + printAndPassToBuffer(ftype); + if (print_) { + printf(", "); + } + } + + if (print_) { + printf("\b\b)\n"); + } + return true; + } + + const std::map& get_frequency_map() { + return frequency_map_; + } + +protected: + void printAndPassToBuffer(apache::thrift::protocol::TType ftype) { + switch (ftype) { + case apache::thrift::protocol::T_BOOL: + { + bool boolv; + piprot_->readBool(boolv); + if (print_) { + printf("%d", boolv); + } + } + break; + case apache::thrift::protocol::T_BYTE: + { + int8_t bytev; + piprot_->readByte(bytev); + if (print_) { + printf("%d", bytev); + } + } + break; + case apache::thrift::protocol::T_I16: + { + int16_t i16; + piprot_->readI16(i16); + if (print_) { + printf("%d", i16); + } + } + break; + case apache::thrift::protocol::T_I32: + { + int32_t i32; + piprot_->readI32(i32); + if (print_) { + printf("%d", i32); + } + } + break; + case apache::thrift::protocol::T_I64: + { + int64_t i64; + piprot_->readI64(i64); + if (print_) { + printf("%ld", i64); + } + } + break; + case apache::thrift::protocol::T_DOUBLE: + { + double dub; + piprot_->readDouble(dub); + if (print_) { + printf("%f", dub); + } + } + break; + case apache::thrift::protocol::T_STRING: + { + std::string str; + piprot_->readString(str); + if (print_) { + printf("%s", str.c_str()); + } + } + break; + case apache::thrift::protocol::T_STRUCT: + { + std::string name; + int16_t fid; + apache::thrift::protocol::TType ftype; + piprot_->readStructBegin(name); + if (print_) { + printf("<"); + } + while (true) { + piprot_->readFieldBegin(name, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + printAndPassToBuffer(ftype); + if (print_) { + printf(","); + } + piprot_->readFieldEnd(); + } + piprot_->readStructEnd(); + if (print_) { + printf("\b>"); + } + } + break; + case apache::thrift::protocol::T_MAP: + { + apache::thrift::protocol::TType keyType; + apache::thrift::protocol::TType valType; + uint32_t i, size; + piprot_->readMapBegin(keyType, valType, size); + if (print_) { + printf("{"); + } + for (i = 0; i < size; i++) { + printAndPassToBuffer(keyType); + if (print_) { + printf("=>"); + } + printAndPassToBuffer(valType); + if (print_) { + printf(","); + } + } + piprot_->readMapEnd(); + if (print_) { + printf("\b}"); + } + } + break; + case apache::thrift::protocol::T_SET: + { + apache::thrift::protocol::TType elemType; + uint32_t i, size; + piprot_->readSetBegin(elemType, size); + if (print_) { + printf("{"); + } + for (i = 0; i < size; i++) { + printAndPassToBuffer(elemType); + if (print_) { + printf(","); + } + } + piprot_->readSetEnd(); + if (print_) { + printf("\b}"); + } + } + break; + case apache::thrift::protocol::T_LIST: + { + apache::thrift::protocol::TType elemType; + uint32_t i, size; + piprot_->readListBegin(elemType, size); + if (print_) { + printf("["); + } + for (i = 0; i < size; i++) { + printAndPassToBuffer(elemType); + if (print_) { + printf(","); + } + } + piprot_->readListEnd(); + if (print_) { + printf("\b]"); + } + } + break; + default: + break; + } + } + + boost::shared_ptr piprot_; + std::map frequency_map_; + + bool print_; + bool frequency_; +}; + +}}} // apache::thrift::processor + +#endif diff --git a/thrift/lib/cpp/processor/TARGETS b/thrift/lib/cpp/processor/TARGETS new file mode 100644 index 0000000000..d5f363053f --- /dev/null +++ b/thrift/lib/cpp/processor/TARGETS @@ -0,0 +1,9 @@ +# TARGETS file for thrift/lib/cpp/processor + +cpp_library ( + name = "processor", + srcs = [ + "PeekProcessor.cpp" + ], + deps = [ "@/thrift/lib/cpp/transport" ] +) diff --git a/thrift/lib/cpp/processor/test/EventLog.h b/thrift/lib/cpp/processor/test/EventLog.h new file mode 100644 index 0000000000..50d0d87164 --- /dev/null +++ b/thrift/lib/cpp/processor/test/EventLog.h @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef _THRIFT_TEST_EVENTLOG_H_ +#define _THRIFT_TEST_EVENTLOG_H_ 1 + +#include "thrift/lib/cpp/concurrency/Monitor.h" + +namespace apache { namespace thrift { namespace test { + +// Initially I made EventType an enum, but using char* results +// in much more readable error messages when there is a mismatch. +// It also lets users of EventLog easily define their own new types. +// Comparing the literal pointer values should be safe, barring any strange +// linking setup that results in duplicate symbols. +typedef const char* EventType; + +struct Event { + Event(EventType type, uint32_t connectionId, uint32_t callId, + const std::string& message) : + type(type), + connectionId(connectionId), + callId(callId), + message(message) {} + + EventType type; + uint32_t connectionId; + uint32_t callId; + std::string message; +}; + +class EventLog { + public: + static EventType ET_LOG_END; + static EventType ET_CONN_CREATED; + static EventType ET_CONN_DESTROYED; + static EventType ET_CALL_STARTED; + static EventType ET_CALL_FINISHED; + static EventType ET_PROCESS; + static EventType ET_PRE_READ; + static EventType ET_POST_READ; + static EventType ET_PRE_WRITE; + static EventType ET_POST_WRITE; + static EventType ET_ASYNC_COMPLETE; + static EventType ET_HANDLER_ERROR; + + static EventType ET_CALL_INCREMENT_GENERATION; + static EventType ET_CALL_GET_GENERATION; + static EventType ET_CALL_ADD_STRING; + static EventType ET_CALL_GET_STRINGS; + static EventType ET_CALL_GET_DATA_WAIT; + static EventType ET_CALL_ONEWAY_WAIT; + static EventType ET_CALL_UNEXPECTED_EXCEPTION_WAIT; + static EventType ET_CALL_EXCEPTION_WAIT; + static EventType ET_WAIT_RETURN; + static EventType ET_CALL_SET_VALUE; + static EventType ET_CALL_GET_VALUE; + + EventLog(); + + void append(EventType type, uint32_t connectionId, uint32_t callId, + const std::string& message = ""); + + Event waitForEvent(int64_t timeout = 500); + Event waitForConnEvent(uint32_t connId, int64_t timeout = 500); + + protected: + typedef std::list EventList; + + concurrency::Monitor monitor_; + EventList events_; + uint32_t id_; + + static uint32_t nextId_; +}; + +}}} // apache::thrift::test + +#endif // _THRIFT_TEST_EVENTLOG_H_ diff --git a/thrift/lib/cpp/processor/test/Handlers.h b/thrift/lib/cpp/processor/test/Handlers.h new file mode 100644 index 0000000000..4dc0e68b2b --- /dev/null +++ b/thrift/lib/cpp/processor/test/Handlers.h @@ -0,0 +1,311 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef _THRIFT_PROCESSOR_TEST_HANDLERS_H_ +#define _THRIFT_PROCESSOR_TEST_HANDLERS_H_ 1 + +#include "thrift/lib/cpp/processor/test/EventLog.h" +#include "thrift/lib/cpp/processor/test/gen-cpp/ChildService.h" + +#include "thrift/lib/cpp/server/TConnectionContext.h" + +namespace apache { namespace thrift { namespace test { + +class ParentHandler : virtual public ParentServiceIf { + public: + ParentHandler(const boost::shared_ptr& log) : + triggerMonitor(&mutex_), + generation_(0), + wait_(false), + log_(log) { } + + int32_t incrementGeneration() { + concurrency::Guard g(mutex_); + log_->append(EventLog::ET_CALL_INCREMENT_GENERATION, 0, 0); + return ++generation_; + } + + int32_t getGeneration() { + concurrency::Guard g(mutex_); + log_->append(EventLog::ET_CALL_GET_GENERATION, 0, 0); + return generation_; + } + + void addString(const std::string& s) { + concurrency::Guard g(mutex_); + log_->append(EventLog::ET_CALL_ADD_STRING, 0, 0); + strings_.push_back(s); + } + + void getStrings(std::vector& _return) { + concurrency::Guard g(mutex_); + log_->append(EventLog::ET_CALL_GET_STRINGS, 0, 0); + _return = strings_; + } + + void getDataWait(std::string& _return, int32_t length) { + concurrency::Guard g(mutex_); + log_->append(EventLog::ET_CALL_GET_DATA_WAIT, 0, 0); + + blockUntilTriggered(); + + _return.append(length, 'a'); + } + + void onewayWait() { + concurrency::Guard g(mutex_); + log_->append(EventLog::ET_CALL_ONEWAY_WAIT, 0, 0); + + blockUntilTriggered(); + } + + void exceptionWait(const std::string& message) { + concurrency::Guard g(mutex_); + log_->append(EventLog::ET_CALL_EXCEPTION_WAIT, 0, 0); + + blockUntilTriggered(); + + MyError e; + e.message = message; + throw e; + } + + void unexpectedExceptionWait(const std::string& message) { + concurrency::Guard g(mutex_); + log_->append(EventLog::ET_CALL_UNEXPECTED_EXCEPTION_WAIT, 0, 0); + + blockUntilTriggered(); + + MyError e; + e.message = message; + throw e; + } + + /** + * After prepareTriggeredCall() is invoked, calls to any of the *Wait() + * functions won't return until triggerPendingCalls() is invoked + * + * This has to be a separate function invoked by the main test thread + * in order to to avoid race conditions. + */ + void prepareTriggeredCall() { + concurrency::Guard g(mutex_); + wait_ = true; + } + + /** + * Wake up all calls waiting in blockUntilTriggered() + */ + void triggerPendingCalls() { + concurrency::Guard g(mutex_); + wait_ = false; + triggerMonitor.notifyAll(); + } + + protected: + /** + * blockUntilTriggered() won't return until triggerPendingCalls() is invoked + * in another thread. + * + * This should only be called when already holding mutex_. + */ + void blockUntilTriggered() { + while (wait_) { + triggerMonitor.waitForever(); + } + + // Log an event when we return + log_->append(EventLog::ET_WAIT_RETURN, 0, 0); + } + + concurrency::Mutex mutex_; + concurrency::Monitor triggerMonitor; + int32_t generation_; + bool wait_; + std::vector strings_; + boost::shared_ptr log_; +}; + +class ChildHandler : public ParentHandler, virtual public ChildServiceIf { + public: + ChildHandler(const boost::shared_ptr& log) : + ParentHandler(log), + value_(0) {} + + int32_t setValue(int32_t value) { + concurrency::Guard g(mutex_); + log_->append(EventLog::ET_CALL_SET_VALUE, 0, 0); + + int32_t oldValue = value_; + value_ = value; + return oldValue; + } + + int32_t getValue() { + concurrency::Guard g(mutex_); + log_->append(EventLog::ET_CALL_GET_VALUE, 0, 0); + + return value_; + } + + protected: + int32_t value_; +}; + +struct ConnContext { + public: + ConnContext(server::TConnectionContext* ctx, uint32_t id) : + ctx(ctx), + id(id) {} + + server::TConnectionContext* ctx; + uint32_t id; +}; + +struct CallContext { + public: + CallContext(ConnContext *context, uint32_t id, const std::string& name) : + connContext(context), + name(name), + id(id) {} + + ConnContext *connContext; + std::string name; + uint32_t id; +}; + +class ServerEventHandler : public server::TServerEventHandler { + public: + ServerEventHandler(const boost::shared_ptr& log) : + nextId_(1), + log_(log) {} + + virtual void preServe(const transport::TSocketAddress*) {} + + virtual void newConnection(server::TConnectionContext* ctx) { + ConnContext* context = new ConnContext(ctx, nextId_); + ++nextId_; + ctx->setUserData(context); + log_->append(EventLog::ET_CONN_CREATED, context->id, 0); + } + + virtual void connectionDestroyed(server::TConnectionContext* ctx) { + ConnContext* context = static_cast(ctx->getUserData()); + + if (ctx != context->ctx) { + abort(); + } + + log_->append(EventLog::ET_CONN_DESTROYED, context->id, 0); + + delete context; + } + + protected: + uint32_t nextId_; + boost::shared_ptr log_; +}; + +class ProcessorEventHandler : public TProcessorEventHandler { + public: + ProcessorEventHandler(const boost::shared_ptr& log) : + nextId_(1), + log_(log) {} + + void* getContext(const char* fnName, TConnectionContext* serverContext) { + ConnContext* connContext = + reinterpret_cast(serverContext->getUserData()); + + CallContext* context = new CallContext(connContext, nextId_, fnName); + ++nextId_; + + log_->append(EventLog::ET_CALL_STARTED, connContext->id, context->id, + fnName); + return context; + } + + void freeContext(void* ctx, const char* fnName) { + CallContext* context = reinterpret_cast(ctx); + checkName(context, fnName); + log_->append(EventLog::ET_CALL_FINISHED, context->connContext->id, + context->id, fnName); + delete context; + } + + void preRead(void* ctx, const char* fnName) { + CallContext* context = reinterpret_cast(ctx); + checkName(context, fnName); + log_->append(EventLog::ET_PRE_READ, context->connContext->id, context->id, + fnName); + } + + void postRead(void* ctx, const char* fnName, uint32_t bytes) { + CallContext* context = reinterpret_cast(ctx); + checkName(context, fnName); + log_->append(EventLog::ET_POST_READ, context->connContext->id, context->id, + fnName); + } + + void preWrite(void* ctx, const char* fnName) { + CallContext* context = reinterpret_cast(ctx); + checkName(context, fnName); + log_->append(EventLog::ET_PRE_WRITE, context->connContext->id, context->id, + fnName); + } + + void postWrite(void* ctx, const char* fnName, uint32_t bytes) { + CallContext* context = reinterpret_cast(ctx); + checkName(context, fnName); + log_->append(EventLog::ET_POST_WRITE, context->connContext->id, + context->id, fnName); + } + + void asyncComplete(void* ctx, const char* fnName) { + CallContext* context = reinterpret_cast(ctx); + checkName(context, fnName); + log_->append(EventLog::ET_ASYNC_COMPLETE, context->connContext->id, + context->id, fnName); + } + + void handlerError(void* ctx, const char* fnName) { + CallContext* context = reinterpret_cast(ctx); + checkName(context, fnName); + log_->append(EventLog::ET_HANDLER_ERROR, context->connContext->id, + context->id, fnName); + } + + protected: + void checkName(const CallContext* context, const char* fnName) { + // Note: we can't use BOOST_CHECK_EQUAL here, since the handler runs in a + // different thread from the test functions. Just abort if the names are + // different + if (context->name != fnName) { + fprintf(stderr, "call context name mismatch: \"%s\" != \"%s\"\n", + context->name.c_str(), fnName); + fflush(stderr); + abort(); + } + } + + uint32_t nextId_; + boost::shared_ptr log_; +}; + +}}} // apache::thrift::test + +#endif // _THRIFT_PROCESSOR_TEST_HANDLERS_H_ diff --git a/thrift/lib/cpp/processor/test/ServerThread.h b/thrift/lib/cpp/processor/test/ServerThread.h new file mode 100644 index 0000000000..c5fcb39c36 --- /dev/null +++ b/thrift/lib/cpp/processor/test/ServerThread.h @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef _THRIFT_TEST_SERVERTHREAD_H_ +#define _THRIFT_TEST_SERVERTHREAD_H_ 1 + +#include "thrift/lib/cpp/TProcessor.h" +#include "thrift/lib/cpp/protocol/TProtocol.h" +#include "thrift/lib/cpp/server/TServer.h" +#include "thrift/lib/cpp/transport/TTransport.h" + +#include "thrift/lib/cpp/processor/test/EventLog.h" + +namespace apache { namespace thrift { namespace test { + +/** + * A helper class to tell ServerThread how to create the server + */ +class ServerState { + public: + virtual ~ServerState() {} + + /** + * Create a server to listen on the specified port. + * + * If the server returned fails to bind to the specified port when serve() is + * called on it, createServer() may be called again on a different port. + */ + virtual boost::shared_ptr createServer(uint16_t port) = 0; + + /** + * Get the TServerEventHandler to set on the server. + * + * This is only called after the server successfully binds and is about to + * start serving traffic. It is invoked from the server thread, rather than + * the main thread. + */ + virtual boost::shared_ptr + getServerEventHandler() { + return boost::shared_ptr(); + } + + /** + * This method is called in the server thread after server binding succeeds. + * + * Subclasses may override this method if they wish to record the final + * port that was used for the server. + */ + virtual void bindSuccessful(uint16_t port) { + } +}; + +/** + * ServerThread starts a thrift server running in a separate thread. + */ +class ServerThread { + public: + ServerThread(const boost::shared_ptr& state, bool autoStart) : + helper_(new Helper(this)), + port_(0), + running_(false), + serving_(false), + error_(false), + serverState_(state) { + if (autoStart) { + start(); + } + } + + void start(); + void stop(); + + uint16_t getPort() const { + return port_; + } + + ~ServerThread() { + if (running_) { + try { + stop(); + } catch (...) { + GlobalOutput.printf("error shutting down server"); + } + } + } + + protected: + // Annoying. thrift forces us to use shared_ptr, so we have to use + // a helper class that we can allocate on the heap and give to thrift. + // It would be simpler if we could just make Runnable and TServerEventHandler + // private base classes of ServerThread. + class Helper : public concurrency::Runnable, + public server::TServerEventHandler { + public: + Helper(ServerThread* serverThread) + : serverThread_(serverThread) {} + + void run() { + serverThread_->run(); + } + + void preServe(const transport::TSocketAddress* address) { + serverThread_->preServe(address); + } + + private: + ServerThread* serverThread_; + }; + + void run(); + void preServe(const transport::TSocketAddress* address); + + boost::shared_ptr helper_; + + uint16_t port_; + bool running_; + bool serving_; + bool error_; + concurrency::Monitor serverMonitor_; + + boost::shared_ptr serverState_; + boost::shared_ptr server_; + boost::shared_ptr thread_; +}; + +}}} // apache::thrift::test + +#endif // _THRIFT_TEST_SERVERTHREAD_H_ diff --git a/thrift/lib/cpp/processor/test/TARGETS b/thrift/lib/cpp/processor/test/TARGETS new file mode 100644 index 0000000000..9cb4d8b224 --- /dev/null +++ b/thrift/lib/cpp/processor/test/TARGETS @@ -0,0 +1,24 @@ +cpp_unittest( + name = 'ProcessorTest', + srcs = [ + 'EventLog.cpp', + 'ProcessorTest.cpp', + 'ServerThread.cpp', + ], + deps = [ + ':thrift', + '@/thrift/lib/cpp/server:nonblocking', + ], + external_deps = [ + # Versions of boost lower than 1.37 don't support BOOST_AUTO_TEST_CASE. + ('boost', '>= 1.37', 'boost_unit_test_framework'), + ], + type = 'boost', + emails = ['thrift-team@lists.facebook.com'], +) + +cpp_library( + name = 'thrift', + thrift_srcs = { 'proc.thrift' : ['ParentService', 'ChildService']}, + thrift_cpp_options = 'templates', +) diff --git a/thrift/lib/cpp/processor/test/proc.thrift b/thrift/lib/cpp/processor/test/proc.thrift new file mode 100644 index 0000000000..ac3c5f9536 --- /dev/null +++ b/thrift/lib/cpp/processor/test/proc.thrift @@ -0,0 +1,22 @@ +namespace cpp apache.thrift.test + +exception MyError { + 1: string message +} + +service ParentService { + i32 incrementGeneration() + i32 getGeneration() + void addString(1: string s) + list getStrings() + + binary getDataWait(1: i32 length) + oneway void onewayWait() + void exceptionWait(1: string message) throws (2: MyError error) + void unexpectedExceptionWait(1: string message) +} + +service ChildService extends ParentService { + i32 setValue(1: i32 value) + i32 getValue() +} diff --git a/thrift/lib/cpp/protocol/TARGETS b/thrift/lib/cpp/protocol/TARGETS new file mode 100644 index 0000000000..cb128d0684 --- /dev/null +++ b/thrift/lib/cpp/protocol/TARGETS @@ -0,0 +1,18 @@ +# TARGETS file for thrift/lib/cpp/protocol + +cpp_library ( + name = "protocol", + srcs = [ + "TBase64Utils.cpp", + "TDebugProtocol.cpp", + "TJSONProtocol.cpp", + "TSimpleJSONProtocol.cpp", + "THeaderProtocol.cpp", + "TPhpSerializeProtocol.cpp", + ], + deps = [ + "@/thrift/lib/cpp/transport", + "@/thrift/lib/cpp/transport:header", + "@/thrift/lib/cpp:thrift_exception", + ], +) diff --git a/thrift/lib/cpp/protocol/TBase64Utils.h b/thrift/lib/cpp/protocol/TBase64Utils.h new file mode 100644 index 0000000000..3def733501 --- /dev/null +++ b/thrift/lib/cpp/protocol/TBase64Utils.h @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_PROTOCOL_TBASE64UTILS_H_ +#define _THRIFT_PROTOCOL_TBASE64UTILS_H_ + +#include +#include + +namespace apache { namespace thrift { namespace protocol { + +// in must be at least len bytes +// len must be 1, 2, or 3 +// buf must be a buffer of at least 4 bytes and may not overlap in +// the data is not padded with '='; the caller can do this if desired +void base64_encode(const uint8_t *in, uint32_t len, uint8_t *buf); + +// buf must be a buffer of at least 4 bytes and contain base64 encoded values +// buf will be changed to contain output bytes +// len is number of bytes to consume from input (must be 2, 3, or 4) +// no '=' padding should be included in the input +void base64_decode(uint8_t *buf, uint32_t len); + +}}} // apache::thrift::protocol + +#endif // #define _THRIFT_PROTOCOL_TBASE64UTILS_H_ diff --git a/thrift/lib/cpp/protocol/TBinaryProtocol.h b/thrift/lib/cpp/protocol/TBinaryProtocol.h new file mode 100644 index 0000000000..9a98d26c17 --- /dev/null +++ b/thrift/lib/cpp/protocol/TBinaryProtocol.h @@ -0,0 +1,302 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef THRIFT_PROTOCOL_TBINARYPROTOCOL_H_ +#define THRIFT_PROTOCOL_TBINARYPROTOCOL_H_ 1 + +#include "thrift/lib/cpp/protocol/TProtocol.h" +#include "thrift/lib/cpp/protocol/TVirtualProtocol.h" + +#include + +namespace apache { namespace thrift { namespace protocol { + +/** + * The default binary protocol for thrift. Writes all data in a very basic + * binary format, essentially just spitting out the raw bytes. + * + */ +template +class TBinaryProtocolT + : public TVirtualProtocol< TBinaryProtocolT > { + + public: + static const int32_t VERSION_MASK = 0xffff0000; + static const int32_t VERSION_1 = 0x80010000; + // VERSION_2 (0x80020000) is taken by TDenseProtocol. + + TBinaryProtocolT(const boost::shared_ptr& trans) : + TVirtualProtocol< TBinaryProtocolT >(trans), + trans_(trans.get()), + string_limit_(0), + container_limit_(0), + strict_read_(false), + strict_write_(true), + string_buf_(NULL), + string_buf_size_(0) {} + + TBinaryProtocolT(const boost::shared_ptr& trans, + int32_t string_limit, + int32_t container_limit, + bool strict_read, + bool strict_write) : + TVirtualProtocol< TBinaryProtocolT >(trans), + trans_(trans.get()), + string_limit_(string_limit), + container_limit_(container_limit), + strict_read_(strict_read), + strict_write_(strict_write), + string_buf_(NULL), + string_buf_size_(0) {} + + /** + * Construct a TBinaryProtocolT using a raw pointer to the transport. + * + * The caller is responsible for ensuring that the transport remains valid + * for the lifetime of the protocol. + */ + TBinaryProtocolT(Transport_* trans) : + TVirtualProtocol< TBinaryProtocolT >(trans), + trans_(trans), + string_limit_(0), + container_limit_(0), + strict_read_(false), + strict_write_(true), + string_buf_(NULL), + string_buf_size_(0) {} + + ~TBinaryProtocolT() { + if (string_buf_ != NULL) { + std::free(string_buf_); + string_buf_size_ = 0; + } + } + + void setStringSizeLimit(int32_t string_limit) { + string_limit_ = string_limit; + } + + void setContainerSizeLimit(int32_t container_limit) { + container_limit_ = container_limit; + } + + void setStrict(bool strict_read, bool strict_write) { + strict_read_ = strict_read; + strict_write_ = strict_write; + } + + /** + * Writing functions. + */ + + /*ol*/ uint32_t writeMessageBegin(const std::string& name, + const TMessageType messageType, + const int32_t seqid); + + /*ol*/ uint32_t writeMessageEnd(); + + + inline uint32_t writeStructBegin(const char* name); + + inline uint32_t writeStructEnd(); + + inline uint32_t writeFieldBegin(const char* name, + const TType fieldType, + const int16_t fieldId); + + inline uint32_t writeFieldEnd(); + + inline uint32_t writeFieldStop(); + + inline uint32_t writeMapBegin(const TType keyType, + const TType valType, + const uint32_t size); + + inline uint32_t writeMapEnd(); + + inline uint32_t writeListBegin(const TType elemType, const uint32_t size); + + inline uint32_t writeListEnd(); + + inline uint32_t writeSetBegin(const TType elemType, const uint32_t size); + + inline uint32_t writeSetEnd(); + + inline uint32_t writeBool(const bool value); + + inline uint32_t writeByte(const int8_t byte); + + inline uint32_t writeI16(const int16_t i16); + + inline uint32_t writeI32(const int32_t i32); + + inline uint32_t writeI64(const int64_t i64); + + inline uint32_t writeDouble(const double dub); + + template + inline uint32_t writeString(const StrType& str); + + inline uint32_t writeBinary(const std::string& str); + + /** + * Reading functions + */ + + + /*ol*/ uint32_t readMessageBegin(std::string& name, + TMessageType& messageType, + int32_t& seqid); + + /*ol*/ uint32_t readMessageEnd(); + + inline uint32_t readStructBegin(std::string& name); + + inline uint32_t readStructEnd(); + + inline uint32_t readFieldBegin(std::string& name, + TType& fieldType, + int16_t& fieldId); + + inline uint32_t readFieldEnd(); + + inline uint32_t readMapBegin(TType& keyType, + TType& valType, + uint32_t& size); + + inline uint32_t readMapEnd(); + + inline uint32_t readListBegin(TType& elemType, uint32_t& size); + + inline uint32_t readListEnd(); + + inline uint32_t readSetBegin(TType& elemType, uint32_t& size); + + inline uint32_t readSetEnd(); + + inline uint32_t readBool(bool& value); + // Provide the default readBool() implementation for std::vector + using TVirtualProtocol< TBinaryProtocolT >::readBool; + + inline uint32_t readByte(int8_t& byte); + + inline uint32_t readI16(int16_t& i16); + + inline uint32_t readI32(int32_t& i32); + + inline uint32_t readI64(int64_t& i64); + + inline uint32_t readDouble(double& dub); + + template + inline uint32_t readString(StrType& str); + + inline uint32_t readBinary(std::string& str); + + protected: + template + uint32_t readStringBody(StrType& str, int32_t sz); + + Transport_* trans_; + + int32_t string_limit_; + int32_t container_limit_; + + // Enforce presence of version identifier + bool strict_read_; + bool strict_write_; + + // Buffer for reading strings, save for the lifetime of the protocol to + // avoid memory churn allocating memory on every string read + uint8_t* string_buf_; + int32_t string_buf_size_; + +}; + +typedef TBinaryProtocolT TBinaryProtocol; + + +class TBinaryProtocolFactoryBase : public TProtocolFactory {}; + +/** + * Constructs binary protocol handlers + */ +template +class TBinaryProtocolFactoryT : public TBinaryProtocolFactoryBase { + public: + TBinaryProtocolFactoryT() : + string_limit_(0), + container_limit_(0), + strict_read_(false), + strict_write_(true) {} + + TBinaryProtocolFactoryT(int32_t string_limit, int32_t container_limit, + bool strict_read, bool strict_write) : + string_limit_(string_limit), + container_limit_(container_limit), + strict_read_(strict_read), + strict_write_(strict_write) {} + + virtual ~TBinaryProtocolFactoryT() {} + + void setStringSizeLimit(int32_t string_limit) { + string_limit_ = string_limit; + } + + void setContainerSizeLimit(int32_t container_limit) { + container_limit_ = container_limit; + } + + void setStrict(bool strict_read, bool strict_write) { + strict_read_ = strict_read; + strict_write_ = strict_write; + } + + boost::shared_ptr getProtocol( + boost::shared_ptr trans) { + boost::shared_ptr specific_trans = + boost::dynamic_pointer_cast(trans); + TProtocol* prot; + if (specific_trans) { + prot = new TBinaryProtocolT(specific_trans, string_limit_, + container_limit_, strict_read_, + strict_write_); + } else { + prot = new TBinaryProtocol(trans, string_limit_, container_limit_, + strict_read_, strict_write_); + } + + return boost::shared_ptr(prot); + } + + private: + int32_t string_limit_; + int32_t container_limit_; + bool strict_read_; + bool strict_write_; + +}; + +typedef TBinaryProtocolFactoryT TBinaryProtocolFactory; + +}}} // apache::thrift::protocol + +#include "TBinaryProtocol.tcc" + +#endif // #ifndef THRIFT_PROTOCOL_TBINARYPROTOCOL_H_ diff --git a/thrift/lib/cpp/protocol/TBinaryProtocol.tcc b/thrift/lib/cpp/protocol/TBinaryProtocol.tcc new file mode 100644 index 0000000000..9d14361c07 --- /dev/null +++ b/thrift/lib/cpp/protocol/TBinaryProtocol.tcc @@ -0,0 +1,441 @@ +// Copyright (c) 2006- Facebook +// Distributed under the Thrift Software License +// +// See accompanying file LICENSE or visit the Thrift site at: +// http://developers.facebook.com/thrift/ + +#ifndef _THRIFT_PROTOCOL_TBINARYPROTOCOL_TCC_ +#define _THRIFT_PROTOCOL_TBINARYPROTOCOL_TCC_ 1 + +#include "thrift/lib/cpp/protocol/TBinaryProtocol.h" + +#include +#include + + +namespace apache { namespace thrift { namespace protocol { + +template +uint32_t TBinaryProtocolT::writeMessageBegin(const std::string& name, + const TMessageType messageType, + const int32_t seqid) { + if (this->strict_write_) { + int32_t version = (VERSION_1) | ((int32_t)messageType); + uint32_t wsize = 0; + wsize += writeI32(version); + wsize += writeString(name); + wsize += writeI32(seqid); + return wsize; + } else { + uint32_t wsize = 0; + wsize += writeString(name); + wsize += writeByte((int8_t)messageType); + wsize += writeI32(seqid); + return wsize; + } +} + +template +uint32_t TBinaryProtocolT::writeMessageEnd() { + return 0; +} + +template +uint32_t TBinaryProtocolT::writeStructBegin(const char* name) { + return 0; +} + +template +uint32_t TBinaryProtocolT::writeStructEnd() { + return 0; +} + +template +uint32_t TBinaryProtocolT::writeFieldBegin(const char* name, + const TType fieldType, + const int16_t fieldId) { + uint32_t wsize = 0; + wsize += writeByte((int8_t)fieldType); + wsize += writeI16(fieldId); + return wsize; +} + +template +uint32_t TBinaryProtocolT::writeFieldEnd() { + return 0; +} + +template +uint32_t TBinaryProtocolT::writeFieldStop() { + return + writeByte((int8_t)T_STOP); +} + +template +uint32_t TBinaryProtocolT::writeMapBegin(const TType keyType, + const TType valType, + const uint32_t size) { + uint32_t wsize = 0; + wsize += writeByte((int8_t)keyType); + wsize += writeByte((int8_t)valType); + wsize += writeI32((int32_t)size); + return wsize; +} + +template +uint32_t TBinaryProtocolT::writeMapEnd() { + return 0; +} + +template +uint32_t TBinaryProtocolT::writeListBegin(const TType elemType, + const uint32_t size) { + uint32_t wsize = 0; + wsize += writeByte((int8_t) elemType); + wsize += writeI32((int32_t)size); + return wsize; +} + +template +uint32_t TBinaryProtocolT::writeListEnd() { + return 0; +} + +template +uint32_t TBinaryProtocolT::writeSetBegin(const TType elemType, + const uint32_t size) { + uint32_t wsize = 0; + wsize += writeByte((int8_t)elemType); + wsize += writeI32((int32_t)size); + return wsize; +} + +template +uint32_t TBinaryProtocolT::writeSetEnd() { + return 0; +} + +template +uint32_t TBinaryProtocolT::writeBool(const bool value) { + uint8_t tmp = value ? 1 : 0; + this->trans_->write(&tmp, 1); + return 1; +} + +template +uint32_t TBinaryProtocolT::writeByte(const int8_t byte) { + this->trans_->write((uint8_t*)&byte, 1); + return 1; +} + +template +uint32_t TBinaryProtocolT::writeI16(const int16_t i16) { + int16_t net = (int16_t)htons(i16); + this->trans_->write((uint8_t*)&net, 2); + return 2; +} + +template +uint32_t TBinaryProtocolT::writeI32(const int32_t i32) { + int32_t net = (int32_t)htonl(i32); + this->trans_->write((uint8_t*)&net, 4); + return 4; +} + +template +uint32_t TBinaryProtocolT::writeI64(const int64_t i64) { + int64_t net = (int64_t)htonll(i64); + this->trans_->write((uint8_t*)&net, 8); + return 8; +} + +template +uint32_t TBinaryProtocolT::writeDouble(const double dub) { + BOOST_STATIC_ASSERT(sizeof(double) == sizeof(uint64_t)); + BOOST_STATIC_ASSERT(std::numeric_limits::is_iec559); + + uint64_t bits = bitwise_cast(dub); + bits = htonll(bits); + this->trans_->write((uint8_t*)&bits, 8); + return 8; +} + + +template +template +uint32_t TBinaryProtocolT::writeString(const StrType& str) { + uint32_t size = str.size(); + uint32_t result = writeI32((int32_t)size); + if (size > 0) { + this->trans_->write((uint8_t*)str.data(), size); + } + return result + size; +} + +template +uint32_t TBinaryProtocolT::writeBinary(const std::string& str) { + return TBinaryProtocolT::writeString(str); +} + +/** + * Reading functions + */ + +template +uint32_t TBinaryProtocolT::readMessageBegin(std::string& name, + TMessageType& messageType, + int32_t& seqid) { + uint32_t result = 0; + int32_t sz; + result += readI32(sz); + + if (sz < 0) { + // Check for correct version number + int32_t version = sz & VERSION_MASK; + if (version != VERSION_1) { + throw TProtocolException(TProtocolException::BAD_VERSION, "Bad version identifier"); + } + messageType = (TMessageType)(sz & 0x000000ff); + result += readString(name); + result += readI32(seqid); + } else { + if (this->strict_read_) { + throw TProtocolException(TProtocolException::BAD_VERSION, "No version identifier... old protocol client in strict mode?"); + } else { + // Handle pre-versioned input + int8_t type; + result += readStringBody(name, sz); + result += readByte(type); + messageType = (TMessageType)type; + result += readI32(seqid); + } + } + return result; +} + +template +uint32_t TBinaryProtocolT::readMessageEnd() { + return 0; +} + +template +uint32_t TBinaryProtocolT::readStructBegin(std::string& name) { + name = ""; + return 0; +} + +template +uint32_t TBinaryProtocolT::readStructEnd() { + return 0; +} + +template +uint32_t TBinaryProtocolT::readFieldBegin(std::string& name, + TType& fieldType, + int16_t& fieldId) { + uint32_t result = 0; + int8_t type; + result += readByte(type); + fieldType = (TType)type; + if (fieldType == T_STOP) { + fieldId = 0; + return result; + } + result += readI16(fieldId); + return result; +} + +template +uint32_t TBinaryProtocolT::readFieldEnd() { + return 0; +} + +template +uint32_t TBinaryProtocolT::readMapBegin(TType& keyType, + TType& valType, + uint32_t& size) { + int8_t k, v; + uint32_t result = 0; + int32_t sizei; + result += readByte(k); + keyType = (TType)k; + result += readByte(v); + valType = (TType)v; + result += readI32(sizei); + if (sizei < 0) { + throw TProtocolException(TProtocolException::NEGATIVE_SIZE); + } else if (this->container_limit_ && sizei > this->container_limit_) { + throw TProtocolException(TProtocolException::SIZE_LIMIT); + } + size = (uint32_t)sizei; + return result; +} + +template +uint32_t TBinaryProtocolT::readMapEnd() { + return 0; +} + +template +uint32_t TBinaryProtocolT::readListBegin(TType& elemType, + uint32_t& size) { + int8_t e; + uint32_t result = 0; + int32_t sizei; + result += readByte(e); + elemType = (TType)e; + result += readI32(sizei); + if (sizei < 0) { + throw TProtocolException(TProtocolException::NEGATIVE_SIZE); + } else if (this->container_limit_ && sizei > this->container_limit_) { + throw TProtocolException(TProtocolException::SIZE_LIMIT); + } + size = (uint32_t)sizei; + return result; +} + +template +uint32_t TBinaryProtocolT::readListEnd() { + return 0; +} + +template +uint32_t TBinaryProtocolT::readSetBegin(TType& elemType, + uint32_t& size) { + int8_t e; + uint32_t result = 0; + int32_t sizei; + result += readByte(e); + elemType = (TType)e; + result += readI32(sizei); + if (sizei < 0) { + throw TProtocolException(TProtocolException::NEGATIVE_SIZE); + } else if (this->container_limit_ && sizei > this->container_limit_) { + throw TProtocolException(TProtocolException::SIZE_LIMIT); + } + size = (uint32_t)sizei; + return result; +} + +template +uint32_t TBinaryProtocolT::readSetEnd() { + return 0; +} + +template +uint32_t TBinaryProtocolT::readBool(bool& value) { + uint8_t b[1]; + this->trans_->readAll(b, 1); + value = *(int8_t*)b != 0; + return 1; +} + +template +uint32_t TBinaryProtocolT::readByte(int8_t& byte) { + uint8_t b[1]; + this->trans_->readAll(b, 1); + byte = *(int8_t*)b; + return 1; +} + +template +uint32_t TBinaryProtocolT::readI16(int16_t& i16) { + uint8_t b[2]; + this->trans_->readAll(b, 2); + i16 = *bitwise_cast(&b); + i16 = (int16_t)ntohs(i16); + return 2; +} + +template +uint32_t TBinaryProtocolT::readI32(int32_t& i32) { + uint8_t b[4]; + this->trans_->readAll(b, 4); + i32 = *bitwise_cast(&b); + i32 = (int32_t)ntohl(i32); + return 4; +} + +template +uint32_t TBinaryProtocolT::readI64(int64_t& i64) { + uint8_t b[8]; + this->trans_->readAll(b, 8); + i64 = *bitwise_cast(&b); + i64 = (int64_t)ntohll(i64); + return 8; +} + +template +uint32_t TBinaryProtocolT::readDouble(double& dub) { + BOOST_STATIC_ASSERT(sizeof(double) == sizeof(uint64_t)); + BOOST_STATIC_ASSERT(std::numeric_limits::is_iec559); + + uint64_t bits; + uint8_t b[8]; + this->trans_->readAll(b, 8); + bits = *bitwise_cast(&b); + bits = ntohll(bits); + dub = bitwise_cast(bits); + return 8; +} + +template +template +uint32_t TBinaryProtocolT::readString(StrType& str) { + uint32_t result; + int32_t size; + result = readI32(size); + return result + readStringBody(str, size); +} + +template +uint32_t TBinaryProtocolT::readBinary(std::string& str) { + return TBinaryProtocolT::readString(str); +} + +template +template +uint32_t TBinaryProtocolT::readStringBody(StrType& str, + int32_t size) { + uint32_t result = 0; + + // Catch error cases + if (size < 0) { + throw TProtocolException(TProtocolException::NEGATIVE_SIZE); + } + if (this->string_limit_ > 0 && size > this->string_limit_) { + throw TProtocolException(TProtocolException::SIZE_LIMIT); + } + + // Catch empty string case + if (size == 0) { + str.clear(); + return result; + } + + // Try to borrow first + const uint8_t* borrow_buf; + uint32_t got = size; + if ((borrow_buf = this->trans_->borrow(NULL, &got))) { + str.assign((const char*)borrow_buf, size); + this->trans_->consume(size); + return size; + } + + // Use the heap here to prevent stack overflow for v. large strings + if (size > this->string_buf_size_ || this->string_buf_ == NULL) { + void* new_string_buf = std::realloc(this->string_buf_, (uint32_t)size); + if (new_string_buf == NULL) { + throw std::bad_alloc(); + } + this->string_buf_ = (uint8_t*)new_string_buf; + this->string_buf_size_ = size; + } + this->trans_->readAll(this->string_buf_, size); + str.assign((char*)this->string_buf_, size); + return (uint32_t)size; +} + +}}} // apache::thrift::protocol + +#endif // #ifndef _THRIFT_PROTOCOL_TBINARYPROTOCOL_TCC_ diff --git a/thrift/lib/cpp/protocol/TCompactProtocol.h b/thrift/lib/cpp/protocol/TCompactProtocol.h new file mode 100644 index 0000000000..ff2b99a74c --- /dev/null +++ b/thrift/lib/cpp/protocol/TCompactProtocol.h @@ -0,0 +1,313 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef THRIFT_PROTOCOL_TCOMPACTPROTOCOL_H_ +#define THRIFT_PROTOCOL_TCOMPACTPROTOCOL_H_ 1 + +#include "thrift/lib/cpp/protocol/TVirtualProtocol.h" + +#include +#include + +namespace apache { namespace thrift { namespace protocol { + +/** + * C++ Implementation of the Compact Protocol as described in THRIFT-110 + */ +template +class TCompactProtocolT + : public TVirtualProtocol< TCompactProtocolT > { + public: + static const int8_t VERSION_N = 2; + static const int8_t VERSION_LOW = 1; + static const int8_t VERSION_DOUBLE_BE = 2; + + protected: + // Normally we can define static const data members of integral + // type here. However there appears to be a gcc issue when the + // high bit is set (http://gcc.gnu.org/bugzilla/show_bug.cgi?id=49896) + // unless we cast to a value that fits in an int8_t (0x82 and 0xE0 are + // uint8_t) + static const int8_t PROTOCOL_ID = static_cast(0x82); + static const int8_t TYPE_MASK = static_cast(0xE0); + + static const int8_t VERSION_MASK = 0x1f; // 0001 1111 + static const int32_t TYPE_SHIFT_AMOUNT = 5; + + Transport_* trans_; + + /** + * (Writing) If we encounter a boolean field begin, save the TField here + * so it can have the value incorporated. + */ + struct { + const char* name; + TType fieldType; + int16_t fieldId; + } booleanField_; + + /** + * (Reading) If we read a field header, and it's a boolean field, save + * the boolean value here so that readBool can use it. + */ + struct { + bool hasBoolValue; + bool boolValue; + } boolValue_; + + /** + * Used to keep track of the last field for the current and previous structs, + * so we can do the delta stuff. + */ + + std::stack lastField_; + int16_t lastFieldId_; + int8_t version_; + + public: + explicit TCompactProtocolT(boost::shared_ptr trans) : + TVirtualProtocol< TCompactProtocolT >(trans), + trans_(trans.get()), + lastFieldId_(0), + version_(VERSION_N), + string_limit_(0), + string_buf_(NULL), + string_buf_size_(0), + container_limit_(0) { + booleanField_.name = NULL; + boolValue_.hasBoolValue = false; + } + + TCompactProtocolT(boost::shared_ptr trans, + int32_t string_limit, + int32_t container_limit) : + TVirtualProtocol< TCompactProtocolT >(trans), + trans_(trans.get()), + lastFieldId_(0), + version_(VERSION_N), + string_limit_(string_limit), + string_buf_(NULL), + string_buf_size_(0), + container_limit_(container_limit) { + booleanField_.name = NULL; + boolValue_.hasBoolValue = false; + } + + virtual ~TCompactProtocolT() { + if (string_buf_) { + free(string_buf_); + } + } + + /** Set this if you need backwards compatibility with an old version */ + void setVersion(const int8_t version) { + version_ = version; + } + + /** + * Writing functions + */ + + virtual uint32_t writeMessageBegin(const std::string& name, + const TMessageType messageType, + const int32_t seqid); + + uint32_t writeStructBegin(const char* name); + + uint32_t writeStructEnd(); + + uint32_t writeFieldBegin(const char* name, + const TType fieldType, + const int16_t fieldId); + + uint32_t writeFieldStop(); + + uint32_t writeListBegin(const TType elemType, + const uint32_t size); + + uint32_t writeSetBegin(const TType elemType, + const uint32_t size); + + virtual uint32_t writeMapBegin(const TType keyType, + const TType valType, + const uint32_t size); + + uint32_t writeBool(const bool value); + + uint32_t writeByte(const int8_t byte); + + uint32_t writeI16(const int16_t i16); + + uint32_t writeI32(const int32_t i32); + + uint32_t writeI64(const int64_t i64); + + uint32_t writeDouble(const double dub); + + uint32_t writeString(const char* str); + + template + uint32_t writeString(const String_& str); + + template + uint32_t writeBinary(const String_& str); + + /** + * These methods are called by structs, but don't actually have any wired + * output or purpose + */ + virtual uint32_t writeMessageEnd() { return 0; } + uint32_t writeMapEnd() { return 0; } + uint32_t writeListEnd() { return 0; } + uint32_t writeSetEnd() { return 0; } + uint32_t writeFieldEnd() { return 0; } + + protected: + int32_t writeFieldBeginInternal(const char* name, + const TType fieldType, + const int16_t fieldId, + int8_t typeOverride); + uint32_t writeCollectionBegin(int8_t elemType, int32_t size); + uint32_t writeVarint32(uint32_t n); + uint32_t writeVarint64(uint64_t n); + uint64_t i64ToZigzag(const int64_t l); + uint32_t i32ToZigzag(const int32_t n); + inline int8_t getCompactType(int8_t ttype); + + public: + uint32_t readMessageBegin(std::string& name, + TMessageType& messageType, + int32_t& seqid); + + uint32_t readStructBegin(std::string& name); + + uint32_t readStructEnd(); + + uint32_t readFieldBegin(std::string& name, + TType& fieldType, + int16_t& fieldId); + + uint32_t readMapBegin(TType& keyType, + TType& valType, + uint32_t& size); + + uint32_t readListBegin(TType& elemType, + uint32_t& size); + + uint32_t readSetBegin(TType& elemType, + uint32_t& size); + + uint32_t readBool(bool& value); + // Provide the default readBool() implementation for std::vector + using TVirtualProtocol< TCompactProtocolT >::readBool; + + uint32_t readByte(int8_t& byte); + + uint32_t readI16(int16_t& i16); + + uint32_t readI32(int32_t& i32); + + uint32_t readI64(int64_t& i64); + + uint32_t readDouble(double& dub); + + template + uint32_t readString(String_& str); + + template + uint32_t readBinary(String_& str); + + /* + *These methods are here for the struct to call, but don't have any wire + * encoding. + */ + uint32_t readMessageEnd() { return 0; } + uint32_t readFieldEnd() { return 0; } + uint32_t readMapEnd() { return 0; } + uint32_t readListEnd() { return 0; } + uint32_t readSetEnd() { return 0; } + + protected: + uint32_t readVarint32(int32_t& i32); + uint32_t readVarint64(int64_t& i64); + int32_t zigzagToI32(uint32_t n); + int64_t zigzagToI64(uint64_t n); + TType getTType(int8_t type); + + // Buffer for reading strings, save for the lifetime of the protocol to + // avoid memory churn allocating memory on every string read + int32_t string_limit_; + uint8_t* string_buf_; + int32_t string_buf_size_; + int32_t container_limit_; +}; + +typedef TCompactProtocolT TCompactProtocol; + +/** + * Constructs compact protocol handlers + */ +template +class TCompactProtocolFactoryT : public TProtocolFactory { + public: + TCompactProtocolFactoryT() : + string_limit_(0), + container_limit_(0) {} + + TCompactProtocolFactoryT(int32_t string_limit, int32_t container_limit) : + string_limit_(string_limit), + container_limit_(container_limit) {} + + virtual ~TCompactProtocolFactoryT() {} + + void setStringSizeLimit(int32_t string_limit) { + string_limit_ = string_limit; + } + + void setContainerSizeLimit(int32_t container_limit) { + container_limit_ = container_limit; + } + + boost::shared_ptr getProtocol(boost::shared_ptr trans) { + boost::shared_ptr specific_trans = + boost::dynamic_pointer_cast(trans); + TProtocol* prot; + if (specific_trans) { + prot = new TCompactProtocolT(specific_trans, string_limit_, + container_limit_); + } else { + prot = new TCompactProtocol(trans, string_limit_, container_limit_); + } + + return boost::shared_ptr(prot); + } + + private: + int32_t string_limit_; + int32_t container_limit_; + +}; + +typedef TCompactProtocolFactoryT TCompactProtocolFactory; + +}}} // apache::thrift::protocol + +#include "TCompactProtocol.tcc" + +#endif diff --git a/thrift/lib/cpp/protocol/TCompactProtocol.tcc b/thrift/lib/cpp/protocol/TCompactProtocol.tcc new file mode 100644 index 0000000000..3551d69eab --- /dev/null +++ b/thrift/lib/cpp/protocol/TCompactProtocol.tcc @@ -0,0 +1,820 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_PROTOCOL_TCOMPACTPROTOCOL_TCC_ +#define THRIFT_PROTOCOL_TCOMPACTPROTOCOL_TCC_ 1 + +#include "thrift/lib/cpp/thrift_config.h" +#include "folly/Likely.h" +#include + +/* + * TCompactProtocol::i*ToZigzag depend on the fact that the right shift + * operator on a signed integer is an arithmetic (sign-extending) shift. + * If this is not the case, the current implementation will not work. + * If anyone encounters this error, we can try to figure out the best + * way to implement an arithmetic right shift on their platform. + */ +#if !defined(SIGNED_RIGHT_SHIFT_IS) || !defined(ARITHMETIC_RIGHT_SHIFT) +# error "Unable to determine the behavior of a signed right shift" +#endif +#if SIGNED_RIGHT_SHIFT_IS != ARITHMETIC_RIGHT_SHIFT +# error "TCompactProtocol currently only works if a signed right shift is arithmetic" +#endif + +namespace apache { namespace thrift { namespace protocol { + +namespace detail { namespace compact { + +enum Types { + CT_STOP = 0x00, + CT_BOOLEAN_TRUE = 0x01, + CT_BOOLEAN_FALSE = 0x02, + CT_BYTE = 0x03, + CT_I16 = 0x04, + CT_I32 = 0x05, + CT_I64 = 0x06, + CT_DOUBLE = 0x07, + CT_BINARY = 0x08, + CT_LIST = 0x09, + CT_SET = 0x0A, + CT_MAP = 0x0B, + CT_STRUCT = 0x0C, +}; + +const int8_t TTypeToCType[16] = { + CT_STOP, // T_STOP + 0, // unused + CT_BOOLEAN_TRUE, // T_BOOL + CT_BYTE, // T_BYTE + CT_DOUBLE, // T_DOUBLE + 0, // unused + CT_I16, // T_I16 + 0, // unused + CT_I32, // T_I32 + 0, // unused + CT_I64, // T_I64 + CT_BINARY, // T_STRING + CT_STRUCT, // T_STRUCT + CT_MAP, // T_MAP + CT_SET, // T_SET + CT_LIST, // T_LIST +}; + +}} // end detail::compact namespace + +template +uint32_t TCompactProtocolT::writeMessageBegin( + const std::string& name, + const TMessageType messageType, + const int32_t seqid) { + uint32_t wsize = 0; + wsize += writeByte(PROTOCOL_ID); + wsize += writeByte((version_ & VERSION_MASK) | (((int32_t)messageType << TYPE_SHIFT_AMOUNT) & TYPE_MASK)); + wsize += writeVarint32(seqid); + wsize += writeString(name); + return wsize; +} + +/** + * Write a field header containing the field id and field type. If the + * difference between the current field id and the last one is small (< 15), + * then the field id will be encoded in the 4 MSB as a delta. Otherwise, the + * field id will follow the type header as a zigzag varint. + */ +template +uint32_t TCompactProtocolT::writeFieldBegin(const char* name, + const TType fieldType, + const int16_t fieldId) { + if (fieldType == T_BOOL) { + booleanField_.name = name; + booleanField_.fieldType = fieldType; + booleanField_.fieldId = fieldId; + } else { + return writeFieldBeginInternal(name, fieldType, fieldId, -1); + } + return 0; +} + +/** + * Write the STOP symbol so we know there are no more fields in this struct. + */ +template +uint32_t TCompactProtocolT::writeFieldStop() { + return writeByte(T_STOP); +} + +/** + * Write a struct begin. This doesn't actually put anything on the wire. We + * use it as an opportunity to put special placeholder markers on the field + * stack so we can get the field id deltas correct. + */ +template +uint32_t TCompactProtocolT::writeStructBegin(const char* name) { + lastField_.push(lastFieldId_); + lastFieldId_ = 0; + return 0; +} + +/** + * Write a struct end. This doesn't actually put anything on the wire. We use + * this as an opportunity to pop the last field from the current struct off + * of the field stack. + */ +template +uint32_t TCompactProtocolT::writeStructEnd() { + lastFieldId_ = lastField_.top(); + lastField_.pop(); + return 0; +} + +/** + * Write a List header. + */ +template +uint32_t TCompactProtocolT::writeListBegin(const TType elemType, + const uint32_t size) { + return writeCollectionBegin(elemType, size); +} + +/** + * Write a set header. + */ +template +uint32_t TCompactProtocolT::writeSetBegin(const TType elemType, + const uint32_t size) { + return writeCollectionBegin(elemType, size); +} + +/** + * Write a map header. If the map is empty, omit the key and value type + * headers, as we don't need any additional information to skip it. + */ +template +uint32_t TCompactProtocolT::writeMapBegin(const TType keyType, + const TType valType, + const uint32_t size) { + uint32_t wsize = 0; + + if (size == 0) { + wsize += writeByte(0); + } else { + wsize += writeVarint32(size); + wsize += writeByte(getCompactType(keyType) << 4 | getCompactType(valType)); + } + return wsize; +} + +/** + * Write a boolean value. Potentially, this could be a boolean field, in + * which case the field header info isn't written yet. If so, decide what the + * right type header is for the value and then write the field header. + * Otherwise, write a single byte. + */ +template +uint32_t TCompactProtocolT::writeBool(const bool value) { + uint32_t wsize = 0; + + if (booleanField_.name != NULL) { + // we haven't written the field header yet + wsize += writeFieldBeginInternal(booleanField_.name, + booleanField_.fieldType, + booleanField_.fieldId, + value ? detail::compact::CT_BOOLEAN_TRUE : + detail::compact::CT_BOOLEAN_FALSE); + booleanField_.name = NULL; + } else { + // we're not part of a field, so just write the value + wsize += writeByte(value ? detail::compact::CT_BOOLEAN_TRUE : + detail::compact::CT_BOOLEAN_FALSE); + } + return wsize; +} + +template +uint32_t TCompactProtocolT::writeByte(const int8_t byte) { + trans_->write((uint8_t*)&byte, 1); + return 1; +} + +/** + * Write an i16 as a zigzag varint. + */ +template +uint32_t TCompactProtocolT::writeI16(const int16_t i16) { + return writeVarint32(i32ToZigzag(i16)); +} + +/** + * Write an i32 as a zigzag varint. + */ +template +uint32_t TCompactProtocolT::writeI32(const int32_t i32) { + return writeVarint32(i32ToZigzag(i32)); +} + +/** + * Write an i64 as a zigzag varint. + */ +template +uint32_t TCompactProtocolT::writeI64(const int64_t i64) { + return writeVarint64(i64ToZigzag(i64)); +} + +/** + * Write a double to the wire as 8 bytes. + */ +template +uint32_t TCompactProtocolT::writeDouble(const double dub) { + BOOST_STATIC_ASSERT(sizeof(double) == sizeof(uint64_t)); + BOOST_STATIC_ASSERT(std::numeric_limits::is_iec559); + + uint64_t bits = bitwise_cast(dub); + if (version_ >= VERSION_DOUBLE_BE) { + bits = htonll(bits); + } else { + bits = htolell(bits); + } + trans_->write((uint8_t*)&bits, 8); + return 8; +} + +/** + * Write a string to the wire with a varint size preceding. + */ +template +uint32_t TCompactProtocolT::writeString(const char* str) { + return writeString(std::string(str)); +} + +template +template +uint32_t TCompactProtocolT::writeString(const String_& str) { + return writeBinary(str); +} + +template +template +uint32_t TCompactProtocolT::writeBinary(const String_& str) { + uint32_t ssize = str.size(); + uint32_t wsize = writeVarint32(ssize) + ssize; + trans_->write((uint8_t*)str.data(), ssize); + return wsize; +} + +// +// Internal Writing methods +// + +/** + * The workhorse of writeFieldBegin. It has the option of doing a + * 'type override' of the type header. This is used specifically in the + * boolean field case. + */ +template +int32_t TCompactProtocolT::writeFieldBeginInternal( + const char* name, + const TType fieldType, + const int16_t fieldId, + int8_t typeOverride) { + uint32_t wsize = 0; + + // if there's a type override, use that. + int8_t typeToWrite = (typeOverride == -1 ? getCompactType(fieldType) : typeOverride); + + // check if we can use delta encoding for the field id + if (fieldId > lastFieldId_ && fieldId - lastFieldId_ <= 15) { + // write them together + wsize += writeByte((fieldId - lastFieldId_) << 4 | typeToWrite); + } else { + // write them separate + wsize += writeByte(typeToWrite); + wsize += writeI16(fieldId); + } + + lastFieldId_ = fieldId; + return wsize; +} + +/** + * Abstract method for writing the start of lists and sets. List and sets on + * the wire differ only by the type indicator. + */ +template +uint32_t TCompactProtocolT::writeCollectionBegin(int8_t elemType, + int32_t size) { + uint32_t wsize = 0; + if (size <= 14) { + wsize += writeByte(size << 4 | getCompactType(elemType)); + } else { + wsize += writeByte(0xf0 | getCompactType(elemType)); + wsize += writeVarint32(size); + } + return wsize; +} + +/** + * Write an i32 as a varint. Results in 1-5 bytes on the wire. + */ +template +uint32_t TCompactProtocolT::writeVarint32(uint32_t n) { + uint8_t buf[5]; + uint32_t wsize = 0; + + while (true) { + if ((n & ~0x7F) == 0) { + buf[wsize++] = (int8_t)n; + break; + } else { + buf[wsize++] = (int8_t)((n & 0x7F) | 0x80); + n >>= 7; + } + } + trans_->write(buf, wsize); + return wsize; +} + +/** + * Write an i64 as a varint. Results in 1-10 bytes on the wire. + */ +template +uint32_t TCompactProtocolT::writeVarint64(uint64_t n) { + uint8_t buf[10]; + uint32_t wsize = 0; + + while (true) { + if ((n & ~0x7FL) == 0) { + buf[wsize++] = (int8_t)n; + break; + } else { + buf[wsize++] = (int8_t)((n & 0x7F) | 0x80); + n >>= 7; + } + } + trans_->write(buf, wsize); + return wsize; +} + +/** + * Convert l into a zigzag long. This allows negative numbers to be + * represented compactly as a varint. + */ +template +uint64_t TCompactProtocolT::i64ToZigzag(const int64_t l) { + return (l << 1) ^ (l >> 63); +} + +/** + * Convert n into a zigzag int. This allows negative numbers to be + * represented compactly as a varint. + */ +template +uint32_t TCompactProtocolT::i32ToZigzag(const int32_t n) { + return (n << 1) ^ (n >> 31); +} + +/** + * Given a TType value, find the appropriate detail::compact::Types value + */ +template +int8_t TCompactProtocolT::getCompactType(int8_t ttype) { + return detail::compact::TTypeToCType[ttype]; +} + +// +// Reading Methods +// + +/** + * Read a message header. + */ +template +uint32_t TCompactProtocolT::readMessageBegin( + std::string& name, + TMessageType& messageType, + int32_t& seqid) { + uint32_t rsize = 0; + int8_t protocolId; + int8_t versionAndType; + + rsize += readByte(protocolId); + if (protocolId != PROTOCOL_ID) { + throw TProtocolException(TProtocolException::BAD_VERSION, "Bad protocol identifier"); + } + + rsize += readByte(versionAndType); + version_ = (int8_t)(versionAndType & VERSION_MASK); + if (!(version_ <= VERSION_N && version_ >= VERSION_LOW)) { + throw TProtocolException(TProtocolException::BAD_VERSION, "Bad protocol version"); + } + + messageType = (TMessageType)((versionAndType >> TYPE_SHIFT_AMOUNT) & 0x03); + rsize += readVarint32(seqid); + rsize += readString(name); + + return rsize; +} + +/** + * Read a struct begin. There's nothing on the wire for this, but it is our + * opportunity to push a new struct begin marker on the field stack. + */ +template +uint32_t TCompactProtocolT::readStructBegin(std::string& name) { + name = ""; + lastField_.push(lastFieldId_); + lastFieldId_ = 0; + return 0; +} + +/** + * Doesn't actually consume any wire data, just removes the last field for + * this struct from the field stack. + */ +template +uint32_t TCompactProtocolT::readStructEnd() { + lastFieldId_ = lastField_.top(); + lastField_.pop(); + return 0; +} + +/** + * Read a field header off the wire. + */ +template +uint32_t TCompactProtocolT::readFieldBegin(std::string& name, + TType& fieldType, + int16_t& fieldId) { + uint32_t rsize = 0; + int8_t byte; + int8_t type; + + rsize += readByte(byte); + type = (byte & 0x0f); + + // if it's a stop, then we can return immediately, as the struct is over. + if (type == T_STOP) { + fieldType = T_STOP; + fieldId = 0; + return rsize; + } + + // mask off the 4 MSB of the type header. it could contain a field id delta. + int16_t modifier = (int16_t)(((uint8_t)byte & 0xf0) >> 4); + if (modifier == 0) { + // not a delta, look ahead for the zigzag varint field id. + rsize += readI16(fieldId); + } else { + fieldId = (int16_t)(lastFieldId_ + modifier); + } + fieldType = getTType(type); + + // if this happens to be a boolean field, the value is encoded in the type + if (type == detail::compact::CT_BOOLEAN_TRUE || + type == detail::compact::CT_BOOLEAN_FALSE) { + // save the boolean value in a special instance variable. + boolValue_.hasBoolValue = true; + boolValue_.boolValue = + (type == detail::compact::CT_BOOLEAN_TRUE ? true : false); + } + + // push the new field onto the field stack so we can keep the deltas going. + lastFieldId_ = fieldId; + return rsize; +} + +/** + * Read a map header off the wire. If the size is zero, skip reading the key + * and value type. This means that 0-length maps will yield TMaps without the + * "correct" types. + */ +template +uint32_t TCompactProtocolT::readMapBegin(TType& keyType, + TType& valType, + uint32_t& size) { + uint32_t rsize = 0; + int8_t kvType = 0; + int32_t msize = 0; + + rsize += readVarint32(msize); + if (msize != 0) + rsize += readByte(kvType); + + if (msize < 0) { + throw TProtocolException(TProtocolException::NEGATIVE_SIZE); + } else if (container_limit_ && msize > container_limit_) { + throw TProtocolException(TProtocolException::SIZE_LIMIT); + } + + keyType = getTType((int8_t)((uint8_t)kvType >> 4)); + valType = getTType((int8_t)((uint8_t)kvType & 0xf)); + size = (uint32_t)msize; + + return rsize; +} + +/** + * Read a list header off the wire. If the list size is 0-14, the size will + * be packed into the element type header. If it's a longer list, the 4 MSB + * of the element type header will be 0xF, and a varint will follow with the + * true size. + */ +template +uint32_t TCompactProtocolT::readListBegin(TType& elemType, + uint32_t& size) { + int8_t size_and_type; + uint32_t rsize = 0; + int32_t lsize; + + rsize += readByte(size_and_type); + + lsize = ((uint8_t)size_and_type >> 4) & 0x0f; + if (lsize == 15) { + rsize += readVarint32(lsize); + } + + if (lsize < 0) { + throw TProtocolException(TProtocolException::NEGATIVE_SIZE); + } else if (container_limit_ && lsize > container_limit_) { + throw TProtocolException(TProtocolException::SIZE_LIMIT); + } + + elemType = getTType((int8_t)(size_and_type & 0x0f)); + size = (uint32_t)lsize; + + return rsize; +} + +/** + * Read a set header off the wire. If the set size is 0-14, the size will + * be packed into the element type header. If it's a longer set, the 4 MSB + * of the element type header will be 0xF, and a varint will follow with the + * true size. + */ +template +uint32_t TCompactProtocolT::readSetBegin(TType& elemType, + uint32_t& size) { + return readListBegin(elemType, size); +} + +/** + * Read a boolean off the wire. If this is a boolean field, the value should + * already have been read during readFieldBegin, so we'll just consume the + * pre-stored value. Otherwise, read a byte. + */ +template +uint32_t TCompactProtocolT::readBool(bool& value) { + if (boolValue_.hasBoolValue == true) { + value = boolValue_.boolValue; + boolValue_.hasBoolValue = false; + return 0; + } else { + int8_t val; + readByte(val); + value = (val == detail::compact::CT_BOOLEAN_TRUE); + return 1; + } +} + +/** + * Read a single byte off the wire. Nothing interesting here. + */ +template +uint32_t TCompactProtocolT::readByte(int8_t& byte) { + uint8_t b[1]; + trans_->readAll(b, 1); + byte = *(int8_t*)b; + return 1; +} + +/** + * Read an i16 from the wire as a zigzag varint. + */ +template +uint32_t TCompactProtocolT::readI16(int16_t& i16) { + int32_t value; + uint32_t rsize = readVarint32(value); + i16 = (int16_t)zigzagToI32(value); + return rsize; +} + +/** + * Read an i32 from the wire as a zigzag varint. + */ +template +uint32_t TCompactProtocolT::readI32(int32_t& i32) { + int32_t value; + uint32_t rsize = readVarint32(value); + i32 = zigzagToI32(value); + return rsize; +} + +/** + * Read an i64 from the wire as a zigzag varint. + */ +template +uint32_t TCompactProtocolT::readI64(int64_t& i64) { + int64_t value; + uint32_t rsize = readVarint64(value); + i64 = zigzagToI64(value); + return rsize; +} + +/** + * No magic here - just read a double off the wire. + */ +template +uint32_t TCompactProtocolT::readDouble(double& dub) { + BOOST_STATIC_ASSERT(sizeof(double) == sizeof(uint64_t)); + BOOST_STATIC_ASSERT(std::numeric_limits::is_iec559); + + union { + uint64_t bits; + uint8_t b[8]; + } u; + trans_->readAll(u.b, 8); + if (version_ >= VERSION_DOUBLE_BE) { + u.bits = ntohll(u.bits); + } else { + u.bits = letohll(u.bits); + } + dub = bitwise_cast(u.bits); + return 8; +} + +template +template +uint32_t TCompactProtocolT::readString(String_& str) { + return readBinary(str); +} + +/** + * Read a byte[] from the wire. + */ +template +template +uint32_t TCompactProtocolT::readBinary(String_& str) { + int32_t rsize = 0; + int32_t size; + + rsize += readVarint32(size); + // Catch empty string case + if (size == 0) { + str = ""; + return rsize; + } + + // Catch error cases + if (size < 0) { + throw TProtocolException(TProtocolException::NEGATIVE_SIZE); + } + if (string_limit_ > 0 && size > string_limit_) { + throw TProtocolException(TProtocolException::SIZE_LIMIT); + } + + // Use the heap here to prevent stack overflow for v. large strings + if (size > string_buf_size_ || string_buf_ == NULL) { + void* new_string_buf = std::realloc(string_buf_, (uint32_t)size); + if (new_string_buf == NULL) { + throw std::bad_alloc(); + } + string_buf_ = (uint8_t*)new_string_buf; + string_buf_size_ = size; + } + trans_->readAll(string_buf_, size); + str.assign((char*)string_buf_, size); + + return rsize + (uint32_t)size; +} + +/** + * Read an i32 from the wire as a varint. The MSB of each byte is set + * if there is another byte to follow. This can read up to 5 bytes. + */ +template +uint32_t TCompactProtocolT::readVarint32(int32_t& i32) { + int64_t val; + uint32_t rsize = readVarint64(val); + i32 = (int32_t)val; + return rsize; +} + +/** + * Read an i64 from the wire as a proper varint. The MSB of each byte is set + * if there is another byte to follow. This can read up to 10 bytes. + */ +template +uint32_t TCompactProtocolT::readVarint64(int64_t& i64) { + uint32_t rsize = 0; + uint64_t val = 0; + int shift = 0; + uint8_t buf[10]; // 64 bits / (7 bits/byte) = 10 bytes. + uint32_t buf_size = sizeof(buf); + const uint8_t* borrowed = trans_->borrow(buf, &buf_size); + + // Fast path. + if (borrowed != NULL) { + while (true) { + uint8_t byte = borrowed[rsize]; + rsize++; + val |= (uint64_t)(byte & 0x7f) << shift; + shift += 7; + if (!(byte & 0x80)) { + i64 = val; + trans_->consume(rsize); + return rsize; + } + // Have to check for invalid data so we don't crash. + if (UNLIKELY(rsize == sizeof(buf))) { + throw TProtocolException(TProtocolException::INVALID_DATA, "Variable-length int over 10 bytes."); + } + } + } + + // Slow path. + else { + while (true) { + uint8_t byte; + rsize += trans_->readAll(&byte, 1); + val |= (uint64_t)(byte & 0x7f) << shift; + shift += 7; + if (!(byte & 0x80)) { + i64 = val; + return rsize; + } + // Might as well check for invalid data on the slow path too. + if (UNLIKELY(rsize >= sizeof(buf))) { + throw TProtocolException(TProtocolException::INVALID_DATA, "Variable-length int over 10 bytes."); + } + } + } +} + +/** + * Convert from zigzag int to int. + */ +template +int32_t TCompactProtocolT::zigzagToI32(uint32_t n) { + return (n >> 1) ^ -(n & 1); +} + +/** + * Convert from zigzag long to long. + */ +template +int64_t TCompactProtocolT::zigzagToI64(uint64_t n) { + return (n >> 1) ^ -(n & 1); +} + +template +TType TCompactProtocolT::getTType(int8_t type) { + switch (type) { + case T_STOP: + return T_STOP; + case detail::compact::CT_BOOLEAN_FALSE: + case detail::compact::CT_BOOLEAN_TRUE: + return T_BOOL; + case detail::compact::CT_BYTE: + return T_BYTE; + case detail::compact::CT_I16: + return T_I16; + case detail::compact::CT_I32: + return T_I32; + case detail::compact::CT_I64: + return T_I64; + case detail::compact::CT_DOUBLE: + return T_DOUBLE; + case detail::compact::CT_BINARY: + return T_STRING; + case detail::compact::CT_LIST: + return T_LIST; + case detail::compact::CT_SET: + return T_SET; + case detail::compact::CT_MAP: + return T_MAP; + case detail::compact::CT_STRUCT: + return T_STRUCT; + default: + throw TLibraryException("don't know what type: " + type); + } + return T_STOP; +} + +}}} // apache::thrift::protocol + +#endif // THRIFT_PROTOCOL_TCOMPACTPROTOCOL_TCC_ diff --git a/thrift/lib/cpp/protocol/TDebugProtocol.h b/thrift/lib/cpp/protocol/TDebugProtocol.h new file mode 100644 index 0000000000..236f9b86e1 --- /dev/null +++ b/thrift/lib/cpp/protocol/TDebugProtocol.h @@ -0,0 +1,360 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef THRIFT_PROTOCOL_TDEBUGPROTOCOL_H +#define THRIFT_PROTOCOL_TDEBUGPROTOCOL_H + +#include "thrift/lib/cpp/protocol/TVirtualProtocol.h" + +#include + +namespace apache { namespace thrift { namespace protocol { + +/* + +!!! EXPERIMENTAL CODE !!! + +This protocol is very much a work in progress. +It doesn't handle many cases properly. +It throws exceptions in many cases. +It probably segfaults in many cases. +Bug reports and feature requests are welcome. +Complaints are not. :R + +*/ + + +/** + * Protocol that prints the payload in a nice human-readable format. + * Reading from this protocol is not supported. + * + */ +class TDebugProtocol : public TVirtualProtocol { + private: + enum write_state_t + { UNINIT + , STRUCT + , LIST + , SET + , MAP_KEY + , MAP_VALUE + }; + + public: + TDebugProtocol(boost::shared_ptr trans) + : TVirtualProtocol(trans) + , trans_(trans.get()) + , string_limit_(DEFAULT_STRING_LIMIT) + , string_prefix_size_(DEFAULT_STRING_PREFIX_SIZE) + { + write_state_.push_back(UNINIT); + } + + static const int32_t DEFAULT_STRING_LIMIT = 256; + static const int32_t DEFAULT_STRING_PREFIX_SIZE = 16; + + void setStringSizeLimit(int32_t string_limit) { + string_limit_ = string_limit; + } + + void setStringPrefixSize(int32_t string_prefix_size) { + string_prefix_size_ = string_prefix_size; + } + + + uint32_t writeMessageBegin(const std::string& name, + const TMessageType messageType, + const int32_t seqid); + + uint32_t writeMessageEnd(); + + + uint32_t writeStructBegin(const char* name); + + uint32_t writeStructEnd(); + + uint32_t writeFieldBegin(const char* name, + const TType fieldType, + const int16_t fieldId); + + uint32_t writeFieldEnd(); + + uint32_t writeFieldStop(); + + uint32_t writeMapBegin(const TType keyType, + const TType valType, + const uint32_t size); + + uint32_t writeMapEnd(); + + uint32_t writeListBegin(const TType elemType, + const uint32_t size); + + uint32_t writeListEnd(); + + uint32_t writeSetBegin(const TType elemType, + const uint32_t size); + + uint32_t writeSetEnd(); + + uint32_t writeBool(const bool value); + + uint32_t writeByte(const int8_t byte); + + uint32_t writeI16(const int16_t i16); + + uint32_t writeI32(const int32_t i32); + + uint32_t writeI64(const int64_t i64); + + uint32_t writeDouble(const double dub); + + uint32_t writeString(const std::string& str); + + template + uint32_t writeString(const StrType& str) { + return writeString(std::string(str.data(), str.size())); + } + + uint32_t writeBinary(const std::string& str); + + + private: + void indentUp(); + void indentDown(); + uint32_t writePlain(const std::string& str); + uint32_t writeIndented(const std::string& str); + uint32_t startItem(); + uint32_t endItem(); + uint32_t writeItem(const std::string& str); + + static std::string fieldTypeName(TType type); + + TTransport* trans_; + + int32_t string_limit_; + int32_t string_prefix_size_; + + std::string indent_str_; + static const int indent_inc = 2; + + std::vector write_state_; + std::vector list_idx_; +}; + +/** + * Constructs debug protocol handlers + */ +class TDebugProtocolFactory : public TProtocolFactory { + public: + TDebugProtocolFactory() {} + virtual ~TDebugProtocolFactory() {} + + boost::shared_ptr getProtocol(boost::shared_ptr trans) { + return boost::shared_ptr(new TDebugProtocol(trans)); + } + +}; + +}}} // apache::thrift::protocol + + +// TODO(dreiss): Move (part of) ThriftDebugString into a .cpp file and remove this. +#include "thrift/lib/cpp/transport/TBufferTransports.h" + +namespace apache { namespace thrift { + +template +struct ThriftTypeTraits { + static const apache::thrift::protocol::TType fieldType_ = + apache::thrift::protocol::TType(99); +}; + +template<> +struct ThriftTypeTraits { + static const apache::thrift::protocol::TType fieldType_ = + apache::thrift::protocol::T_BOOL; +}; + +template<> +struct ThriftTypeTraits { + static const apache::thrift::protocol::TType fieldType_ = + apache::thrift::protocol::T_I08; +}; + +template<> +struct ThriftTypeTraits { + static const apache::thrift::protocol::TType fieldType_ = + apache::thrift::protocol::T_I16; +}; + +template<> +struct ThriftTypeTraits { + static const apache::thrift::protocol::TType fieldType_ = + apache::thrift::protocol::T_I32; +}; + +template<> +struct ThriftTypeTraits { + static const apache::thrift::protocol::TType fieldType_ = + apache::thrift::protocol::T_I64; +}; + +template<> +struct ThriftTypeTraits { + static const apache::thrift::protocol::TType fieldType_ = + apache::thrift::protocol::T_DOUBLE; +}; + +template<> +struct ThriftTypeTraits { + static const apache::thrift::protocol::TType fieldType_ = + apache::thrift::protocol::T_STRING; +}; + +class TDebugProtocolEx : public apache::thrift::protocol::TDebugProtocol { +public: + TDebugProtocolEx( + boost::shared_ptr trans + ) + : TDebugProtocol(trans) { + } + + template + void write(const T& t) { + t.write(this); + } + + template + void write(const std::vector& c) { + writeListBegin(ThriftTypeTraits::fieldType_, c.size()); + typeof(c.begin()) it = c.begin(); + for (; it != c.end(); it++) { + write(*it); + } + writeListEnd(); + } + + template + void write(const std::map& c) { + writeMapBegin( + ThriftTypeTraits::fieldType_, + ThriftTypeTraits::fieldType_, + c.size() + ); + typeof(c.begin()) it = c.begin(); + for (; it != c.end(); it++) { + write(it->first); + write(it->second); + } + writeMapEnd(); + } + + template + void write(const std::multimap& c) { + writeMapBegin( + ThriftTypeTraits::fieldType_, + ThriftTypeTraits::fieldType_, + c.size() + ); + typeof(c.begin()) it = c.begin(); + for (; it != c.end(); it++) { + write(it->first); + write(it->second); + } + writeMapEnd(); + } + + + template + void write(const std::set& c) { + writeSetBegin( + ThriftTypeTraits::fieldType_, + c.size() + ); + typeof(c.begin()) it = c.begin(); + for (; it != c.end(); it++) { + write(*it); + } + writeSetEnd(); + } + + void write(const bool value) { + writeBool(value); + } + + void write(const int8_t byte) { + writeByte(byte); + } + + void write(const int16_t i16) { + writeI16(i16); + } + + void write(const int32_t i32) { + writeI32(i32); + } + + void write(const int64_t i64) { + writeI64(i64); + } + + void write(const double dub) { + writeDouble(dub); + } + + void write(const std::string& str) { + writeString(str); + } + + template + void write(const std::pair& p) { + writeStructBegin("pair"); + writeFieldBegin("first", ThriftTypeTraits::fieldType_, 1); + write(p.first); + writeFieldEnd(); + writeFieldBegin("second", ThriftTypeTraits::fieldType_, 2); + write(p.second); + writeFieldEnd(); + writeStructEnd(); + } +}; + +template +std::string ThriftDebugString(const T& ts) { + using namespace apache::thrift::transport; + using namespace apache::thrift::protocol; + TMemoryBuffer* buffer = new TMemoryBuffer; + boost::shared_ptr trans(buffer); + TDebugProtocolEx protocol(trans); + + protocol.write(ts); + + uint8_t* buf; + uint32_t size; + buffer->getBuffer(&buf, &size); + return std::string((char*)buf, (unsigned int)size); +} + +}} // apache::thrift + + +#endif // #ifndef _THRIFT_PROTOCOL_TDEBUGPROTOCOL_H_ + + diff --git a/thrift/lib/cpp/protocol/THeaderProtocol.h b/thrift/lib/cpp/protocol/THeaderProtocol.h new file mode 100644 index 0000000000..ac9730cc46 --- /dev/null +++ b/thrift/lib/cpp/protocol/THeaderProtocol.h @@ -0,0 +1,370 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef THRIFT_PROTOCOL_THEADERPROTOCOL_H_ +#define THRIFT_PROTOCOL_THEADERPROTOCOL_H_ 1 + +#include "thrift/lib/cpp/protocol/TProtocol.h" +#include "thrift/lib/cpp/protocol/TProtocolTypes.h" +#include "thrift/lib/cpp/protocol/TVirtualProtocol.h" +#include "thrift/lib/cpp/transport/THeaderTransport.h" +#include "thrift/lib/cpp/util/shared_ptr_util.h" + +#include + +#include + +using apache::thrift::transport::THeaderTransport; +using apache::thrift::transport::TTransportPair; + +namespace apache { namespace thrift { namespace protocol { + +/** + * The header protocol for thrift. Reads unframed, framed, header format, + * and http + * + */ +class THeaderProtocol + : public TVirtualProtocol { + public: + explicit THeaderProtocol(const boost::shared_ptr& trans, + std::bitset* clientTypes = NULL, + uint16_t protoId = T_COMPACT_PROTOCOL) : + TVirtualProtocol(getTransportWrapper(trans, + clientTypes)) + , trans_(boost::dynamic_pointer_cast( + this->getTransport())) + , protoId_(protoId) + { + trans_->setProtocolId(protoId); + resetProtocol(); + } + + THeaderProtocol(const boost::shared_ptr& inTrans, + const boost::shared_ptr& outTrans, + std::bitset* clientTypes = NULL, + uint16_t protoId = T_COMPACT_PROTOCOL) : + TVirtualProtocol(getInOutTransportWrapper(inTrans, + outTrans, + clientTypes)) + , trans_(boost::dynamic_pointer_cast( + this->getTransport())) + , protoId_(protoId) + { + trans_->setProtocolId(protoId); + resetProtocol(); + } + + /** + * Construct a THeaderProtocol using a raw pointer to the transport. + * + * The caller is responsible for ensuring that the transport remains valid + * for the lifetime of the protocol. + */ + THeaderProtocol(TTransport* trans, + std::bitset* clientTypes, + uint16_t protoId = T_COMPACT_PROTOCOL) : + TVirtualProtocol( + getTransportWrapper( + boost::shared_ptr(trans, + NoopPtrDestructor()), + clientTypes)) + , trans_(boost::dynamic_pointer_cast( + this->getTransport())) + , protoId_(protoId) + { + trans_->setProtocolId(protoId); + resetProtocol(); + } + + ~THeaderProtocol() {} + + /** + * Functions to work with headers by calling into THeaderTransport + */ + void setProtocolId(uint16_t protoId) { + trans_->setProtocolId(protoId); + resetProtocol(); + } + + void resetProtocol(); + + typedef THeaderTransport::StringToStringMap StringToStringMap; + + // these work with write headers + void setHeader(const std::string& key, const std::string& value) { + trans_->setHeader(key, value); + } + + void setPersistentHeader(const std::string& key, const std::string& value) { + trans_->setPersistentHeader(key, value); + } + + void clearHeaders() { + trans_->clearHeaders(); + } + + void clearPersistentHeaders() { + trans_->clearPersistentHeaders(); + } + + StringToStringMap& getWriteHeaders() { + return trans_->getWriteHeaders(); + } + + StringToStringMap& getPersistentWriteHeaders() { + return trans_->getPersistentWriteHeaders(); + } + + // these work with read headers + const StringToStringMap& getHeaders() const { + return trans_->getHeaders(); + } + + void setTransform(uint16_t trans) { + trans_->setTransform(trans); + } + + std::string getPeerIdentity() const { + return trans_->getPeerIdentity(); + } + void setIdentity(const std::string& identity) { + trans_->setIdentity(identity); + } + + void setHmac(THeaderTransport::MacCallback macCb, + THeaderTransport::VerifyMacCallback verifyCb) { + trans_->setHmac(macCb, verifyCb); + } + + /** + * Writing functions. + */ + + /*ol*/ uint32_t writeMessageBegin(const std::string& name, + const TMessageType messageType, + const int32_t seqId); + + /*ol*/ uint32_t writeMessageEnd(); + + + uint32_t writeStructBegin(const char* name); + + uint32_t writeStructEnd(); + + uint32_t writeFieldBegin(const char* name, + const TType fieldType, + const int16_t fieldId); + + uint32_t writeFieldEnd(); + + uint32_t writeFieldStop(); + + uint32_t writeMapBegin(const TType keyType, + const TType valType, + const uint32_t size); + + uint32_t writeMapEnd(); + + uint32_t writeListBegin(const TType elemType, const uint32_t size); + + uint32_t writeListEnd(); + + uint32_t writeSetBegin(const TType elemType, const uint32_t size); + + uint32_t writeSetEnd(); + + uint32_t writeBool(const bool value); + + uint32_t writeByte(const int8_t byte); + + uint32_t writeI16(const int16_t i16); + + uint32_t writeI32(const int32_t i32); + + uint32_t writeI64(const int64_t i64); + + uint32_t writeDouble(const double dub); + + uint32_t writeString(const std::string& str); + + uint32_t writeBinary(const std::string& str); + + /** + * Reading functions + */ + + + /*ol*/ uint32_t readMessageBegin(std::string& name, + TMessageType& messageType, + int32_t& seqId); + + /*ol*/ uint32_t readMessageEnd(); + + uint32_t readStructBegin(std::string& name); + + uint32_t readStructEnd(); + + uint32_t readFieldBegin(std::string& name, + TType& fieldType, + int16_t& fieldId); + + uint32_t readFieldEnd(); + + uint32_t readMapBegin(TType& keyType, + TType& valType, + uint32_t& size); + + uint32_t readMapEnd(); + + uint32_t readListBegin(TType& elemType, uint32_t& size); + + uint32_t readListEnd(); + + uint32_t readSetBegin(TType& elemType, uint32_t& size); + + uint32_t readSetEnd(); + + uint32_t readBool(bool& value); + // Provide the default readBool() implementation for std::vector + using TVirtualProtocol< THeaderProtocol >::readBool; + + uint32_t readByte(int8_t& byte); + + uint32_t readI16(int16_t& i16); + + uint32_t readI32(int32_t& i32); + + uint32_t readI64(int64_t& i64); + + uint32_t readDouble(double& dub); + + uint32_t readString(std::string& str); + + uint32_t readBinary(std::string& binary); + + protected: + template + uint32_t readStringBody(StrType& str, int32_t sz); + + boost::shared_ptr getTransportWrapper( + const boost::shared_ptr& trans, + std::bitset* clientTypes) { + if (dynamic_cast(trans.get()) != NULL) { + return trans; + } else { + return boost::shared_ptr( + new THeaderTransport(trans, clientTypes)); + } + } + + boost::shared_ptr getInOutTransportWrapper( + const boost::shared_ptr& inTrans, + const boost::shared_ptr& outTrans, + std::bitset* clientTypes) { + assert(dynamic_cast(inTrans.get()) == NULL + && dynamic_cast(outTrans.get()) == NULL); + + return boost::shared_ptr( + new THeaderTransport(inTrans, outTrans, clientTypes) + ); + } + + boost::shared_ptr trans_; + + boost::shared_ptr proto_; + uint32_t protoId_; +}; + +/** + * Constructs header protocol handlers + */ +class THeaderProtocolFactory : public TDuplexProtocolFactory { + public: + explicit THeaderProtocolFactory(uint16_t protoId = T_COMPACT_PROTOCOL, + bool disableIdentity = false) { + protoId_ = protoId; + setIdentity_ = disableIdentity; + } + + virtual ~THeaderProtocolFactory() {} + + void setClientTypes(std::bitset& clientTypes) { + for (int i = 0; i < CLIENT_TYPES_LEN; i++) { + this->clientTypes[i] = clientTypes[i]; + } + } + + void setIdentity(const std::string& identity) { + identity_ = identity; + setIdentity_ = true; + } + + void setTransform(uint16_t trans) { + trans_.push_back(trans); + } + + virtual TProtocolPair getProtocol( + boost::shared_ptr trans) { + THeaderProtocol* prot = new THeaderProtocol(trans, &clientTypes, protoId_); + + if(setIdentity_) { + prot->setIdentity(identity_); + } + + for (auto& t : trans_) { + prot->setTransform(t); + } + + boost::shared_ptr pprot(prot); + return TProtocolPair(pprot, pprot); + } + + virtual TProtocolPair getProtocol(TTransportPair transports) { + THeaderProtocol* prot = new THeaderProtocol(transports.first, + transports.second, + &clientTypes, + protoId_); + + if(setIdentity_) { + prot->setIdentity(identity_); + } + + for (auto& t : trans_) { + prot->setTransform(t); + } + + boost::shared_ptr pprot(prot); + return TProtocolPair(pprot, pprot); + } + + // No implementation of getInputProtocolFactory/getOutputProtocolFactory + // Using base class implementation which return NULL. + + private: + std::bitset clientTypes; + uint16_t protoId_; + bool setIdentity_; + std::vector trans_; + std::string identity_; +}; + +}}} // apache::thrift::protocol + +#endif // #ifndef THRIFT_PROTOCOL_THEADERPROTOCOL_H_ diff --git a/thrift/lib/cpp/protocol/TJSONProtocol.h b/thrift/lib/cpp/protocol/TJSONProtocol.h new file mode 100644 index 0000000000..71ca8e8771 --- /dev/null +++ b/thrift/lib/cpp/protocol/TJSONProtocol.h @@ -0,0 +1,348 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef THRIFT_PROTOCOL_TJSONPROTOCOL_H_ +#define THRIFT_PROTOCOL_TJSONPROTOCOL_H_ 1 + +#include "TVirtualProtocol.h" + +#include + +namespace apache { namespace thrift { namespace protocol { + +// Forward declaration +class TJSONContext; + +/** + * JSON protocol for Thrift. + * + * Implements a protocol which uses JSON as the wire-format. + * + * Thrift types are represented as described below: + * + * 1. Every Thrift integer type is represented as a JSON number. + * + * 2. Thrift doubles are represented as JSON numbers. Some special values are + * represented as strings: + * a. "NaN" for not-a-number values + * b. "Infinity" for positive infinity + * c. "-Infinity" for negative infinity + * + * 3. Thrift string values are emitted as JSON strings, with appropriate + * escaping. + * + * 4. Thrift binary values are encoded into Base64 and emitted as JSON strings. + * The readBinary() method is written such that it will properly skip if + * called on a Thrift string (although it will decode garbage data). + * + * 5. Thrift structs are represented as JSON objects, with the field ID as the + * key, and the field value represented as a JSON object with a single + * key-value pair. The key is a short string identifier for that type, + * followed by the value. The valid type identifiers are: "tf" for bool, + * "i8" for byte, "i16" for 16-bit integer, "i32" for 32-bit integer, "i64" + * for 64-bit integer, "dbl" for double-precision floating point, "str" for + * string (including binary), "rec" for struct ("records"), "map" for map, + * "lst" for list, "set" for set. + * + * 6. Thrift lists and sets are represented as JSON arrays, with the first + * element of the JSON array being the string identifier for the Thrift + * element type and the second element of the JSON array being the count of + * the Thrift elements. The Thrift elements then follow. + * + * 7. Thrift maps are represented as JSON arrays, with the first two elements + * of the JSON array being the string identifiers for the Thrift key type + * and value type, followed by the count of the Thrift pairs, followed by a + * JSON object containing the key-value pairs. Note that JSON keys can only + * be strings, which means that the key type of the Thrift map should be + * restricted to numeric or string types -- in the case of numerics, they + * are serialized as strings. + * + * 8. Thrift messages are represented as JSON arrays, with the protocol + * version #, the message name, the message type, and the sequence ID as + * the first 4 elements. + * + * More discussion of the double handling is probably warranted. The aim of + * the current implementation is to match as closely as possible the behavior + * of Java's Double.toString(), which has no precision loss. Implementors in + * other languages should strive to achieve that where possible. I have not + * yet verified whether boost:lexical_cast, which is doing that work for me in + * C++, loses any precision, but I am leaving this as a future improvement. I + * may try to provide a C component for this, so that other languages could + * bind to the same underlying implementation for maximum consistency. + * + * Note further that JavaScript itself is not capable of representing + * floating point infinities -- presumably when we have a JavaScript Thrift + * client, this would mean that infinities get converted to not-a-number in + * transmission. I don't know of any work-around for this issue. + * + */ +class TJSONProtocol : public TVirtualProtocol { + public: + + TJSONProtocol(boost::shared_ptr ptrans); + + ~TJSONProtocol(); + + private: + + void pushContext(boost::shared_ptr c); + + void popContext(); + + protected: + + uint32_t writeJSONEscapeChar(uint8_t ch); + + uint32_t writeJSONChar(uint8_t ch); + + uint32_t writeJSONString(const std::string &str); + + uint32_t writeJSONBase64(const std::string &str); + + template + uint32_t writeJSONInteger(NumberType num); + + uint32_t writeJSONBool(bool value); + + uint32_t writeJSONDouble(double num); + + uint32_t writeJSONObjectStart() ; + + uint32_t writeJSONObjectEnd(); + + uint32_t writeJSONArrayStart(); + + uint32_t writeJSONArrayEnd(); + + uint32_t readJSONSyntaxChar(uint8_t ch); + + uint32_t readJSONEscapeChar(uint8_t *out); + + uint32_t readJSONString(std::string &str, bool skipContext = false); + + uint32_t readJSONBase64(std::string &str); + + uint32_t readJSONNumericChars(std::string &str); + + template + uint32_t readJSONInteger(NumberType &num); + + uint32_t readJSONDouble(double &num); + + uint32_t readJSONObjectStart(); + + uint32_t readJSONObjectEnd(); + + uint32_t readJSONArrayStart(); + + uint32_t readJSONArrayEnd(); + + public: + + /** + * Writing functions. + */ + + uint32_t writeMessageBegin(const std::string& name, + const TMessageType messageType, + const int32_t seqid); + + uint32_t writeMessageEnd(); + + uint32_t writeStructBegin(const char* name); + + uint32_t writeStructEnd(); + + uint32_t writeFieldBegin(const char* name, + const TType fieldType, + const int16_t fieldId); + + uint32_t writeFieldEnd(); + + uint32_t writeFieldStop(); + + uint32_t writeMapBegin(const TType keyType, + const TType valType, + const uint32_t size); + + uint32_t writeMapEnd(); + + uint32_t writeListBegin(const TType elemType, + const uint32_t size); + + uint32_t writeListEnd(); + + uint32_t writeSetBegin(const TType elemType, + const uint32_t size); + + uint32_t writeSetEnd(); + + uint32_t writeBool(const bool value); + + uint32_t writeByte(const int8_t byte); + + uint32_t writeI16(const int16_t i16); + + uint32_t writeI32(const int32_t i32); + + uint32_t writeI64(const int64_t i64); + + uint32_t writeDouble(const double dub); + + uint32_t writeString(const std::string& str); + + uint32_t writeBinary(const std::string& str); + + /** + * Reading functions + */ + + uint32_t readMessageBegin(std::string& name, + TMessageType& messageType, + int32_t& seqid); + + uint32_t readMessageEnd(); + + uint32_t readStructBegin(std::string& name); + + uint32_t readStructEnd(); + + uint32_t readFieldBegin(std::string& name, + TType& fieldType, + int16_t& fieldId); + + uint32_t readFieldEnd(); + + uint32_t readMapBegin(TType& keyType, + TType& valType, + uint32_t& size); + + uint32_t readMapEnd(); + + uint32_t readListBegin(TType& elemType, + uint32_t& size); + + uint32_t readListEnd(); + + uint32_t readSetBegin(TType& elemType, + uint32_t& size); + + uint32_t readSetEnd(); + + uint32_t readBool(bool& value); + + // Provide the default readBool() implementation for std::vector + using TVirtualProtocol::readBool; + + uint32_t readByte(int8_t& byte); + + uint32_t readI16(int16_t& i16); + + uint32_t readI32(int32_t& i32); + + uint32_t readI64(int64_t& i64); + + uint32_t readDouble(double& dub); + + uint32_t readString(std::string& str); + + uint32_t readBinary(std::string& str); + + class LookaheadReader { + + public: + + LookaheadReader(TTransport &trans) : + trans_(&trans), + hasData_(false) { + } + + uint8_t read() { + if (hasData_) { + hasData_ = false; + } + else { + trans_->readAll(&data_, 1); + } + return data_; + } + + uint8_t peek() { + if (!hasData_) { + trans_->readAll(&data_, 1); + } + hasData_ = true; + return data_; + } + + private: + TTransport *trans_; + bool hasData_; + uint8_t data_; + }; + + private: + TTransport* trans_; + + std::stack > contexts_; + boost::shared_ptr context_; + LookaheadReader reader_; +}; + +/** + * Constructs input and output protocol objects given transports. + */ +class TJSONProtocolFactory : public TProtocolFactory { + public: + TJSONProtocolFactory() {} + + virtual ~TJSONProtocolFactory() {} + + boost::shared_ptr getProtocol(boost::shared_ptr trans) { + return boost::shared_ptr(new TJSONProtocol(trans)); + } +}; + +}}} // apache::thrift::protocol + + +// TODO(dreiss): Move part of ThriftJSONString into a .cpp file and remove this. +#include "thrift/lib/cpp/transport/TBufferTransports.h" + +/*namespace apache { namespace thrift { + +template + std::string ThriftJSONString(const ThriftStruct& ts) { + using namespace apache::thrift::transport; + using namespace apache::thrift::protocol; + TMemoryBuffer* buffer = new TMemoryBuffer; + boost::shared_ptr trans(buffer); + TJSONProtocol protocol(trans); + + ts.write(&protocol); + + uint8_t* buf; + uint32_t size; + buffer->getBuffer(&buf, &size); + return std::string((char*)buf, (unsigned int)size); +} + +}} // apache::thrift +*/ +#endif // #define THRIFT_PROTOCOL_TJSONPROTOCOL_H_ 1 diff --git a/thrift/lib/cpp/protocol/TNeutroniumProtocol.h b/thrift/lib/cpp/protocol/TNeutroniumProtocol.h new file mode 100644 index 0000000000..dfeae1e4a1 --- /dev/null +++ b/thrift/lib/cpp/protocol/TNeutroniumProtocol.h @@ -0,0 +1,287 @@ +/** + * Copyright 2012 Facebook + * @author Tudor Bosman (tudorb@fb.com) + */ + +#ifndef THRIFT_LIB_CPP_PROTOCOL_TNEUTRONIUMPROTOCOL_H_ +#define THRIFT_LIB_CPP_PROTOCOL_TNEUTRONIUMPROTOCOL_H_ + +#include "thrift/lib/cpp/protocol/TProtocol.h" +#include "thrift/lib/cpp/protocol/TVirtualProtocol.h" +#include "thrift/lib/cpp/protocol/neutronium/Encoder.h" +#include "thrift/lib/cpp/protocol/neutronium/Decoder.h" + +namespace apache { namespace thrift { namespace protocol { + +class TNeutroniumProtocol + : public TVirtualProtocol { + + public: + TNeutroniumProtocol(const neutronium::Schema* schema, + neutronium::InternTable* internTable, + folly::IOBuf* buf) + : TVirtualProtocol(nullptr), + enc_(schema, internTable, buf), + dec_(schema, internTable, buf) { + } + + void setRootType(int64_t rootType) { + enc_.setRootType(rootType); + dec_.setRootType(rootType); + } + + uint32_t writeMessageBegin(const std::string& name, + const TMessageType messageType, + const int32_t seqid) { + LOG(FATAL) << "Message encoding / decoding not implemented"; + } + + uint32_t writeMessageEnd() { + LOG(FATAL) << "Message encoding / decoding not implemented"; + } + + uint32_t writeStructBegin(const char* name) { + enc_.writeStructBegin(name); + return 0; + } + + uint32_t writeStructEnd() { + enc_.writeStructEnd(); + return enc_.bytesWritten(); + } + + uint32_t writeFieldBegin(const char* name, + const TType fieldType, + const int16_t fieldId) { + enc_.writeFieldBegin(name, fieldType, fieldId); + return 0; + } + + uint32_t writeFieldEnd() { + enc_.writeFieldEnd(); + return 0; + } + + uint32_t writeFieldStop() { + enc_.writeFieldStop(); + return 0; + } + + uint32_t writeMapBegin(const TType keyType, + const TType valType, + const uint32_t size) { + enc_.writeMapBegin(keyType, valType, size); + return 0; + } + + uint32_t writeMapEnd() { + enc_.writeMapEnd(); + return 0; + } + + uint32_t writeListBegin(const TType elemType, const uint32_t size) { + enc_.writeListBegin(elemType, size); + return 0; + } + + uint32_t writeListEnd() { + enc_.writeListEnd(); + return 0; + } + + uint32_t writeSetBegin(const TType elemType, const uint32_t size) { + enc_.writeSetBegin(elemType, size); + return 0; + } + + uint32_t writeSetEnd() { + enc_.writeSetEnd(); + return 0; + } + + uint32_t writeBool(const bool value) { + enc_.writeBool(value); + return 0; + } + + uint32_t writeByte(const int8_t byte) { + enc_.writeByte(byte); + return 0; + } + + uint32_t writeI16(const int16_t i16) { + enc_.writeI16(i16); + return 0; + } + + uint32_t writeI32(const int32_t i32) { + enc_.writeI32(i32); + return 0; + } + + uint32_t writeI64(const int64_t i64) { + enc_.writeI64(i64); + return 0; + } + + uint32_t writeDouble(const double dub) { + enc_.writeDouble(dub); + return 0; + } + + template + uint32_t writeString(const StrType& str) { + enc_.writeString(str); + return 0; + } + + uint32_t writeBinary(const std::string& str) { + enc_.writeBinary(str); + return 0; + } + + + /** + * Reading functions + */ + + + uint32_t readMessageBegin(std::string& name, + TMessageType& messageType, + int32_t& seqid) { + LOG(FATAL) << "Message encoding / decoding not implemented"; + } + + uint32_t readMessageEnd() { + LOG(FATAL) << "Message encoding / decoding not implemented"; + } + + uint32_t readStructBegin(std::string& name) { + dec_.readStructBegin(); + return 0; + } + + uint32_t readStructEnd() { + dec_.readStructEnd(); + return dec_.bytesRead(); + } + + uint32_t readFieldBegin(std::string& name, + TType& fieldType, + int16_t& fieldId) { + dec_.readFieldBegin(fieldType, fieldId); + return 0; + } + + uint32_t readFieldEnd() { + dec_.readFieldEnd(); + return 0; + } + + uint32_t readMapBegin(TType& keyType, TType& valType, uint32_t& size) { + dec_.readMapBegin(keyType, valType, size); + return 0; + } + + uint32_t readMapEnd() { + dec_.readMapEnd(); + return 0; + } + + uint32_t readListBegin(TType& elemType, uint32_t& size) { + dec_.readListBegin(elemType, size); + return 0; + } + + uint32_t readListEnd() { + dec_.readListEnd(); + return 0; + } + + uint32_t readSetBegin(TType& elemType, uint32_t& size) { + dec_.readSetBegin(elemType, size); + return 0; + } + + uint32_t readSetEnd() { + dec_.readSetEnd(); + return 0; + } + + uint32_t readBool(bool& value) { + dec_.readBool(value); + return 0; + } + // Provide the default readBool() implementation for std::vector + using TVirtualProtocol::readBool; + + uint32_t readByte(int8_t& byte) { + dec_.readByte(byte); + return 0; + } + + uint32_t readI16(int16_t& i16) { + dec_.readI16(i16); + return 0; + } + + uint32_t readI32(int32_t& i32) { + dec_.readI32(i32); + return 0; + } + + uint32_t readI64(int64_t& i64) { + dec_.readI64(i64); + return 0; + } + + uint32_t readDouble(double& dub) { + dec_.readDouble(dub); + return 0; + } + + template + uint32_t readString(StrType& str) { + dec_.readString(str); + return 0; + } + + uint32_t readBinary(std::string& str) { + dec_.readBinary(str); + return 0; + } + + private: + neutronium::Encoder enc_; + neutronium::Decoder dec_; +}; + +class Neutronium { + public: + explicit Neutronium(const neutronium::Schema* schema, + neutronium::InternTable* internTable, + folly::IOBuf* buf) + : proto_(schema, internTable, buf) { + } + + template + uint32_t serialize(const T& obj) { + proto_.setRootType(T::_reflection_id); + return obj.write(&proto_); + } + + template + uint32_t deserialize(T& obj) { + proto_.setRootType(T::_reflection_id); + return obj.read(&proto_); + } + + private: + TNeutroniumProtocol proto_; +}; + + +}}} // apache::thrift::protocol + +#endif /* THRIFT_LIB_CPP_PROTOCOL_TNEUTRONIUMPROTOCOL_H_ */ + diff --git a/thrift/lib/cpp/protocol/TPhpSerializeProtocol.h b/thrift/lib/cpp/protocol/TPhpSerializeProtocol.h new file mode 100644 index 0000000000..d193d423fe --- /dev/null +++ b/thrift/lib/cpp/protocol/TPhpSerializeProtocol.h @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_PROTOCOL_TPHPSERIALIZEPROTOCOL_H_ +#define _THRIFT_PROTOCOL_TPHPSERIALIZEPROTOCOL_H_ 1 + +#include "thrift/lib/cpp/protocol/TVirtualProtocol.h" +#include "thrift/lib/cpp/transport/TBufferTransports.h" + +#include +#include + +namespace apache { namespace thrift { namespace protocol { + + +/** + * A Thrift protocol for serializing Thrift objects into PHP's + * "serialize" format. Should work properly for objects that + * PHP can properly express. Currently, it can silently corrupt + * data that PHP cannot properly express (lists or bools as map keys, + * very large integers on 32-bit systems, and possibly others). + */ + +class TPhpSerializeProtocol : public TVirtualProtocol { + public: + TPhpSerializeProtocol(boost::shared_ptr trans) + : TVirtualProtocol(trans) + , trans_(trans.get()) + {} + + uint32_t writeMessageBegin(const std::string& name, + const TMessageType messageType, + const int32_t seqid); + + uint32_t writeMessageEnd(); + + uint32_t writeStructBegin(const char* name); + + uint32_t writeStructEnd(); + + uint32_t writeFieldBegin(const char* name, + const TType fieldType, + const int16_t fieldId); + + uint32_t writeFieldEnd(); + + uint32_t writeFieldStop(); + + uint32_t writeListBegin(const TType elemType, + const uint32_t size); + + uint32_t writeListEnd(); + + uint32_t writeSetBegin(const TType elemType, + const uint32_t size); + + uint32_t writeSetEnd(); + + uint32_t writeMapBegin(const TType keyType, + const TType valType, + const uint32_t size); + + uint32_t writeMapEnd(); + + uint32_t writeBool(const bool value); + + uint32_t writeByte(const int8_t byte); + + uint32_t writeI16(const int16_t i16); + + uint32_t writeI32(const int32_t i32); + + uint32_t writeI64(const int64_t i64); + + uint32_t writeDouble(const double dub); + + uint32_t writeString(const std::string& str); + + uint32_t writeBinary(const std::string& str); + + protected: + uint32_t doWriteInt(const int64_t i64); + uint32_t doWriteInt(const std::string& val); + uint32_t doWriteString(const std::string& str, bool is_class); + uint32_t doWriteListBegin(uint32_t size, bool is_map); + uint32_t listKey(); + uint32_t write3(const char* v1, const char* v2, const char* v3); + uint32_t write(const char* buf, uint32_t len); + + std::stack listPosStack_; + std::stack structSizeStack_; + std::stack< boost::shared_ptr > structBufferStack_; + + TTransport* trans_; +}; + +}}} // apache::thrift::protocol + +#endif diff --git a/thrift/lib/cpp/protocol/TProtocol.h b/thrift/lib/cpp/protocol/TProtocol.h new file mode 100644 index 0000000000..f35fa09a0a --- /dev/null +++ b/thrift/lib/cpp/protocol/TProtocol.h @@ -0,0 +1,753 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef THRIFT_PROTOCOL_TPROTOCOL_H_ +#define THRIFT_PROTOCOL_TPROTOCOL_H_ 1 + +#include "thrift/lib/cpp/transport/TTransport.h" +#include "thrift/lib/cpp/protocol/TProtocolException.h" +#include "thrift/lib/cpp/util/BitwiseCast.h" +#include "thrift/lib/cpp/util/shared_ptr_util.h" + +#include + +#include +#include +#include +#include +#include + +namespace apache { namespace thrift { namespace protocol { + +using apache::thrift::transport::TTransport; + +#ifdef THRIFT_HAVE_ENDIAN_H +#include +#endif + +#ifndef __BYTE_ORDER +# if defined(BYTE_ORDER) && defined(LITTLE_ENDIAN) && defined(BIG_ENDIAN) +# define __BYTE_ORDER BYTE_ORDER +# define __LITTLE_ENDIAN LITTLE_ENDIAN +# define __BIG_ENDIAN BIG_ENDIAN +# else +# error "Cannot determine endianness" +# endif +#endif + +#if __BYTE_ORDER == __BIG_ENDIAN +# if !defined(htonll) && !defined(ntohll) +# define ntohll(n) (n) +# define htonll(n) (n) +# endif /* !defined(htonll) && !defined(ntohll) */ +# if defined(__GNUC__) && defined(__GLIBC__) +# include +# define htolell(n) bswap_64(n) +# define letohll(n) bswap_64(n) +# else /* GNUC & GLIBC */ +# define bswap_64(n) \ + ( (((n) & 0xff00000000000000ull) >> 56) \ + | (((n) & 0x00ff000000000000ull) >> 40) \ + | (((n) & 0x0000ff0000000000ull) >> 24) \ + | (((n) & 0x000000ff00000000ull) >> 8) \ + | (((n) & 0x00000000ff000000ull) << 8) \ + | (((n) & 0x0000000000ff0000ull) << 24) \ + | (((n) & 0x000000000000ff00ull) << 40) \ + | (((n) & 0x00000000000000ffull) << 56) ) +# define htolell(n) bswap_64(n) +# define letohll(n) bswap_64(n) +# endif /* GNUC & GLIBC */ +#elif __BYTE_ORDER == __LITTLE_ENDIAN +# define htolell(n) (n) +# define letohll(n) (n) +# if !defined(htonll) && !defined(ntohll) +# if defined(__GNUC__) && defined(__GLIBC__) +# include +# define ntohll(n) bswap_64(n) +# define htonll(n) bswap_64(n) +# else /* GNUC & GLIBC */ +# define ntohll(n) ( (((unsigned long long)ntohl(n)) << 32) + ntohl(n >> 32) ) +# define htonll(n) ( (((unsigned long long)htonl(n)) << 32) + htonl(n >> 32) ) +# endif /* GNUC & GLIBC */ +# endif /* !defined(htonll) && !defined(ntohll) */ +#else /* __BYTE_ORDER */ +# error "Can't define htonll or ntohll!" +#endif + +/** + * Enumerated definition of the types that the Thrift protocol supports. + * Take special note of the T_END type which is used specifically to mark + * the end of a sequence of fields. + */ +enum TType { + T_STOP = 0, + T_VOID = 1, + T_BOOL = 2, + T_BYTE = 3, + T_I08 = 3, + T_I16 = 6, + T_I32 = 8, + T_U64 = 9, + T_I64 = 10, + T_DOUBLE = 4, + T_STRING = 11, + T_UTF7 = 11, + T_STRUCT = 12, + T_MAP = 13, + T_SET = 14, + T_LIST = 15, + T_UTF8 = 16, + T_UTF16 = 17 +}; + +/** + * Enumerated definition of the message types that the Thrift protocol + * supports. + */ +enum TMessageType { + T_CALL = 1, + T_REPLY = 2, + T_EXCEPTION = 3, + T_ONEWAY = 4 +}; + + +/** + * Helper template for implementing TProtocol::skip(). + * + * Templatized to avoid having to make virtual function calls. + */ +template +uint32_t skip(Protocol_& prot, TType type) { + switch (type) { + case T_BOOL: + { + bool boolv; + return prot.readBool(boolv); + } + case T_BYTE: + { + int8_t bytev = 0; + return prot.readByte(bytev); + } + case T_I16: + { + int16_t i16; + return prot.readI16(i16); + } + case T_I32: + { + int32_t i32; + return prot.readI32(i32); + } + case T_I64: + { + int64_t i64; + return prot.readI64(i64); + } + case T_DOUBLE: + { + double dub; + return prot.readDouble(dub); + } + case T_STRING: + { + std::string str; + return prot.readBinary(str); + } + case T_STRUCT: + { + uint32_t result = 0; + std::string name; + int16_t fid; + TType ftype; + result += prot.readStructBegin(name); + while (true) { + result += prot.readFieldBegin(name, ftype, fid); + if (ftype == T_STOP) { + break; + } + result += skip(prot, ftype); + result += prot.readFieldEnd(); + } + result += prot.readStructEnd(); + return result; + } + case T_MAP: + { + uint32_t result = 0; + TType keyType; + TType valType; + uint32_t i, size; + result += prot.readMapBegin(keyType, valType, size); + for (i = 0; i < size; i++) { + result += skip(prot, keyType); + result += skip(prot, valType); + } + result += prot.readMapEnd(); + return result; + } + case T_SET: + { + uint32_t result = 0; + TType elemType; + uint32_t i, size; + result += prot.readSetBegin(elemType, size); + for (i = 0; i < size; i++) { + result += skip(prot, elemType); + } + result += prot.readSetEnd(); + return result; + } + case T_LIST: + { + uint32_t result = 0; + TType elemType; + uint32_t i, size; + result += prot.readListBegin(elemType, size); + for (i = 0; i < size; i++) { + result += skip(prot, elemType); + } + result += prot.readListEnd(); + return result; + } + default: + return 0; + } +} + +/** + * Abstract class for a thrift protocol driver. These are all the methods that + * a protocol must implement. Essentially, there must be some way of reading + * and writing all the base types, plus a mechanism for writing out structs + * with indexed fields. + * + * TProtocol objects should not be shared across multiple encoding contexts, + * as they may need to maintain internal state in some protocols (i.e. XML). + * Note that is is acceptable for the TProtocol module to do its own internal + * buffered reads/writes to the underlying TTransport where appropriate (i.e. + * when parsing an input XML stream, reading should be batched rather than + * looking ahead character by character for a close tag). + * + */ +class TProtocol { + public: + virtual ~TProtocol() {} + + virtual void setVersion_virt(const int8_t version) = 0; + + void setVersion(const int8_t version) { + T_VIRTUAL_CALL(); + return setVersion_virt(version); + } + + /** + * Writing functions. + */ + + virtual uint32_t writeMessageBegin_virt(const std::string& name, + const TMessageType messageType, + const int32_t seqid) = 0; + + virtual uint32_t writeMessageEnd_virt() = 0; + + + virtual uint32_t writeStructBegin_virt(const char* name) = 0; + + virtual uint32_t writeStructEnd_virt() = 0; + + virtual uint32_t writeFieldBegin_virt(const char* name, + const TType fieldType, + const int16_t fieldId) = 0; + + virtual uint32_t writeFieldEnd_virt() = 0; + + virtual uint32_t writeFieldStop_virt() = 0; + + virtual uint32_t writeMapBegin_virt(const TType keyType, + const TType valType, + const uint32_t size) = 0; + + virtual uint32_t writeMapEnd_virt() = 0; + + virtual uint32_t writeListBegin_virt(const TType elemType, + const uint32_t size) = 0; + + virtual uint32_t writeListEnd_virt() = 0; + + virtual uint32_t writeSetBegin_virt(const TType elemType, + const uint32_t size) = 0; + + virtual uint32_t writeSetEnd_virt() = 0; + + virtual uint32_t writeBool_virt(const bool value) = 0; + + virtual uint32_t writeByte_virt(const int8_t byte) = 0; + + virtual uint32_t writeI16_virt(const int16_t i16) = 0; + + virtual uint32_t writeI32_virt(const int32_t i32) = 0; + + virtual uint32_t writeI64_virt(const int64_t i64) = 0; + + virtual uint32_t writeDouble_virt(const double dub) = 0; + + virtual uint32_t writeString_virt(const std::string& str) = 0; + + virtual uint32_t writeBinary_virt(const std::string& str) = 0; + + uint32_t writeMessageBegin(const std::string& name, + const TMessageType messageType, + const int32_t seqid) { + T_VIRTUAL_CALL(); + return writeMessageBegin_virt(name, messageType, seqid); + } + + uint32_t writeMessageEnd() { + T_VIRTUAL_CALL(); + return writeMessageEnd_virt(); + } + + + uint32_t writeStructBegin(const char* name) { + T_VIRTUAL_CALL(); + return writeStructBegin_virt(name); + } + + uint32_t writeStructEnd() { + T_VIRTUAL_CALL(); + return writeStructEnd_virt(); + } + + uint32_t writeFieldBegin(const char* name, + const TType fieldType, + const int16_t fieldId) { + T_VIRTUAL_CALL(); + return writeFieldBegin_virt(name, fieldType, fieldId); + } + + uint32_t writeFieldEnd() { + T_VIRTUAL_CALL(); + return writeFieldEnd_virt(); + } + + uint32_t writeFieldStop() { + T_VIRTUAL_CALL(); + return writeFieldStop_virt(); + } + + uint32_t writeMapBegin(const TType keyType, + const TType valType, + const uint32_t size) { + T_VIRTUAL_CALL(); + return writeMapBegin_virt(keyType, valType, size); + } + + uint32_t writeMapEnd() { + T_VIRTUAL_CALL(); + return writeMapEnd_virt(); + } + + uint32_t writeListBegin(const TType elemType, const uint32_t size) { + T_VIRTUAL_CALL(); + return writeListBegin_virt(elemType, size); + } + + uint32_t writeListEnd() { + T_VIRTUAL_CALL(); + return writeListEnd_virt(); + } + + uint32_t writeSetBegin(const TType elemType, const uint32_t size) { + T_VIRTUAL_CALL(); + return writeSetBegin_virt(elemType, size); + } + + uint32_t writeSetEnd() { + T_VIRTUAL_CALL(); + return writeSetEnd_virt(); + } + + uint32_t writeBool(const bool value) { + T_VIRTUAL_CALL(); + return writeBool_virt(value); + } + + uint32_t writeByte(const int8_t byte) { + T_VIRTUAL_CALL(); + return writeByte_virt(byte); + } + + uint32_t writeI16(const int16_t i16) { + T_VIRTUAL_CALL(); + return writeI16_virt(i16); + } + + uint32_t writeI32(const int32_t i32) { + T_VIRTUAL_CALL(); + return writeI32_virt(i32); + } + + uint32_t writeI64(const int64_t i64) { + T_VIRTUAL_CALL(); + return writeI64_virt(i64); + } + + uint32_t writeDouble(const double dub) { + T_VIRTUAL_CALL(); + return writeDouble_virt(dub); + } + + uint32_t writeString(const std::string& str) { + T_VIRTUAL_CALL(); + return writeString_virt(str); + } + + uint32_t writeBinary(const std::string& str) { + T_VIRTUAL_CALL(); + return writeBinary_virt(str); + } + + /** + * Reading functions + */ + + virtual uint32_t readMessageBegin_virt(std::string& name, + TMessageType& messageType, + int32_t& seqid) = 0; + + virtual uint32_t readMessageEnd_virt() = 0; + + virtual uint32_t readStructBegin_virt(std::string& name) = 0; + + virtual uint32_t readStructEnd_virt() = 0; + + virtual uint32_t readFieldBegin_virt(std::string& name, + TType& fieldType, + int16_t& fieldId) = 0; + + virtual uint32_t readFieldEnd_virt() = 0; + + virtual uint32_t readMapBegin_virt(TType& keyType, + TType& valType, + uint32_t& size) = 0; + + virtual uint32_t readMapEnd_virt() = 0; + + virtual uint32_t readListBegin_virt(TType& elemType, + uint32_t& size) = 0; + + virtual uint32_t readListEnd_virt() = 0; + + virtual uint32_t readSetBegin_virt(TType& elemType, + uint32_t& size) = 0; + + virtual uint32_t readSetEnd_virt() = 0; + + virtual uint32_t readBool_virt(bool& value) = 0; + + virtual uint32_t readBool_virt(std::vector::reference value) = 0; + + virtual uint32_t readByte_virt(int8_t& byte) = 0; + + virtual uint32_t readI16_virt(int16_t& i16) = 0; + + virtual uint32_t readI32_virt(int32_t& i32) = 0; + + virtual uint32_t readI64_virt(int64_t& i64) = 0; + + virtual uint32_t readDouble_virt(double& dub) = 0; + + virtual uint32_t readString_virt(std::string& str) = 0; + + virtual uint32_t readBinary_virt(std::string& str) = 0; + + uint32_t readMessageBegin(std::string& name, + TMessageType& messageType, + int32_t& seqid) { + T_VIRTUAL_CALL(); + return readMessageBegin_virt(name, messageType, seqid); + } + + uint32_t readMessageEnd() { + T_VIRTUAL_CALL(); + return readMessageEnd_virt(); + } + + uint32_t readStructBegin(std::string& name) { + T_VIRTUAL_CALL(); + return readStructBegin_virt(name); + } + + uint32_t readStructEnd() { + T_VIRTUAL_CALL(); + return readStructEnd_virt(); + } + + uint32_t readFieldBegin(std::string& name, + TType& fieldType, + int16_t& fieldId) { + T_VIRTUAL_CALL(); + return readFieldBegin_virt(name, fieldType, fieldId); + } + + uint32_t readFieldEnd() { + T_VIRTUAL_CALL(); + return readFieldEnd_virt(); + } + + uint32_t readMapBegin(TType& keyType, TType& valType, uint32_t& size) { + T_VIRTUAL_CALL(); + return readMapBegin_virt(keyType, valType, size); + } + + uint32_t readMapEnd() { + T_VIRTUAL_CALL(); + return readMapEnd_virt(); + } + + uint32_t readListBegin(TType& elemType, uint32_t& size) { + T_VIRTUAL_CALL(); + return readListBegin_virt(elemType, size); + } + + uint32_t readListEnd() { + T_VIRTUAL_CALL(); + return readListEnd_virt(); + } + + uint32_t readSetBegin(TType& elemType, uint32_t& size) { + T_VIRTUAL_CALL(); + return readSetBegin_virt(elemType, size); + } + + uint32_t readSetEnd() { + T_VIRTUAL_CALL(); + return readSetEnd_virt(); + } + + uint32_t readBool(bool& value) { + T_VIRTUAL_CALL(); + return readBool_virt(value); + } + + uint32_t readByte(int8_t& byte) { + T_VIRTUAL_CALL(); + return readByte_virt(byte); + } + + uint32_t readI16(int16_t& i16) { + T_VIRTUAL_CALL(); + return readI16_virt(i16); + } + + uint32_t readI32(int32_t& i32) { + T_VIRTUAL_CALL(); + return readI32_virt(i32); + } + + uint32_t readI64(int64_t& i64) { + T_VIRTUAL_CALL(); + return readI64_virt(i64); + } + + uint32_t readDouble(double& dub) { + T_VIRTUAL_CALL(); + return readDouble_virt(dub); + } + + uint32_t readString(std::string& str) { + T_VIRTUAL_CALL(); + return readString_virt(str); + } + + uint32_t readBinary(std::string& str) { + T_VIRTUAL_CALL(); + return readBinary_virt(str); + } + + /* + * std::vector is specialized for bool, and its elements are individual bits + * rather than bools. We need to define a different version of readBool() + * to work with std::vector. + */ + uint32_t readBool(std::vector::reference value) { + T_VIRTUAL_CALL(); + return readBool_virt(value); + } + + /** + * Method to arbitrarily skip over data. + */ + uint32_t skip(TType type) { + T_VIRTUAL_CALL(); + return skip_virt(type); + } + virtual uint32_t skip_virt(TType type) { + return ::apache::thrift::protocol::skip(*this, type); + } + + inline boost::shared_ptr getTransport() { + return ptrans_; + } + + // TODO: remove these two calls, they are for backwards + // compatibility + inline boost::shared_ptr getInputTransport() { + return ptrans_; + } + inline boost::shared_ptr getOutputTransport() { + return ptrans_; + } + + protected: + explicit TProtocol(boost::shared_ptr ptrans): + ptrans_(ptrans) { + } + + /** + * Construct a TProtocol using a raw TTransport pointer. + * + * It is the callers responsibility to ensure that the TTransport remains + * valid for the lifetime of the TProtocol object. + */ + explicit TProtocol(TTransport* ptrans): + ptrans_(ptrans, NoopPtrDestructor()) { + } + + boost::shared_ptr ptrans_; + + private: + TProtocol() {} +}; + +/** + * Constructs protocol objects given transports. + */ +class TProtocolFactory { + public: + TProtocolFactory() {} + + virtual ~TProtocolFactory() {} + + virtual boost::shared_ptr getProtocol(boost::shared_ptr trans) = 0; +}; + +/** + * Constructs both input and output protocol objects with a given pair of + * input and output transports. + * + * TProtocolPair.first = Input Protocol + * TProtocolPair.second = Output Protocol + */ +typedef std::pair, + boost::shared_ptr > TProtocolPair; + +class TDuplexProtocolFactory { + public: + TDuplexProtocolFactory() {} + + virtual ~TDuplexProtocolFactory() {} + + virtual TProtocolPair getProtocol(transport::TTransportPair transports) = 0; + + virtual boost::shared_ptr getInputProtocolFactory() { + return boost::shared_ptr(); + } + + virtual boost::shared_ptr getOutputProtocolFactory() { + return boost::shared_ptr(); + } +}; + +/** + * Adapts a TProtocolFactory to a TDuplexProtocolFactory that returns + * a new protocol object for both input and output + */ +template +class TSingleProtocolFactory : public TDuplexProtocolFactory { + public: + TSingleProtocolFactory() { + factory_.reset(new Factory_()); + } + + explicit TSingleProtocolFactory(boost::shared_ptr factory) : + factory_(factory) {} + + virtual TProtocolPair getProtocol(transport::TTransportPair transports) { + return std::make_pair(factory_->getProtocol(transports.first), + factory_->getProtocol(transports.second)); + } + + virtual boost::shared_ptr getInputProtocolFactory() { + return factory_; + } + + virtual boost::shared_ptr getOutputProtocolFactory() { + return factory_; + } + + private: + + boost::shared_ptr factory_; +}; + +/** + * Use TDualProtocolFactory to construct input and output protocols from + * different factories. + */ +class TDualProtocolFactory : public TDuplexProtocolFactory { + public: + TDualProtocolFactory( + boost::shared_ptr inputFactory, + boost::shared_ptr outputFactory) : + inputFactory_(inputFactory), + outputFactory_(outputFactory) {} + + virtual TProtocolPair getProtocol(transport::TTransportPair transports) { + return std::make_pair(inputFactory_->getProtocol(transports.first), + outputFactory_->getProtocol(transports.second)); + } + + virtual boost::shared_ptr getInputProtocolFactory() { + return inputFactory_; + } + + virtual boost::shared_ptr getOutputProtocolFactory() { + return outputFactory_; + } + + private: + + boost::shared_ptr inputFactory_; + boost::shared_ptr outputFactory_; +}; + +/** + * Dummy protocol class. + * + * This class does nothing, and should never be instantiated. + * It is used only by the generator code. + */ +class TDummyProtocol : public TProtocol { +}; + +}}} // apache::thrift::protocol + +#endif // #define _THRIFT_PROTOCOL_TPROTOCOL_H_ 1 diff --git a/thrift/lib/cpp/protocol/TProtocolException.h b/thrift/lib/cpp/protocol/TProtocolException.h new file mode 100644 index 0000000000..e9f3856a44 --- /dev/null +++ b/thrift/lib/cpp/protocol/TProtocolException.h @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_PROTOCOL_TPROTOCOLEXCEPTION_H_ +#define _THRIFT_PROTOCOL_TPROTOCOLEXCEPTION_H_ 1 + +#include + +namespace apache { namespace thrift { namespace protocol { + +/** + * Class to encapsulate all the possible types of protocol errors that may + * occur in various protocol systems. This provides a sort of generic + * wrapper around the shitty UNIX E_ error codes that lets a common code + * base of error handling to be used for various types of protocols, i.e. + * pipes etc. + * + */ +class TProtocolException : public apache::thrift::TLibraryException { + public: + + /** + * Error codes for the various types of exceptions. + */ + enum TProtocolExceptionType + { UNKNOWN = 0 + , INVALID_DATA = 1 + , NEGATIVE_SIZE = 2 + , SIZE_LIMIT = 3 + , BAD_VERSION = 4 + , NOT_IMPLEMENTED = 5 + }; + + TProtocolException() : + apache::thrift::TLibraryException(), + type_(UNKNOWN) {} + + TProtocolException(TProtocolExceptionType type) : + apache::thrift::TLibraryException(), + type_(type) {} + + TProtocolException(const std::string& message) : + apache::thrift::TLibraryException(message), + type_(UNKNOWN) {} + + TProtocolException(TProtocolExceptionType type, const std::string& message) : + apache::thrift::TLibraryException(message), + type_(type) {} + + virtual ~TProtocolException() throw() {} + + /** + * Returns an error code that provides information about the type of error + * that has occurred. + * + * @return Error code + */ + TProtocolExceptionType getType() { + return type_; + } + + virtual const char* what() const throw() { + if (message_.empty()) { + switch (type_) { + case UNKNOWN : return "TProtocolException: Unknown protocol exception"; + case INVALID_DATA : return "TProtocolException: Invalid data"; + case NEGATIVE_SIZE : return "TProtocolException: Negative size"; + case SIZE_LIMIT : return "TProtocolException: Exceeded size limit"; + case BAD_VERSION : return "TProtocolException: Invalid version"; + case NOT_IMPLEMENTED : return "TProtocolException: Not implemented"; + default : return "TProtocolException: (Invalid exception type)"; + } + } else { + return message_.c_str(); + } + } + + protected: + /** + * Error code + */ + TProtocolExceptionType type_; + +}; + +}}} // apache::thrift::protocol + +#endif // #ifndef _THRIFT_PROTOCOL_TPROTOCOLEXCEPTION_H_ diff --git a/thrift/lib/cpp/protocol/TProtocolTap.h b/thrift/lib/cpp/protocol/TProtocolTap.h new file mode 100644 index 0000000000..3772bff58c --- /dev/null +++ b/thrift/lib/cpp/protocol/TProtocolTap.h @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_PROTOCOL_TPROTOCOLTAP_H_ +#define _THRIFT_PROTOCOL_TPROTOCOLTAP_H_ 1 + +#include + +namespace apache { namespace thrift { namespace protocol { + +using apache::thrift::transport::TTransport; + +/** + * Puts a wiretap on a protocol object. Any reads to this class are passed + * through to an enclosed protocol object, but also mirrored as write to a + * second protocol object. + * + */ +class TProtocolTap : public TVirtualProtocol { + public: + TProtocolTap(boost::shared_ptr source, + boost::shared_ptr sink) + : TVirtualProtocol(source->getTransport()) + , source_(source) + , sink_(sink) + {} + + uint32_t readMessageBegin(std::string& name, + TMessageType& messageType, + int32_t& seqid) { + uint32_t rv = source_->readMessageBegin(name, messageType, seqid); + sink_->writeMessageBegin(name, messageType, seqid); + return rv; + } + + uint32_t readMessageEnd() { + uint32_t rv = source_->readMessageEnd(); + sink_->writeMessageEnd(); + return rv; + } + + uint32_t readStructBegin(std::string& name) { + uint32_t rv = source_->readStructBegin(name); + sink_->writeStructBegin(name.c_str()); + return rv; + } + + uint32_t readStructEnd() { + uint32_t rv = source_->readStructEnd(); + sink_->writeStructEnd(); + return rv; + } + + uint32_t readFieldBegin(std::string& name, + TType& fieldType, + int16_t& fieldId) { + uint32_t rv = source_->readFieldBegin(name, fieldType, fieldId); + if (fieldType == T_STOP) { + sink_->writeFieldStop(); + } else { + sink_->writeFieldBegin(name.c_str(), fieldType, fieldId); + } + return rv; + } + + + uint32_t readFieldEnd() { + uint32_t rv = source_->readFieldEnd(); + sink_->writeFieldEnd(); + return rv; + } + + uint32_t readMapBegin(TType& keyType, + TType& valType, + uint32_t& size) { + uint32_t rv = source_->readMapBegin(keyType, valType, size); + sink_->writeMapBegin(keyType, valType, size); + return rv; + } + + + uint32_t readMapEnd() { + uint32_t rv = source_->readMapEnd(); + sink_->writeMapEnd(); + return rv; + } + + uint32_t readListBegin(TType& elemType, uint32_t& size) { + uint32_t rv = source_->readListBegin(elemType, size); + sink_->writeListBegin(elemType, size); + return rv; + } + + + uint32_t readListEnd() { + uint32_t rv = source_->readListEnd(); + sink_->writeListEnd(); + return rv; + } + + uint32_t readSetBegin(TType& elemType, uint32_t& size) { + uint32_t rv = source_->readSetBegin(elemType, size); + sink_->writeSetBegin(elemType, size); + return rv; + } + + + uint32_t readSetEnd() { + uint32_t rv = source_->readSetEnd(); + sink_->writeSetEnd(); + return rv; + } + + uint32_t readBool(bool& value) { + uint32_t rv = source_->readBool(value); + sink_->writeBool(value); + return rv; + } + + // Provide the default readBool() implementation for std::vector + using TVirtualProtocol::readBool; + + uint32_t readByte(int8_t& byte) { + uint32_t rv = source_->readByte(byte); + sink_->writeByte(byte); + return rv; + } + + uint32_t readI16(int16_t& i16) { + uint32_t rv = source_->readI16(i16); + sink_->writeI16(i16); + return rv; + } + + uint32_t readI32(int32_t& i32) { + uint32_t rv = source_->readI32(i32); + sink_->writeI32(i32); + return rv; + } + + uint32_t readI64(int64_t& i64) { + uint32_t rv = source_->readI64(i64); + sink_->writeI64(i64); + return rv; + } + + uint32_t readDouble(double& dub) { + uint32_t rv = source_->readDouble(dub); + sink_->writeDouble(dub); + return rv; + } + + uint32_t readString(std::string& str) { + uint32_t rv = source_->readString(str); + sink_->writeString(str); + return rv; + } + + uint32_t readBinary(std::string& str) { + uint32_t rv = source_->readBinary(str); + sink_->writeBinary(str); + return rv; + } + + private: + boost::shared_ptr source_; + boost::shared_ptr sink_; +}; + +}}} // apache::thrift::protocol + +#endif // #define _THRIFT_PROTOCOL_TPROTOCOLTAP_H_ 1 diff --git a/thrift/lib/cpp/protocol/TProtocolTypes.h b/thrift/lib/cpp/protocol/TProtocolTypes.h new file mode 100644 index 0000000000..ca22b545f0 --- /dev/null +++ b/thrift/lib/cpp/protocol/TProtocolTypes.h @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef THRIFT_PROTOCOL_TPROTOCOLTYPES_H_ +#define THRIFT_PROTOCOL_TPROTOCOLTYPES_H_ 1 + +namespace apache { namespace thrift { namespace protocol { + +enum PROTOCOL_TYPES { + T_BINARY_PROTOCOL = 0, + T_JSON_PROTOCOL = 1, + T_COMPACT_PROTOCOL = 2, +}; + +}}} // apache::thrift::protocol + +#endif // #define _THRIFT_PROTOCOL_TPROTOCOLTYPES_H_ 1 + diff --git a/thrift/lib/cpp/protocol/TSimpleJSONProtocol.h b/thrift/lib/cpp/protocol/TSimpleJSONProtocol.h new file mode 100644 index 0000000000..4183d3af59 --- /dev/null +++ b/thrift/lib/cpp/protocol/TSimpleJSONProtocol.h @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef THRIFT_PROTOCOL_TSIMPLEJSONPROTOCOL_H_ +#define THRIFT_PROTOCOL_TSIMPLEJSONPROTOCOL_H_ 1 + +#include "TJSONProtocol.h" + +namespace apache { namespace thrift { namespace protocol { + + +/* + * TsimpleJSONProtocol overrides parts of the regular JSON serialization to + * comply with the Simple JSON format. + * Namely, spitting only field names without verbose field type output + */ + +class TSimpleJSONProtocol : public TVirtualProtocol{ + + public: + + TSimpleJSONProtocol(boost::shared_ptr ptrans); + + ~TSimpleJSONProtocol(); + + + public: + + uint32_t writeFieldBegin(const char* name, + const TType fieldType, + const int16_t fieldId); + + + uint32_t writeFieldEnd(); + + uint32_t writeMapBegin(const TType keyType, const TType valType, + const uint32_t size); + + uint32_t writeMapEnd(); + + uint32_t writeListBegin(const TType elemType, const uint32_t size); + + uint32_t writeSetBegin(const TType elemType, const uint32_t size); + + uint32_t writeBool(const bool value); +}; + +/** + * Constructs input and output protocol objects given transports. + */ +class TSimpleJSONProtocolFactory : public TProtocolFactory { + public: + TSimpleJSONProtocolFactory() {} + + virtual ~TSimpleJSONProtocolFactory() {} + + boost::shared_ptr getProtocol( + boost::shared_ptr trans) { + return boost::shared_ptr(new TSimpleJSONProtocol(trans)); + } +}; + +}}} // apache::thrift::protocol + +#endif // #define THRIFT_PROTOCOL_TJSONPROTOCOL_H_ 1 diff --git a/thrift/lib/cpp/protocol/TVirtualProtocol.h b/thrift/lib/cpp/protocol/TVirtualProtocol.h new file mode 100644 index 0000000000..97298f0043 --- /dev/null +++ b/thrift/lib/cpp/protocol/TVirtualProtocol.h @@ -0,0 +1,523 @@ +// Copyright (c) 2006- Facebook +// Distributed under the Thrift Software License +// +// See accompanying file LICENSE or visit the Thrift site at: +// http://developers.facebook.com/thrift/ + +#ifndef _THRIFT_PROTOCOL_TVIRTUALPROTOCOL_H_ +#define _THRIFT_PROTOCOL_TVIRTUALPROTOCOL_H_ 1 + +#include "thrift/lib/cpp/protocol/TProtocol.h" + +namespace apache { namespace thrift { namespace protocol { + +/** + * Helper class that provides default implementations of TProtocol methods. + * + * This class provides default implementations of the non-virtual TProtocol + * methods. It exists primarily so TVirtualProtocol can derive from it. It + * prevents TVirtualProtocol methods from causing infinite recursion if the + * non-virtual methods are not overridden by the TVirtualProtocol subclass. + * + * You probably don't want to use this class directly. Use TVirtualProtocol + * instead. + */ +class TProtocolDefaults : public TProtocol { + public: + void setVersion(const int8_t version) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "this protocol does not support setVersion (yet)"); + } + + uint32_t readMessageBegin(std::string& name, + TMessageType& messageType, + int32_t& seqid) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "this protocol does not support reading (yet)."); + } + + uint32_t readMessageEnd() { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "this protocol does not support reading (yet)."); + } + + uint32_t readStructBegin(std::string& name) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "this protocol does not support reading (yet)."); + } + + uint32_t readStructEnd() { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "this protocol does not support reading (yet)."); + } + + uint32_t readFieldBegin(std::string& name, + TType& fieldType, + int16_t& fieldId) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "this protocol does not support reading (yet)."); + } + + uint32_t readFieldEnd() { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "this protocol does not support reading (yet)."); + } + + uint32_t readMapBegin(TType& keyType, TType& valType, uint32_t& size) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "this protocol does not support reading (yet)."); + } + + uint32_t readMapEnd() { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "this protocol does not support reading (yet)."); + } + + uint32_t readListBegin(TType& elemType, uint32_t& size) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "this protocol does not support reading (yet)."); + } + + uint32_t readListEnd() { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "this protocol does not support reading (yet)."); + } + + uint32_t readSetBegin(TType& elemType, uint32_t& size) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "this protocol does not support reading (yet)."); + } + + uint32_t readSetEnd() { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "this protocol does not support reading (yet)."); + } + + uint32_t readBool(bool& value) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "this protocol does not support reading (yet)."); + } + + uint32_t readBool(std::vector::reference value) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "this protocol does not support reading (yet)."); + } + + uint32_t readByte(int8_t& byte) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "this protocol does not support reading (yet)."); + } + + uint32_t readI16(int16_t& i16) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "this protocol does not support reading (yet)."); + } + + uint32_t readI32(int32_t& i32) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "this protocol does not support reading (yet)."); + } + + uint32_t readI64(int64_t& i64) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "this protocol does not support reading (yet)."); + } + + uint32_t readDouble(double& dub) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "this protocol does not support reading (yet)."); + } + + uint32_t readString(std::string& str) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "this protocol does not support reading (yet)."); + } + + uint32_t readBinary(std::string& str) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "this protocol does not support reading (yet)."); + } + + uint32_t writeMessageBegin(const std::string& name, + const TMessageType messageType, + const int32_t seqid) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "this protocol does not support writing (yet)."); + } + + uint32_t writeMessageEnd() { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "this protocol does not support writing (yet)."); + } + + + uint32_t writeStructBegin(const char* name) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "this protocol does not support writing (yet)."); + } + + uint32_t writeStructEnd() { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "this protocol does not support writing (yet)."); + } + + uint32_t writeFieldBegin(const char* name, + const TType fieldType, + const int16_t fieldId) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "this protocol does not support writing (yet)."); + } + + uint32_t writeFieldEnd() { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "this protocol does not support writing (yet)."); + } + + uint32_t writeFieldStop() { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "this protocol does not support writing (yet)."); + } + + uint32_t writeMapBegin(const TType keyType, + const TType valType, + const uint32_t size) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "this protocol does not support writing (yet)."); + } + + uint32_t writeMapEnd() { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "this protocol does not support writing (yet)."); + } + + uint32_t writeListBegin(const TType elemType, const uint32_t size) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "this protocol does not support writing (yet)."); + } + + uint32_t writeListEnd() { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "this protocol does not support writing (yet)."); + } + + uint32_t writeSetBegin(const TType elemType, const uint32_t size) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "this protocol does not support writing (yet)."); + } + + uint32_t writeSetEnd() { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "this protocol does not support writing (yet)."); + } + + uint32_t writeBool(const bool value) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "this protocol does not support writing (yet)."); + } + + uint32_t writeByte(const int8_t byte) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "this protocol does not support writing (yet)."); + } + + uint32_t writeI16(const int16_t i16) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "this protocol does not support writing (yet)."); + } + + uint32_t writeI32(const int32_t i32) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "this protocol does not support writing (yet)."); + } + + uint32_t writeI64(const int64_t i64) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "this protocol does not support writing (yet)."); + } + + uint32_t writeDouble(const double dub) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "this protocol does not support writing (yet)."); + } + + uint32_t writeString(const std::string& str) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "this protocol does not support writing (yet)."); + } + + uint32_t writeBinary(const std::string& str) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "this protocol does not support writing (yet)."); + } + + uint32_t skip(TType type) { + return ::apache::thrift::protocol::skip(*this, type); + } + + protected: + TProtocolDefaults(const boost::shared_ptr& ptrans) + : TProtocol(ptrans) + {} + + TProtocolDefaults(transport::TTransport* ptrans) + : TProtocol(ptrans) + {} +}; + +/** + * Concrete TProtocol classes should inherit from TVirtualProtocol + * so they don't have to manually override virtual methods. + * + * @author David Reiss + */ +template +class TVirtualProtocol : public Super_ { + public: + + virtual void setVersion_virt(const int8_t version) { + return static_cast(this)->setVersion(version); + } + /** + * Writing functions. + */ + + virtual uint32_t writeMessageBegin_virt(const std::string& name, + const TMessageType messageType, + const int32_t seqid) { + return static_cast(this)->writeMessageBegin(name, messageType, + seqid); + } + + virtual uint32_t writeMessageEnd_virt() { + return static_cast(this)->writeMessageEnd(); + } + + + virtual uint32_t writeStructBegin_virt(const char* name) { + return static_cast(this)->writeStructBegin(name); + } + + virtual uint32_t writeStructEnd_virt() { + return static_cast(this)->writeStructEnd(); + } + + virtual uint32_t writeFieldBegin_virt(const char* name, + const TType fieldType, + const int16_t fieldId) { + return static_cast(this)->writeFieldBegin(name, fieldType, + fieldId); + } + + virtual uint32_t writeFieldEnd_virt() { + return static_cast(this)->writeFieldEnd(); + } + + virtual uint32_t writeFieldStop_virt() { + return static_cast(this)->writeFieldStop(); + } + + virtual uint32_t writeMapBegin_virt(const TType keyType, + const TType valType, + const uint32_t size) { + return static_cast(this)->writeMapBegin(keyType, valType, size); + } + + virtual uint32_t writeMapEnd_virt() { + return static_cast(this)->writeMapEnd(); + } + + virtual uint32_t writeListBegin_virt(const TType elemType, + const uint32_t size) { + return static_cast(this)->writeListBegin(elemType, size); + } + + virtual uint32_t writeListEnd_virt() { + return static_cast(this)->writeListEnd(); + } + + virtual uint32_t writeSetBegin_virt(const TType elemType, + const uint32_t size) { + return static_cast(this)->writeSetBegin(elemType, size); + } + + virtual uint32_t writeSetEnd_virt() { + return static_cast(this)->writeSetEnd(); + } + + virtual uint32_t writeBool_virt(const bool value) { + return static_cast(this)->writeBool(value); + } + + virtual uint32_t writeByte_virt(const int8_t byte) { + return static_cast(this)->writeByte(byte); + } + + virtual uint32_t writeI16_virt(const int16_t i16) { + return static_cast(this)->writeI16(i16); + } + + virtual uint32_t writeI32_virt(const int32_t i32) { + return static_cast(this)->writeI32(i32); + } + + virtual uint32_t writeI64_virt(const int64_t i64) { + return static_cast(this)->writeI64(i64); + } + + virtual uint32_t writeDouble_virt(const double dub) { + return static_cast(this)->writeDouble(dub); + } + + virtual uint32_t writeString_virt(const std::string& str) { + return static_cast(this)->writeString(str); + } + + virtual uint32_t writeBinary_virt(const std::string& str) { + return static_cast(this)->writeBinary(str); + } + + /** + * Reading functions + */ + + virtual uint32_t readMessageBegin_virt(std::string& name, + TMessageType& messageType, + int32_t& seqid) { + return static_cast(this)->readMessageBegin(name, messageType, + seqid); + } + + virtual uint32_t readMessageEnd_virt() { + return static_cast(this)->readMessageEnd(); + } + + virtual uint32_t readStructBegin_virt(std::string& name) { + return static_cast(this)->readStructBegin(name); + } + + virtual uint32_t readStructEnd_virt() { + return static_cast(this)->readStructEnd(); + } + + virtual uint32_t readFieldBegin_virt(std::string& name, + TType& fieldType, + int16_t& fieldId) { + return static_cast(this)->readFieldBegin(name, fieldType, + fieldId); + } + + virtual uint32_t readFieldEnd_virt() { + return static_cast(this)->readFieldEnd(); + } + + virtual uint32_t readMapBegin_virt(TType& keyType, + TType& valType, + uint32_t& size) { + return static_cast(this)->readMapBegin(keyType, valType, size); + } + + virtual uint32_t readMapEnd_virt() { + return static_cast(this)->readMapEnd(); + } + + virtual uint32_t readListBegin_virt(TType& elemType, + uint32_t& size) { + return static_cast(this)->readListBegin(elemType, size); + } + + virtual uint32_t readListEnd_virt() { + return static_cast(this)->readListEnd(); + } + + virtual uint32_t readSetBegin_virt(TType& elemType, + uint32_t& size) { + return static_cast(this)->readSetBegin(elemType, size); + } + + virtual uint32_t readSetEnd_virt() { + return static_cast(this)->readSetEnd(); + } + + virtual uint32_t readBool_virt(bool& value) { + return static_cast(this)->readBool(value); + } + + virtual uint32_t readBool_virt(std::vector::reference value) { + return static_cast(this)->readBool(value); + } + + virtual uint32_t readByte_virt(int8_t& byte) { + return static_cast(this)->readByte(byte); + } + + virtual uint32_t readI16_virt(int16_t& i16) { + return static_cast(this)->readI16(i16); + } + + virtual uint32_t readI32_virt(int32_t& i32) { + return static_cast(this)->readI32(i32); + } + + virtual uint32_t readI64_virt(int64_t& i64) { + return static_cast(this)->readI64(i64); + } + + virtual uint32_t readDouble_virt(double& dub) { + return static_cast(this)->readDouble(dub); + } + + virtual uint32_t readString_virt(std::string& str) { + return static_cast(this)->readString(str); + } + + virtual uint32_t readBinary_virt(std::string& str) { + return static_cast(this)->readBinary(str); + } + + virtual uint32_t skip_virt(TType type) { + return static_cast(this)->skip(type); + } + + /* + * Provide a default skip() implementation that uses non-virtual read + * methods. + * + * Note: subclasses that use TVirtualProtocol to derive from another protocol + * implementation (i.e., not TProtocolDefaults) should beware that this may + * override any non-default skip() implementation provided by the parent + * transport class. They may need to explicitly redefine skip() to call the + * correct parent implementation, if desired. + */ + uint32_t skip(TType type) { + Protocol_* const prot = static_cast(this); + return ::apache::thrift::protocol::skip(*prot, type); + } + + /* + * Provide a default readBool() implementation for use with + * std::vector, that behaves the same as reading into a normal bool. + * + * Subclasses can override this if desired, but there normally shouldn't + * be a need to. + */ + uint32_t readBool(std::vector::reference value) { + bool b = false; + uint32_t ret = static_cast(this)->readBool(b); + value = b; + return ret; + } + using Super_::readBool; // so we don't hide readBool(bool&) + + protected: + TVirtualProtocol(const boost::shared_ptr& ptrans) + : Super_(ptrans) + {} + + TVirtualProtocol(transport::TTransport* ptrans) + : Super_(ptrans) + {} +}; + +}}} // apache::thrift::protocol + +#endif // #define _THRIFT_PROTOCOL_TVIRTUALPROTOCOL_H_ 1 diff --git a/thrift/lib/cpp/protocol/neutronium/Decoder-inl.h b/thrift/lib/cpp/protocol/neutronium/Decoder-inl.h new file mode 100644 index 0000000000..e38451c637 --- /dev/null +++ b/thrift/lib/cpp/protocol/neutronium/Decoder-inl.h @@ -0,0 +1,438 @@ +/** + * Copyright 2012 Facebook + * @author Tudor Bosman (tudorb@fb.com) + */ + +#ifndef THRIFT_LIB_CPP_PROTOCOL_NEUTRONIUM_DECODER_INL_H_ +#define THRIFT_LIB_CPP_PROTOCOL_NEUTRONIUM_DECODER_INL_H_ + +#ifndef THRIFT_INCLUDE_DECODER_INL +#error This file may only be included from Decoder.h +#endif + +#include "folly/GroupVarint.h" +#include "folly/Conv.h" + +namespace apache { +namespace thrift { +namespace protocol { +namespace neutronium { + +inline const char* CH(const uint8_t* p) { + return reinterpret_cast(p); +} +inline char* CH(uint8_t* p) { + return reinterpret_cast(p); +} + +template +inline auto CH(P p) -> std::pair { + return std::make_pair(CH(p.first), p.second); +} + +inline folly::StringPiece SP(std::pair p) { + return folly::StringPiece(CH(p.first), p.second); +} + +inline void Decoder::setRootType(int64_t rootType) { + CHECK(stack_.empty()); + rootType_ = rootType; +} + +inline void Decoder::readStructBegin() { + push(reflection::TYPE_STRUCT, nextType(), 0); +} + +inline void Decoder::readStructEnd() { + DCHECK_EQ(top().state, IN_STRUCT); + // The last readFieldBegin returned T_STOP and so didn't change the state + // to IN_FIELD + pop(); +} + +inline int64_t Decoder::nextType() { + int64_t type; + if (!stack_.empty()) { + auto& s = top(); + s.nextValue(); + type = s.type.typeVal; + } else { + type = rootType_; + } + return type; +} + +inline void Decoder::readMapBegin(TType& keyType, TType& valType, + uint32_t& size) { + size = peekElementCount(); + push(reflection::TYPE_MAP, nextType(), size); + keyType = top().list.mapKeyType.ttype(); + valType = top().list.valueType.ttype(); + size = top().list.remaining; +} + +inline void Decoder::readMapEnd() { + DCHECK_EQ(top().state, IN_MAP_VALUE); + DCHECK_EQ(top().list.remaining, 0); + pop(); +} + +inline void Decoder::readListBegin(TType& elemType, uint32_t& size) { + size = peekElementCount(); + push(reflection::TYPE_LIST, nextType(), size); + elemType = top().list.valueType.ttype(); + size = top().list.remaining; +} + +inline void Decoder::readListEnd() { + DCHECK_EQ(top().state, IN_LIST_VALUE); + DCHECK_EQ(top().list.remaining, 0); + pop(); +} + +inline void Decoder::readSetBegin(TType& elemType, uint32_t& size) { + size = peekElementCount(); + push(reflection::TYPE_SET, nextType(), size); + elemType = top().list.valueType.ttype(); + size = top().list.remaining; +} + +inline void Decoder::readSetEnd() { + DCHECK_EQ(top().state, IN_SET_VALUE); + DCHECK_EQ(top().list.remaining, 0); + pop(); +} + +inline void Decoder::push(reflection::Type rtype, int64_t type, uint32_t size) { + if (rtype != reflection::getType(type)) { + throw TProtocolException(folly::to( + "Invalid type ", reflection::getType(type), " expected ", rtype)); + } + const DataType* dt = &(schema_->map().at(type)); + stack_.emplace_back(schema_, type, dt, size); + read(); +} + +inline Decoder::TypeInfo::TypeInfo(const Schema* schema, int64_t t) + : typeVal(t), + length(kVariableLength), + dataType(nullptr), + terminator('\0') { + if (!reflection::isBaseType(type())) { + dataType = &(schema->map().at(t)); + length = dataType->fixedSize; + } +} + +inline Decoder::TypeInfo::TypeInfo(int64_t t) + : typeVal(t), + length(kVariableLength), + dataType(nullptr), + terminator('\0') { + DCHECK(reflection::isBaseType(type())); +} + +template +inline void Decoder::DecoderState::setStateList(T& ts) { + if (UNLIKELY(++ts.index == ts.count)) { + throw TProtocolException("type mismatch"); + } +} + +inline void Decoder::DecoderState::nextList() { + switch (state) { + case IN_MAP_KEY: + state = IN_MAP_VALUE; + type = list.valueType; + break; // don't advance to next + case IN_MAP_VALUE: + state = IN_MAP_KEY; + type = list.mapKeyType; + DCHECK_NE(list.remaining, 0); + --list.remaining; + break; + case IN_LIST_VALUE: + case IN_SET_VALUE: + type = list.valueType; + DCHECK_NE(list.remaining, 0); + --list.remaining; + break; + default: + LOG(FATAL) << "Invalid state " << state; + } + // Advance to next element + + switch (type.typeVal) { + case reflection::TYPE_BOOL: + setStateList(bools); + break; + case reflection::TYPE_BYTE: + setStateList(bytes); + break; + case reflection::TYPE_I16: // fallthrough + case reflection::TYPE_I32: + setStateList(ints); + break; + case reflection::TYPE_I64: // fallthrough + case reflection::TYPE_DOUBLE: + setStateList(int64s); + break; + case reflection::TYPE_STRING: // fallthrough + default: + setStateList(strings); + } +} + +template +bool Decoder::DecoderState::setStateStruct( + TS& ts, + const TV& tv, + FieldState nextState) { + if (++ts.index == ts.count) { + str.fieldState = nextState; + return false; + } else { + auto& p = tv[ts.index]; + str.tag = p.first; + type = p.second; + return true; + } +} + +inline bool Decoder::DecoderState::nextStruct() { + switch (str.fieldState) { + case FS_START: + str.fieldState = FS_INT; + ints.index = -1; + // fallthrough + case FS_INT: + if (setStateStruct(ints, str.intTags, FS_INT64)) { + break; + } + // fallthrough + case FS_INT64: + if (setStateStruct(int64s, str.int64Tags, FS_BOOL)) { + break; + } + // fallthrough + case FS_BOOL: + if (setStateStruct(bools, str.boolTags, FS_STRICT_ENUM)) { + break; + } + case FS_STRICT_ENUM: + if (setStateStruct(strictEnums, str.strictEnumTags, FS_BYTE)) { + break; + } + // fallthrough + case FS_BYTE: + if (setStateStruct(bytes, str.byteTags, FS_INTERNED_STRING)) { + break; + } + // fallthrough + case FS_INTERNED_STRING: + if (setStateStruct(internedStrings, str.internedStringTags, FS_STRING)) { + break; + } + case FS_STRING: + if (setStateStruct(strings, str.stringTags, FS_END)) { + break; + } + // fallthrough + case FS_END: + return false; + } + + return true; +} + +inline bool Decoder::DecoderState::nextField() { + DCHECK_EQ(state, IN_STRUCT); + if (!nextStruct()) { + return false; + } + state = IN_FIELD; + setLength(); + return true; +} + +inline void Decoder::DecoderState::nextValue() { + DCHECK_NE(state, IN_STRUCT); + if (state != IN_FIELD) { + nextList(); + setLength(); + } +} + +inline void Decoder::DecoderState::setLength() { + if (type.type() == reflection::TYPE_STRING && + type.length == kVariableLength) { + // Variable size, read length + if (++vars.index == vars.count) { + throw TProtocolException("too many vars"); + } + type.length = vars.values[vars.index]; + } +} + +inline uint32_t Decoder::peekElementCount() { + uint32_t len; + cursor_.gather(folly::GroupVarint32::maxSize(1)); + folly::GroupVarint32Decoder decoder(SP(cursor_.peek())); + uint32_t count; + if (!decoder.next(&count)) { + throw TProtocolException("underflow"); + } + return count; +} + +inline void Decoder::readFieldBegin(TType& fieldType, int16_t& fieldId) { + auto& s = top(); + if (s.nextField()) { + fieldType = s.type.ttype(); + fieldId = s.str.tag; + } else { + fieldType = T_STOP; + fieldId = 0; + } +} + +inline void Decoder::readFieldEnd() { + auto& s = top(); + CHECK_EQ(s.state, IN_FIELD); + s.state = IN_STRUCT; +} + +inline void Decoder::readBool(bool& value) { + auto& s = top(); + s.nextValue(); + if (UNLIKELY(s.type.typeVal != reflection::TYPE_BOOL)) { + throw TProtocolException("invalid type"); + } + value = testBit(s.bools.values, s.boolStartBit + s.bools.index); +} + +inline void Decoder::readByte(int8_t& value) { + auto& s = top(); + s.nextValue(); + if (UNLIKELY(s.type.typeVal != reflection::TYPE_BYTE)) { + throw TProtocolException("invalid type"); + } + value = s.bytes.values[s.bytes.index]; +} + +inline void Decoder::readI16(int16_t& value) { + auto& s = top(); + s.nextValue(); + if (UNLIKELY(s.type.typeVal != reflection::TYPE_I16)) { + throw TProtocolException("invalid type"); + } + value = s.ints.values[s.ints.index]; +} + +inline void Decoder::readI32(int32_t& value) { + auto& s = top(); + s.nextValue(); + // Strict enums only occur in structs + // TODO(tudorb): Relax? + if (s.state == IN_FIELD && s.str.fieldState == DecoderState::FS_STRICT_ENUM) { + value = s.strictEnums.values[s.strictEnums.index]; + } else if (UNLIKELY(s.type.typeVal != reflection::TYPE_I32)) { + throw TProtocolException("invalid type"); + } else { + value = s.ints.values[s.ints.index]; + } +} + +inline void Decoder::readI64(int64_t& value) { + auto& s = top(); + s.nextValue(); + if (UNLIKELY(s.type.typeVal != reflection::TYPE_I64)) { + throw TProtocolException("invalid type"); + } + value = s.int64s.values[s.int64s.index]; +} + +inline void Decoder::readDouble(double& value) { + auto& s = top(); + s.nextValue(); + if (UNLIKELY(s.type.typeVal != reflection::TYPE_DOUBLE)) { + throw TProtocolException("invalid type"); + } + int64_t i64 = s.int64s.values[s.int64s.index]; + value = bitwise_cast(i64); +} + +inline bool Decoder::beginReadString() { + auto& s = top(); + s.nextValue(); + if (UNLIKELY(s.type.typeVal != reflection::TYPE_STRING)) { + throw TProtocolException("invalid type"); + } + // Interned strings only occur in structs, + // TODO(tudorb): Relax? + return (s.state == IN_FIELD && + s.str.fieldState == DecoderState::FS_INTERNED_STRING); +} + +template +inline void Decoder::readString(StrType& str) { + if (beginReadString()) { + // interned + auto& s = top(); + auto sp = s.internedStrings.values[s.internedStrings.index]; + str.assign(sp.data(), sp.size()); + } else { + // not interned + str.clear(); + int64_t len = top().type.length; + DCHECK(len >= 0 || len == kTerminated); + if (len == kTerminated) { + char terminator = top().type.terminator; + const uint8_t* term = nullptr; + while (!term) { + auto p = cursor_.peek(); + if (p.second == 0) throw TLibraryException("eof"); + term = static_cast( + memchr(p.first, terminator, p.second)); + size_t n = term ? term - p.first : p.second; + str.append(reinterpret_cast(p.first), n); + cursor_.skip(n); + top().bytesRead += n; + } + } else { + while (len) { + auto p = cursor_.peek(); + if (p.second == 0) throw TLibraryException("eof"); + auto n = std::min(len, static_cast(p.second)); + str.append(reinterpret_cast(p.first), n); + cursor_.skip(n); + top().bytesRead += n; + len -= n; + } + } + } +} + +inline Decoder::DecoderState& Decoder::top() { + DCHECK(!stack_.empty()); + return stack_.back(); +} + +inline void Decoder::pop() { + DCHECK(!stack_.empty()); + size_t bytesRead = stack_.back().bytesRead; + stack_.pop_back(); + if (stack_.empty()) { + bytesRead_ = bytesRead; + } else { + top().bytesRead += bytesRead; + } +} + +} // namespace neutronium +} // namespace protocol +} // namespace thrift +} // namespace apache + +#endif /* THRIFT_LIB_CPP_PROTOCOL_NEUTRONIUM_DECODER_INL_H_ */ + diff --git a/thrift/lib/cpp/protocol/neutronium/Decoder.h b/thrift/lib/cpp/protocol/neutronium/Decoder.h new file mode 100644 index 0000000000..de0edff68d --- /dev/null +++ b/thrift/lib/cpp/protocol/neutronium/Decoder.h @@ -0,0 +1,239 @@ +/** + * Copyright 2012 Facebook + * @author Tudor Bosman (tudorb@fb.com) + */ + +#ifndef THRIFT_LIB_CPP_PROTOCOL_NEUTRONIUM_DECODER_H_ +#define THRIFT_LIB_CPP_PROTOCOL_NEUTRONIUM_DECODER_H_ + +#include "thrift/lib/cpp/protocol/neutronium/Utils.h" +#include "thrift/lib/cpp/protocol/neutronium/InternTable.h" +#include "thrift/lib/cpp/protocol/neutronium/Schema.h" +#include "thrift/lib/cpp/protocol/TProtocol.h" +#include "folly/FBString.h" +#include "folly/Range.h" +#include "folly/small_vector.h" +#include "folly/experimental/io/Cursor.h" + +namespace apache { +namespace thrift { +namespace protocol { +namespace neutronium { + +class Decoder { + public: + explicit Decoder(const Schema* schema, const InternTable* internTable, + folly::IOBuf* buf); + void setRootType(int64_t type); + + void readStructBegin(); + void readStructEnd(); + + void readFieldBegin(TType& fieldType, int16_t& fieldId); + void readFieldEnd(); + + void readMapBegin(TType& keyType, TType& valType, uint32_t& size); + void readMapEnd(); + + void readListBegin(TType& elemType, uint32_t& size); + void readListEnd(); + + void readSetBegin(TType& elemType, uint32_t& size); + void readSetEnd(); + + void readBool(bool& value); + void readByte(int8_t& value); + void readI16(int16_t& value); + void readI32(int32_t& i32); + void readI64(int64_t& i64); + void readDouble(double& dub); + + template + void readString(StrType& str); + + void readBinary(std::string& str) { + readString(str); + } + + size_t bytesRead() const { + return bytesRead_; + } + + private: + bool beginReadString(); // returns true if interned + + const Schema* schema_; + const InternTable* internTable_; + folly::io::RWPrivateCursor cursor_; + int64_t rootType_; + + enum State { + IN_STRUCT, + IN_FIELD, + IN_MAP_KEY, + IN_MAP_VALUE, + IN_LIST_VALUE, + IN_SET_VALUE, + }; + + static const int64_t kVariableLength = -1; + static const int64_t kTerminated = -2; + + struct TypeInfo { + TypeInfo() + : typeVal(reflection::TYPE_VOID), + length(kVariableLength), + dataType(nullptr), + terminator('\0') { } + TypeInfo(const Schema* schema, int64_t t); + /* implicit */ TypeInfo(int64_t t); + reflection::Type type() const { + return reflection::getType(typeVal); + } + TType ttype() const { + return toTType(type()); + } + + int64_t typeVal; + int64_t length; + const DataType* dataType; + char terminator; + }; + + static const size_t kIntInline = 8; + static const size_t kInt64Inline = 8; + static const size_t kByteInline = 8; + static const size_t kFixedInt16Inline = 8; + static const size_t kFixedInt32Inline = 8; + static const size_t kFixedInt64Inline = 8; + static const size_t kBoolInline = 8; + static const size_t kStringInline = 8; + + struct DecoderState { + DecoderState(const Schema* schema, int64_t type, + const DataType* dataType, uint32_t size); + DecoderState(DecoderState&&) = default; + DecoderState& operator=(DecoderState&&) = default; + + enum FieldState { + FS_START, + FS_INT, + FS_INT64, + FS_BYTE, + FS_BOOL, + FS_STRICT_ENUM, + FS_INTERNED_STRING, + FS_STRING, + FS_END + }; + + const DataType* dataType; + State state; + + TypeInfo type; + + size_t bytesRead; + + struct TypeStateBase { + TypeStateBase() : count(0), index(-1) { } + size_t count; + ssize_t index; + }; + + template + struct TypeState : public TypeStateBase { + folly::small_vector values; + }; + + TypeState ints; // int16_t, int32_t + TypeState int64s; // int64_t, double + TypeState bytes; // int8_t + size_t boolStartBit; // offset of first bit from bools in bools.values + TypeState bools; // bool + TypeState strictEnums; + size_t totalStrictEnumBits; + TypeState vars; // variable-length + TypeState internedStrings; + TypeStateBase strings; // non-interned strings and user-defined types + + // TODO(tudorb): Make Struct and List into a union, but as these types are + // non-POD, the union would have a deleted copy/move constructor, + // copy/move assignment operator, and destructor, which meanns that + // DecoderState would have to have them user-defined in order to insert + // DecoderState in containers, bleh. + + typedef + folly::small_vector, 8> + TagVec; + + typedef + folly::small_vector, 8> + FullTagVec; + + struct Struct { + FieldState fieldState; + int16_t tag; + + TagVec intTags; + TagVec int64Tags; + TagVec byteTags; + TagVec boolTags; + FullTagVec strictEnumTags; + + FullTagVec stringTags; + FullTagVec internedStringTags; + + TagVec fixedInt16Tags; + TagVec fixedInt32Tags; + TagVec fixedInt64Tags; + } str; + struct List { + uint32_t remaining; + TypeInfo mapKeyType; + TypeInfo valueType; + } list; + + template + void setStateList(T& ts); + + template + bool setStateStruct(TS& ts, const TV& tv, FieldState nextState); + + bool nextField(); + void nextValue(); + void addType(TypeInfo& tinfo, + const StructField& field, int16_t tag, uint32_t count); + + private: + void setLength(); + void nextList(); + bool nextStruct(); + }; + + size_t bytesRead_; + static const size_t kStackInline = 8; + folly::small_vector stack_; + + void readBoolsAndStrictEnums(size_t skipBits); + void read(); + void push(reflection::Type expected, int64_t type, uint32_t size); + void pop(); + DecoderState& top(); + uint32_t peekElementCount(); + int64_t nextType(); + std::pair ensure(size_t n); +}; + + +} // namespace neutronium +} // namespace protocol +} // namespace thrift +} // namespace apache + +#define THRIFT_INCLUDE_DECODER_INL +#include "thrift/lib/cpp/protocol/neutronium/Decoder-inl.h" +#undef THRIFT_INCLUDE_DECODER_INL + + +#endif /* THRIFT_LIB_CPP_PROTOCOL_NEUTRONIUM_DECODER_H_ */ + diff --git a/thrift/lib/cpp/protocol/neutronium/Encoder-inl.h b/thrift/lib/cpp/protocol/neutronium/Encoder-inl.h new file mode 100644 index 0000000000..a70a8a9dbf --- /dev/null +++ b/thrift/lib/cpp/protocol/neutronium/Encoder-inl.h @@ -0,0 +1,357 @@ +/** + * Copyright 2012 Facebook + * @author Tudor Bosman (tudorb@fb.com) + */ + +#ifndef THRIFT_LIB_CPP_PROTOCOL_NEUTRONIUM_ENCODER_INL_H_ +#define THRIFT_LIB_CPP_PROTOCOL_NEUTRONIUM_ENCODER_INL_H_ + +#ifndef THRIFT_INCLUDE_ENCODER_INL +#error This file may only be included from Encoder.h +#endif + +#include "folly/Conv.h" + +namespace apache { +namespace thrift { +namespace protocol { +namespace neutronium { + +inline void Encoder::writeFieldBegin(const char* name, TType fieldType, + int16_t fieldId) { + auto& s = top(); + DCHECK(s.state == IN_STRUCT); + s.field = s.dataType->fields.at(fieldId); + s.tag = fieldId; + s.state = IN_FIELD; +} + +inline void Encoder::writeFieldEnd() { + auto& s = top(); + DCHECK(s.state == DONE_FIELD); + s.tag = 0; + s.field.clear(); + s.state = IN_STRUCT; +} + +inline void Encoder::writeFieldStop() { + auto& s = top(); + DCHECK(s.state == IN_STRUCT); + flush(); +} + +inline bool Encoder::EncoderState::inDataState() const { + return (state == IN_FIELD || + state == IN_MAP_KEY || + state == IN_MAP_VALUE || + state == IN_LIST_VALUE || + state == IN_SET_VALUE); +} + +inline bool Encoder::EncoderState::inFlushableState() const { + return (state == IN_STRUCT || + state == IN_MAP_KEY || + state == IN_LIST_VALUE || + state == IN_SET_VALUE); +} + +inline void Encoder::EncoderState::dataWritten() { + switch (state) { + case IN_FIELD: + state = DONE_FIELD; + break; + case IN_MAP_KEY: + state = IN_MAP_VALUE; + field.type = dataType->valueType; + break; + case IN_MAP_VALUE: + state = IN_MAP_KEY; + field.type = dataType->mapKeyType; + break; + case IN_LIST_VALUE: + break; + case IN_SET_VALUE: + break; + default: + LOG(FATAL) << "Invalid state " << state; + } +} + +template +inline size_t findIndex(const C& container, T val) { + auto pos = container.find(val); + if (pos == container.end()) throw std::out_of_range("not found"); + return pos - container.begin(); +} + +inline void Encoder::EncoderState::markFieldSet() { + // only useful for structs + if (state != IN_FIELD || field.isRequired) return; + optionalSet[findIndex(dataType->optionalFields, tag)] = true; +} + +template +inline void Encoder::EncoderState::checkType() const { + throw TLibraryException(folly::to("Invalid type ", field.type)); +} + +template +inline void Encoder::EncoderState::checkType(reflection::Type t, + Args... tail) const { + if (t == reflection::getType(field.type)) return; + checkType(tail...); +} + +template +inline void Encoder::EncoderState::appendToOutput(const Vec& vec) { + size_t bytes = vec.size() * sizeof(typename Vec::value_type::second_type); + for (auto& p : vec) { + appender.writeBE(p.second); + } + bytesWritten += bytes; +} + +inline void Encoder::writeMapBegin(TType keyType, TType valType, + uint32_t size) { + push(reflection::TYPE_MAP, topType(), size); +} + +inline void Encoder::writeMapEnd() { + DCHECK_EQ(top().state, IN_MAP_KEY); + flush(); + pop(); +} + +inline void Encoder::writeListBegin(TType elemType, uint32_t size) { + push(reflection::TYPE_LIST, topType(), size); +} + +inline void Encoder::writeListEnd() { + DCHECK_EQ(top().state, IN_LIST_VALUE); + flush(); + pop(); +} + +inline void Encoder::writeSetBegin(TType elemType, uint32_t size) { + push(reflection::TYPE_SET, topType(), size); +} + +inline void Encoder::writeSetEnd() { + DCHECK_EQ(top().state, IN_SET_VALUE); + flush(); + pop(); +} + +inline void Encoder::writeStructBegin(const char* name) { + push(reflection::TYPE_STRUCT, topType(), 0); +} + +inline void Encoder::writeStructEnd() { + DCHECK_EQ(top().state, FLUSHED); + // writeFieldStop() called flush() + pop(); +} + +inline void Encoder::push(reflection::Type rtype, int64_t type, + uint32_t size) { + DCHECK(stack_.empty() || top().inDataState()); + if (reflection::getType(type) != rtype) { + throw TLibraryException(folly::to( + "Invalid aggregate type ", reflection::getType(type), + " expected ", rtype)); + } + + const DataType* dt = &(schema_->map().at(type)); + stack_.emplace_back(new EncoderState(type, dt, size)); +} + +inline void Encoder::pop() { + auto& s = top(); + DCHECK_EQ(s.state, FLUSHED); + int64_t size = s.dataType->fixedSize; + DCHECK(size == -1 || size == s.buf->computeChainDataLength()); + size_t bytesWritten = s.bytesWritten; + DCHECK_EQ(bytesWritten, s.buf->computeChainDataLength()); + auto buf = std::move(s.buf); + stack_.pop_back(); + if (stack_.empty()) { + outputBuf_->prependChain(std::move(buf)); + bytesWritten_ = bytesWritten; + } else { + writeData(std::move(buf)); + top().bytesWritten += bytesWritten; + } +} + +inline Encoder::EncoderState& Encoder::top() { + DCHECK(!stack_.empty()); + DCHECK(stack_.back()); + return *stack_.back(); +} + +inline const Encoder::EncoderState& Encoder::top() const { + DCHECK(!stack_.empty()); + DCHECK(stack_.back()); + return *stack_.back(); +} + +inline int64_t Encoder::topType() const { + return (stack_.empty() ? rootType_ : top().field.type); +} + +inline void Encoder::writeBool(bool v) { + auto& s = top(); + DCHECK(s.inDataState()); + s.checkType(reflection::TYPE_BOOL); + s.markFieldSet(); + s.bools.emplace_back(s.tag, v); + s.dataWritten(); +} + +inline void Encoder::writeByte(int8_t v) { + auto& s = top(); + DCHECK(s.inDataState()); + s.markFieldSet(); + s.checkType(reflection::TYPE_BYTE); + s.bytes.emplace_back(s.tag, v); + s.dataWritten(); +} + +inline void Encoder::writeI16(int16_t v) { + auto& s = top(); + DCHECK(s.inDataState()); + s.checkType(reflection::TYPE_I16); + s.markFieldSet(); + if (s.field.isFixed) { + s.fixedInt16s.emplace_back(s.tag, v); + } else { + s.varInts.emplace_back(s.tag, v); + } + s.dataWritten(); +} + +inline void Encoder::writeI32(int32_t v) { + auto& s = top(); + DCHECK(s.inDataState()); + s.checkType(reflection::TYPE_I32, reflection::TYPE_ENUM); + s.markFieldSet(); + if (reflection::getType(s.field.type) == reflection::TYPE_ENUM && + s.field.isStrictEnum) { + auto& dt = schema_->map().at(s.field.type); + uint8_t nbits = dt.enumBits(); + uint32_t value = findIndex(dt.enumValues, v); + s.strictEnums.push_back({s.tag, {nbits, value}}); + s.totalStrictEnumBits += nbits; + } else { + if (s.field.isFixed) { + s.fixedInt32s.emplace_back(s.tag, v); + } else { + s.varInts.emplace_back(s.tag, v); + } + } + s.dataWritten(); +} + +inline void Encoder::writeI64(int64_t v) { + innerWriteI64(v, reflection::TYPE_I64); +} + +inline void Encoder::writeDouble(double v) { + innerWriteI64(bitwise_cast(v), reflection::TYPE_DOUBLE); +} + +inline void Encoder::innerWriteI64(int64_t v, reflection::Type expected) { + auto& s = top(); + DCHECK(s.inDataState()); + s.markFieldSet(); + s.checkType(expected); + if (s.field.isFixed) { + s.fixedInt64s.emplace_back(s.tag, v); + } else { + s.varInt64s.emplace_back(s.tag, v); + } + s.dataWritten(); +} + +namespace { +std::unique_ptr copyBufferToSize( + const void* data, size_t dataSize, size_t outSize, char pad) { + auto buf = folly::IOBuf::create(outSize); + dataSize = std::min(dataSize, outSize); + memcpy(buf->writableData(), data, dataSize); + if (dataSize < outSize) { + memset(buf->writableData() + dataSize, pad, outSize - dataSize); + } + buf->append(outSize); + return buf; +} + +std::unique_ptr copyBufferAndTerminate( + const void* data, size_t dataSize, char terminator) { + if (memchr(data, terminator, dataSize)) { + throw TProtocolException("terminator found in terminated string"); + } + // 1 byte of tailroom for the terminator + auto buf = folly::IOBuf::copyBuffer(data, dataSize, 0, 1); + buf->writableTail()[0] = terminator; + buf->append(1); + return buf; +} + +} // namespace + + +// TODO(tudorb): Zero-copy version +inline void Encoder::writeBytes(folly::StringPiece data) { + auto& s = top(); + DCHECK(s.inDataState()); + s.checkType(reflection::TYPE_STRING); + s.markFieldSet(); + if (s.field.isInterned) { + CHECK(internTable_); + s.varInternIds.emplace_back(s.tag, internTable_->add(data)); + } else if (s.field.isFixed) { + s.strings.emplace_back( + s.tag, copyBufferToSize(data.data(), data.size(), + s.field.fixedStringSize, s.field.pad)); + s.bytesWritten += s.field.fixedStringSize; + } else if (s.field.isTerminated) { + s.strings.emplace_back( + s.tag, copyBufferAndTerminate(data.data(), data.size(), + s.field.terminator)); + s.bytesWritten += data.size() + 1; + } else { + s.varLengths.emplace_back(s.tag, data.size()); + s.strings.emplace_back( + s.tag, folly::IOBuf::copyBuffer(data.data(), data.size())); + s.bytesWritten += data.size(); + } + s.dataWritten(); +} + +inline void Encoder::writeData(std::unique_ptr&& data) { + auto& s = top(); + DCHECK(s.inDataState()); + s.markFieldSet(); + s.strings.emplace_back(s.tag, std::move(data)); + s.dataWritten(); +} + +inline void Encoder::flush() { + auto& s = top(); + DCHECK(s.inFlushableState()); + if (s.state == IN_STRUCT) { + // TODO(tudorb): Check that all required fields were actually specified. + flushStruct(); + } + flushData(s.state == IN_STRUCT); + s.state = FLUSHED; +} + +} // namespace neutronium +} // namespace protocol +} // namespace thrift +} // namespace apache + +#endif /* THRIFT_LIB_CPP_PROTOCOL_NEUTRONIUM_ENCODER_INL_H_ */ + diff --git a/thrift/lib/cpp/protocol/neutronium/Encoder.h b/thrift/lib/cpp/protocol/neutronium/Encoder.h new file mode 100644 index 0000000000..be2a5590ea --- /dev/null +++ b/thrift/lib/cpp/protocol/neutronium/Encoder.h @@ -0,0 +1,180 @@ +/** + * Copyright 2012 Facebook + * @author Tudor Bosman (tudorb@fb.com) + */ + +#ifndef THRIFT_LIB_CPP_PROTOCOL_NEUTRONIUM_ENCODER_H_ +#define THRIFT_LIB_CPP_PROTOCOL_NEUTRONIUM_ENCODER_H_ + +#include "thrift/lib/cpp/protocol/neutronium/Utils.h" +#include "thrift/lib/cpp/protocol/neutronium/Schema.h" +#include "thrift/lib/cpp/protocol/neutronium/InternTable.h" +#include "thrift/lib/cpp/protocol/TProtocol.h" +#include "folly/FBString.h" +#include "folly/Range.h" +#include "folly/experimental/io/IOBuf.h" +#include "folly/experimental/io/Cursor.h" + +namespace apache { +namespace thrift { +namespace protocol { +namespace neutronium { + +class Encoder { + public: + Encoder(const Schema* schema, InternTable* internTable, folly::IOBuf* buf); + void setRootType(int64_t rootType); + + // Similar interface to the writing part of TProtocol + void writeStructBegin(const char* name); + void writeStructEnd(); + + void writeFieldBegin(const char* name, TType fieldType, int16_t fieldId); + void writeFieldEnd(); + void writeFieldStop(); + + void writeMapBegin(TType keyType, TType valType, uint32_t size); + void writeMapEnd(); + void writeListBegin(TType elemType, uint32_t size); + void writeListEnd(); + void writeSetBegin(TType elemType, uint32_t size); + void writeSetEnd(); + + void writeBool(bool value); + void writeByte(int8_t byte); + void writeI16(int16_t i16); + void writeI32(int32_t i32); + void writeI64(int64_t i64); + void writeDouble(double dub); + + void writeBinary(const std::string& str) { + writeString(str); + } + template + void writeString(const StrType& str) { + writeBytes(str); + } + + /** + * Return number of bytes written. Non-zero only at root level, after + * the final writeStructEnd(). + */ + size_t bytesWritten() const { + return bytesWritten_; + } + + private: + void innerWriteI64(int64_t i64, reflection::Type expected); + + void writeData(std::unique_ptr&& data); + void markFieldSet(); + + void writeBytes(folly::StringPiece data); + + int32_t intern(folly::StringPiece data); + + const Schema* schema_; + int64_t rootType_; + size_t bytesWritten_; + + enum State { + IN_STRUCT, + IN_FIELD, + DONE_FIELD, + IN_MAP_KEY, + IN_MAP_VALUE, + IN_LIST_VALUE, + IN_SET_VALUE, + FLUSHED, + }; + + struct EncoderState { + EncoderState(int64_t type, const DataType* dt, uint32_t size); + + const DataType* dataType; + State state; + // TODO(tudorb): Check type_ + int16_t tag; + StructField field; + std::unique_ptr buf; + folly::io::Appender appender; + + std::vector> bools; // bool + struct StrictEnum { + uint8_t bits; + uint32_t value; + }; + // strict (bit-field) enums + std::vector> strictEnums; + size_t totalStrictEnumBits; + std::vector> bytes; // byte + + // Integer fields that were requested to be represented as fixed-length + std::vector> fixedInt16s; // i16 + std::vector> fixedInt32s; // i32 + std::vector> fixedInt64s; // i64, double + + // Integer fields that are represented as variable-length (GroupVarint) + std::vector> varInts; // i16, i32 + std::vector> varInt64s; // i64, double + + // Lengths of strings of unknown size, represented as GroupVarint + std::vector> varLengths; + + // Ids of interned strings, represented as GroupVarint + // (see InternTable.h) + std::vector> varInternIds; + + // Strings AND child elements (structs, lists, sets, maps) + // Note that only strings can be of unknown size (and thus use up one + // entry in varLengths, above); we always fully decode children and so + // we determine their size that way + std::vector>> strings; + + // Bitset (see Utils.h) of which optional fields are set; the index is + // the index in DataType::optionalFields. + std::vector optionalSet; + + size_t bytesWritten; + + bool inDataState() const; + void dataWritten(); + + bool inFlushableState() const; + void markFieldSet(); + + template void checkType() const; + template void checkType(reflection::Type t, + Args... tail) const; + + template + void appendToOutput(const Vec& vec); + }; + + std::vector> stack_; + void push(reflection::Type expected, int64_t type, uint32_t size); + void pop(); + int64_t topType() const; + EncoderState& top(); + const EncoderState& top() const; + + void flush(); + void flushBitValues(); + void flushStruct(); + void flushData(bool isStruct); + + InternTable* internTable_; + folly::IOBuf* outputBuf_; +}; + +} // namespace neutronium +} // namespace protocol +} // namespace thrift +} // namespace apache + + +#define THRIFT_INCLUDE_ENCODER_INL +#include "thrift/lib/cpp/protocol/neutronium/Encoder-inl.h" +#undef THRIFT_INCLUDE_ENCODER_INL + +#endif /* THRIFT_LIB_CPP_PROTOCOL_NEUTRONIUM_ENCODER_H_ */ diff --git a/thrift/lib/cpp/protocol/neutronium/InternTable.h b/thrift/lib/cpp/protocol/neutronium/InternTable.h new file mode 100644 index 0000000000..48ad032373 --- /dev/null +++ b/thrift/lib/cpp/protocol/neutronium/InternTable.h @@ -0,0 +1,102 @@ +/** + * Copyright 2012 Facebook + * @author Tudor Bosman (tudorb@fb.com) + */ + +#ifndef THRIFT_LIB_CPP_PROTOCOL_NEUTRONIUM_INTERNTABLE_H_ +#define THRIFT_LIB_CPP_PROTOCOL_NEUTRONIUM_INTERNTABLE_H_ + +#include +#include +#include +#include "folly/Range.h" +#include "folly/experimental/io/IOBuf.h" +#include "folly/experimental/io/Cursor.h" + +namespace apache { +namespace thrift { +namespace protocol { +namespace neutronium { + +/** + * Interned string table. + * + * Strings are stored in a chain of IOBufs, so serializing and deserializing + * the InternTable is fast and easy. + */ +class InternTable { + public: + InternTable(); + + /** + * Add a string to the table, returning the id. + */ + uint32_t add(folly::StringPiece str); + + /** + * Get a string from the table, by id; throws std::out_of_range if the + * id is invalid. + */ + folly::StringPiece get(uint32_t id) const { + return strings_.at(id); + } + + /** + * Serialize the table to a IOBuf chain. + * Note that part of the chain may be shared with the InternTable, so + * you must call unshare() on the returned buffers if you want to + * write to them. + */ + std::unique_ptr serialize(); + + /** + * Load the table from a IOBuf chain. + * Takes ownership of the IOBuf chain. + * InternTable will call unshare() appropriately before writing to + * buffers from the provided chain. + */ + void deserialize(std::unique_ptr&& data); + + /** + * Clear the table. + */ + void clear(); + + private: + // Performance optimization to reduce the number of calls to unshare() / + // unshareOne(), which folly::io::Cursor benchmarks have shown to be + // somewhat expensive. Given that we know that we only write to the tail + // of the IOBuf chain, we can often tell with certainty that the tail + // is private to us and so we don't need to check or call unshareOne(). + bool maybeShared_; + std::unique_ptr head_; + std::vector strings_; + + typedef + std::unordered_map + StringMap; + + // Only allocate writer if needed (when actually add()ing) + struct Writer { + template + /* implicit */ Writer(Args... args) : appender(args...) { } + folly::io::Appender appender; + StringMap map; + }; + std::unique_ptr writer_; + + // Not copiable; not movable + InternTable(const InternTable&) = delete; + InternTable(InternTable&&) = delete; + InternTable& operator=(const InternTable&) = delete; + InternTable& operator=(InternTable&&) = delete; + +}; + +} // namespace neutronium +} // namespace protocol +} // namespace thrift +} // namespace apache + +#endif /* THRIFT_LIB_CPP_PROTOCOL_NEUTRONIUM_INTERNTABLE_H_ */ + diff --git a/thrift/lib/cpp/protocol/neutronium/README b/thrift/lib/cpp/protocol/neutronium/README new file mode 100644 index 0000000000..b5ff365e85 --- /dev/null +++ b/thrift/lib/cpp/protocol/neutronium/README @@ -0,0 +1,67 @@ +Neutronium: a very dense encoding of Thrift objects + +Neutronium is a Thrift encoding format optimized for space at the expense of +speed. It achieves high efficiency in a few ways: + +1. It does not encode type and tag information. This is stored out of line in +a Schema object, which must be provided at both encoding and decoding time. +If encoding many Thrift objects, you can transmit / store the Schema only +once. The Schema (in thrift/lib/thrift/reflection.thrift) is itself a +Thrift object, and can be serialized / deserialized in the usual way. + +2. It encodes data very compactly. Bytes use one byte each; larger numbers +(i16, i32, i64, double) use variable-length encoding (GroupVarint). Booleans +use one bit each. We also encode one bit for every optional field that exists +in the structure definition (indicating whether the field is set or not). +Strings can be encoded in a variety of formats, see below. + +3. Aggregates (lists, maps, sets) are encoded efficiently -- they are encoded +like a structure with a variable number of fields. So list takes +advantage of GroupVarint encoding among consecutive values. + +4. Strings can be interned: when encoding multiple strings, we can detect +duplicates and store only an ID. This requires using an InternTable and +passing the same InternTable to the encoder and decoder (the InternTable +can be easily serialized and deserialized). + +Neutronium is backwards compatible as long as the schema is identical from +encoding to decoding, and the changes you made to the Thrift definition are +backwards compatible (that is, fields were added, removed, or renamed, but +field ids remained the same) + +Configuration: + +Neutronium can be configured by using field attributes in your Thrift +definition: + +struct Foo { + 1: i32 a (neutronium.fixed = 1), +} + +Attributes for number fields: + neutronium.fixed = 1 + Encode the number as a fixed-length value (i16 takes 2 bytes, i32 + takes 4 bytes, i64 takes 8 bytes) instead of using Varint encoding + +Attributes for string fields: + neutronium.fixed = + neutronium.pad = 'X' + Do not encode the string length, assume that all strings have length + . Strings longer than are truncated; strings + shorter than are padded with 'X' (default: the null byte, '\0'). + Use this when you expect that all / most strings have a fixed length. + + neutronium.terminator = 'X' + Do not encode the string length; store strings terminated with a + terminator ('\0' will likely be a popular choice). Encoding strings that + contain the terminator is an error. + + neutronium.intern = 1 + Intern strings; requires a non-NULL InternTable. + +Attributes for enum fields: + neutronium.strict = 1 + Encode the enum using as few bits as necessary to encode all possible + values; note that it becomes an error to encode an enum value that is not + specified in the Thrift definition. + diff --git a/thrift/lib/cpp/protocol/neutronium/Schema-inl.h b/thrift/lib/cpp/protocol/neutronium/Schema-inl.h new file mode 100644 index 0000000000..8ba257e719 --- /dev/null +++ b/thrift/lib/cpp/protocol/neutronium/Schema-inl.h @@ -0,0 +1,32 @@ +/** + * Copyright 2012 Facebook + * @author Tudor Bosman (tudorb@fb.com) + */ + +#ifndef THRIFT_LIB_CPP_PROTOCOL_NEUTRONIUM_SCHEMA_INL_H_ +#define THRIFT_LIB_CPP_PROTOCOL_NEUTRONIUM_SCHEMA_INL_H_ + +#ifndef THRIFT_INCLUDE_SCHEMA_INL +#error This file may only be included from Schema.h +#endif + +namespace apache { +namespace thrift { +namespace protocol { +namespace neutronium { + +namespace detail { +extern const TType typeToTType[]; +} // namespace detail + +inline TType toTType(reflection::Type t) { + return detail::typeToTType[t]; +} + +} // namespace neutronium +} // namespace protocol +} // namespace thrift +} // namespace apache + +#endif /* THRIFT_LIB_CPP_PROTOCOL_NEUTRONIUM_SCHEMA_INL_H_ */ + diff --git a/thrift/lib/cpp/protocol/neutronium/Schema.h b/thrift/lib/cpp/protocol/neutronium/Schema.h new file mode 100644 index 0000000000..aea16993ad --- /dev/null +++ b/thrift/lib/cpp/protocol/neutronium/Schema.h @@ -0,0 +1,111 @@ +/** + * Copyright 2012 Facebook + * @author Tudor Bosman (tudorb@fb.com) + */ + +#ifndef THRIFT_LIB_CPP_PROTOCOL_NEUTRONIUM_SCHEMA_H_ +#define THRIFT_LIB_CPP_PROTOCOL_NEUTRONIUM_SCHEMA_H_ + +#include +#include "boost/interprocess/containers/flat_map.hpp" +#include "boost/interprocess/containers/flat_set.hpp" +#include "thrift/lib/cpp/protocol/TProtocol.h" +#include "thrift/lib/cpp/Reflection.h" +#include "external/glog/logging.h" +#include "folly/Bits.h" +#include "thrift/lib/cpp/protocol/neutronium/Utils.h" + +namespace apache { +namespace thrift { +namespace protocol { +namespace neutronium { + +struct StructField { + StructField() + : type(0), + isRequired(false), + isInterned(false), + isFixed(false), + isTerminated(false), + fixedStringSize(0), + pad('\0'), + terminator('\0') { + } + + void setFlags(const reflection::DataType& rtype, + const reflection::StructField& rfield); + + void clear() { + *this = StructField(); + } + + // from reflection::StructField + int64_t type; + bool isRequired : 1; + bool isInterned : 1; + + // numbers: not varint-encoded + // strings: fixed size, fixedStringSize bytes, padded with 'pad' + bool isFixed : 1; + + // strings: terminated with 'terminator' which may not occur in the string + bool isTerminated : 1; + + // enums: strict (bitfield) encoding + bool isStrictEnum : 1; + + uint32_t fixedStringSize; + char pad; + char terminator; +}; + +struct DataType { + DataType() : mapKeyType(0), valueType(0), fixedSize(-1) { } + + // from reflection::StructField + boost::container::flat_map fields; + int64_t mapKeyType; + int64_t valueType; + // generated + int64_t fixedSize; // -1 = not fixed size + boost::container::flat_set optionalFields; + boost::container::flat_set enumValues; + + // Minimum number of bits needed to represent all enum values + // 0 if there's only one possible value -- there's nothing to encode! + size_t enumBits() const { + return enumValues.empty() ? 0 : folly::findLastSet(enumValues.size() - 1); + } +}; + +class Schema { + public: + typedef std::unordered_map Map; + Schema() { } + explicit Schema(const reflection::Schema& rschema); + void add(const reflection::Schema& rschema); + + const Map& map() const { return map_; } + + private: + int64_t fixedSizeForField(const StructField& field) const; + + int64_t add(const reflection::Schema& rschema, int64_t type); + void add(const reflection::Schema& rschema, int64_t type, + const reflection::DataType& dt); + Map map_; +}; + +TType toTType(reflection::Type t); + +} // namespace neutronium +} // namespace protocol +} // namespace thrift +} // namespace apache + +#define THRIFT_INCLUDE_SCHEMA_INL +#include "thrift/lib/cpp/protocol/neutronium/Schema-inl.h" +#undef THRIFT_INCLUDE_SCHEMA_INL + +#endif /* THRIFT_LIB_CPP_PROTOCOL_NEUTRONIUM_SCHEMA_H_ */ + diff --git a/thrift/lib/cpp/protocol/neutronium/TARGETS b/thrift/lib/cpp/protocol/neutronium/TARGETS new file mode 100644 index 0000000000..130a998e95 --- /dev/null +++ b/thrift/lib/cpp/protocol/neutronium/TARGETS @@ -0,0 +1,24 @@ +# Copyright 2011 Facebook +# @author Tudor Bosman (tudorb@fb.com) + +cpp_library( + name = "neutronium", + srcs = [ + "Schema.cpp", + "Encoder.cpp", + "Decoder.cpp", + "InternTable.cpp", + ], + thrift_srcs = { + "intern_table.thrift": [], + }, + deps = [ + "@/folly", + "@/folly/experimental/io", + "@/thrift/lib/cpp/protocol", + ], + external_deps = [ + ("boost", None, "boost"), + ], +) + diff --git a/thrift/lib/cpp/protocol/neutronium/Utils.h b/thrift/lib/cpp/protocol/neutronium/Utils.h new file mode 100644 index 0000000000..d1f6dc53e2 --- /dev/null +++ b/thrift/lib/cpp/protocol/neutronium/Utils.h @@ -0,0 +1,110 @@ +/** + * Copyright 2012 Facebook + * @author Tudor Bosman (tudorb@fb.com) + */ + +#ifndef THRIFT_LIB_CPP_PROTOCOL_NEUTRONIUM_UTILS_H_ +#define THRIFT_LIB_CPP_PROTOCOL_NEUTRONIUM_UTILS_H_ + +#include +#include +#include +#include + +namespace apache { +namespace thrift { +namespace protocol { +namespace neutronium { + +// Helper functions for using an array of uint8_t values as a bitset + +// byte index +constexpr inline size_t byteIndex(size_t bit) { + return bit / 8; +} + +// offset within byte +constexpr inline size_t bitOffset(size_t bit) { + return bit % 8; +} + +// number of bytes needed to represent bitCount bits (round up) +constexpr inline size_t byteCount(size_t bitCount) { + return bitCount / 8 + (bitCount % 8 != 0); +} + +// set bit +inline void setBit(uint8_t* p, size_t bit) { + p[byteIndex(bit)] |= (1 << bitOffset(bit)); +} + +inline void setBits(uint8_t* p, size_t bit, size_t count, uint32_t value) { + assert(value < (1 << count)); + while (count) { + size_t index = byteIndex(bit); + size_t offset = bitOffset(bit); + size_t countInThisByte = std::min(count, 8 - offset); + // Extract "countInThisByte" bits from value + uint8_t mask = (1 << countInThisByte) - 1; + uint8_t v = value & mask; + // Wipe out these bits, then set them to the new value + p[index] = (p[index] & ~(mask << offset)) | (v << offset); + + value >>= countInThisByte; + bit += countInThisByte; + count -= countInThisByte; + } +} + +inline uint32_t getBits(uint8_t* p, size_t bit, size_t count) { + uint32_t value = 0; + size_t shift = 0; + while (count) { + size_t index = byteIndex(bit); + size_t offset = bitOffset(bit); + size_t countInThisByte = std::min(count, 8 - offset); + uint8_t mask = (1 << countInThisByte) - 1; + uint8_t v = (p[index] & (mask << offset)) >> offset; + + value |= (v << shift); + shift += countInThisByte; + bit += countInThisByte; + count -= countInThisByte; + } + return value; +} + +// clear bit +inline void clearBit(uint8_t* p, size_t bit) { + p[byteIndex(bit)] &= ~(1 << bitOffset(bit)); +} + +// test bit +inline bool testBit(const uint8_t* p, size_t bit) { + return p[byteIndex(bit)] & (1 << bitOffset(bit)); +} + +// Overloads for vector, array, or anything else that has +// a working operator[] +template +inline void setBit(T& p, size_t bit) { + setBit(&p[0], bit); +} + +template +inline void clearBit(T& p, size_t bit) { + clearBit(&p[0], bit); +} + +template +inline bool testBit(const T& p, size_t bit) { + return testBit(&p[0], bit); +} + +} // namespace neutronium +} // namespace protocol +} // namespace thrift +} // namespace apache + +#endif /* THRIFT_LIB_CPP_PROTOCOL_NEUTRONIUM_UTILS_H_ */ + diff --git a/thrift/lib/cpp/protocol/neutronium/intern_table.thrift b/thrift/lib/cpp/protocol/neutronium/intern_table.thrift new file mode 100644 index 0000000000..4951c990c6 --- /dev/null +++ b/thrift/lib/cpp/protocol/neutronium/intern_table.thrift @@ -0,0 +1,6 @@ +namespace cpp apache.thrift.protocol.neutronium + +struct InternTableSizes { + 1: required list sizes, +} + diff --git a/thrift/lib/cpp/protocol/neutronium/test/TARGETS b/thrift/lib/cpp/protocol/neutronium/test/TARGETS new file mode 100644 index 0000000000..df99ef6dfb --- /dev/null +++ b/thrift/lib/cpp/protocol/neutronium/test/TARGETS @@ -0,0 +1,41 @@ +# Copyright 2011 Facebook +# @author Tudor Bosman (tudorb@fb.com) + +thrift_library( + name = "if", + thrift_srcs = { + "neutronium_test.thrift" : [], + }, + languages = ["cpp"], + thrift_cpp_options = "templates=only,neutronium", + cpp_deps = [], +) + +cpp_unittest( + name = "intern_table_test", + srcs = ["InternTableTest.cpp"], + deps = [ + "@/thrift/lib/cpp/protocol/neutronium", + "@/external/gflags", + ], + external_deps = [ + ("gtest", None, "gtest_main"), + ], + type = 'gtest', +) + +cpp_unittest( + name = "neutronium_test", + srcs = ["NeutroniumTest.cpp"], + deps = [ + "@/thrift/lib/cpp/protocol/neutronium", + "@/thrift/lib/cpp/util", + ":if-cpp", + "@/common/fbunit:old_folly_benchmark", + ], + external_deps = [ + ("gtest", None), + ], + type = 'gtest', +) + diff --git a/thrift/lib/cpp/protocol/neutronium/test/neutronium_test.thrift b/thrift/lib/cpp/protocol/neutronium/test/neutronium_test.thrift new file mode 100644 index 0000000000..ec04995365 --- /dev/null +++ b/thrift/lib/cpp/protocol/neutronium/test/neutronium_test.thrift @@ -0,0 +1,80 @@ +namespace cpp apache.thrift.protocol.neutronium.test + +cpp_include "folly/FBString.h" + +struct TestFixedSizeStruct1 { + 1: bool a, + 2: bool b, + 3: bool c, + 4: byte d, +} + +struct TestFixedSizeStruct2 { + 1: i32 a (neutronium.fixed = 1), + 2: i64 b (neutronium.fixed = 1), +} + +struct TestNotFixedSizeStruct2 { + 1: i32 a, + 2: i64 b, +} + +struct TestStruct1 { + 1: bool a, + 2: optional bool b, + 3: i32 c (neutronium.fixed = 1), + 4: optional i32 d, + 5: i64 e, + 6: optional i64 f, + 7: string g +} + +struct TestStruct2 { + 1: i32 a, + 2: TestStruct1 b, + 3: string c, + 4: list d, + 5: list e, + 6: map f, +} + +struct TestStringEncoding1 { + 1: i32 a, + 2: string b, + 3: string c (neutronium.intern = 1), + 4: string d1 (neutronium.fixed = 10, neutronium.pad = 'X'), + 5: string d2 (neutronium.fixed = 10, neutronium.pad = 'X'), + 6: string e (neutronium.terminator = 'X'), +} + +enum Foo { + HELLO = 1, // 0 + WORLD = 42, // 1 + MEOW = 23456, // 3 + GOODBYE = 12345 // 2 +} + +struct TestEnumEncoding1 { + 1: Foo a, + 2: Foo b (neutronium.strict = 1), + 3: bool c, + 4: i32 d +} + +struct TestEnumEncoding2 { + 1: bool a, + 2: Foo b (neutronium.strict = 1), + 4: optional Foo d (neutronium.strict = 1), + 5: Foo e (neutronium.strict = 1), +} + +struct BenchStruct1 { + 1: required i32 a, +} + +struct BenchStruct2 { + 1: optional i32 a, + 4: optional string b, + 5: optional list c +} + diff --git a/thrift/lib/cpp/server/TARGETS b/thrift/lib/cpp/server/TARGETS new file mode 100644 index 0000000000..967deceaf8 --- /dev/null +++ b/thrift/lib/cpp/server/TARGETS @@ -0,0 +1,30 @@ +# TARGETS file for thrift/lib/cpp/server + +cpp_library ( + name = "server", + srcs = [ + "TRpcTransportContext.cpp", + "TServer.cpp", + "TSimpleServer.cpp", + "TThreadedServer.cpp", + "TThreadPoolServer.cpp", + ], + deps = [ + "@/thrift/lib/cpp/concurrency", + "@/thrift/lib/cpp/transport", + ] +) + +cpp_library ( + name = "nonblocking", + srcs = [ "TNonblockingServer.cpp" ], + deps = [ + ":server", + "@/thrift/lib/cpp/concurrency", + "@/thrift/lib/cpp/transport", + "@/thrift/lib/cpp/transport:header", + "@/thrift/lib/cpp/protocol:protocol", + ], + preprocessor_flags = [ "-DTCP_LOW_MIN_RTO=15" ], + external_deps = [ ('libevent', None) ], +) diff --git a/thrift/lib/cpp/server/TConnectionContext.h b/thrift/lib/cpp/server/TConnectionContext.h new file mode 100644 index 0000000000..41da039ccc --- /dev/null +++ b/thrift/lib/cpp/server/TConnectionContext.h @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_TCONNECTIONCONTEXT_H_ +#define THRIFT_TCONNECTIONCONTEXT_H_ 1 + +#include + +#include +#include "thrift/lib/cpp/protocol/TProtocol.h" + +namespace apache { namespace thrift { + +namespace transport { +class TSocketAddress; +} + +namespace protocol { +class TProtocol; +} + +namespace server { + +class TConnectionContext { + public: + TConnectionContext() + : userData_(NULL) + , destructor_(NULL) {} + + virtual ~TConnectionContext() { + cleanupUserData(); + } + + // expose getPeerAddress() defined in TRpcTransportContext + + virtual const transport::TSocketAddress* getPeerAddress() const = 0; + + virtual boost::shared_ptr getInputProtocol() const = 0; + + virtual boost::shared_ptr getOutputProtocol() const = 0; + /** + * Get the user data field. + */ + void* getUserData() const { + return userData_; + } + + /** + * Set the user data field. + * + * @param data The new value for the user data field. + * @param destructor A function pointer to invoke when the connection + * context is destroyed. It will be invoked with the + * contents of the user data field. + * + * @return Returns the old user data value. + */ + virtual void* setUserData(void* data, void (*destructor)(void*) = NULL) { + void* oldData = userData_; + userData_ = data; + destructor_ = destructor; + return oldData; + } + + protected: + void cleanupUserData() { + if (destructor_) { + destructor_(userData_); + destructor_ = NULL; + } + userData_ = NULL; + } + + private: + void* userData_; + void (*destructor_)(void*); +}; + +}}} // apache::thrift::server + +#endif // THRIFT_TCONNECTIONCONTEXT_H_ diff --git a/thrift/lib/cpp/server/TNonblockingServer.h b/thrift/lib/cpp/server/TNonblockingServer.h new file mode 100644 index 0000000000..18dce011a5 --- /dev/null +++ b/thrift/lib/cpp/server/TNonblockingServer.h @@ -0,0 +1,1165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef THRIFT_SERVER_TNONBLOCKINGSERVER_H_ +#define THRIFT_SERVER_TNONBLOCKINGSERVER_H_ 1 + +#include "thrift/lib/cpp/Thrift.h" +#include "thrift/lib/cpp/server/TServer.h" +#include "thrift/lib/cpp/transport/TBufferTransports.h" +#include "thrift/lib/cpp/transport/TSocket.h" +#include "thrift/lib/cpp/concurrency/ThreadManager.h" +#include "thrift/lib/cpp/concurrency/Thread.h" +#include "thrift/lib/cpp/concurrency/PosixThreadFactory.h" +#include "thrift/lib/cpp/concurrency/Mutex.h" +#include "thrift/lib/cpp/concurrency/ThreadLocal.h" +#include +#include +#include +#include +#include +#include +#include +#include // libevent +#include + + +namespace apache { namespace thrift { namespace server { + +using apache::thrift::transport::TMemoryBuffer; +using apache::thrift::transport::TSocket; +using apache::thrift::protocol::TProtocol; +using apache::thrift::concurrency::Runnable; +using apache::thrift::concurrency::ThreadManager; +using apache::thrift::concurrency::PosixThreadFactory; +using apache::thrift::concurrency::ThreadFactory; +using apache::thrift::concurrency::Thread; +using apache::thrift::concurrency::Mutex; +using apache::thrift::concurrency::Guard; + +/** + * This is a non-blocking server in C++ for high performance that + * operates a set of IO threads (by default only one). It assumes that + * all incoming requests are framed with a 4 byte length indicator and + * writes out responses using the same framing. + * + * It does not use the TServerTransport framework, but rather has socket + * operations hardcoded for use with select. + * + */ + + +/// Overload condition actions. +enum TOverloadAction { + T_OVERLOAD_NO_ACTION, ///< Don't handle overload */ + T_OVERLOAD_CLOSE_ON_ACCEPT, ///< Drop new connections immediately */ + T_OVERLOAD_DRAIN_TASK_QUEUE, ///< Drop some tasks from head of task queue */ + T_OVERLOAD_PAUSE_ACCEPTING ///< do not accept any new connections +}; + +class TNonblockingIOThread; +class TNonblockingServerObserver; + +class TNonblockingServer : public TServer { + private: + class TConnection; + + friend class TNonblockingIOThread; + private: + /// Listen backlog + static const int LISTEN_BACKLOG = 1024; + + /// Default limit on size of idle connection pool + static const int CONNECTION_STACK_LIMIT = -1; + + /// Default limit on frame size + static const int MAX_FRAME_SIZE = 256 * 1024 * 1024; + + /// Default limit on total number of connected sockets + static const int MAX_CONNECTIONS = INT_MAX; + + /// Default limit on connections in handler/task processing + static const int MAX_ACTIVE_PROCESSORS = INT_MAX; + + /// Default limit on the number of active requests on the server. + static const int MAX_ACTIVE_REQUESTS = INT_MAX; + + /// Default limit on memory usage + static const uint64_t MAX_MEMORY_USAGE_BYTES = ULONG_MAX; + + /// Default size of write buffer + static const int WRITE_BUFFER_DEFAULT_SIZE = 1024; + + /// Maximum size of read buffer allocated to idle connection (0 = unlimited) + static const int IDLE_READ_BUFFER_LIMIT = 8192; + + /// Maximum size of write buffer allocated to idle connection (0 = unlimited) + static const int IDLE_WRITE_BUFFER_LIMIT = 0; + + /// # of calls before resizing oversized buffers (0 = check only on close) + static const int RESIZE_BUFFER_EVERY_N = 0; + + /// # of IO threads to use by default + static const int DEFAULT_IO_THREADS = 1; + + /// File descriptor of an invalid socket + static const int INVALID_SOCKET = -1; + + /// # of IO threads this server will use + size_t numIOThreads_; + + /// Whether to set high scheduling priority for IO threads + bool useHighPriorityIOThreads_; + + /// Server socket file descriptor + int serverSocket_; + + /// Port server runs on + int port_; + + /// For processing via thread pool, may be NULL + boost::shared_ptr threadManager_; + + /// Is thread pool processing? + bool threadPoolProcessing_; + + // Factory to create the IO threads + boost::shared_ptr ioThreadFactory_; + + // Vector of IOThread objects that will handle our IO + std::vector > ioThreads_; + + // Index of next IO Thread to be used (for round-robin) + int nextIOThread_; + + // Synchronizes access to connection stack and similar data, such as + // numActiveRequests_. + Mutex connMutex_; + + /// Number of TConnection object we've created + size_t numTConnections_; + + /// Number of Connections processing or waiting to process + size_t numActiveProcessors_; + + /// Number of active/outstanding requests on the server. I.e., sum of requests + /// that are being processed, waiting to be processed and are waiting for the + /// reply to be sent out over the wire. + size_t numActiveRequests_; + + /// Limit for the number of requests that are being processed, waiting to be + /// processed, or are waiting for the reply to be sent out over the wire. + size_t maxActiveRequests_; + + /// listen backlog + int32_t listenBacklog_; + + /// Limit for how many TConnection objects to cache + /// 0 = infinite, -1 = none. + size_t connectionStackLimit_; + + /// Limit for number of connections processing or waiting to process + size_t maxActiveProcessors_; + + /// Limit for number of open connections + size_t maxConnections_; + + /// Limit for memory usage + uint64_t maxMemoryUsageBytes_; + + /// Limit for frame size + size_t maxFrameSize_; + + /// Time in milliseconds before an unperformed task expires (0 == infinite). + int64_t taskExpireTime_; + + /// Time in milliseconds to pause accepts for OVERLOAD_PAUSE_ACCEPTING + int pauseAcceptDuration_; + + /** + * Hysteresis for overload state. This is the fraction of the overload + * value that needs to be reached before the overload state is cleared; + * must be <= 1.0. + */ + double overloadHysteresis_; + + /// Action to take when we're overloaded. + TOverloadAction overloadAction_; + + /** + * The write buffer is initialized (and when idleWriteBufferLimit_ is checked + * and found to be exceeded, reinitialized) to this size. + */ + size_t writeBufferDefaultSize_; + + /** + * Max read buffer size for an idle TConnection. When we place an idle + * TConnection into connectionStack_ or on every resizeBufferEveryN_ calls, + * we will free the buffer (such that it will be reinitialized by the next + * received frame) if it has exceeded this limit. 0 disables this check. + */ + size_t idleReadBufferLimit_; + + /** + * Max write buffer size for an idle connection. When we place an idle + * TConnection into connectionStack_ or on every resizeBufferEveryN_ calls, + * we insure that its write buffer is <= to this size; otherwise we + * replace it with a new one of writeBufferDefaultSize_ bytes to insure that + * idle connections don't hog memory. 0 disables this check. + */ + size_t idleWriteBufferLimit_; + + /** + * Every N calls we check the buffer size limits on a connected TConnection. + * 0 disables (i.e. the checks are only done when a connection closes). + */ + int32_t resizeBufferEveryN_; + + /// Set if we are currently in an overloaded state. + bool overloaded_; + + /// Count of connections dropped since overload started + uint32_t nConnectionsDropped_; + + /// Count of connections dropped on overload since server started + uint64_t nTotalConnectionsDropped_; + + // Notification of various server events + boost::shared_ptr observer_; + + /** + * This is a stack of all the objects that have been created but that + * are NOT currently in use. When we close a connection, we place it on this + * stack so that the object can be reused later, rather than freeing the + * memory and reallocating a new object later. + */ + std::stack connectionStack_; + + /** + * This contains all existing TConnections, including the ones in the + * connectionStack_. + */ + std::tr1::unordered_set connectionSet_; + + /** + * Struct to help set socket options for + * TNonblockingServer::TConnection->TSocket sockets + */ + TSocket::Options sockOptHelper_; + + /** + * Thread-local data storage to track the current connection being processed. + */ + typedef concurrency::ThreadLocal< + TConnection, + concurrency::NoopThreadLocalManager > + ThreadLocalConnection; + ThreadLocalConnection currentConnection_; + + /** + * Called when server socket had something happen. We accept all waiting + * client connections on listen socket fd and assign TConnection objects + * to handle those requests. + * + * @param fd the listen socket. + * @param which the event flag that triggered the handler. + */ + void handleEvent(int fd, short which); + + void init(int port) { + serverSocket_ = -1; + numIOThreads_ = DEFAULT_IO_THREADS; + nextIOThread_ = 0; + useHighPriorityIOThreads_ = false; + port_ = port; + threadPoolProcessing_ = false; + numTConnections_ = 0; + numActiveProcessors_ = 0; + numActiveRequests_ = 0; + connectionStackLimit_ = CONNECTION_STACK_LIMIT; + maxActiveProcessors_ = MAX_ACTIVE_PROCESSORS; + maxActiveRequests_ = MAX_ACTIVE_REQUESTS; + maxConnections_ = MAX_CONNECTIONS; + maxMemoryUsageBytes_ = MAX_MEMORY_USAGE_BYTES; + maxFrameSize_ = MAX_FRAME_SIZE; + taskExpireTime_ = 0; + pauseAcceptDuration_ = 1; + overloadHysteresis_ = 0.8; + overloadAction_ = T_OVERLOAD_NO_ACTION; + writeBufferDefaultSize_ = WRITE_BUFFER_DEFAULT_SIZE; + idleReadBufferLimit_ = IDLE_READ_BUFFER_LIMIT; + idleWriteBufferLimit_ = IDLE_WRITE_BUFFER_LIMIT; + resizeBufferEveryN_ = RESIZE_BUFFER_EVERY_N; + overloaded_ = false; + nConnectionsDropped_ = 0; + nTotalConnectionsDropped_ = 0; + listenBacklog_ = LISTEN_BACKLOG; + + } + + size_t getWriteBufferBytesImpl() const; + size_t getReadBufferBytesImpl() const; + + protected: + + /// Increment the count of connections currently processing. + void incrementActiveProcessors() { + Guard g(connMutex_); + ++numActiveProcessors_; + } + + /// Decrement the count of connections currently processing. + void decrementActiveProcessors() { + Guard g(connMutex_); + if (numActiveProcessors_ > 0) { + --numActiveProcessors_; + } + } + + /// Increment the number of active requests on the server. + void incrementNumActiveRequests() { + Guard g(connMutex_); + ++numActiveRequests_; + } + + /// Decrement the number of active requests on the server. + void decrementNumActiveRequests() { + Guard g(connMutex_); + if (numActiveRequests_ > 0) { + --numActiveRequests_; + } + } + + public: + template + TNonblockingServer( + const boost::shared_ptr& processorFactory, + int port, + THRIFT_OVERLOAD_IF(ProcessorFactory, TProcessorFactory)) : + TServer(processorFactory) { + init(port); + } + + template + TNonblockingServer(const boost::shared_ptr& processor, + int port, + THRIFT_OVERLOAD_IF(Processor, TProcessor)) : + TServer(processor) { + init(port); + } + + template + TNonblockingServer( + const boost::shared_ptr& processorFactory, + const boost::shared_ptr& protocolFactory, + int port, + const boost::shared_ptr& threadManager = + boost::shared_ptr(), + THRIFT_OVERLOAD_IF(ProcessorFactory, TProcessorFactory)) : + TServer(processorFactory) { + + init(port); + + setProtocolFactory(protocolFactory); + setThreadManager(threadManager); + } + + template + TNonblockingServer( + const boost::shared_ptr& processorFactory, + const boost::shared_ptr& duplexProtocolFactory, + int port, + const boost::shared_ptr& threadManager = + boost::shared_ptr(), + THRIFT_OVERLOAD_IF(ProcessorFactory, TProcessorFactory)) : + TServer(processorFactory) { + + init(port); + + setDuplexProtocolFactory(duplexProtocolFactory); + setThreadManager(threadManager); + } + + template + TNonblockingServer( + const boost::shared_ptr& processor, + const boost::shared_ptr& protocolFactory, + int port, + const boost::shared_ptr& threadManager = + boost::shared_ptr(), + THRIFT_OVERLOAD_IF(Processor, TProcessor)) : + TServer(processor) { + + init(port); + + setProtocolFactory(protocolFactory); + setThreadManager(threadManager); + } + + template + TNonblockingServer( + const boost::shared_ptr& processor, + const boost::shared_ptr& duplexProtocolFactory, + int port, + const boost::shared_ptr& threadManager = + boost::shared_ptr(), + THRIFT_OVERLOAD_IF(Processor, TProcessor)) : + TServer(processor) { + + init(port); + + setDuplexProtocolFactory(duplexProtocolFactory); + setThreadManager(threadManager); + } + + template + TNonblockingServer( + const boost::shared_ptr& processorFactory, + const boost::shared_ptr& duplexTransportFactory, + const boost::shared_ptr& duplexProtocolFactory, + int port, + const boost::shared_ptr& threadManager = + boost::shared_ptr(), + THRIFT_OVERLOAD_IF(ProcessorFactory, TProcessorFactory)): + TServer(processorFactory) { + + init(port); + + setDuplexTransportFactory(duplexTransportFactory); + setDuplexProtocolFactory(duplexProtocolFactory); + setThreadManager(threadManager); + } + + template + TNonblockingServer( + const boost::shared_ptr& processor, + const boost::shared_ptr& duplexTransportFactory, + const boost::shared_ptr& duplexProtocolFactory, + int port, + const boost::shared_ptr& threadManager = + boost::shared_ptr(), + THRIFT_OVERLOAD_IF(Processor, TProcessor)): + TServer(processor) { + + init(port); + + setDuplexTransportFactory(duplexTransportFactory); + setDuplexProtocolFactory(duplexProtocolFactory); + setThreadManager(threadManager); + } + + void setThreadManager(boost::shared_ptr threadManager); + + boost::shared_ptr getThreadManager() { + return threadManager_; + } + + /** + * Sets the number of IO threads used by this server. Can only be used before + * the call to serve() and has no effect afterwards. We always use a + * PosixThreadFactory for the IO worker threads, because they must joinable + * for clean shutdown. + */ + void setNumIOThreads(size_t numThreads) { + numIOThreads_ = numThreads; + } + + /** Return whether the IO threads will get high scheduling priority */ + bool useHighPriorityIOThreads() const { + return useHighPriorityIOThreads_; + } + + /** Set whether the IO threads will get high scheduling priority. */ + void setUseHighPriorityIOThreads(bool val) { + useHighPriorityIOThreads_ = val; + } + + /** Return the number of IO threads used by this server. */ + size_t getNumIOThreads() const { + return numIOThreads_; + } + + /** + * Get the maximum number of unused TConnection we will hold in reserve. + * + * @return the current limit on TConnection pool size. + */ + size_t getConnectionStackLimit() const { + return connectionStackLimit_; + } + + /** + * Set the maximum number of unused TConnection we will hold in reserve. + * + * @param sz the new limit for TConnection pool size. + */ + void setConnectionStackLimit(size_t sz) { + connectionStackLimit_ = sz; + } + + bool isThreadPoolProcessing() const { + return threadPoolProcessing_; + } + + /** + * set the listen backlog + * + * @param listenBacklog the new listen backlog + */ + void setListenBacklog(int32_t listenBacklog) { + listenBacklog_ = listenBacklog; + } + + /** + * Return the count of sockets currently connected to. + * + * @return count of connected sockets. + */ + size_t getNumConnections() const { + return numTConnections_; + } + + /** + * Return the count of sockets currently connected to. + * + * @return count of connected sockets. + */ + size_t getNumActiveConnections() const { + return getNumConnections() - getNumIdleConnections(); + } + + /** + * Return the bytes in all connection's write buffer. + * + * @return bytes + */ + size_t getWriteBufferBytes() const { + concurrency::Guard g(connMutex_); + return getWriteBufferBytesImpl(); + } + + /** + * Return the bytes in all connection's read buffer. + * + * @return bytes + */ + size_t getReadBufferBytes() const { + concurrency::Guard g(connMutex_); + return getReadBufferBytesImpl(); + } + + /** + * Return the count of connection objects allocated but not in use. + * + * @return count of idle connection objects. + */ + size_t getNumIdleConnections() const { + return connectionStack_.size(); + } + + /** + * Return count of number of connections which are currently processing. + * This is defined as a connection where all data has been received and + * either assigned a task (when threading) or passed to a handler (when + * not threading), and where the handler has not yet returned. + * + * @return # of connections currently processing. + */ + size_t getNumActiveProcessors() const { + return numActiveProcessors_; + } + + /** + * Return the number of active/outstanding requests on the server. This + * includes requests that are processing, waiting to be processed and have + * been processed, but are waiting for the reply to be sent out over the wire. + * + * @return # of currently active requests. + */ + size_t getNumActiveRequests() const { + return numActiveRequests_; + } + + /** + * Get the maximum number of active requests. + * + * @return current setting. + */ + size_t getMaxActiveRequests() const { + return maxActiveRequests_; + } + + /** + * Set the maximum # of active requests allowed before overload. + * + * @param maxActiveRequsts new setting for maximum # of active requests. + */ + void setMaxActiveRequests(size_t maxActiveRequests) { + maxActiveRequests_ = maxActiveRequests; + } + + /** + * Get the maximum # of connections allowed before overload. + * + * @return current setting. + */ + size_t getMaxConnections() const { + return maxConnections_; + } + + /** + * Set the maximum # of connections allowed before overload. + * + * @param maxConnections new setting for maximum # of connections. + */ + void setMaxConnections(size_t maxConnections) { + maxConnections_ = maxConnections; + } + + /** + * Get the maximum # of connections waiting in handler/task before overload. + * + * @return current setting. + */ + size_t getMaxActiveProcessors() const { + return maxActiveProcessors_; + } + + /** + * Set the maximum # of connections waiting in handler/task before overload. + * + * @param maxActiveProcessors new setting for maximum # of active processes. + */ + void setMaxActiveProcessors(size_t maxActiveProcessors) { + maxActiveProcessors_ = maxActiveProcessors; + } + + /** + * Get the maximum memory usage allowed before overload. + * + * @return current setting. + */ + uint64_t getMaxMemoryUsageBytes() const { + return maxMemoryUsageBytes_; + } + + /** + * Set the maximum memory usage allowed before overload. + * + * @param maxMemoryUsage new setting for maximum memory usage. + */ + void setMaxMemoryUsageBytes(uint64_t maxMemoryUsageBytes) { + maxMemoryUsageBytes_ = maxMemoryUsageBytes; + } + + /** + * Get the maximum allowed frame size. + * + * If a client tries to send a message larger than this limit, + * its connection will be closed. + * + * @return Maxium frame size, in bytes. + */ + size_t getMaxFrameSize() const { + return maxFrameSize_; + } + + /** + * Set the maximum allowed frame size, up to 2^31-1 + * + * @param maxFrameSize The new maximum frame size. + */ + void setMaxFrameSize(size_t maxFrameSize) { + if (maxFrameSize > 0x7fffffff) { + maxFrameSize = 0x7fffffff; + } + maxFrameSize_ = maxFrameSize; + } + + /** + * Get fraction of maximum limits before an overload condition is cleared. + * + * @return hysteresis fraction + */ + double getOverloadHysteresis() const { + return overloadHysteresis_; + } + + /** + * Set fraction of maximum limits before an overload condition is cleared. + * A good value would probably be between 0.5 and 0.9. + * + * @param hysteresisFraction fraction <= 1.0. + */ + void setOverloadHysteresis(double hysteresisFraction) { + if (hysteresisFraction <= 1.0 && hysteresisFraction > 0.0) { + overloadHysteresis_ = hysteresisFraction; + } + } + + /** + * Get duration of stopping accepts when overload + * + * @return pauseAcceptDuration_ + */ + int getPauseAcceptDuration() const { + return pauseAcceptDuration_; + } + + /** + * Set the accept pause duration (when overload) in ms + * + * @param pauseDuration > 0 + */ + void setPauseAcceptDuration(int pauseDuration) { + if (pauseDuration > 0) { + pauseAcceptDuration_ = pauseDuration; + } + } + + /** + * Get the action the server will take on overload. + * + * @return a TOverloadAction enum value for the currently set action. + */ + TOverloadAction getOverloadAction() const { + return overloadAction_; + } + + /** + * Set the action the server is to take on overload. + * + * @param overloadAction a TOverloadAction enum value for the action. + */ + void setOverloadAction(TOverloadAction overloadAction) { + overloadAction_ = overloadAction; + } + + /** + * Get the time in milliseconds after which a task expires (0 == infinite). + * + * @return a 64-bit time in milliseconds. + */ + int64_t getTaskExpireTime() const { + return taskExpireTime_; + } + + /** + * Set the time in milliseconds after which a task expires (0 == infinite). + * + * @param taskExpireTime a 64-bit time in milliseconds. + */ + void setTaskExpireTime(int64_t taskExpireTime) { + taskExpireTime_ = taskExpireTime; + } + + /** + * Determine if the server is currently overloaded. + * This function checks the maximums for open connections and connections + * currently in processing, and sets an overload condition if they are + * exceeded. The overload will persist until both values are below the + * current hysteresis fraction of their maximums. + * + * @return true if an overload condition exists, false if not. + */ + bool serverOverloaded(); + + /** + * Check if the server is overloaded and overloadAction_ is + * T_OVERLOAD_PAUSE_ACCEPTING, and if so, pause the accept on the listening + * socket + * + * @return true if overloadAction_ != OVERLOAD_PAUSE_ACCEPTING, or + * if overload condition doesn't exist. + */ + bool canContinueToAccept(); + + /** + * Check if the server is overloaded, and if so, take the necessary action. + * + * @param conn The connection currently being processed that is triggering + * the overload check. + * + * Returns true if the current connection has been closed due to overload + * processing, and false otherwise. (Note that a false return value does not + * mean that the server is not overloaded.) + */ + bool checkForOverload(TConnection* conn); + + + /** Pop and discard next task on threadpool wait queue. + * + * @return true if a task was discarded, false if the wait queue was empty. + */ + bool drainPendingTask(); + + /** + * Get the starting size of a TConnection object's write buffer. + * + * @return # bytes we initialize a TConnection object's write buffer to. + */ + size_t getWriteBufferDefaultSize() const { + return writeBufferDefaultSize_; + } + + /** + * Set the starting size of a TConnection object's write buffer. + * + * @param size # bytes we initialize a TConnection object's write buffer to. + */ + void setWriteBufferDefaultSize(size_t size) { + writeBufferDefaultSize_ = size; + } + + /** + * Get the maximum size of read buffer allocated to idle TConnection objects. + * + * @return # bytes beyond which we will dealloc idle buffer. + */ + size_t getIdleReadBufferLimit() const { + return idleReadBufferLimit_; + } + + /** + * [NOTE: This is for backwards compatibility, use getIdleReadBufferLimit().] + * Get the maximum size of read buffer allocated to idle TConnection objects. + * + * @return # bytes beyond which we will dealloc idle buffer. + */ + size_t getIdleBufferMemLimit() const { + return idleReadBufferLimit_; + } + + /** + * Set the maximum size read buffer allocated to idle TConnection objects. + * If a TConnection object is found (either on connection close or between + * calls when resizeBufferEveryN_ is set) with more than this much memory + * allocated to its read buffer, we free it and allow it to be reinitialized + * on the next received frame. + * + * @param limit of bytes beyond which we will shrink buffers when checked. + */ + void setIdleReadBufferLimit(size_t limit) { + idleReadBufferLimit_ = limit; + } + + /** + * [NOTE: This is for backwards compatibility, use setIdleReadBufferLimit().] + * Set the maximum size read buffer allocated to idle TConnection objects. + * If a TConnection object is found (either on connection close or between + * calls when resizeBufferEveryN_ is set) with more than this much memory + * allocated to its read buffer, we free it and allow it to be reinitialized + * on the next received frame. + * + * @param limit of bytes beyond which we will shrink buffers when checked. + */ + void setIdleBufferMemLimit(size_t limit) { + idleReadBufferLimit_ = limit; + } + + + + /** + * Get the maximum size of write buffer allocated to idle TConnection objects. + * + * @return # bytes beyond which we will reallocate buffers when checked. + */ + size_t getIdleWriteBufferLimit() const { + return idleWriteBufferLimit_; + } + + /** + * Set the maximum size write buffer allocated to idle TConnection objects. + * If a TConnection object is found (either on connection close or between + * calls when resizeBufferEveryN_ is set) with more than this much memory + * allocated to its write buffer, we destroy and construct that buffer with + * writeBufferDefaultSize_ bytes. + * + * @param limit of bytes beyond which we will shrink buffers when idle. + */ + void setIdleWriteBufferLimit(size_t limit) { + idleWriteBufferLimit_ = limit; + } + + /** + * Get # of calls made between buffer size checks. 0 means disabled. + * + * @return # of calls between buffer size checks. + */ + int32_t getResizeBufferEveryN() const { + return resizeBufferEveryN_; + } + + /** + * Check buffer sizes every "count" calls. This allows buffer limits + * to be enforced for persistant connections with a controllable degree + * of overhead. 0 disables checks except at connection close. + * + * @param count the number of calls between checks, or 0 to disable + */ + void setResizeBufferEveryN(int32_t count) { + resizeBufferEveryN_ = count; + } + + /** + * Sets the sockOptHelper_ + * Socket options on underlying TSockets are set by passing + * the sockOptHelper_ to TSocket::setSocketOptions() function + * in the TConnection constructor + */ + void setSocketOptions(const TSocket::Options& oh) { + sockOptHelper_ = oh; + } + + void setObserver( + const boost::shared_ptr& observer) { + observer_ = observer; + } + + /** + * Main workhorse function, starts up the server listening on a port and + * loops over the libevent handler. + */ + void serve(); + + /** + * Causes the server to terminate gracefully (can be called from any thread). + */ + void stop(); + + TConnectionContext* getConnectionContext() const; + + private: + void addTask(boost::shared_ptr task) { + threadManager_->add(task, 0LL, taskExpireTime_); + } + + void setCurrentConnection(TConnection* conn) { + assert(currentConnection_.get() == NULL); + currentConnection_.set(conn); + } + void clearCurrentConnection() { + currentConnection_.clear(); + } + + /** + * Callback function that the threadmanager calls when a task reaches + * its expiration time. It is needed to clean up the expired connection. + * + * @param task the runnable associated with the expired task. + */ + void expireClose(boost::shared_ptr task); + + const boost::shared_ptr& getObserver() const { + return observer_; + } + + /// Creates a socket to listen on and binds it to the local port. + void createAndListenOnSocket(); + + /** + * Takes a socket created by createAndListenOnSocket() and sets various + * options on it to prepare for use in the server. + * + * @param fd descriptor of socket to be initialized/ + */ + void listenSocket(int fd); + /** + * Return an initialized connection object. Creates or recovers from + * pool a TConnection and initializes it with the provided socket FD + * and flags. + * + * @param socket FD of socket associated with this connection. + * @param addr the sockaddr of the client + * @param addrLen the length of addr + * @return pointer to initialized TConnection object. + */ + TConnection* createConnection(int socket, const sockaddr* addr, + socklen_t addrLen); + + /** + * Returns a connection to pool or deletion. If the connection pool + * (a stack) isn't full, place the connection object on it, otherwise + * just delete it. + * + * @param connection the TConection being returned. + */ + void returnConnection(TConnection* connection); +}; + +class TNonblockingIOThread : public Runnable { + public: + // Creates an IO thread and sets up the event base. The listenSocket should + // be a valid FD on which listen() has already been called. If the + // listenSocket is < 0, accepting will not be done. + TNonblockingIOThread(TNonblockingServer* server, + int number, + int listenSocket, + bool useHighPriority); + + ~TNonblockingIOThread(); + + // Returns the event-base for this thread. + event_base* getEventBase() const { return eventBase_; } + + // Returns the server for this thread. + TNonblockingServer* getServer() const { return server_; } + + // Returns the number of this IO thread. + int getThreadNumber() const { return number_; } + + // Returns the thread id associated with this object. This should + // only be called after the thread has been started. + pthread_t getThreadId() const { return threadId_; } + + // Returns the send-fd for task complete notifications. + int getNotificationSendFD() const { return notificationPipeFDs_[1]; } + + // Returns the read-fd for task complete notifications. + int getNotificationRecvFD() const { return notificationPipeFDs_[0]; } + + // Returns the actual thread object associated with this IO thread. + boost::shared_ptr getThread() const { return thread_; } + + // Sets the actual thread object associated with this IO thread. + void setThread(const boost::shared_ptr& t) { thread_ = t; } + + // Used by TConnection objects to indicate processing has finished. + bool notify(TNonblockingServer::TConnection* conn); + + // Enters the event loop and does not return until a call to stop(). + virtual void run(); + + // Exits the event loop as soon as possible. + void stop(); + + // Ensures that the event-loop thread is fully finished and shut down. + void join(); + + /// Maintains a count of requests for sampling purposes + /// request counter is maintained module the sample rate + uint32_t incrementRequestCounter(uint32_t sampleRate); + + /// Return true if the currently running thread is this I/O thread + bool isInIOThread() const { + return pthread_equal(pthread_self(), threadId_); + } + + /// Pauses accept event handling for pauseDuration milliseconds + void pauseAcceptHandling(int pauseDuration); + + private: + /** + * C-callable event handler for signaling task completion. Provides a + * callback that libevent can understand that will read a connection + * object's address from a pipe and call connection->transition() for + * that object. + * + * @param fd the descriptor the event occurred on. + */ + static void notifyHandler(int fd, short which, void* v); + + /** + * C-callable event handler for listener events. Provides a callback + * that libevent can understand which invokes server->handleEvent(). + * + * @param fd the descriptor the event occurred on. + * @param which the flags associated with the event. + * @param v void* callback arg where we placed TNonblockingServer's "this". + */ + static void listenHandler(int fd, short which, void* v) { + ((TNonblockingServer*)v)->handleEvent(fd, which); + } + + /** + * C-callable event handler to re-enable accept handling. + * Provides a callback that libevent can understand. + * + * @param fd unused. + * @param which the flags associated with the event. + * @param v void* callback arg where we placed TNonblockingIOThread's "this". + */ + static void reenableAcceptHandler(int fd, short which, void* v) { + ((TNonblockingIOThread*)v)->reenableAccept(); + } + + /// Re-enables accept event handling. + void reenableAccept(); + + /// Exits the loop ASAP in case of shutdown or error. + void breakLoop(bool error); + + /// Registers the events for the notification & listen sockets + void registerEvents(); + + /// Create the pipe used to notify I/O process of task completion. + void createNotificationPipe(); + + /// Unregisters our events for notification and listen sockets. + void cleanupEvents(); + + /// Sets (or clears) high priority scheduling status for the current thread. + void setCurrentThreadHighPriority(bool value); + + private: + /// associated server + TNonblockingServer* server_; + + /// thread number (for debugging). + const int number_; + + /// The actual physical thread id. + pthread_t threadId_; + + /// If listenSocket_ >= 0, adds an event on the event_base to accept conns + int listenSocket_; + + /// Sets a high scheduling priority when running + bool useHighPriority_; + + /// pointer to eventbase to be used for looping + event_base* eventBase_; + + /// Whether the server is currently accepting connections + bool acceptingConnections_; + + /// Used with eventBase_ for connection events (only in listener thread) + struct event serverEvent_; + + /// Used with eventBase_ for backing off on accept handling until there are + /// available fds + struct event acceptBackoffEvent_; + + /// Used with eventBase_ for task completion notification + struct event notificationEvent_; + + /// File descriptors for pipe used for task completion notification. + int notificationPipeFDs_[2]; + + /// Actual IO Thread + boost::shared_ptr thread_; + + /// Call counter + uint32_t requestCounter_; +}; + + +}}} // apache::thrift::server + +#endif // #ifndef THRIFT_SERVER_TNONBLOCKINGSERVER_H_ diff --git a/thrift/lib/cpp/server/TNonblockingServerObserver.h b/thrift/lib/cpp/server/TNonblockingServerObserver.h new file mode 100644 index 0000000000..594a2849d7 --- /dev/null +++ b/thrift/lib/cpp/server/TNonblockingServerObserver.h @@ -0,0 +1,54 @@ +#ifndef THRIFT_SERVER_TNONBLOCKINGSERVEROBSERVER_H_ +#define THRIFT_SERVER_TNONBLOCKINGSERVEROBSERVER_H_ 1 + +#include + +namespace apache { namespace thrift { namespace server { + +class TNonblockingServerObserver { + public: + + virtual ~TNonblockingServerObserver() {} + + TNonblockingServerObserver() : sampleRate_(0) {} + TNonblockingServerObserver(uint32_t sampleRate) : sampleRate_(sampleRate) {} + + class CallTimestamps { + public: + uint64_t readBegin; + uint64_t readEnd; + uint64_t processBegin; + uint64_t processEnd; + uint64_t writeBegin; + uint64_t writeEnd; + + CallTimestamps() { + init(); + } + + void init() { + readBegin = readEnd = 0; + processBegin = processEnd = 0; + writeBegin = writeEnd = 0; + } + + }; + + // Notifications for various events on the TNonblockingServer + virtual void connDropped() = 0; + + virtual void taskKilled() = 0; + + virtual void callCompleted(const CallTimestamps& runtimes) = 0; + + // The observer has to specify a sample rate for callCompleted notifications + inline uint32_t getSampleRate() const { + return sampleRate_; + } + + protected: + uint32_t sampleRate_; +}; + +}}} // apache::thrift::server +#endif diff --git a/thrift/lib/cpp/server/TRpcTransportContext.h b/thrift/lib/cpp/server/TRpcTransportContext.h new file mode 100644 index 0000000000..cb1a30226e --- /dev/null +++ b/thrift/lib/cpp/server/TRpcTransportContext.h @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_SERVER_TRPCTRANSPORTCONTEXT_H_ +#define THRIFT_SERVER_TRPCTRANSPORTCONTEXT_H_ 1 + +#include "thrift/lib/cpp/server/TConnectionContext.h" + +#include + +namespace apache { namespace thrift { + +namespace transport { +class TRpcTransport; +} + +namespace server { + +class TRpcTransportContext : public TConnectionContext { + public: + TRpcTransportContext(boost::shared_ptr transport) + : transport_(transport) {} + + TRpcTransportContext( + const boost::shared_ptr& transport, + const boost::shared_ptr& iprot, + const boost::shared_ptr& oprot) + : transport_(transport), + iprot_(iprot), + oprot_(oprot) {} + + virtual const transport::TSocketAddress* getPeerAddress() const; + + const boost::shared_ptr& getTransport() const { + return transport_; + } + + virtual boost::shared_ptr getInputProtocol() const { + return iprot_; + } + + virtual boost::shared_ptr getOutputProtocol() const { + return oprot_; + } + + private: + boost::shared_ptr transport_; + boost::shared_ptr iprot_; + boost::shared_ptr oprot_; +}; + +}}} // apache::thrift::server + +#endif // THRIFT_SERVER_TRPCTRANSPORTCONTEXT_H_ diff --git a/thrift/lib/cpp/server/TServer.h b/thrift/lib/cpp/server/TServer.h new file mode 100644 index 0000000000..9623b41d01 --- /dev/null +++ b/thrift/lib/cpp/server/TServer.h @@ -0,0 +1,324 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef THRIFT_SERVER_TSERVER_H +#define THRIFT_SERVER_TSERVER_H 1 + +#include "thrift/lib/cpp/TProcessor.h" +#include "thrift/lib/cpp/transport/TServerTransport.h" +#include "thrift/lib/cpp/protocol/TBinaryProtocol.h" +#include "thrift/lib/cpp/concurrency/Thread.h" +#include "thrift/lib/cpp/util/shared_ptr_util.h" + +namespace apache { namespace thrift { + +namespace transport { +class TSocketAddress; +} + +namespace server { + +using apache::thrift::TProcessor; +using apache::thrift::protocol::TBinaryProtocolFactory; +using apache::thrift::protocol::TProtocol; +using apache::thrift::protocol::TProtocolFactory; +using apache::thrift::protocol::TDuplexProtocolFactory; +using apache::thrift::protocol::TDualProtocolFactory; +using apache::thrift::protocol::TSingleProtocolFactory; +using apache::thrift::transport::TServerTransport; +using apache::thrift::transport::TTransport; +using apache::thrift::transport::TTransportFactory; +using apache::thrift::transport::TDuplexTransportFactory; +using apache::thrift::transport::TDualTransportFactory; +using apache::thrift::transport::TSingleTransportFactory; + +class TConnectionContext; + +/** + * Virtual interface class that can handle events from the server core. To + * use this you should subclass it and implement the methods that you care + * about. Your subclass can also store local data that you may care about, + * such as additional "arguments" to these methods (stored in the object + * instance's state). + */ +class TServerEventHandler { + public: + + virtual ~TServerEventHandler() {} + + /** + * Called before the server begins. + * + * @param address The address on which the server is listening. + */ + virtual void preServe(const transport::TSocketAddress* address) {} + + /** + * Called when a new client has connected and is about to being processing. + * + * @param ctx A pointer to the connection context. The context will remain + * valid until the corresponding connectionDestroyed() call. + */ + virtual void newConnection(TConnectionContext* ctx) { + (void)ctx; + } + + /** + * Called when a client has finished request-handling to delete server + * context. + * + * @param ctx A pointer to the connection context. The context will be + * destroyed after connectionDestroyed() returns. + */ + virtual void connectionDestroyed(TConnectionContext* ctx) { + (void)ctx; + } + + protected: + + /** + * Prevent direct instantiation. + */ + TServerEventHandler() {} + +}; + +/** + * Thrift server. + * + */ +class TServer : public concurrency::Runnable { + public: + + virtual ~TServer() {} + + virtual void serve() = 0; + + virtual void stop() {} + + // This API is intended to stop listening on the server + // socket and stop accepting new connection first while + // still letting the established connections to be + // processed on the server. + virtual void stopListening() {} + + // Allows running the server as a Runnable thread + virtual void run() { + serve(); + } + + boost::shared_ptr getProcessorFactory() { + return processorFactory_; + } + + boost::shared_ptr getServerTransport() { + return serverTransport_; + } + + boost::shared_ptr getDuplexTransportFactory() { + return duplexTransportFactory_; + } + + boost::shared_ptr getDuplexProtocolFactory() { + return duplexProtocolFactory_; + } + + boost::shared_ptr getEventHandler() { + return eventHandler_; + } + + /** + * Get the TConnectionContext for the connection currently being processed. + * + * This is intended to be invoked from within the TProcessor (or the handler + * used by the TProcessor). + * + * Note: Not all server types currently support getConnectionContext(). Some + * servers may always return NULL. + * + * TODO: Eventually this method should be supported by all server types, and + * made into a pure virtual method. + * + * @return Return a pointer to the TConnectionContext for the current + * connection, or NULL if invoked outside of a call to + * TProcessor::process(). The returned object is only guaranteed to + * be valid until process() returns. + */ + virtual TConnectionContext* getConnectionContext() const { + return NULL; + } + +protected: + template + TServer(const boost::shared_ptr& processorFactory, + THRIFT_OVERLOAD_IF(ProcessorFactory, TProcessorFactory)): + processorFactory_(processorFactory) { + setTransportFactory(boost::shared_ptr( + new TTransportFactory())); + setProtocolFactory(boost::shared_ptr( + new TBinaryProtocolFactory())); + } + + template + TServer(const boost::shared_ptr& processor, + THRIFT_OVERLOAD_IF(Processor, TProcessor)): + processorFactory_(new TSingletonProcessorFactory(processor)) { + setTransportFactory(boost::shared_ptr( + new TTransportFactory())); + setProtocolFactory(boost::shared_ptr( + new TBinaryProtocolFactory())); + } + + template + TServer(const boost::shared_ptr& processorFactory, + const boost::shared_ptr& serverTransport, + THRIFT_OVERLOAD_IF(ProcessorFactory, TProcessorFactory)): + processorFactory_(processorFactory), + serverTransport_(serverTransport) { + setTransportFactory(boost::shared_ptr( + new TTransportFactory())); + setProtocolFactory(boost::shared_ptr( + new TBinaryProtocolFactory())); + } + + template + TServer(const boost::shared_ptr& processor, + const boost::shared_ptr& serverTransport, + THRIFT_OVERLOAD_IF(Processor, TProcessor)): + processorFactory_(new TSingletonProcessorFactory(processor)), + serverTransport_(serverTransport) { + setTransportFactory(boost::shared_ptr( + new TTransportFactory())); + setProtocolFactory(boost::shared_ptr( + new TBinaryProtocolFactory())); + } + + template + TServer(const boost::shared_ptr& processorFactory, + const boost::shared_ptr& serverTransport, + const boost::shared_ptr& transportFactory, + const boost::shared_ptr& protocolFactory, + THRIFT_OVERLOAD_IF(ProcessorFactory, TProcessorFactory)): + processorFactory_(processorFactory), + serverTransport_(serverTransport) { + setTransportFactory(transportFactory); + setProtocolFactory(protocolFactory); + } + + template + TServer(const boost::shared_ptr& processor, + const boost::shared_ptr& serverTransport, + const boost::shared_ptr& transportFactory, + const boost::shared_ptr& protocolFactory, + THRIFT_OVERLOAD_IF(Processor, TProcessor)): + processorFactory_(new TSingletonProcessorFactory(processor)), + serverTransport_(serverTransport) { + setTransportFactory(transportFactory); + setProtocolFactory(protocolFactory); + } + + template + TServer( + const boost::shared_ptr& processorFactory, + const boost::shared_ptr& serverTransport, + const boost::shared_ptr& duplexTransportFactory, + const boost::shared_ptr& duplexProtocolFactory, + THRIFT_OVERLOAD_IF(ProcessorFactory, TProcessorFactory)) : + processorFactory_(processorFactory), + serverTransport_(serverTransport), + duplexTransportFactory_(duplexTransportFactory), + duplexProtocolFactory_(duplexProtocolFactory) {} + + template + TServer( + const boost::shared_ptr& processor, + const boost::shared_ptr& serverTransport, + const boost::shared_ptr& duplexTransportFactory, + const boost::shared_ptr& duplexProtocolFactory, + THRIFT_OVERLOAD_IF(Processor, TProcessor)) : + processorFactory_(new TSingletonProcessorFactory(processor)), + serverTransport_(serverTransport), + duplexTransportFactory_(duplexTransportFactory), + duplexProtocolFactory_(duplexProtocolFactory) {} + + /** + * Get a TProcessor to handle calls on a particular connection. + * + * This method should only be called once per connection (never once per + * call). This allows the TProcessorFactory to return a different processor + * for each connection if it desires. + */ + boost::shared_ptr getProcessor(TConnectionContext* ctx) { + return processorFactory_->getProcessor(ctx); + } + + // Class variables + boost::shared_ptr processorFactory_; + boost::shared_ptr serverTransport_; + + boost::shared_ptr duplexTransportFactory_; + boost::shared_ptr duplexProtocolFactory_; + + boost::shared_ptr eventHandler_; + +public: + void setProcessorFactory( + boost::shared_ptr processorFactory) { + processorFactory_ = processorFactory; + } + + void setTransportFactory( + boost::shared_ptr transportFactory) { + duplexTransportFactory_.reset( + new TSingleTransportFactory(transportFactory)); + } + + void setDuplexTransportFactory( + boost::shared_ptr duplexTransportFactory) { + duplexTransportFactory_ = duplexTransportFactory; + } + + void setProtocolFactory(boost::shared_ptr protocolFactory) { + duplexProtocolFactory_.reset( + new TSingleProtocolFactory(protocolFactory)); + } + + void setDuplexProtocolFactory( + boost::shared_ptr duplexProtocolFactory) { + duplexProtocolFactory_ = duplexProtocolFactory; + } + + void setServerEventHandler( + boost::shared_ptr eventHandler) { + eventHandler_ = eventHandler; + } + +}; + +/** + * Helper function to increase the max file descriptors limit + * for the current process and all of its children. + * By default, tries to increase it to as much as 2^24. + */ + int increase_max_fds(int max_fds=(1<<24)); + + +}}} // apache::thrift::server + +#endif // #ifndef THRIFT_SERVER_TSERVER_H diff --git a/thrift/lib/cpp/server/TSimpleServer.h b/thrift/lib/cpp/server/TSimpleServer.h new file mode 100644 index 0000000000..6a35d0d03a --- /dev/null +++ b/thrift/lib/cpp/server/TSimpleServer.h @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef THRIFT_SERVER_TSIMPLESERVER_H +#define THRIFT_SERVER_TSIMPLESERVER_H 1 + +#include "thrift/lib/cpp/server/TServer.h" +#include "thrift/lib/cpp/transport/TServerTransport.h" + +namespace apache { namespace thrift { namespace server { + +/** + * This is the most basic simple server. It is single-threaded and runs a + * continuous loop of accepting a single connection, processing requests on + * that connection until it closes, and then repeating. It is a good example + * of how to extend the TServer interface. + * + */ +class TSimpleServer : public TServer { + public: + template + TSimpleServer( + const boost::shared_ptr& processorFactory, + const boost::shared_ptr& serverTransport, + const boost::shared_ptr& transportFactory, + const boost::shared_ptr& protocolFactory, + THRIFT_OVERLOAD_IF(ProcessorFactory, TProcessorFactory)) : + TServer(processorFactory, serverTransport, transportFactory, + protocolFactory), + stop_(false), + connectionCtx_(NULL) {} + + template + TSimpleServer( + const boost::shared_ptr& processor, + const boost::shared_ptr& serverTransport, + const boost::shared_ptr& transportFactory, + const boost::shared_ptr& protocolFactory, + THRIFT_OVERLOAD_IF(Processor, TProcessor)) : + TServer(processor, serverTransport, transportFactory, protocolFactory), + stop_(false), + connectionCtx_(NULL) {} + + template + TSimpleServer( + const boost::shared_ptr& processorFactory, + const boost::shared_ptr& serverTransport, + const boost::shared_ptr& duplexTransportFactory, + const boost::shared_ptr& duplexProtocolFactory, + THRIFT_OVERLOAD_IF(ProcessorFactory, TProcessorFactory)) : + TServer(processorFactory, serverTransport, + duplexTransportFactory, duplexProtocolFactory), + stop_(false), + connectionCtx_(NULL) {} + + template + TSimpleServer( + const boost::shared_ptr& processor, + const boost::shared_ptr& serverTransport, + const boost::shared_ptr& duplexTransportFactory, + const boost::shared_ptr& duplexProtocolFactory, + THRIFT_OVERLOAD_IF(Processor, TProcessor)) : + TServer(processor, serverTransport, + duplexTransportFactory, duplexProtocolFactory), + stop_(false), + connectionCtx_(NULL) {} + + ~TSimpleServer() {} + + void serve(); + + void stop(); + + TConnectionContext* getConnectionContext() const; + + protected: + bool stop_; + TConnectionContext* connectionCtx_; +}; + +}}} // apache::thrift::server + +#endif // #ifndef THRIFT_SERVER_TSIMPLESERVER_H diff --git a/thrift/lib/cpp/server/TThreadPoolServer.h b/thrift/lib/cpp/server/TThreadPoolServer.h new file mode 100644 index 0000000000..2ce0e2e399 --- /dev/null +++ b/thrift/lib/cpp/server/TThreadPoolServer.h @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef THRIFT_SERVER_TTHREADPOOLSERVER_H +#define THRIFT_SERVER_TTHREADPOOLSERVER_H 1 + +#include "thrift/lib/cpp/concurrency/ThreadManager.h" +#include "thrift/lib/cpp/server/TServer.h" +#include "thrift/lib/cpp/transport/TServerTransport.h" +#include "thrift/lib/cpp/concurrency/ThreadLocal.h" + +#include + +namespace apache { namespace thrift { namespace server { + +using apache::thrift::concurrency::ThreadManager; +using apache::thrift::protocol::TProtocolFactory; +using apache::thrift::transport::TServerTransport; +using apache::thrift::transport::TTransportFactory; + +class TThreadPoolServer : public TServer { + public: + template + TThreadPoolServer( + const boost::shared_ptr& processorFactory, + const boost::shared_ptr& serverTransport, + const boost::shared_ptr& transportFactory, + const boost::shared_ptr& protocolFactory, + const boost::shared_ptr& threadManager, + THRIFT_OVERLOAD_IF(ProcessorFactory, TProcessorFactory)) : + TServer(processorFactory, serverTransport, transportFactory, + protocolFactory), + threadManager_(threadManager), + stop_(false), + timeout_(0) {} + + template + TThreadPoolServer( + const boost::shared_ptr& processor, + const boost::shared_ptr& serverTransport, + const boost::shared_ptr& transportFactory, + const boost::shared_ptr& protocolFactory, + const boost::shared_ptr& threadManager, + THRIFT_OVERLOAD_IF(Processor, TProcessor)) : + TServer(processor, serverTransport, transportFactory, protocolFactory), + threadManager_(threadManager), + stop_(false), + timeout_(0) {} + + template + TThreadPoolServer( + const boost::shared_ptr& processorFactory, + const boost::shared_ptr& serverTransport, + const boost::shared_ptr& duplexTransportFactory, + const boost::shared_ptr& duplexProtocolFactory, + const boost::shared_ptr& threadManager, + THRIFT_OVERLOAD_IF(ProcessorFactory, TProcessorFactory)): + TServer(processorFactory, serverTransport, + duplexTransportFactory, duplexProtocolFactory), + threadManager_(threadManager), + stop_(false), + timeout_(0) {} + + template + TThreadPoolServer( + const boost::shared_ptr& processor, + const boost::shared_ptr& serverTransport, + const boost::shared_ptr& duplexTransportFactory, + const boost::shared_ptr& duplexProtocolFactory, + const boost::shared_ptr& threadManager, + THRIFT_OVERLOAD_IF(Processor, TProcessor)): + TServer(processor, serverTransport, + duplexTransportFactory, duplexProtocolFactory), + threadManager_(threadManager), + stop_(false), + timeout_(0) {} + + virtual ~TThreadPoolServer(); + + virtual void serve(); + + virtual int64_t getTimeout() const; + + virtual void setTimeout(int64_t value); + + virtual void stop() { + stop_ = true; + serverTransport_->interrupt(); + } + + virtual TConnectionContext* getConnectionContext() const; + + protected: + class Task; + typedef concurrency::ThreadLocal > + ThreadLocalTask; + + void setCurrentTask(Task* task); + void clearCurrentTask(); + + boost::shared_ptr threadManager_; + volatile bool stop_; + volatile int64_t timeout_; + + /** + * Thread-local data storage to track the current connection being processed. + */ + ThreadLocalTask currentTask_; +}; + +}}} // apache::thrift::server + +#endif // #ifndef THRIFT_SERVER_TTHREADPOOLSERVER_H diff --git a/thrift/lib/cpp/server/TThreadedServer.h b/thrift/lib/cpp/server/TThreadedServer.h new file mode 100644 index 0000000000..61c5034518 --- /dev/null +++ b/thrift/lib/cpp/server/TThreadedServer.h @@ -0,0 +1,223 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef THRIFT_SERVER_TTHREADEDSERVER_H +#define THRIFT_SERVER_TTHREADEDSERVER_H 1 + +#include "thrift/lib/cpp/server/TServer.h" +#include "thrift/lib/cpp/transport/TServerTransport.h" +#include "thrift/lib/cpp/concurrency/Monitor.h" +#include "thrift/lib/cpp/concurrency/Thread.h" +#include "thrift/lib/cpp/concurrency/ThreadLocal.h" + +#include + +namespace apache { namespace thrift { namespace server { + +using apache::thrift::TProcessor; +using apache::thrift::transport::TServerTransport; +using apache::thrift::transport::TTransportFactory; +using apache::thrift::concurrency::Monitor; +using apache::thrift::concurrency::ThreadFactory; + +class TThreadedServer : public TServer { + + public: + class Task; + + template + TThreadedServer(const boost::shared_ptr& processorFactory, + const boost::shared_ptr& serverTransport, + const boost::shared_ptr& transportFactory, + const boost::shared_ptr& protocolFactory, + THRIFT_OVERLOAD_IF(ProcessorFactory, TProcessorFactory)); + + template + TThreadedServer(const boost::shared_ptr& processorFactory, + const boost::shared_ptr& serverTransport, + const boost::shared_ptr& transportFactory, + const boost::shared_ptr& protocolFactory, + const boost::shared_ptr& threadFactory, + THRIFT_OVERLOAD_IF(ProcessorFactory, TProcessorFactory)); + + template + TThreadedServer( + const boost::shared_ptr& processorFactory, + const boost::shared_ptr& serverTransport, + const boost::shared_ptr& duplexTransportFactory, + const boost::shared_ptr& duplexProtocolFactory, + const boost::shared_ptr& threadFactory, + THRIFT_OVERLOAD_IF(ProcessorFactory, TProcessorFactory)); + + template + TThreadedServer(const boost::shared_ptr& processor, + const boost::shared_ptr& serverTransport, + const boost::shared_ptr& transportFactory, + const boost::shared_ptr& protocolFactory, + THRIFT_OVERLOAD_IF(Processor, TProcessor)); + + template + TThreadedServer(const boost::shared_ptr& processor, + const boost::shared_ptr& serverTransport, + const boost::shared_ptr& transportFactory, + const boost::shared_ptr& protocolFactory, + const boost::shared_ptr& threadFactory, + THRIFT_OVERLOAD_IF(Processor, TProcessor)); + + template + TThreadedServer( + const boost::shared_ptr& processor, + const boost::shared_ptr& serverTransport, + const boost::shared_ptr& duplexTransportFactory, + const boost::shared_ptr& duplexProtocolFactory, + THRIFT_OVERLOAD_IF(Processor, TProcessor)); + + template + TThreadedServer( + const boost::shared_ptr& processor, + const boost::shared_ptr& serverTransport, + const boost::shared_ptr& duplexTransportFactory, + const boost::shared_ptr& duplexProtocolFactory, + const boost::shared_ptr& threadFactory, + THRIFT_OVERLOAD_IF(Processor, TProcessor)); + + virtual ~TThreadedServer(); + + virtual void serve(); + + void stop() { + stop_ = true; + serverTransport_->interrupt(); + } + + virtual TConnectionContext* getConnectionContext() const; + + protected: + typedef concurrency::ThreadLocal > + ThreadLocalTask; + + void init(); + void setCurrentTask(Task* task); + + boost::shared_ptr threadFactory_; + volatile bool stop_; + + Monitor tasksMonitor_; + std::set tasks_; + + /** + * Thread-local data storage to track the current connection being processed. + */ + ThreadLocalTask currentTask_; +}; + +template +TThreadedServer::TThreadedServer( + const boost::shared_ptr& processorFactory, + const boost::shared_ptr& serverTransport, + const boost::shared_ptr& transportFactory, + const boost::shared_ptr& protocolFactory, + THRIFT_OVERLOAD_IF_DEFN(ProcessorFactory, TProcessorFactory)) : + TServer(processorFactory, serverTransport, transportFactory, + protocolFactory) { + init(); +} + +template +TThreadedServer::TThreadedServer( + const boost::shared_ptr& processorFactory, + const boost::shared_ptr& serverTransport, + const boost::shared_ptr& transportFactory, + const boost::shared_ptr& protocolFactory, + const boost::shared_ptr& threadFactory, + THRIFT_OVERLOAD_IF_DEFN(ProcessorFactory, TProcessorFactory)) : + TServer(processorFactory, serverTransport, transportFactory, + protocolFactory), + threadFactory_(threadFactory) { + init(); +} + +template +TThreadedServer::TThreadedServer( + const boost::shared_ptr& processorFactory, + const boost::shared_ptr& serverTransport, + const boost::shared_ptr& duplexTransportFactory, + const boost::shared_ptr& duplexProtocolFactory, + const boost::shared_ptr& threadFactory, + THRIFT_OVERLOAD_IF_DEFN(ProcessorFactory, TProcessorFactory)): + TServer(processorFactory, serverTransport, duplexTransportFactory, + duplexProtocolFactory), + threadFactory_(threadFactory) { + init(); +} + +template +TThreadedServer::TThreadedServer( + const boost::shared_ptr& processor, + const boost::shared_ptr& serverTransport, + const boost::shared_ptr& transportFactory, + const boost::shared_ptr& protocolFactory, + THRIFT_OVERLOAD_IF_DEFN(Processor, TProcessor)) : + TServer(processor, serverTransport, transportFactory, protocolFactory) { + init(); +} + +template +TThreadedServer::TThreadedServer( + const boost::shared_ptr& processor, + const boost::shared_ptr& serverTransport, + const boost::shared_ptr& transportFactory, + const boost::shared_ptr& protocolFactory, + const boost::shared_ptr& threadFactory, + THRIFT_OVERLOAD_IF_DEFN(Processor, TProcessor)) : + TServer(processor, serverTransport, transportFactory, protocolFactory), + threadFactory_(threadFactory) { + init(); +} + +template +TThreadedServer::TThreadedServer( + const boost::shared_ptr& processor, + const boost::shared_ptr& serverTransport, + const boost::shared_ptr& duplexTransportFactory, + const boost::shared_ptr& duplexProtocolFactory, + THRIFT_OVERLOAD_IF_DEFN(Processor, TProcessor)) : + TServer(processor, serverTransport, duplexTransportFactory, + duplexProtocolFactory) { + init(); +} + +template +TThreadedServer::TThreadedServer( + const boost::shared_ptr& processor, + const boost::shared_ptr& serverTransport, + const boost::shared_ptr& duplexTransportFactory, + const boost::shared_ptr& duplexProtocolFactory, + const boost::shared_ptr& threadFactory, + THRIFT_OVERLOAD_IF_DEFN(Processor, TProcessor)): + TServer(processor, serverTransport, duplexTransportFactory, + duplexProtocolFactory), + threadFactory_(threadFactory) { + init(); +} + +}}} // apache::thrift::server + +#endif // #ifndef _THRIFT_SERVER_TTHREADEDSERVER_H_ diff --git a/thrift/lib/cpp/server/test/AggregatorTest.thrift b/thrift/lib/cpp/server/test/AggregatorTest.thrift new file mode 100644 index 0000000000..3f94ba3667 --- /dev/null +++ b/thrift/lib/cpp/server/test/AggregatorTest.thrift @@ -0,0 +1,41 @@ +#!/usr/local/bin/thrift -cpp + +include "common/fb303/if/fb303.thrift" + +namespace cpp apache.thrift.async + +struct StructRequest { + 1:i32 i32Val, + 2:i64 i64Val, + 3:double doubleVal, + 4:string stringVal, +} + +struct StructResponse { + 1:StructRequest request, + 2:i32 errorCode + 3:string answerString, +} + +service AggregatorTest extends fb303.FacebookService { + StructResponse sendStructRecvStruct(1:StructRequest request), + oneway void sendStructNoRecv(1:StructRequest request), + StructResponse sendMultiParamsRecvStruct( + 1:i32 i32Val, + 2:i64 i64Val, + 3:double doubleVal, + 4:string stringVal, + 5:StructRequest structVal, + ), + oneway void sendMultiParamsNoRecv( + 1:i32 i32Val, + 2:i64 i64Val, + 3:double doubleVal, + 4:string stringVal, + 5:StructRequest structVal, + ), + StructResponse noSendRecvStruct(), + oneway void noSendNoRecv(), +} + + diff --git a/thrift/lib/cpp/server/test/AggregatorUtilTest.h b/thrift/lib/cpp/server/test/AggregatorUtilTest.h new file mode 100644 index 0000000000..6020ad80ea --- /dev/null +++ b/thrift/lib/cpp/server/test/AggregatorUtilTest.h @@ -0,0 +1,143 @@ +#ifndef COMMON_CLIENT_MGMT_AGGR_UTILTEST +#define COMMON_CLIENT_MGMT_AGGR_UTILTEST 1 + +#include +#include +#include +#include +#include + +#include +#include + +#include + +#include "thrift/lib/cpp/server/test/gen-cpp/AggregatorTest_types.h" +#include "thrift/lib/cpp/server/test/gen-cpp/AggregatorTest.h" + +#include "common/logging/logging.h" + +static inline void defaultData(apache::thrift::async::StructRequest* res) { + res->i32Val = 32; + res->i64Val = 64; + res->doubleVal = -12.34; + res->stringVal = "string"; +} + +static inline void randomData(apache::thrift::async::StructRequest* res) { + res->i32Val = ::random(); + res->i64Val = ::random(); + res->doubleVal = ::random() * ((::random() % 2) == 0 ? -1 : 1); + res->stringVal = boost::lexical_cast(::random()); +} + +static inline void toAnswerString(std::string* dest, + const int32_t i32Val, + const int64_t i64Val, + const double doubleVal, + const std::string& stringVal) { + *dest += + boost::lexical_cast(i32Val) + + ";" + + boost::lexical_cast(i64Val) + + ";" + + boost::lexical_cast(doubleVal) + + ";" + + boost::lexical_cast(stringVal); +} + +static inline void toAnswerString(std::string* dest, + const apache::thrift::async::StructRequest& request) { + toAnswerString(dest, request.i32Val, + request.i64Val, + request.doubleVal, + request.stringVal); +} + +static inline void toAnswerString(std::string* dest, + const int32_t i32Val, + const int64_t i64Val, + const double doubleVal, + const std::string& stringVal, + const apache::thrift::async::StructRequest& structVal) { + toAnswerString(dest, i32Val, + i64Val, + doubleVal, + stringVal); + *dest += "*"; + toAnswerString(dest, structVal); +} + +static inline void zeroResponse(apache::thrift::async::StructResponse* dest) { + dest->request.i32Val = 0; + dest->request.i64Val = 0; + dest->request.doubleVal = 0.0; + dest->request.stringVal = ""; + dest->errorCode = 0; + dest->answerString = ""; +} + +static inline void printResponse( + const apache::thrift::async::StructResponse& x) { + LOG(INFO) << "request.i32Val: " << x.request.i32Val + << ", request.i64Val: " << x.request.i64Val + << ", request.doubleVal: " << x.request.doubleVal + << ", request.stringVal: " << x.request.stringVal + << ", request.stringVal: " << x.request.stringVal + << ", errorCode: " << x.errorCode + << ", answerString: " << x.answerString; +} + +static inline bool equalResult( + const apache::thrift::async::StructResponse& ethalon, + const apache::thrift::async::StructResponse& response) { + bool res = ethalon.request.i32Val == response.request.i32Val + && ethalon.request.i64Val == response.request.i64Val + && fabs(ethalon.request.doubleVal - response.request.doubleVal) < 0.0001 + && ethalon.request.stringVal == response.request.stringVal + && ethalon.errorCode == response.errorCode + && ethalon.answerString == response.answerString; + if (!res) { + printResponse(ethalon); + printResponse(response); + } + return res; +} + +static inline void addResponse(apache::thrift::async::StructResponse* dest, + const apache::thrift::async::StructResponse& src) { + dest->request.i32Val += src.request.i32Val; + dest->request.i64Val += src.request.i64Val; + dest->request.doubleVal += src.request.doubleVal; + dest->request.stringVal += src.request.stringVal; + dest->errorCode += src.errorCode; + dest->answerString += "/"; + dest->answerString += src.answerString; +} + +static inline void addRequest(apache::thrift::async::StructResponse* dest, + const apache::thrift::async::StructRequest& src) { + apache::thrift::async::StructResponse response; + response.request = src; + response.errorCode = 0; + response.answerString = ""; + toAnswerString(&response.answerString, response.request); + addResponse(dest, response); +} + +static inline void addRequest(apache::thrift::async::StructResponse* dest, + const int32_t i32Val, + const int64_t i64Val, + const double doubleVal, + const std::string& stringVal, + const apache::thrift::async::StructRequest& structVal) { + apache::thrift::async::StructResponse response; + response.request = structVal; + response.errorCode = 0; + response.answerString = ""; + toAnswerString(&response.answerString, + i32Val, i64Val, doubleVal, stringVal, response.request); + addResponse(dest, response); +} + +#endif diff --git a/thrift/lib/cpp/server/test/TARGETS b/thrift/lib/cpp/server/test/TARGETS new file mode 100644 index 0000000000..0cb7f9a60a --- /dev/null +++ b/thrift/lib/cpp/server/test/TARGETS @@ -0,0 +1,60 @@ +cpp_binary ( + name = "aggregatorserver_test", + srcs = [ + "AggregatorServerTest.cpp", + ], + deps = [ + "@/thrift/lib/cpp/async", + "@/thrift/lib/cpp/concurrency", + "@/thrift/lib/cpp/protocol", + "@/thrift/lib/cpp/server:nonblocking", + "@/common/fb303/cpp:fb303", + "@/common/base:profiler", + ":aggregator_test_thrift", + ], + allocator='tcmalloc', +) + +cpp_binary ( + name = "aggregatorclient_test", + srcs = [ + "AggregatorClientTest.cpp", + ], + deps = [ + "@/thrift/lib/cpp/async", + "@/thrift/lib/cpp/concurrency", + "@/thrift/lib/cpp/protocol", + "@/common/fb303/cpp:fb303", + "@/common/base:profiler", + "@/common/client_mgmt:asyncclient", + "@/common/logging:logging", + ":aggregator_test_thrift", + ], + allocator='tcmalloc', +) + +cpp_library ( + name = "aggregator_test_thrift", + thrift_srcs = { "AggregatorTest.thrift" : [ "AggregatorTest" ] }, + thrift_cpp_options = "cob_style", + deps = [ + "@/common/fb303/if:fb303-cpp", + ], +) + +cpp_unittest( + name = "conn_ctx_test", + srcs = [ + "ConnCtxTest.cpp", + ], + thrift_srcs = { "connctx.thrift" : [ "ConnCtxService" ] }, + deps = [ + "@/thrift/lib/cpp/test:util", + "@/thrift/lib/cpp/util:nonblocking", + ], + external_deps = [ + ('boost', '>= 1.37', 'boost_unit_test_framework'), + ], + type = 'boost', + emails = ['thrift-team@lists.facebook.com'], +) diff --git a/thrift/lib/cpp/server/test/connctx.thrift b/thrift/lib/cpp/server/test/connctx.thrift new file mode 100644 index 0000000000..3f65fb36fa --- /dev/null +++ b/thrift/lib/cpp/server/test/connctx.thrift @@ -0,0 +1,7 @@ +exception CtxError { + 1: string message +} (message = "message") + +service ConnCtxService { + binary getClientAddress() throws (1: CtxError error) +} diff --git a/thrift/lib/cpp/test/DebugProtocolTest.thrift b/thrift/lib/cpp/test/DebugProtocolTest.thrift new file mode 100644 index 0000000000..7a7278ecc8 --- /dev/null +++ b/thrift/lib/cpp/test/DebugProtocolTest.thrift @@ -0,0 +1,27 @@ +struct Message { + 1: bool a, + 2: byte b, + 3: i16 c, + 4: i32 d, + 5: i64 e, + 6: double f, + 7: string g +} + +typedef list MsgList +typedef list IntList +typedef list StringList + + +typedef map MsgMap +typedef set MsgSet + +struct Ooo { + 1: MsgList l, + 2: MsgMap m, + 3: MsgSet s +} + +service DebugProtocolService { + void Func(1: MsgList lst); +} diff --git a/thrift/lib/cpp/test/NetworkUtil.h b/thrift/lib/cpp/test/NetworkUtil.h new file mode 100644 index 0000000000..853d413c76 --- /dev/null +++ b/thrift/lib/cpp/test/NetworkUtil.h @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_TEST_NETWORKUTIL_H_ +#define THRIFT_TEST_NETWORKUTIL_H_ 1 + +#include + +namespace apache { namespace thrift { + +namespace transport { +class TSocketAddress; +} + +namespace test { + +/** + * Get a list of all configured local IP addresses. + */ +void getLocalAddresses(std::vector* results); + +}}} // apache::thrift::test + +#endif // THRIFT_TEST_NETWORKUTIL_H_ diff --git a/thrift/lib/cpp/test/STAR.newdev.facebook.com.crt b/thrift/lib/cpp/test/STAR.newdev.facebook.com.crt new file mode 100644 index 0000000000..3b91dcfae1 --- /dev/null +++ b/thrift/lib/cpp/test/STAR.newdev.facebook.com.crt @@ -0,0 +1,34 @@ +-----BEGIN CERTIFICATE----- +MIIF9DCCBNygAwIBAgIKEf4IyQAAAAAt5DANBgkqhkiG9w0BAQUFADBKMRMwEQYK +CZImiZPyLGQBGRYDY29tMRswGQYKCZImiZPyLGQBGRYLVGhlRmFjZWJvb2sxFjAU +BgNVBAMTDVRoZUZhY2Vib29rQ0EwHhcNMTEwNDI4MTkxMjU0WhcNMTMwNDI3MTkx +MjU0WjAgMR4wHAYDVQQDDBUqLm5ld2Rldi5mYWNlYm9vay5jb20wggEiMA0GCSqG +SIb3DQEBAQUAA4IBDwAwggEKAoIBAQC42UW2BSdL/OkaP7OGsPGxmZjKEvkYLnSC +WEJLrs9axKTe35jLD6SZjZtUkQn8DelgeDgKrHo5mbPz7yO4TKDBZaVj0nhJRnto +sCrRXyNinslmgThaIGkQOglNO9zly7fKYl0DQDJ+YG4zX6oV/43ZosjXjeLJNHiD +A14rXfUoooUoFFULb4QeKehI9FZo9KyMMtUofAl4Ms2Yu+eO3oOfq+fK0H3qLRVX +KP0+fQXknx6O/a584CkedLpuc+i2a7Lp6iuT7ZHyJgTpE1Lw2JawlE7MACcAHsve +ODpC5AL53luvu1uvVQqUOeQ6Thz4n0BtPKn+KIzEleDCdeCy5QLXAgMBAAGjggME +MIIDADAdBgNVHQ4EFgQUn79HO8QnhoLhTFmXyIMYpNQLgMowHwYDVR0jBBgwFoAU +/c5mcfzW2IaPBA5cUuOYapXlvScwggEPBgNVHR8EggEGMIIBAjCB/6CB/KCB+YaB +u2xkYXA6Ly8vQ049VGhlRmFjZWJvb2tDQSxDTj1TQy1DQVNVQjAxLENOPUNEUCxD +Tj1QdWJsaWMlMjBLZXklMjBTZXJ2aWNlcyxDTj1TZXJ2aWNlcyxDTj1Db25maWd1 +cmF0aW9uLERDPVRoZUZhY2Vib29rLERDPWNvbT9jZXJ0aWZpY2F0ZVJldm9jYXRp +b25MaXN0P2Jhc2U/b2JqZWN0Q2xhc3M9Y1JMRGlzdHJpYnV0aW9uUG9pbnSGOWh0 +dHA6Ly9jZXJ0cy50aGVmYWNlYm9vay5jb20vQ2VydEVucm9sbC9UaGVGYWNlYm9v +a0NBLmNybDCCAScGCCsGAQUFBwEBBIIBGTCCARUwgbAGCCsGAQUFBzAChoGjbGRh +cDovLy9DTj1UaGVGYWNlYm9va0NBLENOPUFJQSxDTj1QdWJsaWMlMjBLZXklMjBT +ZXJ2aWNlcyxDTj1TZXJ2aWNlcyxDTj1Db25maWd1cmF0aW9uLERDPVRoZUZhY2Vi +b29rLERDPWNvbT9jQUNlcnRpZmljYXRlP2Jhc2U/b2JqZWN0Q2xhc3M9Y2VydGlm +aWNhdGlvbkF1dGhvcml0eTBgBggrBgEFBQcwAoZUaHR0cDovL2NlcnRzLnRoZWZh +Y2Vib29rLmNvbS9DZXJ0RW5yb2xsL1NDLUNBU1VCMDEuVGhlRmFjZWJvb2suY29t +X1RoZUZhY2Vib29rQ0EuY3J0MA4GA1UdDwEB/wQEAwIFoDA+BgkrBgEEAYI3FQcE +MTAvBicrBgEEAYI3FQiCuMAag87Wd4fdgReE4Kg9h9aFb4E4hrepboP/41MCAWQC +AQIwEwYDVR0lBAwwCgYIKwYBBQUHAwEwGwYJKwYBBAGCNxUKBA4wDDAKBggrBgEF +BQcDATANBgkqhkiG9w0BAQUFAAOCAQEAgVv0ngnjVzRNv9tu1pk/a9nW6+tKDMCO +IgqUHFO5MmnlOSGK+PLBV8gflBnzjDl7y+LCtMXPSpdTFA7hBrxsVGsLwMjluttO +Az50eskueoO2jxUe21uQw+7Fiy5EhtVdsDPQ/CXZXA2MNGV0cFVxLIY+IpzddP25 +fz1j/r3YtomHaEtGaJXDkgb7YdcLlJ5XZKM59Idp4KfLRNUTjaZXIUpTEBRcIdYV +WXcbFRNBM9oBRej9YEeXdlx51w38eX1BczQ40pwTsi7ze+1ZNESaK9oqvZuNsisg +CYJunbzTNfHWB6M/uvLiwoWjGihr8cf3/ULtxcq9+IFmsphavEzn7A== +-----END CERTIFICATE----- diff --git a/thrift/lib/cpp/test/STAR.newdev.facebook.com.key b/thrift/lib/cpp/test/STAR.newdev.facebook.com.key new file mode 100644 index 0000000000..eab6acf2f2 --- /dev/null +++ b/thrift/lib/cpp/test/STAR.newdev.facebook.com.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAuNlFtgUnS/zpGj+zhrDxsZmYyhL5GC50glhCS67PWsSk3t+Y +yw+kmY2bVJEJ/A3pYHg4Cqx6OZmz8+8juEygwWWlY9J4SUZ7aLAq0V8jYp7JZoE4 +WiBpEDoJTTvc5cu3ymJdA0AyfmBuM1+qFf+N2aLI143iyTR4gwNeK131KKKFKBRV +C2+EHinoSPRWaPSsjDLVKHwJeDLNmLvnjt6Dn6vnytB96i0VVyj9Pn0F5J8ejv2u +fOApHnS6bnPotmuy6eork+2R8iYE6RNS8NiWsJROzAAnAB7L3jg6QuQC+d5br7tb +r1UKlDnkOk4c+J9AbTyp/iiMxJXgwnXgsuUC1wIDAQABAoIBAQCAFHxLDPMXodxa +hbz9HOm7Z3QIwHC8oqO0FTeKfUvL1uluTHvwZ5xjkU0JoZ3LelA0udgK/UuLUcbw +xMh5UQ8qMiYRLqqOCzpDZZ2liUhZczCVx/+l40MpLS14yHvc9fUZVGKid/8Etghl +rGmL6JD/Nd34DaYQSD0gelrrraTLS/K/f3II1AkJUYUNGoe+FHlBOIMw0rrl2yLX +nvCK7DCCWXkdy5PAolm1sEek3ZWRLvykYQm+Pe7tDDAXb7JEhnp1MCwz8QPv3A+8 +Wotk8LdIsRbqdHjychBaHNu4XfyIwfK2W3+yTBudbs3eHzMOvbY7lznb1tcpAiFQ +ot+YwMJhAoGBAOm0dSj2DbAvf53F49EoqNqRlrXuH5PHRFsPeGPHRJXt+AVYzwMe +lMlDXhU3+hgZmE1aKci9sr2i0z71GpLxTAcIzITC0iyWok2h7NC6HtFW0xM3U/oc +1THDWwbxULTzGEzEuYq5AR5xu54aEfjEpCBVBWP52lyNs3hPTHlcOoLpAoGBAMp7 +pgTPnJr/Nr4Fdny/D3Iy+aKKe2wu1vgEaTqYGGpKq976C+8nEZ1hqHzFTQgdsTIK +QUPGAw2LgtSPcB+jIvJoG8LHPum7n3/7lV3JeEXbDhMMrSX7emJ2KZ+qO5dSRMgT +PDtjeK33egEvXw24j1d4ZSef4smw87xmSNk1Fj+/AoGAOxIezvSqbw8CbamYNMTX +MS+9bUy79IlYcL/PSUETx+W3W/qfkIVJTDnX7PsSHsdRZLPWfePc6uveAzrGtb+B +3QFPWHs5zoVgjsjP584zfew5O9EvS7e/920OdAnIwYP0b2sGDtdGPiC3MHYcdsmV +RorW6vc7R0uVwmIgzUgPjgkCgYBImXD7RJ04Dc3I8Bh9vduB7LkJCHIcZeb54ksA +kUwfwEpDlo/03aM5NKy+n9dw04SXFoo0DKlHkAAGyq8/CLUdCSr6+YmbMISN1NzH +Vddj5Jj1R5RFjBdjek6OSvE23m5+9jagbJf2eEcsaZnBR61NhYNvY6pi+qlJobr4 +9knV7QKBgQDRC1lTIcONxZIWRQMhgIdiVjRL4jfITRmrgAUHnOTDqrDr6pzDmHGO +Js8jRQxq3h8mWHnF2P5HMI4QVmpQ8RuH9bJhycIKbPN0r/QsynsHJeVN/pvWe1ke +Z8U59I9XJnB0KEXRkGEZyftw5GDFX1EqEJBbR3iHojkWZB/SZ/65YQ== +-----END RSA PRIVATE KEY----- diff --git a/thrift/lib/cpp/test/ScopedEventBaseThread.h b/thrift/lib/cpp/test/ScopedEventBaseThread.h new file mode 100644 index 0000000000..8736544da3 --- /dev/null +++ b/thrift/lib/cpp/test/ScopedEventBaseThread.h @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_TEST_SCOPEDEVENTBASETHREAD_H_ +#define THRIFT_TEST_SCOPEDEVENTBASETHREAD_H_ 1 + +#include +#include + +namespace apache { namespace thrift { namespace async { +class TEventBase; +}}} +namespace apache { namespace thrift { namespace concurrency { +class Thread; +}}} + +namespace apache { namespace thrift { namespace test { + +/** + * A helper class to start a new thread running a TEventBase loop. + * + * The new thread will be started by the ScopedEventBaseThread constructor. + * When the ScopedEventBaseThread object is destroyed, the thread will be + * stopped. + */ +class ScopedEventBaseThread { + public: + ScopedEventBaseThread(); + ~ScopedEventBaseThread(); + + ScopedEventBaseThread(ScopedEventBaseThread&& other); + ScopedEventBaseThread &operator=(ScopedEventBaseThread&& other); + + /** + * Get a pointer to the TEventBase driving this thread. + */ + async::TEventBase *getEventBase() const { + return eventBase_.get(); + } + + private: + ScopedEventBaseThread(const ScopedEventBaseThread& other) = delete; + ScopedEventBaseThread& operator=(const ScopedEventBaseThread& other) = delete; + + std::unique_ptr eventBase_; + boost::shared_ptr thread_; +}; + +}}} // apache::thrift::test + +#endif // THRIFT_TEST_SCOPEDEVENTBASETHREAD_H_ diff --git a/thrift/lib/cpp/test/SocketPair.h b/thrift/lib/cpp/test/SocketPair.h new file mode 100644 index 0000000000..eebe534e40 --- /dev/null +++ b/thrift/lib/cpp/test/SocketPair.h @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_TEST_SOCKETPAIR_H_ +#define THRIFT_TEST_SOCKETPAIR_H_ 1 + +namespace apache { namespace thrift { namespace test { + +class SocketPair { + public: + enum Mode { + BLOCKING, + NONBLOCKING + }; + + SocketPair(Mode mode = NONBLOCKING); + ~SocketPair(); + + int operator[](int index) const { + return fds_[index]; + } + + void closeFD0(); + void closeFD1(); + + int extractFD0() { + return extractFD(0); + } + int extractFD1() { + return extractFD(1); + } + int extractFD(int index) { + int fd = fds_[index]; + fds_[index] = -1; + return fd; + } + + private: + int fds_[2]; +}; + +}}} // apache::thrift::test + +#endif // THRIFT_TEST_SOCKETPAIR_H_ diff --git a/thrift/lib/cpp/test/TARGETS b/thrift/lib/cpp/test/TARGETS new file mode 100644 index 0000000000..6c8cb67874 --- /dev/null +++ b/thrift/lib/cpp/test/TARGETS @@ -0,0 +1,167 @@ +cpp_unittest( + name = 'TEventBaseTest', + srcs = [ + 'TEventBaseTest.cpp', + ], + deps = [ + ':util', + '@/thrift/lib/cpp/async', + ], + external_deps = [ + ('boost', '>= 1.37', 'boost_unit_test_framework'), + ], + type = 'boost', + emails = ['thrift-team@lists.facebook.com'], +) + +cpp_unittest( + name = 'TAsyncChannelTest', + srcs = [ + 'TAsyncChannelTest.cpp', + ], + deps = [ + ':util', + '@/thrift/lib/cpp/async', + ], + external_deps = [ + ('boost', '>= 1.37', 'boost_unit_test_framework'), + ], + type = 'boost', + emails = ['thrift-team@lists.facebook.com'], +) + +cpp_unittest( + name = 'TAsyncChannelClientTest', + srcs = [ + 'TAsyncChannelClientTest.cpp', + ], + deps = [ + ':util', + '@/thrift/lib/cpp/async:zlib', + '@/thrift/lib/cpp/util:nonblocking', + '@/thrift/lib/cpp/util:httpparser', + '@/thrift/perf/cpp:load-handler', + ], + external_deps = [ + ('boost', '>= 1.37', 'boost_unit_test_framework'), + ], + type = 'boost', + emails = ['thrift-team@lists.facebook.com'], +) + +cpp_unittest( + name = 'TAsyncSocketTest', + srcs = [ + 'TAsyncSocketTest.cpp', + ], + deps = [ + ':util', + '@/thrift/lib/cpp/async', + ], + external_deps = [ + ('boost', '>= 1.37', 'boost_unit_test_framework'), + ], + type = 'boost', + emails = ['thrift-team@lists.facebook.com'], +) + +cpp_unittest( + name = 'TAsyncSSLSocketTest', + srcs = [ + 'TAsyncSSLSocketTest.cpp', + ], + deps = [ + ':util', + '@/thrift/lib/cpp/async:async_ssl', + ], + external_deps = [ + ('boost', '>= 1.37', 'boost_unit_test_framework'), + ], + type = 'boost', + emails = ['ti-bugs@lists.facebook.com', + 'thrift-team@lists.facebook.com'], +) + +cpp_unittest( + name = 'TAsyncTimeoutSetTest', + srcs = [ + 'TAsyncTimeoutSetTest.cpp', + ], + deps = [ + ':util', + '@/thrift/lib/cpp/async:async_cxx11', + ], + type = 'boost', + emails = ['thrift-team@lists.facebook.com'], +) + +cpp_unittest( + name = 'TNotificationQueueTest', + srcs = [ + 'TNotificationQueueTest.cpp', + ], + deps = [ + ':async_util', + '@/thrift/lib/cpp/async:async_cxx11', + ], + type = 'boost', + emails = ['thrift-team@lists.facebook.com'], +) + +cpp_unittest( + name = 'TNotificationPipeTest', + srcs = [ + 'TNotificationPipeTest.cpp', + ], + deps = [ + ':async_util', + ':util', + '@/thrift/lib/cpp/async', + ], + external_deps = [ + ('boost', '>= 1.37', 'boost_unit_test_framework'), + ], + type = 'boost', + emails = ['thrift-team@lists.facebook.com'], +) + +cpp_library( + name = 'util', + srcs = [ + 'NetworkUtil.cpp', + 'SocketPair.cpp', + 'TimeUtil.cpp', + ], + deps = [ + '@/thrift/lib/cpp/transport', + ], +) + +cpp_library( + name = 'async_util', + srcs = [ + 'ScopedEventBaseThread.cpp', + ], + deps = [ + '@/thrift/lib/cpp/async', + '@/thrift/lib/cpp/concurrency', + ], +) + +cpp_library ( + name = "DebugProtocolLib", + thrift_srcs = { + "DebugProtocolTest.thrift" : "DebugProtocolService" + }, + thrift_cpp_options = "templates" +) + +cpp_binary ( + name = "DebugProtocol", + srcs = [ + "DebugProtocolTest.cpp", + ], + deps = [ + ":DebugProtocolLib", + ], +) diff --git a/thrift/lib/cpp/test/TimeUtil.h b/thrift/lib/cpp/test/TimeUtil.h new file mode 100644 index 0000000000..d4f0fcd72d --- /dev/null +++ b/thrift/lib/cpp/test/TimeUtil.h @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_TEST_TIMEUTIL_H_ +#define THRIFT_TEST_TIMEUTIL_H_ 1 + +#include +#include + +namespace apache { namespace thrift { namespace test { + +class TimePoint { + public: + explicit TimePoint(bool set = true) + : timeStart_(0), + timeEnd_(0), + timeWaiting_(0), + tid_(0) { + if (set) { + reset(); + } + } + + void reset(); + + bool isUnset() const { + return (timeStart_ == 0 && timeEnd_ == 0 && timeWaiting_ == 0); + } + + int64_t getTime() const { + return timeStart_; + } + + int64_t getTimeStart() const { + return timeStart_; + } + + int64_t getTimeEnd() const { + return timeStart_; + } + + int64_t getTimeWaiting() const { + return timeWaiting_; + } + + pid_t getTid() const { + return tid_; + } + + private: + int64_t timeStart_; + int64_t timeEnd_; + int64_t timeWaiting_; + pid_t tid_; +}; + +std::ostream& operator<<(std::ostream& os, const TimePoint& timePoint); + +boost::test_tools::predicate_result checkTimeout(const TimePoint& start, + const TimePoint& end, + int64_t expectedMS, + bool allowSmaller, + int64_t tolerance = 5); + +/** + * Check how long a timeout took to fire. + * + * This method verifies: + * - that the timeout did not fire too early (never less than expectedMS) + * - that the timeout fired within a reasonable period of the expected + * duration. It must fire within the specified tolerance, excluding time + * that this process spent waiting to be scheduled. + * + * @param start A TimePoint object set just before the timeout + * was scheduled. + * @param end A TimePoint object set when the timeout fired. + * @param expectedMS The timeout duration, in milliseconds + * @param tolerance The tolerance, in milliseconds. + */ +#define T_CHECK_TIMEOUT(start, end, expectedMS, ...) \ + BOOST_CHECK(apache::thrift::test::checkTimeout((start), (end), \ + (expectedMS), false, \ + ##__VA_ARGS__)) + +/** + * Verify that an event took less than a specified amount of time. + * + * This is similar to T_CHECK_TIMEOUT, but does not fail if the event took less + * than the allowed time. + */ +#define T_CHECK_TIME_LT(start, end, expectedMS, ...) \ + BOOST_CHECK(apache::thrift::test::checkTimeout((start), (end), \ + (expectedMS), true, \ + ##__VA_ARGS__)) + +}}} // apache::thrift::test + +#endif // THRIFT_TEST_TIMEUTIL_H_ diff --git a/thrift/lib/cpp/test/checks.h b/thrift/lib/cpp/test/checks.h new file mode 100644 index 0000000000..4ee07f5d0c --- /dev/null +++ b/thrift/lib/cpp/test/checks.h @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef THRIFT_TEST_CHECKS_H_ +#define THRIFT_TEST_CHECKS_H_ 1 + +#include +#include "common/logging/logging.h" +#include "thrift/lib/cpp/protocol/TDebugProtocol.h" + + +namespace apache { namespace thrift { namespace test { + +template +inline std::string* formatString(const T1& v1, const T2& v2, + const std::string& op) { + std::stringstream ss; + ss << ThriftDebugString(v1) << " "<< op <<" " << ThriftDebugString(v2); + return new std::string(ss.str()); +} + +template +inline std::string* checkThriftEqImpl(const T1& val1, const T2& val2) +{ + if (val1 == val2) { + return NULL; + } + return formatString(val1, val2, "=="); +} + +template +inline std::string* checkThriftNeImpl(const T1& val1, const T2& val2) +{ + if (val1 != val2) { + return NULL; + } + return formatString(val1, val2, "!="); +} + +template +inline std::string* checkThriftLeImpl(const T1& val1, const T2& val2) +{ + if (val1 <= val2) { + return NULL; + } + return formatString(val1, val2, "<="); +} + +template +inline std::string* checkThriftLtImpl(const T1& val1, const T2& val2) { + + if (val1 < val2) { + return NULL; + } + return formatString(val1, val2, "<"); +} + +template +inline std::string* checkThriftGeImpl(const T1& val1, const T2& val2) +{ + if (val1 >= val2) { + return NULL; + } + return formatString(val1, val2, ">="); +} + +template +inline std::string* checkThriftGtImpl(const T1& val1, const T2& val2) +{ + if (val1 > val2) { + return NULL; + } + return formatString(val1, val2, ">"); +} + +}}} + +#define THRIFT_CHECK_OP(name, val1, val2) \ + while (std::string* _checkResult = \ + apache::thrift::test::checkThrift##name##Impl((val1), (val2))) \ + google::LogMessageFatal(__FILE__, __LINE__, \ + google::CheckOpString(_checkResult)).stream() + +#define THRIFT_CHECK_EQ(val1, val2) THRIFT_CHECK_OP(Eq, val1, val2) +#define THRIFT_CHECK_NE(val1, val2) THRIFT_CHECK_OP(Ne, val1, val2) +#define THRIFT_CHECK_LE(val1, val2) THRIFT_CHECK_OP(Le, val1, val2) +#define THRIFT_CHECK_LT(val1, val2) THRIFT_CHECK_OP(Lt, val1, val2) +#define THRIFT_CHECK_GE(val1, val2) THRIFT_CHECK_OP(Ge, val1, val2) +#define THRIFT_CHECK_GT(val1, val2) THRIFT_CHECK_OP(Gt, val1, val2) + +#endif diff --git a/thrift/lib/cpp/test/loadgen/Controller.h b/thrift/lib/cpp/test/loadgen/Controller.h new file mode 100644 index 0000000000..84805664d6 --- /dev/null +++ b/thrift/lib/cpp/test/loadgen/Controller.h @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_TEST_LOADGEN_CONTROLLER_H_ +#define THRIFT_TEST_LOADGEN_CONTROLLER_H_ 1 + +#include + +#include "thrift/lib/cpp/concurrency/Monitor.h" +#include "thrift/lib/cpp/test/loadgen/LoadConfig.h" +#include "thrift/lib/cpp/test/loadgen/IntervalTimer.h" + +namespace apache { namespace thrift { + +namespace concurrency { + +class PosixThreadFactory; + +} // apache::thrift::concurrency + +namespace loadgen { + +class WorkerFactory; +class WorkerIf; +class Monitor; + +class Controller : private boost::noncopyable { + public: + Controller(WorkerFactory* factory, Monitor* monitor, + boost::shared_ptr config, + apache::thrift::concurrency::PosixThreadFactory* threadFactory = NULL); + + void run(uint32_t numThreads, double monitorInterval = 1.0); + + private: + class WorkerRunner; + typedef std::vector< boost::shared_ptr > WorkerVector; + + void startWorkers(uint32_t numThreads); + void runMonitor(double interval); + boost::shared_ptr createWorker(); + + concurrency::Monitor initMonitor_; + uint32_t numThreads_; + WorkerFactory* workerFactory_; + Monitor* monitor_; + WorkerVector workers_; + IntervalTimer intervalTimer_; + boost::shared_ptr config_; + apache::thrift::concurrency::PosixThreadFactory* threadFactory_; +}; + +}}} // apache::thrift::loadgen + +#endif // THRIFT_TEST_LOADGEN_CONTROLLER_H_ diff --git a/thrift/lib/cpp/test/loadgen/IntervalTimer.h b/thrift/lib/cpp/test/loadgen/IntervalTimer.h new file mode 100644 index 0000000000..92924a07dd --- /dev/null +++ b/thrift/lib/cpp/test/loadgen/IntervalTimer.h @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_TEST_LOADGEN_INTERVALTIMER_H_ +#define THRIFT_TEST_LOADGEN_INTERVALTIMER_H_ 1 + +#include "thrift/lib/cpp/concurrency/Util.h" +#include "thrift/lib/cpp/concurrency/Mutex.h" +#include "thrift/lib/cpp/TLogging.h" + +#include + +namespace apache { namespace thrift { namespace loadgen { + +/** + * IntervalTimer helps perform tasks at a desired rate. + * + * Call sleep() in between each operation, and it will sleep the required + * amount of time to hit the target rate. It accounts for the time required to + * perform each operation, and it also adjusts the subsequent intervals if the + * system sleep call wakes up later than requested. This allows good accuracy + * for the average rate, even when the requested interval is very small. Works + * between multiple threads. + */ +class IntervalTimer { + public: + /** + * Create a new IntervalTimer + * + * @param intervalNsec The desired number of ns each interval should take. + * @param maxBacklog If we can't keep up with the requested rate, reset + * when we fall more than maxBacklog microseconds + * behind. If the rate does eventually recover, this + * will setting helps reduce the amount of time that the + * timer goes too fast in order to catch up to the + * average rate. + */ + IntervalTimer(uint64_t intervalNsec, + uint64_t maxBacklog = 3 * concurrency::Util::US_PER_S) + : numTimes_(0) + , intervalNsec_(intervalNsec) + , intervalStart_(0) + , maxBacklog_(maxBacklog) { } + + void setIntervalNsec(uint64_t interval) { + intervalNsec_ = interval; + } + + void setRatePerSec(uint64_t rate) { + if (rate == 0) intervalNsec_ = 0; + else intervalNsec_ = concurrency::Util::NS_PER_S / rate; + } + + /** + * Start the timer. + * + * Call this method before the first interval. + */ + void start() { + intervalStart_ = concurrency::Util::currentTimeUsec(); + } + + /** + * Sleep until the next interval should start. + * + * @return Returns true during normal operations, and false if the maxBacklog + * was hit and the timer has reset the average rate calculation. + */ + bool sleep() { + // Go as fast as possible when intervalNsec_ is 0 + if (intervalNsec_ == 0) { + return true; + } + + + uint64_t waitUntil, now; + { + concurrency::Guard guard(mutex_); + + // intervalStart_ is when the just previous interval started (or when it + // was supposed to start, if we aren't able to keep up with the requested + // rate). + // + // Update it to be when the next interval is supposed to start + numTimes_++; + now = concurrency::Util::currentTimeUsec(); + + waitUntil = intervalStart_ + (intervalNsec_ * numTimes_) / 1000; + + if (now > waitUntil) { + // If we can't keep up with the requested rate, we'll keep falling + // farther and farther behind. + // + // If we fall farther than maxBacklog_ behind, reset intervalStart_ to + // the current time. This way, if the operations eventually do speed up + // and we are able to meet the requested rate, we won't exceed it for + // too long trying to catch up. + uint64_t delta = now - waitUntil; + if (delta > maxBacklog_) { + intervalStart_ = now; + numTimes_ = 0; + return false; + } + return true; + } + } + + usleep(waitUntil - now); + return true; + } + + private: + uint64_t numTimes_; + uint64_t intervalNsec_; + uint64_t intervalStart_; + uint64_t maxBacklog_; + concurrency::Mutex mutex_; +}; + +}}} // apache::thrift::loadgen + +#endif // THRIFT_TEST_LOADGEN_INTERVALTIMER_H_ diff --git a/thrift/lib/cpp/test/loadgen/LatencyMonitor.h b/thrift/lib/cpp/test/loadgen/LatencyMonitor.h new file mode 100644 index 0000000000..5d6d9522dc --- /dev/null +++ b/thrift/lib/cpp/test/loadgen/LatencyMonitor.h @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_TEST_LOADGEN_LATENCYMONITOR_H_ +#define THRIFT_TEST_LOADGEN_LATENCYMONITOR_H_ 1 + +#include "thrift/lib/cpp/test/loadgen/TerminalMonitor.h" +#include "thrift/lib/cpp/test/loadgen/OpEnabledState.h" +#include "thrift/lib/cpp/test/loadgen/LatencyScoreBoard.h" + +namespace apache { namespace thrift { namespace loadgen { + +class LoadConfig; + +/** + * A Monitor implementation that prints QPS rates and latency information for + * all operations. + */ +class LatencyMonitor : public TerminalMonitor { + public: + /// The various fields that LatencyMonitor knows how to print + enum FieldEnum { + FIELD_COUNT, + FIELD_QPS, + FIELD_LATENCY, + FIELD_ALL_TIME_COUNT, + FIELD_ALL_TIME_QPS, + FIELD_ALL_TIME_LATENCY, + }; + + struct FieldInfo { + FieldInfo(FieldEnum f, int w = -1) + : field(f) + , width(w) + , dynamicWidth(true) {} + + FieldEnum field; + int width; + bool dynamicWidth; + }; + + typedef std::vector FieldInfoVector; + + LatencyMonitor(const boost::shared_ptr& config); + + /** + * Set the fields printed for the specified operation. + */ + void setFields(uint32_t opType, const FieldInfoVector* fields); + + /** + * Set the fields printed for aggregate statistics for all operations. + */ + void setTotalFields(const FieldInfoVector* fields); + + virtual boost::shared_ptr newScoreBoard(int id); + + virtual void initializeInfo(); + virtual uint32_t printHeader(); + virtual uint32_t printInfo(uint64_t intervalUsec); + + private: + typedef std::vector< boost::shared_ptr > ScoreBoardVector; + + void printOpHeader(FieldInfoVector* fields); + void printOpInfo(FieldInfoVector* fields, + const LatencyScoreBoard::OpData* current, + const LatencyScoreBoard::OpData* prev, + const LatencyScoreBoard::OpData* initial, + uint64_t intervalUsec, + uint64_t allTimeUsec); + + uint32_t getFieldVectorWidth(const FieldInfoVector* fields) const; + const char *getFieldName(FieldEnum field) const; + uint32_t getDefaultFieldWidth(FieldEnum field) const; + + void formatFieldValue(FieldEnum field, + char* buf, + size_t buflen, + const LatencyScoreBoard::OpData* current, + const LatencyScoreBoard::OpData* prev, + const LatencyScoreBoard::OpData* initial, + uint64_t intervalUsec, + uint64_t allTimeUsec); + void formatLatency(char* buf, size_t buflen, double avg, double stddev); + + void aggregateWorkerScorboards(LatencyScoreBoard* scoreboard); + void printField(const char* value, int width); + void printField(const char* value, FieldInfo* fieldInfo); + + void setDefaultOpFields(); + void printLegend(); + bool isFieldInUse(FieldEnum field); + + uint32_t numOpTypes_; + + /// The list of fields to print for each operation + std::vector opFields_; + /// The fields to print for the aggregate information over all operations + FieldInfoVector totalFields_; + + int64_t initialTime_; + LatencyScoreBoard initialScoreBoard_; + + /// A scoreboard with information aggregated across all of the workers + LatencyScoreBoard aggregateScoreBoard_; + /// A vector of the actual scoreboards used by the workers + ScoreBoardVector scoreboards_; + boost::shared_ptr config_; +}; + +}}} // apache::thrift::loadgen + +#endif // THRIFT_TEST_LOADGEN_LATENCYMONITOR_H_ diff --git a/thrift/lib/cpp/test/loadgen/LatencyScoreBoard.h b/thrift/lib/cpp/test/loadgen/LatencyScoreBoard.h new file mode 100644 index 0000000000..08037a82bd --- /dev/null +++ b/thrift/lib/cpp/test/loadgen/LatencyScoreBoard.h @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_TEST_LOADGEN_LATENCYSCOREBOARD_H_ +#define THRIFT_TEST_LOADGEN_LATENCYSCOREBOARD_H_ 1 + +#include "thrift/lib/cpp/test/loadgen/ScoreBoard.h" +#include "thrift/lib/cpp/test/loadgen/ScoreBoardOpVector.h" + +namespace apache { namespace thrift { namespace loadgen { + +/** + * A ScoreBoard that tracks number of queries per second, as well as + * information about how long each operation takes. + * + * This ScoreBoard calls gettimeofday() twice for each operation, so it does + * add a small amount of overhead. If you have extremely high performance + * requirements, you could use QpsScoreBoard to track just the QPS rate and + * eliminate the gettimeofday() calls. + */ +class LatencyScoreBoard : public ScoreBoard { + public: + class OpData { + public: + OpData(); + + void addDataPoint(uint64_t latencyUsecs); + + void zero(); + void accumulate(const OpData* other); + + uint64_t getCount() const; + uint64_t getCountSince(const OpData* other) const; + double getLatencyAvg() const; + double getLatencyAvgSince(const OpData* other) const; + double getLatencyStdDev() const; + double getLatencyStdDevSince(const OpData* other) const; + + uint64_t count_; + uint64_t usecSum_; + uint64_t sumOfSquares_; + }; + + LatencyScoreBoard(uint32_t numOpsHint) + : startTime_(0) + , opData_(numOpsHint) {} + + virtual void opStarted(uint32_t opType); + virtual void opSucceeded(uint32_t opType); + virtual void opFailed(uint32_t opType); + + /** + * Get the OpData for a particular operation type + */ + const OpData* getOpData(uint32_t opType); + + /** + * Compute an OpData object with aggregate information over all operation + * types. + * + * @param result A pointer to the OpData object to fill in with aggregate + * information. + */ + void computeOpAggregate(OpData* result) const; + + /** + * Zero out the statistics. + */ + void zero(); + + /** + * Add the counters from another scoreboard to this one. + */ + void accumulate(const LatencyScoreBoard* other); + + private: + int64_t startTime_; + ScoreBoardOpVector opData_; +}; + +}}} // apache::thrift::loadgen + +#endif // THRIFT_TEST_LOADGEN_LATENCYSCOREBOARD_H_ diff --git a/thrift/lib/cpp/test/loadgen/LoadConfig.h b/thrift/lib/cpp/test/loadgen/LoadConfig.h new file mode 100644 index 0000000000..90e2995d23 --- /dev/null +++ b/thrift/lib/cpp/test/loadgen/LoadConfig.h @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_TEST_LOADGEN_LOADCONFIG_H_ +#define THRIFT_TEST_LOADGEN_LOADCONFIG_H_ 1 + +#include +#include +#include + +namespace apache { namespace thrift { namespace loadgen { + +class LoadConfig { + public: + virtual ~LoadConfig() {} + + virtual uint32_t getNumOpTypes() const = 0; + + virtual uint32_t pickOpType() = 0; + virtual uint32_t pickOpsPerConnection() = 0; + + /** + * Return a human-readable name for an operation type. + * + * By default, just converts the integer value to a string, but this may be + * overridden by subclasses. + */ + virtual std::string getOpName(uint32_t opType); + + /** + * Get the number of worker threads to run. + */ + virtual uint32_t getNumWorkerThreads() const = 0; + + /** + * Get the desired number of queries per second. + * + * The workers should attempt to perform only this many operations per + * second. (It is possible that they will perform less than this if they + * cannot drive the requested qps rate.) + * + * @return Return the desired qps rate, or 0 if operations should be + * performed as fast as possible. + */ + virtual uint64_t getDesiredQPS() const { + return 0; + } +}; + +}}} // apache::thrift::loadgen + +#endif // THRIFT_TEST_LOADGEN_LOADCONFIG_H_ diff --git a/thrift/lib/cpp/test/loadgen/Monitor.h b/thrift/lib/cpp/test/loadgen/Monitor.h new file mode 100644 index 0000000000..418a9701ff --- /dev/null +++ b/thrift/lib/cpp/test/loadgen/Monitor.h @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_TEST_LOADGEN_MONITOR_H_ +#define THRIFT_TEST_LOADGEN_MONITOR_H_ 1 + +#include +#include +#include + +namespace apache { namespace thrift { namespace loadgen { + +class ScoreBoard; + +class Monitor { + public: + Monitor() {} + virtual ~Monitor() {} + + /** + * Create a new ScoreBoard object. + * + * This method will be called once for each worker thread. All calls to + * newScoreBoard() will be made before run() is invoked. + * + * For each Worker, newScoreBoard() will be called in the thread that will + * run that worker. However, the caller holds a lock and ensures that it + * will only be called in one thread at a time, so newScoreBoard() doesn't + * need to perform any locking internally. + * + * @param id The id of the Worker that will use this ScoreBoard. + */ + virtual boost::shared_ptr newScoreBoard(int id) = 0; + + /** + * Initialize monitoring information. + * + * This method is called once when the workers start, just before the initial + * monitoring interval. It can be used to get initial counter values from + * all of the Workers, so that statistics reported in the first call to + * redisplay() are accurate. + */ + virtual void initializeInfo() {} + + /** + * Redisplay the monitor information. + * + * @param intervalUsec The number of microseconds since the last call to + * redisplay() or initializeInfo(). + */ + virtual void redisplay(uint64_t intervalUsec) = 0; +}; + +}}} // apache::thrift::loadgen + +#endif // THRIFT_TEST_LOADGEN_MONITOR_H_ diff --git a/thrift/lib/cpp/test/loadgen/OpEnabledState.h b/thrift/lib/cpp/test/loadgen/OpEnabledState.h new file mode 100644 index 0000000000..5e834d9db4 --- /dev/null +++ b/thrift/lib/cpp/test/loadgen/OpEnabledState.h @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_TEST_LOADGEN_OPENABLEDSTATE_H_ +#define THRIFT_TEST_LOADGEN_OPENABLEDSTATE_H_ 1 + +#include + +namespace apache { namespace thrift { namespace loadgen { + +/** + * This class tracks a boolean for each operation type, to tell if an operation + * is enabled or not. + * + * This is mainly intended to be used by Monitor implementations, so they can + * avoid monitoring (or at least printing) information for operations that are + * disabled. + */ +class OpEnabledState { + public: + OpEnabledState(uint32_t numOpTypes) { + enabled_.resize(numOpTypes, true); + } + + void setEnabled(uint32_t opType, bool enabled) { + assert(opType < enabled_.size()); + enabled_[opType] = enabled; + } + + void setAllOpsEnabled(bool enabled) { + for (std::vector::iterator it = enabled_.begin(); + it != enabled_.end(); + ++it) { + *it = enabled; + } + } + + bool isEnabled(uint32_t opType) const { + assert(opType < enabled_.size()); + return enabled_[opType]; + } + + uint32_t getNumEnabled() const { + uint32_t num = 0; + for (std::vector::const_iterator it = enabled_.begin(); + it != enabled_.end(); + ++it) { + if (*it) { + ++num; + } + } + + return num; + } + + private: + std::vector enabled_; +}; + +}}} // apache::thrift::loadgen + +#endif // THRIFT_TEST_LOADGEN_OPENABLEDSTATE_H_ diff --git a/thrift/lib/cpp/test/loadgen/QpsMonitor.h b/thrift/lib/cpp/test/loadgen/QpsMonitor.h new file mode 100644 index 0000000000..e4e11dfde8 --- /dev/null +++ b/thrift/lib/cpp/test/loadgen/QpsMonitor.h @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_TEST_LOADGEN_QPSMONITOR_H_ +#define THRIFT_TEST_LOADGEN_QPSMONITOR_H_ 1 + +#include "thrift/lib/cpp/test/loadgen/TerminalMonitor.h" +#include "thrift/lib/cpp/test/loadgen/OpEnabledState.h" +#include "thrift/lib/cpp/test/loadgen/QpsScoreBoard.h" + +namespace apache { namespace thrift { namespace loadgen { + +class LoadConfig; + +/** + * A Monitor implementation that prints QPS rates for all operations. + */ +class QpsMonitor : public TerminalMonitor { + public: + QpsMonitor(const boost::shared_ptr& config); + + virtual boost::shared_ptr newScoreBoard(int id); + + virtual void initializeInfo(); + virtual uint32_t printHeader(); + virtual uint32_t printInfo(uint64_t intervalUsec); + + OpEnabledState* getEnabledState() { + return &enabledState_; + } + + const OpEnabledState* getEnabledState() const { + return &enabledState_; + } + + void printAllTimeQps(bool enabled) { + printAllTime_ = enabled; + } + + private: + typedef std::vector< boost::shared_ptr > ScoreBoardVector; + + void computeAggregate(QpsScoreBoard* scoreboard); + + int64_t initialTime_; + uint64_t initialSum_; + + bool printAllTime_; + OpEnabledState enabledState_; + + QpsScoreBoard aggregateScoreBoard_; + ScoreBoardVector scoreboards_; + boost::shared_ptr config_; +}; + +}}} // apache::thrift::loadgen + +#endif // THRIFT_TEST_LOADGEN_QPSMONITOR_H_ diff --git a/thrift/lib/cpp/test/loadgen/QpsScoreBoard.h b/thrift/lib/cpp/test/loadgen/QpsScoreBoard.h new file mode 100644 index 0000000000..39ed6cb43e --- /dev/null +++ b/thrift/lib/cpp/test/loadgen/QpsScoreBoard.h @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_TEST_LOADGEN_QPSSCOREBOARD_H_ +#define THRIFT_TEST_LOADGEN_QPSSCOREBOARD_H_ 1 + +#include "thrift/lib/cpp/test/loadgen/ScoreBoard.h" +#include "thrift/lib/cpp/test/loadgen/ScoreBoardOpVector.h" + +namespace apache { namespace thrift { namespace loadgen { + +/** + * A ScoreBoard that tracks number of queries per second. + * + * This is a very simple scoreboard, that adds very little overhead. + */ +class QpsScoreBoard : public ScoreBoard { + public: + QpsScoreBoard(uint32_t numOpsHint) : opData_(numOpsHint) {} + + virtual void opStarted(uint32_t opType); + virtual void opSucceeded(uint32_t opType); + virtual void opFailed(uint32_t opType); + + /** + * Get the number of operations performed for a specific operation type. + */ + uint64_t getCount(uint32_t opType) const; + + /** + * Compute the total number of operations performed, for all operation types. + */ + uint64_t computeTotalCount() const; + + /** + * Zero out the statistics. + */ + void zero(); + + /** + * Add the counters from another scoreboard to this one. + */ + void accumulate(const QpsScoreBoard* other); + + private: + struct OpData { + OpData() : count(0) {} + + void zero() { + count = 0; + } + + void accumulate(const OpData* other) { + count += other->count; + } + + uint64_t count; + }; + + ScoreBoardOpVector opData_; +}; + +}}} // apache::thrift::loadgen + +#endif // THRIFT_TEST_LOADGEN_QPSSCOREBOARD_H_ diff --git a/thrift/lib/cpp/test/loadgen/RNG.h b/thrift/lib/cpp/test/loadgen/RNG.h new file mode 100644 index 0000000000..fb91056583 --- /dev/null +++ b/thrift/lib/cpp/test/loadgen/RNG.h @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_TEST_LOADGEN_RNG_H_ +#define THRIFT_TEST_LOADGEN_RNG_H_ 1 + +#include + +namespace apache { namespace thrift { namespace loadgen { + +/** + * A random number generator to use for load tests. + * + * We keep one RNG per thread, to avoid having to perform locking when getting + * random numbers. + * + * RNG also implements the boost random number generator interface, so it can + * be used as an engine for boost random distributions. + * + * + * This class is essentially just a wrapper around boost::mt19937. + * We have to wrap it because some of the boost code copy-constructs the + * generators, causing the state of the original generator to never be updated. + * (In particular uniform_01 copy constructs its generator argument, and + * variate_generator may end up using uniform_01.) + * + * It's rather annoying that we have to define our own wrapper class for this. + */ +class RNG { + public: + // Use boost::mt19937 as the underlying RNG + typedef boost::mt19937 BoostRNG; + + // result_type and has_fixed_range are required for the boost interfaces + typedef BoostRNG::result_type result_type; + static const bool has_fixed_range = false; + + RNG(BoostRNG* rng) : rng_(rng) {} + + /** + * Get the thread-local RNG. + */ + static RNG& getRNG(); + + /** + * Set the RNG seed. + * + * This value is used to pick seeds for new thread-local RNGs. + * It won't affect thread-local RNGs that have already been created. + * + * Note that that the seed value used for each thread-local RNG depends on + * the order in which the thread-local RNGs are created. You may not get + * consistent results across runs if your threads are not initialized in a + * consistent order. + */ + static void setGlobalSeed(result_type s); + + /** + * Re-seed this RNG + */ + void seed(result_type s) { + rng_->seed(s); + } + + /** + * Get a random number. + * + * Part of the boost random generator interface. + */ + result_type operator()() { + return (*rng_)(); + } + + /** + * Get the minimum value that can be returned. + * + * Part of the boost random generator interface. + */ + result_type min() const { + return rng_->min(); + } + + /** + * Get the maximum value that can be returned. + * + * Part of the boost random generator interface. + */ + result_type max() const { + return rng_->max(); + } + + /* + * Helper functions to pick random uint32_t values + */ + static uint32_t getU32(); + static uint32_t getU32(uint32_t max); + static uint32_t getU32(uint32_t min, uint32_t max); + + /** + * Helper function to pick random double values in the range [0.0, 1.0) + */ + static double getReal(); + + /** + * Helper function to pick random double values in the range [min, max) + */ + static double getReal(double min, double max); + + /** + * Helper function to pick random values in a log-normal distribution. + * + * @param mean The mean value for the log-normal distribution. + * @param sigma The sigma value for the log-normal distribution. + * (This controls how spread-out the distribution will be.) + * If negative, defaults to half the mean. + */ + static double getLogNormal(double mean, double sigma = -1.0); + + private: + BoostRNG* rng_; +}; + +}}} // apache::thrift::test + +#endif // THRIFT_TEST_LOADGEN_RNG_H_ diff --git a/thrift/lib/cpp/test/loadgen/ScoreBoard.h b/thrift/lib/cpp/test/loadgen/ScoreBoard.h new file mode 100644 index 0000000000..fd5cd33361 --- /dev/null +++ b/thrift/lib/cpp/test/loadgen/ScoreBoard.h @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_TEST_LOADGEN_SCOREBOARD_H_ +#define THRIFT_TEST_LOADGEN_SCOREBOARD_H_ 1 + +#include + +namespace apache { namespace thrift { namespace loadgen { + +/** + * A ScoreBoard object keeps statistics for a Worker. + * + * There is intended to be one ScoreBoard object per Worker, so that it doesn't + * need to perform locking when recording statistics. + * + * The Monitor object is responsible for aggregating the statistics from the + * various ScoreBoards. + */ +class ScoreBoard { + public: + virtual ~ScoreBoard() {} + + /** + * opStarted() is invoked just before each call to + * Worker::performOperation(). + */ + virtual void opStarted(uint32_t opType) = 0; + + /** + * opSucceeded() is invoked after each successful call to + * Worker::performOperation(). + */ + virtual void opSucceeded(uint32_t opType) = 0; + + /** + * opFailed() is invoked if Worker::performOperation() throws an exception. + */ + virtual void opFailed(uint32_t opType) = 0; +}; + +}}} // apache::thrift::loadgen + +#endif // THRIFT_TEST_LOADGEN_SCOREBOARD_H_ diff --git a/thrift/lib/cpp/test/loadgen/ScoreBoardOpVector.h b/thrift/lib/cpp/test/loadgen/ScoreBoardOpVector.h new file mode 100644 index 0000000000..1935a84627 --- /dev/null +++ b/thrift/lib/cpp/test/loadgen/ScoreBoardOpVector.h @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_TEST_LOADGEN_SCOREBOARDOPVECTOR_H_ +#define THRIFT_TEST_LOADGEN_SCOREBOARDOPVECTOR_H_ 1 + +#include "thrift/lib/cpp/test/loadgen/ScoreBoard.h" + +#include +#include +#include + +namespace apache { namespace thrift { namespace loadgen { + +/** + * A vector of per-operation scoreboard data. + * + * This is useful for implementing ScoreBoard classes. + * + * The OpDataT class must have: + * - a default constructor + * - a zero() method + * - an accumulate() method + */ +template +class ScoreBoardOpVector { + private: + typedef std::vector DataVector; + + public: + typedef typename DataVector::iterator Iterator; + typedef typename DataVector::const_iterator ConstIterator; + + ScoreBoardOpVector(uint32_t numOpsHint) { + resize(numOpsHint); + } + + OpDataT* getOpData(uint32_t opType) { + if (opType >= opData_.size()) { + resize(opType + 1); + } + return &opData_[opType]; + } + + const OpDataT* getOpDataOrNull(uint32_t opType) const { + if (opType >= opData_.size()) { + return NULL; + } + return &opData_[opType]; + } + + void zero() { + for (Iterator it = opData_.begin(); it != opData_.end(); ++it) { + it->zero(); + } + } + + void accumulate(const ScoreBoardOpVector* other) { + if (other->opData_.size() > opData_.size()) { + resize(other->opData_.size()); + } + + uint32_t index = 0; + for (ConstIterator it = other->opData_.begin(); + it != other->opData_.end(); + ++it, ++index) { + opData_[index].accumulate(&(*it)); + } + } + + void accumulateOverOps(OpDataT* result) const { + result->zero(); + for (ConstIterator it = opData_.begin(); it != opData_.end(); ++it) { + result->accumulate(&(*it)); + } + } + + Iterator begin() { + return opData_.begin(); + } + Iterator end() { + return opData_.end(); + } + + ConstIterator begin() const { + return opData_.begin(); + } + ConstIterator end() const { + return opData_.end(); + } + + private: + void resize(uint32_t numOps) { + assert(numOps > opData_.size()); + + // We could add some padding here. In the past, when using glibc malloc + // with QpsScoreBoard and just 1 operation, I've seen the opData for two + // different worker threads end up being allocated on the same cache line. + // This hurts peformance, since the two worker threads run on different + // CPUs and they each keep evicting the cache line from the other CPU. + // With jemalloc this doesn't seem to happen anymore, and allocating more + // space then necessary here can very slightly slow down performance. + + opData_.resize(numOps); + } + + DataVector opData_; +}; + +}}} // apache::thrift::loadgen + +#endif // THRIFT_TEST_LOADGEN_SCOREBOARDOPVECTOR_H_ diff --git a/thrift/lib/cpp/test/loadgen/TARGETS b/thrift/lib/cpp/test/loadgen/TARGETS new file mode 100644 index 0000000000..a2db31670f --- /dev/null +++ b/thrift/lib/cpp/test/loadgen/TARGETS @@ -0,0 +1,32 @@ +cpp_library( + name = 'loadgen', + srcs = [ + 'Controller.cpp', + 'LatencyMonitor.cpp', + 'LatencyScoreBoard.cpp', + 'LoadConfig.cpp', + 'loadgen.cpp', + 'QpsMonitor.cpp', + 'QpsScoreBoard.cpp', + 'RNG.cpp', + 'TerminalMonitor.cpp', + 'WeightedLoadConfig.cpp', + ], + deps = [ + '@/common/time:time', + ], + external_deps = [ + ('boost', None, 'boost'), + ], +) + +cpp_binary( + name = 'perftest', + srcs = [ + 'PerfTest.cpp', + ], + deps = [ + ':loadgen', + '@/common/config', + ], +) diff --git a/thrift/lib/cpp/test/loadgen/TerminalMonitor.h b/thrift/lib/cpp/test/loadgen/TerminalMonitor.h new file mode 100644 index 0000000000..f06847dbd1 --- /dev/null +++ b/thrift/lib/cpp/test/loadgen/TerminalMonitor.h @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_TEST_LOADGEN_TERMINALMONITOR_H_ +#define THRIFT_TEST_LOADGEN_TERMINALMONITOR_H_ 1 + +#include "thrift/lib/cpp/test/loadgen/Monitor.h" + +namespace apache { namespace thrift { namespace loadgen { + +/** + * A Monitor object that prints statistics to a terminal. + * + * It handles printing header lines, and re-printing the header whenever it + * scrolls off of the screen. + */ +class TerminalMonitor : public Monitor { + public: + TerminalMonitor(); + + virtual void redisplay(uint64_t intervalUsec); + + /** + * Initialize monitoring information. + * + * This method is called immediately after all of the Workers have started, + * just before the initial monitoring interval. It can be used to get + * initial counter values from all of the Workers, so that statistics + * reported in the first call to printInfo() are accurate. + * + * If subclasses override initializeInfo(), they should make sure to call + * TerminalMonitor::initializeInfo() in their method. + */ + virtual void initializeInfo(); + + /** + * Print header lines. + * + * This is called when monitoring first starts. If the screen height can + * be determined, this is also called every time the previous header scrolls + * off the screen. + * + * @return Returns the number of lines printed. + */ + virtual uint32_t printHeader() = 0; + + /** + * Print monitoring information. + * + * This is called once every specified interval. + * + * @param intervalUsec The number of microseconds since the previous call to + * printInfo(). The first time printInfo() is called, + * this is the number of microseconds since + * initializeInfo() was called. + * + * @return Returns the number of lines printed. + */ + virtual uint32_t printInfo(uint64_t intervalUsec) = 0; + + protected: + int32_t getScreenHeight(); + + int32_t screenHeight_; + uint32_t linesPrinted_; +}; + +}}} // apache::thrift::loadgen + +#endif // THRIFT_TEST_LOADGEN_TERMINALMONITOR_H_ diff --git a/thrift/lib/cpp/test/loadgen/WeightedLoadConfig.h b/thrift/lib/cpp/test/loadgen/WeightedLoadConfig.h new file mode 100644 index 0000000000..b76522d710 --- /dev/null +++ b/thrift/lib/cpp/test/loadgen/WeightedLoadConfig.h @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_TEST_LOADGEN_WEIGHTEDLOADCONFIG_H_ +#define THRIFT_TEST_LOADGEN_WEIGHTEDLOADCONFIG_H_ 1 + +#include "thrift/lib/cpp/test/loadgen/LoadConfig.h" + +#include +#include + +namespace apache { namespace thrift { namespace loadgen { + +class OpEnabledState; + +class WeightedLoadConfig : public LoadConfig { + public: + WeightedLoadConfig(uint32_t numOps); + + void setOpInfo(uint32_t opType, + const std::string& name, + uint32_t weight); + uint32_t getOpWeight(uint32_t opType); + + virtual uint32_t getNumOpTypes() const; + virtual uint32_t pickOpType(); + virtual uint32_t pickOpsPerConnection() = 0; + + virtual std::string getOpName(uint32_t opType); + + /** + * Update an OpEnabledState object to enable only operations that have a + * non-zero weight. + */ + virtual void configureEnabledState(OpEnabledState* enabledState) const; + + private: + uint32_t weightsSum_; + std::vector weights_; + std::vector names_; +}; + +}}} // apache::thrift::loadgen + +#endif // THRIFT_TEST_LOADGEN_WEIGHTEDLOADCONFIG_H_ diff --git a/thrift/lib/cpp/test/loadgen/Worker.h b/thrift/lib/cpp/test/loadgen/Worker.h new file mode 100644 index 0000000000..04db52fa48 --- /dev/null +++ b/thrift/lib/cpp/test/loadgen/Worker.h @@ -0,0 +1,275 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_TEST_LOADGEN_WORKER_H_ +#define THRIFT_TEST_LOADGEN_WORKER_H_ 1 + +#include "thrift/lib/cpp/test/loadgen/WorkerIf.h" + +#include "thrift/lib/cpp/test/loadgen/IntervalTimer.h" +#include "thrift/lib/cpp/test/loadgen/LoadConfig.h" +#include "thrift/lib/cpp/test/loadgen/ScoreBoard.h" +#include "thrift/lib/cpp/concurrency/Util.h" +#include "thrift/lib/cpp/TLogging.h" + +#include +#include + +namespace apache { namespace thrift { namespace loadgen { + +/** + * Main Worker implementation + * + * If you are implementing a new load generator, you should define your own + * subclass of Worker, and implement the createConnection() and + * performOperation() methods. + * + * This is templatized on the client type. + * + * The Config type is also templatized for convenience. This allows + * subclasses to store their own more-specific configuration type that derives + * from LoadConfig. + */ +template +class Worker : public WorkerIf, public boost::noncopyable { + public: + typedef ClientT ClientType; + typedef ConfigT ConfigType; + + enum ErrorAction { + EA_CONTINUE, + EA_NEXT_CONNECTION, + EA_DROP_THREAD, + EA_ABORT + }; + + Worker() + : id_(-1) + , alive_(false) + , intervalTimer_(NULL) + , config_() + , scoreboard_() {} + + /** + * Initialize the Worker. + * + * This is separate from the constructor so that developers writing new + * Worker implementations don't have to pass through additional constructor + * arguments. If the subclass doesn't need any special initialization, they + * can just use the default constructor. + * + * If a Worker implementation does need to perform additional implementation- + * specific initialization after the config object has been set, it can + * override init(). + */ + void init(int id, + const boost::shared_ptr& config, + const boost::shared_ptr& scoreboard, + IntervalTimer* itimer) { + assert(id_ == -1); + assert(!config_); + id_ = id; + config_ = config; + scoreboard_ = scoreboard; + intervalTimer_ = itimer; + alive_ = true; + } + + virtual ~Worker() {} + + int getID() const { + return id_; + } + + /** + * Create a new connection to the server. + * + * Subclasses must implement this method. + */ + virtual boost::shared_ptr createConnection() = 0; + + /** + * Perform an operation on a connection. + * + * Subclasses must implement this method. + */ + virtual void performOperation(const boost::shared_ptr& client, + uint32_t opType) = 0; + + /** + * Determine how to handle an exception raised by createConnection(). + * + * The default behavior is to log an error message and abort. + * Subclasses may override this function to provide alternate behavior. + */ + virtual ErrorAction handleConnError(const std::exception& ex) { + T_ERROR("worker %d caught %s exception while connecting: %s", + id_, typeid(ex).name(), ex.what()); + return EA_ABORT; + } + + /** + * Determine how to handle an exception raised by performOperation(). + * + * The default behavior is to log an error message and continue processing on + * a new connection. Subclasses may override this function to provide + * alternate behavior. + */ + virtual ErrorAction handleOpError(uint32_t opType, const std::exception& ex) { + T_ERROR("worker %d caught %s exception performing operation %s: %s", + id_, typeid(ex).name(), config_->getOpName(opType).c_str(), + ex.what()); + return EA_NEXT_CONNECTION; + } + + /** + * Get the LoadConfig for this worker. + * + * (Returns a templatized config type for convenience, so subclasses can + * store a subclass of LoadConfig, and retrieve it without having to cast it + * back to the subclass type.) + */ + const boost::shared_ptr& getConfig() const { + return config_; + } + + /** + * The main worker method. + * + * Loop forever creating connections and performing operations on them. + * (May return if an error occurs and the error handler returns + * EA_DROP_THREAD.) + */ + virtual void run() { + while (true) { + // Create a new connection + boost::shared_ptr client; + try { + client = createConnection(); + } catch (const std::exception& ex) { + ErrorAction action = handleConnError(ex); + if (action == EA_CONTINUE || action == EA_NEXT_CONNECTION) { + // continue the next connection loop + continue; + } else if (action == EA_DROP_THREAD) { + T_ERROR("worker %d exiting after connection error", id_); + alive_ = false; + return; + } else if (action == EA_ABORT) { + T_ERROR("worker %d causing abort after connection error", id_); + abort(); + } else { + T_ERROR("worker %d received unknown conn error action %d; aborting", + id_, action); + abort(); + } + } + + // Determine how many operations to perform on this connection + uint32_t nops = config_->pickOpsPerConnection(); + + // Perform operations on the connection + for (uint32_t n = 0; n < nops; ++n) { + // Only send as fast as requested + if (!intervalTimer_->sleep()) { + T_ERROR("can't keep up with requested QPS rate"); + } + uint32_t opType = config_->pickOpType(); + scoreboard_->opStarted(opType); + try { + performOperation(client, opType); + scoreboard_->opSucceeded(opType); + } catch (const std::exception& ex) { + scoreboard_->opFailed(opType); + ErrorAction action = handleOpError(opType, ex); + if (action == EA_CONTINUE) { + // nothing to do; continue trying to use this connection + } else if (action == EA_NEXT_CONNECTION) { + // break out of the op loop, + // continue the next connection loop + break; + } else if (action == EA_DROP_THREAD) { + T_ERROR("worker %d exiting after op %d error", id_, opType); + // return from run() + alive_ = false; + return; + } else if (action == EA_ABORT) { + T_ERROR("worker %d causing abort after op %d error", id_, opType); + abort(); + } else { + T_ERROR("worker %d received unknown op error action %d; aborting", + id_, action); + abort(); + } + } + } + } + + assert(false); + alive_ = false; + } + + bool isAlive() const { + return alive_; + } + + protected: + // Methods needed for overriding ::run + const boost::shared_ptr& getScoreBoard() const { + return scoreboard_; + } + + void stopWorker() { + alive_ = false; + } + + private: + int id_; + bool alive_; + IntervalTimer* intervalTimer_; + boost::shared_ptr config_; + boost::shared_ptr scoreboard_; +}; + + +/** + * Default WorkerFactory implementation. + * + * This factory creates Worker objects using the default constructor, + * then calls init(id, config, scoreboard) on each worker before returning it. + */ +template +class SimpleWorkerFactory : public WorkerFactory { + public: + SimpleWorkerFactory(const boost::shared_ptr& config) + : config_(config) {} + + virtual WorkerT* newWorker(int id, + const boost::shared_ptr& scoreboard, + IntervalTimer* itimer) { + std::auto_ptr worker(new WorkerT); + worker->init(id, config_, scoreboard, itimer); + return worker.release(); + } + + boost::shared_ptr config_; +}; + +}}} // apache::thrift::loadgen + +#endif // THRIFT_TEST_LOADGEN_WORKER_H_ diff --git a/thrift/lib/cpp/test/loadgen/WorkerIf.h b/thrift/lib/cpp/test/loadgen/WorkerIf.h new file mode 100644 index 0000000000..a5e65fc560 --- /dev/null +++ b/thrift/lib/cpp/test/loadgen/WorkerIf.h @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_TEST_LOADGEN_WORKERIF_H_ +#define THRIFT_TEST_LOADGEN_WORKERIF_H_ 1 + +#include + +#include "thrift/lib/cpp/test/loadgen/IntervalTimer.h" + +namespace apache { namespace thrift { namespace loadgen { + +class ScoreBoard; + +/** + * Interface class for all Worker types to inherit from + * + * If you are implementing a new load generator, you generally should derive + * from Worker instead of using WorkerIf directly. WorkerIf + * only exists to provide a common base class for all Worker template + * instantiations. + */ +class WorkerIf { + public: + virtual ~WorkerIf() {} + + /** + * Run the worker + */ + virtual void run() = 0; + + /** + * Determine if this worker is still running. + * + * Returns false if this worker's run() function has returned. + */ + virtual bool isAlive() const = 0; +}; + + +/** + * WorkerFactory + */ +class WorkerFactory { + public: + virtual ~WorkerFactory() {} + + virtual WorkerIf* newWorker(int id, + const boost::shared_ptr& sb, + IntervalTimer* itimer) = 0; +}; + +}}} // apache::thrift::loadgen + +#endif // THRIFT_TEST_LOADGEN_WORKERIF_H_ diff --git a/thrift/lib/cpp/test/loadgen/loadgen.h b/thrift/lib/cpp/test/loadgen/loadgen.h new file mode 100644 index 0000000000..8158cace73 --- /dev/null +++ b/thrift/lib/cpp/test/loadgen/loadgen.h @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_TEST_LOADGEN_LOADGEN_H_ +#define THRIFT_TEST_LOADGEN_LOADGEN_H_ 1 + +#include "thrift/lib/cpp/test/loadgen/Worker.h" + +namespace apache { namespace thrift { + +namespace concurrency { + +class PosixThreadFactory; + +} // apache::thrift::concurrency + +namespace loadgen { + +class LoadConfig; +class Monitor; + +/** + * Run load generation. + * + * @param factory The WorkerFactory to use to create each worker thread. + * @param config The LoadConfig object that describes the operations. + * @param interval The number of seconds between each line of statistics + * output. + * @param monitor The Monitor to use for printing statistics. If NULL, + * a LatencyMonitor will be used. + */ +void runLoadGen(WorkerFactory* factory, + const boost::shared_ptr& config, + double interval = 1.0, + Monitor* monitor = NULL, + apache::thrift::concurrency::PosixThreadFactory* threadFactory = NULL); + +/** + * Run load generation. + * + * This is a helper function around runLoadGen() that automatically creates a + * SimpleWorkerFactory. + */ +template +void runLoadGen(const boost::shared_ptr& config, + double interval = 1.0, + Monitor* monitor = NULL, + apache::thrift::concurrency::PosixThreadFactory* threadFactory = NULL) { + SimpleWorkerFactory factory(config); + runLoadGen(&factory, config, interval, monitor, threadFactory); +} + +/** + * Run load generation. + * + * This is a helper function around runLoadGen() that automatically creates a + * SimpleWorkerFactory. + */ +template +void runLoadGen(const boost::shared_ptr& config, + double interval = 1.0, + Monitor* monitor = NULL, + apache::thrift::concurrency::PosixThreadFactory* threadFactory = NULL) { + runLoadGen(config, interval, monitor, threadFactory); +} + +}}} // apache::thrift::loadgen + +#endif // THRIFT_TEST_LOADGEN_LOADGEN_H_ diff --git a/thrift/lib/cpp/thrift_config.h b/thrift/lib/cpp/thrift_config.h new file mode 100644 index 0000000000..6a84630cc8 --- /dev/null +++ b/thrift/lib/cpp/thrift_config.h @@ -0,0 +1,61 @@ +#ifndef THRIFT_THRIFT_CONFIG_H_ +#define THRIFT_THRIFT_CONFIG_H_ + +#include + +/* Define to 1 if you have the `clock_gettime' function. */ +#define THRIFT_HAVE_CLOCK_GETTIME 1 + +/* Define to 1 if you have the header file. */ +#define THRIFT_HAVE_ENDIAN_H 1 + +/* Define to 1 if you have the header file. */ +#define THRIFT_HAVE_INTTYPES_H 1 + +/* Define to 1 if you have the header file. */ +#define THRIFT_HAVE_STDINT_H 1 + +/* Possible value for SIGNED_RIGHT_SHIFT_IS */ +#define ARITHMETIC_RIGHT_SHIFT 1 + +/* Possible value for SIGNED_RIGHT_SHIFT_IS */ +#define LOGICAL_RIGHT_SHIFT 2 + +/* Possible value for SIGNED_RIGHT_SHIFT_IS */ +#define UNKNOWN_RIGHT_SHIFT 3 + +/* Indicates the effect of the right shift operator on negative signed + integers */ +#define SIGNED_RIGHT_SHIFT_IS 1 + +/* + * Define to noexcept if the compiler supports noexcept + * + * If the compiler does not support noexcept, we define to the empty string + * in optimized builds. In debug builds, we define to throw(), so that the + * compiler will complain if a child class does not use THRIFT_NOEXCEPT when + * overriding a virtual method originally declared with THRIFT_NOEXCEPT. In + * debug mode, the program will also immediately call unexpected() if a + * THRIFT_NOEXCEPT function does throw an exception. + */ +#ifdef NDEBUG +#define THRIFT_NOEXCEPT +#else +#define THRIFT_NOEXCEPT throw() +#endif + +/* + * We have std::unique_ptr if we're compiling with gcc-4.4 or greater + * and C++0x features are enabled. + */ +#ifdef __GNUC__ +#if __GNUC_PREREQ(4, 4) +#define THRIFT_HAVE_UNIQUE_PTR __GXX_EXPERIMENTAL_CXX0X__ +#else +#define THRIFT_HAVE_UNIQUE_PTR 0 +#endif +#else /* ! __GNUC__ */ +#define THRIFT_HAVE_UNIQUE_PTR 0 +#endif /* __GNUC__ */ + +#endif /* THRIFT_THRIFT_CONFIG_H_ */ diff --git a/thrift/lib/cpp/transport/HDFS.h b/thrift/lib/cpp/transport/HDFS.h new file mode 100644 index 0000000000..41fab86432 --- /dev/null +++ b/thrift/lib/cpp/transport/HDFS.h @@ -0,0 +1,61 @@ +// Copyright (c) 2009- Facebook +// Distributed under the Thrift Software License +// +// See accompanying file LICENSE or visit the Thrift site at: +// http://developers.facebook.com/thrift/ + +#ifndef _THRIFT_TRANSPORT_HDFS_H +#define _THRIFT_TRANSPORT_HDFS_H + +#include +#include + +/** + * Dead-simple wrappers around hdfs and hdfsFile descriptors. + * The wrappers take responsibility of descriptor allocation/release in ctor/dtor. + * + * @author Li Zhang + */ + +class HDFS { + public: + HDFS(); + HDFS(const std::string& host, uint16_t port); + bool disconnect(); + ~HDFS(); + void* getHandle() const { + return hdfs_; + } + bool isConnected() const { + return hdfs_ != NULL; + } + protected: + void* hdfs_; +}; + +class HDFSFile { + public: + enum AccessPolicy { + OPEN_FOR_READ = 0, + CREATE_FOR_WRITE = 1, + OPEN_FOR_APPEND = 2, + }; + HDFSFile(boost::shared_ptr hdfs, const std::string& path, AccessPolicy ap, + int bufferSize = 0, short replication = 0, int32_t blocksize = 0); + bool close(); + void* getHandle() const { + return file_; + } + boost::shared_ptr getFS() const { + return hdfs_; + } + bool isOpen() const { + return file_ != NULL; + } + ~HDFSFile(); + protected: + boost::shared_ptr hdfs_; + void* file_; +}; + +#endif diff --git a/thrift/lib/cpp/transport/TARGETS b/thrift/lib/cpp/transport/TARGETS new file mode 100644 index 0000000000..d85bead36f --- /dev/null +++ b/thrift/lib/cpp/transport/TARGETS @@ -0,0 +1,88 @@ +# TARGETS file for thrift/lib/cpp/transport + +cpp_library( + name = "transport", + srcs = [ + "TFileTransport.cpp", + "TFDTransport.cpp", + "THttpClient.cpp", + "THttpServer.cpp", + "THttpTransport.cpp", + "TServerSocket.cpp", + "TSimpleFileTransport.cpp", + "TSocket.cpp", + "TSocketAddress.cpp", + "TSocketPool.cpp", + "TTransportException.cpp", + "TTransportUtils.cpp", + "TBufferTransports.cpp", + "TMemPagedFactory.cpp", + "TMemPagedTransport.cpp", + ], + deps = [ + "@/thrift/lib/cpp:thrift_base", + "@/thrift/lib/cpp/concurrency", + "@/external/google_base", + ], + preprocessor_flags = [ "-DTCP_LOW_MIN_RTO=15" ], +) + +cpp_library( + name = "transport_ssl", + srcs = [ + "TSSLSocket.cpp", + "TSSLServerSocket.cpp", + ], + deps = [ + ":transport", + "@/thrift/lib/cpp/concurrency", + ], + external_deps = [ ('openssl', None, 'ssl') ], +) + +cpp_library( + name = "header", + srcs = [ "THeaderTransport.cpp", + "THeader.cpp" ], + deps = [ + ":transport", + "@/folly/experimental/io:io", + "@/thrift/lib/cpp/util:internal_util", + ], + external_deps = [ ('zlib', None, 'z'), + ('snappy', None) + ], +) + +cpp_library( + name = "zlib", + srcs = [ "TZlibTransport.cpp" ], + deps = [ + ":transport", + ], + external_deps = [ ('zlib', None, 'z') ], +) + +cpp_library( + name = "hdfs_transport", + srcs = [ "HDFS.cpp", + "THDFSFileTransport.cpp", + ], + + system_deps = [ "-Lexternal/hadoop/latest/", + "-L$(JAVA_HOME)/jre/lib/amd64/server/", + "-Lexternal/jvm/", + "-ljvm", + "-lhdfs" + ], + preprocessor_flags = [ "-I@@@JAVA_HOME@@@/include", + "-I@@@JAVA_HOME@@@/include/linux", + "-Iexternal/hadoop/latest", + ], + shared_system_deps = [ "hdfs", + "jvm", + ], + deps = [ + "@/thrift/lib/cpp:thrift_base", + ], +) diff --git a/thrift/lib/cpp/transport/TBufferTransports.h b/thrift/lib/cpp/transport/TBufferTransports.h new file mode 100644 index 0000000000..0797dd88db --- /dev/null +++ b/thrift/lib/cpp/transport/TBufferTransports.h @@ -0,0 +1,927 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef THRIFT_TRANSPORT_TBUFFERTRANSPORTS_H_ +#define THRIFT_TRANSPORT_TBUFFERTRANSPORTS_H_ 1 + +#include +#include "boost/scoped_array.hpp" +#include "folly/Likely.h" + +#include "thrift/lib/cpp/transport/TTransport.h" +#include "thrift/lib/cpp/transport/TVirtualTransport.h" + +namespace apache { namespace thrift { namespace transport { + + +/** + * Base class for all transports that use read/write buffers for performance. + * + * TBufferBase is designed to implement the fast-path "memcpy" style + * operations that work in the common case. It does so with small and + * (eventually) nonvirtual, inlinable methods. TBufferBase is an abstract + * class. Subclasses are expected to define the "slow path" operations + * that have to be done when the buffers are full or empty. + * + */ +class TBufferBase : public TVirtualTransport { + + public: + + /** + * Fast-path read. + * + * When we have enough data buffered to fulfill the read, we can satisfy it + * with a single memcpy, then adjust our internal pointers. If the buffer + * is empty, we call out to our slow path, implemented by a subclass. + * This method is meant to eventually be nonvirtual and inlinable. + */ + uint32_t read(uint8_t* buf, uint32_t len) { + uint8_t* new_rBase = rBase_ + len; + if (LIKELY(new_rBase <= rBound_)) { + std::memcpy(buf, rBase_, len); + rBase_ = new_rBase; + return len; + } + return readSlow(buf, len); + } + + /** + * Shortcutted version of readAll. + */ + uint32_t readAll(uint8_t* buf, uint32_t len) { + uint8_t* new_rBase = rBase_ + len; + if (LIKELY(new_rBase <= rBound_)) { + std::memcpy(buf, rBase_, len); + rBase_ = new_rBase; + return len; + } + return apache::thrift::transport::readAll(*this, buf, len); + } + + /** + * Fast-path write. + * + * When we have enough empty space in our buffer to accommodate the write, we + * can satisfy it with a single memcpy, then adjust our internal pointers. + * If the buffer is full, we call out to our slow path, implemented by a + * subclass. This method is meant to eventually be nonvirtual and + * inlinable. + */ + void write(const uint8_t* buf, uint32_t len) { + uint8_t* new_wBase = wBase_ + len; + if (LIKELY(new_wBase <= wBound_)) { + std::memcpy(wBase_, buf, len); + wBase_ = new_wBase; + return; + } + writeSlow(buf, len); + } + + /** + * Fast-path borrow. A lot like the fast-path read. + */ + const uint8_t* borrow(uint8_t* buf, uint32_t* len) { + if (LIKELY(static_cast(*len) <= rBound_ - rBase_)) { + // With strict aliasing, writing to len shouldn't force us to + // refetch rBase_ from memory. TODO(dreiss): Verify this. + *len = rBound_ - rBase_; + return rBase_; + } + return borrowSlow(buf, len); + } + + /** + * Consume doesn't require a slow path. + * When the read ptr catches up with write ptr, invokes consumeEnd(). + */ + void consume(uint32_t len) { + if (LIKELY(static_cast(len) <= rBound_ - rBase_)) { + rBase_ += len; + if (rBase_ == wBase_) { + // This is required for TMemoryBuffer. + consumeEnd(); + } + } else { + throw TTransportException(TTransportException::BAD_ARGS, + "consume did not follow a borrow."); + } + } + + + protected: + + /// Slow path read. + virtual uint32_t readSlow(uint8_t* buf, uint32_t len) = 0; + + /// Slow path write. + virtual void writeSlow(const uint8_t* buf, uint32_t len) = 0; + + /// consumeEnd, invoked when all data has been consumed. + virtual void consumeEnd() { } + /** + * Slow path borrow. + * + * POSTCONDITION: return == NULL || rBound_ - rBase_ >= *len + */ + virtual const uint8_t* borrowSlow(uint8_t* buf, uint32_t* len) = 0; + + /** + * Trivial constructor. + * + * Initialize pointers safely. Constructing is not a very + * performance-sensitive operation, so it is okay to just leave it to + * the concrete class to set up pointers correctly. + */ + TBufferBase() + : rBase_(NULL) + , rBound_(NULL) + , wBase_(NULL) + , wBound_(NULL) + {} + + /// Convenience mutator for setting the read buffer. + void setReadBuffer(uint8_t* buf, uint32_t len) { + rBase_ = buf; + rBound_ = buf+len; + } + + /// Convenience mutator for setting the write buffer. + void setWriteBuffer(uint8_t* buf, uint32_t len) { + wBase_ = buf; + wBound_ = buf+len; + } + + virtual ~TBufferBase() {} + + /// Reads begin here. + uint8_t* rBase_; + /// Reads may extend to just before here. + uint8_t* rBound_; + + /// Writes begin here. + uint8_t* wBase_; + /// Writes may extend to just before here. + uint8_t* wBound_; +}; + + +/** + * Buffered transport. For reads it will read more data than is requested + * and will serve future data out of a local buffer. For writes, data is + * stored to an in memory buffer before being written out. + * + */ +class TBufferedTransport + : public TVirtualTransport { + public: + + static const int DEFAULT_BUFFER_SIZE = 512; + + /// Use default buffer sizes. + explicit TBufferedTransport(boost::shared_ptr transport) + : transport_(transport) + , rBufSize_(DEFAULT_BUFFER_SIZE) + , wBufSize_(DEFAULT_BUFFER_SIZE) + , rBuf_(new uint8_t[rBufSize_]) + , wBuf_(new uint8_t[wBufSize_]) + { + initPointers(); + } + + /// Use specified buffer sizes. + TBufferedTransport(boost::shared_ptr transport, uint32_t sz) + : transport_(transport) + , rBufSize_(sz) + , wBufSize_(sz) + , rBuf_(new uint8_t[rBufSize_]) + , wBuf_(new uint8_t[wBufSize_]) + { + initPointers(); + } + + /// Use specified read and write buffer sizes. + TBufferedTransport(boost::shared_ptr transport, uint32_t rsz, uint32_t wsz) + : transport_(transport) + , rBufSize_(rsz) + , wBufSize_(wsz) + , rBuf_(new uint8_t[rBufSize_]) + , wBuf_(new uint8_t[wBufSize_]) + { + initPointers(); + } + + // Tries to put some data back in the beginning of the read buffer. + void putBack(uint8_t* buf, uint32_t len); + + void open() { + transport_->open(); + } + + bool isOpen() { + return transport_->isOpen(); + } + + bool peek() { + if (rBase_ == rBound_) { + setReadBuffer(rBuf_.get(), transport_->read(rBuf_.get(), rBufSize_)); + } + return (rBound_ > rBase_); + } + + void close() { + flush(); + transport_->close(); + } + + virtual uint32_t readSlow(uint8_t* buf, uint32_t len); + + virtual void writeSlow(const uint8_t* buf, uint32_t len); + + void flush(); + + + /** + * The following behavior is currently implemented by TBufferedTransport, + * but that may change in a future version: + * 1/ If len is at most rBufSize_, borrow will never return NULL. + * Depending on the underlying transport, it could throw an exception + * or hang forever. + * 2/ Some borrow requests may copy bytes internally. However, + * if len is at most rBufSize_/2, none of the copied bytes + * will ever have to be copied again. For optimal performance, + * stay under this limit. + */ + virtual const uint8_t* borrowSlow(uint8_t* buf, uint32_t* len); + + boost::shared_ptr getUnderlyingTransport() { + return transport_; + } + + /* + * TVirtualTransport provides a default implementation of readAll(). + * We want to use the TBufferBase version instead. + */ + using TBufferBase::readAll; + + protected: + void initPointers() { + setReadBuffer(rBuf_.get(), 0); + setWriteBuffer(wBuf_.get(), wBufSize_); + // Write size never changes. + } + + boost::shared_ptr transport_; + + uint32_t rBufSize_; + uint32_t wBufSize_; + boost::scoped_array rBuf_; + boost::scoped_array wBuf_; +}; + + +/** + * Wraps a transport into a buffered one. + * + */ +class TBufferedTransportFactory : public TTransportFactory { + public: + TBufferedTransportFactory() {} + + virtual ~TBufferedTransportFactory() {} + + /** + * Wraps the transport into a buffered one. + */ + virtual boost::shared_ptr getTransport(boost::shared_ptr trans) { + return boost::shared_ptr(new TBufferedTransport(trans)); + } + +}; + + +/** + * Framed transport. All writes go into an in-memory buffer until flush is + * called, at which point the transport writes the length of the entire + * binary chunk followed by the data payload. This allows the receiver on the + * other end to always do fixed-length reads. + * + */ +class TFramedTransport + : public TVirtualTransport { + + public: + + static const int DEFAULT_BUFFER_SIZE = 512; + + + /// Use default buffer sizes. + explicit TFramedTransport(boost::shared_ptr transport) + : transport_(transport) + , rBufSize_(0) + , wBufSize_(DEFAULT_BUFFER_SIZE) + , rBuf_() + , wBuf_(new uint8_t[wBufSize_]) + { + initPointers(); + } + + TFramedTransport(boost::shared_ptr transport, uint32_t sz) + : transport_(transport) + , rBufSize_(0) + , wBufSize_(sz) + , rBuf_() + , wBuf_(new uint8_t[wBufSize_]) + { + initPointers(); + } + + void open() { + transport_->open(); + } + + bool isOpen() { + return transport_->isOpen(); + } + + bool peek() { + return (rBase_ < rBound_) || transport_->peek(); + } + + void close() { + flush(); + transport_->close(); + } + + virtual uint32_t readSlow(uint8_t* buf, uint32_t len); + + virtual void writeSlow(const uint8_t* buf, uint32_t len); + + virtual void flush(); + + uint32_t readEnd(); + + uint32_t writeEnd(); + + const uint8_t* borrowSlow(uint8_t* buf, uint32_t* len); + + boost::shared_ptr getUnderlyingTransport() { + return transport_; + } + + /* + * TVirtualTransport provides a default implementation of readAll(). + * We want to use the TBufferBase version instead. + */ + using TBufferBase::readAll; + + protected: + /// Constructor for subclassing. + TFramedTransport() + : rBufSize_(0) + , wBufSize_(DEFAULT_BUFFER_SIZE) + , rBuf_() + , wBuf_(new uint8_t[wBufSize_]) + { + initPointers(); + } + + /** + * Reads a frame of input from the underlying stream. + * + * Returns true if a frame was read successfully, or false on EOF. + * (Raises a TTransportException if EOF occurs after a partial frame.) + * + * @param req_sz The size of the requested data. readFrame may read more + * than this amount, but should not read less. + */ + virtual bool readFrame(uint32_t min_frame_sz); + + void initPointers() { + setReadBuffer(NULL, 0); + setWriteBuffer(wBuf_.get(), wBufSize_); + + // Pad the buffer so we can insert the size later. + int32_t pad = 0; + this->write((uint8_t*)&pad, sizeof(pad)); + } + + boost::shared_ptr transport_; + + uint32_t rBufSize_; + uint32_t wBufSize_; + boost::scoped_array rBuf_; + boost::scoped_array wBuf_; +}; + +/** + * Wraps a transport into a framed one. + * + */ +class TFramedTransportFactory : public TTransportFactory { + public: + TFramedTransportFactory() {} + + virtual ~TFramedTransportFactory() {} + + /** + * Wraps the transport into a framed one. + */ + virtual boost::shared_ptr getTransport(boost::shared_ptr trans) { + return boost::shared_ptr(new TFramedTransport(trans)); + } + +}; + + +/** + * A memory buffer is a transport that simply reads from and writes to an + * in memory buffer. Anytime you call write on it, the data is simply placed + * into a buffer, and anytime you call read, data is read from that buffer. + * + * The buffers are allocated using C constructs malloc,realloc, and the size + * doubles as necessary. We've considered using scoped + * + */ +class TMemoryBuffer : public TVirtualTransport { + private: + + TMemoryBuffer(const TMemoryBuffer&); + TMemoryBuffer &operator=(const TMemoryBuffer&); + + // Common initialization done by all constructors. + void initCommon(uint8_t* buf, uint32_t size, bool owner, uint32_t wPos) { + if (buf == NULL && size != 0) { + assert(owner); + buf = (uint8_t*)std::malloc(size); + if (buf == NULL) { + throw std::bad_alloc(); + } + } + + buffer_ = buf; + bufferSize_ = size; + + rBase_ = buffer_; + rBound_ = buffer_ + wPos; + // TODO(dreiss): Investigate NULL-ing this if !owner. + wBase_ = buffer_ + wPos; + wBound_ = buffer_ + bufferSize_; + + owner_ = owner; + + linkedBuffer_ = NULL; + observerCount_ = 0; + // rBound_ is really an artifact. In principle, it should always be + // equal to wBase_. We update it in a few places (computeRead, etc.). + } + + public: + static const uint32_t defaultSize = 1024; + + /** + * This enum specifies how a TMemoryBuffer should treat + * memory passed to it via constructors or resetBuffer. + * + * OBSERVE: + * TMemoryBuffer will simply store a pointer to the memory. + * It is the callers responsibility to ensure that the pointer + * remains valid for the lifetime of the TMemoryBuffer, + * and that it is properly cleaned up. + * Note that no data can be written to observed buffers. + * + * COPY: + * TMemoryBuffer will make an internal copy of the buffer. + * The caller has no responsibilities. + * + * TAKE_OWNERSHIP: + * TMemoryBuffer will become the "owner" of the buffer, + * and will be responsible for freeing it. + * The memory must have been allocated with malloc. + */ + enum MemoryPolicy + { OBSERVE = 1 + , COPY = 2 + , TAKE_OWNERSHIP = 3 + }; + + /** + * Construct a TMemoryBuffer with a default-sized buffer, + * owned by the TMemoryBuffer object. + */ + TMemoryBuffer() { + initCommon(NULL, defaultSize, true, 0); + } + + /** + * Construct a TMemoryBuffer with a buffer of a specified size, + * owned by the TMemoryBuffer object. + * + * @param sz The initial size of the buffer. + */ + explicit TMemoryBuffer(uint32_t sz) { + initCommon(NULL, sz, true, 0); + } + + /** + * Construct a TMemoryBuffer with buf as its initial contents. + * + * @param buf The initial contents of the buffer. + * Note that, while buf is a non-const pointer, + * TMemoryBuffer will not write to it if policy == OBSERVE, + * so it is safe to const_cast(whatever). + * @param sz The size of @c buf. + * @param policy See @link MemoryPolicy @endlink . + */ + TMemoryBuffer(uint8_t* buf, uint32_t sz, MemoryPolicy policy = OBSERVE) { + if (buf == NULL && sz != 0) { + throw TTransportException(TTransportException::BAD_ARGS, + "TMemoryBuffer given null buffer with non-zero size."); + } + + switch (policy) { + case OBSERVE: + case TAKE_OWNERSHIP: + initCommon(buf, sz, policy == TAKE_OWNERSHIP, sz); + break; + case COPY: + initCommon(NULL, sz, true, 0); + this->write(buf, sz); + break; + default: + throw TTransportException(TTransportException::BAD_ARGS, + "Invalid MemoryPolicy for TMemoryBuffer"); + } + } + + explicit TMemoryBuffer(TMemoryBuffer *buffer) { + initCommon(buffer->rBase_, buffer->available_read(), false, + buffer->available_read()); + // Have to set buffer_ appropriately so this buffer can take ownership + // later if necessary. initCommon sets all other state correctly. + buffer_ = buffer->buffer_; + linkedBuffer_ = buffer; + linkedBuffer_->observe(this); + } + + ~TMemoryBuffer() { + cleanup(); + } + + // Set this buffer to observe the next length bytes of buffer. + void link(TMemoryBuffer *buffer, uint32_t length) { + assert(length <= buffer->available_read()); + cleanup(); + initCommon(buffer->rBase_, length, false, length); + buffer_ = buffer->buffer_; + linkedBuffer_ = buffer; + linkedBuffer_->observe(this); + } + + bool isOpen() { + return true; + } + + bool peek() { + return (rBase_ < wBase_); + } + + void open() {} + + void close() {} + + uint32_t getBufferSize() const { return bufferSize_; } + + // TODO(dreiss): Make bufPtr const. + void getBuffer(uint8_t** bufPtr, uint32_t* sz) { + *bufPtr = rBase_; + *sz = wBase_ - rBase_; + } + + std::string getBufferAsString() { + if (buffer_ == NULL) { + return ""; + } + uint8_t* buf; + uint32_t sz; + getBuffer(&buf, &sz); + return std::string((char*)buf, (std::string::size_type)sz); + } + + void appendBufferToString(std::string& str) { + if (buffer_ == NULL) { + return; + } + uint8_t* buf; + uint32_t sz; + getBuffer(&buf, &sz); + str.append((char*)buf, sz); + } + + void resetBuffer() { + rBase_ = buffer_; + rBound_ = buffer_; + wBase_ = buffer_; + // It isn't safe to write into a buffer we don't own. + if (!owner_) { + wBound_ = wBase_; + bufferSize_ = 0; + } + } + + // Register an observer of the memory in this buffer. + // + // A TMemoryBuffer with one or more observers has special handling + // for operations that would invalidate the data being observed. If + // an operation would invalidate the buffer pointer (eg: realloc, free), + // ownership of that buffer is first transferred to one of the observing + // TMemoryBuffers. + // + // Calls that reset an apparently empty buffer are delayed until all + // observers are detached. + void observe(TMemoryBuffer *observer) { + if (observer != NULL) { + observers_.push_back(observer); + } + observerCount_++; + } + + // Remove an observer of the memory in this buffer. If there is a pending + // consumeEnd call, process it + void unobserve(TMemoryBuffer *observer) { + if (UNLIKELY(observerCount_ == 0)) { + throw TTransportException("Tried to unobserve a buffer with no " + "observers"); + } + bool found = false; + if (observer != NULL) { + for(TMemoryBufferContainer::iterator it = observers_.begin(); + it != observers_.end(); ++it) { + if (*it == observer) { + observers_.erase(it); + found = true; + break; + } + } + } + if (!found) { + throw TTransportException("Tried to remove an observer that was not " + "observing the buffer"); + } + observerCount_--; + if (observerCount_ == 0 && rBase_ == wBase_) { + consumeEnd(); + } + } + + // Remove link to an observed memory buffer. If the buffer is not linked + // this is a no-op. + void unlink() { + if (linkedBuffer_ != NULL) { + linkedBuffer_->unobserve(this); + linkedBuffer_ = NULL; + } + } + + /// See constructor documentation. + void resetBuffer(uint8_t* buf, uint32_t sz, MemoryPolicy policy = OBSERVE) { + // Use a variant of the copy-and-swap trick for assignment operators. + // This is sub-optimal in terms of performance for two reasons: + // 1/ The constructing and swapping of the (small) values + // in the temporary object takes some time, and is not necessary. + // 2/ If policy == COPY, we allocate the new buffer before + // freeing the old one, precluding the possibility of + // reusing that memory. + // I doubt that either of these problems could be optimized away, + // but the second is probably no a common case, and the first is minor. + // I don't expect resetBuffer to be a common operation, so I'm willing to + // bite the performance bullet to make the method this simple. + + // Construct the new buffer. + TMemoryBuffer new_buffer(buf, sz, policy); + // Move it into ourself. + this->swap(new_buffer); + // Our old self gets destroyed. + } + + /// See constructor documentation. + void resetBuffer(uint32_t sz) { + // Construct the new buffer. + TMemoryBuffer new_buffer(sz); + // Move it into ourself. + this->swap(new_buffer); + // Our old self gets destroyed. + } + + std::string readAsString(uint32_t len) { + std::string str; + (void)readAppendToString(str, len); + return str; + } + + uint32_t readAppendToString(std::string& str, uint32_t len); + + // return number of bytes read + uint32_t readEnd() { + uint32_t bytes = rBase_ - buffer_; + if (rBase_ == wBase_) { + resetBuffer(); + } + return bytes; + } + + // Return number of bytes written + uint32_t writeEnd() { + return wBase_ - buffer_; + } + + uint32_t available_read() const { + // Remember, wBase_ is the real rBound_. + return wBase_ - rBase_; + } + + uint32_t available_write() const { + return wBound_ - wBase_; + } + + // Returns a pointer to where the client can write data to append to + // the TMemoryBuffer, and ensures the buffer is big enough to accommodate a + // write of the provided length. The returned pointer is very convenient for + // passing to read(), recv(), or similar. You must call wroteBytes() as soon + // as data is written or the buffer will not be aware that data has changed. + uint8_t* getWritePtr(uint32_t len) { + ensureCanWrite(len); + return wBase_; + } + + // Informs the buffer that the client has written 'len' bytes into storage + // that had been provided by getWritePtr(). + void wroteBytes(uint32_t len); + + /* + * TVirtualTransport provides a default implementation of readAll(). + * We want to use the TBufferBase version instead. + */ + using TBufferBase::readAll; + + /** + * Extract the read buffer from the TMemoryBuffer, or make a copy if this + * buffer does not own it. It's not safe to assume that a buffer created + * with TAKE_OWNERSHIP or COPY is still the owner of it's underlying buffer, + * because some TMemoryBuffer APIs (resetBuffer, link) change the ownership. + * Because of that, it's more consistent to make a copy rather than throw. + * + * The TMemoryBuffer's internal buffer is returned, and the caller is + * given ownership of the buffer. The caller is responsible for eventually + * freeing the buffer using free(). + * + * The TMemoryBuffer itself will be reset and will allocate a new buffer the + * next time data is written into it. + * + * @param buf The returned buffer pointer is stored at the location + * specified by this argument. + * @param buflen This parameter is used to return the number of bytes of + * readable data available at the start of the returned buffer. + * (Note that this is the amount of available data, not the + * buffer capacity.) + */ + void extractReadBuffer(uint8_t** buf, uint32_t* buflen) { + *buflen = available_read(); + if (owner_) { + if (rBase_ != buffer_) { + memmove(buffer_, rBase_, *buflen); + } + *buf = buffer_; + } else { + *buf = (uint8_t *)malloc(*buflen); + if (*buf == NULL) { + throw std::bad_alloc(); + } + memcpy(*buf, rBase_, *buflen); + cleanup(); + } + initCommon(NULL, 0, true, 0); + } + + protected: + void swap(TMemoryBuffer& that) { + using std::swap; + swap(buffer_, that.buffer_); + swap(bufferSize_, that.bufferSize_); + + swap(rBase_, that.rBase_); + swap(rBound_, that.rBound_); + swap(wBase_, that.wBase_); + swap(wBound_, that.wBound_); + + swap(owner_, that.owner_); + swap(linkedBuffer_, that.linkedBuffer_); + swap(observerCount_, that.observerCount_); + observers_.swap(that.observers_); + for (auto buf: observers_) { + buf->linkedBuffer_ = this; + } + for (auto buf: that.observers_) { + buf->linkedBuffer_ = &that; + } + if (linkedBuffer_ != NULL) { + linkedBuffer_->unobserve(&that); + linkedBuffer_->observe(this); + } + if (that.linkedBuffer_ != NULL) { + that.linkedBuffer_->unobserve(this); + that.linkedBuffer_->observe(&that); + } + } + + void transferOwnership() { + TMemoryBuffer *newOwner = NULL; + assert(!observers_.empty()); + // The optimal case is to make the new owner the TMemoryBuffer + // with the longest life-span. Use FIFO as a hueristic and assign + // the new owner to the last registered observer. + newOwner = observers_.back(); + observers_.pop_back(); + newOwner->owner_ = true; + newOwner->linkedBuffer_ = NULL; + for (TMemoryBufferContainer::iterator it = observers_.begin(); + it != observers_.end(); ++it) { + (*it)->linkedBuffer_ = newOwner; + // It's possible that an observer has other observers + newOwner->observers_.push_back(*it); + newOwner->observerCount_++; + } + observers_.clear(); + observerCount_ = 0; + } + + void cleanup() { + if (owner_) { + if (observerCount_ > 0) { + transferOwnership(); + } else { + std::free(buffer_); + } + } + if (linkedBuffer_ != NULL) { + linkedBuffer_->unobserve(this); + } + } + + void consumeEnd() { + if (observerCount_ == 0) { + resetBuffer(); + } + } + + // Make sure there's at least 'len' bytes available for writing. + void ensureCanWrite(uint32_t len); + + // Compute the position and available data for reading. + void computeRead(uint32_t len, uint8_t** out_start, uint32_t* out_give); + + uint32_t readSlow(uint8_t* buf, uint32_t len); + + void writeSlow(const uint8_t* buf, uint32_t len); + + const uint8_t* borrowSlow(uint8_t* buf, uint32_t* len); + + // Data buffer + uint8_t* buffer_; + + // Allocated buffer size + uint32_t bufferSize_; + + // Is this object the owner of the buffer? + bool owner_; + + TMemoryBuffer *linkedBuffer_; + uint32_t observerCount_; + // The container is a vector (rather than a list) for performance + // reasons despite the O(N) penalty in unobserve. The vec keeps the + // overhead lower in the common case, which is a single observer. + typedef std::vector TMemoryBufferContainer; + TMemoryBufferContainer observers_; + // Don't forget to update constrctors, initCommon, and swap if + // you add new members. +}; + +}}} // apache::thrift::transport + +#endif // #ifndef THRIFT_TRANSPORT_TBUFFERTRANSPORTS_H_ diff --git a/thrift/lib/cpp/transport/TFDTransport.h b/thrift/lib/cpp/transport/TFDTransport.h new file mode 100644 index 0000000000..1d1a3e6842 --- /dev/null +++ b/thrift/lib/cpp/transport/TFDTransport.h @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_TRANSPORT_TFDTRANSPORT_H_ +#define _THRIFT_TRANSPORT_TFDTRANSPORT_H_ 1 + +#include +#include + +#include "TTransport.h" +#include "TVirtualTransport.h" + +namespace apache { namespace thrift { namespace transport { + +/** + * Dead-simple wrapper around a file descriptor. + * + */ +class TFDTransport : public TVirtualTransport { + public: + enum ClosePolicy + { NO_CLOSE_ON_DESTROY = 0 + , CLOSE_ON_DESTROY = 1 + }; + + TFDTransport(int fd, ClosePolicy close_policy = NO_CLOSE_ON_DESTROY) + : fd_(fd) + , close_policy_(close_policy) + {} + + ~TFDTransport() { + if (close_policy_ == CLOSE_ON_DESTROY) { + close(); + } + } + + bool isOpen() { return fd_ >= 0; } + + void open() {} + + void close(); + + uint32_t read(uint8_t* buf, uint32_t len); + + void write(const uint8_t* buf, uint32_t len); + + void setFD(int fd) { fd_ = fd; } + int getFD() { return fd_; } + + protected: + int fd_; + ClosePolicy close_policy_; +}; + +}}} // apache::thrift::transport + +#endif // #ifndef _THRIFT_TRANSPORT_TFDTRANSPORT_H_ diff --git a/thrift/lib/cpp/transport/TFileTransport.h b/thrift/lib/cpp/transport/TFileTransport.h new file mode 100644 index 0000000000..381e7dc20d --- /dev/null +++ b/thrift/lib/cpp/transport/TFileTransport.h @@ -0,0 +1,462 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_TRANSPORT_TFILETRANSPORT_H_ +#define _THRIFT_TRANSPORT_TFILETRANSPORT_H_ 1 + +#include "thrift/lib/cpp/transport/TTransport.h" +#include "thrift/lib/cpp/Thrift.h" +#include "thrift/lib/cpp/TProcessor.h" + +#include +#include + +#include +#include +#include + +namespace apache { namespace thrift { namespace transport { + +using apache::thrift::TProcessor; +using apache::thrift::protocol::TProtocolFactory; + +// Data pertaining to a single event +typedef struct eventInfo { + uint8_t* eventBuff_; + uint32_t eventSize_; + uint32_t eventBuffPos_; + + eventInfo():eventBuff_(NULL), eventSize_(0), eventBuffPos_(0){}; + ~eventInfo() { + if (eventBuff_) { + delete[] eventBuff_; + } + } +} eventInfo; + +// information about current read state +typedef struct readState { + eventInfo* event_; + + // keep track of event size + uint8_t eventSizeBuff_[4]; + uint8_t eventSizeBuffPos_; + bool readingSize_; + + // read buffer variables + int32_t bufferPtr_; + int32_t bufferLen_; + + // last successful dispatch point + int32_t lastDispatchPtr_; + + void resetState(uint32_t lastDispatchPtr) { + readingSize_ = true; + eventSizeBuffPos_ = 0; + lastDispatchPtr_ = lastDispatchPtr; + } + + void resetAllValues() { + resetState(0); + bufferPtr_ = 0; + bufferLen_ = 0; + if (event_) { + delete(event_); + } + event_ = 0; + } + + readState() { + event_ = 0; + resetAllValues(); + } + + ~readState() { + if (event_) { + delete(event_); + } + } + +} readState; + +/** + * TFileTransportBuffer - buffer class used by TFileTransport for queuing up events + * to be written to disk. Should be used in the following way: + * 1) Buffer created + * 2) Buffer written to (addEvent) + * 3) Buffer read from (getNext) + * 4) Buffer reset (reset) + * 5) Go back to 2, or destroy buffer + * + * The buffer should never be written to after it is read from, unless it is reset first. + * Note: The above rules are enforced mainly for debugging its sole client TFileTransport + * which uses the buffer in this way. + * + */ +class TFileTransportBuffer { + public: + TFileTransportBuffer(uint32_t size); + ~TFileTransportBuffer(); + + bool addEvent(eventInfo *event); + eventInfo* getNext(); + void reset(); + bool isFull(); + bool isEmpty(); + + private: + TFileTransportBuffer(); // should not be used + + enum mode { + WRITE, + READ + }; + mode bufferMode_; + + uint32_t writePoint_; + uint32_t readPoint_; + uint32_t size_; + eventInfo** buffer_; +}; + +/** + * Abstract interface for transports used to read files + */ +class TFileReaderTransport : virtual public TTransport { + public: + virtual int32_t getReadTimeout() = 0; + virtual void setReadTimeout(int32_t readTimeout) = 0; + + virtual uint32_t getNumChunks() = 0; + virtual uint32_t getCurChunk() = 0; + virtual void seekToChunk(int32_t chunk) = 0; + virtual void seekToEnd() = 0; +}; + +/** + * Abstract interface for transports used to write files + */ +class TFileWriterTransport : virtual public TTransport { + public: + virtual uint32_t getChunkSize() = 0; + virtual void setChunkSize(uint32_t chunkSize) = 0; +}; + +/** + * File implementation of a transport. Reads and writes are done to a + * file on disk. + * + */ +class TFileTransport : public TFileReaderTransport, + public TFileWriterTransport { + public: + TFileTransport(std::string path, bool readOnly=false); + ~TFileTransport(); + + // TODO: what is the correct behavior for this? + // the log file is generally always open + bool isOpen() { + return true; + } + + void write(const uint8_t* buf, uint32_t len); + void flush(); + + uint32_t readAll(uint8_t* buf, uint32_t len); + bool peek(); + uint32_t read(uint8_t* buf, uint32_t len); + + // log-file specific functions + void seekToChunk(int32_t chunk); + void seekToEnd(); + uint32_t getNumChunks(); + uint32_t getCurChunk(); + + // for changing the output file + void resetOutputFile(int fd, std::string filename, int64_t offset); + + // Setter/Getter functions for user-controllable options + void setReadBuffSize(uint32_t readBuffSize) { + if (readBuffSize) { + readBuffSize_ = readBuffSize; + } + } + uint32_t getReadBuffSize() { + return readBuffSize_; + } + + static const int32_t TAIL_READ_TIMEOUT = -1; + static const int32_t NO_TAIL_READ_TIMEOUT = 0; + void setReadTimeout(int32_t readTimeout) { + readTimeout_ = readTimeout; + } + int32_t getReadTimeout() { + return readTimeout_; + } + + void setChunkSize(uint32_t chunkSize) { + if (chunkSize) { + chunkSize_ = chunkSize; + } + } + uint32_t getChunkSize() { + return chunkSize_; + } + + void setEventBufferSize(uint32_t bufferSize) { + if (bufferAndThreadInitialized_) { + GlobalOutput("Cannot change the buffer size after writer thread started"); + return; + } + eventBufferSize_ = bufferSize; + } + + uint32_t getEventBufferSize() { + return eventBufferSize_; + } + + void setFlushMaxUs(uint32_t flushMaxUs) { + if (flushMaxUs) { + flushMaxUs_ = flushMaxUs; + } + } + uint32_t getFlushMaxUs() { + return flushMaxUs_; + } + + void setFlushMaxBytes(uint32_t flushMaxBytes) { + if (flushMaxBytes) { + flushMaxBytes_ = flushMaxBytes; + } + } + uint32_t getFlushMaxBytes() { + return flushMaxBytes_; + } + + void setMaxEventSize(uint32_t maxEventSize) { + maxEventSize_ = maxEventSize; + } + uint32_t getMaxEventSize() { + return maxEventSize_; + } + + void setMaxCorruptedEvents(uint32_t maxCorruptedEvents) { + maxCorruptedEvents_ = maxCorruptedEvents; + } + uint32_t getMaxCorruptedEvents() { + return maxCorruptedEvents_; + } + + void setEofSleepTimeUs(uint32_t eofSleepTime) { + if (eofSleepTime) { + eofSleepTime_ = eofSleepTime; + } + } + uint32_t getEofSleepTimeUs() { + return eofSleepTime_; + } + + /* + * Override TTransport *_virt() functions to invoke our implementations. + * We cannot use TVirtualTransport to provide these, since we need to inherit + * virtually from TTransport. + */ + virtual uint32_t read_virt(uint8_t* buf, uint32_t len) { + return this->read(buf, len); + } + virtual uint32_t readAll_virt(uint8_t* buf, uint32_t len) { + return this->readAll(buf, len); + } + virtual void write_virt(const uint8_t* buf, uint32_t len) { + this->write(buf, len); + } + + private: + // helper functions for writing to a file + void enqueueEvent(const uint8_t* buf, uint32_t eventLen); + bool swapEventBuffers(struct timespec* deadline); + bool initBufferAndWriteThread(); + + // control for writer thread + static void* startWriterThread(void* ptr) { + (((TFileTransport*)ptr)->writerThread()); + return 0; + } + void writerThread(); + + // helper functions for reading from a file + eventInfo* readEvent(); + + // event corruption-related functions + bool isEventCorrupted(); + void performRecovery(); + + // Utility functions + void openLogFile(); + void getNextFlushTime(struct timespec* ts_next_flush); + + // Class variables + readState readState_; + uint8_t* readBuff_; + eventInfo* currentEvent_; + + uint32_t readBuffSize_; + static const uint32_t DEFAULT_READ_BUFF_SIZE = 1 * 1024 * 1024; + + int32_t readTimeout_; + static const int32_t DEFAULT_READ_TIMEOUT_MS = 200; + + // size of chunks that file will be split up into + uint32_t chunkSize_; + static const uint32_t DEFAULT_CHUNK_SIZE = 16 * 1024 * 1024; + + // size of event buffers + uint32_t eventBufferSize_; + static const uint32_t DEFAULT_EVENT_BUFFER_SIZE = 10000; + + // max number of microseconds that can pass without flushing + uint32_t flushMaxUs_; + static const uint32_t DEFAULT_FLUSH_MAX_US = 3000000; + + // max number of bytes that can be written without flushing + uint32_t flushMaxBytes_; + static const uint32_t DEFAULT_FLUSH_MAX_BYTES = 1000 * 1024; + + // max event size + uint32_t maxEventSize_; + static const uint32_t DEFAULT_MAX_EVENT_SIZE = 0; + + // max number of corrupted events per chunk + uint32_t maxCorruptedEvents_; + static const uint32_t DEFAULT_MAX_CORRUPTED_EVENTS = 0; + + // sleep duration when EOF is hit + uint32_t eofSleepTime_; + static const uint32_t DEFAULT_EOF_SLEEP_TIME_US = 500 * 1000; + + // sleep duration when a corrupted event is encountered + uint32_t corruptedEventSleepTime_; + static const uint32_t DEFAULT_CORRUPTED_SLEEP_TIME_US = 1 * 1000 * 1000; + + // sleep duration in seconds when an IO error is encountered in the writer thread + uint32_t writerThreadIOErrorSleepTime_; + static const uint32_t DEFAULT_WRITER_THREAD_SLEEP_TIME_US = 60 * 1000 * 1000; + + // writer thread id + pthread_t writerThreadId_; + + // buffers to hold data before it is flushed. Each element of the buffer stores a msg that + // needs to be written to the file. The buffers are swapped by the writer thread. + TFileTransportBuffer *dequeueBuffer_; + TFileTransportBuffer *enqueueBuffer_; + + // conditions used to block when the buffer is full or empty + pthread_cond_t notFull_, notEmpty_; + volatile bool closing_; + + // To keep track of whether the buffer has been flushed + pthread_cond_t flushed_; + bool forceFlush_; + + // Mutex that is grabbed when enqueuing and swapping the read/write buffers + pthread_mutex_t mutex_; + + // File information + std::string filename_; + int fd_; + + // Whether the writer thread and buffers have been initialized + bool bufferAndThreadInitialized_; + + // Offset within the file + off_t offset_; + + // event corruption information + uint32_t lastBadChunk_; + uint32_t numCorruptedEventsInChunk_; + + bool readOnly_; +}; + +// Exception thrown when EOF is hit +class TEOFException : public TTransportException { + public: + TEOFException(): + TTransportException(TTransportException::END_OF_FILE) {}; +}; + + +// wrapper class to process events from a file containing thrift events +class TFileProcessor { + public: + /** + * Constructor that defaults output transport to null transport + * + * @param processor processes log-file events + * @param protocolFactory protocol factory + * @param inputTransport file transport + */ + TFileProcessor(boost::shared_ptr processor, + boost::shared_ptr protocolFactory, + boost::shared_ptr inputTransport); + + TFileProcessor(boost::shared_ptr processor, + boost::shared_ptr inputProtocolFactory, + boost::shared_ptr outputProtocolFactory, + boost::shared_ptr inputTransport); + + /** + * Constructor + * + * @param processor processes log-file events + * @param protocolFactory protocol factory + * @param inputTransport input file transport + * @param output output transport + */ + TFileProcessor(boost::shared_ptr processor, + boost::shared_ptr protocolFactory, + boost::shared_ptr inputTransport, + boost::shared_ptr outputTransport); + + /** + * processes events from the file + * + * @param numEvents number of events to process (0 for unlimited) + * @param tail tails the file if true + */ + void process(uint32_t numEvents, bool tail); + + /** + * process events until the end of the chunk + * + */ + void processChunk(); + + private: + boost::shared_ptr processor_; + boost::shared_ptr inputProtocolFactory_; + boost::shared_ptr outputProtocolFactory_; + boost::shared_ptr inputTransport_; + boost::shared_ptr outputTransport_; +}; + + +}}} // apache::thrift::transport + +#endif // _THRIFT_TRANSPORT_TFILETRANSPORT_H_ diff --git a/thrift/lib/cpp/transport/THDFSFileTransport.h b/thrift/lib/cpp/transport/THDFSFileTransport.h new file mode 100644 index 0000000000..64414fa83a --- /dev/null +++ b/thrift/lib/cpp/transport/THDFSFileTransport.h @@ -0,0 +1,52 @@ +// Copyright (c) 2009- Facebook +// Distributed under the Thrift Software License +// +// See accompanying file LICENSE or visit the Thrift site at: +// http://developers.facebook.com/thrift/ + +#ifndef _THRIFT_TRANSPORT_THDFSFILETRANSPORT_H_ +#define _THRIFT_TRANSPORT_THDFSFILETRANSPORT_H_ + +#include "thrift/lib/cpp/transport/TTransport.h" +#include "thrift/lib/cpp/transport/TVirtualTransport.h" +#include "HDFS.h" +#include + +#include +#include + +namespace apache { namespace thrift { namespace transport { +/** + * Dead-simple wrapper around libhdfs. + * THDFSFileTransport only takes care of read/write, + * and leaves allocation/release to HDFS and HDFSFile. + * @author Li Zhang + */ +class THDFSFileTransport : public TVirtualTransport { + public: + + THDFSFileTransport(boost::shared_ptr hdfsFile) : hdfsFile_(hdfsFile) { + } + + ~THDFSFileTransport() { + } + + bool isOpen() { + return hdfsFile_->isOpen(); + } + + void open(); + + void close(); + + uint32_t read(uint8_t* buf, uint32_t len); + + void write(const uint8_t* buf, uint32_t len); + + protected: + boost::shared_ptr hdfsFile_; +}; + +}}} // apache::thrift::transport + +#endif // _THRIFT_TRANSPORT_THDFSFILETRANSPORT_H_ diff --git a/thrift/lib/cpp/transport/THeader.h b/thrift/lib/cpp/transport/THeader.h new file mode 100644 index 0000000000..62e13d7eaa --- /dev/null +++ b/thrift/lib/cpp/transport/THeader.h @@ -0,0 +1,286 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef THRIFT_TRANSPORT_THEADER_H_ +#define THRIFT_TRANSPORT_THEADER_H_ 1 + +#include + +#include "thrift/lib/cpp/protocol/TBinaryProtocol.h" +#include "thrift/lib/cpp/protocol/TProtocolTypes.h" + +#include "folly/experimental/io/IOBuf.h" +#include "folly/experimental/io/IOBufQueue.h" + +#include +#include "boost/scoped_array.hpp" +#include +#include + +// Don't include the unknown client. +#define CLIENT_TYPES_LEN 4 + +enum CLIENT_TYPE { + THRIFT_HEADER_CLIENT_TYPE = 0, + THRIFT_FRAMED_DEPRECATED = 1, + THRIFT_UNFRAMED_DEPRECATED = 2, + THRIFT_HTTP_CLIENT_TYPE = 3, + THRIFT_UNKNOWN_CLIENT_TYPE = 4, +}; + +namespace apache { namespace thrift { namespace transport { + +using apache::thrift::protocol::T_COMPACT_PROTOCOL; + +/** + * Class that will take an IOBuf and wrap it in some thrift headers. + * see thrift/doc/HeaderFormat.txt for details. + * + * Supports transforms: zlib snappy hmac + * Supports headers: http-style key/value per request and per connection + * other: Protocol Id and seq ID in header. + * + * Backwards compatible with TFramed format, and unframed format, assuming + * your server transport is compatible (some server types require 4-byte size + * at the start, such as TNonblockingServer). + */ +class THeader { + public: + + virtual ~THeader() {} + + explicit THeader() + : queue_(new folly::IOBufQueue) + , protoId(T_COMPACT_PROTOCOL) + , clientType(THRIFT_HEADER_CLIENT_TYPE) + , seqId(0) + , flags(0) + , identity(s_identity) + { + initSupportedClients(NULL); + } + + explicit THeader(std::bitset const* clientTypes) + : queue_(new folly::IOBufQueue) + , protoId(T_COMPACT_PROTOCOL) + , clientType(THRIFT_HEADER_CLIENT_TYPE) + , seqId(0) + , flags(0) + , identity(s_identity) + { + initSupportedClients(clientTypes); + } + + uint16_t getProtocolId() const; + void setProtocolId(uint16_t protoId) { this->protoId = protoId; } + + virtual void resetProtocol(); + + /** + * We know we got a packet in header format here, try to parse the header + * + * @param IObuf of the header + data. Untransforms the data as appropriate. + * @return Just the data section in an IOBuf + */ + std::unique_ptr readHeaderFormat(std::unique_ptr); + + /** + * Untransform the data based on the received header flags + * On conclusion of function, setReadBuffer is called with the + * untransformed data. + * + * @param IOBuf input data section + * @return IOBuf output data section + */ + std::unique_ptr untransform(std::unique_ptr); + + /** + * Transform the data based on our write transform flags + * At conclusion of function the write buffer is set to the + * transformed data. + * + * @param IOBuf to transform. Returns transformed IOBuf (or chain) + * @return transformed data IOBuf + */ + std::unique_ptr transform(std::unique_ptr); + + uint16_t getNumTransforms() const { + int trans = writeTrans_.size(); + if (macCallback_) { + trans += 1; + } + return trans; + } + + void setTransform(uint16_t transId) { writeTrans_.push_back(transId); } + + // Info headers + typedef std::map StringToStringMap; + + // these work with write headers + void setHeader(const std::string& key, const std::string& value); + void setPersistentHeader(const std::string& key, const std::string& value); + void clearHeaders(); + /** + * this function only clears the local persistent + * header. does not affect the persistent header + * that already set + */ + void clearPersistentHeaders(); + + StringToStringMap& getWriteHeaders() { + return writeHeaders_; + } + + StringToStringMap& getPersistentWriteHeaders() { + return persisWriteHeaders_; + } + + // these work with read headers + const StringToStringMap& getHeaders() const { + return readHeaders_; + } + + std::string getPeerIdentity(); + void setIdentity(const std::string& identity); + + // accessors for seqId + int32_t getSequenceNumber() const { return seqId; } + void setSequenceNumber(int32_t seqId) { this->seqId = seqId; } + + enum TRANSFORMS { + ZLIB_TRANSFORM = 0x01, + HMAC_TRANSFORM = 0x02, + SNAPPY_TRANSFORM = 0x03, + }; + + /** + * Callbacks to get and verify a mac transform. + * + * If a mac callback is provided, it will be called with the outgoing packet, + * with the returned string appended at the end of the data. + * + * If a verify callback is provided, all incoming packets will be called with + * their mac data and packet data to verify. If false is returned, an + * exception is thrown. Packets without any mac also throw an exception if a + * verify function is provided. + * + * If no verify callback is provided, and an incoming packet contains a mac, + * the mac is ignored. + * + **/ + typedef std::tr1::function MacCallback; + typedef std::tr1::function< + bool(const std::string&, const std::string)> VerifyMacCallback; + + void setHmac(MacCallback macCb, VerifyMacCallback verifyCb) { + macCallback_ = macCb; + verifyCallback_ = verifyCb; + } + + /* IOBuf interface */ + + /** + * Adds the header based on the type of transport: + * unframed - does nothing. + * framed - prepends frame size + * header - prepends header, optionally appends mac + * http - only supported for sync case, prepends http header. + * + * @return IOBuf chain with header _and_ data. Data is not copied + */ + std::unique_ptr addHeader(std::unique_ptr); + /** + * Given an IOBuf Chain, remove the header. Supports unframed (sync + * only), framed, header, and http (sync case only). + * + * @param IOBufQueue - queue to try to read message from. + * + * @param needed - if the return is NULL (i.e. we didn't read a full + * message), needed is set to the number of bytes needed + * before you should call removeHeader again. + * + * @return IOBuf - the message chain. May be shared, may be chained. + * If NULL, we didn't get enough data for a whole message, + * call removeHeader again after reading needed more bytes. + */ + std::unique_ptr removeHeader(folly::IOBufQueue*, + size_t& needed); + + protected: + + std::bitset supported_clients; + + void initSupportedClients(std::bitset const*); + + std::unique_ptr queue_; + + // 0 and 16th bits must be 0 to differentiate from framed & unframed + static const uint32_t HEADER_MAGIC = 0x0FFF0000; + static const uint32_t HEADER_MASK = 0xFFFF0000; + static const uint32_t FLAGS_MASK = 0x0000FFFF; + static const uint32_t HTTP_MAGIC = 0x504F5354; // POST + + static const uint32_t MAX_FRAME_SIZE = 0x3FFFFFFF; + + int16_t protoId; + uint16_t clientType; + uint32_t seqId; + uint16_t flags; + std::string identity; + + std::vector readTrans_; + std::vector writeTrans_; + + // Map to use for headers + StringToStringMap readHeaders_; + StringToStringMap writeHeaders_; + + // Map to use for persistent headers + StringToStringMap persisReadHeaders_; + StringToStringMap persisWriteHeaders_; + + static const std::string IDENTITY_HEADER; + static const std::string ID_VERSION_HEADER; + static const std::string ID_VERSION; + + static std::string s_identity; + + MacCallback macCallback_; + VerifyMacCallback verifyCallback_; + + /** + * Returns the maximum number of bytes that write k/v headers can take + */ + size_t getMaxWriteHeadersSize() const; + + struct infoIdType { + enum idType { + // start at 1 to avoid confusing header padding for an infoId + KEYVALUE = 1, + // for persistent header + PKEYVALUE = 2, + END // signal the end of infoIds we can handle + }; + }; +}; + +}}} // apache::thrift::transport + +#endif // #ifndef THRIFT_TRANSPORT_THEADER_H_ diff --git a/thrift/lib/cpp/transport/THeaderTransport.h b/thrift/lib/cpp/transport/THeaderTransport.h new file mode 100644 index 0000000000..ddad1af00b --- /dev/null +++ b/thrift/lib/cpp/transport/THeaderTransport.h @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef THRIFT_TRANSPORT_THEADERTRANSPORT_H_ +#define THRIFT_TRANSPORT_THEADERTRANSPORT_H_ 1 + +#include + +#include "thrift/lib/cpp/transport/TBufferTransports.h" +#include "thrift/lib/cpp/transport/THeader.h" +#include "thrift/lib/cpp/transport/THttpServer.h" +#include "thrift/lib/cpp/transport/TTransport.h" +#include "thrift/lib/cpp/transport/TVirtualTransport.h" + +#include "folly/experimental/io/IOBuf.h" +#include "folly/experimental/io/IOBufQueue.h" + +#include +#include "boost/scoped_array.hpp" +#include +#include + +namespace apache { namespace thrift { namespace transport { + +/** + * Header transport. All writes go into an in-memory buffer until flush is + * called, at which point the transport writes the length of the entire + * binary chunk followed by the data payload. This allows the receiver on the + * other end to always do fixed-length reads. + * + * Subclass TFramedTransport because most of the read/write methods are similar + * and need similar buffers. Major changes are readFrame & flush. + */ +class THeaderTransport + : public TVirtualTransport + , public THeader { + public: + + explicit THeaderTransport(const boost::shared_ptr transport) + : THeader() + ,transport_(transport) + , outTransport_(transport) + , httpTransport_(transport) + { + initBuffers(); + } + + THeaderTransport(const boost::shared_ptr transport, + std::bitset const* clientTypes) + : THeader(clientTypes) + , transport_(transport) + , outTransport_(transport) + , httpTransport_(transport) + { + initBuffers(); + } + + THeaderTransport(const boost::shared_ptr inTransport, + const boost::shared_ptr outTransport, + std::bitset const* clientTypes) + : THeader(clientTypes) + , transport_(inTransport) + , outTransport_(outTransport) + , httpTransport_(outTransport) + { + initBuffers(); + } + + THeaderTransport(const boost::shared_ptr transport, uint32_t sz, + std::bitset const* clientTypes) + : THeader(clientTypes) + , transport_(transport) + , outTransport_(transport) + , httpTransport_(transport) + { + initBuffers(); + } + + void resetProtocol(); + + void open() { + transport_->open(); + } + + bool isOpen() { + return transport_->isOpen(); + } + + bool peek() { + return (this->rBase_ < this->rBound_) || transport_->peek(); + } + + void close() { + flush(); + transport_->close(); + } + + virtual uint32_t readSlow(uint8_t* buf, uint32_t len); + virtual uint32_t readAll(uint8_t* buf, uint32_t len); + virtual void flush(); + + boost::shared_ptr getUnderlyingTransport() { + return transport_; + } + + /* + * TVirtualTransport provides a default implementation of readAll(). + * We want to use the TBufferBase version instead. + */ + using TBufferBase::readAll; + + protected: + /** + * Reads a frame of input from the underlying stream. + * + * Returns true if a frame was read successfully, or false on EOF. + * (Raises a TTransportException if EOF occurs after a partial frame.) + */ + bool readFrame(uint32_t minFrameSize); + + void allocateReadBuffer(uint32_t sz); + uint32_t getWriteBytes(); + + void initBuffers() { + setReadBuffer(NULL, 0); + setWriteBuffer(wBuf_.get(), wBufSize_); + } + + boost::shared_ptr transport_; + boost::shared_ptr outTransport_; + + boost::shared_ptr httpTransport_; + + // Buffer to use for readFrame/flush processing + std::unique_ptr readBuf_; +}; + +/** + * Wraps a transport into a header one. + * + */ +class THeaderTransportFactory : public TTransportFactory { + public: + THeaderTransportFactory() {} + + virtual ~THeaderTransportFactory() {} + + /** + * Wraps the transport into a header one. + */ + virtual boost::shared_ptr + getTransport(boost::shared_ptr trans) { + return boost::shared_ptr( + new THeaderTransport(trans, &clientTypes)); + } + + void setClientTypes(const std::bitset& clientTypes) { + for (int i = 0; i < CLIENT_TYPES_LEN; ++i) { + this->clientTypes[i] = clientTypes[i]; + } + } + +private: + std::bitset clientTypes; +}; + +}}} // apache::thrift::transport + +#endif // #ifndef THRIFT_TRANSPORT_THEADERTRANSPORT_H_ diff --git a/thrift/lib/cpp/transport/THttpClient.h b/thrift/lib/cpp/transport/THttpClient.h new file mode 100644 index 0000000000..91e04a9cdb --- /dev/null +++ b/thrift/lib/cpp/transport/THttpClient.h @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef THRIFT_TRANSPORT_THTTPCLIENT_H_ +#define THRIFT_TRANSPORT_THTTPCLIENT_H_ 1 + +#include + +namespace apache { namespace thrift { namespace transport { + +class THttpClient : public THttpTransport { + public: + /* + * Create for a given host and port. The version that doesn't take + * a transport constructs its own TSocket as a transport. + * + * The path must be non-empty and start with "/". + */ + THttpClient(const boost::shared_ptr& transport, + const std::string& host, + const std::string& path); + THttpClient(const std::string& host, int port, const std::string& path); + + virtual ~THttpClient(); + + void setUserAgent(const std::string&); + + virtual void flush(); + + virtual void close() { + connectionClosedByServer_ = false; + THttpTransport::close(); + } + + virtual void init() { + /* + * HTTP requires that the `abs_path' component of a POST message start + * with '/' (see rfc2616 and rfc2396). + */ + assert(!path_.empty() && path_[0] == '/'); + + THttpTransport::init(); + } + + protected: + + const std::string host_; + const std::string path_; + std::string userAgent_; + bool connectionClosedByServer_; + + virtual void parseHeader(char* header); + virtual bool parseStatusLine(char* status); + +}; + +}}} // apache::thrift::transport + +#endif // #ifndef THRIFT_TRANSPORT_THTTPCLIENT_H_ diff --git a/thrift/lib/cpp/transport/THttpServer.h b/thrift/lib/cpp/transport/THttpServer.h new file mode 100644 index 0000000000..65234084a9 --- /dev/null +++ b/thrift/lib/cpp/transport/THttpServer.h @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_TRANSPORT_THTTPSERVER_H_ +#define _THRIFT_TRANSPORT_THTTPSERVER_H_ 1 + +#include + +namespace apache { namespace thrift { namespace transport { + +class THttpServer : public THttpTransport { + public: + THttpServer(boost::shared_ptr transport); + + virtual ~THttpServer(); + + virtual void flush(); + + protected: + + void readHeaders(); + virtual void parseHeader(char* header); + virtual bool parseStatusLine(char* status); + std::string getTimeRFC1123(); + +}; + +/** + * Wraps a transport into HTTP protocol + */ +class THttpServerTransportFactory : public TTransportFactory { + public: + THttpServerTransportFactory() {} + + virtual ~THttpServerTransportFactory() {} + + /** + * Wraps the transport into a buffered one. + */ + virtual boost::shared_ptr getTransport(boost::shared_ptr trans) { + return boost::shared_ptr(new THttpServer(trans)); + } + +}; + +}}} // apache::thrift::transport + +#endif // #ifndef _THRIFT_TRANSPORT_THTTPSERVER_H_ diff --git a/thrift/lib/cpp/transport/THttpTransport.h b/thrift/lib/cpp/transport/THttpTransport.h new file mode 100644 index 0000000000..68d6f33502 --- /dev/null +++ b/thrift/lib/cpp/transport/THttpTransport.h @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef THRIFT_TRANSPORT_THTTPTRANSPORT_H_ +#define THRIFT_TRANSPORT_THTTPTRANSPORT_H_ 1 + +#include +#include "TVirtualTransport.h" + +namespace apache { namespace thrift { namespace transport { + +/** + * HTTP implementation of the thrift transport. This was irritating + * to write, but the alternatives in C++ land are daunting. Linking CURL + * requires 23 dynamic libraries last time I checked (WTF?!?). All we have + * here is a VERY basic HTTP/1.1 client which supports HTTP 100 Continue, + * chunked transfer encoding, keepalive, etc. Tested against Apache. + */ +class THttpTransport : public TVirtualTransport { + public: + THttpTransport(boost::shared_ptr transport); + + virtual ~THttpTransport(); + + void open() { + transport_->open(); + } + + bool isOpen() { + return transport_->isOpen(); + } + + bool peek() { + return transport_->peek(); + } + + virtual void close() { + transport_->close(); + } + + uint32_t read(uint8_t* buf, uint32_t len); + + uint32_t readEnd(); + + void write(const uint8_t* buf, uint32_t len); + + virtual void flush() = 0; + + protected: + + boost::shared_ptr transport_; + + TMemoryBuffer writeBuffer_; + TMemoryBuffer readBuffer_; + + bool readHeaders_; + bool chunked_; + bool chunkedDone_; + uint32_t chunkSize_; + uint32_t contentLength_; + + char* httpBuf_; + uint32_t httpPos_; + uint32_t httpBufLen_; + uint32_t httpBufSize_; + + virtual void init(); + + uint32_t readMoreData(); + char* readLine(); + + void readHeaders(); + virtual void parseHeader(char* header) = 0; + virtual bool parseStatusLine(char* status) = 0; + + uint32_t readChunked(); + void readChunkedFooters(); + uint32_t parseChunkSize(char* line); + + uint32_t readContent(uint32_t size); + + virtual void refill(); + void shift(); + + static const char* CRLF; + static const int CRLF_LEN; +}; + +}}} // apache::thrift::transport + +#endif // #ifndef THRIFT_TRANSPORT_THTTPCLIENT_H_ diff --git a/thrift/lib/cpp/transport/TMemPagedFactory.h b/thrift/lib/cpp/transport/TMemPagedFactory.h new file mode 100644 index 0000000000..3ed65c475d --- /dev/null +++ b/thrift/lib/cpp/transport/TMemPagedFactory.h @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_TRANSPORT_TMEMPAGEDFACTORY_H_ +#define _THRIFT_TRANSPORT_TMEMPAGEDFACTORY_H_ 1 + +#include "external/google_base/spinlock.h" + +namespace apache { namespace thrift { namespace transport { + +/* + * struct keeps information about memory usage for specific page + */ +struct FixedSizeMemoryPage { + FixedSizeMemoryPage* next_; //!< chain hook + uint8_t buffer_[]; //!< pointer to the buffer +}; + +/** + * Fixed size memory pages factory + * Thread safe + * + */ +class FixedSizeMemoryPageFactory { + public: + /** + * Constructor + */ + FixedSizeMemoryPageFactory(size_t pageSize, //!< page size in bytes + size_t maxMemoryUsage, //!< max memory usage + size_t cacheMemorySize); //!< max memory in cache + /** + * Destructor + */ + ~FixedSizeMemoryPageFactory(); + + /** + * Releases all unused memory + */ + void releaseMemory(); + + /** + * Requests for a page allocation + */ + FixedSizeMemoryPage* getPage(bool throwOnError = true); + + /** + * Returns current page size + */ + size_t getPageSize() const; + + /** + * Returns page back to factory + */ + void returnPage(FixedSizeMemoryPage* page); + private: + const size_t pageSize_; + const size_t maxMemoryUsage_; + const size_t cacheMemorySize_; + size_t numAllocatedPages_; + size_t numCachedPages_; + FixedSizeMemoryPage* cachedPages_; + mutable facebook::SpinLock lock_; +}; + + +}}} // apache::thrift::transport + +#endif // #ifndef _THRIFT_TRANSPORT_TMEMPAGEDFACTORY_H_ diff --git a/thrift/lib/cpp/transport/TMemPagedTransport.h b/thrift/lib/cpp/transport/TMemPagedTransport.h new file mode 100644 index 0000000000..d9b4e5b9a9 --- /dev/null +++ b/thrift/lib/cpp/transport/TMemPagedTransport.h @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_TRANSPORT_TMEMPAGEDTRANSPORT_H_ +#define _THRIFT_TRANSPORT_TMEMPAGEDTRANSPORT_H_ 1 + +#include "thrift/lib/cpp/transport/TVirtualTransport.h" +#include "thrift/lib/cpp/transport/TMemPagedFactory.h" +#include + +namespace apache { namespace thrift { namespace transport { + +using namespace boost; + +/** + * Class TTransport for direct usage of paged memory + * NOT thread safe - for a higher performance trade-off + */ +class TMemPagedTransport : public TVirtualTransport { + public: + /** + * Constructor + */ + TMemPagedTransport(shared_ptr& factory); + + /** + * Destructor + */ + ~TMemPagedTransport(); + + /** + * Whether this transport is open. + */ + virtual bool isOpen(); + + /** + * Opens the transport. + */ + virtual void open(); + + /** + * Closes the transport. + */ + virtual void close(); + + /** + * Called when read is completed. + * This can be over-ridden to perform a transport-specific action + * e.g. logging the request to a file + * + * @return number of bytes read if available, 0 otherwise. + */ + virtual uint32_t readEnd(); + + /** + * Called when write is completed. + * This can be over-ridden to perform a transport-specific action + * at the end of a request. + * + * @return number of bytes written if available, 0 otherwise + */ + virtual uint32_t writeEnd(); + + /** + * Implementations of the base functions + */ + inline uint32_t read(uint8_t* buf, uint32_t len); + inline void write(const uint8_t* buf, uint32_t len); + inline const uint8_t* borrow(uint8_t* buf, uint32_t* len); + inline void consume(uint32_t len); + inline uint32_t readAll(uint8_t* buf, uint32_t len); + /** + * interface extension, not part of the TTrasport + * make sure there are some bytes for write at the same page available + */ + inline uint8_t* reserve(uint32_t* len, bool throwOnError = true); + /** + * interface extension, not part of the TTrasport + * similar to consume but for the write + */ + inline void skip(uint32_t len); + + /** + * Reset buffer for read + * Sets read offset to zero + */ + inline void resetForRead(); + + /** + * Reset buffer for write + * Sets write offset to zero and releases extra memory pages + */ + inline void resetForWrite(bool releaseAllPages = false); + + /** + * Returns number of bytes already read from buffer + */ + inline uint32_t getReadBytes() const; + + /** + * Returns current bytes written in buffer + */ + inline uint32_t getWrittenBytes() const; + + private: + /** + * helper function + * write or skip bytes + */ + inline void set(const uint8_t* buf, uint32_t len); + + /** + * helper function + * read or skip bytes + */ + inline uint32_t get(uint8_t* buf, uint32_t len); + + + private: + shared_ptr factory_; //!< memory pages factory + const uint32_t pageSize_; //!< page size + FixedSizeMemoryPage* headPage_; //!< head of the stack + FixedSizeMemoryPage* currentReadPage_; //!< current read page + uint32_t currentReadOffset_; //!< bytes already read + FixedSizeMemoryPage* currentWritePage_; //!< current write page + uint32_t currentWriteOffset_; //!< bytes already written +}; + +}}} // apache::thrift::transport + +#endif // #ifndef _THRIFT_TRANSPORT_TMEMPAGEDTRANSPORT_H_ diff --git a/thrift/lib/cpp/transport/TMemPagedTransport.tcc b/thrift/lib/cpp/transport/TMemPagedTransport.tcc new file mode 100644 index 0000000000..480ec8da0e --- /dev/null +++ b/thrift/lib/cpp/transport/TMemPagedTransport.tcc @@ -0,0 +1,349 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_TRANSPORT_TMEMPAGEDTRANSPORT_HPP_ +#define _THRIFT_TRANSPORT_TMEMPAGEDTRANSPORT_HPP_ 1 + +#include "thrift/lib/cpp/transport/TMemPagedTransport.h" +#include + +namespace apache { namespace thrift { namespace transport { + +/** + * Reset buffer for read + */ +inline void TMemPagedTransport::resetForRead() { + currentReadPage_ = headPage_; + currentReadOffset_ = 0; +} + +/** + * Reset buffer for write + */ +inline void TMemPagedTransport::resetForWrite(bool releaseAllPages) { + FixedSizeMemoryPage* discardPage = NULL; + if (releaseAllPages) { // release all pages + discardPage = headPage_; + headPage_ = NULL; + } else { // make sure head page left or created + if (!headPage_) { // allocate page if any + headPage_ = factory_->getPage(); + } else { + discardPage = headPage_->next_; + headPage_->next_ = NULL; + } + } + while (discardPage) { + FixedSizeMemoryPage* pageForDeletion = discardPage; + discardPage = discardPage->next_; + factory_->returnPage(pageForDeletion); + } + + // reset + currentReadPage_ = currentWritePage_ = headPage_; + currentReadOffset_ = currentWriteOffset_ = 0; +} + +/** + * Returns number of bytes already read from buffer + */ +inline uint32_t TMemPagedTransport::getReadBytes() const { + uint32_t ret = 0; + FixedSizeMemoryPage* page = headPage_; + while (page) { + if (page == currentReadPage_) { + ret += currentReadOffset_; + break; + } else { + ret += pageSize_; + } + page = page->next_; + } + // return result + return ret; +} + +/** + * Returns current bytes written in buffer + */ +inline uint32_t TMemPagedTransport::getWrittenBytes() const { + uint32_t ret = 0; + FixedSizeMemoryPage* page = headPage_; + while (page) { + if (page == currentWritePage_) { + ret += currentWriteOffset_; + break; + } else { + ret += pageSize_; + } + page = page->next_; + } + // return result + return ret; +} + +/** + * Implementations of the base functions + */ +inline uint32_t TMemPagedTransport::read(uint8_t* buf, uint32_t len) { + if (!buf) { + throw TTransportException(TTransportException::BAD_ARGS); + } + return get(buf, len); +} + +inline void TMemPagedTransport::write(const uint8_t* buf, uint32_t len) { + if (!buf) { + throw TTransportException(TTransportException::BAD_ARGS); + } + set(buf, len); +} + +/** + * interface extension, not part of the TTrasport + * similar to consume but for the write + */ +inline void TMemPagedTransport::skip(uint32_t len) { + set(NULL, len); +} + +inline const uint8_t* TMemPagedTransport::borrow(uint8_t* buf, uint32_t* len) { + // - possible scenarios + // - buf is NULL + // -- available bytes for current page >= len - (return internal pointer) + // -- available bytes for current page < len - return NULL, set len 0 + // - buf is not NULL + // -- available bytes for current page >= len - (return internal pointer) + // -- available bytes for current page < len - copy to the external buffer + FixedSizeMemoryPage* page = currentReadPage_; + uint32_t readOffset = currentReadOffset_; + if (!page) { // no memory available + *len = 0; + return NULL; + } + + uint32_t capacity = *len; // requested capacity + uint32_t available = 0; + if (page == currentWritePage_) { + available = currentWriteOffset_ - readOffset; + } else { + available = pageSize_ - readOffset; + } + + if (available >= capacity) { // enough available bytes + *len = available; // assign available bytes + return page->buffer_ + readOffset; // return readable memory pointer + } + // not enough available bytes on the current page + if (!buf) { // buffer is not provided + *len = 0; + return NULL; + } + // buffer is provided, make memory copy from current page if any + available = std::min(available, capacity); + if (available) { + std::memcpy(buf + *len - capacity, + page->buffer_ + readOffset, + available); + // adjust capacity + capacity -= available; + // adjust offset + readOffset += available; + } + // copy the rest if available + while (capacity) { // read until all buffer filled + if (page == currentWritePage_ && + readOffset == currentWriteOffset_) { + *len = 0; + return NULL; // no more available bytes for read + } else { + // if read and write on the same page + if (page == currentWritePage_) { + available = currentWriteOffset_ - readOffset; + } else if (!(available = pageSize_ - readOffset)) { + // move to next page + page = page->next_; + assert(page); + readOffset = 0; + continue; + } + } + // how many bytes we can read + available = std::min(capacity, available); + // memory copy + std::memcpy(buf + *len - capacity, + page->buffer_ + readOffset, + available); + // adjust capacity + capacity -= available; + // adjust page read offset + readOffset += available; + // move to the next page if end is reached + // and more pages for read available + if (readOffset == pageSize_ && + page != currentWritePage_) { + page = page->next_; + assert(page); + // reset offset + readOffset = 0; + } + } + // return result, leave len as it is + return buf; +} + +/** + * interface extension, not part of the TTrasport + * make sure there are some bytes for write at the same page available + */ +inline uint8_t* TMemPagedTransport::reserve(uint32_t* len, bool throwOnError) { + if (!currentWritePage_ || pageSize_ == currentWriteOffset_) { + FixedSizeMemoryPage* newPage = factory_->getPage(throwOnError); + if (!newPage) { // no memory + return NULL; + } + if (currentWritePage_) { + currentWritePage_->next_ = newPage; + } else { + headPage_ = currentReadPage_ = newPage; + } + currentWritePage_ = newPage; + // reset write offset + currentWriteOffset_ = 0; + } + if (len) { // it can be a call just to reserve a new page + *len = pageSize_ - currentWriteOffset_; + } + + return currentWritePage_->buffer_ + currentWriteOffset_; +} + +inline void TMemPagedTransport::consume(uint32_t len) { + get(NULL, len); +} + +inline uint32_t TMemPagedTransport::readAll(uint8_t* buf, uint32_t len) { + // try to read requested bytes + uint32_t res = read(buf, len); + if (res != len) { // not all bytes can be read + // interface description requires throwing exception + throw TTransportException(TTransportException::END_OF_FILE); + } + return res; +} + + +/** + * helper function + * write or skip bytes + */ +inline void TMemPagedTransport::set(const uint8_t* buf, uint32_t len) { + uint32_t capacity = len; // requested capacity + // prepare for rollback in case of out of memory + FixedSizeMemoryPage* saveWritePage = currentWritePage_; + uint32_t saveWriteOffset = currentWriteOffset_; + while (capacity) { // write until all buffer bytes get sent + uint32_t available = 0; + if (!reserve(&available, false)) { // reserve page if any + // rollback + currentWriteOffset_ = saveWriteOffset; + if ((currentWritePage_ = saveWritePage) != NULL) { // there were pages + // remove extra allocated pages if any + // even we remove previously existed pages it won't hurt + FixedSizeMemoryPage* pageForDelete = saveWritePage->next_; + currentWritePage_->next_ = NULL; // cut the tail + while (pageForDelete) { + saveWritePage = pageForDelete; + pageForDelete = pageForDelete->next_; + factory_->returnPage(saveWritePage); + } + } else { + // remove all pages + resetForWrite(true); + } + // throw + throw TTransportException(TTransportException::INTERNAL_ERROR); + } + // how many bytes we can write + available = std::min(capacity, available); + if (buf) { + // memory copy if any + std::memcpy(currentWritePage_->buffer_ + currentWriteOffset_, + buf + len - capacity, + available); + } + // adjust capacity + capacity -= available; + // adjust page write iterator + currentWriteOffset_ += available; + } // while +} + +/** + * Implementations of the base functions + */ +inline uint32_t TMemPagedTransport::get(uint8_t* buf, uint32_t len) { + uint32_t capacity = len; // requested memory + uint32_t available = 0; + if (!currentReadPage_) { + return 0; // nothing to read, did not reset or write bytes + } + while (capacity) { // read until all buffer filled + if (currentReadPage_ == currentWritePage_) { + // if read and write on the same page + if (!(available = currentWriteOffset_ - currentReadOffset_)) { + break; // no more available bytes for read + } + } else if (!(available = pageSize_ - currentReadOffset_)) { + // not the last page + // move to next page + currentReadPage_ = currentReadPage_->next_; + assert(currentReadPage_); + currentReadOffset_ = 0; + available = pageSize_ - currentReadOffset_; + } + // how many bytes we can read + available = std::min(capacity, available); + // memory copy if any + if (buf) { + std::memcpy(buf + len - capacity, + currentReadPage_->buffer_ + currentReadOffset_, + available); + } + // adjust capacity + capacity -= available; + // adjust page read iterator + currentReadOffset_ += available; + // move to the next page if end is reached + // and more pages for read available + if (currentReadOffset_ == pageSize_ && + currentReadPage_ != currentWritePage_) { + currentReadPage_ = currentReadPage_->next_; + assert(currentReadPage_); + // reset offset + currentReadOffset_ = 0; + } + } + // return result + return len - capacity; +} + +}}} // apache::thrift::transport + +#endif // #ifndef _THRIFT_TRANSPORT_TMEMPAGEDTRANSPORT_HPP_ diff --git a/thrift/lib/cpp/transport/TRpcTransport.h b/thrift/lib/cpp/transport/TRpcTransport.h new file mode 100644 index 0000000000..b24e5db93f --- /dev/null +++ b/thrift/lib/cpp/transport/TRpcTransport.h @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_TRANSPORT_TRPCTRANSPORT_H_ +#define THRIFT_TRANSPORT_TRPCTRANSPORT_H_ 1 + +#include "thrift/lib/cpp/transport/TTransport.h" + +namespace apache { namespace thrift { namespace transport { + +class TSocketAddress; + +/** + * A TRpcTransport adds a getPeerAddress() method to the base TTransport + * interface. + */ +class TRpcTransport : public TTransport { + public: + /** + * Get the address of the peer to which this transport is connected. + * + * @return Returns a pointer to a TSocketAddress. This struct is owned by + * the TRpcTransport and is guaranteed to remain valid for the + * lifetime of the TRpcTransport. It is guaranteed to return + * non-NULL. (On error, a TTransportException will be raised.) + */ + virtual const TSocketAddress* getPeerAddress() = 0; +}; + +}}} // apache::thrift::transport + +#endif // THRIFT_TRANSPORT_TRPCTRANSPORT_H_ diff --git a/thrift/lib/cpp/transport/TSSLServerSocket.h b/thrift/lib/cpp/transport/TSSLServerSocket.h new file mode 100644 index 0000000000..178b362fe9 --- /dev/null +++ b/thrift/lib/cpp/transport/TSSLServerSocket.h @@ -0,0 +1,46 @@ +// Copyright (c) 2009- Facebook +// Distributed under the Thrift Software License +// +// See accompanying file LICENSE or visit the Thrift site at: +// http://developers.facebook.com/thrift/ + +#ifndef _THRIFT_TRANSPORT_TSSLSERVERSOCKET_H_ +#define _THRIFT_TRANSPORT_TSSLSERVERSOCKET_H_ 1 + +#include +#include "thrift/lib/cpp/transport/TServerSocket.h" + +namespace apache { namespace thrift { namespace transport { + +class TSSLSocketFactory; + +/** + * Server socket that accepts SSL connections. + */ +class TSSLServerSocket: public TServerSocket { + public: + /** + * Constructor. + * + * @param port Listening port + * @param factory SSL socket factory implementation + */ + TSSLServerSocket(int port, boost::shared_ptr factory); + /** + * Constructor. + * + * @param port Listening port + * @param sendTimeout Socket send timeout + * @param recvTimeout Socket receive timeout + * @param factory SSL socket factory implementation + */ + TSSLServerSocket(int port, int sendTimeout, int recvTimeout, + boost::shared_ptr factory); + protected: + boost::shared_ptr createSocket(int socket); + boost::shared_ptr factory_; +}; + +}}} + +#endif diff --git a/thrift/lib/cpp/transport/TSSLSocket.h b/thrift/lib/cpp/transport/TSSLSocket.h new file mode 100644 index 0000000000..66131b87b7 --- /dev/null +++ b/thrift/lib/cpp/transport/TSSLSocket.h @@ -0,0 +1,365 @@ +// Copyright (c) 2009- Facebook +// Distributed under the Thrift Software License +// +// See accompanying file LICENSE or visit the Thrift site at: +// http://developers.facebook.com/thrift/ + +#ifndef THRIFT_TRANSPORT_TSSLSOCKET_H_ +#define THRIFT_TRANSPORT_TSSLSOCKET_H_ 1 + +#include +#include +#include +#include "thrift/lib/cpp/concurrency/Mutex.h" +#include "thrift/lib/cpp/transport/TSocket.h" + +namespace apache { namespace thrift { namespace transport { + +class PasswordCollector; +class SSLContext; +class TSocketAddress; + +/** + * OpenSSL implementation for SSL socket interface. + */ +class TSSLSocket: public TVirtualTransport { + public: + /** + * Constructor. + */ + explicit TSSLSocket(const boost::shared_ptr& ctx); + /** + * Constructor, create an instance of TSSLSocket given an existing socket. + * + * @param socket An existing socket + */ + TSSLSocket(const boost::shared_ptr& ctx, int socket); + /** + * Constructor. + * + * @param host Remote host name + * @param port Remote port number + */ + TSSLSocket(const boost::shared_ptr& ctx, + const std::string& host, + int port); + /** + * Constructor. + */ + TSSLSocket(const boost::shared_ptr& ctx, + const TSocketAddress& address); + /** + * Destructor. + */ + ~TSSLSocket(); + + /** + * TTransport interface. + */ + bool isOpen(); + bool peek(); + void open(); + void close(); + uint32_t read(uint8_t* buf, uint32_t len); + void write(const uint8_t* buf, uint32_t len); + void flush(); + + /** + * Set whether to use client or server side SSL handshake protocol. + * + * @param flag Use server side handshake protocol if true. + */ + void server(bool flag) { server_ = flag; } + /** + * Determine whether the SSL socket is server or client mode. + */ + bool server() const { return server_; } + +protected: + /** + * Verify peer certificate after SSL handshake completes. + */ + virtual void verifyCertificate(); + + /** + * Initiate SSL handshake if not already initiated. + */ + void checkHandshake(); + + bool server_; + SSL* ssl_; + boost::shared_ptr ctx_; +}; + +/** + * SSL socket factory. SSL sockets should be created via SSL factory. + */ +class TSSLSocketFactory { + public: + /** + * Constructor/Destructor + */ + explicit TSSLSocketFactory(const boost::shared_ptr& context); + virtual ~TSSLSocketFactory(); + + /** + * Create an instance of TSSLSocket with a fresh new socket. + */ + virtual boost::shared_ptr createSocket(); + /** + * Create an instance of TSSLSocket with the given socket. + * + * @param socket An existing socket. + */ + virtual boost::shared_ptr createSocket(int socket); + /** + * Create an instance of TSSLSocket. + * + * @param host Remote host to be connected to + * @param port Remote port to be connected to + */ + virtual boost::shared_ptr createSocket(const std::string& host, + int port); + /** + * Set/Unset server mode. + * + * @param flag Server mode if true + */ + virtual void server(bool flag) { server_ = flag; } + /** + * Determine whether the socket is in server or client mode. + * + * @return true, if server mode, or, false, if client mode + */ + virtual bool server() const { return server_; } + + private: + boost::shared_ptr ctx_; + bool server_; +}; + +/** + * SSL exception. + */ +class TSSLException: public TTransportException { + public: + explicit TSSLException(const std::string& message): + TTransportException(TTransportException::INTERNAL_ERROR, message) {} + + virtual const char* what() const throw() { + if (message_.empty()) { + return "TSSLException"; + } else { + return message_.c_str(); + } + } +}; + +/** + * Wrap OpenSSL SSL_CTX into a class. + */ +class SSLContext { + public: + + enum SSLVersion { + SSLv2, + SSLv3, + TLSv1 + }; + + /** + * Constructor. + * + * @param version The lowest or oldest SSL version to support. + */ + explicit SSLContext(SSLVersion version = TLSv1); + virtual ~SSLContext(); + + /** + * Set ciphers to be used in SSL handshake process. + * + * @param ciphers A list of ciphers + */ + virtual void ciphers(const std::string& enable); + /** + * Enable/Disable authentication. Peer name validation can only be done + * if checkPeerCert is true. + * + * @param checkPeerCert If true, require peer to present valid certificate + * @param checkPeerName If true, validate that the certificate common name + * or alternate name(s) of peer matches the hostname + * used to connect. + * @param peerName If non-empty, validate that the certificate common + * name of peer matches the given string (altername + * name(s) are not used in this case). + */ + virtual void authenticate(bool checkPeerCert, bool checkPeerName, + const std::string& peerName = std::string()); + /** + * Load server certificate. + * + * @param path Path to the certificate file + * @param format Certificate file format + */ + virtual void loadCertificate(const char* path, const char* format = "PEM"); + /** + * Load private key. + * + * @param path Path to the private key file + * @param format Private key file format + */ + virtual void loadPrivateKey(const char* path, const char* format = "PEM"); + /** + * Load trusted certificates from specified file. + * + * @param path Path to trusted certificate file + */ + virtual void loadTrustedCertificates(const char* path); + /** + * Load trusted certificates from specified X509 certificate store. + * + * @param store X509 certificate store. + */ + virtual void loadTrustedCertificates(X509_STORE* store); + /** + * Default randomize method. + */ + virtual void randomize(); + /** + * Override default OpenSSL password collector. + * + * @param collector Instance of user defined password collector + */ + virtual void passwordCollector(boost::shared_ptr collector); + /** + * Obtain password collector. + * + * @return User defined password collector + */ + virtual boost::shared_ptr passwordCollector() { + return collector_; + } + + /** + * Create an SSL object from this context. + */ + SSL* createSSL() const; + + /** + * Possibly validate the peer's certificate name, depending on how this + * SSLContext was configured by authenticate(). + * + * @return True if the peer's name is acceptable, false otherwise + */ + bool validatePeerName(TSSLSocket* sock, SSL* ssl) const; + + /** + * Set the options on the SSL_CTX object. + */ + void setOptions(long options); + +#ifdef OPENSSL_NPN_NEGOTIATED + /** + * Set the list of protocols that a TLS server should advertise for + * Next Protocol Negotiation (NPN). + * + * @param protocols List of protocol names, or NULL to disable NPN. + * Note: if non-null, this method makes a copy, so + * the caller needn't keep the list in scope after + * the call completes. + */ + void setAdvertisedNextProtocols(const std::list* protocols); +#endif // OPENSSL_NPN_NEGOTIATED + + /** + * Gets the underlying SSL_CTX for advanced usage + */ + SSL_CTX *getSSLCtx() const { + return ctx_; + } + + enum SSLLockType { + LOCK_MUTEX, + LOCK_SPINLOCK, + LOCK_NONE + }; + + /** + * Set preferences for how to treat locks in OpenSSL. This must be + * called before the instantiation of any SSLContext objects, otherwise + * the defaults will be used. + * + * OpenSSL has a lock for each module rather than for each object or + * data that needs locking. Some locks protect only refcounts, and + * might be better as spinlocks rather than mutexes. Other locks + * may be totally unnecessary if the objects being protected are not + * shared between threads in the application. + * + * By default, all locks are initialized as mutexes. OpenSSL's lock usage + * may change from version to version and you should know what you are doing + * before disabling any locks entirely. + * + * Example: if you don't share SSL sessions between threads in your + * application, you may be able to do this + * + * setSSLLockTypes({{CRYPTO_LOCK_SSL_SESSION, SSLContext::LOCK_NONE}}) + */ + static void setSSLLockTypes(std::map lockTypes); + + protected: + SSL_CTX* ctx_; + + private: + bool checkPeerName_; + std::string peerFixedName_; + boost::shared_ptr collector_; + + static concurrency::Mutex mutex_; + static uint64_t count_; + +#ifdef OPENSSL_NPN_NEGOTIATED + /** + * Wire-format list of advertised protocols for use in NPN. + */ + unsigned char* advertisedNextProtocols_; + unsigned advertisedNextProtocolsLength_; + + static int advertisedNextProtocolCallback(SSL* ssl, + const unsigned char** out, unsigned int* outlen, void* data); +#endif // OPENSSL_NPN_NEGOTIATED + + static int passwordCallback(char* password, int size, int, void* data); + + static void initializeOpenSSL(); + static void cleanupOpenSSL(); + + /** + * Helper to match a hostname versus a pattern. + */ + static bool matchName(const char* host, const char* pattern, int size); +}; + +typedef boost::shared_ptr SSLContextPtr; + +/** + * Override the default password collector. + */ +class PasswordCollector { + public: + virtual ~PasswordCollector() {} + /** + * Interface for customizing how to collect private key password. + * + * By default, OpenSSL prints a prompt on screen and request for password + * while loading private key. To implement a custom password collector, + * implement this interface and register it with TSSLSocketFactory. + * + * @param password Pass collected password back to OpenSSL + * @param size Maximum length of password including NULL character + */ + virtual void getPassword(std::string& password, int size) = 0; +}; + +}}} + +#endif diff --git a/thrift/lib/cpp/transport/TServerSocket.h b/thrift/lib/cpp/transport/TServerSocket.h new file mode 100644 index 0000000000..992b92d459 --- /dev/null +++ b/thrift/lib/cpp/transport/TServerSocket.h @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_TRANSPORT_TSERVERSOCKET_H_ +#define _THRIFT_TRANSPORT_TSERVERSOCKET_H_ 1 + +#include "thrift/lib/cpp/transport/TServerTransport.h" +#include + +namespace apache { namespace thrift { namespace transport { + +class TSocket; + +/** + * Server socket implementation of TServerTransport. Wrapper around a unix + * socket listen and accept calls. + * + */ +class TServerSocket : public TServerTransport { + public: + TServerSocket(int port); + TServerSocket(int port, int sendTimeout, int recvTimeout); + + ~TServerSocket(); + + int getPort() const { return port_; } + + void setSendTimeout(int sendTimeout); + void setRecvTimeout(int recvTimeout); + + void setAcceptTimeout(int accTimeout); + + void setRetryLimit(int retryLimit); + void setRetryDelay(int retryDelay); + + void setTcpSendBuffer(int tcpSendBuffer); + void setTcpRecvBuffer(int tcpRecvBuffer); + + void listen(); + void close(); + + void interrupt(); + + void getAddress(TSocketAddress* address); + + protected: + boost::shared_ptr acceptImpl(); + virtual boost::shared_ptr createSocket(int client); + + private: + int port_; + int serverSocket_; + int acceptBacklog_; + int sendTimeout_; + int recvTimeout_; + int accTimeout_; + int retryLimit_; + int retryDelay_; + int tcpSendBuffer_; + int tcpRecvBuffer_; + + int intSock1_; + int intSock2_; +}; + +}}} // apache::thrift::transport + +#endif // #ifndef _THRIFT_TRANSPORT_TSERVERSOCKET_H_ diff --git a/thrift/lib/cpp/transport/TServerTransport.h b/thrift/lib/cpp/transport/TServerTransport.h new file mode 100644 index 0000000000..cf275a86be --- /dev/null +++ b/thrift/lib/cpp/transport/TServerTransport.h @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_TRANSPORT_TSERVERTRANSPORT_H_ +#define _THRIFT_TRANSPORT_TSERVERTRANSPORT_H_ 1 + +#include "thrift/lib/cpp/transport/TRpcTransport.h" +#include "thrift/lib/cpp/transport/TTransportException.h" +#include + +namespace apache { namespace thrift { namespace transport { + +/** + * Server transport framework. A server needs to have some facility for + * creating base transports to read/write from. + * + */ +class TServerTransport { + public: + virtual ~TServerTransport() {} + + /** + * Starts the server transport listening for new connections. Prior to this + * call most transports will not return anything when accept is called. + * + * @throws TTransportException if we were unable to listen + */ + virtual void listen() {} + + /** + * Gets a new dynamically allocated transport object and passes it to the + * caller. Note that it is the explicit duty of the caller to free the + * allocated object. The returned TRpcTransport object must always be in the + * opened state. NULL should never be returned, instead an Exception should + * always be thrown. + * + * @return A new TRpcTransport object + * @throws TTransportException if there is an error + */ + boost::shared_ptr accept() { + boost::shared_ptr result = acceptImpl(); + if (result == NULL) { + throw TTransportException("accept() may not return NULL"); + } + return result; + } + + /** + * For "smart" TServerTransport implementations that work in a multi + * threaded context this can be used to break out of an accept() call. + * It is expected that the transport will throw a TTransportException + * with the interrupted error code. + */ + virtual void interrupt() {} + + /** + * Closes this transport such that future calls to accept will do nothing. + */ + virtual void close() = 0; + + /** + * Get the address on which this transport is listening. + * + * Should only be called after calling listen(). May raise + * TTransportException on error. + * + * @param address The server's address will be stored in the TSocketAddress + * pointed to by this argument. + */ + virtual void getAddress(TSocketAddress* address) = 0; + + protected: + TServerTransport() {} + + /** + * Subclasses should implement this function for accept. + * + * @return A newly allocated TRpcTransport object + * @throw TTransportException If an error occurs + */ + virtual boost::shared_ptr acceptImpl() = 0; + +}; + +}}} // apache::thrift::transport + +#endif // #ifndef _THRIFT_TRANSPORT_TSERVERTRANSPORT_H_ diff --git a/thrift/lib/cpp/transport/TShortReadTransport.h b/thrift/lib/cpp/transport/TShortReadTransport.h new file mode 100644 index 0000000000..cdfc35a0b3 --- /dev/null +++ b/thrift/lib/cpp/transport/TShortReadTransport.h @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_TRANSPORT_TSHORTREADTRANSPORT_H_ +#define _THRIFT_TRANSPORT_TSHORTREADTRANSPORT_H_ 1 + +#include + +#include "thrift/lib/cpp/transport/TTransport.h" +#include "thrift/lib/cpp/transport/TVirtualTransport.h" + +namespace apache { namespace thrift { namespace transport { namespace test { + +/** + * This class is only meant for testing. It wraps another transport. + * Calls to read are passed through with some probability. Otherwise, + * the read amount is randomly reduced before being passed through. + * + */ +class TShortReadTransport : public TVirtualTransport { + public: + TShortReadTransport(boost::shared_ptr transport, double full_prob) + : transport_(transport) + , fullProb_(full_prob) + {} + + bool isOpen() { + return transport_->isOpen(); + } + + bool peek() { + return transport_->peek(); + } + + void open() { + transport_->open(); + } + + void close() { + transport_->close(); + } + + uint32_t read(uint8_t* buf, uint32_t len) { + if (len == 0) { + return 0; + } + + if (rand()/(double)RAND_MAX >= fullProb_) { + len = 1 + rand()%len; + } + return transport_->read(buf, len); + } + + void write(const uint8_t* buf, uint32_t len) { + transport_->write(buf, len); + } + + void flush() { + transport_->flush(); + } + + const uint8_t* borrow(uint8_t* buf, uint32_t* len) { + return transport_->borrow(buf, len); + } + + void consume(uint32_t len) { + return transport_->consume(len); + } + + boost::shared_ptr getUnderlyingTransport() { + return transport_; + } + + protected: + boost::shared_ptr transport_; + double fullProb_; +}; + +}}}} // apache::thrift::transport::test + +#endif // #ifndef _THRIFT_TRANSPORT_TSHORTREADTRANSPORT_H_ diff --git a/thrift/lib/cpp/transport/TSimpleFileTransport.h b/thrift/lib/cpp/transport/TSimpleFileTransport.h new file mode 100644 index 0000000000..6bfb879ae8 --- /dev/null +++ b/thrift/lib/cpp/transport/TSimpleFileTransport.h @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_TRANSPORT_TSIMPLEFILETRANSPORT_H_ +#define _THRIFT_TRANSPORT_TSIMPLEFILETRANSPORT_H_ 1 + +#include "thrift/lib/cpp/transport/TFDTransport.h" + +namespace apache { namespace thrift { namespace transport { + +/** + * Dead-simple wrapper around a file. + * + * @param path file name and path + * @param read whether to open file for reading + * @param write whether to open file for writing(using O_APPEND) + * + * @author David Braginsky + */ +class TSimpleFileTransport : public TFDTransport { + public: + TSimpleFileTransport(const std::string& path, + bool read = true, + bool write = false); +}; + +}}} // apache::thrift::transport + +#endif // _THRIFT_TRANSPORT_TSIMPLEFILETRANSPORT_H_ diff --git a/thrift/lib/cpp/transport/TSocket.h b/thrift/lib/cpp/transport/TSocket.h new file mode 100644 index 0000000000..62bad5aab6 --- /dev/null +++ b/thrift/lib/cpp/transport/TSocket.h @@ -0,0 +1,382 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_TRANSPORT_TSOCKET_H_ +#define _THRIFT_TRANSPORT_TSOCKET_H_ 1 + +#include +#include + +#include "thrift/lib/cpp/transport/TRpcTransport.h" +#include "thrift/lib/cpp/transport/TVirtualTransport.h" +#include "thrift/lib/cpp/transport/TServerSocket.h" +#include "thrift/lib/cpp/transport/TSocketAddress.h" + +namespace apache { namespace thrift { namespace transport { + +/** + * TCP Socket implementation of the TTransport interface. + * + */ +class TSocket : public TVirtualTransport< TSocket, + TTransportDefaults > { + +public: + /** + * Struct that contains socket options related stuff + */ + struct Options { + + Options(): + connTimeout(0), + sendTimeout(0), + recvTimeout(0), + sendBufSize(0), + recvBufSize(0), + lingerOn(false), + lingerVal(0), + noDelay(true), + reuseAddr(false) { + } + + /** Connect timeout in ms */ + int connTimeout; + + /** Send timeout in ms */ + int sendTimeout; + + /** Recv timeout in ms */ + int recvTimeout; + + /** Send Buffer Size in Bytes */ + size_t sendBufSize; + + /** Recv Buffer Size in Bytes */ + size_t recvBufSize; + + /** Linger on */ + bool lingerOn; + + /** Linger val */ + int lingerVal; + + /** Nodelay */ + bool noDelay; + + /** SO_REUSEADDR **/ + bool reuseAddr; + }; + + /** + * Constructs a new socket. Note that this does NOT actually connect the + * socket. + * + */ + TSocket(); + + /** + * Constructs a new socket. Note that this does NOT actually connect the + * socket. + * + * If a hostname is provided, and it resolves to multiple IPs, connect() will + * attempt to connect to each one in sequence, until one connection succeeds. + * + * @param host An IP address or hostname to connect to + * @param port The port to connect on + */ + TSocket(std::string host, int port); + + /** + * Constructs a new socket. Note that this does NOT actually connect the + * socket. + * + * @param adddress The address to connect to + */ + explicit TSocket(const TSocketAddress* address); + + /** + * Constructs a new socket. Note that this does NOT actually connect the + * socket. + * + * @param adddress The address to connect to + */ + explicit TSocket(const TSocketAddress& address); + + /** + * Constructor to create socket from raw UNIX handle. + * + * This is used by the TServerSocket class to create a TSocket from file + * descriptors returned by accept(). + */ + explicit TSocket(int socket); + + + /** + * Destroyes the socket object, closing it if necessary. + */ + virtual ~TSocket(); + + /** + * Whether the socket is alive. + * + * @return Is the socket alive? + */ + bool isOpen(); + + /** + * Calls select on the socket to see if there is more data available. + */ + bool peek(); + + /** + * Creates and opens the UNIX socket. + * + * @throws TTransportException If the socket could not connect + */ + virtual void open(); + + /** + * Shuts down communications on the socket. + */ + virtual void close(); + + /** + * Reads from the underlying socket. + */ + uint32_t read(uint8_t* buf, uint32_t len); + + /** + * Writes to the underlying socket. Loops until done or fail. + */ + void write(const uint8_t* buf, uint32_t len); + + /** + * Writes to the underlying socket. Does single send() and returns result. + */ + uint32_t write_partial(const uint8_t* buf, uint32_t len); + + /** + * Get the host that the socket is connected to + * + * @return string host identifier + */ + std::string getHost(); + + /** + * Get the port that the socket is connected to + * + * @return int port number + */ + int getPort(); + + /** + * Set the host that socket will connect to + * + * @param host host identifier + */ + void setHost(std::string host); + + /** + * Set the port that socket will connect to + * + * @param port port number + */ + void setPort(int port); + + + /** + * Sets the socket options enabled in the + * TSocket::Options object options_; + * Note you can try to individually set any option + * using the methods provided below e.g. setSendBufSize + */ + void setSocketOptions(const Options& oh ); + + /** + * get the options_ object + * + * @return TSocket::Options options_; + */ + TSocket::Options getSocketOptions(); + + /** + * get the currently set socket options + * this function returns the currently set socket options + * using the getsockopt() function rather than trusting + * what is set in the options_ member + * setting socket options via setSocketOption calls can fail + * because of settings in the sysctl + * e.g. trying to set the sendBufSize to some value greater + * than the wmem_max value in sysctl + * + * @return TSocket::Options + */ + TSocket::Options getCurrentSocketOptions(); + + /** + * Controls whether the linger option is set on the socket. + * + * @param on Whether SO_LINGER is on + * @param linger If linger is active, the number of seconds to linger for + */ + void setLinger(bool on, int linger); + + /** + * Whether to enable/disable Nagle's algorithm. + * + * @param noDelay Whether or not to disable the algorithm. + * @return + */ + void setNoDelay(bool noDelay); + + /** + * Set the connect timeout + */ + void setConnTimeout(int ms); + + /** + * Set the receive timeout + */ + void setRecvTimeout(int ms); + + /** + * Set the send bufsize + */ + void setSendBufSize(size_t bufsize); + + /** + * Set the recv bufsize + */ + void setRecvBufSize(size_t bufsize); + + /** + * Set the send timeout + */ + void setSendTimeout(int ms); + + /** + * Set the max number of recv retries in case of an EAGAIN + * error + */ + void setMaxRecvRetries(int maxRecvRetries); + + /** + * Set the SO_REUSEADDR socket option. + */ + void setReuseAddress(bool reuseAddr); + + /** + * Get socket information formated as a string + */ + std::string getSocketInfo(); + + /* + * Returns the address of the host to which the socket is connected + */ + const TSocketAddress* getPeerAddress(); + + /** + * Returns the DNS name of the host to which the socket is connected + */ + std::string getPeerHost(); + + /** + * Returns a string representation of the IP address to which the + * socket is connected + */ + std::string getPeerAddressStr(); + + /** + * Returns the port of the host to which the socket is connected + **/ + uint16_t getPeerPort(); + + /** + * Returns the underlying socket file descriptor. + */ + int getSocketFD() { + return socket_; + } + + /** + * (Re-)initialize a TSocket for the supplied descriptor. This is only + * intended for use by TNonblockingServer -- other use may result in + * unfortunate surprises. + * + * @param fd the descriptor for an already-connected socket + */ + void setSocketFD(int fd); + + /** + * Sets whether to use a low minimum TCP retransmission timeout. + */ + static void setUseLowMinRto(bool useLowMinRto); + + /** + * Gets whether to use a low minimum TCP retransmission timeout. + */ + static bool getUseLowMinRto(); + + /** + * Set a cache of the peer address (used when trivially available: e.g. + * accept() or connect()). Only caches IPV4 and IPV6; unset for others. + */ + void setCachedAddress(const sockaddr* addr, socklen_t len); + + protected: + /** connect, called by open */ + void openConnection(struct addrinfo *res); + + /** Host to connect to */ + std::string host_; + + /** Peer hostname */ + std::string peerHost_; + + /** Peer address */ + std::string peerAddressStr_; + + /** Port number to connect on */ + int port_; + + /** Underlying UNIX socket handle */ + int socket_; + + /** Socket Options Helper */ + Options options_; + + /** Recv EGAIN retries */ + int maxRecvRetries_; + + /** Cached peer address */ + TSocketAddress cachedPeerAddr_; + + /** Connection start time */ + timespec startTime_; + + /** Whether to use low minimum TCP retransmission timeout */ + static bool useLowMinRto_; +}; + +std::ostream& operator<<(std::ostream& os, const TSocket::Options& o); + +}}} // apache::thrift::transport + +#endif // #ifndef _THRIFT_TRANSPORT_TSOCKET_H_ + diff --git a/thrift/lib/cpp/transport/TSocketAddress.h b/thrift/lib/cpp/transport/TSocketAddress.h new file mode 100644 index 0000000000..a1ad514cc1 --- /dev/null +++ b/thrift/lib/cpp/transport/TSocketAddress.h @@ -0,0 +1,646 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_TRANSPORT_TSOCKETADDRESS_H_ +#define THRIFT_TRANSPORT_TSOCKETADDRESS_H_ 1 + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace apache { namespace thrift { namespace transport { + +class TSocketAddress { + public: + TSocketAddress() { + storage_.addr.sa_family = AF_UNSPEC; + } + + /** + * Construct a TSocketAddress from a hostname and port. + * + * Note: If the host parameter is not a numeric IP address, hostname + * resolution will be performed, which can be quite slow. + * + * Raises TTransportException on error. + * + * @param host The IP address (or hostname, if allowNameLookup is true) + * @param port The port (in host byte order) + * @pram allowNameLookup If true, attempt to perform hostname lookup + * if the hostname does not appear to be a numeric IP address. + * This is potentially a very slow operation, so is disabled by + * default. + */ + TSocketAddress(const char* host, uint16_t port, + bool allowNameLookup = false) { + // Initialize the address family first, + // since setFromHostPort() and setFromIpPort() will check it. + storage_.addr.sa_family = AF_UNSPEC; + + if (allowNameLookup) { + setFromHostPort(host, port); + } else { + setFromIpPort(host, port); + } + } + + TSocketAddress(const std::string& host, uint16_t port, + bool allowNameLookup = false) { + // Initialize the address family first, + // since setFromHostPort() and setFromIpPort() will check it. + storage_.addr.sa_family = AF_UNSPEC; + + if (allowNameLookup) { + setFromHostPort(host.c_str(), port); + } else { + setFromIpPort(host.c_str(), port); + } + } + + TSocketAddress(const TSocketAddress& addr) { + storage_ = addr.storage_; + if (addr.getFamily() == AF_UNIX) { + storage_.un.init(addr.storage_.un); + } + } + + TSocketAddress& operator=(const TSocketAddress& addr) { + if (getFamily() != AF_UNIX) { + if (addr.getFamily() != AF_UNIX) { + storage_ = addr.storage_; + } else { + storage_ = addr.storage_; + storage_.un.init(addr.storage_.un); + } + } else { + if (addr.getFamily() == AF_UNIX) { + storage_.un.copy(addr.storage_.un); + } else { + storage_.un.free(); + storage_ = addr.storage_; + } + } + return *this; + } + +#if __GXX_EXPERIMENTAL_CXX0X__ + TSocketAddress(TSocketAddress&& addr) { + storage_ = addr.storage_; + addr.storage_.addr.sa_family = AF_UNSPEC; + } + +#if __GNUC_PREREQ(4, 5) + TSocketAddress& operator=(TSocketAddress&& addr) { + std::swap(storage_, addr.storage_); + return *this; + } +#endif +#endif + + ~TSocketAddress() { + if (storage_.addr.sa_family == AF_UNIX) { + storage_.un.free(); + } + } + + bool isInitialized() const { + return (storage_.addr.sa_family != AF_UNSPEC); + } + + /** + * Return whether this address is within private network. + * + * According to RFC1918, the 10/8 prefix, 172.16/12 prefix, and 192.168/16 + * prefix are reserved for private networks. + * fc00::/7 is the IPv6 version, defined in RFC4139. IPv6 link-local + * addresses (fe80::/10) are also considered private addresses. + * + * The loopback addresses 127/8 and ::1 are also regarded as private networks + * for the purpose of this function. + * + * Returns true if this is a private network address, and false otherwise. + */ + bool isPrivateAddress() const; + + /** + * Return whether this address is a loopback address. + */ + bool isLoopbackAddress() const; + + void reset() { + prepFamilyChange(AF_UNSPEC); + storage_.addr.sa_family = AF_UNSPEC; + } + + /** + * Initialize this TSocketAddress from a hostname and port. + * + * Note: If the host parameter is not a numeric IP address, hostname + * resolution will be performed, which can be quite slow. + * + * If the hostname resolves to multiple addresses, only the first will be + * returned. + * + * Raises TTransportException on error. + * + * @param host The hostname or IP address + * @param port The port (in host byte order) + */ + void setFromHostPort(const char* host, uint16_t port); + + void setFromHostPort(const std::string& host, uint16_t port) { + setFromHostPort(host.c_str(), port); + } + + /** + * Initialize this TSocketAddress from an IP address and port. + * + * This is similar to setFromHostPort(), but only accepts numeric IP + * addresses. If the IP string does not look like an IP address, it throws a + * TTransportException rather than trying to perform a hostname resolution. + * + * Raises TTransportException on error. + * + * @param ip The IP address, as a human-readable string. + * @param port The port (in host byte order) + */ + void setFromIpPort(const char* ip, uint16_t port); + + void setFromIpPort(const std::string& ip, uint16_t port) { + setFromIpPort(ip.c_str(), port); + } + + /** + * Initialize this TSocketAddress from a local port number. + * + * This is intended to be used by server code to determine the address to + * listen on. + * + * If the current machine has any IPv6 addresses configured, an IPv6 address + * will be returned (since connections from IPv4 clients can be mapped to the + * IPv6 address). If the machine does not have any IPv6 addresses, an IPv4 + * address will be returned. + */ + void setFromLocalPort(uint16_t port); + + /** + * Initialize this TSocketAddress from a local port number. + * + * This version of setFromLocalPort() accepts the port as a string. A + * TTransportException will be raised if the string does not refer to a port + * number. Non-numeric service port names are not accepted. + */ + void setFromLocalPort(const char* port); + void setFromLocalPort(const std::string& port) { + return setFromLocalPort(port.c_str()); + } + + /** + * Initialize this TSocketAddress from a local port number and optional IP + * address. + * + * The addressAndPort string may be specified either as ":", or + * just as "". If the IP is not specified, the address will be + * initialized to 0, so that a server socket bound to this address will + * accept connections on all local IP addresses. + * + * Both the IP address and port number must be numeric. DNS host names and + * non-numeric service port names are not accepted. + */ + void setFromLocalIpPort(const char* addressAndPort); + void setFromLocalIpPort(const std::string& addressAndPort) { + return setFromLocalIpPort(addressAndPort.c_str()); + } + + /** + * Initialize this TSocketAddress from an IP address and port number. + * + * The addressAndPort string must be of the form ":". E.g., + * "10.0.0.1:1234". + * + * Both the IP address and port number must be numeric. DNS host names and + * non-numeric service port names are not accepted. + */ + void setFromIpPort(const char* addressAndPort); + void setFromIpPort(const std::string& addressAndPort) { + return setFromIpPort(addressAndPort.c_str()); + } + + /** + * Initialize this TSocketAddress from a host name and port number. + * + * The addressAndPort string must be of the form ":". E.g., + * "www.facebook.com:443". + * + * If the host name is not a numeric IP address, a DNS lookup will be + * performed. Beware that the DNS lookup may be very slow. The port number + * must be numeric; non-numeric service port names are not accepted. + */ + void setFromHostPort(const char* hostAndPort); + void setFromHostPort(const std::string& hostAndPort) { + return setFromHostPort(hostAndPort.c_str()); + } + + /** + * Initialize this TSocketAddress from a local unix path. + * + * Raises TTransportException on error. + */ + void setFromPath(const char* path) { + setFromPath(path, strlen(path)); + } + + void setFromPath(const std::string& path) { + setFromPath(path.data(), path.length()); + } + + void setFromPath(const char* path, size_t length); + + /** + * Initialize this TSocketAddress from a socket's peer address. + * + * Raises TTransportException on error. + */ + void setFromPeerAddress(int socket); + + /** + * Initialize this TSocketAddress from a socket's local address. + * + * Raises TTransportException on error. + */ + void setFromLocalAddress(int socket); + + /** + * Initialize this TSocketAddress from a struct sockaddr. + * + * Raises TTransportException on error. + * + * This method is not supported for AF_UNIX addresses. For unix addresses, + * the address length must be explicitly specified. + * + * @param address A struct sockaddr. The size of the address is implied + * from address->sa_family. + */ + void setFromSockaddr(const struct sockaddr* address); + + /** + * Initialize this TSocketAddress from a struct sockaddr. + * + * Raises TTransportException on error. + * + * @param address A struct sockaddr. + * @param addrlen The length of address data available. This must be long + * enough for the full address type required by + * address->sa_family. + */ + void setFromSockaddr(const struct sockaddr* address, + socklen_t addrlen); + + /** + * Initialize this TSocketAddress from a struct sockaddr_in. + */ + void setFromSockaddr(const struct sockaddr_in* address); + + /** + * Initialize this TSocketAddress from a struct sockaddr_in6. + */ + void setFromSockaddr(const struct sockaddr_in6* address); + + /** + * Initialize this TSocketAddress from a struct sockaddr_un. + * + * Note that the addrlen parameter is necessary to properly detect anonymous + * addresses, which have 0 valid path bytes, and may not even have a NUL + * character at the start of the path. + * + * @param address A struct sockaddr_un. + * @param addrlen The length of address data. This should include all of + * the valid bytes of sun_path, not including any NUL + * terminator. + */ + void setFromSockaddr(const struct sockaddr_un* address, + socklen_t addrlen); + + /** + * Get a pointer to the struct sockaddr data that can be used for manually + * modifying the data. + * + * addressUpdated() must be called after you finish modifying the socket data + * before you perform any other operations on the TSocketAddress. + * + * For example, to use this to store the address returned by an accept() + * call: + * + * socklen_t addrlen; + * struct sockaddr *storage = addr.getMutableAddress(AF_INET, &addrlen); + * int newSock = accept(sock, storage, &addrlen); + * if (newSock < 0) { + * // error handling + * } + * addr.addressUpdated(AF_INET, addrlen); + * + * @param family The type of address data you plan to put in the + * sockaddr. This is necessary since some address families + * require more storage than others. + * @param sizeReturn The length of the returned sockaddr will be returned via + * this argument. + */ + struct sockaddr* getMutableAddress(sa_family_t family, + socklen_t *sizeReturn); + + /** + * Indicate that the address data was updated after a call to + * getMutableAddress(). + * + * @param expectedFamily This must be the same value that you passed to + * the getMutableAddress() call. This is used to + * verify that the address data written into the + * sockaddr is actually of the same type that you + * specified when you called getMutableAddress(). + * @param addrlen The length of the new address data written into the + * sockaddr. + */ + void addressUpdated(sa_family_t expectedFamily, socklen_t addrlen) { + if (getFamily() != expectedFamily) { + // This should pretty much never happen. + addressUpdateFailure(expectedFamily); + } + if (getFamily() == AF_UNIX) { + updateUnixAddressLength(addrlen); + } + } + + const struct sockaddr* getAddress() const { + if (getFamily() != AF_UNIX) { + return &storage_.addr; + } else { + return reinterpret_cast(storage_.un.addr); + } + } + + /** + * Return the total number of bytes available for address storage. + */ + socklen_t getStorageSize() const { + if (getFamily() != AF_UNIX) { + return sizeof(storage_); + } else { + return sizeof(*storage_.un.addr); + } + } + + /** + * Return the number of bytes actually used for this address. + * + * For an uninitialized socket, this returns sizeof(struct sockaddr), + * even though some of those bytes may not be initialized. + */ + socklen_t getActualSize() const; + + sa_family_t getFamily() const { + return storage_.addr.sa_family; + } + + /** + * Get a string representation of the IPv4 or IPv6 address. + * + * Raises TTransportException if an error occurs (for example, if the address + * is not an IPv4 or IPv6 address). + */ + std::string getAddressStr() const; + + /** + * Get a string representation of the IPv4 or IPv6 address. + * + * Raises TTransportException if an error occurs (for example, if the address + * is not an IPv4 or IPv6 address). + */ + void getAddressStr(char* buf, size_t buflen) const; + + /** + * Get the IPv4 or IPv6 port for this address. + * + * Raises TTransportException if this is not an IPv4 or IPv6 address. + * + * @return Returns the port, in host byte order. + */ + uint16_t getPort() const; + + /** + * Set the IPv4 or IPv6 port for this address. + * + * Raises TTransportException if this is not an IPv4 or IPv6 address. + */ + void setPort(uint16_t port); + + /** + * Return true if this is an IPv4-mapped IPv6 address. + */ + bool isIPv4Mapped() const { + return (storage_.addr.sa_family == AF_INET6 && + IN6_IS_ADDR_V4MAPPED(&storage_.ipv6.sin6_addr)); + } + + /** + * Convert an IPv4-mapped IPv6 address to an IPv4 address. + * + * Raises TTransportException if this is not an IPv4-mapped IPv6 address. + */ + void convertToIPv4(); + + /** + * Try to convert an address to IPv4. + * + * This attempts to convert an address to an IPv4 address if possible. + * If the address is an IPv4-mapped IPv6 address, it is converted to an IPv4 + * address and true is returned. Otherwise nothing is done, and false is + * returned. + */ + bool tryConvertToIPv4(); + + /** + * Get string representation of the host name (or IP address if the host name + * cannot be resolved). + * + * Warning: Using this method is strongly discouraged. It performs a + * DNS lookup, which may block for many seconds. + * + * Raises TTransportException if an error occurs. + */ + std::string getHostStr() const; + + /** + * Get the path name for a Unix domain socket. + * + * Returns a std::string containing the path. For anonymous sockets, an + * empty string is returned. + * + * For addresses in the abstract namespace (Linux-specific), a std::string + * containing binary data is returned. In this case the first character will + * always be a NUL character. + * + * Raises TTransportException if called on a non-Unix domain socket. + */ + std::string getPath() const; + + /** + * Get human-readable string representation of the address. + * + * This prints a string representation of the address, for human consumption. + * For IP addresses, the string is of the form ":". + */ + std::string describe() const; + + bool operator==(const TSocketAddress& other) const; + bool operator!=(const TSocketAddress& other) const { + return !(*this == other); + } + + /** + * Check whether the first N bits of this address match the first N + * bits of another address. + * @note returns false if the addresses are not from the same + * address family or if the family is neither IPv4 nor IPv6 + */ + bool prefixMatch(const TSocketAddress& other, unsigned prefixLength) const; + + /** + * Use this operator for storing maps based on TSocketAddress. + */ + bool operator<(const TSocketAddress& other) const; + + /** + * Compuate a hash of a TSocketAddress. + */ + size_t hash() const; + + private: + /** + * Unix socket addresses require more storage than IPv4 and IPv6 addresses, + * and are comparatively little-used. + * + * Therefore TSocketAddress' internal storage_ member variable doesn't + * contain room for a full unix address, to avoid wasting space in the common + * case. When we do need to store a Unix socket address, we use this + * ExternalUnixAddr structure to allocate a struct sockaddr_un separately on + * the heap. + */ + struct ExternalUnixAddr { + sa_family_t family; + struct sockaddr_un *addr; + socklen_t len; + + socklen_t pathLength() const { + return len - offsetof(struct sockaddr_un, sun_path); + } + + void init() { + family = AF_UNIX; + addr = new sockaddr_un; + addr->sun_family = AF_UNIX; + len = 0; + } + void init(const ExternalUnixAddr &other) { + family = AF_UNIX; + addr = new sockaddr_un; + len = other.len; + memcpy(addr, other.addr, len); + // Fill the rest with 0s, just for safety + memset(reinterpret_cast(addr) + len, 0, + sizeof(struct sockaddr_un) - len); + } + void copy(const ExternalUnixAddr &other) { + len = other.len; + memcpy(addr, other.addr, len); + } + void free() { + family = AF_UNSPEC; + delete addr; + } + }; + + struct addrinfo* getAddrInfo(const char* host, uint16_t port, int flags); + struct addrinfo* getAddrInfo(const char* host, const char* port, int flags); + void setFromAddrInfo(const struct addrinfo* results); + void setFromLocalAddr(const struct addrinfo* results); + int setFromSocket(int socket, int (*fn)(int, struct sockaddr*, socklen_t*)); + std::string getIpString(int flags) const; + void getIpString(char *buf, size_t buflen, int flags) const; + + void addressUpdateFailure(sa_family_t expectedFamily); + void updateUnixAddressLength(socklen_t addrlen); + + void prepFamilyChange(sa_family_t newFamily) { + if (newFamily != AF_UNIX) { + if (getFamily() == AF_UNIX) { + storage_.un.free(); + } + } else { + if (getFamily() != AF_UNIX) { + storage_.un.init(); + } + } + } + + /* + * storage_ contains room for a full IPv4 or IPv6 address, so they can be + * stored inline without a separate allocation on the heap. + * + * If we need to store a Unix socket address, ExternalUnixAddr is a shim to + * track a struct sockaddr_un allocated separately on the heap. + */ + union { + sockaddr addr; + sockaddr_in ipv4; + sockaddr_in6 ipv6; + ExternalUnixAddr un; + } storage_; +}; + +/** + * Hash a TSocketAddress object. + * + * boost::hash uses hash_value(), so this allows boost::hash to automatically + * work for TSocketAddress. + */ +size_t hash_value(const TSocketAddress& address); + +std::ostream& operator<<(std::ostream& os, const TSocketAddress& addr); + +}}} // apache::thrift::transport + +namespace std { + +// Provide an implementation for std::hash +template<> +struct hash { + size_t operator()( + const apache::thrift::transport::TSocketAddress& addr) const { + return addr.hash(); + } +}; + +} + +#endif // THRIFT_TRANSPORT_TSOCKETADDRESS_H_ diff --git a/thrift/lib/cpp/transport/TSocketPool.h b/thrift/lib/cpp/transport/TSocketPool.h new file mode 100644 index 0000000000..b2f9ee79f7 --- /dev/null +++ b/thrift/lib/cpp/transport/TSocketPool.h @@ -0,0 +1,218 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef THRIFT_TRANSPORT_TSOCKETPOOL_H_ +#define THRIFT_TRANSPORT_TSOCKETPOOL_H_ 1 + +#include +#include +#include "thrift/lib/cpp/transport/TSocket.h" + +namespace apache { namespace thrift { namespace transport { + + /** + * Class to hold server information for TSocketPool + * + */ +class TSocketPoolServer { + + public: + /** + * Default constructor for server info + */ + TSocketPoolServer(); + + /** + * Constructor for TSocketPool server + */ + TSocketPoolServer(const std::string &host, int port); + + // Host name + std::string host_; + + // Port to connect on + int port_; + + // Socket for the server + int socket_; + + // Last time connecting to this server failed + int lastFailTime_; + + // Number of consecutive times connecting to this server failed + int consecutiveFailures_; +}; + +/** + * TCP Socket implementation of the TTransport interface. + * + */ +class TSocketPool : public TSocket { + + public: + + /** + * Socket pool constructor + */ + TSocketPool(); + + /** + * Socket pool constructor + * + * @param hosts list of host names + * @param ports list of port names + */ + TSocketPool(const std::vector &hosts, + const std::vector &ports); + + /** + * Socket pool constructor + * + * @param servers list of pairs of host name and port + */ + /* implicit */ + TSocketPool(const std::vector >& servers); + + /** + * Socket pool constructor + * + * @param servers list of TSocketPoolServers + */ + /* implicit */ + TSocketPool(const + std::vector< boost::shared_ptr >& servers); + + /** + * Socket pool constructor + * + * @param host single host + * @param port single port + */ + TSocketPool(const std::string& host, int port); + + /** + * Destroys the socket object, closing it if necessary. + */ + virtual ~TSocketPool(); + + /** + * Add a server to the pool + */ + void addServer(const std::string& host, int port); + + /** + * Add a server to the pool + */ + void addServer(boost::shared_ptr &server); + + /** + * Set list of servers in this pool + */ + void setServers(const std::vector< boost::shared_ptr >& servers); + + /** + * Get list of servers in this pool + */ + void getServers(std::vector< boost::shared_ptr >& servers); + + /** + * Get port of the current server + */ + int getCurrentServerPort(); + + /** + * Get host of the current server + */ + std::string getCurrentServerHost(); + + /** + * Sets how many times to keep retrying a host in the connect function. + */ + void setNumRetries(int numRetries); + + /** + * Sets how long to wait until retrying a host if it was marked down + */ + void setRetryInterval(int retryInterval); + + /** + * Sets how many times to keep retrying a host before marking it as down. + */ + void setMaxConsecutiveFailures(int maxConsecutiveFailures); + + /** + * Turns randomization in connect order on or off. + */ + void setRandomize(bool randomize); + + /** + * Whether to always try the last server. + */ + void setAlwaysTryLast(bool alwaysTryLast); + + /** + * Sets the max number of servers to try in open + */ + void setMaxServersToTry(unsigned int maxServersToTry); + + /** + * Creates and opens the UNIX socket. + */ + void open(); + + /* + * Closes the UNIX socket + */ + void close(); + + protected: + + void setCurrentServer(const boost::shared_ptr &server); + + /** List of servers to connect to */ + std::vector< boost::shared_ptr > servers_; + + /** Current server */ + boost::shared_ptr currentServer_; + + /** How many times to retry each host in connect */ + int numRetries_; + + /** Retry interval in seconds, how long to not try a host if it has been + * marked as down. + */ + int retryInterval_; + + /** Max consecutive failures before marking a host down. */ + int maxConsecutiveFailures_; + + /** Try hosts in order? or Randomized? */ + bool randomize_; + + /** Always try last host, even if marked down? */ + bool alwaysTryLast_; + + /** Number of servers to try in open(), default is UINT_MAX */ + unsigned int maxServersToTry_; +}; + +}}} // apache::thrift::transport + +#endif // #ifndef _THRIFT_TRANSPORT_TSOCKETPOOL_H_ + diff --git a/thrift/lib/cpp/transport/TTransport.h b/thrift/lib/cpp/transport/TTransport.h new file mode 100644 index 0000000000..5dadfda263 --- /dev/null +++ b/thrift/lib/cpp/transport/TTransport.h @@ -0,0 +1,353 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef THRIFT_TRANSPORT_TTRANSPORT_H +#define THRIFT_TRANSPORT_TTRANSPORT_H 1 + +#include "thrift/lib/cpp/Thrift.h" +#include +#include "thrift/lib/cpp/transport/TTransportException.h" +#include + +namespace apache { namespace thrift { namespace transport { + +/** + * Helper template to hoist readAll implementation out of TTransport + */ +template +uint32_t readAll(Transport_ &trans, uint8_t* buf, uint32_t len) { + uint32_t have = 0; + uint32_t get = 0; + + while (have < len) { + get = trans.read(buf+have, len-have); + if (get <= 0) { + throw TTransportException(TTransportException::END_OF_FILE, + "No more data to read."); + } + have += get; + } + + return have; +} + + +/** + * Generic interface for a method of transporting data. A TTransport may be + * capable of either reading or writing, but not necessarily both. + * + */ +class TTransport { + public: + /** + * Virtual deconstructor. + */ + virtual ~TTransport() {} + + /** + * Whether this transport is open. + */ + virtual bool isOpen() { + return false; + } + + /** + * Tests whether there is more data to read or if the remote side is + * still open. By default this is true whenever the transport is open, + * but implementations should add logic to test for this condition where + * possible (i.e. on a socket). + * This is used by a server to check if it should listen for another + * request. + */ + virtual bool peek() { + return isOpen(); + } + + /** + * Opens the transport for communications. + * + * @return bool Whether the transport was successfully opened + * @throws TTransportException if opening failed + */ + virtual void open() { + throw TTransportException(TTransportException::NOT_OPEN, "Cannot open base TTransport."); + } + + /** + * Closes the transport. + */ + virtual void close() { + throw TTransportException(TTransportException::NOT_OPEN, "Cannot close base TTransport."); + } + + /** + * Attempt to read up to the specified number of bytes into the string. + * + * @param buf Reference to the location to write the data + * @param len How many bytes to read + * @return How many bytes were actually read + * @throws TTransportException If an error occurs + */ + uint32_t read(uint8_t* buf, uint32_t len) { + T_VIRTUAL_CALL(); + return read_virt(buf, len); + } + virtual uint32_t read_virt(uint8_t* /* buf */, uint32_t /* len */) { + throw TTransportException(TTransportException::NOT_OPEN, + "Base TTransport cannot read."); + } + + /** + * Reads the given amount of data in its entirety no matter what. + * + * @param s Reference to location for read data + * @param len How many bytes to read + * @return How many bytes read, which must be equal to size + * @throws TTransportException If insufficient data was read + */ + uint32_t readAll(uint8_t* buf, uint32_t len) { + T_VIRTUAL_CALL(); + return readAll_virt(buf, len); + } + virtual uint32_t readAll_virt(uint8_t* buf, uint32_t len) { + return apache::thrift::transport::readAll(*this, buf, len); + } + + /** + * Called when read is completed. + * This can be over-ridden to perform a transport-specific action + * e.g. logging the request to a file + * + * @return number of bytes read if available, 0 otherwise. + */ + virtual uint32_t readEnd() { + // default behavior is to do nothing + return 0; + } + + /** + * Writes the string in its entirety to the buffer. + * + * Note: You must call flush() to ensure the data is actually written, + * and available to be read back in the future. Destroying a TTransport + * object does not automatically flush pending data--if you destroy a + * TTransport object with written but unflushed data, that data may be + * discarded. + * + * @param buf The data to write out + * @throws TTransportException if an error occurs + */ + void write(const uint8_t* buf, uint32_t len) { + T_VIRTUAL_CALL(); + write_virt(buf, len); + } + virtual void write_virt(const uint8_t* /* buf */, uint32_t /* len */) { + throw TTransportException(TTransportException::NOT_OPEN, + "Base TTransport cannot write."); + } + + /** + * Called when write is completed. + * This can be over-ridden to perform a transport-specific action + * at the end of a request. + * + * @return number of bytes written if available, 0 otherwise + */ + virtual uint32_t writeEnd() { + // default behaviour is to do nothing + return 0; + } + + /** + * Flushes any pending data to be written. Typically used with buffered + * transport mechanisms. + * + * @throws TTransportException if an error occurs + */ + virtual void flush() { + // default behaviour is to do nothing + } + + /** + * Attempts to return a pointer to \c len bytes, possibly copied into \c buf. + * Does not consume the bytes read (i.e.: a later read will return the same + * data). This method is meant to support protocols that need to read + * variable-length fields. They can attempt to borrow the maximum amount of + * data that they will need, then consume (see next method) what they + * actually use. Some transports will not support this method and others + * will fail occasionally, so protocols must be prepared to use read if + * borrow fails. + * + * @oaram buf A buffer where the data can be stored if needed. + * If borrow doesn't return buf, then the contents of + * buf after the call are undefined. This parameter may be + * NULL to indicate that the caller is not supplying storage, + * but would like a pointer into an internal buffer, if + * available. + * @param len *len should initially contain the number of bytes to borrow. + * If borrow succeeds, *len will contain the number of bytes + * available in the returned pointer. This will be at least + * what was requested, but may be more if borrow returns + * a pointer to an internal buffer, rather than buf. + * If borrow fails, the contents of *len are undefined. + * @return If the borrow succeeds, return a pointer to the borrowed data. + * This might be equal to \c buf, or it might be a pointer into + * the transport's internal buffers. + * @throws TTransportException if an error occurs + */ + const uint8_t* borrow(uint8_t* buf, uint32_t* len) { + T_VIRTUAL_CALL(); + return borrow_virt(buf, len); + } + virtual const uint8_t* borrow_virt(uint8_t* /* buf */, uint32_t* /* len */) { + return NULL; + } + + /** + * Remove len bytes from the transport. This should always follow a borrow + * of at least len bytes, and should always succeed. + * TODO(dreiss): Is there any transport that could borrow but fail to + * consume, or that would require a buffer to dump the consumed data? + * + * @param len How many bytes to consume + * @throws TTransportException If an error occurs + */ + void consume(uint32_t len) { + T_VIRTUAL_CALL(); + consume_virt(len); + } + virtual void consume_virt(uint32_t /* len */) { + throw TTransportException(TTransportException::NOT_OPEN, + "Base TTransport cannot consume."); + } + + protected: + /** + * Simple constructor. + */ + TTransport() {} +}; + +/** + * Generic factory class to make an input and output transport out of a + * source transport. Commonly used inside servers to make input and output + * streams out of raw clients. + * + */ +class TTransportFactory { + public: + TTransportFactory() {} + + virtual ~TTransportFactory() {} + + /** + * Default implementation does nothing, just returns the transport given. + */ + virtual boost::shared_ptr getTransport(boost::shared_ptr trans) { + return trans; + } + +}; + +/** + * A duplex transport factory used to make input and output transports in a + * single call. This can be used to ensure the input and output transports + * are the pointers to the same object, for example. + * + * TTransportPair.first = Input Transport + * TTransportPair.second = Output Transport + */ +typedef std::pair, + boost::shared_ptr > TTransportPair; + +class TDuplexTransportFactory { + public: + TDuplexTransportFactory() {} + + virtual ~TDuplexTransportFactory() {} + + virtual TTransportPair getTransport(boost::shared_ptr trans) { + return std::make_pair(trans, trans); + } + + virtual TTransportPair getTransport(TTransportPair transports) { + return std::make_pair(transports.first, transports.second); + } + +}; + +/** + * Adapts a TTransportFactory to a TDuplexTransportFactory that returns + * a new transport object for both input and output + */ +template +class TSingleTransportFactory : public TDuplexTransportFactory { + public: + TSingleTransportFactory() { + factory_.reset(new Factory_()); + } + + explicit TSingleTransportFactory( + boost::shared_ptr factory) : + factory_(factory) {} + + virtual TTransportPair getTransport(boost::shared_ptr trans) { + return std::make_pair(factory_->getTransport(trans), + factory_->getTransport(trans)); + } + + virtual TTransportPair getTransport(TTransportPair transports) { + return std::make_pair(factory_->getTransport(transports.first), + factory_->getTransport(transports.second)); + } + private: + + boost::shared_ptr factory_; +}; + +/** + * Use TDualTransportFactory to construct input and output transports from + * different factories. + */ +class TDualTransportFactory : public TDuplexTransportFactory { + public: + TDualTransportFactory( + boost::shared_ptr inputFactory, + boost::shared_ptr outputFactory) : + inputFactory_(inputFactory), + outputFactory_(outputFactory) {} + + virtual TTransportPair getTransport(boost::shared_ptr trans) { + return std::make_pair(inputFactory_->getTransport(trans), + outputFactory_->getTransport(trans)); + } + + virtual TTransportPair getTransport(TTransportPair transports) { + return std::make_pair(inputFactory_->getTransport(transports.first), + outputFactory_->getTransport(transports.second)); + } + private: + + boost::shared_ptr inputFactory_; + boost::shared_ptr outputFactory_; +}; + +}}} // apache::thrift::transport + +#endif // #ifndef THRIFT_TRANSPORT_TTRANSPORT_H diff --git a/thrift/lib/cpp/transport/TTransportException.h b/thrift/lib/cpp/transport/TTransportException.h new file mode 100644 index 0000000000..28f5d19534 --- /dev/null +++ b/thrift/lib/cpp/transport/TTransportException.h @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_TRANSPORT_TTRANSPORTEXCEPTION_H_ +#define _THRIFT_TRANSPORT_TTRANSPORTEXCEPTION_H_ 1 + +#include +#include "thrift/lib/cpp/Thrift.h" + +namespace apache { namespace thrift { namespace transport { + +/** + * Class to encapsulate all the possible types of transport errors that may + * occur in various transport systems. This provides a sort of generic + * wrapper around the shitty UNIX E_ error codes that lets a common code + * base of error handling to be used for various types of transports, i.e. + * pipes etc. + * + */ +class TTransportException : public apache::thrift::TLibraryException { + public: + /** + * Error codes for the various types of exceptions. + */ + enum TTransportExceptionType + { UNKNOWN = 0 + , NOT_OPEN = 1 + , ALREADY_OPEN = 2 + , TIMED_OUT = 3 + , END_OF_FILE = 4 + , INTERRUPTED = 5 + , BAD_ARGS = 6 + , CORRUPTED_DATA = 7 + , INTERNAL_ERROR = 8 + , NOT_SUPPORTED = 9 + , INVALID_STATE = 10 + , INVALID_FRAME_SIZE = 11 + , SSL_ERROR = 12 + }; + + TTransportException() : + apache::thrift::TLibraryException(), + type_(UNKNOWN), errno_(0) {} + + TTransportException(TTransportExceptionType type) : + apache::thrift::TLibraryException(), + type_(type), errno_(0) {} + + TTransportException(const std::string& message) : + apache::thrift::TLibraryException(message), + type_(UNKNOWN), errno_(0) {} + + TTransportException(TTransportExceptionType type, const std::string& message) : + apache::thrift::TLibraryException(message), + type_(type), errno_(0) {} + + TTransportException(TTransportExceptionType type, + const std::string& message, + int errno_copy) : + apache::thrift::TLibraryException(getMessage(message, errno_copy)), + type_(type), errno_(errno_copy) {} + + virtual ~TTransportException() throw() {} + + /** + * Returns an error code that provides information about the type of error + * that has occurred. + * + * @return Error code + */ + TTransportExceptionType getType() const throw() { + return type_; + } + + virtual const char* what() const throw() { + if (message_.empty()) { + switch (type_) { + case UNKNOWN : return "TTransportException: Unknown transport exception"; + case NOT_OPEN : return "TTransportException: Transport not open"; + case ALREADY_OPEN : return "TTransportException: Transport already open"; + case TIMED_OUT : return "TTransportException: Timed out"; + case END_OF_FILE : return "TTransportException: End of file"; + case INTERRUPTED : return "TTransportException: Interrupted"; + case BAD_ARGS : return "TTransportException: Invalid arguments"; + case CORRUPTED_DATA : return "TTransportException: Corrupted Data"; + case INTERNAL_ERROR : return "TTransportException: Internal error"; + case NOT_SUPPORTED : return "TTransportException: Not supported"; + case INVALID_STATE : return "TTransportException: Invalid state"; + case INVALID_FRAME_SIZE: + return "TTransportException: Invalid frame size"; + case SSL_ERROR : return "TTransportException: SSL error"; + default : return "TTransportException: (Invalid exception type)"; + } + } else { + return message_.c_str(); + } + } + + int getErrno() const { return errno_; } + + protected: + /** Just like strerror_r but returns a C++ string object. */ + std::string strerror_s(int errno_copy); + + /** Return a message based on the input. */ + static std::string getMessage(const std::string &message, + int errno_copy) { + if (errno_copy != 0) { + return message + ": " + TOutput::strerror_s(errno_copy); + } else { + return message; + } + } + + /** Error code */ + TTransportExceptionType type_; + + /** A copy of the errno. */ + int errno_; +}; + +}}} // apache::thrift::transport + +#endif // #ifndef _THRIFT_TRANSPORT_TTRANSPORTEXCEPTION_H_ diff --git a/thrift/lib/cpp/transport/TTransportUtils.h b/thrift/lib/cpp/transport/TTransportUtils.h new file mode 100644 index 0000000000..8dfa4919f1 --- /dev/null +++ b/thrift/lib/cpp/transport/TTransportUtils.h @@ -0,0 +1,330 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_TRANSPORT_TTRANSPORTUTILS_H_ +#define _THRIFT_TRANSPORT_TTRANSPORTUTILS_H_ 1 + +#include +#include +#include +#include +#include "thrift/lib/cpp/transport/TTransport.h" +// Include the buffered transports that used to be defined here. +#include "thrift/lib/cpp/transport/TBufferTransports.h" +#include "thrift/lib/cpp/transport/TFileTransport.h" + +namespace apache { namespace thrift { namespace transport { + +/** + * The null transport is a dummy transport that doesn't actually do anything. + * It's sort of an analogy to /dev/null, you can never read anything from it + * and it will let you write anything you want to it, though it won't actually + * go anywhere. + * + */ +class TNullTransport : public TVirtualTransport { + public: + TNullTransport() {} + + ~TNullTransport() {} + + bool isOpen() { + return true; + } + + void open() {} + + void write(const uint8_t* /* buf */, uint32_t /* len */) { + return; + } + +}; + + +/** + * TPipedTransport. This transport allows piping of a request from one + * transport to another either when readEnd() or writeEnd(). The typical + * use case for this is to log a request or a reply to disk. + * The underlying buffer expands to a keep a copy of the entire + * request/response. + * + */ +class TPipedTransport : virtual public TTransport { + public: + TPipedTransport(boost::shared_ptr srcTrans, + boost::shared_ptr dstTrans) : + srcTrans_(srcTrans), + dstTrans_(dstTrans), + rBufSize_(512), rPos_(0), rLen_(0), + wBufSize_(512), wLen_(0) { + + // default is to to pipe the request when readEnd() is called + pipeOnRead_ = true; + pipeOnWrite_ = false; + + rBuf_ = (uint8_t*) std::malloc(sizeof(uint8_t) * rBufSize_); + if (rBuf_ == NULL) { + throw std::bad_alloc(); + } + wBuf_ = (uint8_t*) std::malloc(sizeof(uint8_t) * wBufSize_); + if (wBuf_ == NULL) { + throw std::bad_alloc(); + } + } + + TPipedTransport(boost::shared_ptr srcTrans, + boost::shared_ptr dstTrans, + uint32_t sz) : + srcTrans_(srcTrans), + dstTrans_(dstTrans), + rBufSize_(512), rPos_(0), rLen_(0), + wBufSize_(sz), wLen_(0) { + + rBuf_ = (uint8_t*) std::malloc(sizeof(uint8_t) * rBufSize_); + if (rBuf_ == NULL) { + throw std::bad_alloc(); + } + wBuf_ = (uint8_t*) std::malloc(sizeof(uint8_t) * wBufSize_); + if (wBuf_ == NULL) { + throw std::bad_alloc(); + } + } + + ~TPipedTransport() { + std::free(rBuf_); + std::free(wBuf_); + } + + bool isOpen() { + return srcTrans_->isOpen(); + } + + bool peek() { + if (rPos_ >= rLen_) { + // Double the size of the underlying buffer if it is full + if (rLen_ == rBufSize_) { + rBufSize_ *=2; + rBuf_ = (uint8_t *)std::realloc(rBuf_, sizeof(uint8_t) * rBufSize_); + } + + // try to fill up the buffer + rLen_ += srcTrans_->read(rBuf_+rPos_, rBufSize_ - rPos_); + } + return (rLen_ > rPos_); + } + + + void open() { + srcTrans_->open(); + } + + void close() { + srcTrans_->close(); + } + + void setPipeOnRead(bool pipeVal) { + pipeOnRead_ = pipeVal; + } + + void setPipeOnWrite(bool pipeVal) { + pipeOnWrite_ = pipeVal; + } + + uint32_t read(uint8_t* buf, uint32_t len); + + uint32_t readEnd() { + + if (pipeOnRead_) { + dstTrans_->write(rBuf_, rPos_); + dstTrans_->flush(); + } + + srcTrans_->readEnd(); + + // If requests are being pipelined, copy down our read-ahead data, + // then reset our state. + int read_ahead = rLen_ - rPos_; + uint32_t bytes = rPos_; + memcpy(rBuf_, rBuf_ + rPos_, read_ahead); + rPos_ = 0; + rLen_ = read_ahead; + + return bytes; + } + + void write(const uint8_t* buf, uint32_t len); + + uint32_t writeEnd() { + if (pipeOnWrite_) { + dstTrans_->write(wBuf_, wLen_); + dstTrans_->flush(); + } + return wLen_; + } + + void flush(); + + boost::shared_ptr getTargetTransport() { + return dstTrans_; + } + + /* + * Override TTransport *_virt() functions to invoke our implementations. + * We cannot use TVirtualTransport to provide these, since we need to inherit + * virtually from TTransport. + */ + virtual uint32_t read_virt(uint8_t* buf, uint32_t len) { + return this->read(buf, len); + } + virtual void write_virt(const uint8_t* buf, uint32_t len) { + this->write(buf, len); + } + + protected: + boost::shared_ptr srcTrans_; + boost::shared_ptr dstTrans_; + + uint8_t* rBuf_; + uint32_t rBufSize_; + uint32_t rPos_; + uint32_t rLen_; + + uint8_t* wBuf_; + uint32_t wBufSize_; + uint32_t wLen_; + + bool pipeOnRead_; + bool pipeOnWrite_; +}; + + +/** + * Wraps a transport into a pipedTransport instance. + * + */ +class TPipedTransportFactory : public TTransportFactory { + public: + TPipedTransportFactory() {} + TPipedTransportFactory(boost::shared_ptr dstTrans) { + initializeTargetTransport(dstTrans); + } + virtual ~TPipedTransportFactory() {} + + /** + * Wraps the base transport into a piped transport. + */ + virtual boost::shared_ptr getTransport(boost::shared_ptr srcTrans) { + return boost::shared_ptr(new TPipedTransport(srcTrans, dstTrans_)); + } + + virtual void initializeTargetTransport(boost::shared_ptr dstTrans) { + if (dstTrans_.get() == NULL) { + dstTrans_ = dstTrans; + } else { + throw TLibraryException("Target transport already initialized"); + } + } + + protected: + boost::shared_ptr dstTrans_; +}; + +/** + * TPipedFileTransport. This is just like a TTransport, except that + * it is a templatized class, so that clients who rely on a specific + * TTransport can still access the original transport. + * + */ +class TPipedFileReaderTransport : public TPipedTransport, + public TFileReaderTransport { + public: + TPipedFileReaderTransport(boost::shared_ptr srcTrans, boost::shared_ptr dstTrans); + + ~TPipedFileReaderTransport(); + + // TTransport functions + bool isOpen(); + bool peek(); + void open(); + void close(); + uint32_t read(uint8_t* buf, uint32_t len); + uint32_t readAll(uint8_t* buf, uint32_t len); + uint32_t readEnd(); + void write(const uint8_t* buf, uint32_t len); + uint32_t writeEnd(); + void flush(); + + // TFileReaderTransport functions + int32_t getReadTimeout(); + void setReadTimeout(int32_t readTimeout); + uint32_t getNumChunks(); + uint32_t getCurChunk(); + void seekToChunk(int32_t chunk); + void seekToEnd(); + + /* + * Override TTransport *_virt() functions to invoke our implementations. + * We cannot use TVirtualTransport to provide these, since we need to inherit + * virtually from TTransport. + */ + virtual uint32_t read_virt(uint8_t* buf, uint32_t len) { + return this->read(buf, len); + } + virtual uint32_t readAll_virt(uint8_t* buf, uint32_t len) { + return this->readAll(buf, len); + } + virtual void write_virt(const uint8_t* buf, uint32_t len) { + this->write(buf, len); + } + + protected: + // shouldn't be used + TPipedFileReaderTransport(); + boost::shared_ptr srcTrans_; +}; + +/** + * Creates a TPipedFileReaderTransport from a filepath and a destination transport + * + */ +class TPipedFileReaderTransportFactory : public TPipedTransportFactory { + public: + TPipedFileReaderTransportFactory() {} + TPipedFileReaderTransportFactory(boost::shared_ptr dstTrans) + : TPipedTransportFactory(dstTrans) + {} + virtual ~TPipedFileReaderTransportFactory() {} + + boost::shared_ptr getTransport(boost::shared_ptr srcTrans) { + boost::shared_ptr pFileReaderTransport = boost::dynamic_pointer_cast(srcTrans); + if (pFileReaderTransport.get() != NULL) { + return getFileReaderTransport(pFileReaderTransport); + } else { + return boost::shared_ptr(); + } + } + + boost::shared_ptr getFileReaderTransport(boost::shared_ptr srcTrans) { + return boost::shared_ptr(new TPipedFileReaderTransport(srcTrans, dstTrans_)); + } +}; + +}}} // apache::thrift::transport + +#endif // #ifndef _THRIFT_TRANSPORT_TTRANSPORTUTILS_H_ diff --git a/thrift/lib/cpp/transport/TVirtualTransport.h b/thrift/lib/cpp/transport/TVirtualTransport.h new file mode 100644 index 0000000000..26ab13a8b5 --- /dev/null +++ b/thrift/lib/cpp/transport/TVirtualTransport.h @@ -0,0 +1,134 @@ +// Copyright (c) 2006- Facebook +// Distributed under the Thrift Software License +// +// See accompanying file LICENSE or visit the Thrift site at: +// http://developers.facebook.com/thrift/ + +#ifndef _THRIFT_TRANSPORT_TVIRTUALTRANSPORT_H_ +#define _THRIFT_TRANSPORT_TVIRTUALTRANSPORT_H_ 1 + +#include "thrift/lib/cpp/transport/TTransport.h" + +namespace apache { namespace thrift { namespace transport { + + +/** + * Helper class that provides default implementations of TTransport methods. + * + * This class provides default implementations of read(), readAll(), write(), + * borrow() and consume(). + * + * In the TTransport base class, each of these methods simply invokes its + * virtual counterpart. This class overrides them to always perform the + * default behavior, without a virtual function call. + * + * The primary purpose of this class is to serve as a base class for + * TVirtualTransport, and prevent infinite recursion if one of its subclasses + * does not override the TTransport implementation of these methods. (Since + * TVirtualTransport::read_virt() calls read(), and TTransport::read() calls + * read_virt().) + */ +template +class TTransportDefaults : public Super_ { + public: + /* + * TTransport *_virt() methods provide reasonable default implementations. + * Invoke them non-virtually. + */ + uint32_t read(uint8_t* buf, uint32_t len) { + return this->TTransport::read_virt(buf, len); + } + uint32_t readAll(uint8_t* buf, uint32_t len) { + return this->TTransport::readAll_virt(buf, len); + } + void write(const uint8_t* buf, uint32_t len) { + this->TTransport::write_virt(buf, len); + } + const uint8_t* borrow(uint8_t* buf, uint32_t* len) { + return this->TTransport::borrow_virt(buf, len); + } + void consume(uint32_t len) { + this->TTransport::consume_virt(len); + } + + protected: + TTransportDefaults() {} +}; + +/** + * Helper class to provide polymorphism for subclasses of TTransport. + * + * This class implements *_virt() methods of TTransport, to call the + * non-virtual versions of these functions in the proper subclass. + * + * To define your own transport class using TVirtualTransport: + * 1) Derive your subclass from TVirtualTransport + * e.g: class MyTransport : public TVirtualTransport { + * 2) Provide your own implementations of read(), readAll(), etc. + * These methods should be non-virtual. + * + * Transport implementations that need to use virtual inheritance when + * inheriting from TTransport cannot use TVirtualTransport. + * + * @author Chad Walters + */ +template < class Transport_, class Super_=TTransportDefaults > +class TVirtualTransport : public Super_ { + public: + /* + * Implementations of the *_virt() functions, to call the subclass's + * non-virtual implementation function. + */ + virtual uint32_t read_virt(uint8_t* buf, uint32_t len) { + return static_cast(this)->read(buf, len); + } + + virtual uint32_t readAll_virt(uint8_t* buf, uint32_t len) { + return static_cast(this)->readAll(buf, len); + } + + virtual void write_virt(const uint8_t* buf, uint32_t len) { + static_cast(this)->write(buf, len); + } + + virtual const uint8_t* borrow_virt(uint8_t* buf, uint32_t* len) { + return static_cast(this)->borrow(buf, len); + } + + virtual void consume_virt(uint32_t len) { + static_cast(this)->consume(len); + } + + /* + * Provide a default readAll() implementation that invokes + * read() non-virtually. + * + * Note: subclasses that use TVirtualTransport to derive from another + * transport implementation (i.e., not TTransportDefaults) should beware that + * this may override any non-default readAll() implementation provided by + * the parent transport class. They may need to redefine readAll() to call + * the correct parent implementation, if desired. + */ + uint32_t readAll(uint8_t* buf, uint32_t len) { + Transport_* trans = static_cast(this); + return ::apache::thrift::transport::readAll(*trans, buf, len); + } + + protected: + TVirtualTransport() {} + + /* + * Templatized constructors, to allow arguments to be passed to the Super_ + * constructor. Currently we only support 0, 1, or 2 arguments, but + * additional versions can be added as needed. + */ + template + TVirtualTransport(Arg_ const& arg) : Super_(arg) { } + + template + TVirtualTransport(Arg1_ const& a1, Arg2_ const& a2) : Super_(a1, a2) { } +}; + +}}} // apache::thrift::transport + +#endif // #ifndef _THRIFT_TRANSPORT_TVIRTUALTRANSPORT_H_ diff --git a/thrift/lib/cpp/transport/TZlibTransport.h b/thrift/lib/cpp/transport/TZlibTransport.h new file mode 100644 index 0000000000..01cfb38bfb --- /dev/null +++ b/thrift/lib/cpp/transport/TZlibTransport.h @@ -0,0 +1,292 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_TRANSPORT_TZLIBTRANSPORT_H_ +#define _THRIFT_TRANSPORT_TZLIBTRANSPORT_H_ 1 + +#include +#include "thrift/lib/cpp/transport/TBufferTransports.h" +#include "thrift/lib/cpp/transport/TVirtualTransport.h" + +struct z_stream_s; + +namespace apache { namespace thrift { namespace transport { + +class TZlibTransportException : public TTransportException { + public: + TZlibTransportException(int status, const char* msg) : + TTransportException(TTransportException::INTERNAL_ERROR, + errorMessage(status, msg)), + zlib_status_(status), + zlib_msg_(msg == NULL ? "(null)" : msg) {} + + virtual ~TZlibTransportException() throw() {} + + int getZlibStatus() { return zlib_status_; } + std::string getZlibMessage() { return zlib_msg_; } + + static std::string errorMessage(int status, const char* msg) { + std::string rv = "zlib error: "; + if (msg) { + rv += msg; + } else { + rv += "(no message)"; + } + rv += " (status = "; + rv += boost::lexical_cast(status); + rv += ")"; + return rv; + } + + int zlib_status_; + std::string zlib_msg_; +}; + +/** + * This transport uses zlib's compressed format on the "far" side. + * + * There are two kinds of TZlibTransport objects: + * - Standalone objects are used to encode self-contained chunks of data + * (like structures). They include checksums. + * - Non-standalone transports are used for RPC. They are not implemented yet. + * + * TODO(dreiss): Don't do an extra copy of the compressed data if + * the underlying transport is TBuffered or TMemory. + * + */ +class TZlibTransport : public TVirtualTransport { + public: + + /** + * @param transport The transport to read compressed data from + * and write compressed data to. + * @param urbuf_size Uncompressed buffer size for reading. + * @param crbuf_size Compressed buffer size for reading. + * @param uwbuf_size Uncompressed buffer size for writing. + * @param cwbuf_size Compressed buffer size for writing. + * + * TODO(dreiss): Write a constructor that isn't a pain. + */ + TZlibTransport(boost::shared_ptr transport, + int urbuf_size = DEFAULT_URBUF_SIZE, + int crbuf_size = DEFAULT_CRBUF_SIZE, + int uwbuf_size = DEFAULT_UWBUF_SIZE, + int cwbuf_size = DEFAULT_CWBUF_SIZE) : + transport_(transport), + urpos_(0), + uwpos_(0), + input_ended_(false), + output_finished_(false), + urbuf_size_(urbuf_size), + crbuf_size_(crbuf_size), + uwbuf_size_(uwbuf_size), + cwbuf_size_(cwbuf_size), + urbuf_(NULL), + crbuf_(NULL), + uwbuf_(NULL), + cwbuf_(NULL), + rstream_(NULL), + wstream_(NULL) + { + if (uwbuf_size_ < MIN_DIRECT_DEFLATE_SIZE) { + // Have to copy this into a local because of a linking issue. + int minimum = MIN_DIRECT_DEFLATE_SIZE; + throw TTransportException( + TTransportException::BAD_ARGS, + "TZLibTransport: uncompressed write buffer must be at least" + + boost::lexical_cast(minimum) + "."); + } + + try { + urbuf_ = new uint8_t[urbuf_size]; + crbuf_ = new uint8_t[crbuf_size]; + uwbuf_ = new uint8_t[uwbuf_size]; + cwbuf_ = new uint8_t[cwbuf_size]; + + // Don't call this outside of the constructor. + initZlib(); + + } catch (...) { + delete[] urbuf_; + delete[] crbuf_; + delete[] uwbuf_; + delete[] cwbuf_; + throw; + } + } + + // Don't call this outside of the constructor. + void initZlib(); + + /** + * TZlibTransport destructor. + * + * Warning: Destroying a TZlibTransport object may discard any written but + * unflushed data. You must explicitly call flush() or finish() to ensure + * that data is actually written and flushed to the underlying transport. + */ + ~TZlibTransport(); + + bool isOpen(); + bool peek(); + + void open() { + transport_->open(); + } + + void close() { + transport_->close(); + } + + uint32_t read(uint8_t* buf, uint32_t len); + + void write(const uint8_t* buf, uint32_t len); + + void flush(); + + /** + * Finalize the zlib stream. + * + * This causes zlib to flush any pending write data and write end-of-stream + * information, including the checksum. Once finish() has been called, no + * new data can be written to the stream. + */ + void finish(); + + const uint8_t* borrow(uint8_t* buf, uint32_t* len); + + void consume(uint32_t len); + + /** + * Verify the checksum at the end of the zlib stream. + * + * This may only be called after all data has been read. + * It verifies the checksum that was written by the finish() call. + */ + void verifyChecksum(); + + /** + * TODO(someone_smart): Choose smart defaults. + */ + static const int DEFAULT_URBUF_SIZE = 128; + static const int DEFAULT_CRBUF_SIZE = 1024; + static const int DEFAULT_UWBUF_SIZE = 128; + static const int DEFAULT_CWBUF_SIZE = 1024; + + protected: + + inline void checkZlibRv(int status, const char* msg); + inline void checkZlibRvNothrow(int status, const char* msg); + inline int readAvail(); + void flushToTransport(int flush); + void flushToZlib(const uint8_t* buf, int len, int flush); + bool readFromZlib(); + + private: + // Deprecated constructor signature. + // + // This used to be the constructor signature. If you are getting a compile + // error because you are trying to use this constructor, you need to update + // your code as follows: + // - Remove the use_for_rpc argument in the constructur. + // There is no longer any distinction between RPC and standalone zlib + // transports. (Previously, only standalone was allowed, anyway.) + // - Replace TZlibTransport::flush() calls with TZlibTransport::finish() + // in your code. Previously, flush() used to finish the zlib stream. + // Now flush() only flushes out pending data, so more writes can be + // performed after a flush(). The finish() method can be used to finalize + // the zlib stream. + // + // If we don't declare this constructor, old code written as + // TZlibTransport(trans, false) still compiles but behaves incorrectly. + // The second bool argument is converted to an integer and used as the + // urbuf_size. + TZlibTransport(boost::shared_ptr transport, + bool use_for_rpc, + int urbuf_size = DEFAULT_URBUF_SIZE, + int crbuf_size = DEFAULT_CRBUF_SIZE, + int uwbuf_size = DEFAULT_UWBUF_SIZE, + int cwbuf_size = DEFAULT_CWBUF_SIZE); + + protected: + // Writes smaller than this are buffered up. + // Larger (or equal) writes are dumped straight to zlib. + static const int MIN_DIRECT_DEFLATE_SIZE = 32; + + boost::shared_ptr transport_; + + int urpos_; + int uwpos_; + + /// True iff zlib has reached the end of the input stream. + bool input_ended_; + /// True iff we have finished the output stream. + bool output_finished_; + + int urbuf_size_; + int crbuf_size_; + int uwbuf_size_; + int cwbuf_size_; + + uint8_t* urbuf_; + uint8_t* crbuf_; + uint8_t* uwbuf_; + uint8_t* cwbuf_; + + struct z_stream_s* rstream_; + struct z_stream_s* wstream_; +}; + + +/** + * Wraps a transport into a zlibbed one. + * + */ +class TZlibTransportFactory : public TTransportFactory { + public: + TZlibTransportFactory() {} + + virtual ~TZlibTransportFactory() {} + + virtual boost::shared_ptr getTransport( + boost::shared_ptr trans) { + return boost::shared_ptr(new TZlibTransport(trans)); + } +}; + +/** + * Wraps a transport into a framed, zlibbed one. + */ +class TFramedZlibTransportFactory : public TTransportFactory { + public: + TFramedZlibTransportFactory() {} + + virtual ~TFramedZlibTransportFactory() {} + + virtual boost::shared_ptr getTransport( + boost::shared_ptr trans) { + boost::shared_ptr framedTransport(new TFramedTransport(trans)); + return boost::shared_ptr(new TZlibTransport(framedTransport)); + } +}; + + +}}} // apache::thrift::transport + +#endif // #ifndef _THRIFT_TRANSPORT_TZLIBTRANSPORT_H_ diff --git a/thrift/lib/cpp/transport/test/TARGETS b/thrift/lib/cpp/transport/test/TARGETS new file mode 100644 index 0000000000..e024909eb4 --- /dev/null +++ b/thrift/lib/cpp/transport/test/TARGETS @@ -0,0 +1,69 @@ +cpp_unittest( + name = 'THeaderTest', + srcs = [ + 'THeaderTest.cpp', + ], + deps = [ + '@/thrift/lib/cpp/transport:header', + ], + external_deps = [ + ('boost', '>= 1.37', 'boost_unit_test_framework'), + ], + type = 'boost', + emails = ['thrift-team@lists.facebook.com'], +) + +cpp_unittest( + name = 'TSocketAddressTest', + srcs = [ + 'TSocketAddressTest.cpp', + ], + deps = [ + '@/thrift/lib/cpp/transport', + ], + external_deps = [ + ('boost', '>= 1.37', 'boost_unit_test_framework'), + ], + type = 'boost', + emails = ['thrift-team@lists.facebook.com'], +) + +cpp_unittest( + name = 'TSSLSocketTest', + srcs = [ + 'TSSLSocketTest.cpp', + ], + deps = [ + '@/thrift/lib/cpp/transport:transport_ssl', + '@/thrift/lib/cpp/util', + ], + external_deps = [ + ('boost', '>= 1.37', 'boost_unit_test_framework'), + ], + type = 'boost', + emails = [ + 'ti-bugs@lists.facebook.com', + 'thrift-team@lists.facebook.com', + ], +) + +cpp_unittest ( + name = "TSocketPoolTest", + srcs = [ + "TSocketPoolTest.cpp", + ], + deps = [ + '@/common/fbunit', + '@/thrift/lib/cpp/transport', + ], +) + +cpp_binary ( + name = "ttransport_test", + srcs = [ + "TransportTest.cpp", + ], + deps = [ + "@/thrift/lib/cpp/transport", + ] +) diff --git a/thrift/lib/cpp/util/AsyncClientUtil.h b/thrift/lib/cpp/util/AsyncClientUtil.h new file mode 100644 index 0000000000..9cd68bb72e --- /dev/null +++ b/thrift/lib/cpp/util/AsyncClientUtil.h @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_UTIL_ASYNCCLIENTUTIL_H_ +#define THRIFT_UTIL_ASYNCCLIENTUTIL_H_ 1 + +#include "thrift/lib/cpp/async/TAsyncSocket.h" +#include "thrift/lib/cpp/async/TFramedAsyncChannel.h" +#include "thrift/lib/cpp/protocol/TBinaryProtocol.h" +#include "thrift/lib/cpp/transport/TBufferTransports.h" + +namespace apache { namespace thrift { namespace util { + +/* + * Create an async client from a TSocketAddress + */ +template, + typename ChannelT = async::TFramedAsyncChannel> +ClientT* createClient(async::TEventBase* eventBase, + const transport::TSocketAddress& address) { + boost::shared_ptr transport( + async::TAsyncSocket::newSocket(eventBase, address)); + boost::shared_ptr channel( + ChannelT::newChannel(transport)); + + ProtocolFactoryT protocolFactory; + return new ClientT(channel, &protocolFactory); +} + +/* + * Create an async client from an IP and port + */ +template, + typename ChannelT = async::TFramedAsyncChannel> +ClientT* createClient(async::TEventBase* eventBase, + const std::string& ip, uint16_t port) { + // Note that we intentionally use setFromIpPort() and not setFromHostPort() + // here. If users want asynchronous operation they almost certainly don't + // want us to perform a blocking DNS lookup operation that may take a long + // time. + transport::TSocketAddress address; + address.setFromIpPort(ip, port); + return createClient( + eventBase, address); +} + +}}} // apache::thrift::util + +#endif // THRIFT_UTIL_ASYNCCLIENTUTIL_H_ diff --git a/thrift/lib/cpp/util/BitwiseCast.h b/thrift/lib/cpp/util/BitwiseCast.h new file mode 100644 index 0000000000..0c9461fc38 --- /dev/null +++ b/thrift/lib/cpp/util/BitwiseCast.h @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef THRIFT_UTIL_BITWISECAST_H_ +#define THRIFT_UTIL_BITWISECAST_H_ + +#include + +// Use this to get around strict aliasing rules. +// For example, uint64_t i = bitwise_cast(returns_double()); +// The most obvious implementation is to just cast a pointer, +// but that doesn't work. +// For a pretty in-depth explanation of the problem, see +// http://cellperformance.beyond3d.com/articles/2006/06/ (...) +// understanding-strict-aliasing.html +template +static inline To bitwise_cast(From from) { + BOOST_STATIC_ASSERT(sizeof(From) == sizeof(To)); + + // BAD!!! These are all broken with -O2. + //return *reinterpret_cast(&from); // BAD!!! + //return *static_cast(static_cast(&from)); // BAD!!! + //return *(To*)(void*)&from; // BAD!!! + + // Super clean and partially blessed by section 3.9 of the standard. + //unsigned char c[sizeof(from)]; + //memcpy(c, &from, sizeof(from)); + //To to; + //memcpy(&to, c, sizeof(c)); + //return to; + + // Slightly more questionable. + // Same code emitted by GCC. + //To to; + //memcpy(&to, &from, sizeof(from)); + //return to; + + // Technically undefined, but almost universally supported, + // and the most efficient implementation. + union { + From f; + To t; + } u; + u.f = from; + return u.t; +} + +#endif diff --git a/thrift/lib/cpp/util/EnumUtils.h b/thrift/lib/cpp/util/EnumUtils.h new file mode 100644 index 0000000000..0bf43afb05 --- /dev/null +++ b/thrift/lib/cpp/util/EnumUtils.h @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_UTIL_ENUMUTILS_H_ +#define THRIFT_UTIL_ENUMUTILS_H_ 1 + +namespace apache { namespace thrift { + +namespace util { + +/** + * Parses an enum name to the enum type + */ +template +bool tryParseEnum(const String& name, EnumType* out) { + return TEnumTraits::findValue(name.c_str(), out); +}; + +template +bool tryParseEnum(const char* name, EnumType* out) { + return TEnumTraits::findValue(name, out); +}; + +/** + * Returns the human-readable name for an Enum type. + */ +template +const char* enumName(EnumType value, + const char* defaultName = nullptr) { + const char* name = TEnumTraits::findName(value); + if (name) { + return name; + } else { + return defaultName; + } +}; + +}}} // apache::thrift::util + +#endif // THRIFT_UTIL_ENUMUTILS_H_ 1 diff --git a/thrift/lib/cpp/util/ScopedServerThread.h b/thrift/lib/cpp/util/ScopedServerThread.h new file mode 100644 index 0000000000..fe537bd360 --- /dev/null +++ b/thrift/lib/cpp/util/ScopedServerThread.h @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_UTIL_SCOPEDSERVERTHREAD_H_ +#define THRIFT_UTIL_SCOPEDSERVERTHREAD_H_ 1 + +#include +#include + +namespace apache { namespace thrift { + +namespace concurrency { +class Thread; +} +namespace server { +class TServer; +} +namespace transport { +class TSocketAddress; +} + +namespace util { + +class ServerCreator; + +/** + * ScopedServerThread spawns a thrift server in a new thread. + * + * The server is stopped automatically when the ScopedServerThread is + * destroyed. + */ +class ScopedServerThread : public boost::noncopyable { + public: + /** + * Create a new, unstarted ScopedServerThread object. + */ + ScopedServerThread(); + + /** + * Create a ScopedServerThread object and automatically start it. + */ + ScopedServerThread(ServerCreator* serverCreator); + + /** + * Create a ScopedServerThread object and automatically start it. + */ + ScopedServerThread(const boost::shared_ptr& server); + + virtual ~ScopedServerThread(); + + /** + * Start the server thread. + * + * This method does not return until the server has successfully started. + * + * @param serverCreator The ServerCreator object to use to create the server. + */ + void start(ServerCreator* serverCreator); + + /** + * Start the server thread. + * + * This method does not return until the server has successfully started. + * + * @param server The server to run in the new thread. + */ + void start(const boost::shared_ptr& server); + + /** + * Stop the server thread. + */ + void stop(); + + /** + * Get the address on which the server is listening. + */ + const transport::TSocketAddress* getAddress() const; + + private: + class Helper; + + boost::shared_ptr helper_; + boost::shared_ptr thread_; +}; + +}}} // apache::thrift::util + +#endif // THRIFT_UTIL_SCOPEDSERVERTHREAD_H_ diff --git a/thrift/lib/cpp/util/ServerCreator.h b/thrift/lib/cpp/util/ServerCreator.h new file mode 100644 index 0000000000..b573f084ec --- /dev/null +++ b/thrift/lib/cpp/util/ServerCreator.h @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_UTIL_SERVERCREATOR_H_ +#define THRIFT_UTIL_SERVERCREATOR_H_ 1 + +#include + +namespace apache { namespace thrift { + +namespace server { +class TServer; +} + +namespace util { + +/** + * ServerCreator is an abstract class for creating a thrift server. + */ +class ServerCreator { + public: + virtual ~ServerCreator() {} + + /** + * Create a new server. + */ + virtual boost::shared_ptr createServer() = 0; + + protected: + ServerCreator() {} +}; + +}}} // apache::thrift::util + +#endif // THRIFT_UTIL_SERVERCREATOR_H_ diff --git a/thrift/lib/cpp/util/ServerCreatorBase.h b/thrift/lib/cpp/util/ServerCreatorBase.h new file mode 100644 index 0000000000..0d7d35e45d --- /dev/null +++ b/thrift/lib/cpp/util/ServerCreatorBase.h @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_UTIL_SERVERCREATORBASE_H_ +#define THRIFT_UTIL_SERVERCREATORBASE_H_ 1 + +#include "thrift/lib/cpp/util/ServerCreator.h" + +namespace apache { namespace thrift { + +namespace protocol { +class TProtocolFactory; +class TDuplexProtocolFactory; +} +namespace server { +class TServerEventHandler; +} + +namespace util { + +/** + * ServerCreatorBase is a helper class for subclass that wish to implement the + * ServerCreator interface. + * + * ServerCreatorBase provides functionality that pretty much all ServerCreators + * need. It provides default protocol settings, and also + * setServerEventHandler(). + */ +class ServerCreatorBase : public ServerCreator { + public: + /** + * Use strict read by default. + * + * We should always use strict read and strict write for all new code. + * Only legacy services should allow requests without a thrift ID. + */ + static const bool DEFAULT_STRICT_READ = true; + + /** + * Use strict write by default. + */ + static const bool DEFAULT_STRICT_WRITE = true; + + /** + * Default string limit. Set a reasonably small value so that servers won't + * fall over when sent garbage data. + */ + static const int32_t DEFAULT_STRING_LIMIT = 256*1024*1024; + + /** + * Default container limit. Set a reasonably small value so that servers + * won't fall over when sent garbage data. + */ + static const int32_t DEFAULT_CONTAINER_LIMIT = 256*1024*1024; + + /** + * Set the TServerEventHandler to use with the new server. + * + * This event handler will be set on servers created by this ServerCreator. + */ + void setServerEventHandler( + const boost::shared_ptr& eventHandler) { + serverEventHandler_ = eventHandler; + } + + /** + * Set whether or not TBinaryProtocol should include an identifier field. + * + * This field helps verify that incoming data is a valid thrift request, + * rather than random garbage data. + * + * This setting is ignored if you explicitly specify a TProtocolFactory to + * use. It is only honored if you let the ServerCreator create a new + * TBinaryProtocol for you. + * + * If unspecified, it defaults to true for both strictRead and strictWrite. + * + * @param strictRead Reject incoming requests that don't include a thrift + * identifier. + * @param strictWrite Send a thrift identifier in outgoing responses. + */ + void setStrictProtocol(bool strictRead, bool strictWrite); + + /** + * Set the maximum string size allowed by TBinaryProtocol. + * + * This setting helps prevent TBinaryProtocol from trying to allocate an + * extremely large buffer if a request contains a bogus string length field. + * + * This setting is ignored if you explicitly specify a TProtocolFactory to + * use. It is only honored if you let the ServerCreator create a new + * TBinaryProtocol for you. + * + * If unspecified, it defaults to 256MB. + */ + void setStringSizeLimit(int32_t stringLimit); + + /** + * Set the maximum container size allowed by TBinaryProtocol. + * + * This setting helps prevent TBinaryProtocol from trying to allocate an + * extremely large buffer if a request contains a bogus container length + * field. (e.g., for list, map, and set fields.) + * + * This setting is ignored if you explicitly specify a TProtocolFactory to + * use. It is only honored if you let the ServerCreator create a new + * TBinaryProtocol for you. + * + * If unspecified, it defaults to 256MB. + */ + void setContainerSizeLimit(int32_t containerLimit); + + /** + * Set the protocol factory to use. + * + * This causes the ServerCreator to use the specified protocol factory. + * This overrides the TBinaryProtocol-related settings that would normally be + * used by the ServerCreator. + */ + void setProtocolFactory(const boost::shared_ptr&); + + /** + * Set the protocol factory to use. + * + * This causes the ServerCreator to use the specified protocol factory + * to construct custom input and output protocols. This overrides the + * TBinaryProtocol-related settings that would normally be used by the + * ServerCreator. + */ + void setDuplexProtocolFactory( + const boost::shared_ptr&); + + /** + * Create a new server. + */ + virtual boost::shared_ptr createServer() = 0; + + protected: + ServerCreatorBase(); + + /** + * Configure the TServer as desired. + * + * Subclasses should call this method in createServer(). + */ + virtual void configureServer(const boost::shared_ptr& srv); + + /** + * Get the TProtocolFactory to use for the server. + * + * Subclasses should call this method in createServer(). + */ + virtual boost::shared_ptr getProtocolFactory(); + + bool strictRead_; + bool strictWrite_; + int32_t stringLimit_; + int32_t containerLimit_; + boost::shared_ptr serverEventHandler_; + boost::shared_ptr protocolFactory_; + boost::shared_ptr duplexProtocolFactory_; +}; + +}}} // apache::thrift::util + +#endif // THRIFT_UTIL_SERVERCREATORBASE_H_ diff --git a/thrift/lib/cpp/util/SyncServerCreator.h b/thrift/lib/cpp/util/SyncServerCreator.h new file mode 100644 index 0000000000..bf988017ff --- /dev/null +++ b/thrift/lib/cpp/util/SyncServerCreator.h @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_UTIL_SYNCSERVERCREATOR_H_ +#define THRIFT_UTIL_SYNCSERVERCREATOR_H_ 1 + +#include "thrift/lib/cpp/util/ServerCreatorBase.h" + +#include + +namespace apache { namespace thrift { + +class TProcessor; + +namespace transport { +class TTransportFactory; +class TDuplexTransportFactory; +class TServerSocket; +} + +namespace util { + +/** + * Helper class for the standard synchronous server types (TSimpleServer, + * TThreadedServer, TThreadPoolServer) + */ +class SyncServerCreator : public ServerCreatorBase { + public: + /** + * Default to a 30 second receive timeout. + * + * Almost everyone should use a receive timeout, to avoid tracking connection + * state indefinitely if a client drops of the network without closing the + * connection. + */ + static const int DEFAULT_RECEIVE_TIMEOUT = 30000; + + /** + * Default to a 5 second send timeout. + * + * This is still a fairly large value. Some users may want to decrease this. + */ + static const int DEFAULT_SEND_TIMEOUT = 5000; + + /** + * Don't use an accept timeout by default. + * + * Almost everyone wants this. With a timeout, serve() will raise an + * exception if a new connection doesn't arrive within the timeout interval. + */ + static const int DEFAULT_ACCEPT_TIMEOUT = -1; + + /** + * By default, don't mess with the kernel's default TCP send buffer size. + */ + static const int DEFAULT_TCP_SEND_BUFFER = -1; + + /** + * By default, don't mess with the kernel's default TCP receive buffer size. + */ + static const int DEFAULT_TCP_RECEIVE_BUFFER = -1; + + /** + * Create a new SyncServerCreator + */ + SyncServerCreator(const boost::shared_ptr& processor, + uint16_t port, bool framed = true); + + /** + * Create a new SyncServerCreator + */ + SyncServerCreator(const boost::shared_ptr& processor, + uint16_t port, + boost::shared_ptr& tf, + boost::shared_ptr& pf); + + /** + * Set the send timeout. + * + * Connections will be dropped if no progress is made sending a response + * for this many milliseconds. + */ + void setSendTimeout(int milliseconds); + + /** + * Set the receive timeout. + * + * Connections will be dropped if there are no requests for this many + * milliseconds. + */ + void setRecvTimeout(int milliseconds); + + /** + * Set the accept timeout. + * + * serve() will raise an exception if no new connections are received for + * this many milliseconds. + */ + void setAcceptTimeout(int milliseconds); + + /** + * Set the TCP send buffer size. + */ + void setTcpSendBuffer(int tcpSendBuffer); + + /** + * Set the TCP receive buffer size. + */ + void setTcpRecvBuffer(int tcpRecvBuffer); + + /** + * Set the transport factory + */ + void setTransportFactory( + const boost::shared_ptr& tf) { + transportFactory_ = tf; + } + + /** + * Set the duplex transport factory. This overrides the base + * transportFactory_ if one was specified + */ + void setDuplexTransportFactory( + const boost::shared_ptr& tf) { + duplexTransportFactory_ = tf; + } + + /** + * Get the duplex transport factory, instantiating one from the base + * transportFactory_ if needed + */ + boost::shared_ptr getDuplexTransportFactory(); + /** + * Create a new server. + */ + virtual boost::shared_ptr createServer() = 0; + + protected: + void init(const boost::shared_ptr& processor, uint16_t port); + + boost::shared_ptr createServerSocket(); + + uint16_t port_; + int sendTimeout_; + int recvTimeout_; + int acceptTimeout_; + int tcpSendBuffer_; + int tcpRecvBuffer_; + boost::shared_ptr processor_; + boost::shared_ptr transportFactory_; + boost::shared_ptr duplexTransportFactory_; +}; + +}}} // apache::thrift::util + +#endif // THRIFT_UTIL_SYNCSERVERCREATOR_H_ diff --git a/thrift/lib/cpp/util/TARGETS b/thrift/lib/cpp/util/TARGETS new file mode 100644 index 0000000000..0f0995f952 --- /dev/null +++ b/thrift/lib/cpp/util/TARGETS @@ -0,0 +1,74 @@ +cpp_library( + name = 'util', + srcs = [ + 'ScopedServerThread.cpp', + 'ServerCreatorBase.cpp', + 'SyncServerCreator.cpp', + 'TSimpleServerCreator.cpp', + 'TThreadedServerCreator.cpp', + 'TThreadPoolServerCreator.cpp', + ], + deps = [ + '@/thrift/lib/cpp/server', + '@/thrift/lib/cpp/protocol', + '@/folly', + ], +) + +cpp_library( + name = 'internal_util', + srcs = [ + 'VarintUtils.cpp', + ], + deps = [ + '@/folly', + ], + external_deps = [ + ('gflags', None, 'gflags'), + ('glog', None, 'glog'), + ], +) + +cpp_library( + name = 'nonblocking', + srcs = [ + 'TNonblockingServerCreator.cpp', + ], + deps = [ + ':util', + '@/thrift/lib/cpp/server:nonblocking', + ], +) + +cpp_library( + name = 'async', + srcs = [ + 'TEventServerCreator.cpp', + ], + deps = [ + ':util', + '@/thrift/lib/cpp/async', + ], +) + +cpp_library( + name = 'httpparser', + srcs = [ + 'THttpParser.cpp', + ], + deps = [ + '@/thrift/lib/cpp/transport', +], +) + +cpp_unittest( + name = 'ThriftSerializerTest', + srcs = [ 'ThriftSerializerTest.cpp' ], + thrift_srcs = { 'ThriftSerializerTest.thrift' : [ ] }, + thrift_cpp_options = 'templates', + deps = [ '@/common/strings:strings', + '@/common/fbunit', + ], +) + + diff --git a/thrift/lib/cpp/util/TEventServerCreator.h b/thrift/lib/cpp/util/TEventServerCreator.h new file mode 100644 index 0000000000..0c2dc53f97 --- /dev/null +++ b/thrift/lib/cpp/util/TEventServerCreator.h @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_UTIL_TEVENTSERVERCREATOR_H_ +#define THRIFT_UTIL_TEVENTSERVERCREATOR_H_ 1 + +#include "thrift/lib/cpp/util/ServerCreatorBase.h" + +#include + +namespace apache { namespace thrift { + +namespace async { +class TAsyncProcessor; +class TEventServer; +} + +namespace util { + +class TEventServerCreator : public ServerCreatorBase { + public: + typedef async::TEventServer ServerType; + + /// Use 8 worker threads by default. + static const size_t DEFAULT_NUM_THREADS = 8; + + /// Default limit on the size of each worker's idle connection pool + static const uint32_t DEFAULT_CONN_POOL_SIZE = 64; + + /// By default, close connections after they have are idle for 60 seconds + static const int DEFAULT_RECV_TIMEOUT = 60000; + + /** + * By default, reject requests over 64MB. + * + * This avoids allocating giant buffers if a client sends a bogus frame + * length. + */ + static const size_t DEFAULT_MAX_FRAME_SIZE = 64 * 1024 * 1024; + + /** + * Start dropping connections to reduce load if a worker's event loop begins + * taking longer than 2 seconds to process a single event loop. + */ + static const int64_t DEFAULT_WORKER_LATENCY = 2000; + + /// Default size of each connection's write buffer + static const size_t DEFAULT_WRITE_BUFFER_SIZE = 1024; + + /// Default size of each connection's read buffer + static const size_t DEFAULT_READ_BUFFER_SIZE = 1024; + + /// Maximum size of read buffer allocated to each idle connection + static const size_t DEFAULT_IDLE_READ_BUF_LIMIT = 8192; + + /// Maximum size of write buffer allocated to each idle connection + static const size_t DEFAULT_IDLE_WRITE_BUF_LIMIT = 8192; + + /** + * By check to see if we should shrink oversized read/write buffers after + * every 64 calls on connection. + */ + static const int DEFAULT_RESIZE_EVERY_N = 64; + + /** + * Create a new TEventServerCreator. + */ + TEventServerCreator(const boost::shared_ptr& proc, + uint16_t port, + size_t numThreads = DEFAULT_NUM_THREADS); + + /** + * Set the number of threads to use. + */ + void setNumThreads(size_t numThreads) { + numThreads_ = numThreads; + } + + /** + * Set the maximum number of TEventConnection objects to cache in each worker + * thread. + * + * By default, when a connection is closed the worker thread caches the + * TEventConnection object and re-uses it when a new connection comes in, + * instead of deleting it and allocating a new one for the next connection. + * This value caps the number of unused TEventConnection objects that will be + * cached. + */ + void setMaxConnectionPoolSize(uint32_t size) { + maxConnPoolSize_ = size; + } + + /** + * Set the maximum amount of time a connection may be idle before it is + * closed. + */ + void setRecvTimeout(int milliseconds) { + recvTimeout_ = milliseconds; + } + + /// Set the maximum allowed request frame size. + void setMaxFrameSize(uint32_t maxFrameSize) { + maxFrameSize_ = maxFrameSize; + } + + /// Set the default write buffer size for new connections + void setReadBufferDefaultSize(size_t size) { + defaultReadBufferSize_ = size; + } + + /// Set the default write buffer size for new connections + void setWriteBufferDefaultSize(size_t size) { + defaultWriteBufferSize_ = size; + } + + /** + * Set the maximum read buffer size for idle connections. + * + * This value is checked when the connection is closed, and also every N + * connections if setResizeBufferEveryN() has been set. If the read buffer + * is larger than the allowed limit, free it. + * + * This prevents a single large request on a connection from continuing to + * taking up a lot of buffer space. + */ + void setIdleReadBufferLimit(size_t limit) { + idleReadBufferLimit_ = limit; + } + + /** + * Set the maximum write buffer size for idle connections. + * + * This value is checked when the connection is closed, and also every N + * connections if setResizeBufferEveryN() has been set. If the write buffer + * is larger than the allowed limit, free it. + * + * This prevents a single large response on a connection from continuing to + * taking up a lot of buffer space. + */ + void setIdleWriteBufferLimit(size_t limit) { + idleWriteBufferLimit_ = limit; + } + + /// Check a connection's buffer size limits every count requests. + void setResizeBufferEveryN(int32_t count) { + resizeBufferEveryN_ = count; + } + + /** + * Set the worker latency limit. + * + * Begin taking overload actions when it takes longer than this amount of + * time to process a single iteration of the event loop. + */ + void setWorkerLatencyLimit(int64_t milliseconds) { + workerLatencyLimit_ = milliseconds; + } + + virtual boost::shared_ptr createServer(); + + boost::shared_ptr createEventServer(); + + private: + boost::shared_ptr processor_; + uint16_t port_; + size_t numThreads_; + uint32_t maxConnPoolSize_; + int recvTimeout_; + uint32_t maxFrameSize_; + size_t defaultReadBufferSize_; + size_t defaultWriteBufferSize_; + size_t idleReadBufferLimit_; + size_t idleWriteBufferLimit_; + int32_t resizeBufferEveryN_; + int32_t workerLatencyLimit_; +}; + +}}} // apache::thrift::util + +#endif // THRIFT_UTIL_TEVENTSERVERCREATOR_H_ diff --git a/thrift/lib/cpp/util/THttpParser.h b/thrift/lib/cpp/util/THttpParser.h new file mode 100644 index 0000000000..de953f5118 --- /dev/null +++ b/thrift/lib/cpp/util/THttpParser.h @@ -0,0 +1,130 @@ +#ifndef THRIFT_TRANSPORT_THTTPPARSER_H_ +#define THRIFT_TRANSPORT_THTTPPARSER_H_ 1 + +#include + +namespace apache { namespace thrift { namespace util { + +class THttpParser { + protected: + enum HttpParseState { + HTTP_PARSE_START = 0, + HTTP_PARSE_HEADER, + HTTP_PARSE_CHUNK, + HTTP_PARSE_CONTENT, + HTTP_PARSE_CHUNKFOOTER, + HTTP_PARSE_TRAILING + }; + + enum HttpParseResult { + HTTP_PARSE_RESULT_CONTINUE, + HTTP_PARSE_RESULT_BLOCK + }; + + public: + THttpParser(); + ~THttpParser(); + + void getReadBuffer(void** bufReturn, size_t* lenReturn); + bool readDataAvailable(size_t len); + void setDataBuffer(apache::thrift::transport::TMemoryBuffer* buffer) { + dataBuf_ = buffer; + } + void unsetDataBuffer() { + dataBuf_ = NULL; + } + void setMaxSize(uint32_t size) { + maxSize_ = size; + } + uint32_t getMaxSize() { + return maxSize_; + } + bool hasReadAheadData() { + return (state_ == HTTP_PARSE_START) && (httpBufLen_ > httpPos_); + } + bool hasPartialMessage() { + return partialMessageSize_ > 0; + } + virtual int constructHeader(iovec* ops, + int opsLen, + int contentLength, + char* contentLengthBuf) = 0; + + protected: + HttpParseResult parseStart(); + HttpParseResult parseHeader(); + HttpParseResult parseContent(); + HttpParseResult parseChunk(); + HttpParseResult parseChunkFooter(); + HttpParseResult parseTrailing(); + + virtual bool parseStatusLine(const char* status) = 0; + virtual void parseHeaderLine(const char* header) = 0; + + void shift(); + char* readLine(); + void checkMessageSize(uint32_t more, bool added); + + char* httpBuf_; + uint32_t httpPos_; + uint32_t httpBufLen_; + uint32_t httpBufSize_; + + HttpParseState state_; + + // for read header + bool statusLine_; + bool finished_; + bool chunked_; + + size_t contentLength_; + + // max http message size + uint32_t maxSize_; + uint32_t partialMessageSize_; + + apache::thrift::transport::TMemoryBuffer* dataBuf_; + + static const char* CRLF; + static const int CRLF_LEN; +}; + +class THttpClientParser : public THttpParser { + public: + THttpClientParser(std::string host, std::string path) { + host_ = host; + path_ = path; + userAgent_ = "C++/THttpClient"; + } + void setHost(const std::string& host) { host_ = host; } + void setPath(const std::string& path) { path_ = path; } + void resetConnectClosedByServer(); + bool isConnectClosedByServer(); + void setUserAgent(std::string userAgent) { + userAgent_ = userAgent; + } + virtual int constructHeader(iovec* ops, + int opsLen, + int contentLength, + char* contentLengthBuf); + + protected: + virtual void parseHeaderLine(const char* header); + virtual bool parseStatusLine(const char* status); + void setiovec(iovec* ops, const char* data, int size) { + ops->iov_base = (void*)data; + ops->iov_len = size; + } + + private: + bool connectionClosedByServer_; + std::string host_; + std::string path_; + std::string userAgent_; +}; + + +}}} // apache::thrift::util + +#endif // #ifndef THRIFT_TRANSPORT_THTTPPARSER_H_ + diff --git a/thrift/lib/cpp/util/TNonblockingServerCreator.h b/thrift/lib/cpp/util/TNonblockingServerCreator.h new file mode 100644 index 0000000000..4bd962855b --- /dev/null +++ b/thrift/lib/cpp/util/TNonblockingServerCreator.h @@ -0,0 +1,275 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_UTIL_TNONBLOCKINGSERVERCREATOR_H_ +#define THRIFT_UTIL_TNONBLOCKINGSERVERCREATOR_H_ 1 + +#include "thrift/lib/cpp/util/ServerCreatorBase.h" + +#include "thrift/lib/cpp/server/TNonblockingServer.h" +#include "thrift/lib/cpp/transport/TSocket.h" + +namespace apache { namespace thrift { + +class TProcessor; + +namespace util { + +class TNonblockingServerCreator : public ServerCreatorBase { + public: + typedef server::TNonblockingServer ServerType; + + /// Use 8 task threads by default. + static const size_t DEFAULT_NUM_TASK_THREADS = 8; + + /// Use 1 I/O thread by default. + static const size_t DEFAULT_NUM_IO_THREADS = 1; + + /** + * By default, just run the I/O threads with the same priority as other + * threads. + */ + static const bool DEFAULT_HI_PRI_IO_THREADS = false; + + /// Default limit on the size of the idle connection pool + static const size_t DEFAULT_CONN_STACK_LIMIT = 1024; + + /// Default limit on the total number of connected sockets + static const size_t DEFAULT_MAX_CONNECTIONS = INT_MAX; + + /// Default limit on the number of outstanding requests + static const size_t DEFAULT_MAX_ACTIVE_PROCESSORS = INT_MAX; + + /** + * By default, reject requests over 256MB. + * + * This avoids allocating giant buffers if a client sends a bogus frame + * length. + */ + static const size_t DEFAULT_MAX_FRAME_SIZE = 256 * 1024 * 1024; + + /// Default overload hysteresis fraction + static const double DEFAULT_HYSTERESIS_FRACTION; + + /** + * Default overload action. + * + * TODO: None of the TNonblockingServer OverloadAction behaviors seem like + * reasonable defaults. If the overload is occurring because we hit the + * maxConnections limit, it seems like the best behavior is to try and close + * idle connections. If the overload is occurring due to the + * maxActiveProcessors limit, it seems like the best behavior may be to stop + * calling accept() and stop reading on sockets until the # of active + * processors drops to a reasonable limit. + */ + static const server::TOverloadAction DEFAULT_OVERLOAD_ACTION = + server::T_OVERLOAD_NO_ACTION; + + /** + * By default, give up on a request if we can't find a thread to process it + * within 30 seconds. + */ + static const int64_t DEFAULT_TASK_EXPIRE_TIME = 30000; + + /// Default size of write buffer + static const size_t DEFAULT_WRITE_BUFFER_SIZE = 1024; + + /// Maximum size of read buffer allocated to each idle connection + static const size_t DEFAULT_IDLE_READ_BUF_LIMIT = 8192; + + /// Maximum size of write buffer allocated to each idle connection + static const size_t DEFAULT_IDLE_WRITE_BUF_LIMIT = 8192; + + /** + * By check to see if we should shrink oversized read/write buffers after + * every 64 calls on connection. + */ + static const int DEFAULT_RESIZE_EVERY_N = 64; + + /// listen backlog + static const size_t DEFAULT_LISTEN_BACKLOG = 1024; + + /** + * Create a new TNonblockingServerCreator. + */ + TNonblockingServerCreator(const boost::shared_ptr& processor, + uint16_t port, + size_t numTaskThreads = DEFAULT_NUM_TASK_THREADS); + + /** + * Set the number of threads to use for processing requests. + * + * Setting this to 0 causes all requests to be processed in the I/O + * thread(s). You should only set this to 0 if the handler responds to all + * requests immediately, without ever blocking. + */ + void setNumTaskThreads(size_t numThreads) { + numTaskThreads_ = numThreads; + } + + /** + * Set the ThreadFactory to use for creating the task threads. + */ + void setTaskThreadFactory( + const boost::shared_ptr& threadFactory) { + taskThreadFactory_ = threadFactory; + } + + /** + * Set the number of threads to use for performing I/O. + */ + void setNumIOThreads(size_t numThreads) { + numIOThreads_ = numThreads; + } + + /** + * Set whether or not the I/O threads should be given a higher scheduling + * priority. + */ + void setUseHighPriorityIOThreads(bool useHiPri) { + useHiPriIOThreads_ = useHiPri; + } + + /** + * Set the maximum number of TConnection objects to cache. + * + * By default, when a connection is closed TNonblockingServer caches the + * TConnection object and re-uses it when a new connection comes in, instead + * of deleting it and allocating a new one for the next connection. This + * value caps the number of unused TConnection objects that will be cached. + */ + void setConnectionStackLimit(size_t limit) { + connectionStackLimit_ = limit; + } + + /// Set the maximum number of open connections + void setMaxConnections(size_t maxConnections) { + maxConnections_ = maxConnections; + } + + /// Set the maximum number of active requests + void setMaxActiveProcessors(size_t maxActiveProcessors) { + maxActiveProcessors_ = maxActiveProcessors; + } + + /// Set the maximum allowed request frame size. + void setMaxFrameSize(size_t maxFrameSize) { + maxFrameSize_ = maxFrameSize; + } + + /// Set the overload hysteresis fraction + void setOverloadHysteresis(double hysteresisFraction) { + hysteresisFraction_ = hysteresisFraction; + } + + /// Set the overload action + void setOverloadAction(server::TOverloadAction action) { + overloadAction_ = action; + } + + /** + * Set the task expiration time. + * + * If no task thread is available to process a request within taskExpireTime + * milliseconds, close the connection rather than continuing to wait for a + * thread. If the server is overloaded, this prevents it from processing + * old requests that the client has already given up on. + */ + void setTaskExpireTime(int64_t taskExpireTime) { + taskExpireTime_ = taskExpireTime; + } + + /** + * Set the socket options to use for accepted connections + */ + void setSocketOptions(const transport::TSocket::Options& options) { + socketOptions_ = options; + } + + /// Set the default write buffer size for new connections + void setWriteBufferDefaultSize(size_t size) { + defaultWriteBufferSize_ = size; + } + + /** + * Set the maximum read buffer size for idle connections. + * + * This value is checked when the connection is closed, and also every N + * connections if setResizeBufferEveryN() has been set. If the read buffer + * is larger than the allowed limit, free it. + * + * This prevents a single large request on a connection from continuing to + * taking up a lot of buffer space. + */ + void setIdleReadBufferLimit(size_t limit) { + idleReadBufferLimit_ = limit; + } + + /** + * Set the maximum write buffer size for idle connections. + * + * This value is checked when the connection is closed, and also every N + * connections if setResizeBufferEveryN() has been set. If the write buffer + * is larger than the allowed limit, free it. + * + * This prevents a single large response on a connection from continuing to + * taking up a lot of buffer space. + */ + void setIdleWriteBufferLimit(size_t limit) { + idleWriteBufferLimit_ = limit; + } + + /// Check a connection's buffer size limits every count requests. + void setResizeBufferEveryN(int32_t count) { + resizeBufferEveryN_ = count; + } + + /// listen backlog + void setListenBacklog(int32_t listenBacklog) { + listenBacklog_ = listenBacklog; + } + + virtual boost::shared_ptr createServer(); + + boost::shared_ptr createNonblockingServer(); + + private: + boost::shared_ptr processor_; + uint16_t port_; + size_t numTaskThreads_; + size_t numIOThreads_; + bool useHiPriIOThreads_; + size_t connectionStackLimit_; + size_t maxConnections_; + size_t maxActiveProcessors_; + size_t maxFrameSize_; + double hysteresisFraction_; + server::TOverloadAction overloadAction_; + int64_t taskExpireTime_; + size_t defaultWriteBufferSize_; + size_t idleReadBufferLimit_; + size_t idleWriteBufferLimit_; + int32_t resizeBufferEveryN_; + int32_t listenBacklog_; + transport::TSocket::Options socketOptions_; + boost::shared_ptr taskThreadFactory_; +}; + +}}} // apache::thrift::util + +#endif // THRIFT_UTIL_TNONBLOCKINGSERVERCREATOR_H_ diff --git a/thrift/lib/cpp/util/TSimpleServerCreator.h b/thrift/lib/cpp/util/TSimpleServerCreator.h new file mode 100644 index 0000000000..8204eecf86 --- /dev/null +++ b/thrift/lib/cpp/util/TSimpleServerCreator.h @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_UTIL_TSIMPLESERVERCREATOR_H_ +#define THRIFT_UTIL_TSIMPLESERVERCREATOR_H_ 1 + +#include "thrift/lib/cpp/util/SyncServerCreator.h" + +namespace apache { namespace thrift { + +namespace server { +class TSimpleServer; +} + +namespace util { + +class TSimpleServerCreator : public SyncServerCreator { + public: + typedef server::TSimpleServer ServerType; + + /** + * Create a new TSimpleServerCreator. + */ + TSimpleServerCreator(const boost::shared_ptr& processor, + uint16_t port, + bool framed = true) + : SyncServerCreator(processor, port, framed) {} + + /** + * Create a new TSimpleServerCreator. + */ + TSimpleServerCreator(const boost::shared_ptr& processor, + uint16_t port, + boost::shared_ptr& tf, + boost::shared_ptr& pf) + : SyncServerCreator(processor, port, tf, pf) {} + + virtual boost::shared_ptr createServer(); + + boost::shared_ptr createSimpleServer(); +}; + +}}} // apache::thrift::util + +#endif // THRIFT_UTIL_TSIMPLESERVERCREATOR_H_ diff --git a/thrift/lib/cpp/util/TThreadPoolServerCreator.h b/thrift/lib/cpp/util/TThreadPoolServerCreator.h new file mode 100644 index 0000000000..9732e3b672 --- /dev/null +++ b/thrift/lib/cpp/util/TThreadPoolServerCreator.h @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_UTIL_TTHREADPOOLSERVERCREATOR_H_ +#define THRIFT_UTIL_TTHREADPOOLSERVERCREATOR_H_ 1 + +#include "thrift/lib/cpp/util/SyncServerCreator.h" + +namespace apache { namespace thrift { + +namespace concurrency { +class ThreadFactory; +} +namespace server { +class TThreadPoolServer; +} + +namespace util { + +class TThreadPoolServerCreator : public SyncServerCreator { + public: + typedef server::TThreadPoolServer ServerType; + + enum OverloadAction { + OVERLOAD_STOP_ACCEPTING, + OVERLOAD_ACCEPT_AND_CLOSE, + }; + + /** + * By default, stop calling accept() once we reach the maximum number pending + * tasks. + */ + static const OverloadAction DEFAULT_OVERLOAD_ACTION = + OVERLOAD_STOP_ACCEPTING; + + /** + * Use 30 threads by default. + * + * Make sure to set this number large enough for your service. If there are + * more active connections than number of threads, new connections will + * block until one of the existing connections is closed! Typically you + * should also use a small receive timeout with TThreadPoolServer, to + * quickly expire idle connections. + */ + static const size_t DEFAULT_NUM_THREADS = 30; + + /** + * By default, only allow up to 30 pending connections waiting for a thread + * to run. + * + * Once all threads are in use, at most this many additional tasks may be + * queued before the server starts taking the OverloadAction. + */ + static const size_t DEFAULT_MAX_PENDING_TASKS = 30; + + /** + * Create a new TThreadPoolServerCreator. + */ + TThreadPoolServerCreator(const boost::shared_ptr& processor, + uint16_t port, + bool framed = true); + + /** + * Create a new TThreadPoolServerCreator. + */ + TThreadPoolServerCreator(const boost::shared_ptr& processor, + uint16_t port, + int numThreads, + bool framed = true); + + /** + * Create a new TThreadPoolServerCreator. + */ + TThreadPoolServerCreator(const boost::shared_ptr& processor, + uint16_t port, + boost::shared_ptr& tf, + boost::shared_ptr& pf); + + /** + * Set the action to take once all threads are in use and the pending task + * queue is full. + */ + void setOverloadAction(OverloadAction action) { + overloadAction_ = action; + } + + /** + * Set the number of threads to use for the thread pool + */ + void setNumThreads(size_t numThreads) { + numThreads_ = numThreads; + } + + /** + * Set the maximum number of pending tasks. + */ + void setMaxPendingTasks(size_t numTasks) { + maxPendingTasks_ = numTasks; + } + + /** + * Set the thread factory + */ + void setThreadFactory(const boost::shared_ptr&); + + virtual boost::shared_ptr createServer(); + + boost::shared_ptr createThreadPoolServer(); + + private: + OverloadAction overloadAction_; + size_t numThreads_; + size_t maxPendingTasks_; + boost::shared_ptr threadFactory_; +}; + +}}} // apache::thrift::util + +#endif // THRIFT_UTIL_TTHREADPOOLSERVERCREATOR_H_ diff --git a/thrift/lib/cpp/util/TThreadedServerCreator.h b/thrift/lib/cpp/util/TThreadedServerCreator.h new file mode 100644 index 0000000000..6155a23ba4 --- /dev/null +++ b/thrift/lib/cpp/util/TThreadedServerCreator.h @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_UTIL_TTHREADEDSERVERCREATOR_H_ +#define THRIFT_UTIL_TTHREADEDSERVERCREATOR_H_ 1 + +#include "thrift/lib/cpp/util/SyncServerCreator.h" + +namespace apache { namespace thrift { + +namespace concurrency { +class ThreadFactory; +} +namespace server { +class TThreadedServer; +} + +namespace util { + +class TThreadedServerCreator : public SyncServerCreator { + public: + typedef server::TThreadedServer ServerType; + + /** + * Create a new TThreadedServerCreator. + */ + TThreadedServerCreator(const boost::shared_ptr& processor, + uint16_t port, + bool framed = true) + : SyncServerCreator(processor, port, framed) {} + + /** + * Create a new TThreadedServerCreator. + */ + TThreadedServerCreator(const boost::shared_ptr& processor, + uint16_t port, + boost::shared_ptr& tf, + boost::shared_ptr& pf) + : SyncServerCreator(processor, port, tf, pf) {} + + /** + * Set the thread factory + */ + void setThreadFactory(const boost::shared_ptr&); + + virtual boost::shared_ptr createServer(); + + boost::shared_ptr createThreadedServer(); + + private: + boost::shared_ptr threadFactory_; +}; + +}}} // apache::thrift::util + +#endif // THRIFT_UTIL_TTHREADEDSERVERCREATOR_H_ diff --git a/thrift/lib/cpp/util/ThriftSerializer-inl.h b/thrift/lib/cpp/util/ThriftSerializer-inl.h new file mode 100644 index 0000000000..fac416d3a0 --- /dev/null +++ b/thrift/lib/cpp/util/ThriftSerializer-inl.h @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// @author Karl Voskuil (karl@facebook.com) +// @author Mark Rabkin (mrabkin@facebook.com) +// + +#ifndef COMMON_STRINGS_THRIFT_SERIALIZER_INL_H +#define COMMON_STRINGS_THRIFT_SERIALIZER_INL_H + +#include +#include +#include "common/logging/logging.h" +#include "thrift/lib/cpp/util/ThriftSerializer.h" + +namespace apache { namespace thrift { namespace util { + +template +template +void +ThriftSerializer::serialize(const T& fields, String* serialized) +{ + // prepare or reset buffer + // + // note: Three cases: + // + // 1) The buffer has never been prepared. Call prepare(). + // + // 2) The buffer was last used for a deserialize() call. In this + // case, the buffer is pointing to a now-invalid constant + // string, and the buffer should be reallocated. + // + // 3) The buffer was last used for a serialize() call. In this + // case, the buffer is still pointing to a valid buffer, but it + // contains the serialization from the last serialize(), and + // the buffer just needs to be reset. + // + // This would be a little simpler if we allocated separate buffers + // and protocols for serialization and deserialization, but it + // seemed to me that this fit the common use cases better. The case + // that performs poorly (20% slower) is doing many alternating + // serializations and deserializations. + if (!prepared_ || lastDeserialized_) { + prepare(); + } else { + buffer_->resetBuffer(); + } + lastDeserialized_ = false; + + // serialize fields into buffer + fields.write(protocol_.get()); + + // assign buffer to string + uint8_t *byteBuffer; + uint32_t byteBufferSize; + buffer_->getBuffer(&byteBuffer, &byteBufferSize); + serialized->assign((const char*)byteBuffer, byteBufferSize); +} + +template +void +ThriftSerializer::serialize(const T& fields, + const uint8_t** serializedBuffer, + size_t* serializedLen) { + CHECK(serializedBuffer); + CHECK(serializedLen); + + // prepare or reset buffer + if (!prepared_ || lastDeserialized_) { + prepare(); + } else { + buffer_->resetBuffer(); + } + lastDeserialized_ = false; + + // serialize fields into buffer + fields.write(protocol_.get()); + + // assign buffer to string + uint8_t *byteBuffer; + uint32_t byteBufferSize; + buffer_->getBuffer(&byteBuffer, &byteBufferSize); + *serializedBuffer = byteBuffer; + *serializedLen = byteBufferSize; +} + + +// Deserializes a thrift object, but assumes that the object passed in the +// fields argument is "clean". If your thrift class contains optional fields, +// you should use deserialize() instead (see below), otherwise optional fields +// that were previously set by other calls to this function won't get reset, +// and will appear to have been desearialized along with the other fields. +template +template +uint32_t +ThriftSerializer::deserializeClean(const String& serialized, T* fields) +{ + // prepare buffer if necessary + if (!prepared_) { + prepare(); + } + lastDeserialized_ = true; + + // reset buffer transport to passed string + buffer_->resetBuffer((uint8_t*)serialized.data(), serialized.size()); + + // deserialize buffer into fields + return fields->read(protocol_.get()); +} + +template +uint32_t +ThriftSerializer::deserialize(const uint8_t* serializedBuffer, + size_t length, + T* fields) +{ + // prepare buffer if necessary + if (!prepared_) { + prepare(); + } + lastDeserialized_ = true; + + // reset buffer transport to passed string + buffer_->resetBuffer((uint8_t*)serializedBuffer, length); + + // need to clean the existing structure, as fields->read() will not overwrite + // the optional fields that were not set in the serialized object. + T emptyFields; + swap(emptyFields, *fields); + + // deserialize buffer into fields + return fields->read(protocol_.get()); +} + +template +void +ThriftSerializer::setVersion(int8_t version) +{ + version_ = version; + setVersion_ = true; +} + +template +void +ThriftSerializer::prepare() +{ + // create memory buffer to use as transport + // + // note: TMemoryBuffer won't write to the buffer unless it is the + // owner of it, which means that we can't pass in our own local + // buffer. Either must malloc a buffer which TMemoryBuffer will + // free, or else just let it malloc its own. This has the added + // benefit that TMemoryBuffer will grow the buffer size as + // necessary. + // + // The initial allocation of the memory buffer might seem + // unnecessary in case of parsing (since the memory buffer will be + // reset to point to the string passed to parseSerializer()), but + // it's apparently necessary for some protocols. + buffer_.reset(new TMemoryBuffer()); + + // create a protocol for the memory buffer transport + protocol_.reset(new Protocol(buffer_)); + if (setVersion_) { + protocol_->setVersion(version_); + } + + prepared_ = true; +} + + +}}} // namespace apache::thrift::util + + +#endif diff --git a/thrift/lib/cpp/util/ThriftSerializer.h b/thrift/lib/cpp/util/ThriftSerializer.h new file mode 100644 index 0000000000..80099b9a16 --- /dev/null +++ b/thrift/lib/cpp/util/ThriftSerializer.h @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// @author Karl Voskuil (karl@facebook.com) +// @author Mark Rabkin (mrabkin@facebook.com) +// + +#ifndef COMMON_STRINGS_THRIFT_SERIALIZER_H +#define COMMON_STRINGS_THRIFT_SERIALIZER_H + +#include +#include + +#include "thrift/lib/cpp/protocol/TBinaryProtocol.h" // for serialization +#include "thrift/lib/cpp/protocol/TCompactProtocol.h" // for serialization +#include "thrift/lib/cpp/protocol/TJSONProtocol.h" // for serialization +#include "thrift/lib/cpp/protocol/TSimpleJSONProtocol.h" // for serialization +#include "thrift/lib/cpp/transport/TTransportUtils.h" // for serialization + +namespace apache { namespace thrift { namespace util { + +using apache::thrift::protocol::TBinaryProtocolT; +using apache::thrift::protocol::TCompactProtocolT; +using apache::thrift::protocol::TCompactProtocol; +using apache::thrift::protocol::TSimpleJSONProtocol; +using apache::thrift::protocol::TJSONProtocol; +using apache::thrift::transport::TBufferBase; +using apache::thrift::transport::TTransport; +using apache::thrift::transport::TMemoryBuffer; +using boost::shared_ptr; + +/** + * Encapsulates methods to serialize or deserialize a code-gen-ed + * Thrift class to and from a string. For example: + * + * extern StatsMcValue data; + * + * ThriftSerializerBinary serializer; + * string serialized; + * serializer.serialize(data, &serizalied); + * + * StatsMcValue result; + * try { + * if (serializer.deserialize(serialized, &result) != + * serialized.length()) { + * // Handle deserialization error, not all data consumed. + * } + * } catch (TProtocolException& tpe) { + * // Handle deserialization error, underlying protocol threw. + * } + * // Use deserialized thrift object. + * ... + * + * @author Karl Voskuil + */ + +template +class ThriftSerializer { + public: + ThriftSerializer() + : prepared_(false) + , lastDeserialized_(false) + , setVersion_(false) {} + + /** + * Serializes the passed type into the passed string. + * + * @author Karl Voskuil + */ + template + void serialize(const T& fields, String* serialized); + + /** + * Serializes the passed type into the internal buffer + * and returns a pointer to the internal buffer and its size. + * + * @author Yuri Putivsky + */ + void serialize(const T& fields, const uint8_t** serializedBuffer, + size_t* serializedLen); + + /** + * Deserializes the passed string into the passed type, returns the number of + * bytes that have been consumed from the passed string. + * + * The return value can be used to verify if the deserialization is successful + * or not. When a type is serialized then deserialized back, the number of + * consumed bytes must equal to the size of serialized string. But please + * note, equality of these two values doesn't guarantee the serialized string + * isn't corrupt. It's up to the underlying implementation of the type and + * thrift protocol to detect and handle invalid serialized string, they may + * throw exception or just ignore the unrecognized data. + * @author Karl Voskuil + */ + template + uint32_t deserialize(const String& serialized, T* fields) + { + return deserialize((const uint8_t*)serialized.data(), + serialized.size(), + fields); + } + + /** + * Deserializes the passed char array into the passed type, returns the number + * of bytes that have been consumed from the passed string. + * + * See notes on return value for: + * deserilize(const String* serialized, T* fields) + */ + uint32_t deserialize(const uint8_t* serializedBuffer, + size_t length, + T* fields); + + /** + * Same as deserialize() above, but won't touch/reset any optional fields + * that are not present in 'serialized'. So, if T has any optional fields, + * the caller is responsible for resetting those (or somehow handling the + * potentially dirty data.) + * + * Use this method if: + * 1) your thrift class doesn't contain any optional fields, and + * 2) you are trying to avoid memory allocations/fragmentation during + * deserialization of thift objects + * + * @author Rafael Sagula + */ + template + uint32_t deserializeClean(const String& serialized, T* fields); + + /** + * Set version of protocol data to read/write. This is only necessary + * for data that will be saved to disk between protocol versions! + * You probably don't need to use this unless you know what you are doing + * + * @author davejwatson + */ + void setVersion(int8_t version); + + private: + void prepare(); + + private: + typedef P Protocol; + + bool prepared_; + bool lastDeserialized_; + shared_ptr buffer_; + shared_ptr protocol_; + int8_t version_; + bool setVersion_; +}; + +template +struct ThriftSerializerBinary + : public ThriftSerializer > +{ }; + +template +struct ThriftSerializerCompact + : public ThriftSerializer > +{ }; + +/** + * This version is deprecated. Please do not use it anymore, + * unless you have data already serialized to disk in this format. + * Doubles are not serialized in correct network order, so making RPC + * calls with this data will not work. + */ +template +struct ThriftSerializerCompactDeprecated + : public ThriftSerializer > +{ + public: + ThriftSerializerCompactDeprecated() { + this->setVersion(TCompactProtocol::VERSION_LOW); + } +}; + +template +struct ThriftSerializerJson + : public ThriftSerializer +{ }; + +template +struct ThriftSerializerSimpleJson + : public ThriftSerializer +{ }; + +}}} // namespace apache::thrift:util + +namespace apache { namespace thrift { + +template + std::string ThriftJSONString(const ThriftStruct& ts) { + using namespace apache::thrift::protocol; + using namespace apache::thrift::util; + ThriftSerializer serializer; + std::string serialized; + serializer.serialize(ts, &serialized); + return serialized; +} + +}} // apache::thrift + + +#include "thrift/lib/cpp/util/ThriftSerializer-inl.h" + +#endif diff --git a/thrift/lib/cpp/util/ThriftSerializerTest.thrift b/thrift/lib/cpp/util/ThriftSerializerTest.thrift new file mode 100644 index 0000000000..0f7d11d3ec --- /dev/null +++ b/thrift/lib/cpp/util/ThriftSerializerTest.thrift @@ -0,0 +1,29 @@ +struct TestValue { + 1: i64 legalClicks, + 2: i64 rawImpressions, + 3: i64 lastLegalClickTime, + 4: i64 lastLegalImpressionTime, + 5: i64 socialClicks, + 6: i64 socialImpressions, + 7: double throttle, + 8: i64 pageOtherImpressions, + 9: i64 pageCanvasImpressions, + 10: i64 pageProfileImpressions, + 11: i64 pageSearchImpressions, + 12: i64 pageEventImpressions, + 13: i64 pageGroupImpressions, + 14: i64 pagePhotoImpressions, + 15: i64 pageHomeImpressions, + 16: i64 position1Impressions, + 17: i64 position2Impressions, + 18: i64 position3Impressions, + 19: i64 feedbackLegalLikes, + 20: i64 feedbackLegalDislikes, + 21: list rollingWindowLegalClicks, + 22: list rollingWindowRawImpressions, + 23: list rollingWindowFeedbackLegalLikes, + 24: list rollingWindowFeedbackLegalDislikes, + 25: i64 pageAlbumImpressions, + 26: i64 pageAdBoardImpressions, + 27: string text, +} diff --git a/thrift/lib/cpp/util/VarintUtils.h b/thrift/lib/cpp/util/VarintUtils.h new file mode 100644 index 0000000000..a12b8fc0fa --- /dev/null +++ b/thrift/lib/cpp/util/VarintUtils.h @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_UTIL_VARINTUTILS_H_ +#define THRIFT_UTIL_VARINTUTILS_H_ 1 + +#include + +namespace apache { namespace thrift { + +namespace util { + +/** + * Read an i16 from the wire as a varint. The MSB of each byte is set + * if there is another byte to follow. This can read up to 3 bytes. + */ +uint32_t readVarint16(uint8_t const* ptr, int16_t* i16, + uint8_t const* boundary); + +/** + * Read an i32 from the wire as a varint. The MSB of each byte is set + * if there is another byte to follow. This can read up to 5 bytes. + */ +uint32_t readVarint32(uint8_t const* ptr, int32_t* i32, + uint8_t const* boundary); + +/** + * Read an i64 from the wire as a proper varint. The MSB of each byte is set + * if there is another byte to follow. This can read up to 10 bytes. + * Caller is responsible for advancing ptr after call. + */ +uint32_t readVarint64(uint8_t const* ptr, int64_t* i64, + uint8_t const* boundary); + +/** + * Write an i32 as a varint. Results in 1-5 bytes on the wire. + */ +uint32_t writeVarint32(uint32_t n, uint8_t* pkt); + +/** + * Write an i16 as a varint. Results in 1-3 bytes on the wire. + */ +uint32_t writeVarint16(uint16_t n, uint8_t* pkt); + +}}} // apache::thrift::util + +#include "VarintUtils.tcc" + +#endif // THRIFT_UTIL_VARINTUTILS_H_ diff --git a/thrift/lib/cpp/util/VarintUtils.tcc b/thrift/lib/cpp/util/VarintUtils.tcc new file mode 100644 index 0000000000..c4d9217b2a --- /dev/null +++ b/thrift/lib/cpp/util/VarintUtils.tcc @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "folly/experimental/io/Cursor.h" + +namespace apache { namespace thrift { + +namespace util { + +template ::value, + bool>::type = false> +T readVarint(CursorT& c) { + T retVal = 0; + uint8_t shift = 0; + uint8_t rsize = 0; + while (true) { + uint8_t byte; + c.pull(&byte, sizeof(byte)); + rsize++; + retVal |= (uint64_t)(byte & 0x7f) << shift; + shift += 7; + if (!(byte & 0x80)) { + return retVal; + } + if (rsize > sizeof(T)) { + // Too big for return type + throw std::out_of_range("invalid varint read"); + } + } +} + +template +void writeVarint(folly::io::RWPrivateCursor& c, T value) { + while (true) { + if ((value & ~0x7F) == 0) { + c.write((int8_t)value); + break; + } else { + c.write((int8_t)((value & 0x7F) | 0x80)); + value = (unsigned)value >> 7; + } + } +} + +}}} // apache::thrift::util diff --git a/thrift/lib/cpp/util/shared_ptr_util.h b/thrift/lib/cpp/util/shared_ptr_util.h new file mode 100644 index 0000000000..282deb0a66 --- /dev/null +++ b/thrift/lib/cpp/util/shared_ptr_util.h @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef THRIFT_UTIL_SHARED_PTR_UTIL_H_ +#define THRIFT_UTIL_SHARED_PTR_UTIL_H_ 1 + +#include +#include +#include + +/** + * Helper macros to allow function overloading even when using + * boost::shared_ptr. + * + * shared_ptr makes overloading really annoying, since shared_ptr defines + * constructor methods to allow one shared_ptr type to be constructed from any + * other shared_ptr type. (Even if it would be a compile error to actually try + * to instantiate the constructor.) These macros add an extra argument to the + * function to cause it to only be instantiated if a pointer of type T is + * convertible to a pointer of type U. + * + * THRIFT_OVERLOAD_IF should be used in function declarations. + * THRIFT_OVERLOAD_IF_DEFN should be used in the function definition, if it is + * defined separately from where it is declared. + */ +#define THRIFT_OVERLOAD_IF_DEFN(T, Y) \ + typename ::boost::enable_if::type, \ + void*>::type + +#define THRIFT_OVERLOAD_IF(T, Y) \ + THRIFT_OVERLOAD_IF_DEFN(T, Y) = NULL + +namespace apache { namespace thrift { + +/** + * A helper functor that can be used as a shared_ptr destructor, but that does + * nothing at all. + * + * This can be used to create a shared_ptr to an object that shouldn't really + * be destroyed when the last shared_ptr to it goes away. This is useful in + * some situations to call thrift APIs that require a shared_ptr argument when + * you have an object on the stack or otherwise owned by some other part of the + * code. + * + * WARNING: If at all possible, please avoid using this class to create + * shared_ptr. Attempting to use shared_ptrs with manually managed objects is + * generally a bad idea. This class should only be used in a few rare cases + * where a temporary shared_ptr is needed, and you can guarantee that all + * shared_ptrs to the object will be destroyed before the object itself is + * destroyed. + */ +template +class NoopPtrDestructor { + public: + void operator()(T* obj) {}; +}; + +}} // apache::thrift + +#endif // THRIFT_UTIL_SHARED_PTR_UTIL_H_ diff --git a/thrift/lib/cpp/util/test/TARGETS b/thrift/lib/cpp/util/test/TARGETS new file mode 100644 index 0000000000..5b56e3846c --- /dev/null +++ b/thrift/lib/cpp/util/test/TARGETS @@ -0,0 +1,17 @@ +cpp_unittest( + name = 'ServerCreatorTest', + srcs = [ + 'ServerCreatorTest.cpp', + ], + deps = [ + '@/thrift/lib/cpp/util:async', + '@/thrift/lib/cpp/util:nonblocking', + '@/thrift/perf/cpp:load-handler', + '@/thrift/perf/cpp:async-load-handler', + ], + external_deps = [ + ('boost', '>= 1.37', 'boost_unit_test_framework'), + ], + type = 'boost', + emails = ['thrift-team@lists.facebook.com'], +) diff --git a/thrift/openhandles.h b/thrift/openhandles.h new file mode 100644 index 0000000000..4b2f40832f --- /dev/null +++ b/thrift/openhandles.h @@ -0,0 +1,227 @@ +/** + * Thrift server for leveldb + * @author Dhruba Borthakur (dhruba@gmail.com) + * Copyright 2012 Facebook + */ + +#include +#include +#include "DB.h" +#include +#include +#include +#include +#include + +#include "leveldb/db.h" +#include "util/random.h" + +using boost::shared_ptr; +using std::unordered_map; + +using namespace ::Tleveldb; + +// List of snapshots. Each entry has a unique snapshot id. +struct snapshotEntry { + int64_t snapshotid; + const leveldb::Snapshot* lsnap; + + snapshotEntry() : snapshotid(-1), lsnap(NULL) { + } + + private: + snapshotEntry(const snapshotEntry&); + snapshotEntry& operator= (const snapshotEntry&); +}; + +// List of iterators. Each entry has a unique iterator id. +struct iteratorEntry { + int64_t iteratorid; + leveldb::Iterator* liter; + + iteratorEntry() : iteratorid(-1), liter(NULL) { + } + + private: + iteratorEntry(const iteratorEntry&); + iteratorEntry& operator= (const iteratorEntry&); +}; + + +// +// This is the information stored for each open database. Each open instance +// of the database has a list of snapshots and a list of iterators that are +// currenty open +// +struct onehandle { + Text name; + leveldb::DB* onedb; // locate the localleveldb instance + int refcount; // currently not used + int64_t uniqueid; // unique identifier + std::atomic currentSnapshotId; // valid snapshotids > 0 + std::atomic currentIteratorId; // valid iterators > 0 + unordered_map snaplist; + // list of snapshots for this database + unordered_map iterlist; + // list of iterators for this database + + onehandle() : currentSnapshotId(1), currentIteratorId(1) { + } + + // stores a new leveldb snapshot and returns an unique id + int64_t addSnapshot(const leveldb::Snapshot* l) { + struct snapshotEntry* news = new snapshotEntry; + news->snapshotid = currentSnapshotId++; + news->lsnap = l; + snaplist[news->snapshotid] = news; + return news->snapshotid; + } + + // lookup a snapshot from its ids + const leveldb::Snapshot* lookupSnapshot(int64_t id) { + auto p = snaplist.find(id); + if (p == snaplist.end()) { + fprintf(stderr, "get:No snaphot with id %d\n", id); + return NULL; + } + return p->second->lsnap; + } + + // remove a snapshot from this database + const leveldb::Snapshot* removeSnapshot(int64_t id) { + const leveldb::Snapshot* l = lookupSnapshot(id); + if (l != NULL) { + int numRemoved = snaplist.erase(id); + assert(numRemoved == 1); + return l; + } + return NULL; // not found + } + + // stores a new leveldb iterator and returns an unique id + int64_t addIterator(leveldb::Iterator* l) { + struct iteratorEntry* news = new iteratorEntry; + news->iteratorid = currentIteratorId++; + news->liter = l; + iterlist[news->iteratorid] = news; + return news->iteratorid; + } + + // lookup a iterator from its ids + leveldb::Iterator* lookupIterator(int64_t id) { + auto p = iterlist.find(id); + if (p == iterlist.end()) { + fprintf(stderr, "lookupIterator:No iterator with id %d\n", id); + return NULL; + } + return p->second->liter; + } + + // remove a iterator from this database + leveldb::Iterator* removeIterator(int64_t id) { + leveldb::Iterator* i = lookupIterator(id); + if (i != NULL) { + int numRemoved = iterlist.erase(id); + assert(numRemoved == 1); + return i; + } + return NULL; // not found + } + + private: + onehandle(const onehandle&); + onehandle& operator= (const onehandle&); +}; + +class OpenHandles { + public: + + OpenHandles() : dbnum_(1) { + } + + // Inserts a new database into the list. + // If the database is already open, increase refcount. + // If the database is not already open, open and insert into list. + int64_t add(leveldb::Options& options, Text dbname) { + struct onehandle* found = head_[dbname]; + if (found == NULL) { + found = new onehandle; + found->name = dbname; + found->uniqueid = dbnum_++; + fprintf(stderr, "openhandle.add: Opening leveldb directory %s\n", + dbname.c_str()); + leveldb::Status status = leveldb::DB::Open(options, dbname, &found->onedb); + if (!status.ok()) { + LeveldbException e; + e.errorCode = Code::kIOError; + e.message = "Unable to open database"; + fprintf(stderr, "openhandle.add: Unable to open database %s\n", + dbname.c_str()); + throw e; + } + assert(found->onedb != NULL); + head_[dbname] = found; + } + found->refcount++; + return found->uniqueid; + } + + leveldb::DB* get(Text dbname, int64_t uniqueid, struct onehandle** f) { + auto p = head_.find(dbname); + if (p == head_.end()) { + fprintf(stderr, "get:No db with name\n"); + return NULL; + } + struct onehandle* found = p->second; + if (found->uniqueid != uniqueid) { + fprintf(stderr, "get:Uniqueid does not match\n."); + return NULL; + } + if (found->refcount <= 0) { + fprintf(stderr, "get:bad refcount\n."); + return NULL; + } + // returns the onehandle if asked to do so + if (f != NULL) { + *f = found; + } + return found->onedb; + } + + bool remove(Text dbname, int64_t uniqueid) { + auto p = head_.find(dbname); + if (p == head_.end()) { + fprintf(stderr, "get:No db with name\n"); + return false; + } + struct onehandle* found = p->second; + if (found->uniqueid != uniqueid) { + fprintf(stderr, "remove:Uniqueid does not match\n."); + return false; + } + if (found->refcount == 1) { + delete found->onedb; // close database + int numRemoved = head_.erase(dbname); + assert (numRemoved == 1); + } else { + found->refcount--; // decrement refcount + } + return true; + } + + private: + unordered_map head_; // all open databases + std::atomic dbnum_; + + struct onehandle* lookup(Text dbname) { + auto p = head_.find(dbname); + if (p == head_.end()) { + fprintf(stderr, "get:No db with name\n"); + return NULL; + } + return p->second; + } +}; + + + diff --git a/thrift/server.cpp b/thrift/server.cpp new file mode 100644 index 0000000000..5ffca33ac3 --- /dev/null +++ b/thrift/server.cpp @@ -0,0 +1,46 @@ +/** + * Thrift server for leveldb + * @author Dhruba Borthakur (dhruba@gmail.com) + * Copyright 2012 Facebook + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "openhandles.h" + +#include "leveldb/db.h" +#include "leveldb/write_batch.h" + +using namespace apache::thrift; +using namespace apache::thrift::protocol; +using namespace apache::thrift::transport; +using namespace apache::thrift::server; +using namespace Tleveldb; +using boost::shared_ptr; + +extern "C" void startServer(int port); +extern "C" void stopServer(int port); + +static int port = 9090; + +void signal_handler(int sig) { + switch (sig) { + case SIGINT: + fprintf(stderr, "Received SIGINT, stopping leveldb server"); + stopServer(port); + break; + } +} + +int main(int argc, char **argv) { + signal(SIGINT, signal_handler); + startServer(port); + return 0; +} + diff --git a/thrift/server_utils.cpp b/thrift/server_utils.cpp new file mode 100644 index 0000000000..d2f325e5cf --- /dev/null +++ b/thrift/server_utils.cpp @@ -0,0 +1,400 @@ +/** + * Thrift server for leveldb + * @author Dhruba Borthakur (dhruba@gmail.com) + * Copyright 2012 Facebook + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "openhandles.h" + +#include "leveldb/db.h" +#include "leveldb/write_batch.h" + +using namespace apache::thrift; +using namespace apache::thrift::util; +using namespace apache::thrift::protocol; +using namespace apache::thrift::transport; +using namespace apache::thrift::server; +using namespace apache::thrift::async; +using namespace Tleveldb; +using boost::shared_ptr; + +extern "C" void startServer(int port); +extern "C" void stopServer(int port); + +static boost::shared_ptr tServer; + +class DBHandler : virtual public DBIf { + public: + DBHandler() { + openHandles = new OpenHandles(); + } + + void Open(DBHandle& _return, const Text& dbname, + const DBOptions& dboptions) { + printf("Open\n"); + leveldb::Options options; + leveldb::DB* db; + options.create_if_missing = dboptions.create_if_missing; + options.error_if_exists = dboptions.error_if_exists; + options.write_buffer_size = dboptions.write_buffer_size; + options.max_open_files = dboptions.max_open_files; + options.block_size = dboptions.block_size; + options.block_restart_interval = dboptions.block_restart_interval; + if (dboptions.compression == kNoCompression) { + options.compression = leveldb::kNoCompression; + } else if (dboptions.compression == kSnappyCompression) { + options.compression = leveldb::kSnappyCompression; + } + int64_t session = openHandles->add(options, dbname); + _return.dbname = dbname; + _return.handleid = session; + } + + Code Close(const DBHandle& dbhandle, const Text& dbname) { + /** + * We do not close any handles for now, otherwise we have to do + * some locking that will degrade performance in the normal case. + if (openHandles->remove(dbname, dbhandle.handleid) == false) { + return Code::kIOError; + } + */ + return Code::kNotSupported; + } + + Code Put(const DBHandle& dbhandle, const kv& kv, + const WriteOptions& options) { + leveldb::WriteOptions woptions; + woptions.sync = options.sync; + leveldb::Slice key, value; + key.data_ = kv.key.data.data(); + key.size_ = kv.key.size; + value.data_ = kv.value.data.data(); + value.size_ = kv.value.size; + leveldb::DB* db = openHandles->get(dbhandle.dbname, dbhandle.handleid, NULL); + if (db == NULL) { + return Code::kNotFound; + } + leveldb::Status status = db->Put(woptions, key, value); + if (status.ok()) { + return Code::kOk; + } + return Code::kIOError; + } + + Code Delete(const DBHandle& dbhandle, const Slice& kv, + const WriteOptions& options) { + leveldb::WriteOptions woptions; + woptions.sync = options.sync; + leveldb::Slice key; + key.data_ = kv.data.data(); + key.size_ = kv.size; + leveldb::DB* db = openHandles->get(dbhandle.dbname, dbhandle.handleid, NULL); + if (db == NULL) { + return Code::kNotFound; + } + leveldb::Status status = db->Delete(woptions, key); + if (status.ok()) { + return Code::kOk; + } + return Code::kIOError; + } + + Code Write(const DBHandle& dbhandle, const std::vector & batch, + const WriteOptions& options) { + leveldb::WriteOptions woptions; + leveldb::WriteBatch lbatch; + woptions.sync = options.sync; + leveldb::Slice key, value; + for (int i = 0; i < batch.size(); i++) { + kv one = batch[i]; + key.data_ = one.key.data.data(); + key.size_ = one.key.size; + value.data_ = one.value.data.data(); + value.size_ = one.value.size; + lbatch.Put(key, value); + } + leveldb::DB* db = openHandles->get(dbhandle.dbname, dbhandle.handleid, NULL); + if (db == NULL) { + return Code::kNotFound; + } + leveldb::Status status = db->Write(woptions, &lbatch); + if (status.ok()) { + return Code::kOk; + } + return Code::kIOError; + } + + void Get(ResultItem& _return, const DBHandle& dbhandle, const Slice& inputkey, + const ReadOptions& options) { + struct onehandle* thishandle; + _return.status = Code::kNotFound; + std::string ret; + leveldb::Slice ikey; + ikey.data_ = inputkey.data.data(); + ikey.size_ = inputkey.size; + leveldb::DB* db = openHandles->get(dbhandle.dbname, dbhandle.handleid, + &thishandle); + if (db == NULL) { + return; + } + assert(thishandle != NULL); + const leveldb::Snapshot* s = NULL; + if (options.snapshot.snapshotid > 0) { + s = thishandle->lookupSnapshot(options.snapshot.snapshotid); + assert(s != NULL); + if (s == NULL) { + return; + } + } + leveldb::ReadOptions roptions; + roptions.verify_checksums = options.verify_checksums; + roptions.fill_cache = options.fill_cache; + roptions.snapshot = s; + + leveldb::Status status = db->Get(roptions, ikey, &ret); + if (status.ok()) { + _return.value.data = ret.data(); + _return.value.size = ret.size(); + _return.status = Code::kOk; + } + } + + void NewIterator(ResultIterator& _return, const DBHandle& dbhandle, + const ReadOptions& options, IteratorType iteratorType, + const Slice& target) { + struct onehandle* thishandle; + _return.status = Code::kNotFound; + leveldb::DB* db = openHandles->get(dbhandle.dbname, dbhandle.handleid, + &thishandle); + if (db == NULL) { + return; + } + assert(thishandle != NULL); + + // check to see if snapshot is specified + const leveldb::Snapshot* s = NULL; + if (options.snapshot.snapshotid > 0) { + s = thishandle->lookupSnapshot(options.snapshot.snapshotid); + assert(s != NULL); + if (s == NULL) { + return; + } + } + + // create leveldb iterator + leveldb::ReadOptions roptions; + roptions.verify_checksums = options.verify_checksums; + roptions.fill_cache = options.fill_cache; + roptions.snapshot = s; + leveldb::Iterator* iter = db->NewIterator(roptions); + if (iter == NULL) { + return; + } + + // position iterator at right place + if (iteratorType == IteratorType::seekToFirst) { + iter->SeekToFirst(); + } else if (iteratorType == IteratorType::seekToLast) { + iter->SeekToLast(); + } else if (iteratorType == IteratorType::seekToKey) { + leveldb::Slice key; + key.data_ = target.data.data(); + key.size_ = target.size; + iter->Seek(key); + } else { + delete iter; + _return.status = Code::kInvalidArgument; + return; + } + + // insert iterator into openhandle list, get unique id + int64_t id = thishandle->addIterator(iter); + _return.iterator.iteratorid = id; + _return.status = kOk; + } + + // Delete existing iterator + Code DeleteIterator(const DBHandle& dbhandle, const Iterator& iterator) { + // find the db + struct onehandle* thishandle; + leveldb::DB* db = openHandles->get(dbhandle.dbname, dbhandle.handleid, + &thishandle); + if (db == NULL) { + return kNotFound; + } + assert(thishandle != NULL); + + // find the leveldb iterator for this db + leveldb::Iterator* it = + thishandle->lookupIterator(iterator.iteratorid); + if (it == NULL) { + // this must have been cleaned up by the last call to GetNext + return kOk; + } + thishandle->removeIterator(iterator.iteratorid); + delete it; // cleanup + return kOk; + } + + // read the next value from the iterator + void GetAnother(ResultPair& _return, const DBHandle& dbhandle, + const Iterator& iterator, const bool doNext) { + + // find the db + struct onehandle* thishandle; + _return.status = Code::kNotFound; + leveldb::DB* db = openHandles->get(dbhandle.dbname, dbhandle.handleid, + &thishandle); + if (db == NULL) { + return; + } + assert(thishandle != NULL); + + // find the leveldb iterator for this db + leveldb::Iterator* it = + thishandle->lookupIterator(iterator.iteratorid); + assert(it != NULL); + if (it == NULL) { + return; + } + + // If the iterator has reached the endm close it rightaway. + // There is no need for the application to make another thrift + // call to cleanup the iterator. + if (!it->Valid()) { + thishandle->removeIterator(iterator.iteratorid); + delete it; // cleanup + _return.status = Code::kEnd; // no more elements + return; + } + + // find current key-value + leveldb::Slice key = it->key(); + leveldb::Slice value = it->value(); + + // move to next or previous value + if (doNext) { + it->Next(); + } else { + it->Prev(); + } + + // pack results back to client + _return.keyvalue.key.data = key.data_; + _return.keyvalue.key.size = key.size_; + _return.keyvalue.value.data = value.data_; + _return.keyvalue.value.size = value.size_; + _return.status = Code::kOk; // success + } + + // read the next value from the iterator + void GetNext(ResultPair& _return, const DBHandle& dbhandle, + const Iterator& iterator) { + GetAnother(_return, dbhandle, iterator, 1); + } + + // read the prev value from the iterator + void GetPrev(ResultPair& _return, const DBHandle& dbhandle, + const Iterator& iterator) { + GetAnother(_return, dbhandle, iterator, 0); + } + + void GetSnapshot(ResultSnapshot& _return, const DBHandle& dbhandle) { + _return.status = kIOError; + struct onehandle* thishandle; + leveldb::DB* db = openHandles->get(dbhandle.dbname, dbhandle.handleid, + &thishandle); + if (db == NULL) { + return; + } + // create leveldb snapshot + const leveldb::Snapshot* s = db->GetSnapshot(); + + // store snapshot in dbhandle, get unique id. + int64_t id = thishandle->addSnapshot(s); + _return.snapshot.snapshotid = id; + _return.status = kOk; + } + + Code ReleaseSnapshot(const DBHandle& dbhandle, const Snapshot& snapshot) { + struct onehandle* thishandle; + leveldb::DB* db = openHandles->get(dbhandle.dbname, dbhandle.handleid, + &thishandle); + if (db == NULL) { + return kNotFound; + } + const leveldb::Snapshot* s = thishandle->removeSnapshot(snapshot.snapshotid); + if (s == NULL) { + return Code::kNotFound; + } + db->ReleaseSnapshot(s); // release leveldb snapshot + return Code::kOk; + } + + Code CompactRange(const DBHandle& dbhandle, const Slice& begin, + const Slice& end) { + leveldb::DB* db = openHandles->get(dbhandle.dbname, dbhandle.handleid, NULL); + if (db == NULL) { + return Code::kNotFound; + } + leveldb::Slice k1, *start = &k1; + k1.data_ = begin.data.data(); + k1.size_ = begin.size; + leveldb::Slice k2, *stop = &k2; + k2.data_ = begin.data.data(); + k2.size_ = begin.size; + + // check special ranges. + if (start->size_ == 0) { + start == NULL; + } + if (stop->size_ == 0) { + stop == NULL; + } + db->CompactRange(start, stop); + return Code::kOk; + } + + private: + OpenHandles* openHandles; +}; + +// Starts a very simple thrift server +void startServer(int port) { + shared_ptr handler(new DBHandler()); + shared_ptr processor(new DBProcessor(handler)); + shared_ptr serverTransport(new TServerSocket(port)); + shared_ptr transportFactory(new TBufferedTransportFactory()); + shared_ptr protocolFactory(new TBinaryProtocolFactory()); + + TSimpleServer tServer(processor, serverTransport, transportFactory, protocolFactory); + tServer.serve(); +} + +/** +void startEventServer(int port) { + shared_ptr handler(new DBHandler()); + shared_ptr processor(new DBProcessor(handler)); + shared_ptr asyncProcessor(new TSyncToAsyncProcessor(processor)); + TEventServerCreator creator(asyncProcessor, (uint16_t)port, 2); + tServer = creator.createServer(); + tServer.serve(); +} +**/ + +// Stops the thrift server +void stopServer(int port) { + tServer->stop(); +} diff --git a/thrift/test/simpletest.cpp b/thrift/test/simpletest.cpp new file mode 100644 index 0000000000..43d805ed29 --- /dev/null +++ b/thrift/test/simpletest.cpp @@ -0,0 +1,212 @@ +/** + * Tests for Thrift server for leveldb + * @author Dhruba Borthakur (dhruba@gmail.com) + * Copyright 2012 Facebook + */ +#include +#include +#include +#include +#include + +using namespace apache::thrift; +using namespace apache::thrift::protocol; +using namespace apache::thrift::transport; +using boost::shared_ptr; +using namespace Tleveldb; + +extern "C" void startServer(int port); +extern "C" void stopServer(int port); + +static DBHandle dbhandle; +static DBClient* dbclient; +static const Text dbname = "/tmp/leveldb/test"; +static const int myport = 9091; +static pthread_t serverThread; + +static void createDatabase() { + DBOptions options; + options.create_if_missing = true; // create + options.error_if_exists = false; // overwrite + options.write_buffer_size = (4<<20); // 4 MB + options.max_open_files = 1000; + options.block_size = 4096; + options.block_restart_interval = 16; + options.compression = kSnappyCompression; + + // create the database + dbclient->Open(dbhandle, dbname, options); +} + +static void testClient(int port) { + boost::shared_ptr socket(new TSocket("localhost", port)); + boost::shared_ptr transport(new TBufferedTransport(socket)); + boost::shared_ptr protocol(new TBinaryProtocol(transport)); + WriteOptions writeOptions; + + // open database + dbclient = new DBClient(protocol); + transport->open(); + createDatabase(); + printf("Database created.\n"); + + // insert record into leveldb + Slice key; + key.data = "Key1"; + key.size = 4; + Slice value; + value.data = "value1"; + value.size = 6; + kv keyvalue; + keyvalue.key = key; + keyvalue.value = value; + int ret = dbclient->Put(dbhandle, keyvalue, writeOptions); + assert(ret == Code::kOk); + printf("Put Key1 suceeded.\n"); + + //read it back + ReadOptions readOptions; + ResultItem rValue; + dbclient->Get(rValue, dbhandle, key, readOptions); + assert(rValue.status == Code::kOk); + assert(value.data.compare(rValue.value.data) == 0); + assert(value.size == rValue.value.size); + printf("Get suceeded.\n"); + + // get a snapshot + ResultSnapshot rsnap; + dbclient->GetSnapshot(rsnap, dbhandle); + assert(rsnap.status == Code::kOk); + assert(rsnap.snapshot.snapshotid > 0); + printf("Snapshot suceeded.\n"); + + // insert a new record into leveldb + Slice key2; + key2.data = "Key2"; + key2.size = 4; + Slice value2; + value2.data = "value2"; + value2.size = 6; + keyvalue.key = key2; + keyvalue.value = value2; + ret = dbclient->Put(dbhandle, keyvalue, writeOptions); + assert(ret == Code::kOk); + printf("Put Key2 suceeded.\n"); + + // verify that a get done with a previous snapshot does not find Key2 + readOptions.snapshot = rsnap.snapshot; + dbclient->Get(rValue, dbhandle, key2, readOptions); + assert(rValue.status == Code::kNotFound); + printf("Get with snapshot succeeded.\n"); + + // release snapshot + ret = dbclient->ReleaseSnapshot(dbhandle, rsnap.snapshot); + assert(ret == Code::kOk); + printf("Snapshot released.\n"); + + // if we try to re-release the same snapshot, it should fail + ret = dbclient->ReleaseSnapshot(dbhandle, rsnap.snapshot); + assert(ret == Code::kNotFound); + + // compact whole database + Slice range; + range.size = 0; + ret = dbclient->CompactRange(dbhandle, range, range); + assert(ret == Code::kOk); + printf("Compaction trigger suceeded.\n"); + + // create a new iterator to scan all keys from the start + Slice target; + ResultIterator ri; + readOptions.snapshot.snapshotid = 0; + dbclient->NewIterator(ri, dbhandle, readOptions, + IteratorType::seekToFirst, target); + assert(ri.status == Code::kOk); + int foundPairs = 0; + while (true) { + ResultPair pair; + dbclient->GetNext(pair, dbhandle, ri.iterator); + if (pair.status == Code::kOk) { + foundPairs++; + } else { + break; + } + } + assert(foundPairs == 2); + ret = dbclient->DeleteIterator(dbhandle, ri.iterator); + assert(ret == Code::kOk); + printf("Iterator scan-all forward passes.\n"); + + // create a new iterator, position at end and scan backwards + readOptions.snapshot.snapshotid = 0; + dbclient->NewIterator(ri, dbhandle, readOptions, + IteratorType::seekToLast, target); + assert(ri.status == Code::kOk); + foundPairs = 0; + while (true) { + ResultPair pair; + dbclient->GetPrev(pair, dbhandle, ri.iterator); + if (pair.status == Code::kOk) { + foundPairs++; + } else { + break; + } + } + assert(foundPairs == 2); + ret = dbclient->DeleteIterator(dbhandle, ri.iterator); + assert(ret == Code::kOk); + printf("Iterator scan-all backward passes.\n"); + + // create a new iterator, position at middle + readOptions.snapshot.snapshotid = 0; + target = key; + dbclient->NewIterator(ri, dbhandle, readOptions, + IteratorType::seekToKey, target); + assert(ri.status == Code::kOk); + foundPairs = 0; + while (true) { + ResultPair pair; + dbclient->GetPrev(pair, dbhandle, ri.iterator); + if (pair.status == Code::kOk) { + foundPairs++; + } else { + break; + } + } + assert(foundPairs == 1); + ret = dbclient->DeleteIterator(dbhandle, ri.iterator); + assert(ret == Code::kOk); + printf("Iterator scan-selective backward passes.\n"); + + // close database + dbclient->Close(dbhandle, dbname); + transport->close(); +} + + +static void* startTestServer(void *arg) { + printf("Server starting on port %d...\n", myport); + startServer(myport); +} + +int main(int argc, char **argv) { + + // remove old data, if any + int ret = unlink(dbname.c_str()); + char* cleanup = new char[100]; + snprintf(cleanup, 100, "rm -rf %s", dbname.c_str()); + system(cleanup); + + // create a server + int rc = pthread_create(&serverThread, NULL, startTestServer, NULL); + printf("Server thread created.\n"); + + // give some time to the server to initialize itself + sleep(1); + + // test client + testClient(myport); + + return 0; +} + diff --git a/util/options.cc b/util/options.cc index c222334a10..0ad742ce20 100644 --- a/util/options.cc +++ b/util/options.cc @@ -37,5 +37,4 @@ Options::Options() statistics(NULL) { } - } // namespace leveldb