2016-02-09 23:12:00 +00:00
|
|
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
2017-07-15 23:03:42 +00:00
|
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
|
|
// (found in the LICENSE.Apache file in the root directory).
|
2015-05-29 21:36:35 +00:00
|
|
|
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
|
2017-08-07 23:07:40 +00:00
|
|
|
#include "utilities/transactions/optimistic_transaction.h"
|
2015-05-29 21:36:35 +00:00
|
|
|
|
|
|
|
#include <string>
|
|
|
|
|
|
|
|
#include "db/column_family.h"
|
2019-05-31 18:52:59 +00:00
|
|
|
#include "db/db_impl/db_impl.h"
|
2015-05-29 21:36:35 +00:00
|
|
|
#include "rocksdb/comparator.h"
|
|
|
|
#include "rocksdb/db.h"
|
|
|
|
#include "rocksdb/status.h"
|
|
|
|
#include "rocksdb/utilities/optimistic_transaction_db.h"
|
2017-07-28 23:23:50 +00:00
|
|
|
#include "util/cast_util.h"
|
2015-05-29 21:36:35 +00:00
|
|
|
#include "util/string_util.h"
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 00:37:33 +00:00
|
|
|
#include "utilities/transactions/transaction_util.h"
|
2020-01-07 22:19:06 +00:00
|
|
|
#include "utilities/transactions/optimistic_transaction.h"
|
|
|
|
#include "utilities/transactions/optimistic_transaction_db_impl.h"
|
2015-05-29 21:36:35 +00:00
|
|
|
|
2020-02-20 20:07:53 +00:00
|
|
|
namespace ROCKSDB_NAMESPACE {
|
2015-05-29 21:36:35 +00:00
|
|
|
|
|
|
|
struct WriteOptions;
|
|
|
|
|
2017-08-07 23:07:40 +00:00
|
|
|
OptimisticTransaction::OptimisticTransaction(
|
2015-05-29 21:36:35 +00:00
|
|
|
OptimisticTransactionDB* txn_db, const WriteOptions& write_options,
|
|
|
|
const OptimisticTransactionOptions& txn_options)
|
2015-08-21 22:47:21 +00:00
|
|
|
: TransactionBaseImpl(txn_db->GetBaseDB(), write_options), txn_db_(txn_db) {
|
2016-03-04 00:33:26 +00:00
|
|
|
Initialize(txn_options);
|
|
|
|
}
|
|
|
|
|
2017-08-07 23:07:40 +00:00
|
|
|
void OptimisticTransaction::Initialize(
|
2016-03-04 00:33:26 +00:00
|
|
|
const OptimisticTransactionOptions& txn_options) {
|
2015-05-29 21:36:35 +00:00
|
|
|
if (txn_options.set_snapshot) {
|
|
|
|
SetSnapshot();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-08-07 23:07:40 +00:00
|
|
|
void OptimisticTransaction::Reinitialize(
|
2016-03-04 00:33:26 +00:00
|
|
|
OptimisticTransactionDB* txn_db, const WriteOptions& write_options,
|
|
|
|
const OptimisticTransactionOptions& txn_options) {
|
|
|
|
TransactionBaseImpl::Reinitialize(txn_db->GetBaseDB(), write_options);
|
|
|
|
Initialize(txn_options);
|
|
|
|
}
|
|
|
|
|
2017-08-16 23:49:11 +00:00
|
|
|
OptimisticTransaction::~OptimisticTransaction() {}
|
2015-07-31 22:18:27 +00:00
|
|
|
|
2017-08-16 23:49:11 +00:00
|
|
|
void OptimisticTransaction::Clear() { TransactionBaseImpl::Clear(); }
|
2015-05-29 21:36:35 +00:00
|
|
|
|
2017-08-07 23:07:40 +00:00
|
|
|
Status OptimisticTransaction::Prepare() {
|
2016-04-18 18:15:50 +00:00
|
|
|
return Status::InvalidArgument(
|
|
|
|
"Two phase commit not supported for optimistic transactions.");
|
|
|
|
}
|
|
|
|
|
2017-08-07 23:07:40 +00:00
|
|
|
Status OptimisticTransaction::Commit() {
|
2020-01-07 22:19:06 +00:00
|
|
|
auto txn_db_impl = static_cast_with_check<OptimisticTransactionDBImpl,
|
|
|
|
OptimisticTransactionDB>(txn_db_);
|
|
|
|
assert(txn_db_impl);
|
2020-02-14 02:50:04 +00:00
|
|
|
switch (txn_db_impl->GetValidatePolicy()) {
|
|
|
|
case OccValidationPolicy::kValidateParallel:
|
|
|
|
return CommitWithParallelValidate();
|
|
|
|
case OccValidationPolicy::kValidateSerial:
|
|
|
|
return CommitWithSerialValidate();
|
|
|
|
default:
|
|
|
|
assert(0);
|
2020-01-07 22:19:06 +00:00
|
|
|
}
|
|
|
|
// unreachable, just void compiler complain
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
|
|
|
Status OptimisticTransaction::CommitWithSerialValidate() {
|
2015-05-29 21:36:35 +00:00
|
|
|
// Set up callback which will call CheckTransactionForConflicts() to
|
|
|
|
// check whether this transaction is safe to be committed.
|
|
|
|
OptimisticTransactionCallback callback(this);
|
|
|
|
|
2017-07-28 23:23:50 +00:00
|
|
|
DBImpl* db_impl = static_cast_with_check<DBImpl, DB>(db_->GetRootDB());
|
2015-05-29 21:36:35 +00:00
|
|
|
|
|
|
|
Status s = db_impl->WriteWithCallback(
|
2016-01-28 01:11:44 +00:00
|
|
|
write_options_, GetWriteBatch()->GetWriteBatch(), &callback);
|
2015-05-29 21:36:35 +00:00
|
|
|
|
|
|
|
if (s.ok()) {
|
2015-08-25 02:13:18 +00:00
|
|
|
Clear();
|
2015-05-29 21:36:35 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
2020-01-07 22:19:06 +00:00
|
|
|
Status OptimisticTransaction::CommitWithParallelValidate() {
|
|
|
|
auto txn_db_impl = static_cast_with_check<OptimisticTransactionDBImpl,
|
|
|
|
OptimisticTransactionDB>(txn_db_);
|
|
|
|
assert(txn_db_impl);
|
|
|
|
DBImpl* db_impl = static_cast_with_check<DBImpl, DB>(db_->GetRootDB());
|
|
|
|
assert(db_impl);
|
|
|
|
const size_t space = txn_db_impl->GetLockBucketsSize();
|
|
|
|
std::set<size_t> lk_idxes;
|
|
|
|
std::vector<std::unique_lock<std::mutex>> lks;
|
|
|
|
for (auto& cfit : GetTrackedKeys()) {
|
|
|
|
for (auto& keyit : cfit.second) {
|
|
|
|
lk_idxes.insert(fastrange64(GetSliceNPHash64(keyit.first), space));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// NOTE: in a single txn, all bucket-locks are taken in ascending order.
|
|
|
|
// In this way, txns from different threads all obey this rule so that
|
|
|
|
// deadlock can be avoided.
|
|
|
|
for (auto v : lk_idxes) {
|
|
|
|
lks.emplace_back(txn_db_impl->LockBucket(v));
|
|
|
|
}
|
|
|
|
|
|
|
|
Status s = TransactionUtil::CheckKeysForConflicts(db_impl, GetTrackedKeys(),
|
|
|
|
true /* cache_only */);
|
|
|
|
if (!s.ok()) {
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
|
|
|
s = db_impl->Write(write_options_, GetWriteBatch()->GetWriteBatch());
|
|
|
|
if (s.ok()) {
|
|
|
|
Clear();
|
|
|
|
}
|
|
|
|
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
2017-08-07 23:07:40 +00:00
|
|
|
Status OptimisticTransaction::Rollback() {
|
2016-04-18 18:15:50 +00:00
|
|
|
Clear();
|
|
|
|
return Status::OK();
|
|
|
|
}
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 00:37:33 +00:00
|
|
|
|
2015-05-29 21:36:35 +00:00
|
|
|
// Record this key so that we can check it for conflicts at commit time.
|
2016-12-06 01:18:14 +00:00
|
|
|
//
|
|
|
|
// 'exclusive' is unused for OptimisticTransaction.
|
2017-08-07 23:07:40 +00:00
|
|
|
Status OptimisticTransaction::TryLock(ColumnFamilyHandle* column_family,
|
2017-08-16 23:49:11 +00:00
|
|
|
const Slice& key, bool read_only,
|
2018-12-07 01:46:57 +00:00
|
|
|
bool exclusive, const bool do_validate,
|
|
|
|
const bool assume_tracked) {
|
|
|
|
assert(!assume_tracked); // not supported
|
|
|
|
(void)assume_tracked;
|
|
|
|
if (!do_validate) {
|
2015-08-21 22:47:21 +00:00
|
|
|
return Status::OK();
|
|
|
|
}
|
2015-05-29 21:36:35 +00:00
|
|
|
uint32_t cfh_id = GetColumnFamilyID(column_family);
|
|
|
|
|
2015-09-28 19:12:17 +00:00
|
|
|
SetSnapshotIfNeeded();
|
|
|
|
|
2015-05-29 21:36:35 +00:00
|
|
|
SequenceNumber seq;
|
|
|
|
if (snapshot_) {
|
2016-01-28 01:11:44 +00:00
|
|
|
seq = snapshot_->GetSequenceNumber();
|
2015-05-29 21:36:35 +00:00
|
|
|
} else {
|
|
|
|
seq = db_->GetLatestSequenceNumber();
|
|
|
|
}
|
|
|
|
|
|
|
|
std::string key_str = key.ToString();
|
|
|
|
|
2017-04-10 22:47:20 +00:00
|
|
|
TrackKey(cfh_id, key_str, seq, read_only, exclusive);
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 00:37:33 +00:00
|
|
|
|
2015-08-21 22:47:21 +00:00
|
|
|
// Always return OK. Confilct checking will happen at commit time.
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 00:37:33 +00:00
|
|
|
return Status::OK();
|
2015-05-29 21:36:35 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Returns OK if it is safe to commit this transaction. Returns Status::Busy
|
|
|
|
// if there are read or write conflicts that would prevent us from committing OR
|
|
|
|
// if we can not determine whether there would be any such conflicts.
|
|
|
|
//
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 00:37:33 +00:00
|
|
|
// Should only be called on writer thread in order to avoid any race conditions
|
Use SST files for Transaction conflict detection
Summary:
Currently, transactions can fail even if there is no actual write conflict. This is due to relying on only the memtables to check for write-conflicts. Users have to tune memtable settings to try to avoid this, but it's hard to figure out exactly how to tune these settings.
With this diff, TransactionDB will use both memtables and SST files to determine if there are any write conflicts. This relies on the fact that BlockBasedTable stores sequence numbers for all writes that happen after any open snapshot. Also, D50295 is needed to prevent SingleDelete from disappearing writes (the TODOs in this test code will be fixed once the other diff is approved and merged).
Note that Optimistic transactions will still rely on tuning memtable settings as we do not want to read from SST while on the write thread. Also, memtable settings can still be used to reduce how often TransactionDB needs to read SST files.
Test Plan: unit tests, db bench
Reviewers: rven, yhchiang, kradhakrishnan, IslamAbdelRahman, sdong
Reviewed By: sdong
Subscribers: dhruba, leveldb, yoshinorim
Differential Revision: https://reviews.facebook.net/D50475
2015-10-15 23:37:15 +00:00
|
|
|
// in detecting write conflicts.
|
2017-08-07 23:07:40 +00:00
|
|
|
Status OptimisticTransaction::CheckTransactionForConflicts(DB* db) {
|
2015-05-29 21:36:35 +00:00
|
|
|
Status result;
|
|
|
|
|
2017-07-28 23:23:50 +00:00
|
|
|
auto db_impl = static_cast_with_check<DBImpl, DB>(db);
|
2015-05-29 21:36:35 +00:00
|
|
|
|
Use SST files for Transaction conflict detection
Summary:
Currently, transactions can fail even if there is no actual write conflict. This is due to relying on only the memtables to check for write-conflicts. Users have to tune memtable settings to try to avoid this, but it's hard to figure out exactly how to tune these settings.
With this diff, TransactionDB will use both memtables and SST files to determine if there are any write conflicts. This relies on the fact that BlockBasedTable stores sequence numbers for all writes that happen after any open snapshot. Also, D50295 is needed to prevent SingleDelete from disappearing writes (the TODOs in this test code will be fixed once the other diff is approved and merged).
Note that Optimistic transactions will still rely on tuning memtable settings as we do not want to read from SST while on the write thread. Also, memtable settings can still be used to reduce how often TransactionDB needs to read SST files.
Test Plan: unit tests, db bench
Reviewers: rven, yhchiang, kradhakrishnan, IslamAbdelRahman, sdong
Reviewed By: sdong
Subscribers: dhruba, leveldb, yoshinorim
Differential Revision: https://reviews.facebook.net/D50475
2015-10-15 23:37:15 +00:00
|
|
|
// Since we are on the write thread and do not want to block other writers,
|
|
|
|
// we will do a cache-only conflict check. This can result in TryAgain
|
|
|
|
// getting returned if there is not sufficient memtable history to check
|
|
|
|
// for conflicts.
|
|
|
|
return TransactionUtil::CheckKeysForConflicts(db_impl, GetTrackedKeys(),
|
|
|
|
true /* cache_only */);
|
2015-08-25 02:13:18 +00:00
|
|
|
}
|
|
|
|
|
2017-08-07 23:07:40 +00:00
|
|
|
Status OptimisticTransaction::SetName(const TransactionName& /* unused */) {
|
2016-04-18 18:15:50 +00:00
|
|
|
return Status::InvalidArgument("Optimistic transactions cannot be named.");
|
|
|
|
}
|
|
|
|
|
2020-02-20 20:07:53 +00:00
|
|
|
} // namespace ROCKSDB_NAMESPACE
|
2015-05-29 21:36:35 +00:00
|
|
|
|
|
|
|
#endif // ROCKSDB_LITE
|