mirror of https://github.com/facebook/rocksdb.git
Sync with upstream @23860137.
Fix GCC -Wshadow warnings in LevelDB's public header files, reported by Dustin. Add in-memory Env implementation (helpers/memenv/*). This enables users to create LevelDB databases in-memory. Initialize ShardedLRUCache::last_id_ to zero. This fixes a Valgrind warning. (Also delete port/sha1_* which were removed upstream some time ago.)
This commit is contained in:
Hans Wennborg
parent
7263023651
commit
213a68eb68
12
Makefile
12
Makefile
|
|
@ -92,6 +92,7 @@ TESTS = \
|
|||
env_test \
|
||||
filename_test \
|
||||
log_test \
|
||||
memenv_test \
|
||||
skiplist_test \
|
||||
table_test \
|
||||
version_edit_test \
|
||||
|
|
@ -102,6 +103,7 @@ PROGRAMS = db_bench $(TESTS)
|
|||
BENCHMARKS = db_bench_sqlite3 db_bench_tree_db
|
||||
|
||||
LIBRARY = libleveldb.a
|
||||
MEMENVLIBRARY = libmemenv.a
|
||||
|
||||
all: $(LIBRARY)
|
||||
|
||||
|
|
@ -109,7 +111,7 @@ check: $(PROGRAMS) $(TESTS)
|
|||
for t in $(TESTS); do echo "***** Running $$t"; ./$$t || exit 1; done
|
||||
|
||||
clean:
|
||||
-rm -f $(PROGRAMS) $(BENCHMARKS) $(LIBRARY) */*.o */*/*.o ios-x86/*/*.o ios-arm/*/*.o
|
||||
-rm -f $(PROGRAMS) $(BENCHMARKS) $(LIBRARY) $(MEMENVLIBRARY) */*.o */*/*.o ios-x86/*/*.o ios-arm/*/*.o
|
||||
-rm -rf ios-x86/* ios-arm/*
|
||||
-rm build_config.mk
|
||||
|
||||
|
|
@ -174,6 +176,13 @@ version_set_test: db/version_set_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
|||
write_batch_test: db/write_batch_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CC) $(LDFLAGS) db/write_batch_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@
|
||||
|
||||
$(MEMENVLIBRARY) : helpers/memenv/memenv.o
|
||||
rm -f $@
|
||||
$(AR) -rs $@ helpers/memenv/memenv.o
|
||||
|
||||
memenv_test : helpers/memenv/memenv_test.o $(MEMENVLIBRARY) $(LIBRARY) $(TESTHARNESS)
|
||||
$(CC) $(LDFLAGS) helpers/memenv/memenv_test.o $(MEMENVLIBRARY) $(LIBRARY) $(TESTHARNESS) -o $@
|
||||
|
||||
ifeq ($(PLATFORM), IOS)
|
||||
# For iOS, create universal object files to be used on both the simulator and
|
||||
# a device.
|
||||
|
|
@ -202,4 +211,3 @@ else
|
|||
.c.o:
|
||||
$(CC) $(CFLAGS) $< -o $@
|
||||
endif
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,374 @@
|
|||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "helpers/memenv/memenv.h"
|
||||
|
||||
#include "leveldb/env.h"
|
||||
#include "leveldb/status.h"
|
||||
#include "port/port.h"
|
||||
#include "util/mutexlock.h"
|
||||
#include <map>
|
||||
#include <string.h>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
namespace {
|
||||
|
||||
class FileState {
|
||||
public:
|
||||
// FileStates are reference counted. The initial reference count is zero
|
||||
// and the caller must call Ref() at least once.
|
||||
FileState() : refs_(0), size_(0) {}
|
||||
|
||||
// Increase the reference count.
|
||||
void Ref() {
|
||||
MutexLock lock(&refs_mutex_);
|
||||
++refs_;
|
||||
}
|
||||
|
||||
// Decrease the reference count. Delete if this is the last reference.
|
||||
void Unref() {
|
||||
bool do_delete = false;
|
||||
|
||||
{
|
||||
MutexLock lock(&refs_mutex_);
|
||||
--refs_;
|
||||
assert(refs_ >= 0);
|
||||
if (refs_ <= 0) {
|
||||
do_delete = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (do_delete) {
|
||||
delete this;
|
||||
}
|
||||
}
|
||||
|
||||
uint64_t Size() const { return size_; }
|
||||
|
||||
Status Read(uint64_t offset, size_t n, Slice* result, char* scratch) const {
|
||||
if (offset > size_) {
|
||||
return Status::IOError("Offset greater than file size.");
|
||||
}
|
||||
const uint64_t available = size_ - offset;
|
||||
if (n > available) {
|
||||
n = available;
|
||||
}
|
||||
if (n == 0) {
|
||||
*result = Slice();
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
size_t block = offset / kBlockSize;
|
||||
size_t block_offset = offset % kBlockSize;
|
||||
|
||||
if (n <= kBlockSize - block_offset) {
|
||||
// The requested bytes are all in the first block.
|
||||
*result = Slice(blocks_[block] + block_offset, n);
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
size_t bytes_to_copy = n;
|
||||
char* dst = scratch;
|
||||
|
||||
while (bytes_to_copy > 0) {
|
||||
size_t avail = kBlockSize - block_offset;
|
||||
if (avail > bytes_to_copy) {
|
||||
avail = bytes_to_copy;
|
||||
}
|
||||
memcpy(dst, blocks_[block] + block_offset, avail);
|
||||
|
||||
bytes_to_copy -= avail;
|
||||
dst += avail;
|
||||
block++;
|
||||
block_offset = 0;
|
||||
}
|
||||
|
||||
*result = Slice(scratch, n);
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status Append(const Slice& data) {
|
||||
const char* src = data.data();
|
||||
size_t src_len = data.size();
|
||||
|
||||
while (src_len > 0) {
|
||||
size_t avail;
|
||||
size_t offset = size_ % kBlockSize;
|
||||
|
||||
if (offset != 0) {
|
||||
// There is some room in the last block.
|
||||
avail = kBlockSize - offset;
|
||||
} else {
|
||||
// No room in the last block; push new one.
|
||||
blocks_.push_back(new char[kBlockSize]);
|
||||
avail = kBlockSize;
|
||||
}
|
||||
|
||||
if (avail > src_len) {
|
||||
avail = src_len;
|
||||
}
|
||||
memcpy(blocks_.back() + offset, src, avail);
|
||||
src_len -= avail;
|
||||
src += avail;
|
||||
size_ += avail;
|
||||
}
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
private:
|
||||
// Private since only Unref() should be used to delete it.
|
||||
~FileState() {
|
||||
for (std::vector<char*>::iterator i = blocks_.begin(); i != blocks_.end();
|
||||
++i) {
|
||||
delete [] *i;
|
||||
}
|
||||
}
|
||||
|
||||
// No copying allowed.
|
||||
FileState(const FileState&);
|
||||
void operator=(const FileState&);
|
||||
|
||||
port::Mutex refs_mutex_;
|
||||
int refs_; // Protected by refs_mutex_;
|
||||
|
||||
// The following fields are not protected by any mutex. They are only mutable
|
||||
// while the file is being written, and concurrent access is not allowed
|
||||
// to writable files.
|
||||
std::vector<char*> blocks_;
|
||||
uint64_t size_;
|
||||
|
||||
enum { kBlockSize = 8 * 1024 };
|
||||
};
|
||||
|
||||
class SequentialFileImpl : public SequentialFile {
|
||||
public:
|
||||
explicit SequentialFileImpl(FileState* file) : file_(file), pos_(0) {
|
||||
file_->Ref();
|
||||
}
|
||||
|
||||
~SequentialFileImpl() {
|
||||
file_->Unref();
|
||||
}
|
||||
|
||||
virtual Status Read(size_t n, Slice* result, char* scratch) {
|
||||
Status s = file_->Read(pos_, n, result, scratch);
|
||||
if (s.ok()) {
|
||||
pos_ += result->size();
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
virtual Status Skip(uint64_t n) {
|
||||
if (pos_ > file_->Size()) {
|
||||
return Status::IOError("pos_ > file_->Size()");
|
||||
}
|
||||
const size_t available = file_->Size() - pos_;
|
||||
if (n > available) {
|
||||
n = available;
|
||||
}
|
||||
pos_ += n;
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
private:
|
||||
FileState* file_;
|
||||
size_t pos_;
|
||||
};
|
||||
|
||||
class RandomAccessFileImpl : public RandomAccessFile {
|
||||
public:
|
||||
explicit RandomAccessFileImpl(FileState* file) : file_(file) {
|
||||
file_->Ref();
|
||||
}
|
||||
|
||||
~RandomAccessFileImpl() {
|
||||
file_->Unref();
|
||||
}
|
||||
|
||||
virtual Status Read(uint64_t offset, size_t n, Slice* result,
|
||||
char* scratch) const {
|
||||
return file_->Read(offset, n, result, scratch);
|
||||
}
|
||||
|
||||
private:
|
||||
FileState* file_;
|
||||
};
|
||||
|
||||
class WritableFileImpl : public WritableFile {
|
||||
public:
|
||||
WritableFileImpl(FileState* file) : file_(file) {
|
||||
file_->Ref();
|
||||
}
|
||||
|
||||
~WritableFileImpl() {
|
||||
file_->Unref();
|
||||
}
|
||||
|
||||
virtual Status Append(const Slice& data) {
|
||||
return file_->Append(data);
|
||||
}
|
||||
|
||||
virtual Status Close() { return Status::OK(); }
|
||||
virtual Status Flush() { return Status::OK(); }
|
||||
virtual Status Sync() { return Status::OK(); }
|
||||
|
||||
private:
|
||||
FileState* file_;
|
||||
};
|
||||
|
||||
class InMemoryEnv : public EnvWrapper {
|
||||
public:
|
||||
explicit InMemoryEnv(Env* base_env) : EnvWrapper(base_env) { }
|
||||
|
||||
virtual ~InMemoryEnv() {
|
||||
for (FileSystem::iterator i = file_map_.begin(); i != file_map_.end(); ++i){
|
||||
i->second->Unref();
|
||||
}
|
||||
}
|
||||
|
||||
// Partial implementation of the Env interface.
|
||||
virtual Status NewSequentialFile(const std::string& fname,
|
||||
SequentialFile** result) {
|
||||
MutexLock lock(&mutex_);
|
||||
if (file_map_.find(fname) == file_map_.end()) {
|
||||
*result = NULL;
|
||||
return Status::IOError(fname, "File not found");
|
||||
}
|
||||
|
||||
*result = new SequentialFileImpl(file_map_[fname]);
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
virtual Status NewRandomAccessFile(const std::string& fname,
|
||||
RandomAccessFile** result) {
|
||||
MutexLock lock(&mutex_);
|
||||
if (file_map_.find(fname) == file_map_.end()) {
|
||||
*result = NULL;
|
||||
return Status::IOError(fname, "File not found");
|
||||
}
|
||||
|
||||
*result = new RandomAccessFileImpl(file_map_[fname]);
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
virtual Status NewWritableFile(const std::string& fname,
|
||||
WritableFile** result) {
|
||||
MutexLock lock(&mutex_);
|
||||
if (file_map_.find(fname) != file_map_.end()) {
|
||||
DeleteFileInternal(fname);
|
||||
}
|
||||
|
||||
FileState* file = new FileState();
|
||||
file->Ref();
|
||||
file_map_[fname] = file;
|
||||
|
||||
*result = new WritableFileImpl(file);
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
virtual bool FileExists(const std::string& fname) {
|
||||
MutexLock lock(&mutex_);
|
||||
return file_map_.find(fname) != file_map_.end();
|
||||
}
|
||||
|
||||
virtual Status GetChildren(const std::string& dir,
|
||||
std::vector<std::string>* result) {
|
||||
MutexLock lock(&mutex_);
|
||||
result->clear();
|
||||
|
||||
for (FileSystem::iterator i = file_map_.begin(); i != file_map_.end(); ++i){
|
||||
const std::string& filename = i->first;
|
||||
|
||||
if (filename.size() >= dir.size() + 1 && filename[dir.size()] == '/' &&
|
||||
Slice(filename).starts_with(Slice(dir))) {
|
||||
result->push_back(filename.substr(dir.size() + 1));
|
||||
}
|
||||
}
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
void DeleteFileInternal(const std::string& fname) {
|
||||
if (file_map_.find(fname) == file_map_.end()) {
|
||||
return;
|
||||
}
|
||||
|
||||
file_map_[fname]->Unref();
|
||||
file_map_.erase(fname);
|
||||
}
|
||||
|
||||
virtual Status DeleteFile(const std::string& fname) {
|
||||
MutexLock lock(&mutex_);
|
||||
if (file_map_.find(fname) == file_map_.end()) {
|
||||
return Status::IOError(fname, "File not found");
|
||||
}
|
||||
|
||||
DeleteFileInternal(fname);
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
virtual Status CreateDir(const std::string& dirname) {
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
virtual Status DeleteDir(const std::string& dirname) {
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
virtual Status GetFileSize(const std::string& fname, uint64_t* file_size) {
|
||||
MutexLock lock(&mutex_);
|
||||
if (file_map_.find(fname) == file_map_.end()) {
|
||||
return Status::IOError(fname, "File not found");
|
||||
}
|
||||
|
||||
*file_size = file_map_[fname]->Size();
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
virtual Status RenameFile(const std::string& src,
|
||||
const std::string& target) {
|
||||
MutexLock lock(&mutex_);
|
||||
if (file_map_.find(src) == file_map_.end()) {
|
||||
return Status::IOError(src, "File not found");
|
||||
}
|
||||
|
||||
DeleteFileInternal(target);
|
||||
file_map_[target] = file_map_[src];
|
||||
file_map_.erase(src);
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
virtual Status LockFile(const std::string& fname, FileLock** lock) {
|
||||
*lock = new FileLock;
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
virtual Status UnlockFile(FileLock* lock) {
|
||||
delete lock;
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
virtual Status GetTestDirectory(std::string* path) {
|
||||
*path = "/test";
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
private:
|
||||
// Map from filenames to FileState objects, representing a simple file system.
|
||||
typedef std::map<std::string, FileState*> FileSystem;
|
||||
port::Mutex mutex_;
|
||||
FileSystem file_map_; // Protected by mutex_.
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
Env* NewMemEnv(Env* base_env) {
|
||||
return new InMemoryEnv(base_env);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#ifndef STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_
|
||||
#define STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
class Env;
|
||||
|
||||
// Returns a new environment that stores its data in memory and delegates
|
||||
// all non-file-storage tasks to base_env. The caller must delete the result
|
||||
// when it is no longer needed.
|
||||
// *base_env must remain live while the result is in use.
|
||||
Env* NewMemEnv(Env* base_env);
|
||||
|
||||
}
|
||||
|
||||
#endif // STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_
|
||||
|
|
@ -0,0 +1,232 @@
|
|||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "helpers/memenv/memenv.h"
|
||||
|
||||
#include "db/db_impl.h"
|
||||
#include "leveldb/db.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "util/testharness.h"
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
class MemEnvTest {
|
||||
public:
|
||||
Env* env_;
|
||||
|
||||
MemEnvTest()
|
||||
: env_(NewMemEnv(Env::Default())) {
|
||||
}
|
||||
~MemEnvTest() {
|
||||
delete env_;
|
||||
}
|
||||
};
|
||||
|
||||
TEST(MemEnvTest, Basics) {
|
||||
size_t file_size;
|
||||
WritableFile* writable_file;
|
||||
std::vector<std::string> children;
|
||||
|
||||
ASSERT_OK(env_->CreateDir("/dir"));
|
||||
|
||||
// Check that the directory is empty.
|
||||
ASSERT_TRUE(!env_->FileExists("/dir/non_existent"));
|
||||
ASSERT_TRUE(!env_->GetFileSize("/dir/non_existent", &file_size).ok());
|
||||
ASSERT_OK(env_->GetChildren("/dir", &children));
|
||||
ASSERT_EQ(0, children.size());
|
||||
|
||||
// Create a file.
|
||||
ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file));
|
||||
delete writable_file;
|
||||
|
||||
// Check that the file exists.
|
||||
ASSERT_TRUE(env_->FileExists("/dir/f"));
|
||||
ASSERT_OK(env_->GetFileSize("/dir/f", &file_size));
|
||||
ASSERT_EQ(0, file_size);
|
||||
ASSERT_OK(env_->GetChildren("/dir", &children));
|
||||
ASSERT_EQ(1, children.size());
|
||||
ASSERT_EQ("f", children[0]);
|
||||
|
||||
// Write to the file.
|
||||
ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file));
|
||||
ASSERT_OK(writable_file->Append("abc"));
|
||||
delete writable_file;
|
||||
|
||||
// Check for expected size.
|
||||
ASSERT_OK(env_->GetFileSize("/dir/f", &file_size));
|
||||
ASSERT_EQ(3, file_size);
|
||||
|
||||
// Check that renaming works.
|
||||
ASSERT_TRUE(!env_->RenameFile("/dir/non_existent", "/dir/g").ok());
|
||||
ASSERT_OK(env_->RenameFile("/dir/f", "/dir/g"));
|
||||
ASSERT_TRUE(!env_->FileExists("/dir/f"));
|
||||
ASSERT_TRUE(env_->FileExists("/dir/g"));
|
||||
ASSERT_OK(env_->GetFileSize("/dir/g", &file_size));
|
||||
ASSERT_EQ(3, file_size);
|
||||
|
||||
// Check that opening non-existent file fails.
|
||||
SequentialFile* seq_file;
|
||||
RandomAccessFile* rand_file;
|
||||
ASSERT_TRUE(!env_->NewSequentialFile("/dir/non_existent", &seq_file).ok());
|
||||
ASSERT_TRUE(!seq_file);
|
||||
ASSERT_TRUE(!env_->NewRandomAccessFile("/dir/non_existent", &rand_file).ok());
|
||||
ASSERT_TRUE(!rand_file);
|
||||
|
||||
// Check that deleting works.
|
||||
ASSERT_TRUE(!env_->DeleteFile("/dir/non_existent").ok());
|
||||
ASSERT_OK(env_->DeleteFile("/dir/g"));
|
||||
ASSERT_TRUE(!env_->FileExists("/dir/g"));
|
||||
ASSERT_OK(env_->GetChildren("/dir", &children));
|
||||
ASSERT_EQ(0, children.size());
|
||||
ASSERT_OK(env_->DeleteDir("/dir"));
|
||||
}
|
||||
|
||||
TEST(MemEnvTest, ReadWrite) {
|
||||
WritableFile* writable_file;
|
||||
SequentialFile* seq_file;
|
||||
RandomAccessFile* rand_file;
|
||||
Slice result;
|
||||
char scratch[100];
|
||||
|
||||
ASSERT_OK(env_->CreateDir("/dir"));
|
||||
|
||||
ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file));
|
||||
ASSERT_OK(writable_file->Append("hello "));
|
||||
ASSERT_OK(writable_file->Append("world"));
|
||||
delete writable_file;
|
||||
|
||||
// Read sequentially.
|
||||
ASSERT_OK(env_->NewSequentialFile("/dir/f", &seq_file));
|
||||
ASSERT_OK(seq_file->Read(5, &result, scratch)); // Read "hello".
|
||||
ASSERT_EQ(0, result.compare("hello"));
|
||||
ASSERT_OK(seq_file->Skip(1));
|
||||
ASSERT_OK(seq_file->Read(1000, &result, scratch)); // Read "world".
|
||||
ASSERT_EQ(0, result.compare("world"));
|
||||
ASSERT_OK(seq_file->Read(1000, &result, scratch)); // Try reading past EOF.
|
||||
ASSERT_EQ(0, result.size());
|
||||
ASSERT_OK(seq_file->Skip(100)); // Try to skip past end of file.
|
||||
ASSERT_OK(seq_file->Read(1000, &result, scratch));
|
||||
ASSERT_EQ(0, result.size());
|
||||
delete seq_file;
|
||||
|
||||
// Random reads.
|
||||
ASSERT_OK(env_->NewRandomAccessFile("/dir/f", &rand_file));
|
||||
ASSERT_OK(rand_file->Read(6, 5, &result, scratch)); // Read "world".
|
||||
ASSERT_EQ(0, result.compare("world"));
|
||||
ASSERT_OK(rand_file->Read(0, 5, &result, scratch)); // Read "hello".
|
||||
ASSERT_EQ(0, result.compare("hello"));
|
||||
ASSERT_OK(rand_file->Read(10, 100, &result, scratch)); // Read "d".
|
||||
ASSERT_EQ(0, result.compare("d"));
|
||||
|
||||
// Too high offset.
|
||||
ASSERT_TRUE(!rand_file->Read(1000, 5, &result, scratch).ok());
|
||||
delete rand_file;
|
||||
}
|
||||
|
||||
TEST(MemEnvTest, Locks) {
|
||||
FileLock* lock;
|
||||
|
||||
// These are no-ops, but we test they return success.
|
||||
ASSERT_OK(env_->LockFile("some file", &lock));
|
||||
ASSERT_OK(env_->UnlockFile(lock));
|
||||
}
|
||||
|
||||
TEST(MemEnvTest, Misc) {
|
||||
std::string test_dir;
|
||||
ASSERT_OK(env_->GetTestDirectory(&test_dir));
|
||||
ASSERT_TRUE(!test_dir.empty());
|
||||
|
||||
WritableFile* writable_file;
|
||||
ASSERT_OK(env_->NewWritableFile("/a/b", &writable_file));
|
||||
|
||||
// These are no-ops, but we test they return success.
|
||||
ASSERT_OK(writable_file->Sync());
|
||||
ASSERT_OK(writable_file->Flush());
|
||||
ASSERT_OK(writable_file->Close());
|
||||
delete writable_file;
|
||||
}
|
||||
|
||||
TEST(MemEnvTest, LargeWrite) {
|
||||
const size_t kWriteSize = 300 * 1024;
|
||||
char* scratch = new char[kWriteSize * 2];
|
||||
|
||||
std::string write_data;
|
||||
for (size_t i = 0; i < kWriteSize; ++i) {
|
||||
write_data.append(1, static_cast<char>(i));
|
||||
}
|
||||
|
||||
WritableFile* writable_file;
|
||||
ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file));
|
||||
ASSERT_OK(writable_file->Append("foo"));
|
||||
ASSERT_OK(writable_file->Append(write_data));
|
||||
delete writable_file;
|
||||
|
||||
SequentialFile* seq_file;
|
||||
Slice result;
|
||||
ASSERT_OK(env_->NewSequentialFile("/dir/f", &seq_file));
|
||||
ASSERT_OK(seq_file->Read(3, &result, scratch)); // Read "foo".
|
||||
ASSERT_EQ(0, result.compare("foo"));
|
||||
|
||||
size_t read = 0;
|
||||
std::string read_data;
|
||||
while (read < kWriteSize) {
|
||||
ASSERT_OK(seq_file->Read(kWriteSize - read, &result, scratch));
|
||||
read_data.append(result.data(), result.size());
|
||||
read += result.size();
|
||||
}
|
||||
ASSERT_TRUE(write_data == read_data);
|
||||
delete seq_file;
|
||||
delete [] scratch;
|
||||
}
|
||||
|
||||
TEST(MemEnvTest, DBTest) {
|
||||
Options options;
|
||||
options.create_if_missing = true;
|
||||
options.env = env_;
|
||||
DB* db;
|
||||
|
||||
const Slice keys[] = {Slice("aaa"), Slice("bbb"), Slice("ccc")};
|
||||
const Slice vals[] = {Slice("foo"), Slice("bar"), Slice("baz")};
|
||||
|
||||
ASSERT_OK(DB::Open(options, "/dir/db", &db));
|
||||
for (size_t i = 0; i < 3; ++i) {
|
||||
ASSERT_OK(db->Put(WriteOptions(), keys[i], vals[i]));
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < 3; ++i) {
|
||||
std::string res;
|
||||
ASSERT_OK(db->Get(ReadOptions(), keys[i], &res));
|
||||
ASSERT_TRUE(res == vals[i]);
|
||||
}
|
||||
|
||||
Iterator* iterator = db->NewIterator(ReadOptions());
|
||||
iterator->SeekToFirst();
|
||||
for (size_t i = 0; i < 3; ++i) {
|
||||
ASSERT_TRUE(iterator->Valid());
|
||||
ASSERT_TRUE(keys[i] == iterator->key());
|
||||
ASSERT_TRUE(vals[i] == iterator->value());
|
||||
iterator->Next();
|
||||
}
|
||||
ASSERT_TRUE(!iterator->Valid());
|
||||
delete iterator;
|
||||
|
||||
DBImpl* dbi = reinterpret_cast<DBImpl*>(db);
|
||||
ASSERT_OK(dbi->TEST_CompactMemTable());
|
||||
|
||||
for (size_t i = 0; i < 3; ++i) {
|
||||
std::string res;
|
||||
ASSERT_OK(db->Get(ReadOptions(), keys[i], &res));
|
||||
ASSERT_TRUE(res == vals[i]);
|
||||
}
|
||||
|
||||
delete db;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
return leveldb::test::RunAllTests();
|
||||
}
|
||||
|
|
@ -258,8 +258,8 @@ extern Status ReadFileToString(Env* env, const std::string& fname,
|
|||
// functionality of another Env.
|
||||
class EnvWrapper : public Env {
|
||||
public:
|
||||
// Initialize an EnvWrapper that delegates all calls to *target
|
||||
explicit EnvWrapper(Env* target) : target_(target) { }
|
||||
// Initialize an EnvWrapper that delegates all calls to *t
|
||||
explicit EnvWrapper(Env* t) : target_(t) { }
|
||||
virtual ~EnvWrapper();
|
||||
|
||||
// Return the target to which this Env forwards all calls
|
||||
|
|
|
|||
|
|
@ -27,8 +27,8 @@ class Slice {
|
|||
// Create an empty slice.
|
||||
Slice() : data_(""), size_(0) { }
|
||||
|
||||
// Create a slice that refers to data[0,n-1].
|
||||
Slice(const char* data, size_t n) : data_(data), size_(n) { }
|
||||
// Create a slice that refers to d[0,n-1].
|
||||
Slice(const char* d, size_t n) : data_(d), size_(n) { }
|
||||
|
||||
// Create a slice that refers to the contents of "s"
|
||||
Slice(const std::string& s) : data_(s.data()), size_(s.size()) { }
|
||||
|
|
|
|||
|
|
@ -1,298 +0,0 @@
|
|||
// Portions copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
//
|
||||
// This module provides a slow but portable implementation of
|
||||
// the SHA1 hash function.
|
||||
//
|
||||
// It is adapted from free code written by Paul E. Jones
|
||||
// <paulej@packetizer.com>. See http://www.packetizer.com/security/sha1/
|
||||
//
|
||||
// The license for the original code is:
|
||||
/*
|
||||
Copyright (C) 1998, 2009
|
||||
Paul E. Jones <paulej@packetizer.com>
|
||||
|
||||
Freeware Public License (FPL)
|
||||
|
||||
This software is licensed as "freeware." Permission to distribute
|
||||
this software in source and binary forms, including incorporation
|
||||
into other products, is hereby granted without a fee. THIS SOFTWARE
|
||||
IS PROVIDED 'AS IS' AND WITHOUT ANY EXPRESSED OR IMPLIED WARRANTIES,
|
||||
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||
AND FITNESS FOR A PARTICULAR PURPOSE. THE AUTHOR SHALL NOT BE HELD
|
||||
LIABLE FOR ANY DAMAGES RESULTING FROM THE USE OF THIS SOFTWARE, EITHER
|
||||
DIRECTLY OR INDIRECTLY, INCLUDING, BUT NOT LIMITED TO, LOSS OF DATA
|
||||
OR DATA BEING RENDERED INACCURATE.
|
||||
*/
|
||||
|
||||
#include "port/sha1_portable.h"
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <stdint.h>
|
||||
|
||||
namespace leveldb {
|
||||
namespace port {
|
||||
|
||||
/*
|
||||
* Description:
|
||||
* This class implements the Secure Hashing Standard as defined
|
||||
* in FIPS PUB 180-1 published April 17, 1995.
|
||||
*/
|
||||
|
||||
/*
|
||||
* This structure will hold context information for the hashing
|
||||
* operation
|
||||
*/
|
||||
typedef struct SHA1Context {
|
||||
unsigned Message_Digest[5]; /* Message Digest (output) */
|
||||
|
||||
unsigned Length_Low; /* Message length in bits */
|
||||
unsigned Length_High; /* Message length in bits */
|
||||
|
||||
unsigned char Message_Block[64]; /* 512-bit message blocks */
|
||||
int Message_Block_Index; /* Index into message block array */
|
||||
|
||||
bool Computed; /* Is the digest computed? */
|
||||
bool Corrupted; /* Is the message digest corruped? */
|
||||
} SHA1Context;
|
||||
|
||||
/*
|
||||
* Portability Issues:
|
||||
* SHA-1 is defined in terms of 32-bit "words". This code was
|
||||
* written with the expectation that the processor has at least
|
||||
* a 32-bit machine word size. If the machine word size is larger,
|
||||
* the code should still function properly. One caveat to that
|
||||
* is that the input functions taking characters and character
|
||||
* arrays assume that only 8 bits of information are stored in each
|
||||
* character.
|
||||
*/
|
||||
|
||||
/*
|
||||
* Define the circular shift macro
|
||||
*/
|
||||
#define SHA1CircularShift(bits,word) \
|
||||
((((word) << (bits)) & 0xFFFFFFFF) | \
|
||||
((word) >> (32-(bits))))
|
||||
|
||||
/* Function prototypes */
|
||||
static void SHA1ProcessMessageBlock(SHA1Context *);
|
||||
static void SHA1PadMessage(SHA1Context *);
|
||||
|
||||
// Initialize the SHA1Context in preparation for computing a new
|
||||
// message digest.
|
||||
static void SHA1Reset(SHA1Context* context) {
|
||||
context->Length_Low = 0;
|
||||
context->Length_High = 0;
|
||||
context->Message_Block_Index = 0;
|
||||
|
||||
context->Message_Digest[0] = 0x67452301;
|
||||
context->Message_Digest[1] = 0xEFCDAB89;
|
||||
context->Message_Digest[2] = 0x98BADCFE;
|
||||
context->Message_Digest[3] = 0x10325476;
|
||||
context->Message_Digest[4] = 0xC3D2E1F0;
|
||||
|
||||
context->Computed = false;
|
||||
context->Corrupted = false;
|
||||
}
|
||||
|
||||
// This function will return the 160-bit message digest into the
|
||||
// Message_Digest array within the SHA1Context provided
|
||||
static bool SHA1Result(SHA1Context *context) {
|
||||
if (context->Corrupted) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!context->Computed) {
|
||||
SHA1PadMessage(context);
|
||||
context->Computed = true;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// This function accepts an array of bytes as the next portion of
|
||||
// the message.
|
||||
static void SHA1Input(SHA1Context *context,
|
||||
const unsigned char *message_array,
|
||||
unsigned length) {
|
||||
if (!length) return;
|
||||
|
||||
if (context->Computed || context->Corrupted) {
|
||||
context->Corrupted = true;
|
||||
return;
|
||||
}
|
||||
|
||||
while(length-- && !context->Corrupted) {
|
||||
context->Message_Block[context->Message_Block_Index++] =
|
||||
(*message_array & 0xFF);
|
||||
|
||||
context->Length_Low += 8;
|
||||
/* Force it to 32 bits */
|
||||
context->Length_Low &= 0xFFFFFFFF;
|
||||
if (context->Length_Low == 0) {
|
||||
context->Length_High++;
|
||||
/* Force it to 32 bits */
|
||||
context->Length_High &= 0xFFFFFFFF;
|
||||
if (context->Length_High == 0)
|
||||
{
|
||||
/* Message is too long */
|
||||
context->Corrupted = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (context->Message_Block_Index == 64)
|
||||
{
|
||||
SHA1ProcessMessageBlock(context);
|
||||
}
|
||||
|
||||
message_array++;
|
||||
}
|
||||
}
|
||||
|
||||
// This function will process the next 512 bits of the message stored
|
||||
// in the Message_Block array.
|
||||
static void SHA1ProcessMessageBlock(SHA1Context *context) {
|
||||
const unsigned K[] = // Constants defined in SHA-1
|
||||
{
|
||||
0x5A827999,
|
||||
0x6ED9EBA1,
|
||||
0x8F1BBCDC,
|
||||
0xCA62C1D6
|
||||
};
|
||||
int t; // Loop counter
|
||||
unsigned temp; // Temporary word value
|
||||
unsigned W[80]; // Word sequence
|
||||
unsigned A, B, C, D, E; // Word buffers
|
||||
|
||||
// Initialize the first 16 words in the array W
|
||||
for(t = 0; t < 16; t++) {
|
||||
W[t] = ((unsigned) context->Message_Block[t * 4]) << 24;
|
||||
W[t] |= ((unsigned) context->Message_Block[t * 4 + 1]) << 16;
|
||||
W[t] |= ((unsigned) context->Message_Block[t * 4 + 2]) << 8;
|
||||
W[t] |= ((unsigned) context->Message_Block[t * 4 + 3]);
|
||||
}
|
||||
|
||||
for(t = 16; t < 80; t++) {
|
||||
W[t] = SHA1CircularShift(1,W[t-3] ^ W[t-8] ^ W[t-14] ^ W[t-16]);
|
||||
}
|
||||
|
||||
A = context->Message_Digest[0];
|
||||
B = context->Message_Digest[1];
|
||||
C = context->Message_Digest[2];
|
||||
D = context->Message_Digest[3];
|
||||
E = context->Message_Digest[4];
|
||||
|
||||
for(t = 0; t < 20; t++) {
|
||||
temp = SHA1CircularShift(5,A) +
|
||||
((B & C) | ((~B) & D)) + E + W[t] + K[0];
|
||||
temp &= 0xFFFFFFFF;
|
||||
E = D;
|
||||
D = C;
|
||||
C = SHA1CircularShift(30,B);
|
||||
B = A;
|
||||
A = temp;
|
||||
}
|
||||
|
||||
for(t = 20; t < 40; t++) {
|
||||
temp = SHA1CircularShift(5,A) + (B ^ C ^ D) + E + W[t] + K[1];
|
||||
temp &= 0xFFFFFFFF;
|
||||
E = D;
|
||||
D = C;
|
||||
C = SHA1CircularShift(30,B);
|
||||
B = A;
|
||||
A = temp;
|
||||
}
|
||||
|
||||
for(t = 40; t < 60; t++) {
|
||||
temp = SHA1CircularShift(5,A) +
|
||||
((B & C) | (B & D) | (C & D)) + E + W[t] + K[2];
|
||||
temp &= 0xFFFFFFFF;
|
||||
E = D;
|
||||
D = C;
|
||||
C = SHA1CircularShift(30,B);
|
||||
B = A;
|
||||
A = temp;
|
||||
}
|
||||
|
||||
for(t = 60; t < 80; t++) {
|
||||
temp = SHA1CircularShift(5,A) + (B ^ C ^ D) + E + W[t] + K[3];
|
||||
temp &= 0xFFFFFFFF;
|
||||
E = D;
|
||||
D = C;
|
||||
C = SHA1CircularShift(30,B);
|
||||
B = A;
|
||||
A = temp;
|
||||
}
|
||||
|
||||
context->Message_Digest[0] = (context->Message_Digest[0] + A) & 0xFFFFFFFF;
|
||||
context->Message_Digest[1] = (context->Message_Digest[1] + B) & 0xFFFFFFFF;
|
||||
context->Message_Digest[2] = (context->Message_Digest[2] + C) & 0xFFFFFFFF;
|
||||
context->Message_Digest[3] = (context->Message_Digest[3] + D) & 0xFFFFFFFF;
|
||||
context->Message_Digest[4] = (context->Message_Digest[4] + E) & 0xFFFFFFFF;
|
||||
|
||||
context->Message_Block_Index = 0;
|
||||
}
|
||||
|
||||
// According to the standard, the message must be padded to an even
|
||||
// 512 bits. The first padding bit must be a '1'. The last 64 bits
|
||||
// represent the length of the original message. All bits in between
|
||||
// should be 0. This function will pad the message according to those
|
||||
// rules by filling the Message_Block array accordingly. It will also
|
||||
// call SHA1ProcessMessageBlock() appropriately. When it returns, it
|
||||
// can be assumed that the message digest has been computed.
|
||||
static void SHA1PadMessage(SHA1Context *context) {
|
||||
// Check to see if the current message block is too small to hold
|
||||
// the initial padding bits and length. If so, we will pad the
|
||||
// block, process it, and then continue padding into a second block.
|
||||
if (context->Message_Block_Index > 55) {
|
||||
context->Message_Block[context->Message_Block_Index++] = 0x80;
|
||||
while(context->Message_Block_Index < 64) {
|
||||
context->Message_Block[context->Message_Block_Index++] = 0;
|
||||
}
|
||||
|
||||
SHA1ProcessMessageBlock(context);
|
||||
|
||||
while(context->Message_Block_Index < 56) {
|
||||
context->Message_Block[context->Message_Block_Index++] = 0;
|
||||
}
|
||||
} else {
|
||||
context->Message_Block[context->Message_Block_Index++] = 0x80;
|
||||
while(context->Message_Block_Index < 56) {
|
||||
context->Message_Block[context->Message_Block_Index++] = 0;
|
||||
}
|
||||
}
|
||||
|
||||
// Store the message length as the last 8 octets
|
||||
context->Message_Block[56] = (context->Length_High >> 24) & 0xFF;
|
||||
context->Message_Block[57] = (context->Length_High >> 16) & 0xFF;
|
||||
context->Message_Block[58] = (context->Length_High >> 8) & 0xFF;
|
||||
context->Message_Block[59] = (context->Length_High) & 0xFF;
|
||||
context->Message_Block[60] = (context->Length_Low >> 24) & 0xFF;
|
||||
context->Message_Block[61] = (context->Length_Low >> 16) & 0xFF;
|
||||
context->Message_Block[62] = (context->Length_Low >> 8) & 0xFF;
|
||||
context->Message_Block[63] = (context->Length_Low) & 0xFF;
|
||||
|
||||
SHA1ProcessMessageBlock(context);
|
||||
}
|
||||
|
||||
|
||||
void SHA1_Hash_Portable(const char* data, size_t len, char* hash_array) {
|
||||
SHA1Context context;
|
||||
SHA1Reset(&context);
|
||||
SHA1Input(&context, reinterpret_cast<const unsigned char*>(data), len);
|
||||
bool ok = SHA1Result(&context);
|
||||
if (!ok) {
|
||||
fprintf(stderr, "Unexpected error in SHA1_Hash_Portable code\n");
|
||||
exit(1);
|
||||
}
|
||||
for (int i = 0; i < 5; i++) {
|
||||
uint32_t value = context.Message_Digest[i];
|
||||
hash_array[i*4 + 0] = (value >> 24) & 0xff;
|
||||
hash_array[i*4 + 1] = (value >> 16) & 0xff;
|
||||
hash_array[i*4 + 2] = (value >> 8) & 0xff;
|
||||
hash_array[i*4 + 3] = value & 0xff;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#ifndef STORAGE_LEVELDB_PORT_SHA1_PORTABLE_H_
|
||||
#define STORAGE_LEVELDB_PORT_SHA1_PORTABLE_H_
|
||||
|
||||
#include <stddef.h>
|
||||
|
||||
namespace leveldb {
|
||||
namespace port {
|
||||
|
||||
// Compute the SHA1 hash value of "data[0..len-1]" and store it in
|
||||
// "hash_array[0..19]". hash_array must have 20 bytes of space available.
|
||||
//
|
||||
// This function is portable but may not be as fast as a version
|
||||
// optimized for your platform. It is provided as a default method
|
||||
// that can be used when porting leveldb to a new platform if no
|
||||
// better SHA1 hash implementation is available.
|
||||
void SHA1_Hash_Portable(const char* data, size_t len, char* hash_array);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
#endif // STORAGE_LEVELDB_PORT_SHA1_PORTABLE_H_
|
||||
|
|
@ -1,39 +0,0 @@
|
|||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "port/port.h"
|
||||
#include "util/testharness.h"
|
||||
|
||||
namespace leveldb {
|
||||
namespace port {
|
||||
|
||||
class SHA1 { };
|
||||
|
||||
static std::string TestSHA1(const char* data, size_t len) {
|
||||
char hash_val[20];
|
||||
SHA1_Hash(data, len, hash_val);
|
||||
char buf[41];
|
||||
for (int i = 0; i < 20; i++) {
|
||||
snprintf(buf + i * 2, 41 - i * 2,
|
||||
"%02x",
|
||||
static_cast<unsigned int>(static_cast<unsigned char>(
|
||||
hash_val[i])));
|
||||
}
|
||||
return std::string(buf, 40);
|
||||
}
|
||||
|
||||
TEST(SHA1, Simple) {
|
||||
ASSERT_EQ("da39a3ee5e6b4b0d3255bfef95601890afd80709", TestSHA1("", 0));
|
||||
ASSERT_EQ("aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d", TestSHA1("hello", 5));
|
||||
std::string x(10000, 'x');
|
||||
ASSERT_EQ("f8c5cde791c5056cf515881e701c8a9ecb439a75",
|
||||
TestSHA1(x.data(), x.size()));
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
return leveldb::test::RunAllTests();
|
||||
}
|
||||
|
|
@ -285,7 +285,8 @@ class ShardedLRUCache : public Cache {
|
|||
}
|
||||
|
||||
public:
|
||||
explicit ShardedLRUCache(size_t capacity) {
|
||||
explicit ShardedLRUCache(size_t capacity)
|
||||
: last_id_(0) {
|
||||
const size_t per_shard = (capacity + (kNumShards - 1)) / kNumShards;
|
||||
for (int s = 0; s < kNumShards; s++) {
|
||||
shard_[s].SetCapacity(per_shard);
|
||||
|
|
|
|||
Loading…
Reference in New Issue