From ef4b87f1b2ee49ac602697786db06e9d5105903d Mon Sep 17 00:00:00 2001 From: Dmitri Smirnov Date: Tue, 7 Jul 2015 16:58:20 -0700 Subject: [PATCH] Commit both PR and internal code review changes --- CMakeLists.txt | 3 +- INSTALL.md | 3 + WINDOWS_PORT.md | 4 +- build_tools/build_detect_version.bat | 24 ---- build_tools/runall.bat | 99 --------------- db/c.cc | 2 +- db/file_indexer.h | 3 +- include/rocksdb/c.h | 14 +-- include/rocksdb/slice.h | 20 +-- include/rocksdb/thread_status.h | 8 -- include/rocksdb/utilities/spatial_db.h | 2 +- include/utilities/document_db.h | 6 +- include/utilities/geo_db.h | 6 +- include/utilities/json_document.h | 6 +- include/utilities/stackable_db.h | 5 +- port/port_posix.h | 5 + port/sys_time.h | 2 +- port/win/env_win.cc | 19 ++- port/win/port_win.h | 48 ++++---- port/win/win_logger.cc | 154 ++++++++++++------------ port/win/win_logger.h | 40 +++--- table/block_based_table_builder.cc | 2 +- table/format.h | 26 ++-- third-party/fbson/FbsonStream.h | 5 + util/arena.cc | 9 +- util/autovector.h | 1 - util/dynamic_bloom.h | 3 +- util/env.cc | 2 +- util/log_buffer.cc | 4 +- util/options_helper.cc | 2 + util/posix_logger.h | 2 +- util/slice.cc | 23 ++++ util/thread_local.cc | 18 +-- utilities/checkpoint/checkpoint_test.cc | 4 +- 34 files changed, 245 insertions(+), 329 deletions(-) delete mode 100644 build_tools/build_detect_version.bat delete mode 100644 build_tools/runall.bat diff --git a/CMakeLists.txt b/CMakeLists.txt index 1fbf1cef18..fcf24fbed8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,8 +1,9 @@ # This cmake build is for Windows only. # # Prerequisites: -# You must have Visual Studio 2013 installed. Start the Developer Command Prompt window that is a part of Visual Studio installation. +# You must have Visual Studio 2013 Update 4 installed. Start the Developer Command Prompt window that is a part of Visual Studio installation. # Run the build commands from within the Developer Command Prompt window to have paths to the compiler and runtime libraries set. +# Make sure that Git is in your PATH # # To build Rocksdb for Windows is as easy as 1-2-3-4-5: # diff --git a/INSTALL.md b/INSTALL.md index c1b13986fe..d10d82a7cc 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -80,3 +80,6 @@ your make commands, like this: `PORTABLE=1 make static_lib` * **iOS**: * Run: `TARGET_OS=IOS make static_lib`. When building the project which uses rocksdb iOS library, make sure to define two important pre-processing macros: `ROCKSDB_LITE` and `IOS_CROSS_COMPILE`. + +* **Windows**: + * Read the follow the instructions at CMakeLists.txt diff --git a/WINDOWS_PORT.md b/WINDOWS_PORT.md index 7f5d3e0403..a0fe1fe11f 100644 --- a/WINDOWS_PORT.md +++ b/WINDOWS_PORT.md @@ -15,7 +15,7 @@ These notes describe some decisions and changes we had to make with regards to p We are open for comments and improvements. ## OS specifics -All of the porting, testing and benchmarking was done on Windows Server 2012 R2 Datacenter but to the best of our knowledge there is not a specific API we used during porting that is unsupported on other Windows OS after Vista. +All of the porting, testing and benchmarking was done on Windows Server 2012 R2 Datacenter 64-bit but to the best of our knowledge there is not a specific API we used during porting that is unsupported on other Windows OS after Vista. ## Porting goals We strive to achieve the following goals: @@ -34,6 +34,8 @@ At the same time it generates Visual Studio projects that are both usable from a The top-level CMakeLists.txt file contains description of all targets and build rules. It also provides brief instructions on how to build the software for Windows. One more build related file is thirdparty.inc that also resides on the top level. This file must be edited to point to actual third party libraries location. We think that it would be beneficial to merge the existing make-based build system and the new cmake-based build system into a single one to use on all platforms. +All building and testing was done for 64-bit. We have not conducted any testing for 32-bit and early reports indicate that it will not run on 32-bit. + ## C++ and STL notes We had to make some minimum changes within the portable files that either account for OS differences or the shortcomings of C++11 support in the current version of the MS compiler. Most or all of them are expected to be fixed in the upcoming compiler releases. diff --git a/build_tools/build_detect_version.bat b/build_tools/build_detect_version.bat deleted file mode 100644 index 41066af9e5..0000000000 --- a/build_tools/build_detect_version.bat +++ /dev/null @@ -1,24 +0,0 @@ -@echo off - -REM Record the version of the source that we are compiling. -REM We keep a record of the git revision in util/version.cc. This source file -REM is then built as a regular source file as part of the compilation process. -REM One can run "strings executable_filename | grep _build_" to find the version of -REM the source that we used to build the executable file. - -set CONFIGURATION=%1 - -pushd "%~dp0" -set "OUTFILE="..\util\build_version_%CONFIGURATION%.cc" - -REM GIT_SHA="" -REM if command -v git >/dev/null 2>&1; then -REM GIT_SHA=$(git rev-parse HEAD 2>/dev/null) -REM fi - -@echo #include "build_version.h" > %OUTFILE% -@echo const char* rocksdb_build_git_sha = "rocksdb_build_git_sha:${GIT_SHA}"; >> %OUTFILE% -@echo const char* rocksdb_build_git_datetime = "rocksdb_build_git_datetime:$(date)"; >> %OUTFILE% -@echo const char* rocksdb_build_compile_date = __DATE__; >> %OUTFILE% - -@popd diff --git a/build_tools/runall.bat b/build_tools/runall.bat deleted file mode 100644 index 039fb690cd..0000000000 --- a/build_tools/runall.bat +++ /dev/null @@ -1,99 +0,0 @@ -@echo off -call :init -call :runtest arena_test.exe -call :runtest autovector_test.exe -call :runtest auto_roll_logger_test.exe -call :runtest backupable_db_test.exe -rem call :runtest benchharness_test.exe -call :runtest block_based_filter_block_test.exe -call :runtest block_hash_index_test.exe -call :runtest block_test.exe -call :runtest bloom_test.exe -call :runtest cache_test.exe -call :runtest coding_test.exe -call :runtest column_family_test.exe -call :runtest compaction_job_test.exe -call :runtest compaction_picker_test.exe -call :runtest comparator_db_test.exe -call :runtest corruption_test.exe -call :runtest crc32c_test.exe -call :runtest cuckoo_table_builder_test.exe -call :runtest cuckoo_table_db_test.exe -call :runtest cuckoo_table_reader_test.exe -call :runtest dbformat_test.exe -call :runtest db_iter_test.exe -call :runtest db_test.exe -call :runtest deletefile_test.exe -call :runtest dynamic_bloom_test.exe -call :runtest env_test.exe -call :runtest fault_injection_test.exe -call :runtest filelock_test.exe -call :runtest filename_test.exe -call :runtest file_indexer_test.exe -call :runtest full_filter_block_test.exe -call :runtest histogram_test.exe -call :runtest listener_test.exe -call :runtest log_test.exe -call :runtest manual_compaction_test.exe -call :runtest memenv_test.exe -call :runtest merger_test.exe -call :runtest merge_test.exe -call :runtest mock_env_test.exe -call :runtest options_test.exe -call :runtest perf_context_test.exe -call :runtest plain_table_db_test.exe -call :runtest prefix_test.exe -call :runtest rate_limiter_test.exe -call :runtest redis_lists_test.exe -rem call :runtest signal_test.exe -call :runtest skiplist_test.exe -call :runtest slice_transform_test.exe -call :runtest sst_dump_test.exe -call :runtest stringappend_test.exe -call :runtest table_properties_collector_test.exe -call :runtest table_test.exe -call :runtest thread_list_test.exe -call :runtest thread_local_test.exe -call :runtest ttl_test.exe -call :runtest version_builder_test.exe -call :runtest version_edit_test.exe -call :runtest version_set_test.exe -call :runtest wal_manager_test.exe -call :runtest write_batch_test.exe -rem call :runtest write_batch_with_index_test.exe -call :runtest write_controller_test.exe -call :stat -goto :eof - -:init -set tests=0 -set passed=0 -set failed=0 -goto :eof - -:runtest -set /A tests=%tests% + 1 -echo|set /p=Running %1... -%1 > %1.log 2>&1 -findstr /C:"PASSED" %1.log > nul 2>&1 -IF ERRORLEVEL 1 ( - findstr /C:"Passed all tests" %1.log > nul 2>&1 - IF ERRORLEVEL 1 ( - echo ***FAILED*** - set /A failed=%failed% + 1 - ) ELSE ( - echo OK - set /A passed=%passed% + 1 - ) -) ELSE ( - echo OK - set /A passed=%passed% + 1 -) -goto :eof - -:stat -echo ================= -echo Total tests : %tests% -echo Passed : %passed% -echo Failed : %failed% -goto :eof diff --git a/db/c.cc b/db/c.cc index 96a5882d89..318b92f2e1 100644 --- a/db/c.cc +++ b/db/c.cc @@ -484,7 +484,7 @@ static bool SaveError(char** errptr, const Status& s) { *errptr = strdup(s.ToString().c_str()); } else { // TODO(sanjay): Merge with existing error? - // This is a bug if *errptr is not create by malloc() + // This is a bug if *errptr is not created by malloc() free(*errptr); *errptr = strdup(s.ToString().c_str()); } diff --git a/db/file_indexer.h b/db/file_indexer.h index 748669a82a..617634192a 100644 --- a/db/file_indexer.h +++ b/db/file_indexer.h @@ -12,6 +12,7 @@ #include #include #include +#include "port/port.h" #include "util/arena.h" #include "util/autovector.h" @@ -59,7 +60,7 @@ class FileIndexer { enum { // MSVC version 1800 still does not have constexpr for ::max() - kLevelMaxIndex = INT32_MAX + kLevelMaxIndex = rocksdb::port::LevelMaxIndex }; private: diff --git a/include/rocksdb/c.h b/include/rocksdb/c.h index fc36372b26..42641c3ea8 100644 --- a/include/rocksdb/c.h +++ b/include/rocksdb/c.h @@ -192,13 +192,13 @@ extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open_for_read_only_column_families unsigned char error_if_log_file_exist, char** errptr); -ROCKSDB_LIBRARY_API char** rocksdb_list_column_families( +extern ROCKSDB_LIBRARY_API char** rocksdb_list_column_families( const rocksdb_options_t* options, const char* name, size_t* lencf, char** errptr); -ROCKSDB_LIBRARY_API void rocksdb_list_column_families_destroy(char** list, size_t len); +extern ROCKSDB_LIBRARY_API void rocksdb_list_column_families_destroy(char** list, size_t len); extern ROCKSDB_LIBRARY_API rocksdb_column_family_handle_t* rocksdb_create_column_family( rocksdb_t* db, @@ -236,7 +236,7 @@ extern ROCKSDB_LIBRARY_API void rocksdb_delete( const char* key, size_t keylen, char** errptr); -void ROCKSDB_LIBRARY_API rocksdb_delete_cf( +extern ROCKSDB_LIBRARY_API void rocksdb_delete_cf( rocksdb_t* db, const rocksdb_writeoptions_t* options, rocksdb_column_family_handle_t* column_family, @@ -467,16 +467,16 @@ extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_delete_cf( rocksdb_writebatch_t*, rocksdb_column_family_handle_t* column_family, const char* key, size_t klen); -ROCKSDB_LIBRARY_API void rocksdb_writebatch_deletev( +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_deletev( rocksdb_writebatch_t* b, int num_keys, const char* const* keys_list, const size_t* keys_list_sizes); -ROCKSDB_LIBRARY_API void rocksdb_writebatch_deletev_cf( +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_deletev_cf( rocksdb_writebatch_t* b, rocksdb_column_family_handle_t* column_family, int num_keys, const char* const* keys_list, const size_t* keys_list_sizes); -ROCKSDB_LIBRARY_API extern void rocksdb_writebatch_put_log_data( +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_put_log_data( rocksdb_writebatch_t*, const char* blob, size_t len); extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_iterate( @@ -986,7 +986,7 @@ extern ROCKSDB_LIBRARY_API void rocksdb_get_options_from_string( rocksdb_options_t* new_options, char** errptr); -// refering to convention (3), this should be used by client +// referring to convention (3), this should be used by client // to free memory that was malloc()ed extern ROCKSDB_LIBRARY_API void rocksdb_free( void* ptr); diff --git a/include/rocksdb/slice.h b/include/rocksdb/slice.h index c07f627557..084e846e78 100644 --- a/include/rocksdb/slice.h +++ b/include/rocksdb/slice.h @@ -24,12 +24,6 @@ #include #include #include -#include - -// Do not want to include the whole /port/port.h here for one define -#ifdef OS_WIN -# define snprintf _snprintf -#endif namespace rocksdb { @@ -80,19 +74,7 @@ class Slice { } // Return a string that contains the copy of the referenced data. - std::string ToString(bool hex = false) const { - if (hex) { - std::string result; - char buf[10]; - for (size_t i = 0; i < size_; i++) { - snprintf(buf, 10, "%02X", (unsigned char)data_[i]); - result += buf; - } - return result; - } else { - return std::string(data_, size_); - } - } + std::string ToString(bool hex = false) const; // Three-way comparison. Returns value: // < 0 iff "*this" < "b", diff --git a/include/rocksdb/thread_status.h b/include/rocksdb/thread_status.h index 282c9eb2ee..4d53b28aa9 100644 --- a/include/rocksdb/thread_status.h +++ b/include/rocksdb/thread_status.h @@ -32,15 +32,11 @@ namespace rocksdb { // TODO(yhchiang): remove this function once c++14 is available // as std::max will be able to cover this. -#ifndef OS_WIN -constexpr int constexpr_max(int a, int b) { return a > b ? a : b; } -#else // Current MS compiler does not support constexpr template struct constexpr_max { static const int result = (A > B) ? A : B; }; -#endif // A structure that describes the current status of a thread. // The status of active threads can be fetched using @@ -100,11 +96,7 @@ struct ThreadStatus { // The maximum number of properties of an operation. // This number should be set to the biggest NUM_XXX_PROPERTIES. static const int kNumOperationProperties = -#ifndef OS_WIN - constexpr_max(NUM_COMPACTION_PROPERTIES, NUM_FLUSH_PROPERTIES); -#else constexpr_max::result; -#endif // The type used to refer to a thread state. // A state describes lower-level action of a thread diff --git a/include/rocksdb/utilities/spatial_db.h b/include/rocksdb/utilities/spatial_db.h index 979968cb6e..894930a682 100644 --- a/include/rocksdb/utilities/spatial_db.h +++ b/include/rocksdb/utilities/spatial_db.h @@ -80,7 +80,7 @@ struct Variant { const std::string& get_string() const { return *GetStringPtr(data_); } bool operator==(const Variant& other) const; - bool operator!=(const Variant& rhs) const { return !(*this == rhs); } + bool operator!=(const Variant& other) const { return !(*this == other); } private: diff --git a/include/utilities/document_db.h b/include/utilities/document_db.h index 1d1330bcae..9c4a388ea7 100644 --- a/include/utilities/document_db.h +++ b/include/utilities/document_db.h @@ -4,5 +4,9 @@ // of patent rights can be found in the PATENTS file in the same directory. #pragma once -#warning This file was moved to rocksdb/utilities/document_db.h + +#include "pragma_error.h" + +ROCKSDB_WARNING("This file was moved to rocksdb/utilities/document_db.h") + #include "rocksdb/utilities/document_db.h" diff --git a/include/utilities/geo_db.h b/include/utilities/geo_db.h index 48957d4075..ae2e8c47fa 100644 --- a/include/utilities/geo_db.h +++ b/include/utilities/geo_db.h @@ -4,5 +4,9 @@ // of patent rights can be found in the PATENTS file in the same directory. #pragma once -#warning This file was moved to rocksdb/utilities/geo_db.h + +#include "pragma_error.h" + +ROCKSDB_WARNING("This file was moved to rocksdb/utilities/geo_db.h") + #include "rocksdb/utilities/geo_db.h" diff --git a/include/utilities/json_document.h b/include/utilities/json_document.h index f3f93969d1..fb8b7d304f 100644 --- a/include/utilities/json_document.h +++ b/include/utilities/json_document.h @@ -3,5 +3,9 @@ // LICENSE file in the root directory of this source tree. An additional grant // of patent rights can be found in the PATENTS file in the same directory. #pragma once -#warning This file was moved to rocksdb/utilities/json_document.h + +#include "pragma_error.h" + +ROCKSDB_WARNING("This file was moved to rocksdb/utilities/json_document.h") + #include "rocksdb/utilities/json_document.h" diff --git a/include/utilities/stackable_db.h b/include/utilities/stackable_db.h index 435818d2b6..8f72b04a6a 100644 --- a/include/utilities/stackable_db.h +++ b/include/utilities/stackable_db.h @@ -3,5 +3,8 @@ // found in the LICENSE file. See the AUTHORS file for names of contributors. #pragma once -#warning This file was moved to rocksdb/utilities/stackable_db.h +#include "pragma_error.h" + +ROCKSDB_WARNING("This file was moved to rocksdb/utilities/stackable_db.h") + #include "rocksdb/utilities/stackable_db.h" diff --git a/port/port_posix.h b/port/port_posix.h index e9af0978d7..75ae996c0a 100644 --- a/port/port_posix.h +++ b/port/port_posix.h @@ -71,9 +71,14 @@ #define fdatasync fsync #endif +#include + namespace rocksdb { namespace port { +// For use at db/file_indexer.h kLevelMaxIndex +const int LevelMaxIndex = std::numeric_limits::max(); + static const bool kLittleEndian = PLATFORM_IS_LITTLE_ENDIAN; #undef PLATFORM_IS_LITTLE_ENDIAN diff --git a/port/sys_time.h b/port/sys_time.h index f416159e22..0ebf298aa0 100644 --- a/port/sys_time.h +++ b/port/sys_time.h @@ -12,7 +12,7 @@ #ifndef STORAGE_LEVELDB_PORT_SYS_TIME_H_ #define STORAGE_LEVELDB_PORT_SYS_TIME_H_ -#if defined(_WIN32) && defined(_MSC_VER) +#if defined(OS_WIN) && defined(_MSC_VER) #include diff --git a/port/win/env_win.cc b/port/win/env_win.cc index c56873ce20..27c5c3fbda 100644 --- a/port/win/env_win.cc +++ b/port/win/env_win.cc @@ -229,10 +229,9 @@ size_t Roundup(size_t x, size_t y) { } -// Can only truncate or reserve to a sector size aligned if -// used on files that are opened with Unbuffered I/O -// Normally it does not present a problem since in memory mapped files -// we do not disable buffering +// SetFileInformationByHandle() is capable of fast pre-allocates. +// However, this does not change the file end position unless the file is +// truncated and the pre-allocated space is not considered filled with zeros. inline Status fallocate(const std::string& filename, HANDLE hFile, uint64_t to_size) { @@ -394,7 +393,7 @@ class WinMmapReadableFile : public RandomAccessFile { const size_t length_; public: - // base[0,length-1] contains the mmapped contents of the file. + // mapped_region_[0,length-1] contains the mmapped contents of the file. WinMmapReadableFile(const std::string &fileName, HANDLE hFile, HANDLE hMap, const void* mapped_region, size_t length) : fileName_(fileName), hFile_(hFile), hMap_(hMap), mapped_region_(mapped_region), length_(length) { @@ -1523,7 +1522,7 @@ public: NULL); } - if (hFile == INVALID_HANDLE_VALUE) { + if (INVALID_HANDLE_VALUE == hFile) { auto lastError = GetLastError(); s = IOErrorFromWindowsError("Failed to open NewSequentialFile" + fname, lastError); } else { @@ -1565,7 +1564,7 @@ public: /// Shared access is necessary for corruption test to pass // almost all tests would work with a possible exception of fault_injection - HANDLE hFile; + HANDLE hFile = 0; { IOSTATS_TIMER_GUARD(open_nanos); hFile = CreateFileA( @@ -1717,8 +1716,8 @@ public: FILE_ATTRIBUTE_NORMAL, NULL); } - - if (hFile == INVALID_HANDLE_VALUE) { + + if (INVALID_HANDLE_VALUE == hFile) { auto lastError = GetLastError(); s = IOErrorFromWindowsError("Failed to Open/Create NewRandomRWFile" + fname, lastError); } @@ -2018,7 +2017,7 @@ public: NULL); } - if (hFile == INVALID_HANDLE_VALUE) { + if (INVALID_HANDLE_VALUE == hFile) { auto lastError = GetLastError(); s = IOErrorFromWindowsError("Failed to open LogFile" + fname, lastError); } else { diff --git a/port/win/port_win.h b/port/win/port_win.h index 1c638878be..6e1a40bcd8 100644 --- a/port/win/port_win.h +++ b/port/win/port_win.h @@ -62,7 +62,7 @@ typedef SSIZE_T ssize_t; #endif #ifdef SNAPPY -#include "snappy.h" +#include #endif // Thread local storage on Linux @@ -80,6 +80,9 @@ namespace rocksdb { namespace port { +// For use at db/file_indexer.h kLevelMaxIndex +const int LevelMaxIndex = INT32_MAX; + const bool kLittleEndian = true; class CondVar; @@ -87,8 +90,7 @@ class CondVar; class Mutex { public: - /* implicit */ - Mutex(bool adaptive = false); + /* implicit */ Mutex(bool adaptive = false); ~Mutex(); void Lock(); @@ -97,8 +99,8 @@ public: // this will assert if the mutex is not locked // it does NOT verify that mutex is held by a calling thread void AssertHeld(); - std::unique_lock& getLock() - { + + std::unique_lock& getLock() { return lock; } @@ -117,38 +119,36 @@ private: class RWMutex { -private: - SRWLOCK srwLock_; public: - RWMutex(){ - InitializeSRWLock(&srwLock_); + RWMutex() { + InitializeSRWLock(&srwLock_); } void ReadLock() { - AcquireSRWLockShared(&srwLock_); + AcquireSRWLockShared(&srwLock_); } void WriteLock() { - AcquireSRWLockExclusive(&srwLock_); + AcquireSRWLockExclusive(&srwLock_); } void ReadUnlock() { - ReleaseSRWLockShared(&srwLock_); + ReleaseSRWLockShared(&srwLock_); } void WriteUnlock() { - ReleaseSRWLockExclusive(&srwLock_); + ReleaseSRWLockExclusive(&srwLock_); } - void AssertHeld() { - //TODO: psrao - should be implemented - } + // Empty as in POSIX + void AssertHeld() { } private: - // No copying allowed - RWMutex(const RWMutex&); - void operator=(const RWMutex&); + SRWLOCK srwLock_; + // No copying allowed + RWMutex(const RWMutex&); + void operator=(const RWMutex&); }; class CondVar @@ -520,7 +520,7 @@ int pthread_key_create(pthread_key_t *key, void(*destructor)(void*)) { (void)destructor; pthread_key_t k = TlsAlloc(); - if (k == TLS_OUT_OF_INDEXES) { + if (TLS_OUT_OF_INDEXES == k) { return ENOMEM; } @@ -530,7 +530,7 @@ int pthread_key_create(pthread_key_t *key, void(*destructor)(void*)) { inline int pthread_key_delete(pthread_key_t key) { - if(!TlsFree(key)) { + if (!TlsFree(key)) { return EINVAL; } return 0; @@ -538,7 +538,7 @@ int pthread_key_delete(pthread_key_t key) { inline int pthread_setspecific(pthread_key_t key, const void *value) { - if(!TlsSetValue(key, const_cast(value))) { + if (!TlsSetValue(key, const_cast(value))) { return ENOMEM; } return 0; @@ -547,8 +547,8 @@ int pthread_setspecific(pthread_key_t key, const void *value) { inline void* pthread_getspecific(pthread_key_t key) { void* result = TlsGetValue(key); - if(!result) { - if(GetLastError() != ERROR_SUCCESS) { + if (!result) { + if (GetLastError() != ERROR_SUCCESS) { errno = EINVAL; } else { errno = NOERROR; diff --git a/port/win/win_logger.cc b/port/win/win_logger.cc index 1ef58e21b8..44c279af3f 100644 --- a/port/win/win_logger.cc +++ b/port/win/win_logger.cc @@ -24,8 +24,6 @@ namespace rocksdb { -//const int kDebugLogChunkSize = 128 * 1024; - WinLogger::WinLogger(uint64_t (*gettid)(), Env* env, FILE * file, const InfoLogLevel log_level) : Logger(log_level), gettid_(gettid), @@ -61,91 +59,93 @@ void WinLogger::Flush() { } void WinLogger::Logv(const char* format, va_list ap) { - const uint64_t thread_id = (*gettid_)(); - IOSTATS_TIMER_GUARD(logger_nanos); - // We try twice: the first time with a fixed-size stack allocated buffer, - // and the second time with a much larger dynamically allocated buffer. - char buffer[500]; - std::unique_ptr largeBuffer; - for (int iter = 0; iter < 2; ++iter) { - char* base; - int bufsize; - if (iter == 0) { - bufsize = sizeof(buffer); - base = buffer; - } else { - bufsize = 30000; - largeBuffer.reset(new char[bufsize]); - base = largeBuffer.get(); - } + IOSTATS_TIMER_GUARD(logger_nanos); - char* p = base; - char* limit = base + bufsize; + const uint64_t thread_id = (*gettid_)(); - struct timeval now_tv; - gettimeofday(&now_tv, nullptr); - const time_t seconds = now_tv.tv_sec; - struct tm t; - localtime_s(&t, &seconds); - p += snprintf(p, limit - p, "%04d/%02d/%02d-%02d:%02d:%02d.%06d %llx ", t.tm_year + 1900, t.tm_mon + 1, t.tm_mday, - t.tm_hour, - t.tm_min, - t.tm_sec, - static_cast(now_tv.tv_usec), - static_cast(thread_id)); + // We try twice: the first time with a fixed-size stack allocated buffer, + // and the second time with a much larger dynamically allocated buffer. + char buffer[500]; + std::unique_ptr largeBuffer; + for (int iter = 0; iter < 2; ++iter) { + char* base; + int bufsize; + if (iter == 0) { + bufsize = sizeof(buffer); + base = buffer; + } else { + bufsize = 30000; + largeBuffer.reset(new char[bufsize]); + base = largeBuffer.get(); + } - // Print the message - if (p < limit) { - va_list backup_ap; - va_copy(backup_ap, ap); - int done = vsnprintf(p, limit - p, format, backup_ap); - if (done > 0){ - p += done; - } else { - continue; - } - va_end(backup_ap); - } + char* p = base; + char* limit = base + bufsize; - // Truncate to available space if necessary - if (p >= limit) { - if (iter == 0) - { - continue; // Try again with larger buffer - } else { - p = limit - 1; - } - } + struct timeval now_tv; + gettimeofday(&now_tv, nullptr); + const time_t seconds = now_tv.tv_sec; + struct tm t; + localtime_s(&t, &seconds); + p += snprintf(p, limit - p, "%04d/%02d/%02d-%02d:%02d:%02d.%06d %llx ", t.tm_year + 1900, t.tm_mon + 1, t.tm_mday, + t.tm_hour, + t.tm_min, + t.tm_sec, + static_cast(now_tv.tv_usec), + static_cast(thread_id)); - // Add newline if necessary - if (p == base || p[-1] != '\n') { - *p++ = '\n'; - } + // Print the message + if (p < limit) { + va_list backup_ap; + va_copy(backup_ap, ap); + int done = vsnprintf(p, limit - p, format, backup_ap); + if (done > 0){ + p += done; + } else { + continue; + } + va_end(backup_ap); + } - assert(p <= limit); - const size_t write_size = p - base; + // Truncate to available space if necessary + if (p >= limit) { + if (iter == 0) + { + continue; // Try again with larger buffer + } else { + p = limit - 1; + } + } - size_t sz = fwrite(base, 1, write_size, file_); - if (sz == 0) { - perror("fwrite .. [BAD]"); - } + // Add newline if necessary + if (p == base || p[-1] != '\n') { + *p++ = '\n'; + } - flush_pending_ = true; - assert(sz == write_size); - if (sz > 0) { - log_size_ += write_size; - } + assert(p <= limit); + const size_t write_size = p - base; - uint64_t now_micros = static_cast(now_tv.tv_sec) * 1000000 + - now_tv.tv_usec; - if (now_micros - last_flush_micros_ >= flush_every_seconds_ * 1000000) { - flush_pending_ = false; - fflush(file_); - last_flush_micros_ = now_micros; - } - break; - } + size_t sz = fwrite(base, 1, write_size, file_); + if (sz == 0) { + perror("fwrite .. [BAD]"); + } + + flush_pending_ = true; + assert(sz == write_size); + if (sz > 0) { + log_size_ += write_size; + } + + uint64_t now_micros = static_cast(now_tv.tv_sec) * 1000000 + + now_tv.tv_usec; + if (now_micros - last_flush_micros_ >= flush_every_seconds_ * 1000000) { + flush_pending_ = false; + fflush(file_); + last_flush_micros_ = now_micros; + } + break; + } } size_t WinLogger::GetLogFileSize() const { diff --git a/port/win/win_logger.h b/port/win/win_logger.h index 0a9dabf4a1..b7c85100ba 100644 --- a/port/win/win_logger.h +++ b/port/win/win_logger.h @@ -23,30 +23,36 @@ class Env; const int kDebugLogChunkSize = 128 * 1024; class WinLogger : public rocksdb::Logger { -private: - FILE* file_; - uint64_t (*gettid_)(); // Return the thread id for the current thread - std::atomic_size_t log_size_; - std::atomic_uint_fast64_t last_flush_micros_; - Env* env_; - bool flush_pending_; - - const static uint64_t flush_every_seconds_ = 5; - public: - WinLogger(uint64_t(*gettid)(), Env* env, FILE * file, const InfoLogLevel log_level = InfoLogLevel::ERROR_LEVEL); + WinLogger(uint64_t(*gettid)(), Env* env, FILE * file, + const InfoLogLevel log_level = InfoLogLevel::ERROR_LEVEL); - virtual ~WinLogger(); + virtual ~WinLogger(); - void close(); + WinLogger(const WinLogger&) = delete; - void Flush() override; + WinLogger& operator=(const WinLogger&) = delete; - void Logv(const char* format, va_list ap) override; + void close(); - size_t GetLogFileSize() const override; + void Flush() override; - void DebugWriter(const char* str, int len); + void Logv(const char* format, va_list ap) override; + + size_t GetLogFileSize() const override; + + void DebugWriter(const char* str, int len); + +private: + + FILE* file_; + uint64_t(*gettid_)(); // Return the thread id for the current thread + std::atomic_size_t log_size_; + std::atomic_uint_fast64_t last_flush_micros_; + Env* env_; + bool flush_pending_; + + const static uint64_t flush_every_seconds_ = 5; }; } // namespace rocksdb diff --git a/table/block_based_table_builder.cc b/table/block_based_table_builder.cc index 006be6fdea..4b58e93823 100644 --- a/table/block_based_table_builder.cc +++ b/table/block_based_table_builder.cc @@ -374,7 +374,7 @@ Slice CompressBlock(const Slice& raw, // kBlockBasedTableMagicNumber was picked by running // echo rocksdb.table.block_based | sha1sum // and taking the leading 64 bits. -// Please note that kBlockBasedTableMagicNumber may also be accessed by +// Please note that kBlockBasedTableMagicNumber may also be accessed by other .cc files // for that reason we declare it extern in the header but to get the space allocated // it must be not extern in one place. const uint64_t kBlockBasedTableMagicNumber = 0x88e241b785f4cff7ull; diff --git a/table/format.h b/table/format.h index 06eef47cd5..7b231da28e 100644 --- a/table/format.h +++ b/table/format.h @@ -182,26 +182,26 @@ struct BlockContents { BlockContents() : cachable(false), compression_type(kNoCompression) {} BlockContents(const Slice& _data, bool _cachable, - CompressionType _compression_type) - : data(_data), cachable(_cachable), compression_type(_compression_type) {} + CompressionType _compression_type) + : data(_data), cachable(_cachable), compression_type(_compression_type) {} BlockContents(std::unique_ptr&& _data, size_t _size, bool _cachable, - CompressionType _compression_type) - : data(_data.get(), _size), - cachable(_cachable), - compression_type(_compression_type), - allocation(std::move(_data)) {} + CompressionType _compression_type) + : data(_data.get(), _size), + cachable(_cachable), + compression_type(_compression_type), + allocation(std::move(_data)) {} BlockContents(BlockContents&& other) { - *this = std::move(other); + *this = std::move(other); } BlockContents& operator=(BlockContents&& other) { - data = std::move(other.data); - cachable = other.cachable; - compression_type = other.compression_type; - allocation = std::move(other.allocation); - return *this; + data = std::move(other.data); + cachable = other.cachable; + compression_type = other.compression_type; + allocation = std::move(other.allocation); + return *this; } }; diff --git a/third-party/fbson/FbsonStream.h b/third-party/fbson/FbsonStream.h index 6ac132bae5..bd59ea1728 100644 --- a/third-party/fbson/FbsonStream.h +++ b/third-party/fbson/FbsonStream.h @@ -30,6 +30,11 @@ #define __STDC_FORMAT_MACROS #endif +#if defined OS_WIN && !defined snprintf +# define snprintf _snprintf +#endif + + #include #include diff --git a/util/arena.cc b/util/arena.cc index 0a0acd55d1..09c2bcedc6 100644 --- a/util/arena.cc +++ b/util/arena.cc @@ -17,6 +17,11 @@ namespace rocksdb { +// MSVC complains that it is already defined since it is static in the header. +#ifndef OS_WIN +const size_t Arena::kInlineSize; +#endif + const size_t Arena::kMinBlockSize = 4096; const size_t Arena::kMaxBlockSize = 2 << 30; static const int kAlignUnit = sizeof(void*); @@ -54,8 +59,8 @@ Arena::~Arena() { for (const auto& block : blocks_) { delete[] block; } -// yuslepukhin: this needs to be addressed as it previously was under #ifdef -#ifndef OS_WIN + +#ifdef MAP_HUGETLB for (const auto& mmap_info : huge_blocks_) { auto ret = munmap(mmap_info.addr_, mmap_info.length_); if (ret != 0) { diff --git a/util/autovector.h b/util/autovector.h index 3d0820bf4c..11fc4667b0 100644 --- a/util/autovector.h +++ b/util/autovector.h @@ -240,7 +240,6 @@ class autovector { } void push_back(const T& item) { - //psrao: causes infinite recursion with VC if (num_stack_items_ < kSize) { values_[num_stack_items_++] = item; } diff --git a/util/dynamic_bloom.h b/util/dynamic_bloom.h index 58cad7c86f..e2ac56e764 100644 --- a/util/dynamic_bloom.h +++ b/util/dynamic_bloom.h @@ -9,8 +9,7 @@ #include "rocksdb/slice.h" - -#include +#include "port/port.h" #include #include diff --git a/util/env.cc b/util/env.cc index 95b54c3837..d98d926d84 100644 --- a/util/env.cc +++ b/util/env.cc @@ -10,7 +10,7 @@ #include "rocksdb/env.h" #include -#include +#include "port/sys_time.h" #include "rocksdb/options.h" #include "util/arena.h" diff --git a/util/log_buffer.cc b/util/log_buffer.cc index f4f016b84d..7d15cf22ec 100644 --- a/util/log_buffer.cc +++ b/util/log_buffer.cc @@ -5,8 +5,8 @@ #include "util/log_buffer.h" -#include -#include +#include "port/sys_time.h" +#include "port/port.h" namespace rocksdb { diff --git a/util/options_helper.cc b/util/options_helper.cc index 4de39f652e..fdbedc7cd3 100644 --- a/util/options_helper.cc +++ b/util/options_helper.cc @@ -417,6 +417,8 @@ bool ParseColumnFamilyOption(const std::string& name, const std::string& value, new_options->memtable_factory.reset(new_mem_factory); } else if (name == "min_write_buffer_number_to_merge") { new_options->min_write_buffer_number_to_merge = ParseInt(value); + } else if (name == "max_write_buffer_number_to_maintain") { + new_options->max_write_buffer_number_to_maintain = ParseInt(value); } else if (name == "compression") { new_options->compression = ParseCompressionType(value); } else if (name == "compression_per_level") { diff --git a/util/posix_logger.h b/util/posix_logger.h index 1d495973bb..55cb34a864 100644 --- a/util/posix_logger.h +++ b/util/posix_logger.h @@ -13,7 +13,7 @@ #pragma once #include #include -#include +#include "port/sys_time.h" #include #include #ifdef OS_LINUX diff --git a/util/slice.cc b/util/slice.cc index 371272e0e5..03d89e27f9 100644 --- a/util/slice.cc +++ b/util/slice.cc @@ -11,6 +11,7 @@ #include "rocksdb/slice_transform.h" #include "rocksdb/slice.h" #include "util/string_util.h" +#include namespace rocksdb { @@ -93,6 +94,28 @@ class NoopTransform : public SliceTransform { } +// Do not want to include the whole /port/port.h here for one define +#ifdef OS_WIN +# define snprintf _snprintf +#endif + +// Return a string that contains the copy of the referenced data. +std::string Slice::ToString(bool hex) const { + std::string result; // RVO/NRVO/move + if (hex) { + char buf[10]; + for (size_t i = 0; i < size_; i++) { + snprintf(buf, 10, "%02X", (unsigned char)data_[i]); + result += buf; + } + return result; + } else { + result.assign(data_, size_); + return result; + } +} + + const SliceTransform* NewFixedPrefixTransform(size_t prefix_len) { return new FixedPrefixTransform(prefix_len); } diff --git a/util/thread_local.cc b/util/thread_local.cc index ca64a77d98..ab70c4a6f7 100644 --- a/util/thread_local.cc +++ b/util/thread_local.cc @@ -42,15 +42,15 @@ pthread_key_t thread_local_key = -1; // Static callback function to call with each thread termination. void NTAPI WinOnThreadExit(PVOID module, DWORD reason, PVOID reserved) { - // We decided to punt on PROCESS_EXIT - if (DLL_THREAD_DETACH == reason) { - if (thread_local_key != -1 && thread_local_inclass_routine != nullptr) { - void* tls = pthread_getspecific(thread_local_key); - if(tls != nullptr) { - thread_local_inclass_routine(tls); - } - } - } + // We decided to punt on PROCESS_EXIT + if (DLL_THREAD_DETACH == reason) { + if (thread_local_key != -1 && thread_local_inclass_routine != nullptr) { + void* tls = pthread_getspecific(thread_local_key); + if(tls != nullptr) { + thread_local_inclass_routine(tls); + } + } + } } } // wintlscleanup diff --git a/utilities/checkpoint/checkpoint_test.cc b/utilities/checkpoint/checkpoint_test.cc index c143831c89..b01fe647e5 100644 --- a/utilities/checkpoint/checkpoint_test.cc +++ b/utilities/checkpoint/checkpoint_test.cc @@ -8,7 +8,7 @@ // found in the LICENSE file. See the AUTHORS file for names of contributors. // Syncpoint prevents us building and running tests in release -#if !defined( NDEBUG) || !defined (OS_WIN) +#if !defined(NDEBUG) || !defined (OS_WIN) #ifndef OS_WIN # include @@ -351,7 +351,7 @@ TEST_F(DBTest, CheckpointCF) { #endif int main(int argc, char** argv) { -#if !defined( NDEBUG) || !defined (OS_WIN) +#if !defined(NDEBUG) || !defined (OS_WIN) rocksdb::port::InstallStackTraceHandler(); ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS();