diff --git a/.gitignore b/.gitignore index 7072f34935..6a92b5d537 100644 --- a/.gitignore +++ b/.gitignore @@ -38,7 +38,7 @@ coverage/COVERAGE_REPORT .gdbhistory package/ .phutil_module_cache -unity +unity.a tags rocksdb_dump rocksdb_undump @@ -52,6 +52,8 @@ java/include/org_rocksdb_*.h .idea/ *.iml +rocksdb.cc +rocksdb.h unity.cc java/crossbuild/.vagrant .vagrant/ diff --git a/Makefile b/Makefile index c96e61cd0d..da05412358 100644 --- a/Makefile +++ b/Makefile @@ -206,6 +206,7 @@ util/build_version.cc: FORCE else mv -f $@-t $@; fi LIBOBJECTS = $(LIB_SOURCES:.cc=.o) +LIBOBJECTS += $(TOOL_SOURCES:.cc=.o) MOCKOBJECTS = $(MOCK_SOURCES:.cc=.o) GTEST = $(GTEST_DIR)/gtest/gtest-all.o @@ -592,14 +593,17 @@ CLEAN_FILES += unity.cc unity.cc: Makefile rm -f $@ $@-t for source_file in $(LIB_SOURCES); do \ - echo "#include <$$source_file>" >> $@-t; \ + echo "#include \"$$source_file\"" >> $@-t; \ done - echo 'int main(int argc, char** argv){ return 0; }' >> $@-t chmod a=r $@-t mv $@-t $@ -unity: unity.o - $(AM_LINK) +unity.a: unity.o + $(AM_V_AR)rm -f $@ + $(AM_V_at)$(AR) $(ARFLAGS) $@ unity.o + +rocksdb.h rocksdb.cc: build_tools/amalgamate.py Makefile $(LIB_SOURCES) unity.cc + build_tools/amalgamate.py -I. -i./include unity.cc -x include/rocksdb/c.h -H rocksdb.h -o rocksdb.cc clean: rm -f $(BENCHMARKS) $(TOOLS) $(TESTS) $(LIBRARY) $(SHARED) diff --git a/build_tools/amalgamate.py b/build_tools/amalgamate.py new file mode 100755 index 0000000000..548b1e8cec --- /dev/null +++ b/build_tools/amalgamate.py @@ -0,0 +1,110 @@ +#!/usr/bin/python + +# amalgamate.py creates an amalgamation from a unity build. +# It can be run with either Python 2 or 3. +# An amalgamation consists of a header that includes the contents of all public +# headers and a source file that includes the contents of all source files and +# private headers. +# +# This script works by starting with the unity build file and recursively expanding +# #include directives. If the #include is found in a public include directory, +# that header is expanded into the amalgamation header. +# +# A particular header is only expanded once, so this script will +# break if there are multiple inclusions of the same header that are expected to +# expand differently. Similarly, this type of code causes issues: +# +# #ifdef FOO +# #include "bar.h" +# // code here +# #else +# #include "bar.h" // oops, doesn't get expanded +# // different code here +# #endif +# +# The solution is to move the include out of the #ifdef. + +from __future__ import print_function + +import argparse +from os import path +import re +import sys + +include_re = re.compile('^[ \t]*#include[ \t]+"(.*)"[ \t]*$') +included = set() +excluded = set() + +def find_header(name, abs_path, include_paths): + samedir = path.join(path.dirname(abs_path), name) + if path.exists(samedir): + return samedir + for include_path in include_paths: + include_path = path.join(include_path, name) + if path.exists(include_path): + return include_path + return None + +def expand_include(include_path, f, abs_path, source_out, header_out, include_paths, public_include_paths): + if include_path in included: + return False + + included.add(include_path) + with open(include_path) as f: + print('#line 1 "{}"'.format(include_path), file=source_out) + process_file(f, include_path, source_out, header_out, include_paths, public_include_paths) + return True + +def process_file(f, abs_path, source_out, header_out, include_paths, public_include_paths): + for (line, text) in enumerate(f): + m = include_re.match(text) + if m: + filename = m.groups()[0] + # first check private headers + include_path = find_header(filename, abs_path, include_paths) + if include_path: + if include_path in excluded: + source_out.write(text) + expanded = False + else: + expanded = expand_include(include_path, f, abs_path, source_out, header_out, include_paths, public_include_paths) + else: + # now try public headers + include_path = find_header(filename, abs_path, public_include_paths) + if include_path: + # found public header + expanded = False + if include_path in excluded: + source_out.write(text) + else: + expand_include(include_path, f, abs_path, header_out, None, public_include_paths, []) + else: + sys.exit("unable to find {}, included in {} on line {}".format(filename, abs_path, line)) + + if expanded: + print('#line {} "{}"'.format(line+1, abs_path), file=source_out) + elif text != "#pragma once\n": + source_out.write(text) + +def main(): + parser = argparse.ArgumentParser(description="Transform a unity build into an amalgamation") + parser.add_argument("source", help="source file") + parser.add_argument("-I", action="append", dest="include_paths", help="include paths for private headers") + parser.add_argument("-i", action="append", dest="public_include_paths", help="include paths for public headers") + parser.add_argument("-x", action="append", dest="excluded", help="excluded header files") + parser.add_argument("-o", dest="source_out", help="output C++ file", required=True) + parser.add_argument("-H", dest="header_out", help="output C++ header file", required=True) + args = parser.parse_args() + + include_paths = list(map(path.abspath, args.include_paths or [])) + public_include_paths = list(map(path.abspath, args.public_include_paths or [])) + excluded.update(map(path.abspath, args.excluded or [])) + filename = args.source + abs_path = path.abspath(filename) + with open(filename) as f, open(args.source_out, 'w') as source_out, open(args.header_out, 'w') as header_out: + print('#line 1 "{}"'.format(filename), file=source_out) + print('#include "{}"'.format(header_out.name), file=source_out) + process_file(f, abs_path, source_out, header_out, include_paths, public_include_paths) + +if __name__ == "__main__": + main() diff --git a/src.mk b/src.mk index 9e86c01f74..4a5fd335f8 100644 --- a/src.mk +++ b/src.mk @@ -130,8 +130,6 @@ LIB_SOURCES = \ utilities/write_batch_with_index/write_batch_with_index.cc \ utilities/write_batch_with_index/write_batch_with_index_internal.cc \ util/event_logger.cc \ - util/ldb_cmd.cc \ - util/ldb_tool.cc \ util/log_buffer.cc \ util/logging.cc \ util/memenv.cc \ @@ -146,7 +144,6 @@ LIB_SOURCES = \ util/rate_limiter.cc \ util/skiplistrep.cc \ util/slice.cc \ - util/sst_dump_tool.cc \ util/statistics.cc \ util/status.cc \ util/status_message.cc \ @@ -162,6 +159,11 @@ LIB_SOURCES = \ util/xfunc.cc \ util/xxhash.cc \ +TOOL_SOURCES = \ + util/ldb_cmd.cc \ + util/ldb_tool.cc \ + util/sst_dump_tool.cc \ + MOCK_SOURCES = \ table/mock_table.cc \ util/mock_env.cc diff --git a/util/env_hdfs.cc b/util/env_hdfs.cc index 77055aafb4..30e796245f 100644 --- a/util/env_hdfs.cc +++ b/util/env_hdfs.cc @@ -3,6 +3,10 @@ // LICENSE file in the root directory of this source tree. An additional grant // of patent rights can be found in the PATENTS file in the same directory. // + +#include "rocksdb/env.h" +#include "hdfs/env_hdfs.h" + #ifdef USE_HDFS #ifndef ROCKSDB_HDFS_FILE_C #define ROCKSDB_HDFS_FILE_C @@ -13,9 +17,7 @@ #include #include #include -#include "rocksdb/env.h" #include "rocksdb/status.h" -#include "hdfs/env_hdfs.h" #define HDFS_EXISTS 0 #define HDFS_DOESNT_EXIST -1 @@ -598,8 +600,6 @@ Status HdfsEnv::NewLogger(const std::string& fname, #else // USE_HDFS // dummy placeholders used when HDFS is not available -#include "rocksdb/env.h" -#include "hdfs/env_hdfs.h" namespace rocksdb { Status HdfsEnv::NewSequentialFile(const std::string& fname, unique_ptr* result, diff --git a/util/xfunc.cc b/util/xfunc.cc index d80565247d..98de1c5944 100644 --- a/util/xfunc.cc +++ b/util/xfunc.cc @@ -10,7 +10,6 @@ #include "db/write_callback.h" #include "rocksdb/db.h" #include "rocksdb/options.h" -#include "rocksdb/utilities/optimistic_transaction.h" #include "rocksdb/utilities/optimistic_transaction_db.h" #include "rocksdb/write_batch.h" #include "util/xfunc.h"