dolysis/benchmark
3
0
Fork 0
mirror of https://github.com/google/benchmark.git synced 2024-11-28 15:34:33 +00:00
Code Issues Packages Projects Releases Wiki Activity
benchmark/test/CMakeLists.txt

310 lines
12 KiB
CMake
Raw Normal View History

Step one towards merging timer changes. This patch cleans up our use of generic macros and also merges changes in the build system. It adds options -DBENCHMARK_ENABLE_TESTING and -DBENCHMARK_ENABLE_SHARED.
2015-03-06 17:35:00 +00:00
# Enable the tests
annotate and export public symbols (#1321)
2022-02-14 10:48:53 +00:00
set(THREADS_PREFER_PTHREAD_FLAG ON)
Move pthread dependency to test CMakeLists.txt
2014-10-11 21:57:45 +00:00
find_package(Threads REQUIRED)
Fix #342: DoNotOptimize causes compile errors on older GCC versions. (#398) * Fix #342: DoNotOptimize causes compile errors on older GCC versions. DoNotOptimize uses inline assembly contraints to tell the compiler what the type of the input variable. The 'g' operand allows the input to be any register, memory, or immediate integer operand. However this constraint seems to be too weak on older GCC versions, and certain inputs will cause compile errors. This patch changes the constraint to 'X', which is documented as "any operand whatsoever is allowed". This appears to fix the issues with older GCC versions. However Clang doesn't seem to like "X", and will attempt to put the input into a register even when it can't/shouldn't; causing a compile error. However using "g" seems to work like "X" with GCC, so for this reason Clang still uses "g". * Try alternative formulation to placate GCC
2017-06-02 22:47:24 +00:00
include(CheckCXXCompilerFlag)
Move pthread dependency to test CMakeLists.txt
2014-10-11 21:57:45 +00:00
Enable <cassert> by removing -DNDEBUG when running the tests. In non-debug builds CMake automatically adds -DNDEBUG, this means that uses of `assert` in the tests are disabled for non-debug builds. Obviously we want these tests to run, regardless of configuration. This patch strips -DNDEBUG during non-debug builds and adds -UNDEBUG just to be sure.
2017-04-18 02:49:51 +00:00
# NOTE: Some tests use `<cassert>` to perform the test. Therefore we must
# strip -DNDEBUG from the default CMake flags in DEBUG mode.
string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE)
if( NOT uppercase_CMAKE_BUILD_TYPE STREQUAL "DEBUG" )
add_definitions( -UNDEBUG )
add_definitions(-DTEST_BENCHMARK_LIBRARY_HAS_NO_ASSERTIONS)
# Also remove /D NDEBUG to avoid MSVC warnings about conflicting defines.
foreach (flags_var_to_scrub
CMAKE_CXX_FLAGS_RELEASE
CMAKE_CXX_FLAGS_RELWITHDEBINFO
CMAKE_CXX_FLAGS_MINSIZEREL
CMAKE_C_FLAGS_RELEASE
CMAKE_C_FLAGS_RELWITHDEBINFO
CMAKE_C_FLAGS_MINSIZEREL)
string (REGEX REPLACE "(^| )[/-]D *NDEBUG($| )" " "
"${flags_var_to_scrub}" "${${flags_var_to_scrub}}")
endforeach()
endif()
Stop generating the export header and just check it in (#1435) * Stop generating the export header and just check it in * format the new header * support windows * format the header again * avoid depending on internal macro * ensure we define the right thing for windows static builds * support older cmake * and for tests
2022-07-20 19:34:39 +00:00
if (NOT BUILD_SHARED_LIBS)
add_definitions(-DBENCHMARK_STATIC_DEFINE)
endif()
Add tests to verify assembler output -- Fix DoNotOptimize. (#530) * Add tests to verify assembler output -- Fix DoNotOptimize. For things like `DoNotOptimize`, `ClobberMemory`, and even `KeepRunning()`, it is important exactly what assembly they generate. However, we currently have no way to test this. Instead it must be manually validated every time a change occurs -- including a change in compiler version. This patch attempts to introduce a way to test the assembled output automatically. It's mirrors how LLVM verifies compiler output, and it uses LLVM FileCheck to run the tests in a similar way. The tests function by generating the assembly for a test in CMake, and then using FileCheck to verify the // CHECK lines in the source file are found in the generated assembly. Currently, the tests only run on 64-bit x86 systems under GCC and Clang, and when FileCheck is found on the system. Additionally, this patch tries to improve the code gen from DoNotOptimize. This should probably be a separate change, but I needed something to test. * Disable assembly tests on Bazel for now * Link FIXME to github issue * Fix Tests on OS X * fix strip_asm.py to work on both Linux and OS X like targets
2018-03-23 22:10:47 +00:00
check_cxx_compiler_flag(-O3 BENCHMARK_HAS_O3_FLAG)
set(BENCHMARK_O3_FLAG "")
if (BENCHMARK_HAS_O3_FLAG)
set(BENCHMARK_O3_FLAG "-O3")
endif()
Add -DBENCHMARK_USE_LIBCXX:BOOL=<value> option. (#280) * Add BENCHMARK_USE_LIBCXX option. * Add comments * re-add zero null pointer warning
2016-08-29 18:43:30 +00:00
# NOTE: These flags must be added after find_package(Threads REQUIRED) otherwise
# they will break the configuration check.
if (DEFINED BENCHMARK_CXX_LINKER_FLAGS)
list(APPEND CMAKE_EXE_LINKER_FLAGS ${BENCHMARK_CXX_LINKER_FLAGS})
endif()
Add output_test.h to output_test_helper for VisualStudio editing.
2017-05-01 21:33:44 +00:00
add_library(output_test_helper STATIC output_test_helper.cc output_test.h)
annotate and export public symbols (#1321)
2022-02-14 10:48:53 +00:00
target_link_libraries(output_test_helper PRIVATE benchmark::benchmark)
Refactor output test runner into standalone module. (#277) * refactor * Move default substitutions into library * Move default substitutions to the *right* place in the library * Fix init order issues that caused test failures * improve diagnostics * add missing include * general cleanup * Address review comments
2016-08-28 19:24:16 +00:00
Step one towards merging timer changes. This patch cleans up our use of generic macros and also merges changes in the build system. It adds options -DBENCHMARK_ENABLE_TESTING and -DBENCHMARK_ENABLE_SHARED.
2015-03-06 17:35:00 +00:00
macro(compile_benchmark_test name)
add_executable(${name} "${name}.cc")
link to benchmark directly for tests that aren't link_main_test (#1576)
2023-03-14 13:35:32 +00:00
target_link_libraries(${name} benchmark::benchmark ${CMAKE_THREAD_LIBS_INIT})
Address warnings on NVIDIA nvc++ (#1573) * Address warnings on NVIDIA nvc++ Types of warnings were being generated: 1. Deprecated warnings - solved by defining the relevant BENCHMARK_* macros for nvc++ and adding pragma suppress on a couple of .cc files 2. Setup/TearDown const vs non-const partial override - solved by adding non-const version 3. Static but not referenced - added diagnostic suppress for that file * Modified manually to comply with CD/CI * Revert partial override * Suppress warnings from tests if compiler is NVHPC --------- Co-authored-by: dominic <510002+dmah42@users.noreply.github.com>
2023-03-13 12:34:12 +00:00
if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "NVHPC")
target_compile_options( ${name} PRIVATE --diag_suppress partial_override )
endif()
Step one towards merging timer changes. This patch cleans up our use of generic macros and also merges changes in the build system. It adds options -DBENCHMARK_ENABLE_TESTING and -DBENCHMARK_ENABLE_SHARED.
2015-03-06 17:35:00 +00:00
endmacro(compile_benchmark_test)
Add benchmark_main target. (#601) * Add benchmark_main library with support for Bazel. * fix newline at end of file * Add CMake support for benchmark_main. * Mention optionally using benchmark_main in README.
2018-05-25 10:18:58 +00:00
macro(compile_benchmark_test_with_main name)
add_executable(${name} "${name}.cc")
Alias CMake Targets. Fixes #921 (#926) * add Jordan Williams to both CONTRIBUTORS and AUTHORS * alias benchmark libraries Provide aliased CMake targets for the benchmark and benchmark_main targets. The alias targets are namespaced under benchmark::, which is the namespace when they are exported. I chose not to use either the PROJECT_NAME or the namespace variable but to hard-code the namespace. This is because the benchmark and benchmark_main targets are hard-coded by name themselves. Hard-coding the namespace is also much cleaner and easier to read. * link to aliased benchmark targets It is safer to link against namespaced targets because of how CMake interprets the double colon. Typo's will be caught by CMake at configuration-time instead of during compile / link time. * document the provided alias targets * add "Usage with CMake" section in documentation This section covers linking against the alias/import CMake targets and including them using either find_package or add_subdirectory. * format the "Usage with CMake" README section Added a newline after the "Usage with CMake" section header. Dropped the header level of the section by one to make it a direct subsection of the "Usage" section. Wrapped lines to be no longer than 80 characters in length.
2020-01-14 20:21:24 +00:00
target_link_libraries(${name} benchmark::benchmark_main)
Add benchmark_main target. (#601) * Add benchmark_main library with support for Bazel. * fix newline at end of file * Add CMake support for benchmark_main. * Mention optionally using benchmark_main in README.
2018-05-25 10:18:58 +00:00
endmacro(compile_benchmark_test_with_main)
Refactor output test runner into standalone module. (#277) * refactor * Move default substitutions into library * Move default substitutions to the *right* place in the library * Fix init order issues that caused test failures * improve diagnostics * add missing include * general cleanup * Address review comments
2016-08-28 19:24:16 +00:00
macro(compile_output_test name)
CMake: add headers to the list of source files. (#284) cmake does not require this, but IDEs such as VisualStudio or QtCreator need this to add the headers to the generated project.
2016-08-31 22:28:43 +00:00
add_executable(${name} "${name}.cc" output_test.h)
annotate and export public symbols (#1321)
2022-02-14 10:48:53 +00:00
target_link_libraries(${name} output_test_helper benchmark::benchmark_main
Add -DBENCHMARK_USE_LIBCXX:BOOL=<value> option. (#280) * Add BENCHMARK_USE_LIBCXX option. * Add comments * re-add zero null pointer warning
2016-08-29 18:43:30 +00:00
${BENCHMARK_CXX_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT})
Refactor output test runner into standalone module. (#277) * refactor * Move default substitutions into library * Move default substitutions to the *right* place in the library * Fix init order issues that caused test failures * improve diagnostics * add missing include * general cleanup * Address review comments
2016-08-28 19:24:16 +00:00
endmacro(compile_output_test)
Setup targets from each directory, rather than the top-level.
2014-04-23 07:47:07 +00:00
# Demonstration executable
Step one towards merging timer changes. This patch cleans up our use of generic macros and also merges changes in the build system. It adds options -DBENCHMARK_ENABLE_TESTING and -DBENCHMARK_ENABLE_SHARED.
2015-03-06 17:35:00 +00:00
compile_benchmark_test(benchmark_test)
Allow specifying number of iterations via --benchmark_min_time. (#1525) * Allow specifying number of iterations via --benchmark_min_time. Make the flag accept two new suffixes: + <integer>x: number of iterations + <floag>s: minimum number of seconds. This matches the internal benchmark API. * forgot to change flag type to string * used tagged union instead of std::variant, which is not available pre C++14 * update decl in benchmark_runner.h too * fixed errors * refactor * backward compat * typo * use IterationCount type * fixed test * const_cast * ret type * remove extra _ * debug * fixed bug from reporting that caused the new configs not to be included in the final report * addressed review comments * restore unnecessary changes in test/BUILD * fix float comparisons warnings from Release builds * clang format * fix visibility warning * remove misc file * removed backup files * addressed review comments * fix shorten in warning * use suffix for existing min_time specs to silent warnings in tests * fix leaks * use default min-time value in flag decl for consistency * removed double kMinTimeDecl from benchmark.h * dont need to preserve errno * add death tests * Add BENCHMARK_EXPORT to hopefully fix missing def errors * only enable death tests in debug mode because bm_check is no-op in release mode * guard death tests with additional support-check macros * Add additional guard to prevent running in Release mode --------- Co-authored-by: dominic <510002+dmah42@users.noreply.github.com>
2023-02-07 11:45:18 +00:00
add_test(NAME benchmark COMMAND benchmark_test --benchmark_min_time=0.01s)
Partially resolve google/benchmark#17 by fixing regular expression leak. This adds a unit test to validate the wrapper without running the entirety of benchmark_test.
2014-04-23 07:56:17 +00:00
[RFC] Adding API for setting/getting benchmark_filter flag? (#1254) * [RFC] Adding API for setting/getting benchmark_filter flag? This PR is more of a Request-for-comment - open to other ideas/suggestions as well. Details: This flag has different implementations(absl vs benchmark) and since the proposal to add absl as a dependency was rejected, it would be nice to have a reliable (and less hacky) way to access this flag internally. (Actually, reading it isn't much a problem but setting it is). Internally, we have a sizeable number users to use absl::SetFlags to set this flag. This will not work with benchmark-flags. Another motivation is that not all users use the command line flag. Some prefer to programmatically set this value. * fixed build errors * fix lints again * per discussion: add additional RunSpecifiedBenchmarks instead. * add tests * fix up tests * clarify comment * fix stray : in test * more assertion in test * add test file to test/CMakeLists.txt * more test * make test ISO C++ compliant * fix up BUILD file to pass the flag
2021-10-27 07:52:57 +00:00
compile_benchmark_test(spec_arg_test)
add_test(NAME spec_arg COMMAND spec_arg_test --benchmark_filter=BM_NotChosen)
Add option to get the verbosity provided by commandline flag -v (#1330) (#1397) * Add option to get the verbosity provided by commandline flag -v (#1330) * replace assert with test failure asserts are stripped out in non debug builds, and we run tests in non-debug CI bots. * clang-format my own tweak Co-authored-by: Dominic Hamon <dominichamon@users.noreply.github.com>
2022-05-17 16:59:36 +00:00
compile_benchmark_test(spec_arg_verbosity_test)
add_test(NAME spec_arg_verbosity COMMAND spec_arg_verbosity_test --v=42)
Add Setup/Teardown option on Benchmark. (#1269) * Add Setup/Teardown option on Benchmark. Motivations: - feature parity with our internal library. (which has ~718 callers) - more flexible than cordinating setup/teardown inside the benchmark routine. * change Setup/Teardown callback type to raw function pointers * add test file to cmake file * move b.Teardown() up * add const to param of Setup/Teardown callbacks * fix comment and add doc to user_guide * fix typo * fix doc, fix test and add bindings to python/benchmark.cc * fix binding again * remove explicit C cast - that was wrong * change policy to reference_internal * try removing the bindinds ... * clean up * add more tests with repetitions and fixtures * more comments * init setup/teardown callbacks to NULL * s/nullptr/NULL * removed unused var * change assertion on fixture_interaction::fixture_setup * move NULL init to .cc file
2021-11-17 16:51:55 +00:00
compile_benchmark_test(benchmark_setup_teardown_test)
add_test(NAME benchmark_setup_teardown COMMAND benchmark_setup_teardown_test)
Add filter test, remove re test, and googletest deps
2015-03-10 03:30:14 +00:00
compile_benchmark_test(filter_test)
redo filter test
2015-03-31 04:05:02 +00:00
macro(add_filter_test name filter expect)
Allow specifying number of iterations via --benchmark_min_time. (#1525) * Allow specifying number of iterations via --benchmark_min_time. Make the flag accept two new suffixes: + <integer>x: number of iterations + <floag>s: minimum number of seconds. This matches the internal benchmark API. * forgot to change flag type to string * used tagged union instead of std::variant, which is not available pre C++14 * update decl in benchmark_runner.h too * fixed errors * refactor * backward compat * typo * use IterationCount type * fixed test * const_cast * ret type * remove extra _ * debug * fixed bug from reporting that caused the new configs not to be included in the final report * addressed review comments * restore unnecessary changes in test/BUILD * fix float comparisons warnings from Release builds * clang format * fix visibility warning * remove misc file * removed backup files * addressed review comments * fix shorten in warning * use suffix for existing min_time specs to silent warnings in tests * fix leaks * use default min-time value in flag decl for consistency * removed double kMinTimeDecl from benchmark.h * dont need to preserve errno * add death tests * Add BENCHMARK_EXPORT to hopefully fix missing def errors * only enable death tests in debug mode because bm_check is no-op in release mode * guard death tests with additional support-check macros * Add additional guard to prevent running in Release mode --------- Co-authored-by: dominic <510002+dmah42@users.noreply.github.com>
2023-02-07 11:45:18 +00:00
add_test(NAME ${name} COMMAND filter_test --benchmark_min_time=0.01s --benchmark_filter=${filter} ${expect})
CMake: use full add_test(NAME <> COMMAND <>) signature (#901) * CTest must use proper paths to executables With the following syntax: ``` add_test(NAME <name> COMMAND <command> [<arg>...]) ``` if `<command>` specifies an executable target it will automatically be replaced by the location of the executable created at build time. This is important if a `<Configuration>_POSTFIX` like `_d` is used. * Fix typo in ctest invocation Instead of `-c` the uppercase `-C` must be used to select a config. But better use the longopt.
2019-11-05 19:46:13 +00:00
add_test(NAME ${name}_list_only COMMAND filter_test --benchmark_list_tests --benchmark_filter=${filter} ${expect})
redo filter test
2015-03-31 04:05:02 +00:00
endmacro(add_filter_test)
Allow specifying number of iterations via --benchmark_min_time. (#1525) * Allow specifying number of iterations via --benchmark_min_time. Make the flag accept two new suffixes: + <integer>x: number of iterations + <floag>s: minimum number of seconds. This matches the internal benchmark API. * forgot to change flag type to string * used tagged union instead of std::variant, which is not available pre C++14 * update decl in benchmark_runner.h too * fixed errors * refactor * backward compat * typo * use IterationCount type * fixed test * const_cast * ret type * remove extra _ * debug * fixed bug from reporting that caused the new configs not to be included in the final report * addressed review comments * restore unnecessary changes in test/BUILD * fix float comparisons warnings from Release builds * clang format * fix visibility warning * remove misc file * removed backup files * addressed review comments * fix shorten in warning * use suffix for existing min_time specs to silent warnings in tests * fix leaks * use default min-time value in flag decl for consistency * removed double kMinTimeDecl from benchmark.h * dont need to preserve errno * add death tests * Add BENCHMARK_EXPORT to hopefully fix missing def errors * only enable death tests in debug mode because bm_check is no-op in release mode * guard death tests with additional support-check macros * Add additional guard to prevent running in Release mode --------- Co-authored-by: dominic <510002+dmah42@users.noreply.github.com>
2023-02-07 11:45:18 +00:00
compile_benchmark_test(benchmark_min_time_flag_time_test)
add_test(NAME min_time_flag_time COMMAND benchmark_min_time_flag_time_test)
compile_benchmark_test(benchmark_min_time_flag_iters_test)
add_test(NAME min_time_flag_iters COMMAND benchmark_min_time_flag_iters_test)
redo filter test
2015-03-31 04:05:02 +00:00
add_filter_test(filter_simple "Foo" 3)
Issue 571: Allow support for negative regex filtering (#576) * Allow support for negative regex filtering This patch allows one to apply a negation to the entire regex filter by appending it with a '-' character, much in the same style as GoogleTest uses. * Address issues in PR * Add unit tests for negative filtering
2018-04-26 09:56:06 +00:00
add_filter_test(filter_simple_negative "-Foo" 2)
redo filter test
2015-03-31 04:05:02 +00:00
add_filter_test(filter_suffix "BM_.*" 4)
Issue 571: Allow support for negative regex filtering (#576) * Allow support for negative regex filtering This patch allows one to apply a negation to the entire regex filter by appending it with a '-' character, much in the same style as GoogleTest uses. * Address issues in PR * Add unit tests for negative filtering
2018-04-26 09:56:06 +00:00
add_filter_test(filter_suffix_negative "-BM_.*" 1)
redo filter test
2015-03-31 04:05:02 +00:00
add_filter_test(filter_regex_all ".*" 5)
Issue 571: Allow support for negative regex filtering (#576) * Allow support for negative regex filtering This patch allows one to apply a negation to the entire regex filter by appending it with a '-' character, much in the same style as GoogleTest uses. * Address issues in PR * Add unit tests for negative filtering
2018-04-26 09:56:06 +00:00
add_filter_test(filter_regex_all_negative "-.*" 0)
redo filter test
2015-03-31 04:05:02 +00:00
add_filter_test(filter_regex_blank "" 5)
Issue 571: Allow support for negative regex filtering (#576) * Allow support for negative regex filtering This patch allows one to apply a negation to the entire regex filter by appending it with a '-' character, much in the same style as GoogleTest uses. * Address issues in PR * Add unit tests for negative filtering
2018-04-26 09:56:06 +00:00
add_filter_test(filter_regex_blank_negative "-" 0)
redo filter test
2015-03-31 04:05:02 +00:00
add_filter_test(filter_regex_none "monkey" 0)
Issue 571: Allow support for negative regex filtering (#576) * Allow support for negative regex filtering This patch allows one to apply a negation to the entire regex filter by appending it with a '-' character, much in the same style as GoogleTest uses. * Address issues in PR * Add unit tests for negative filtering
2018-04-26 09:56:06 +00:00
add_filter_test(filter_regex_none_negative "-monkey" 5)
redo filter test
2015-03-31 04:05:02 +00:00
add_filter_test(filter_regex_wildcard ".*Foo.*" 3)
Issue 571: Allow support for negative regex filtering (#576) * Allow support for negative regex filtering This patch allows one to apply a negation to the entire regex filter by appending it with a '-' character, much in the same style as GoogleTest uses. * Address issues in PR * Add unit tests for negative filtering
2018-04-26 09:56:06 +00:00
add_filter_test(filter_regex_wildcard_negative "-.*Foo.*" 2)
redo filter test
2015-03-31 04:05:02 +00:00
add_filter_test(filter_regex_begin "^BM_.*" 4)
Issue 571: Allow support for negative regex filtering (#576) * Allow support for negative regex filtering This patch allows one to apply a negation to the entire regex filter by appending it with a '-' character, much in the same style as GoogleTest uses. * Address issues in PR * Add unit tests for negative filtering
2018-04-26 09:56:06 +00:00
add_filter_test(filter_regex_begin_negative "-^BM_.*" 1)
redo filter test
2015-03-31 04:05:02 +00:00
add_filter_test(filter_regex_begin2 "^N" 1)
Issue 571: Allow support for negative regex filtering (#576) * Allow support for negative regex filtering This patch allows one to apply a negation to the entire regex filter by appending it with a '-' character, much in the same style as GoogleTest uses. * Address issues in PR * Add unit tests for negative filtering
2018-04-26 09:56:06 +00:00
add_filter_test(filter_regex_begin2_negative "-^N" 4)
redo filter test
2015-03-31 04:05:02 +00:00
add_filter_test(filter_regex_end ".*Ba$" 1)
Issue 571: Allow support for negative regex filtering (#576) * Allow support for negative regex filtering This patch allows one to apply a negation to the entire regex filter by appending it with a '-' character, much in the same style as GoogleTest uses. * Address issues in PR * Add unit tests for negative filtering
2018-04-26 09:56:06 +00:00
add_filter_test(filter_regex_end_negative "-.*Ba$" 4)
Adopt new benchmark timing internals. This patch adopts a new internal structure for how timings are performed. Currently every iteration of a benchmark checks to see if it has been running for an appropriate amount of time. Checking the clock introduces noise into the timings and this can cause inconsistent output from each benchmark. Now every iteration of a benchmark only checks an iteration count to see if it should stop running. The iteration count is determined before hand by testing the benchmark on a series of increasing iteration counts until a suitable count is found. This increases the amount of time it takes to run the actual benchmarks but it also greatly increases the accuracy of the results. This patch introduces some breaking changes. The notable breaking changes are: 1. Benchmarks run on multiple threads no generate a report per thread. Instead only a single report is generated. 2. ::benchmark::UseRealTime() was removed and replaced with State::UseRealTime().
2015-03-12 22:03:33 +00:00
Change the available benchmark options
2015-03-27 03:37:26 +00:00
compile_benchmark_test(options_test)
Allow specifying number of iterations via --benchmark_min_time. (#1525) * Allow specifying number of iterations via --benchmark_min_time. Make the flag accept two new suffixes: + <integer>x: number of iterations + <floag>s: minimum number of seconds. This matches the internal benchmark API. * forgot to change flag type to string * used tagged union instead of std::variant, which is not available pre C++14 * update decl in benchmark_runner.h too * fixed errors * refactor * backward compat * typo * use IterationCount type * fixed test * const_cast * ret type * remove extra _ * debug * fixed bug from reporting that caused the new configs not to be included in the final report * addressed review comments * restore unnecessary changes in test/BUILD * fix float comparisons warnings from Release builds * clang format * fix visibility warning * remove misc file * removed backup files * addressed review comments * fix shorten in warning * use suffix for existing min_time specs to silent warnings in tests * fix leaks * use default min-time value in flag decl for consistency * removed double kMinTimeDecl from benchmark.h * dont need to preserve errno * add death tests * Add BENCHMARK_EXPORT to hopefully fix missing def errors * only enable death tests in debug mode because bm_check is no-op in release mode * guard death tests with additional support-check macros * Add additional guard to prevent running in Release mode --------- Co-authored-by: dominic <510002+dmah42@users.noreply.github.com>
2023-02-07 11:45:18 +00:00
add_test(NAME options_benchmarks COMMAND options_test --benchmark_min_time=0.01s)
Change the available benchmark options
2015-03-27 03:37:26 +00:00
Adopt new benchmark timing internals. This patch adopts a new internal structure for how timings are performed. Currently every iteration of a benchmark checks to see if it has been running for an appropriate amount of time. Checking the clock introduces noise into the timings and this can cause inconsistent output from each benchmark. Now every iteration of a benchmark only checks an iteration count to see if it should stop running. The iteration count is determined before hand by testing the benchmark on a series of increasing iteration counts until a suitable count is found. This increases the amount of time it takes to run the actual benchmarks but it also greatly increases the accuracy of the results. This patch introduces some breaking changes. The notable breaking changes are: 1. Benchmarks run on multiple threads no generate a report per thread. Instead only a single report is generated. 2. ::benchmark::UseRealTime() was removed and replaced with State::UseRealTime().
2015-03-12 22:03:33 +00:00
compile_benchmark_test(basic_test)
Allow specifying number of iterations via --benchmark_min_time. (#1525) * Allow specifying number of iterations via --benchmark_min_time. Make the flag accept two new suffixes: + <integer>x: number of iterations + <floag>s: minimum number of seconds. This matches the internal benchmark API. * forgot to change flag type to string * used tagged union instead of std::variant, which is not available pre C++14 * update decl in benchmark_runner.h too * fixed errors * refactor * backward compat * typo * use IterationCount type * fixed test * const_cast * ret type * remove extra _ * debug * fixed bug from reporting that caused the new configs not to be included in the final report * addressed review comments * restore unnecessary changes in test/BUILD * fix float comparisons warnings from Release builds * clang format * fix visibility warning * remove misc file * removed backup files * addressed review comments * fix shorten in warning * use suffix for existing min_time specs to silent warnings in tests * fix leaks * use default min-time value in flag decl for consistency * removed double kMinTimeDecl from benchmark.h * dont need to preserve errno * add death tests * Add BENCHMARK_EXPORT to hopefully fix missing def errors * only enable death tests in debug mode because bm_check is no-op in release mode * guard death tests with additional support-check macros * Add additional guard to prevent running in Release mode --------- Co-authored-by: dominic <510002+dmah42@users.noreply.github.com>
2023-02-07 11:45:18 +00:00
add_test(NAME basic_benchmark COMMAND basic_test --benchmark_min_time=0.01s)
add C++03 test and update README
2015-03-18 20:34:43 +00:00
Ensure that we print repetition count even when it was specified via flag `--benchmark_repetitions=`
2021-06-02 09:34:00 +00:00
compile_output_test(repetitions_test)
Allow specifying number of iterations via --benchmark_min_time. (#1525) * Allow specifying number of iterations via --benchmark_min_time. Make the flag accept two new suffixes: + <integer>x: number of iterations + <floag>s: minimum number of seconds. This matches the internal benchmark API. * forgot to change flag type to string * used tagged union instead of std::variant, which is not available pre C++14 * update decl in benchmark_runner.h too * fixed errors * refactor * backward compat * typo * use IterationCount type * fixed test * const_cast * ret type * remove extra _ * debug * fixed bug from reporting that caused the new configs not to be included in the final report * addressed review comments * restore unnecessary changes in test/BUILD * fix float comparisons warnings from Release builds * clang format * fix visibility warning * remove misc file * removed backup files * addressed review comments * fix shorten in warning * use suffix for existing min_time specs to silent warnings in tests * fix leaks * use default min-time value in flag decl for consistency * removed double kMinTimeDecl from benchmark.h * dont need to preserve errno * add death tests * Add BENCHMARK_EXPORT to hopefully fix missing def errors * only enable death tests in debug mode because bm_check is no-op in release mode * guard death tests with additional support-check macros * Add additional guard to prevent running in Release mode --------- Co-authored-by: dominic <510002+dmah42@users.noreply.github.com>
2023-02-07 11:45:18 +00:00
add_test(NAME repetitions_benchmark COMMAND repetitions_test --benchmark_min_time=0.01s --benchmark_repetitions=3)
Ensure that we print repetition count even when it was specified via flag `--benchmark_repetitions=`
2021-06-02 09:34:00 +00:00
Add SkipWithError(...) function.
2016-05-24 01:24:56 +00:00
compile_benchmark_test(diagnostics_test)
Allow specifying number of iterations via --benchmark_min_time. (#1525) * Allow specifying number of iterations via --benchmark_min_time. Make the flag accept two new suffixes: + <integer>x: number of iterations + <floag>s: minimum number of seconds. This matches the internal benchmark API. * forgot to change flag type to string * used tagged union instead of std::variant, which is not available pre C++14 * update decl in benchmark_runner.h too * fixed errors * refactor * backward compat * typo * use IterationCount type * fixed test * const_cast * ret type * remove extra _ * debug * fixed bug from reporting that caused the new configs not to be included in the final report * addressed review comments * restore unnecessary changes in test/BUILD * fix float comparisons warnings from Release builds * clang format * fix visibility warning * remove misc file * removed backup files * addressed review comments * fix shorten in warning * use suffix for existing min_time specs to silent warnings in tests * fix leaks * use default min-time value in flag decl for consistency * removed double kMinTimeDecl from benchmark.h * dont need to preserve errno * add death tests * Add BENCHMARK_EXPORT to hopefully fix missing def errors * only enable death tests in debug mode because bm_check is no-op in release mode * guard death tests with additional support-check macros * Add additional guard to prevent running in Release mode --------- Co-authored-by: dominic <510002+dmah42@users.noreply.github.com>
2023-02-07 11:45:18 +00:00
add_test(NAME diagnostics_test COMMAND diagnostics_test --benchmark_min_time=0.01s)
Add SkipWithError(...) function.
2016-05-24 01:24:56 +00:00
compile_benchmark_test(skip_with_error_test)
Allow specifying number of iterations via --benchmark_min_time. (#1525) * Allow specifying number of iterations via --benchmark_min_time. Make the flag accept two new suffixes: + <integer>x: number of iterations + <floag>s: minimum number of seconds. This matches the internal benchmark API. * forgot to change flag type to string * used tagged union instead of std::variant, which is not available pre C++14 * update decl in benchmark_runner.h too * fixed errors * refactor * backward compat * typo * use IterationCount type * fixed test * const_cast * ret type * remove extra _ * debug * fixed bug from reporting that caused the new configs not to be included in the final report * addressed review comments * restore unnecessary changes in test/BUILD * fix float comparisons warnings from Release builds * clang format * fix visibility warning * remove misc file * removed backup files * addressed review comments * fix shorten in warning * use suffix for existing min_time specs to silent warnings in tests * fix leaks * use default min-time value in flag decl for consistency * removed double kMinTimeDecl from benchmark.h * dont need to preserve errno * add death tests * Add BENCHMARK_EXPORT to hopefully fix missing def errors * only enable death tests in debug mode because bm_check is no-op in release mode * guard death tests with additional support-check macros * Add additional guard to prevent running in Release mode --------- Co-authored-by: dominic <510002+dmah42@users.noreply.github.com>
2023-02-07 11:45:18 +00:00
add_test(NAME skip_with_error_test COMMAND skip_with_error_test --benchmark_min_time=0.01s)
Add SkipWithError(...) function.
2016-05-24 01:24:56 +00:00
Force DoNotOptimize operand to memory for both gcc and clang
2016-05-25 06:31:20 +00:00
compile_benchmark_test(donotoptimize_test)
Fix passing non-const lvalue refs to DoNotOptimize (#1622)
2023-07-05 17:05:08 +00:00
# Enable errors for deprecated deprecations (DoNotOptimize(Tp const& value)).
check_cxx_compiler_flag(-Werror=deprecated-declarations BENCHMARK_HAS_DEPRECATED_DECLARATIONS_FLAG)
if (BENCHMARK_HAS_DEPRECATED_DECLARATIONS_FLAG)
target_compile_options (donotoptimize_test PRIVATE "-Werror=deprecated-declarations")
endif()
Fix #342: DoNotOptimize causes compile errors on older GCC versions. (#398) * Fix #342: DoNotOptimize causes compile errors on older GCC versions. DoNotOptimize uses inline assembly contraints to tell the compiler what the type of the input variable. The 'g' operand allows the input to be any register, memory, or immediate integer operand. However this constraint seems to be too weak on older GCC versions, and certain inputs will cause compile errors. This patch changes the constraint to 'X', which is documented as "any operand whatsoever is allowed". This appears to fix the issues with older GCC versions. However Clang doesn't seem to like "X", and will attempt to put the input into a register even when it can't/shouldn't; causing a compile error. However using "g" seems to work like "X" with GCC, so for this reason Clang still uses "g". * Try alternative formulation to placate GCC
2017-06-02 22:47:24 +00:00
# Some of the issues with DoNotOptimize only occur when optimization is enabled
check_cxx_compiler_flag(-O3 BENCHMARK_HAS_O3_FLAG)
if (BENCHMARK_HAS_O3_FLAG)
set_target_properties(donotoptimize_test PROPERTIES COMPILE_FLAGS "-O3")
endif()
Allow specifying number of iterations via --benchmark_min_time. (#1525) * Allow specifying number of iterations via --benchmark_min_time. Make the flag accept two new suffixes: + <integer>x: number of iterations + <floag>s: minimum number of seconds. This matches the internal benchmark API. * forgot to change flag type to string * used tagged union instead of std::variant, which is not available pre C++14 * update decl in benchmark_runner.h too * fixed errors * refactor * backward compat * typo * use IterationCount type * fixed test * const_cast * ret type * remove extra _ * debug * fixed bug from reporting that caused the new configs not to be included in the final report * addressed review comments * restore unnecessary changes in test/BUILD * fix float comparisons warnings from Release builds * clang format * fix visibility warning * remove misc file * removed backup files * addressed review comments * fix shorten in warning * use suffix for existing min_time specs to silent warnings in tests * fix leaks * use default min-time value in flag decl for consistency * removed double kMinTimeDecl from benchmark.h * dont need to preserve errno * add death tests * Add BENCHMARK_EXPORT to hopefully fix missing def errors * only enable death tests in debug mode because bm_check is no-op in release mode * guard death tests with additional support-check macros * Add additional guard to prevent running in Release mode --------- Co-authored-by: dominic <510002+dmah42@users.noreply.github.com>
2023-02-07 11:45:18 +00:00
add_test(NAME donotoptimize_test COMMAND donotoptimize_test --benchmark_min_time=0.01s)
Force DoNotOptimize operand to memory for both gcc and clang
2016-05-25 06:31:20 +00:00
add ability to use benchmark filters
2015-04-06 21:00:06 +00:00
compile_benchmark_test(fixture_test)
Allow specifying number of iterations via --benchmark_min_time. (#1525) * Allow specifying number of iterations via --benchmark_min_time. Make the flag accept two new suffixes: + <integer>x: number of iterations + <floag>s: minimum number of seconds. This matches the internal benchmark API. * forgot to change flag type to string * used tagged union instead of std::variant, which is not available pre C++14 * update decl in benchmark_runner.h too * fixed errors * refactor * backward compat * typo * use IterationCount type * fixed test * const_cast * ret type * remove extra _ * debug * fixed bug from reporting that caused the new configs not to be included in the final report * addressed review comments * restore unnecessary changes in test/BUILD * fix float comparisons warnings from Release builds * clang format * fix visibility warning * remove misc file * removed backup files * addressed review comments * fix shorten in warning * use suffix for existing min_time specs to silent warnings in tests * fix leaks * use default min-time value in flag decl for consistency * removed double kMinTimeDecl from benchmark.h * dont need to preserve errno * add death tests * Add BENCHMARK_EXPORT to hopefully fix missing def errors * only enable death tests in debug mode because bm_check is no-op in release mode * guard death tests with additional support-check macros * Add additional guard to prevent running in Release mode --------- Co-authored-by: dominic <510002+dmah42@users.noreply.github.com>
2023-02-07 11:45:18 +00:00
add_test(NAME fixture_test COMMAND fixture_test --benchmark_min_time=0.01s)
add ability to use benchmark filters
2015-04-06 21:00:06 +00:00
Add `RegisterBenchmark(name, func, args...)` for creating/registering benchmarks. (#259) * Add RegisterBenchmark * fix test inputs * fix UB caused by unitialized value * Add RegisterBenchmark * fix test inputs * fix UB caused by unitialized value * Work around GCC 4.6/4.7/4.8 bug
2016-08-02 23:22:46 +00:00
compile_benchmark_test(register_benchmark_test)
Allow specifying number of iterations via --benchmark_min_time. (#1525) * Allow specifying number of iterations via --benchmark_min_time. Make the flag accept two new suffixes: + <integer>x: number of iterations + <floag>s: minimum number of seconds. This matches the internal benchmark API. * forgot to change flag type to string * used tagged union instead of std::variant, which is not available pre C++14 * update decl in benchmark_runner.h too * fixed errors * refactor * backward compat * typo * use IterationCount type * fixed test * const_cast * ret type * remove extra _ * debug * fixed bug from reporting that caused the new configs not to be included in the final report * addressed review comments * restore unnecessary changes in test/BUILD * fix float comparisons warnings from Release builds * clang format * fix visibility warning * remove misc file * removed backup files * addressed review comments * fix shorten in warning * use suffix for existing min_time specs to silent warnings in tests * fix leaks * use default min-time value in flag decl for consistency * removed double kMinTimeDecl from benchmark.h * dont need to preserve errno * add death tests * Add BENCHMARK_EXPORT to hopefully fix missing def errors * only enable death tests in debug mode because bm_check is no-op in release mode * guard death tests with additional support-check macros * Add additional guard to prevent running in Release mode --------- Co-authored-by: dominic <510002+dmah42@users.noreply.github.com>
2023-02-07 11:45:18 +00:00
add_test(NAME register_benchmark_test COMMAND register_benchmark_test --benchmark_min_time=0.01s)
Add `RegisterBenchmark(name, func, args...)` for creating/registering benchmarks. (#259) * Add RegisterBenchmark * fix test inputs * fix UB caused by unitialized value * Add RegisterBenchmark * fix test inputs * fix UB caused by unitialized value * Work around GCC 4.6/4.7/4.8 bug
2016-08-02 23:22:46 +00:00
Pass const state to Fixture::SetUp. Add map_test.
2016-02-15 11:04:19 +00:00
compile_benchmark_test(map_test)
Allow specifying number of iterations via --benchmark_min_time. (#1525) * Allow specifying number of iterations via --benchmark_min_time. Make the flag accept two new suffixes: + <integer>x: number of iterations + <floag>s: minimum number of seconds. This matches the internal benchmark API. * forgot to change flag type to string * used tagged union instead of std::variant, which is not available pre C++14 * update decl in benchmark_runner.h too * fixed errors * refactor * backward compat * typo * use IterationCount type * fixed test * const_cast * ret type * remove extra _ * debug * fixed bug from reporting that caused the new configs not to be included in the final report * addressed review comments * restore unnecessary changes in test/BUILD * fix float comparisons warnings from Release builds * clang format * fix visibility warning * remove misc file * removed backup files * addressed review comments * fix shorten in warning * use suffix for existing min_time specs to silent warnings in tests * fix leaks * use default min-time value in flag decl for consistency * removed double kMinTimeDecl from benchmark.h * dont need to preserve errno * add death tests * Add BENCHMARK_EXPORT to hopefully fix missing def errors * only enable death tests in debug mode because bm_check is no-op in release mode * guard death tests with additional support-check macros * Add additional guard to prevent running in Release mode --------- Co-authored-by: dominic <510002+dmah42@users.noreply.github.com>
2023-02-07 11:45:18 +00:00
add_test(NAME map_test COMMAND map_test --benchmark_min_time=0.01s)
Pass const state to Fixture::SetUp. Add map_test.
2016-02-15 11:04:19 +00:00
Support multiple ranges in the benchmark (#257) * Support multiple ranges in the benchmark google-benchmark library allows to provide up to two ranges to the benchmark method (range_x and range_y). However, in many cases it's not sufficient. The patch introduces multi-range features, so user can easily define multiple ranges by passing a vector of integers, and access values through the method range(i). * Remove redundant API Functions State::range_x() and State::range_y() have been removed. They should be replaced by State::range(0) and State::range(1). Functions Benchmark::ArgPair() and Benchmark::RangePair() have been removed. They should be replaced by Benchmark::Args() and Benchmark::Ranges().
2016-08-04 19:30:14 +00:00
compile_benchmark_test(multiple_ranges_test)
Allow specifying number of iterations via --benchmark_min_time. (#1525) * Allow specifying number of iterations via --benchmark_min_time. Make the flag accept two new suffixes: + <integer>x: number of iterations + <floag>s: minimum number of seconds. This matches the internal benchmark API. * forgot to change flag type to string * used tagged union instead of std::variant, which is not available pre C++14 * update decl in benchmark_runner.h too * fixed errors * refactor * backward compat * typo * use IterationCount type * fixed test * const_cast * ret type * remove extra _ * debug * fixed bug from reporting that caused the new configs not to be included in the final report * addressed review comments * restore unnecessary changes in test/BUILD * fix float comparisons warnings from Release builds * clang format * fix visibility warning * remove misc file * removed backup files * addressed review comments * fix shorten in warning * use suffix for existing min_time specs to silent warnings in tests * fix leaks * use default min-time value in flag decl for consistency * removed double kMinTimeDecl from benchmark.h * dont need to preserve errno * add death tests * Add BENCHMARK_EXPORT to hopefully fix missing def errors * only enable death tests in debug mode because bm_check is no-op in release mode * guard death tests with additional support-check macros * Add additional guard to prevent running in Release mode --------- Co-authored-by: dominic <510002+dmah42@users.noreply.github.com>
2023-02-07 11:45:18 +00:00
add_test(NAME multiple_ranges_test COMMAND multiple_ranges_test --benchmark_min_time=0.01s)
Support multiple ranges in the benchmark (#257) * Support multiple ranges in the benchmark google-benchmark library allows to provide up to two ranges to the benchmark method (range_x and range_y). However, in many cases it's not sufficient. The patch introduces multi-range features, so user can easily define multiple ranges by passing a vector of integers, and access values through the method range(i). * Remove redundant API Functions State::range_x() and State::range_y() have been removed. They should be replaced by State::range(0) and State::range(1). Functions Benchmark::ArgPair() and Benchmark::RangePair() have been removed. They should be replaced by Benchmark::Args() and Benchmark::Ranges().
2016-08-04 19:30:14 +00:00
Add CartesianProduct with associated test (#1029) * Add CartesianProduct with associated test * Use CartesianProduct in Ranges to avoid code duplication * Add new cartesian_product_test to CMakeLists.txt * Update AUTHORS & CONTRIBUTORS * Rename CartesianProduct to ArgsProduct * Rename test & fixture accordingly * Add example for ArgsProduct to README
2020-08-25 12:47:44 +00:00
compile_benchmark_test(args_product_test)
Allow specifying number of iterations via --benchmark_min_time. (#1525) * Allow specifying number of iterations via --benchmark_min_time. Make the flag accept two new suffixes: + <integer>x: number of iterations + <floag>s: minimum number of seconds. This matches the internal benchmark API. * forgot to change flag type to string * used tagged union instead of std::variant, which is not available pre C++14 * update decl in benchmark_runner.h too * fixed errors * refactor * backward compat * typo * use IterationCount type * fixed test * const_cast * ret type * remove extra _ * debug * fixed bug from reporting that caused the new configs not to be included in the final report * addressed review comments * restore unnecessary changes in test/BUILD * fix float comparisons warnings from Release builds * clang format * fix visibility warning * remove misc file * removed backup files * addressed review comments * fix shorten in warning * use suffix for existing min_time specs to silent warnings in tests * fix leaks * use default min-time value in flag decl for consistency * removed double kMinTimeDecl from benchmark.h * dont need to preserve errno * add death tests * Add BENCHMARK_EXPORT to hopefully fix missing def errors * only enable death tests in debug mode because bm_check is no-op in release mode * guard death tests with additional support-check macros * Add additional guard to prevent running in Release mode --------- Co-authored-by: dominic <510002+dmah42@users.noreply.github.com>
2023-02-07 11:45:18 +00:00
add_test(NAME args_product_test COMMAND args_product_test --benchmark_min_time=0.01s)
Add CartesianProduct with associated test (#1029) * Add CartesianProduct with associated test * Use CartesianProduct in Ranges to avoid code duplication * Add new cartesian_product_test to CMakeLists.txt * Update AUTHORS & CONTRIBUTORS * Rename CartesianProduct to ArgsProduct * Rename test & fixture accordingly * Add example for ArgsProduct to README
2020-08-25 12:47:44 +00:00
Add benchmark_main target. (#601) * Add benchmark_main library with support for Bazel. * fix newline at end of file * Add CMake support for benchmark_main. * Mention optionally using benchmark_main in README.
2018-05-25 10:18:58 +00:00
compile_benchmark_test_with_main(link_main_test)
Allow specifying number of iterations via --benchmark_min_time. (#1525) * Allow specifying number of iterations via --benchmark_min_time. Make the flag accept two new suffixes: + <integer>x: number of iterations + <floag>s: minimum number of seconds. This matches the internal benchmark API. * forgot to change flag type to string * used tagged union instead of std::variant, which is not available pre C++14 * update decl in benchmark_runner.h too * fixed errors * refactor * backward compat * typo * use IterationCount type * fixed test * const_cast * ret type * remove extra _ * debug * fixed bug from reporting that caused the new configs not to be included in the final report * addressed review comments * restore unnecessary changes in test/BUILD * fix float comparisons warnings from Release builds * clang format * fix visibility warning * remove misc file * removed backup files * addressed review comments * fix shorten in warning * use suffix for existing min_time specs to silent warnings in tests * fix leaks * use default min-time value in flag decl for consistency * removed double kMinTimeDecl from benchmark.h * dont need to preserve errno * add death tests * Add BENCHMARK_EXPORT to hopefully fix missing def errors * only enable death tests in debug mode because bm_check is no-op in release mode * guard death tests with additional support-check macros * Add additional guard to prevent running in Release mode --------- Co-authored-by: dominic <510002+dmah42@users.noreply.github.com>
2023-02-07 11:45:18 +00:00
add_test(NAME link_main_test COMMAND link_main_test --benchmark_min_time=0.01s)
Add benchmark_main target. (#601) * Add benchmark_main library with support for Bazel. * fix newline at end of file * Add CMake support for benchmark_main. * Mention optionally using benchmark_main in README.
2018-05-25 10:18:58 +00:00
Refactor output test runner into standalone module. (#277) * refactor * Move default substitutions into library * Move default substitutions to the *right* place in the library * Fix init order issues that caused test failures * improve diagnostics * add missing include * general cleanup * Address review comments
2016-08-28 19:24:16 +00:00
compile_output_test(reporter_output_test)
Allow specifying number of iterations via --benchmark_min_time. (#1525) * Allow specifying number of iterations via --benchmark_min_time. Make the flag accept two new suffixes: + <integer>x: number of iterations + <floag>s: minimum number of seconds. This matches the internal benchmark API. * forgot to change flag type to string * used tagged union instead of std::variant, which is not available pre C++14 * update decl in benchmark_runner.h too * fixed errors * refactor * backward compat * typo * use IterationCount type * fixed test * const_cast * ret type * remove extra _ * debug * fixed bug from reporting that caused the new configs not to be included in the final report * addressed review comments * restore unnecessary changes in test/BUILD * fix float comparisons warnings from Release builds * clang format * fix visibility warning * remove misc file * removed backup files * addressed review comments * fix shorten in warning * use suffix for existing min_time specs to silent warnings in tests * fix leaks * use default min-time value in flag decl for consistency * removed double kMinTimeDecl from benchmark.h * dont need to preserve errno * add death tests * Add BENCHMARK_EXPORT to hopefully fix missing def errors * only enable death tests in debug mode because bm_check is no-op in release mode * guard death tests with additional support-check macros * Add additional guard to prevent running in Release mode --------- Co-authored-by: dominic <510002+dmah42@users.noreply.github.com>
2023-02-07 11:45:18 +00:00
add_test(NAME reporter_output_test COMMAND reporter_output_test --benchmark_min_time=0.01s)
Add macros for create benchmark with templated fixture (#451) * Add macros for create benchmark with templated fixture * Add info about templated fixtures to README.md * Add tests for templated fixtures
2017-10-09 19:10:37 +00:00
compile_output_test(templated_fixture_test)
Allow specifying number of iterations via --benchmark_min_time. (#1525) * Allow specifying number of iterations via --benchmark_min_time. Make the flag accept two new suffixes: + <integer>x: number of iterations + <floag>s: minimum number of seconds. This matches the internal benchmark API. * forgot to change flag type to string * used tagged union instead of std::variant, which is not available pre C++14 * update decl in benchmark_runner.h too * fixed errors * refactor * backward compat * typo * use IterationCount type * fixed test * const_cast * ret type * remove extra _ * debug * fixed bug from reporting that caused the new configs not to be included in the final report * addressed review comments * restore unnecessary changes in test/BUILD * fix float comparisons warnings from Release builds * clang format * fix visibility warning * remove misc file * removed backup files * addressed review comments * fix shorten in warning * use suffix for existing min_time specs to silent warnings in tests * fix leaks * use default min-time value in flag decl for consistency * removed double kMinTimeDecl from benchmark.h * dont need to preserve errno * add death tests * Add BENCHMARK_EXPORT to hopefully fix missing def errors * only enable death tests in debug mode because bm_check is no-op in release mode * guard death tests with additional support-check macros * Add additional guard to prevent running in Release mode --------- Co-authored-by: dominic <510002+dmah42@users.noreply.github.com>
2023-02-07 11:45:18 +00:00
add_test(NAME templated_fixture_test COMMAND templated_fixture_test --benchmark_min_time=0.01s)
Only compile and run the C++03 test when -std=c++03 is supported. Fixes #164
2016-05-24 02:44:10 +00:00
User counter tests: first version.
2017-04-27 18:25:20 +00:00
compile_output_test(user_counters_test)
Allow specifying number of iterations via --benchmark_min_time. (#1525) * Allow specifying number of iterations via --benchmark_min_time. Make the flag accept two new suffixes: + <integer>x: number of iterations + <floag>s: minimum number of seconds. This matches the internal benchmark API. * forgot to change flag type to string * used tagged union instead of std::variant, which is not available pre C++14 * update decl in benchmark_runner.h too * fixed errors * refactor * backward compat * typo * use IterationCount type * fixed test * const_cast * ret type * remove extra _ * debug * fixed bug from reporting that caused the new configs not to be included in the final report * addressed review comments * restore unnecessary changes in test/BUILD * fix float comparisons warnings from Release builds * clang format * fix visibility warning * remove misc file * removed backup files * addressed review comments * fix shorten in warning * use suffix for existing min_time specs to silent warnings in tests * fix leaks * use default min-time value in flag decl for consistency * removed double kMinTimeDecl from benchmark.h * dont need to preserve errno * add death tests * Add BENCHMARK_EXPORT to hopefully fix missing def errors * only enable death tests in debug mode because bm_check is no-op in release mode * guard death tests with additional support-check macros * Add additional guard to prevent running in Release mode --------- Co-authored-by: dominic <510002+dmah42@users.noreply.github.com>
2023-02-07 11:45:18 +00:00
add_test(NAME user_counters_test COMMAND user_counters_test --benchmark_min_time=0.01s)
User counter tests: first version.
2017-04-27 18:25:20 +00:00
Support optional, user-directed collection of performance counters (#1114) * Support optional, user-directed collection of performance counters The patch allows an engineer wishing to drill into the root causes of a regression, for example. Currently, only single threaded runs are supported. The feature is a build-time opt in, and then a runtime opt in. The engineer may run the benchmark executable, passing a list of performance counter names (using libpfm's naming scheme) at the command line. The counter values will then be collected and reported back as UserCounters. This is different from #240 in that it is a benchmark user opt-in, and the counter collection is transparent to the benchmark. Currently, this is only supported on platforms where libpfm is supported. libpfm: http://perfmon2.sourceforge.net/ * 'Use' values param in Snapshot when BENCHMARK_OS_WINDOWS This is to avoid unused parameter warning-as-error * Added missing include for <vector> in perf_counters.cc * Moved doc to docs * Added license blurbs
2021-04-28 08:25:29 +00:00
compile_output_test(perf_counters_test)
Allow specifying number of iterations via --benchmark_min_time. (#1525) * Allow specifying number of iterations via --benchmark_min_time. Make the flag accept two new suffixes: + <integer>x: number of iterations + <floag>s: minimum number of seconds. This matches the internal benchmark API. * forgot to change flag type to string * used tagged union instead of std::variant, which is not available pre C++14 * update decl in benchmark_runner.h too * fixed errors * refactor * backward compat * typo * use IterationCount type * fixed test * const_cast * ret type * remove extra _ * debug * fixed bug from reporting that caused the new configs not to be included in the final report * addressed review comments * restore unnecessary changes in test/BUILD * fix float comparisons warnings from Release builds * clang format * fix visibility warning * remove misc file * removed backup files * addressed review comments * fix shorten in warning * use suffix for existing min_time specs to silent warnings in tests * fix leaks * use default min-time value in flag decl for consistency * removed double kMinTimeDecl from benchmark.h * dont need to preserve errno * add death tests * Add BENCHMARK_EXPORT to hopefully fix missing def errors * only enable death tests in debug mode because bm_check is no-op in release mode * guard death tests with additional support-check macros * Add additional guard to prevent running in Release mode --------- Co-authored-by: dominic <510002+dmah42@users.noreply.github.com>
2023-02-07 11:45:18 +00:00
add_test(NAME perf_counters_test COMMAND perf_counters_test --benchmark_min_time=0.01s --benchmark_perf_counters=CYCLES,BRANCHES)
Support optional, user-directed collection of performance counters (#1114) * Support optional, user-directed collection of performance counters The patch allows an engineer wishing to drill into the root causes of a regression, for example. Currently, only single threaded runs are supported. The feature is a build-time opt in, and then a runtime opt in. The engineer may run the benchmark executable, passing a list of performance counter names (using libpfm's naming scheme) at the command line. The counter values will then be collected and reported back as UserCounters. This is different from #240 in that it is a benchmark user opt-in, and the counter collection is transparent to the benchmark. Currently, this is only supported on platforms where libpfm is supported. libpfm: http://perfmon2.sourceforge.net/ * 'Use' values param in Snapshot when BENCHMARK_OS_WINDOWS This is to avoid unused parameter warning-as-error * Added missing include for <vector> in perf_counters.cc * Moved doc to docs * Added license blurbs
2021-04-28 08:25:29 +00:00
Add process_time for better OpenMP and user-managed thread timing * Google Benchmark now works with OpenMP and other user-managed threading.
2019-04-09 12:01:33 +00:00
compile_output_test(internal_threading_test)
Allow specifying number of iterations via --benchmark_min_time. (#1525) * Allow specifying number of iterations via --benchmark_min_time. Make the flag accept two new suffixes: + <integer>x: number of iterations + <floag>s: minimum number of seconds. This matches the internal benchmark API. * forgot to change flag type to string * used tagged union instead of std::variant, which is not available pre C++14 * update decl in benchmark_runner.h too * fixed errors * refactor * backward compat * typo * use IterationCount type * fixed test * const_cast * ret type * remove extra _ * debug * fixed bug from reporting that caused the new configs not to be included in the final report * addressed review comments * restore unnecessary changes in test/BUILD * fix float comparisons warnings from Release builds * clang format * fix visibility warning * remove misc file * removed backup files * addressed review comments * fix shorten in warning * use suffix for existing min_time specs to silent warnings in tests * fix leaks * use default min-time value in flag decl for consistency * removed double kMinTimeDecl from benchmark.h * dont need to preserve errno * add death tests * Add BENCHMARK_EXPORT to hopefully fix missing def errors * only enable death tests in debug mode because bm_check is no-op in release mode * guard death tests with additional support-check macros * Add additional guard to prevent running in Release mode --------- Co-authored-by: dominic <510002+dmah42@users.noreply.github.com>
2023-02-07 11:45:18 +00:00
add_test(NAME internal_threading_test COMMAND internal_threading_test --benchmark_min_time=0.01s)
Add process_time for better OpenMP and user-managed thread timing * Google Benchmark now works with OpenMP and other user-managed threading.
2019-04-09 12:01:33 +00:00
*Display* aggregates only. (#665) There is a flag https://github.com/google/benchmark/blob/d9cab612e40017af10bddaa5b60c7067032a9e1c/src/benchmark.cc#L75-L78 and a call https://github.com/google/benchmark/blob/d9cab612e40017af10bddaa5b60c7067032a9e1c/include/benchmark/benchmark.h#L837-L840 But that affects everything, every reporter, destination: https://github.com/google/benchmark/blob/d9cab612e40017af10bddaa5b60c7067032a9e1c/src/benchmark.cc#L316 It would be quite useful to have an ability to be more picky. More specifically, i would like to be able to only see the aggregates in the on-screen output, but for the file output to still contain everything. The former is useful in case of a lot of repetition (or even more so if every iteration is reported separately), while the former is **great** for tooling. Fixes https://github.com/google/benchmark/issues/664
2018-09-12 13:26:17 +00:00
compile_output_test(report_aggregates_only_test)
Allow specifying number of iterations via --benchmark_min_time. (#1525) * Allow specifying number of iterations via --benchmark_min_time. Make the flag accept two new suffixes: + <integer>x: number of iterations + <floag>s: minimum number of seconds. This matches the internal benchmark API. * forgot to change flag type to string * used tagged union instead of std::variant, which is not available pre C++14 * update decl in benchmark_runner.h too * fixed errors * refactor * backward compat * typo * use IterationCount type * fixed test * const_cast * ret type * remove extra _ * debug * fixed bug from reporting that caused the new configs not to be included in the final report * addressed review comments * restore unnecessary changes in test/BUILD * fix float comparisons warnings from Release builds * clang format * fix visibility warning * remove misc file * removed backup files * addressed review comments * fix shorten in warning * use suffix for existing min_time specs to silent warnings in tests * fix leaks * use default min-time value in flag decl for consistency * removed double kMinTimeDecl from benchmark.h * dont need to preserve errno * add death tests * Add BENCHMARK_EXPORT to hopefully fix missing def errors * only enable death tests in debug mode because bm_check is no-op in release mode * guard death tests with additional support-check macros * Add additional guard to prevent running in Release mode --------- Co-authored-by: dominic <510002+dmah42@users.noreply.github.com>
2023-02-07 11:45:18 +00:00
add_test(NAME report_aggregates_only_test COMMAND report_aggregates_only_test --benchmark_min_time=0.01s)
*Display* aggregates only. (#665) There is a flag https://github.com/google/benchmark/blob/d9cab612e40017af10bddaa5b60c7067032a9e1c/src/benchmark.cc#L75-L78 and a call https://github.com/google/benchmark/blob/d9cab612e40017af10bddaa5b60c7067032a9e1c/include/benchmark/benchmark.h#L837-L840 But that affects everything, every reporter, destination: https://github.com/google/benchmark/blob/d9cab612e40017af10bddaa5b60c7067032a9e1c/src/benchmark.cc#L316 It would be quite useful to have an ability to be more picky. More specifically, i would like to be able to only see the aggregates in the on-screen output, but for the file output to still contain everything. The former is useful in case of a lot of repetition (or even more so if every iteration is reported separately), while the former is **great** for tooling. Fixes https://github.com/google/benchmark/issues/664
2018-09-12 13:26:17 +00:00
compile_output_test(display_aggregates_only_test)
Allow specifying number of iterations via --benchmark_min_time. (#1525) * Allow specifying number of iterations via --benchmark_min_time. Make the flag accept two new suffixes: + <integer>x: number of iterations + <floag>s: minimum number of seconds. This matches the internal benchmark API. * forgot to change flag type to string * used tagged union instead of std::variant, which is not available pre C++14 * update decl in benchmark_runner.h too * fixed errors * refactor * backward compat * typo * use IterationCount type * fixed test * const_cast * ret type * remove extra _ * debug * fixed bug from reporting that caused the new configs not to be included in the final report * addressed review comments * restore unnecessary changes in test/BUILD * fix float comparisons warnings from Release builds * clang format * fix visibility warning * remove misc file * removed backup files * addressed review comments * fix shorten in warning * use suffix for existing min_time specs to silent warnings in tests * fix leaks * use default min-time value in flag decl for consistency * removed double kMinTimeDecl from benchmark.h * dont need to preserve errno * add death tests * Add BENCHMARK_EXPORT to hopefully fix missing def errors * only enable death tests in debug mode because bm_check is no-op in release mode * guard death tests with additional support-check macros * Add additional guard to prevent running in Release mode --------- Co-authored-by: dominic <510002+dmah42@users.noreply.github.com>
2023-02-07 11:45:18 +00:00
add_test(NAME display_aggregates_only_test COMMAND display_aggregates_only_test --benchmark_min_time=0.01s)
*Display* aggregates only. (#665) There is a flag https://github.com/google/benchmark/blob/d9cab612e40017af10bddaa5b60c7067032a9e1c/src/benchmark.cc#L75-L78 and a call https://github.com/google/benchmark/blob/d9cab612e40017af10bddaa5b60c7067032a9e1c/include/benchmark/benchmark.h#L837-L840 But that affects everything, every reporter, destination: https://github.com/google/benchmark/blob/d9cab612e40017af10bddaa5b60c7067032a9e1c/src/benchmark.cc#L316 It would be quite useful to have an ability to be more picky. More specifically, i would like to be able to only see the aggregates in the on-screen output, but for the file output to still contain everything. The former is useful in case of a lot of repetition (or even more so if every iteration is reported separately), while the former is **great** for tooling. Fixes https://github.com/google/benchmark/issues/664
2018-09-12 13:26:17 +00:00
Add first unit test for benchmark_tabular_counters.
2017-05-02 19:33:28 +00:00
compile_output_test(user_counters_tabular_test)
Allow specifying number of iterations via --benchmark_min_time. (#1525) * Allow specifying number of iterations via --benchmark_min_time. Make the flag accept two new suffixes: + <integer>x: number of iterations + <floag>s: minimum number of seconds. This matches the internal benchmark API. * forgot to change flag type to string * used tagged union instead of std::variant, which is not available pre C++14 * update decl in benchmark_runner.h too * fixed errors * refactor * backward compat * typo * use IterationCount type * fixed test * const_cast * ret type * remove extra _ * debug * fixed bug from reporting that caused the new configs not to be included in the final report * addressed review comments * restore unnecessary changes in test/BUILD * fix float comparisons warnings from Release builds * clang format * fix visibility warning * remove misc file * removed backup files * addressed review comments * fix shorten in warning * use suffix for existing min_time specs to silent warnings in tests * fix leaks * use default min-time value in flag decl for consistency * removed double kMinTimeDecl from benchmark.h * dont need to preserve errno * add death tests * Add BENCHMARK_EXPORT to hopefully fix missing def errors * only enable death tests in debug mode because bm_check is no-op in release mode * guard death tests with additional support-check macros * Add additional guard to prevent running in Release mode --------- Co-authored-by: dominic <510002+dmah42@users.noreply.github.com>
2023-02-07 11:45:18 +00:00
add_test(NAME user_counters_tabular_test COMMAND user_counters_tabular_test --benchmark_counters_tabular=true --benchmark_min_time=0.01s)
Add first unit test for benchmark_tabular_counters.
2017-05-02 19:33:28 +00:00
Counter(): add 'one thousand' param. (#657) * Counter(): add 'one thousand' param. Needed for https://github.com/google/benchmark/pull/654 Custom user counters are quite custom. It is not guaranteed that the user *always* expects for these to have 1k == 1000. If the counter represents bytes/memory/etc, 1k should be 1024. Some bikeshedding points: 1. Is this sufficient, or do we really want to go full on into custom types with names? I think just the '1000' is sufficient for now. 2. Should there be a helper benchmark::Counter::Counter{1000,1024}() static 'constructor' functions, since these two, by far, will be the most used? 3. In the future, we should be somehow encoding this info into JSON. * Counter(): use std::pair<> to represent 'one thousand' * Counter(): just use a new enum with two values 1000 vs 1024. Simpler is better. If someone comes up with a real reason to need something more advanced, it can be added later on. * Counter: just store the 1000 or 1024 in the One_K values directly * Counter: s/One_K/OneK/
2018-08-29 18:11:06 +00:00
compile_output_test(user_counters_thousands_test)
Allow specifying number of iterations via --benchmark_min_time. (#1525) * Allow specifying number of iterations via --benchmark_min_time. Make the flag accept two new suffixes: + <integer>x: number of iterations + <floag>s: minimum number of seconds. This matches the internal benchmark API. * forgot to change flag type to string * used tagged union instead of std::variant, which is not available pre C++14 * update decl in benchmark_runner.h too * fixed errors * refactor * backward compat * typo * use IterationCount type * fixed test * const_cast * ret type * remove extra _ * debug * fixed bug from reporting that caused the new configs not to be included in the final report * addressed review comments * restore unnecessary changes in test/BUILD * fix float comparisons warnings from Release builds * clang format * fix visibility warning * remove misc file * removed backup files * addressed review comments * fix shorten in warning * use suffix for existing min_time specs to silent warnings in tests * fix leaks * use default min-time value in flag decl for consistency * removed double kMinTimeDecl from benchmark.h * dont need to preserve errno * add death tests * Add BENCHMARK_EXPORT to hopefully fix missing def errors * only enable death tests in debug mode because bm_check is no-op in release mode * guard death tests with additional support-check macros * Add additional guard to prevent running in Release mode --------- Co-authored-by: dominic <510002+dmah42@users.noreply.github.com>
2023-02-07 11:45:18 +00:00
add_test(NAME user_counters_thousands_test COMMAND user_counters_thousands_test --benchmark_min_time=0.01s)
Counter(): add 'one thousand' param. (#657) * Counter(): add 'one thousand' param. Needed for https://github.com/google/benchmark/pull/654 Custom user counters are quite custom. It is not guaranteed that the user *always* expects for these to have 1k == 1000. If the counter represents bytes/memory/etc, 1k should be 1024. Some bikeshedding points: 1. Is this sufficient, or do we really want to go full on into custom types with names? I think just the '1000' is sufficient for now. 2. Should there be a helper benchmark::Counter::Counter{1000,1024}() static 'constructor' functions, since these two, by far, will be the most used? 3. In the future, we should be somehow encoding this info into JSON. * Counter(): use std::pair<> to represent 'one thousand' * Counter(): just use a new enum with two values 1000 vs 1024. Simpler is better. If someone comes up with a real reason to need something more advanced, it can be added later on. * Counter: just store the 1000 or 1024 in the One_K values directly * Counter: s/One_K/OneK/
2018-08-29 18:11:06 +00:00
Track two more details about runs - the aggregate name, and run name. (#675) This is related to @BaaMeow's work in https://github.com/google/benchmark/pull/616 but is not based on it. Two new fields are tracked, and dumped into JSON: * If the run is an aggregate, the aggregate's name is stored. It can be RMS, BigO, mean, median, stddev, or any custom stat name. * The aggregate-name-less run name is additionally stored. I.e. not some name of the benchmark function, but the actual name, but without the 'aggregate name' suffix. This way one can group/filter all the runs, and filter by the particular aggregate type. I *might* need this for further tooling improvement. Or maybe not. But this is certainly worthwhile for custom tooling.
2018-09-13 12:08:15 +00:00
compile_output_test(memory_manager_test)
Allow specifying number of iterations via --benchmark_min_time. (#1525) * Allow specifying number of iterations via --benchmark_min_time. Make the flag accept two new suffixes: + <integer>x: number of iterations + <floag>s: minimum number of seconds. This matches the internal benchmark API. * forgot to change flag type to string * used tagged union instead of std::variant, which is not available pre C++14 * update decl in benchmark_runner.h too * fixed errors * refactor * backward compat * typo * use IterationCount type * fixed test * const_cast * ret type * remove extra _ * debug * fixed bug from reporting that caused the new configs not to be included in the final report * addressed review comments * restore unnecessary changes in test/BUILD * fix float comparisons warnings from Release builds * clang format * fix visibility warning * remove misc file * removed backup files * addressed review comments * fix shorten in warning * use suffix for existing min_time specs to silent warnings in tests * fix leaks * use default min-time value in flag decl for consistency * removed double kMinTimeDecl from benchmark.h * dont need to preserve errno * add death tests * Add BENCHMARK_EXPORT to hopefully fix missing def errors * only enable death tests in debug mode because bm_check is no-op in release mode * guard death tests with additional support-check macros * Add additional guard to prevent running in Release mode --------- Co-authored-by: dominic <510002+dmah42@users.noreply.github.com>
2023-02-07 11:45:18 +00:00
add_test(NAME memory_manager_test COMMAND memory_manager_test --benchmark_min_time=0.01s)
Track two more details about runs - the aggregate name, and run name. (#675) This is related to @BaaMeow's work in https://github.com/google/benchmark/pull/616 but is not based on it. Two new fields are tracked, and dumped into JSON: * If the run is an aggregate, the aggregate's name is stored. It can be RMS, BigO, mean, median, stddev, or any custom stat name. * The aggregate-name-less run name is additionally stored. I.e. not some name of the benchmark function, but the actual name, but without the 'aggregate name' suffix. This way one can group/filter all the runs, and filter by the particular aggregate type. I *might* need this for further tooling improvement. Or maybe not. But this is certainly worthwhile for custom tooling.
2018-09-13 12:08:15 +00:00
annotate and export public symbols (#1321)
2022-02-14 10:48:53 +00:00
# MSVC does not allow to set the language standard to C++98/03.
if(NOT CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
Only compile and run the C++03 test when -std=c++03 is supported. Fixes #164
2016-05-24 02:44:10 +00:00
compile_benchmark_test(cxx03_test)
set_target_properties(cxx03_test
Suppress -Wodr on C++03 tests when LTO is enabled. The benchmark library is compiled as C++11, but certain tests are compiled as C++03. When -flto is enabled GCC 5.4 and above will diagnose an ODR violation in libstdc++'s <map>. This ODR violation, although real, should likely be benign. For this reason it seems sensible to simply suppress -Wodr when building the C++03 test. This patch fixes #420 and supersede's PR #424.
2017-07-31 00:41:57 +00:00
PROPERTIES
Fix cxx03 standard selection, option override in CMake 3.13+. Fixes #933 (#934)
2020-01-31 10:16:25 +00:00
CXX_STANDARD 98
CXX_STANDARD_REQUIRED YES)
Suppress -Wodr on C++03 tests when LTO is enabled. The benchmark library is compiled as C++11, but certain tests are compiled as C++03. When -flto is enabled GCC 5.4 and above will diagnose an ODR violation in libstdc++'s <map>. This ODR violation, although real, should likely be benign. For this reason it seems sensible to simply suppress -Wodr when building the C++03 test. This patch fixes #420 and supersede's PR #424.
2017-07-31 00:41:57 +00:00
# libstdc++ provides different definitions within <map> between dialects. When
# LTO is enabled and -Werror is specified GCC diagnoses this ODR violation
# causing the test to fail to compile. To prevent this we explicitly disable
# the warning.
check_cxx_compiler_flag(-Wno-odr BENCHMARK_HAS_WNO_ODR)
annotate and export public symbols (#1321)
2022-02-14 10:48:53 +00:00
check_cxx_compiler_flag(-Wno-lto-type-mismatch BENCHMARK_HAS_WNO_LTO_TYPE_MISMATCH)
# Cannot set_target_properties multiple times here because the warnings will
# be overwritten on each call
set (DISABLE_LTO_WARNINGS "")
if (BENCHMARK_HAS_WNO_ODR)
set(DISABLE_LTO_WARNINGS "${DISABLE_LTO_WARNINGS} -Wno-odr")
endif()
if (BENCHMARK_HAS_WNO_LTO_TYPE_MISMATCH)
set(DISABLE_LTO_WARNINGS "${DISABLE_LTO_WARNINGS} -Wno-lto-type-mismatch")
Suppress -Wodr on C++03 tests when LTO is enabled. The benchmark library is compiled as C++11, but certain tests are compiled as C++03. When -flto is enabled GCC 5.4 and above will diagnose an ODR violation in libstdc++'s <map>. This ODR violation, although real, should likely be benign. For this reason it seems sensible to simply suppress -Wodr when building the C++03 test. This patch fixes #420 and supersede's PR #424.
2017-07-31 00:41:57 +00:00
endif()
annotate and export public symbols (#1321)
2022-02-14 10:48:53 +00:00
set_target_properties(cxx03_test PROPERTIES LINK_FLAGS "${DISABLE_LTO_WARNINGS}")
Allow specifying number of iterations via --benchmark_min_time. (#1525) * Allow specifying number of iterations via --benchmark_min_time. Make the flag accept two new suffixes: + <integer>x: number of iterations + <floag>s: minimum number of seconds. This matches the internal benchmark API. * forgot to change flag type to string * used tagged union instead of std::variant, which is not available pre C++14 * update decl in benchmark_runner.h too * fixed errors * refactor * backward compat * typo * use IterationCount type * fixed test * const_cast * ret type * remove extra _ * debug * fixed bug from reporting that caused the new configs not to be included in the final report * addressed review comments * restore unnecessary changes in test/BUILD * fix float comparisons warnings from Release builds * clang format * fix visibility warning * remove misc file * removed backup files * addressed review comments * fix shorten in warning * use suffix for existing min_time specs to silent warnings in tests * fix leaks * use default min-time value in flag decl for consistency * removed double kMinTimeDecl from benchmark.h * dont need to preserve errno * add death tests * Add BENCHMARK_EXPORT to hopefully fix missing def errors * only enable death tests in debug mode because bm_check is no-op in release mode * guard death tests with additional support-check macros * Add additional guard to prevent running in Release mode --------- Co-authored-by: dominic <510002+dmah42@users.noreply.github.com>
2023-02-07 11:45:18 +00:00
add_test(NAME cxx03 COMMAND cxx03_test --benchmark_min_time=0.01s)
Only compile and run the C++03 test when -std=c++03 is supported. Fixes #164
2016-05-24 02:44:10 +00:00
endif()
Coverage support with LCOV and GCOV This patch adds a `coverage` target that allows coverage statisitics to be retrieved for the project. It requires that lcov and gcov is installed and that the generator is unix makefiles but this can be improved upon in future releases. To make it work use the coverage build type: ``` cmake -DCMAKE_BUILD_TYPE=Coverage . make coverage ```
2015-04-30 13:28:04 +00:00
Increase complexity_test.cc runtime to attempt to avoid appveyor failures
2016-09-03 06:06:51 +00:00
# Attempt to work around flaky test failures when running on Appveyor servers.
if (DEFINED ENV{APPVEYOR})
Allow specifying number of iterations via --benchmark_min_time. (#1525) * Allow specifying number of iterations via --benchmark_min_time. Make the flag accept two new suffixes: + <integer>x: number of iterations + <floag>s: minimum number of seconds. This matches the internal benchmark API. * forgot to change flag type to string * used tagged union instead of std::variant, which is not available pre C++14 * update decl in benchmark_runner.h too * fixed errors * refactor * backward compat * typo * use IterationCount type * fixed test * const_cast * ret type * remove extra _ * debug * fixed bug from reporting that caused the new configs not to be included in the final report * addressed review comments * restore unnecessary changes in test/BUILD * fix float comparisons warnings from Release builds * clang format * fix visibility warning * remove misc file * removed backup files * addressed review comments * fix shorten in warning * use suffix for existing min_time specs to silent warnings in tests * fix leaks * use default min-time value in flag decl for consistency * removed double kMinTimeDecl from benchmark.h * dont need to preserve errno * add death tests * Add BENCHMARK_EXPORT to hopefully fix missing def errors * only enable death tests in debug mode because bm_check is no-op in release mode * guard death tests with additional support-check macros * Add additional guard to prevent running in Release mode --------- Co-authored-by: dominic <510002+dmah42@users.noreply.github.com>
2023-02-07 11:45:18 +00:00
set(COMPLEXITY_MIN_TIME "0.5s")
Increase complexity_test.cc runtime to attempt to avoid appveyor failures
2016-09-03 06:06:51 +00:00
else()
Allow specifying number of iterations via --benchmark_min_time. (#1525) * Allow specifying number of iterations via --benchmark_min_time. Make the flag accept two new suffixes: + <integer>x: number of iterations + <floag>s: minimum number of seconds. This matches the internal benchmark API. * forgot to change flag type to string * used tagged union instead of std::variant, which is not available pre C++14 * update decl in benchmark_runner.h too * fixed errors * refactor * backward compat * typo * use IterationCount type * fixed test * const_cast * ret type * remove extra _ * debug * fixed bug from reporting that caused the new configs not to be included in the final report * addressed review comments * restore unnecessary changes in test/BUILD * fix float comparisons warnings from Release builds * clang format * fix visibility warning * remove misc file * removed backup files * addressed review comments * fix shorten in warning * use suffix for existing min_time specs to silent warnings in tests * fix leaks * use default min-time value in flag decl for consistency * removed double kMinTimeDecl from benchmark.h * dont need to preserve errno * add death tests * Add BENCHMARK_EXPORT to hopefully fix missing def errors * only enable death tests in debug mode because bm_check is no-op in release mode * guard death tests with additional support-check macros * Add additional guard to prevent running in Release mode --------- Co-authored-by: dominic <510002+dmah42@users.noreply.github.com>
2023-02-07 11:45:18 +00:00
set(COMPLEXITY_MIN_TIME "0.01s")
Increase complexity_test.cc runtime to attempt to avoid appveyor failures
2016-09-03 06:06:51 +00:00
endif()
Refactor output test runner into standalone module. (#277) * refactor * Move default substitutions into library * Move default substitutions to the *right* place in the library * Fix init order issues that caused test failures * improve diagnostics * add missing include * general cleanup * Address review comments
2016-08-28 19:24:16 +00:00
compile_output_test(complexity_test)
CMake: use full add_test(NAME <> COMMAND <>) signature (#901) * CTest must use proper paths to executables With the following syntax: ``` add_test(NAME <name> COMMAND <command> [<arg>...]) ``` if `<command>` specifies an executable target it will automatically be replaced by the location of the executable created at build time. This is important if a `<Configuration>_POSTFIX` like `_d` is used. * Fix typo in ctest invocation Instead of `-c` the uppercase `-C` must be used to select a config. But better use the longopt.
2019-11-05 19:46:13 +00:00
add_test(NAME complexity_benchmark COMMAND complexity_test --benchmark_min_time=${COMPLEXITY_MIN_TIME})
Coverage support with LCOV and GCOV This patch adds a `coverage` target that allows coverage statisitics to be retrieved for the project. It requires that lcov and gcov is installed and that the generator is unix makefiles but this can be improved upon in future releases. To make it work use the coverage build type: ``` cmake -DCMAKE_BUILD_TYPE=Coverage . make coverage ```
2015-04-30 13:28:04 +00:00
Add support for GTest based unit tests. (#485) * Add support for GTest based unit tests. As Dominic and I have previously discussed, there is some need/desire to improve the testing situation in Google Benchmark. One step to fixing this problem is to make it easier to write unit tests by adding support for GTest, which is what this patch does. By default it looks for an installed version of GTest. However the user can specify -DBENCHMARK_BUILD_EXTERNAL_GTEST=ON to instead download, build, and use copy of gtest from source. This is quite useful when Benchmark is being built in non-standard configurations, such as against libc++ or in 32 bit mode.
2017-12-13 23:26:47 +00:00
###############################################################################
# GoogleTest Unit Tests
###############################################################################
if (BENCHMARK_ENABLE_GTEST_TESTS)
macro(compile_gtest name)
add_executable(${name} "${name}.cc")
Alias CMake Targets. Fixes #921 (#926) * add Jordan Williams to both CONTRIBUTORS and AUTHORS * alias benchmark libraries Provide aliased CMake targets for the benchmark and benchmark_main targets. The alias targets are namespaced under benchmark::, which is the namespace when they are exported. I chose not to use either the PROJECT_NAME or the namespace variable but to hard-code the namespace. This is because the benchmark and benchmark_main targets are hard-coded by name themselves. Hard-coding the namespace is also much cleaner and easier to read. * link to aliased benchmark targets It is safer to link against namespaced targets because of how CMake interprets the double colon. Typo's will be caught by CMake at configuration-time instead of during compile / link time. * document the provided alias targets * add "Usage with CMake" section in documentation This section covers linking against the alias/import CMake targets and including them using either find_package or add_subdirectory. * format the "Usage with CMake" README section Added a newline after the "Usage with CMake" section header. Dropped the header level of the section by one to make it a direct subsection of the "Usage" section. Wrapped lines to be no longer than 80 characters in length.
2020-01-14 20:21:24 +00:00
target_link_libraries(${name} benchmark::benchmark
CMake: codedrop of googletest cmake magic from me (#809) https://github.com/google/benchmark/pull/801 is stuck with some cryptic cmake failure due to some linking issue between googletest and threading libraries. I suspect that is mostly happening because of the, uhm, intentionally extremely twisted-in-the-brains approach that is being used to actually build the library as part of the buiild, except without actually building it as part of the build. If we do actually build it as part of the build, then all the transitive dependencies should magically be in order, and maybe everything will just work. This new version of cmake magic was written by me in https://github.com/darktable-org/rawspeed/blob/0e22f085c57cd629983c7351c598abd744b421b5/cmake/Modules/GoogleTest.cmake.in https://github.com/darktable-org/rawspeed/blob/0e22f085c57cd629983c7351c598abd744b421b5/cmake/Modules/GoogleTest.cmake, based on the official googletest docs and LOTS of experimentation.
2019-04-30 10:36:29 +00:00
gmock_main ${CMAKE_THREAD_LIBS_INIT})
Add support for GTest based unit tests. (#485) * Add support for GTest based unit tests. As Dominic and I have previously discussed, there is some need/desire to improve the testing situation in Google Benchmark. One step to fixing this problem is to make it easier to write unit tests by adding support for GTest, which is what this patch does. By default it looks for an installed version of GTest. However the user can specify -DBENCHMARK_BUILD_EXTERNAL_GTEST=ON to instead download, build, and use copy of gtest from source. This is quite useful when Benchmark is being built in non-standard configurations, such as against libc++ or in 32 bit mode.
2017-12-13 23:26:47 +00:00
endmacro(compile_gtest)
macro(add_gtest name)
compile_gtest(${name})
CMake: use full add_test(NAME <> COMMAND <>) signature (#901) * CTest must use proper paths to executables With the following syntax: ``` add_test(NAME <name> COMMAND <command> [<arg>...]) ``` if `<command>` specifies an executable target it will automatically be replaced by the location of the executable created at build time. This is important if a `<Configuration>_POSTFIX` like `_d` is used. * Fix typo in ctest invocation Instead of `-c` the uppercase `-C` must be used to select a config. But better use the longopt.
2019-11-05 19:46:13 +00:00
add_test(NAME ${name} COMMAND ${name})
Add support for GTest based unit tests. (#485) * Add support for GTest based unit tests. As Dominic and I have previously discussed, there is some need/desire to improve the testing situation in Google Benchmark. One step to fixing this problem is to make it easier to write unit tests by adding support for GTest, which is what this patch does. By default it looks for an installed version of GTest. However the user can specify -DBENCHMARK_BUILD_EXTERNAL_GTEST=ON to instead download, build, and use copy of gtest from source. This is quite useful when Benchmark is being built in non-standard configurations, such as against libc++ or in 32 bit mode.
2017-12-13 23:26:47 +00:00
endmacro()
Allow AddRange to work with int64_t. (#548) * Allow AddRange to work with int64_t. Fixes #516 Also, tweak how we manage per-test build needs, and create a standard _gtest suffix for googletest to differentiate from non-googletest tests. I also ran clang-format on the files that I changed (but not the benchmark include or main src as they have too many clang-format issues). * Add benchmark_gtest to cmake * Set(Items|Bytes)Processed now take int64_t
2018-04-03 22:12:47 +00:00
add_gtest(benchmark_gtest)
BENCHMARK_CAPTURE() and Complexity() - naming problem (#761) Created BenchmarkName class which holds the full benchmark name and allows specifying and retrieving different components of the name (e.g. ARGS, THREADS etc.) Fixes #730.
2019-03-17 13:38:51 +00:00
add_gtest(benchmark_name_gtest)
Random interleaving of benchmark repetitions - the sequel (fixes #1051) (#1163) Inspired by the original implementation by Hai Huang @haih-g from https://github.com/google/benchmark/pull/1105. The original implementation had design deficiencies that weren't really addressable without redesign, so it was reverted. In essence, the original implementation consisted of two separateable parts: * reducing the amount time each repetition is run for, and symmetrically increasing repetition count * running the repetitions in random order While it worked fine for the usual case, it broke down when user would specify repetitions (it would completely ignore that request), or specified per-repetition min time (while it would still adjust the repetition count, it would not adjust the per-repetition time, leading to much greater run times) Here, like i was originally suggesting in the original review, i'm separating the features, and only dealing with a single one - running repetitions in random order. Now that the runs/repetitions are no longer in-order, the tooling may wish to sort the output, and indeed `compare.py` has been updated to do that: #1168.
2021-06-03 18:16:54 +00:00
add_gtest(benchmark_random_interleaving_gtest)
Add command line flags tests (#793) Increase coverage
2019-04-17 16:08:52 +00:00
add_gtest(commandlineflags_gtest)
Allow AddRange to work with int64_t. (#548) * Allow AddRange to work with int64_t. Fixes #516 Also, tweak how we manage per-test build needs, and create a standard _gtest suffix for googletest to differentiate from non-googletest tests. I also ran clang-format on the files that I changed (but not the benchmark include or main src as they have too many clang-format issues). * Add benchmark_gtest to cmake * Set(Items|Bytes)Processed now take int64_t
2018-04-03 22:12:47 +00:00
add_gtest(statistics_gtest)
Fix compilation on Android with GNU STL (#596) * Fix compilation on Android with GNU STL GNU STL in Android NDK lacks string conversion functions from C++11, including std::stoul, std::stoi, and std::stod. This patch reimplements these functions in benchmark:: namespace using C-style equivalents from C++03. * Avoid use of log2 which doesn't exist in Android GNU STL GNU STL in Android NDK lacks log2 function from C99/C++11. This patch replaces their use in the code with double log(double) function.
2018-06-05 10:36:26 +00:00
add_gtest(string_util_gtest)
Support optional, user-directed collection of performance counters (#1114) * Support optional, user-directed collection of performance counters The patch allows an engineer wishing to drill into the root causes of a regression, for example. Currently, only single threaded runs are supported. The feature is a build-time opt in, and then a runtime opt in. The engineer may run the benchmark executable, passing a list of performance counter names (using libpfm's naming scheme) at the command line. The counter values will then be collected and reported back as UserCounters. This is different from #240 in that it is a benchmark user opt-in, and the counter collection is transparent to the benchmark. Currently, this is only supported on platforms where libpfm is supported. libpfm: http://perfmon2.sourceforge.net/ * 'Use' values param in Snapshot when BENCHMARK_OS_WINDOWS This is to avoid unused parameter warning-as-error * Added missing include for <vector> in perf_counters.cc * Moved doc to docs * Added license blurbs
2021-04-28 08:25:29 +00:00
add_gtest(perf_counters_gtest)
Allow setting the default time unit globally (#1337) * Add option to set the default time unit globally This commit introduces the `--benchmark_time_unit={ns|us|ms|s}` command line argument. The argument only affects benchmarks where the time unit is not set explicitly. * Update AUTHORS and CONTRIBUTORS * Test `SetDefaultTimeUnit` * clang format * Use `GetDefaultTimeUnit()` for initializing `TimeUnit` variables * Review fixes * Export functions * Add comment
2022-03-04 11:07:01 +00:00
add_gtest(time_unit_gtest)
Allow specifying number of iterations via --benchmark_min_time. (#1525) * Allow specifying number of iterations via --benchmark_min_time. Make the flag accept two new suffixes: + <integer>x: number of iterations + <floag>s: minimum number of seconds. This matches the internal benchmark API. * forgot to change flag type to string * used tagged union instead of std::variant, which is not available pre C++14 * update decl in benchmark_runner.h too * fixed errors * refactor * backward compat * typo * use IterationCount type * fixed test * const_cast * ret type * remove extra _ * debug * fixed bug from reporting that caused the new configs not to be included in the final report * addressed review comments * restore unnecessary changes in test/BUILD * fix float comparisons warnings from Release builds * clang format * fix visibility warning * remove misc file * removed backup files * addressed review comments * fix shorten in warning * use suffix for existing min_time specs to silent warnings in tests * fix leaks * use default min-time value in flag decl for consistency * removed double kMinTimeDecl from benchmark.h * dont need to preserve errno * add death tests * Add BENCHMARK_EXPORT to hopefully fix missing def errors * only enable death tests in debug mode because bm_check is no-op in release mode * guard death tests with additional support-check macros * Add additional guard to prevent running in Release mode --------- Co-authored-by: dominic <510002+dmah42@users.noreply.github.com>
2023-02-07 11:45:18 +00:00
add_gtest(min_time_parse_gtest)
Add support for GTest based unit tests. (#485) * Add support for GTest based unit tests. As Dominic and I have previously discussed, there is some need/desire to improve the testing situation in Google Benchmark. One step to fixing this problem is to make it easier to write unit tests by adding support for GTest, which is what this patch does. By default it looks for an installed version of GTest. However the user can specify -DBENCHMARK_BUILD_EXTERNAL_GTEST=ON to instead download, build, and use copy of gtest from source. This is quite useful when Benchmark is being built in non-standard configurations, such as against libc++ or in 32 bit mode.
2017-12-13 23:26:47 +00:00
endif(BENCHMARK_ENABLE_GTEST_TESTS)
Add tests to verify assembler output -- Fix DoNotOptimize. (#530) * Add tests to verify assembler output -- Fix DoNotOptimize. For things like `DoNotOptimize`, `ClobberMemory`, and even `KeepRunning()`, it is important exactly what assembly they generate. However, we currently have no way to test this. Instead it must be manually validated every time a change occurs -- including a change in compiler version. This patch attempts to introduce a way to test the assembled output automatically. It's mirrors how LLVM verifies compiler output, and it uses LLVM FileCheck to run the tests in a similar way. The tests function by generating the assembly for a test in CMake, and then using FileCheck to verify the // CHECK lines in the source file are found in the generated assembly. Currently, the tests only run on 64-bit x86 systems under GCC and Clang, and when FileCheck is found on the system. Additionally, this patch tries to improve the code gen from DoNotOptimize. This should probably be a separate change, but I needed something to test. * Disable assembly tests on Bazel for now * Link FIXME to github issue * Fix Tests on OS X * fix strip_asm.py to work on both Linux and OS X like targets
2018-03-23 22:10:47 +00:00
###############################################################################
# Assembly Unit Tests
###############################################################################
if (BENCHMARK_ENABLE_ASSEMBLY_TESTS)
if (NOT LLVM_FILECHECK_EXE)
message(FATAL_ERROR "LLVM FileCheck is required when including this file")
endif()
include(AssemblyTests.cmake)
add_filecheck_test(donotoptimize_assembly_test)
add_filecheck_test(state_assembly_test)
add_filecheck_test(clobber_memory_assembly_test)
endif()
###############################################################################
# Code Coverage Configuration
###############################################################################
Add support for GTest based unit tests. (#485) * Add support for GTest based unit tests. As Dominic and I have previously discussed, there is some need/desire to improve the testing situation in Google Benchmark. One step to fixing this problem is to make it easier to write unit tests by adding support for GTest, which is what this patch does. By default it looks for an installed version of GTest. However the user can specify -DBENCHMARK_BUILD_EXTERNAL_GTEST=ON to instead download, build, and use copy of gtest from source. This is quite useful when Benchmark is being built in non-standard configurations, such as against libc++ or in 32 bit mode.
2017-12-13 23:26:47 +00:00
Coverage support with LCOV and GCOV This patch adds a `coverage` target that allows coverage statisitics to be retrieved for the project. It requires that lcov and gcov is installed and that the generator is unix makefiles but this can be improved upon in future releases. To make it work use the coverage build type: ``` cmake -DCMAKE_BUILD_TYPE=Coverage . make coverage ```
2015-04-30 13:28:04 +00:00
# Add the coverage command(s)
Fixing cmake issue with empty cmake_build_type
2015-05-22 21:15:19 +00:00
if(CMAKE_BUILD_TYPE)
string(TOLOWER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_LOWER)
endif()
Coverage support with LCOV and GCOV This patch adds a `coverage` target that allows coverage statisitics to be retrieved for the project. It requires that lcov and gcov is installed and that the generator is unix makefiles but this can be improved upon in future releases. To make it work use the coverage build type: ``` cmake -DCMAKE_BUILD_TYPE=Coverage . make coverage ```
2015-04-30 13:28:04 +00:00
if (${CMAKE_BUILD_TYPE_LOWER} MATCHES "coverage")
find_program(GCOV gcov)
find_program(LCOV lcov)
find_program(GENHTML genhtml)
find_program(CTEST ctest)
if (GCOV AND LCOV AND GENHTML AND CTEST AND HAVE_CXX_FLAG_COVERAGE)
add_custom_command(
OUTPUT ${CMAKE_BINARY_DIR}/lcov/index.html
COMMAND ${LCOV} -q -z -d .
COMMAND ${LCOV} -q --no-external -c -b "${CMAKE_SOURCE_DIR}" -d . -o before.lcov -i
COMMAND ${CTEST} --force-new-ctest-process
COMMAND ${LCOV} -q --no-external -c -b "${CMAKE_SOURCE_DIR}" -d . -o after.lcov
COMMAND ${LCOV} -q -a before.lcov -a after.lcov --output-file final.lcov
COMMAND ${LCOV} -q -r final.lcov "'${CMAKE_SOURCE_DIR}/test/*'" -o final.lcov
COMMAND ${GENHTML} final.lcov -o lcov --demangle-cpp --sort -p "${CMAKE_BINARY_DIR}" -t benchmark
Added test file and Complexity() interface
2016-05-18 17:59:34 +00:00
DEPENDS filter_test benchmark_test options_test basic_test fixture_test cxx03_test complexity_test
Coverage support with LCOV and GCOV This patch adds a `coverage` target that allows coverage statisitics to be retrieved for the project. It requires that lcov and gcov is installed and that the generator is unix makefiles but this can be improved upon in future releases. To make it work use the coverage build type: ``` cmake -DCMAKE_BUILD_TYPE=Coverage . make coverage ```
2015-04-30 13:28:04 +00:00
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
COMMENT "Running LCOV"
)
add_custom_target(coverage
DEPENDS ${CMAKE_BINARY_DIR}/lcov/index.html
COMMENT "LCOV report at lcov/index.html"
)
message(STATUS "Coverage command added")
else()
if (HAVE_CXX_FLAG_COVERAGE)
set(CXX_FLAG_COVERAGE_MESSAGE supported)
else()
set(CXX_FLAG_COVERAGE_MESSAGE unavailable)
endif()
message(WARNING
"Coverage not available:\n"
" gcov: ${GCOV}\n"
" lcov: ${LCOV}\n"
" genhtml: ${GENHTML}\n"
" ctest: ${CTEST}\n"
" --coverage flag: ${CXX_FLAG_COVERAGE_MESSAGE}")
endif()
endif()
Reference in a new issue Copy permalink