dolysis
/
rocksdb
mirror of https://github.com/facebook/rocksdb.git
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Avoid duplicate task creation for RocksDB contruns 

Summary: We rely on a basic evaluation logic: when using an expression `A || B`, the `B` part will only get evaluated when `A` fails. Therefore the task creation tool is guaranteed to run if previous build/test step failed. To indicate the correct return value to shell, the task creation tool will call `exit(1)` which will cause Sandcastle to mark it as a failure.

Test Plan:
- Land the changes.
- Trigger a RocksDB contrun - observe the results.
- Look at the results from the nightly runs and fix issues as necessary.

Current testing so far has been in isolation for various components.

Reviewers: andrewkr, IslamAbdelRahman, sdong

Reviewed By: sdong

Subscribers: andrewkr, dhruba

Differential Revision: https://reviews.facebook.net/D60909
This commit is contained in:
Gunnar Kudrjavets 2016-07-27 16:37:12 -07:00
parent d4c45428af
commit 9c8ac144bd
1 changed files with 45 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
build_tools/rocksdb-lego-determinator
View File

@ -8,7 +8,7 @@
# Input Value
# -------------------------------------------------------------------------
# EMAIL Email address to report on trigger conditions
# ONCAL Email address to raise a task on failure
# ONCALL Email address to raise a task on failure
# TRIGGER Trigger conditions for email. Valid values are fail, warn, all
# SUBSCRIBER Email addresss to add as subscriber for task
#
@ -42,6 +42,9 @@ if [ ! -z $ONCALL ]; then
},"
fi
# For now, create the tasks using only the dedicated task creation tool.
CREATE_TASK=
REPORT=
if [[ ! -z $REPORT_EMAIL || ! -z $CREATE_TASK ]]; then
REPORT="'report': [
@ -89,6 +92,19 @@ HTTP_PROXY="https_proxy=http://fwdproxy.29.prn1:8080 http_proxy=http://fwdproxy.
SETUP_JAVA_ENV="export $HTTP_PROXY; export JAVA_HOME=/usr/local/jdk-7u10-64/; export PATH=\$PATH:\$JAVA_HOME/bin"
PARSER="'parser':'python build_tools/error_filter.py $1'"
CONTRUN_NAME="ROCKSDB_CONTRUN_NAME"
# This code is getting called under various scenarios. What we care about is to
# understand when it's called from nightly contruns because in that case we'll
# create tasks for any failures. To follow the existing pattern, we'll check
# the value of $ONCALL. If it's a diff then just call `false` to make sure
# that errors will be properly propagated to the caller.
if [ ! -z $ONCALL ]; then
TASK_CREATION_TOOL="/usr/local/bin/mysql_mtr_filter --rocksdb"
else
TASK_CREATION_TOOL="false"
fi
ARTIFACTS=" 'artifacts': [
{
'name':'database',
@ -126,14 +142,14 @@ PARALLEL_UNIT_TEST_COMMANDS="[
$CLEANUP_ENV,
{
'name':'Build and test RocksDB debug version',
'shell':'$DEBUG make -j$(nproc) all && $SHM make check > /dev/null 2>&1 || cat t/log-*',
'shell':'$DEBUG make -j$(nproc) all && $SHM make check > /dev/null 2>&1 || $CONTRUN_NAME=punit_check $TASK_CREATION_TOOL || cat t/log-*',
'user':'root',
$PARSER
},
$CLEANUP_ENV,
{
'name':'Build and test RocksDB debug version under gcc-4.8.1',
'shell':'$GCC_481 $DEBUG make -j$(nproc) all && $SHM make check > /dev/null 2>&1 || cat t/log-*',
'shell':'$GCC_481 $DEBUG make -j$(nproc) all && $SHM make check > /dev/null 2>&1 || $CONTRUN_NAME=punit_check_gcc481 $TASK_CREATION_TOOL || cat t/log-*',
'user':'root',
$PARSER
},
@ -153,7 +169,7 @@ UNIT_TEST_COMMANDS="[
$CLEANUP_ENV,
{
'name':'Build and test RocksDB debug version',
'shell':'$SHM $DEBUG make $PARALLELISM check',
'shell':'$SHM $DEBUG make $PARALLELISM check || $CONTRUN_NAME=check $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
@ -175,7 +191,7 @@ UNIT_TEST_NON_SHM_COMMANDS="[
{
'name':'Build and test RocksDB debug version',
'timeout': 86400,
'shell':'$DEBUG make $PARALLELISM check',
'shell':'$DEBUG make $PARALLELISM check || $CONTRUN_NAME=non_shm_check $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
@ -195,7 +211,7 @@ RELEASE_BUILD_COMMANDS="[
$CLEANUP_ENV,
{
'name':'Build RocksDB release',
'shell':'make $PARALLEL_j release',
'shell':'make $PARALLEL_j release || $CONTRUN_NAME=release $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
@ -215,7 +231,7 @@ UNIT_TEST_COMMANDS_481="[
$CLEANUP_ENV,
{
'name':'Build and test RocksDB debug version',
'shell':'$SHM $GCC_481 $DEBUG make $PARALLELISM check',
'shell':'$SHM $GCC_481 $DEBUG make $PARALLELISM check || $CONTRUN_NAME=unit_gcc_481_check $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
@ -235,7 +251,7 @@ RELEASE_BUILD_COMMANDS_481="[
$CLEANUP_ENV,
{
'name':'Build RocksDB release on GCC 4.8.1',
'shell':'$GCC_481 make $PARALLEL_j release',
'shell':'$GCC_481 make $PARALLEL_j release || $CONTRUN_NAME=release_gcc481 $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
@ -255,7 +271,7 @@ CLANG_UNIT_TEST_COMMANDS="[
$CLEANUP_ENV,
{
'name':'Build and test RocksDB debug',
'shell':'$CLANG $SHM $DEBUG make $PARALLELISM check',
'shell':'$CLANG $SHM $DEBUG make $PARALLELISM check || $CONTRUN_NAME=clang_check $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
@ -275,7 +291,7 @@ CLANG_RELEASE_BUILD_COMMANDS="[
$CLEANUP_ENV,
{
'name':'Build RocksDB release',
'shell':'$CLANG make $PARALLEL_j release',
'shell':'$CLANG make $PARALLEL_j release|| $CONTRUN_NAME=clang_release $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
@ -295,7 +311,7 @@ CLANG_ANALYZE_COMMANDS="[
$CLEANUP_ENV,
{
'name':'RocksDB build and analyze',
'shell':'$CLANG $SHM $DEBUG make $PARALLEL_j analyze',
'shell':'$CLANG $SHM $DEBUG make $PARALLEL_j analyze || $CONTRUN_NAME=clang_analyze $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
@ -315,7 +331,7 @@ CODE_COV_COMMANDS="[
$CLEANUP_ENV,
{
'name':'Build, test and collect code coverage info',
'shell':'$SHM $DEBUG make $PARALLELISM coverage',
'shell':'$SHM $DEBUG make $PARALLELISM coverage || $CONTRUN_NAME=coverage $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
@ -335,7 +351,7 @@ UNITY_COMMANDS="[
$CLEANUP_ENV,
{
'name':'Build, test unity test',
'shell':'$SHM $DEBUG V=1 make J=1 unity_test',
'shell':'$SHM $DEBUG V=1 make J=1 unity_test || $CONTRUN_NAME=unity_test $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
@ -355,7 +371,7 @@ LITE_BUILD_COMMANDS="[
$CLEANUP_ENV,
{
'name':'Build RocksDB debug version',
'shell':'$LITE make J=1 static_lib',
'shell':'$LITE make J=1 static_lib || $CONTRUN_NAME=lite_static_lib $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
@ -375,7 +391,7 @@ LITE_UNIT_TEST_COMMANDS="[
$CLEANUP_ENV,
{
'name':'Build RocksDB debug version',
'shell':'$SHM $LITE make J=1 check',
'shell':'$SHM $LITE make J=1 check || $CONTRUN_NAME=lite_check $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
@ -396,14 +412,14 @@ STRESS_CRASH_TEST_COMMANDS="[
$CLEANUP_ENV,
{
'name':'Build and run RocksDB debug stress tests',
'shell':'$SHM $DEBUG make J=1 db_stress',
'shell':'$SHM $DEBUG make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
{
'name':'Build and run RocksDB debug crash tests',
'timeout': 86400,
'shell':'$SHM $DEBUG make J=1 crash_test',
'shell':'$SHM $DEBUG make J=1 crash_test || $CONTRUN_NAME=crash_test $TASK_CREATION_TOOL',
'user':'root',
$PARSER
}
@ -424,7 +440,7 @@ WRITE_STRESS_COMMANDS="[
$CLEANUP_ENV,
{
'name':'Build and run RocksDB write stress tests',
'shell':'make write_stress && python tools/write_stress_runner.py --runtime_sec=3600 --db=/tmp/rocksdb_write_stress',
'shell':'make write_stress && python tools/write_stress_runner.py --runtime_sec=3600 --db=/tmp/rocksdb_write_stress || $CONTRUN_NAME=write_stress $TASK_CREATION_TOOL',
'user':'root',
$PARSER
}
@ -446,7 +462,7 @@ ASAN_TEST_COMMANDS="[
$CLEANUP_ENV,
{
'name':'Test RocksDB debug under ASAN',
'shell':'set -o pipefail && $SHM $ASAN $DEBUG make $PARALLELISM asan_check |& /usr/facebook/ops/scripts/asan_symbolize.py -d',
'shell':'set -o pipefail && ($SHM $ASAN $DEBUG make $PARALLELISM asan_check || $CONTRUN_NAME=asan_check $TASK_CREATION_TOOL) |& /usr/facebook/ops/scripts/asan_symbolize.py -d',
'user':'root',
$PARSER
}
@ -468,7 +484,7 @@ ASAN_CRASH_TEST_COMMANDS="[
{
'name':'Build and run RocksDB debug asan_crash_test',
'timeout': 86400,
'shell':'$SHM $DEBUG make J=1 asan_crash_test',
'shell':'$SHM $DEBUG make J=1 asan_crash_test || $CONTRUN_NAME=asan_crash_test $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
@ -488,7 +504,7 @@ UBSAN_TEST_COMMANDS="[
$CLEANUP_ENV,
{
'name':'Test RocksDB debug under UBSAN',
'shell':'set -o pipefail && $SHM $UBSAN $DEBUG make $PARALLELISM ubsan_check',
'shell':'set -o pipefail && $SHM $UBSAN $DEBUG make $PARALLELISM ubsan_check || $CONTRUN_NAME=ubsan_check $TASK_CREATION_TOOL',
'user':'root',
$PARSER
}
@ -510,7 +526,7 @@ UBSAN_CRASH_TEST_COMMANDS="[
{
'name':'Build and run RocksDB debug ubsan_crash_test',
'timeout': 86400,
'shell':'$SHM $DEBUG make J=1 ubsan_crash_test',
'shell':'$SHM $DEBUG make J=1 ubsan_crash_test || $CONTRUN_NAME=ubsan_crash_test $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
@ -532,7 +548,7 @@ VALGRIND_TEST_COMMANDS="[
{
'name':'Run RocksDB debug unit tests',
'timeout': 86400,
'shell':'$DISABLE_JEMALLOC $SHM $DEBUG make $PARALLELISM valgrind_check',
'shell':'$DISABLE_JEMALLOC $SHM $DEBUG make $PARALLELISM valgrind_check || $CONTRUN_NAME=valgrind_check $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
@ -554,7 +570,7 @@ TSAN_UNIT_TEST_COMMANDS="[
{
'name':'Run RocksDB debug unit test',
'timeout': 86400,
'shell':'set -o pipefail && $SHM $DEBUG $TSAN make $PARALLELISM check',
'shell':'set -o pipefail && $SHM $DEBUG $TSAN make $PARALLELISM check || $CONTRUN_NAME=tsan_check $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
@ -576,7 +592,7 @@ TSAN_CRASH_TEST_COMMANDS="[
{
'name':'Compile and run',
'timeout': 86400,
'shell':'set -o pipefail && $SHM $DEBUG $TSAN CRASH_TEST_KILL_ODD=1887 make J=1 crash_test',
'shell':'set -o pipefail && $SHM $DEBUG $TSAN CRASH_TEST_KILL_ODD=1887 make J=1 crash_test || $CONTRUN_NAME=tsan_crash_test $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
@ -638,7 +654,7 @@ FORMAT_COMPATIBLE_COMMANDS="[
$CLEANUP_ENV,
{
'name':'Run RocksDB debug unit test',
'shell':'build_tools/rocksdb-lego-determinator run_format_compatible',
'shell':'build_tools/rocksdb-lego-determinator run_format_compatible || $CONTRUN_NAME=run_format_compatible $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
@ -671,7 +687,7 @@ NO_COMPRESSION_COMMANDS="[
$CLEANUP_ENV,
{
'name':'Run RocksDB debug unit test',
'shell':'build_tools/rocksdb-lego-determinator run_no_compression',
'shell':'build_tools/rocksdb-lego-determinator run_no_compression || $CONTRUN_NAME=run_no_compression $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
@ -728,7 +744,7 @@ REGRESSION_COMMANDS="[
$CLEANUP_ENV,
{
'name':'Make and run script',
'shell':'build_tools/rocksdb-lego-determinator run_regression',
'shell':'build_tools/rocksdb-lego-determinator run_regression || $CONTRUN_NAME=run_regression $TASK_CREATION_TOOL'',
'user':'root',
$PARSER
},
@ -748,7 +764,7 @@ JAVA_BUILD_TEST_COMMANDS="[
$CLEANUP_ENV,
{
'name':'Build RocksDB for Java',
'shell':'$SETUP_JAVA_ENV; $SHM make rocksdbjava',
'shell':'$SETUP_JAVA_ENV; $SHM make rocksdbjava || $CONTRUN_NAME=rocksdbjava $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},