From 4a56c09712a459bd7b084677f4500e961098baf7 Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Tue, 5 Apr 2016 16:18:41 +0100 Subject: Keep Jenkins job script in Mono repo This means we can change the script in different branches (right now, all pull requests against any branch share the same job script) --- scripts/babysitter | 409 ---------------------------------------------- scripts/ci/babysitter | 409 ++++++++++++++++++++++++++++++++++++++++++++++ scripts/ci/run-jenkins.sh | 101 ++++++++++++ scripts/ci/run-step.sh | 56 +++++++ 4 files changed, 566 insertions(+), 409 deletions(-) delete mode 100755 scripts/babysitter create mode 100755 scripts/ci/babysitter create mode 100755 scripts/ci/run-jenkins.sh create mode 100755 scripts/ci/run-step.sh diff --git a/scripts/babysitter b/scripts/babysitter deleted file mode 100755 index f8b5a6c345c..00000000000 --- a/scripts/babysitter +++ /dev/null @@ -1,409 +0,0 @@ -#!/usr/bin/env python - -# Mimics GNU timeout, but does some fancy tracking based on custom features in mono nunit24. - -import argparse -import subprocess -import re -import signal -import time -import sys -import os.path -import copy -import tempfile -import calendar -import json - -### Constants - -# Here is how the communication with nunit works. It has to work with two constraints: -# - We don't invoke nunit directly. We invoke some Makefile which invokes some other Makefile -# and at some point down the line someone calls nunit. -# - nunit has to be able to report back to us even if (especially if) it terminates improperly. -# To deal with all this, communication babysitter->nunit is done by environment variables, -# and communication nunit->babysitter is done by leaving behind files in known locations. - -# Filenames - -CURRENT_TEST_FILE = "babysitter_report_current_test_file.txt" -RAN_TEST_FILE = "babysitter_report_ran_test_file.txt" -FAILED_TEST_FILE = "babysitter_report_failed_test_file.txt" -LOGGING_FILE = "babysitter_report.json_lines" - -# Environment keys - -# Keys used for Babysitter<->Nunit IPC -CURRENT_TEST_KEY = 'MONO_BABYSITTER_NUNIT_CURRENT_TEST_FILE' # Tell nunit where to leave files -RAN_TEST_KEY = 'MONO_BABYSITTER_NUNIT_RAN_TEST_FILE' -FAILED_TEST_KEY = 'MONO_BABYSITTER_NUNIT_FAILED_TEST_FILE' -RUN_KEY = 'MONO_BABYSITTER_NUNIT_RUN_TEST' # Semicolon-separated list of test names -RUN_MODE_KEY = 'MONO_BABYSITTER_NUNIT_RUN_MODE' # Equal to either RUN or AFTER - -# Keys used for script configuration (see --help text) -LOG_FILE_KEY = 'MONO_BABYSITTER_LOG_FILE' # Path -RETRY_KEY = 'MONO_BABYSITTER_RETRY' # Equal to an integer -VERBOSE_KEY = 'MONO_BABYSITTER_VERBOSE' # "Undocumented"-- used for debugging babysitter - -# JSON keys - -DATE_JSON = 'date' # POSIX timestamp of test suite run -INVOKE_JSON = 'invocation' -COUNT_JSON = 'iteration' # How many times was command executed? -LIMIT_JSON = 'failure_max' -SUPPORT_JSON = 'babysitter_protocol' # Was the test suite running with a babysitter-aware nunit? -FINAL_CODE_JSON = 'final_code' -TESTS_JSON = 'tests' # Holds dictionary of (test case name)->(dict with TEST_ keys below) -TEST_FAILURES = 'normal_failures' -TEST_CRASH_FAILURES = 'crash_failures' -TEST_TIMEOUT_FAILURES = 'timeout_failures' - -### Interpret arguments - -scriptname = sys.argv[0] - -# This is very silly: This program needs to switch from argparse to manual parsing -# after the second positional argument, but argparse doesn't let you do this. -# I work around this with a custom subclass which can selectively swallow errors: -class Hesitate(Exception): - pass -class HesitantParser(argparse.ArgumentParser): - def __init__(s, *args, **kwargs): - s.hesitating = True # Errors will be swallowed until this is set false - argparse.ArgumentParser.__init__(s, *args, **kwargs) - def error(s, *args, **kwargs): - if s.hesitating: - raise Hesitate() # Bail out before errors are printed. - argparse.ArgumentParser.error(s, *args, **kwargs) - -# Define args -argparser = HesitantParser(description="""\ -Run a test suite with a timeout.\n -Durations are floating point numbers followed by an optional unit:\n -'s' for seconds (the default) -'m' for minutes -'h' for hours -'d' for days\n -supported environment variables: - %s: File to write logs to (as line-delimited JSON) - %s: If set to a number, failed test cases will be rerun this many times (NUnit test suites only)""" % - (LOG_FILE_KEY, RETRY_KEY), - formatter_class=argparse.RawTextHelpFormatter) -argparser.add_argument('-s', '--signal', dest='signal', metavar='signal', default='TERM', - help="Send this signal to the command on timeout, instead of TERM.") -argparser.add_argument('-k', '--kill-after-duration', dest='kill_after', metavar='duration', - help="If process continues running after signal, send KILL after this duration.") -argparser.add_argument('duration', - help="Time to run before sending signal.") -argparser.add_argument('command', nargs="+", help="Command+args to run.") - -# Repeatedly parse the given args until we find the shortest prefix for which parsing succeeds. -argc = len(sys.argv) -extra_args = [] -for limit in range(1,argc+1): - try: - if limit == argc: # On the final pass, parse for real - argparser.hesitating = False - args = argparser.parse_args(sys.argv[1:limit]) - # If we're still here, parse_args succeeded. - # The final parsed arg is the command; remaining argv items are its args. - extra_args = sys.argv[limit:] - break - except Hesitate: # Before the final pass, if parse_args fails, skip - pass - -argparser.hesitating = False # Just act like a normal argparser from here - -durationre = re.compile(r'(\d+)([smhd]?)') -def parse_duration(duration): # Accept units - match = durationre.match(duration) - if not match: - argparser.error("Could not understand duration %s" % duration) - time, units = match.group(1), match.group(2) - time = int(time) - if units == 'm': - time *= 60 - elif units == 'h': - time *= 60*60 - elif units == 'd': - time *= 60*60*24 - return time - -def parse_signal(sig): # Accept names - if sig.isdigit(): - return int(sig) - for k,v in signal.__dict__.iteritems(): - if k == ("SIG%s" % sig): - return v - argparser.error("Could not understand signal name %s" % sig) - -# Final interpretation of arguments -duration = parse_duration(args.duration) -kill_after = parse_duration(args.kill_after) if args.kill_after is not None else None -timeout_signal = parse_signal(args.signal) -command = args.command + extra_args - -# Process environment -global_env = copy.deepcopy( os.environ ) - -verbose = VERBOSE_KEY in global_env -logging = LOG_FILE_KEY in global_env -logfile = global_env[LOG_FILE_KEY] if logging else None -crash_resuming = True # TODO: Consider exposing this option, or adding a retry_on_crash option. -failmax = int(global_env[RETRY_KEY]) if RETRY_KEY in global_env else 0 -babysitting = True # If false, babysitter becomes a timeout clone with no env manipulation or anything. -if babysitting: - babysitter_dir = tempfile.mkdtemp() - global_env[CURRENT_TEST_KEY] = os.path.join(babysitter_dir, CURRENT_TEST_FILE) - global_env[RAN_TEST_KEY] = os.path.join(babysitter_dir, RAN_TEST_FILE) - global_env[FAILED_TEST_KEY] = os.path.join(babysitter_dir, FAILED_TEST_FILE) - -have_unix_process_groups = 'killpg' in os.__dict__ -have_windows_process_groups = 'CREATE_NEW_PROCESS_GROUP' in subprocess.__dict__ - -### Timeout implementation - -def wait(proc, duration): - # TODO: If we detect Python 3.3, Popen objects have a wait(timeout) method we can use - start = time.time() - while True: - code = proc.poll() - if code is not None: - return code - if time.time()-start > duration: - return None - time.sleep(0.05) - -# Popen and send_signal can't be called in their basic forms because we want to -# send signals to all children, not just to the immediately spawned process. -# Unfortunately the way to do this varies by operating system. -def popen(*args, **kwargs): - if have_unix_process_groups: # Call function on spawn to become process group leader - kwargs['preexec_fn'] = os.setsid - elif have_windows_process_groups: # Set magic flag for Windows process groups - kwargs['creationflags'] = subprocess.CREATE_NEW_PROCESS_GROUP - return subprocess.Popen(*args, **kwargs) - -def send_signal(proc, sig): - if have_unix_process_groups: # UNIX - # For compatibility with GNU timeout, pre-send the signal to just the monitored process - os.kill(proc.pid, sig) - # Send signal to entire group - os.killpg(proc.pid, sig) - # For compatibility with GNU Timeout, send a SIGCONT after the signal - # (so delivery has a chance to occur even for stopped processes) - if sig != signal.SIGKILL and sig != signal.SIGCONT: - os.kill(proc.pid, signal.SIGCONT) - elif have_windows_process_groups: # Windows with Python 2.7 or better - os.kill(proc.pid, sig) # Becuase CREATE_NEW_PROCESS_GROUP, will go to entire group - else: # Windows with Python 2.6-- CREATE_NEW_PROCESS_GROUP not supported - proc.send_signal(sig) # No way to contact group, just kill process - -### Utility functions - -def attemptDelete(path): - try: - os.remove(path) - except OSError: - pass - -def attemptLines(path): - try: - with open(path) as f: - return map(lambda s: s.strip('\r\n'), f.readlines()) - except (OSError, IOError): - return [] - -def attemptFirstLine(path): - lines = attemptLines(path) - if lines: - return lines[0] - return None - -def posixtime(): # Amazingly, time.time() does not guarantee an epoch in the docs. However this does: - return calendar.timegm(time.gmtime()) - -failcount = {} -def failure_may_retry(test): - if test not in failcount: - failcount[test] = 0 - failcount[test] += 1 - return failcount[test] < failmax - -def verbose_print(arg): - if (verbose): - print(arg) - -def failure_annotate(test): - return "%s (failure #%d of %d allowed)" % (test, failcount[test], failmax) - -def pluralize(lst): - return "s" if len(lst) > 1 else "" - -### Run command - -def run(): # Returns exit code - resume_after = [] - retry_these = [] - ever_completed = False - died_politely = False - proc = None - code = None - - # Set up logging - log = {DATE_JSON: posixtime(), COUNT_JSON:0, LIMIT_JSON:failmax, SUPPORT_JSON:False, - INVOKE_JSON: " ".join(command)} - - def log_value(key, set=None, add=None, target=log): # Call to add toplevel value to log - if add is not None: - if key not in target: - target[key] = 0 - target[key] += add - else: - target[key] = set - - def log_test(testname, key, set=None, add=None): # Call to add test-case-level value to log - if TESTS_JSON not in log: - log[TESTS_JSON] = {} - if testname not in log[TESTS_JSON]: - log[TESTS_JSON][testname] = {} - log_value(key, set=set, add=add, target=log[TESTS_JSON][testname]) - - # Ready to run tests - try: - while True: - env = copy.copy(global_env) - if ever_completed: - retry_next = [] - else: # Persist reported failures list until first non-crash run - retry_next = retry_these - - log_value(COUNT_JSON, add=1) - - # Prepare environment/filesystem - if babysitting: - for key in [CURRENT_TEST_KEY, RAN_TEST_KEY, FAILED_TEST_KEY]: - attemptDelete(env[key]) - if resume_after: - env[RUN_KEY] = ";".join(resume_after) - env[RUN_MODE_KEY] = "EXCLUDE" - elif retry_these: - env[RUN_KEY] = ";".join(retry_these) - env[RUN_MODE_KEY] = "RUN" - - # Run test suite - try: - proc = popen(command, env=env) - except OSError: - died_politely = True - sys.stderr.write("%s: Could not execute command `%s`\n" % (scriptname, command[0])) - sys.exit(127) - - code = wait(proc, duration) - timed_out = code is None - if timed_out: # Initial timeout - send_signal(proc, timeout_signal) - if kill_after is not None: # Kill-after timeout - code = wait(proc, kill_after) - if code is None: - send_signal(proc, signal.SIGKILL) - code = proc.wait() # Waits forever - sys.stderr.write("%s: Command `%s` timed out\n" % (scriptname, command[0])) - died_politely = True - - # The test suite has now run, and what happens next varies: - # 1. The suite either completed fully without failures, or timed out: Just quit. - # 2. The suite crashed (halted without completing): - # Remember any failures for later and rerun, using a blacklist of testcases we have completed. - # 3. The suite completed, but there were failures reported: - # Rerun, using a whitelist of only reported-failed testcases. - # 4. The suite crashed partway through a run with a whitelist: - # Rerun, using a whitelist consisting of the previous whitelist minus successful testcases. - - crashed_at = attemptFirstLine(env[CURRENT_TEST_KEY]) - failed_tests = attemptLines(env[FAILED_TEST_KEY]) - ran_tests = attemptLines(env[RAN_TEST_KEY]) - bailout = False - - if crashed_at or failed_tests or ran_tests: # Test suite follows the babysitter protocol - log_value(SUPPORT_JSON, True) - - if not crashed_at and not ever_completed: # The resume_after whitelist is only - resume_after = [] # used before the first noncrashing run - ever_completed = True - - if timed_out: # Currently no retries after timeout - bailout = True - code = 124 # See GNU timeout manpage - - if code or crashed_at: # Process failures - # Handle crash failures - if crashed_at and not timed_out: - log_test(crashed_at, TEST_CRASH_FAILURES, add=1) - if not crash_resuming: - bailout = True - - if failure_may_retry(crashed_at): - if ever_completed: # Rerun with whitelist next time - for test in retry_these: # Prepopulate with last whitelist minus run testcases - if test == crashed_at or test not in ran_tests: # (plus crashed testcase) - retry_next.append(test) - else: # Rerun with blacklist next time - for test in ran_tests: # Add testcases we just ran to blacklist - if test != crashed_at: # (except for the crashed testcase) - resume_after.append(test) - else: - bailout = True - - # Handle reported failures - for test in failed_tests: - log_test(test, TEST_FAILURES, add=1) - if failure_may_retry(test): - retry_next.append(test) - else: - bailout = True - - # Report human-readable failures for stdout log. - message = "%s:" % (scriptname) - if timed_out: - message += " Saw timeout in test case %s (never allowed)." % (crashed_at) - elif crashed_at: - message += " Saw crash in test case %s." % (failure_annotate(crashed_at)) - if failed_tests: - message += " Saw test failure in test case%s %s." % (pluralize(failed_tests), "; ".join(map(failure_annotate, failed_tests))) - if not (timed_out or crashed_at or failed_tests): - message += " Test suite terminated with code %d, " % (code) - if log[SUPPORT_JSON]: - message += "but failure did not occur during a test case. Halting." - else: - message += "and suite cannot report test case data. Halting." - elif bailout: - message += " Will halt testing." - print(message) - - if bailout or not (resume_after or retry_next): # If not retrying - return code - - # If we got here, a retry is occurring - retry_these = retry_next - - # Report human-readable retry notice for stdout log. - message = "%s: Will rerun test suite" % (scriptname) - if log[COUNT_JSON] > 1: - message += " (iteration #%d)" % (log[COUNT_JSON]) - if resume_after: - message += ", resuming at crashed testcase %s." % (crashed_at) - else: - message += ", running previously failed testcase%s: %s." % (pluralize(retry_these), "; ".join(retry_these)) - print(message) - finally: - # Emergency: Ensure command does not outlive this script - if proc is not None and not died_politely: - send_signal(proc, signal.SIGKILL) - - # Write out logs - log_value(FINAL_CODE_JSON, "EXCEPTION" if code is None else code) - if logging: - with open(logfile, "a") as f: - f.write(json.dumps(log) + os.linesep) - -sys.exit( run() ) diff --git a/scripts/ci/babysitter b/scripts/ci/babysitter new file mode 100755 index 00000000000..f8b5a6c345c --- /dev/null +++ b/scripts/ci/babysitter @@ -0,0 +1,409 @@ +#!/usr/bin/env python + +# Mimics GNU timeout, but does some fancy tracking based on custom features in mono nunit24. + +import argparse +import subprocess +import re +import signal +import time +import sys +import os.path +import copy +import tempfile +import calendar +import json + +### Constants + +# Here is how the communication with nunit works. It has to work with two constraints: +# - We don't invoke nunit directly. We invoke some Makefile which invokes some other Makefile +# and at some point down the line someone calls nunit. +# - nunit has to be able to report back to us even if (especially if) it terminates improperly. +# To deal with all this, communication babysitter->nunit is done by environment variables, +# and communication nunit->babysitter is done by leaving behind files in known locations. + +# Filenames + +CURRENT_TEST_FILE = "babysitter_report_current_test_file.txt" +RAN_TEST_FILE = "babysitter_report_ran_test_file.txt" +FAILED_TEST_FILE = "babysitter_report_failed_test_file.txt" +LOGGING_FILE = "babysitter_report.json_lines" + +# Environment keys + +# Keys used for Babysitter<->Nunit IPC +CURRENT_TEST_KEY = 'MONO_BABYSITTER_NUNIT_CURRENT_TEST_FILE' # Tell nunit where to leave files +RAN_TEST_KEY = 'MONO_BABYSITTER_NUNIT_RAN_TEST_FILE' +FAILED_TEST_KEY = 'MONO_BABYSITTER_NUNIT_FAILED_TEST_FILE' +RUN_KEY = 'MONO_BABYSITTER_NUNIT_RUN_TEST' # Semicolon-separated list of test names +RUN_MODE_KEY = 'MONO_BABYSITTER_NUNIT_RUN_MODE' # Equal to either RUN or AFTER + +# Keys used for script configuration (see --help text) +LOG_FILE_KEY = 'MONO_BABYSITTER_LOG_FILE' # Path +RETRY_KEY = 'MONO_BABYSITTER_RETRY' # Equal to an integer +VERBOSE_KEY = 'MONO_BABYSITTER_VERBOSE' # "Undocumented"-- used for debugging babysitter + +# JSON keys + +DATE_JSON = 'date' # POSIX timestamp of test suite run +INVOKE_JSON = 'invocation' +COUNT_JSON = 'iteration' # How many times was command executed? +LIMIT_JSON = 'failure_max' +SUPPORT_JSON = 'babysitter_protocol' # Was the test suite running with a babysitter-aware nunit? +FINAL_CODE_JSON = 'final_code' +TESTS_JSON = 'tests' # Holds dictionary of (test case name)->(dict with TEST_ keys below) +TEST_FAILURES = 'normal_failures' +TEST_CRASH_FAILURES = 'crash_failures' +TEST_TIMEOUT_FAILURES = 'timeout_failures' + +### Interpret arguments + +scriptname = sys.argv[0] + +# This is very silly: This program needs to switch from argparse to manual parsing +# after the second positional argument, but argparse doesn't let you do this. +# I work around this with a custom subclass which can selectively swallow errors: +class Hesitate(Exception): + pass +class HesitantParser(argparse.ArgumentParser): + def __init__(s, *args, **kwargs): + s.hesitating = True # Errors will be swallowed until this is set false + argparse.ArgumentParser.__init__(s, *args, **kwargs) + def error(s, *args, **kwargs): + if s.hesitating: + raise Hesitate() # Bail out before errors are printed. + argparse.ArgumentParser.error(s, *args, **kwargs) + +# Define args +argparser = HesitantParser(description="""\ +Run a test suite with a timeout.\n +Durations are floating point numbers followed by an optional unit:\n +'s' for seconds (the default) +'m' for minutes +'h' for hours +'d' for days\n +supported environment variables: + %s: File to write logs to (as line-delimited JSON) + %s: If set to a number, failed test cases will be rerun this many times (NUnit test suites only)""" % + (LOG_FILE_KEY, RETRY_KEY), + formatter_class=argparse.RawTextHelpFormatter) +argparser.add_argument('-s', '--signal', dest='signal', metavar='signal', default='TERM', + help="Send this signal to the command on timeout, instead of TERM.") +argparser.add_argument('-k', '--kill-after-duration', dest='kill_after', metavar='duration', + help="If process continues running after signal, send KILL after this duration.") +argparser.add_argument('duration', + help="Time to run before sending signal.") +argparser.add_argument('command', nargs="+", help="Command+args to run.") + +# Repeatedly parse the given args until we find the shortest prefix for which parsing succeeds. +argc = len(sys.argv) +extra_args = [] +for limit in range(1,argc+1): + try: + if limit == argc: # On the final pass, parse for real + argparser.hesitating = False + args = argparser.parse_args(sys.argv[1:limit]) + # If we're still here, parse_args succeeded. + # The final parsed arg is the command; remaining argv items are its args. + extra_args = sys.argv[limit:] + break + except Hesitate: # Before the final pass, if parse_args fails, skip + pass + +argparser.hesitating = False # Just act like a normal argparser from here + +durationre = re.compile(r'(\d+)([smhd]?)') +def parse_duration(duration): # Accept units + match = durationre.match(duration) + if not match: + argparser.error("Could not understand duration %s" % duration) + time, units = match.group(1), match.group(2) + time = int(time) + if units == 'm': + time *= 60 + elif units == 'h': + time *= 60*60 + elif units == 'd': + time *= 60*60*24 + return time + +def parse_signal(sig): # Accept names + if sig.isdigit(): + return int(sig) + for k,v in signal.__dict__.iteritems(): + if k == ("SIG%s" % sig): + return v + argparser.error("Could not understand signal name %s" % sig) + +# Final interpretation of arguments +duration = parse_duration(args.duration) +kill_after = parse_duration(args.kill_after) if args.kill_after is not None else None +timeout_signal = parse_signal(args.signal) +command = args.command + extra_args + +# Process environment +global_env = copy.deepcopy( os.environ ) + +verbose = VERBOSE_KEY in global_env +logging = LOG_FILE_KEY in global_env +logfile = global_env[LOG_FILE_KEY] if logging else None +crash_resuming = True # TODO: Consider exposing this option, or adding a retry_on_crash option. +failmax = int(global_env[RETRY_KEY]) if RETRY_KEY in global_env else 0 +babysitting = True # If false, babysitter becomes a timeout clone with no env manipulation or anything. +if babysitting: + babysitter_dir = tempfile.mkdtemp() + global_env[CURRENT_TEST_KEY] = os.path.join(babysitter_dir, CURRENT_TEST_FILE) + global_env[RAN_TEST_KEY] = os.path.join(babysitter_dir, RAN_TEST_FILE) + global_env[FAILED_TEST_KEY] = os.path.join(babysitter_dir, FAILED_TEST_FILE) + +have_unix_process_groups = 'killpg' in os.__dict__ +have_windows_process_groups = 'CREATE_NEW_PROCESS_GROUP' in subprocess.__dict__ + +### Timeout implementation + +def wait(proc, duration): + # TODO: If we detect Python 3.3, Popen objects have a wait(timeout) method we can use + start = time.time() + while True: + code = proc.poll() + if code is not None: + return code + if time.time()-start > duration: + return None + time.sleep(0.05) + +# Popen and send_signal can't be called in their basic forms because we want to +# send signals to all children, not just to the immediately spawned process. +# Unfortunately the way to do this varies by operating system. +def popen(*args, **kwargs): + if have_unix_process_groups: # Call function on spawn to become process group leader + kwargs['preexec_fn'] = os.setsid + elif have_windows_process_groups: # Set magic flag for Windows process groups + kwargs['creationflags'] = subprocess.CREATE_NEW_PROCESS_GROUP + return subprocess.Popen(*args, **kwargs) + +def send_signal(proc, sig): + if have_unix_process_groups: # UNIX + # For compatibility with GNU timeout, pre-send the signal to just the monitored process + os.kill(proc.pid, sig) + # Send signal to entire group + os.killpg(proc.pid, sig) + # For compatibility with GNU Timeout, send a SIGCONT after the signal + # (so delivery has a chance to occur even for stopped processes) + if sig != signal.SIGKILL and sig != signal.SIGCONT: + os.kill(proc.pid, signal.SIGCONT) + elif have_windows_process_groups: # Windows with Python 2.7 or better + os.kill(proc.pid, sig) # Becuase CREATE_NEW_PROCESS_GROUP, will go to entire group + else: # Windows with Python 2.6-- CREATE_NEW_PROCESS_GROUP not supported + proc.send_signal(sig) # No way to contact group, just kill process + +### Utility functions + +def attemptDelete(path): + try: + os.remove(path) + except OSError: + pass + +def attemptLines(path): + try: + with open(path) as f: + return map(lambda s: s.strip('\r\n'), f.readlines()) + except (OSError, IOError): + return [] + +def attemptFirstLine(path): + lines = attemptLines(path) + if lines: + return lines[0] + return None + +def posixtime(): # Amazingly, time.time() does not guarantee an epoch in the docs. However this does: + return calendar.timegm(time.gmtime()) + +failcount = {} +def failure_may_retry(test): + if test not in failcount: + failcount[test] = 0 + failcount[test] += 1 + return failcount[test] < failmax + +def verbose_print(arg): + if (verbose): + print(arg) + +def failure_annotate(test): + return "%s (failure #%d of %d allowed)" % (test, failcount[test], failmax) + +def pluralize(lst): + return "s" if len(lst) > 1 else "" + +### Run command + +def run(): # Returns exit code + resume_after = [] + retry_these = [] + ever_completed = False + died_politely = False + proc = None + code = None + + # Set up logging + log = {DATE_JSON: posixtime(), COUNT_JSON:0, LIMIT_JSON:failmax, SUPPORT_JSON:False, + INVOKE_JSON: " ".join(command)} + + def log_value(key, set=None, add=None, target=log): # Call to add toplevel value to log + if add is not None: + if key not in target: + target[key] = 0 + target[key] += add + else: + target[key] = set + + def log_test(testname, key, set=None, add=None): # Call to add test-case-level value to log + if TESTS_JSON not in log: + log[TESTS_JSON] = {} + if testname not in log[TESTS_JSON]: + log[TESTS_JSON][testname] = {} + log_value(key, set=set, add=add, target=log[TESTS_JSON][testname]) + + # Ready to run tests + try: + while True: + env = copy.copy(global_env) + if ever_completed: + retry_next = [] + else: # Persist reported failures list until first non-crash run + retry_next = retry_these + + log_value(COUNT_JSON, add=1) + + # Prepare environment/filesystem + if babysitting: + for key in [CURRENT_TEST_KEY, RAN_TEST_KEY, FAILED_TEST_KEY]: + attemptDelete(env[key]) + if resume_after: + env[RUN_KEY] = ";".join(resume_after) + env[RUN_MODE_KEY] = "EXCLUDE" + elif retry_these: + env[RUN_KEY] = ";".join(retry_these) + env[RUN_MODE_KEY] = "RUN" + + # Run test suite + try: + proc = popen(command, env=env) + except OSError: + died_politely = True + sys.stderr.write("%s: Could not execute command `%s`\n" % (scriptname, command[0])) + sys.exit(127) + + code = wait(proc, duration) + timed_out = code is None + if timed_out: # Initial timeout + send_signal(proc, timeout_signal) + if kill_after is not None: # Kill-after timeout + code = wait(proc, kill_after) + if code is None: + send_signal(proc, signal.SIGKILL) + code = proc.wait() # Waits forever + sys.stderr.write("%s: Command `%s` timed out\n" % (scriptname, command[0])) + died_politely = True + + # The test suite has now run, and what happens next varies: + # 1. The suite either completed fully without failures, or timed out: Just quit. + # 2. The suite crashed (halted without completing): + # Remember any failures for later and rerun, using a blacklist of testcases we have completed. + # 3. The suite completed, but there were failures reported: + # Rerun, using a whitelist of only reported-failed testcases. + # 4. The suite crashed partway through a run with a whitelist: + # Rerun, using a whitelist consisting of the previous whitelist minus successful testcases. + + crashed_at = attemptFirstLine(env[CURRENT_TEST_KEY]) + failed_tests = attemptLines(env[FAILED_TEST_KEY]) + ran_tests = attemptLines(env[RAN_TEST_KEY]) + bailout = False + + if crashed_at or failed_tests or ran_tests: # Test suite follows the babysitter protocol + log_value(SUPPORT_JSON, True) + + if not crashed_at and not ever_completed: # The resume_after whitelist is only + resume_after = [] # used before the first noncrashing run + ever_completed = True + + if timed_out: # Currently no retries after timeout + bailout = True + code = 124 # See GNU timeout manpage + + if code or crashed_at: # Process failures + # Handle crash failures + if crashed_at and not timed_out: + log_test(crashed_at, TEST_CRASH_FAILURES, add=1) + if not crash_resuming: + bailout = True + + if failure_may_retry(crashed_at): + if ever_completed: # Rerun with whitelist next time + for test in retry_these: # Prepopulate with last whitelist minus run testcases + if test == crashed_at or test not in ran_tests: # (plus crashed testcase) + retry_next.append(test) + else: # Rerun with blacklist next time + for test in ran_tests: # Add testcases we just ran to blacklist + if test != crashed_at: # (except for the crashed testcase) + resume_after.append(test) + else: + bailout = True + + # Handle reported failures + for test in failed_tests: + log_test(test, TEST_FAILURES, add=1) + if failure_may_retry(test): + retry_next.append(test) + else: + bailout = True + + # Report human-readable failures for stdout log. + message = "%s:" % (scriptname) + if timed_out: + message += " Saw timeout in test case %s (never allowed)." % (crashed_at) + elif crashed_at: + message += " Saw crash in test case %s." % (failure_annotate(crashed_at)) + if failed_tests: + message += " Saw test failure in test case%s %s." % (pluralize(failed_tests), "; ".join(map(failure_annotate, failed_tests))) + if not (timed_out or crashed_at or failed_tests): + message += " Test suite terminated with code %d, " % (code) + if log[SUPPORT_JSON]: + message += "but failure did not occur during a test case. Halting." + else: + message += "and suite cannot report test case data. Halting." + elif bailout: + message += " Will halt testing." + print(message) + + if bailout or not (resume_after or retry_next): # If not retrying + return code + + # If we got here, a retry is occurring + retry_these = retry_next + + # Report human-readable retry notice for stdout log. + message = "%s: Will rerun test suite" % (scriptname) + if log[COUNT_JSON] > 1: + message += " (iteration #%d)" % (log[COUNT_JSON]) + if resume_after: + message += ", resuming at crashed testcase %s." % (crashed_at) + else: + message += ", running previously failed testcase%s: %s." % (pluralize(retry_these), "; ".join(retry_these)) + print(message) + finally: + # Emergency: Ensure command does not outlive this script + if proc is not None and not died_politely: + send_signal(proc, signal.SIGKILL) + + # Write out logs + log_value(FINAL_CODE_JSON, "EXCEPTION" if code is None else code) + if logging: + with open(logfile, "a") as f: + f.write(json.dumps(log) + os.linesep) + +sys.exit( run() ) diff --git a/scripts/ci/run-jenkins.sh b/scripts/ci/run-jenkins.sh new file mode 100755 index 00000000000..a8a93ae26ef --- /dev/null +++ b/scripts/ci/run-jenkins.sh @@ -0,0 +1,101 @@ +#!/bin/bash -e + +TESTCMD=`dirname "${BASH_SOURCE[0]}"`/run-step.sh + +export TEST_HARNESS_VERBOSE=1 + +if [[ ${label} == 'osx-i386' ]]; then EXTRA_CONF_FLAGS="--with-libgdiplus=/Library/Frameworks/Mono.framework/Versions/Current/lib/libgdiplus.dylib --enable-nls=no --build=i386-apple-darwin11.2.0"; fi +if [[ ${label} == 'osx-amd64' ]]; then EXTRA_CONF_FLAGS="--with-libgdiplus=/Library/Frameworks/Mono.framework/Versions/Current/lib/libgdiplus.dylib --enable-nls=no"; fi +if [[ ${label} == 'w32' ]]; then PLATFORM=Win32; EXTRA_CONF_FLAGS="--host=i686-pc-mingw32"; export MONO_EXECUTABLE="`cygpath -u ${WORKSPACE}\\\msvc\\\Win32\\\bin\\\Release_SGen\\\mono-sgen.exe`";fi +if [[ ${label} == 'w64' ]]; then PLATFORM=x64; EXTRA_CONF_FLAGS="--host=i686-pc-mingw32"; export MONO_EXECUTABLE="`cygpath -u ${WORKSPACE}\\\msvc\\\x64\\\bin\\\Release_SGen\\\mono-sgen.exe`"; fi + +${TESTCMD} --label=configure --timeout=60m --fatal ./autogen.sh --with-monodroid --with-monotouch --with-monotouch_watch --with-monotouch_tv --with-xammac --with-mobile_static $EXTRA_CONF_FLAGS +if [[ ${label} == w* ]]; + then + ${TESTCMD} --label=make-msvc --timeout=60m --fatal /cygdrive/c/Program\ Files\ \(x86\)/MSBuild/12.0/Bin/MSBuild.exe /p:Platform=${PLATFORM} /p:Configuration=Release msvc/mono.sln + ${TESTCMD} --label=make-msvc-sgen --timeout=60m --fatal /cygdrive/c/Program\ Files\ \(x86\)/MSBuild/12.0/Bin/MSBuild.exe /p:Platform=${PLATFORM} /p:Configuration=Release_SGen msvc/mono.sln +fi +${TESTCMD} --label=make --timeout=300m --fatal make -w V=1 +if [[ -n "${ghprbPullId}" ]] && [[ ${label} == w* ]]; + then + exit 0 + # we don't run the test suite on Windows PRs, we just ensure the build succeeds, so end here +fi +${TESTCMD} --label=mini --timeout=5m make -w -C mono/mini -k check +${TESTCMD} --label=runtime --timeout=120m make -w -C mono/tests -k test-wrench V=1 CI=1 +${TESTCMD} --label=corlib --timeout=30m make -w -C mcs/class/corlib run-test +${TESTCMD} --label=verify --timeout=15m make -w -C runtime mcs-compileall +${TESTCMD} --label=profiler --timeout=30m make -w -C mono/profiler -k check +${TESTCMD} --label=compiler --timeout=30m make -w -C mcs/tests run-test +${TESTCMD} --label=compiler-errors --timeout=30m make -w -C mcs/errors run-test +${TESTCMD} --label=System --timeout=10m make -w -C mcs/class/System run-test +${TESTCMD} --label=System.XML --timeout=5m make -w -C mcs/class/System.XML run-test +${TESTCMD} --label=Mono.Security --timeout=5m make -w -C mcs/class/Mono.Security run-test +${TESTCMD} --label=System.Security --timeout=5m make -w -C mcs/class/System.Security run-test +${TESTCMD} --label=System.Drawing --timeout=5m make -w -C mcs/class/System.Drawing run-test +if [[ ${label} == osx-* ]] +then ${TESTCMD} --label=Windows.Forms --skip +else ${TESTCMD} --label=Windows.Forms --timeout=5m make -w -C mcs/class/System.Windows.Forms run-test +fi +${TESTCMD} --label=System.Data --timeout=5m make -w -C mcs/class/System.Data run-test +${TESTCMD} --label=System.Data.OracleClient --timeout=5m make -w -C mcs/class/System.Data.OracleClient run-test +${TESTCMD} --label=System.Design --timeout=5m make -w -C mcs/class/System.Design run-test +${TESTCMD} --label=Mono.Posix --timeout=5m make -w -C mcs/class/Mono.Posix run-test +${TESTCMD} --label=System.Web --timeout=30m make -w -C mcs/class/System.Web run-test +${TESTCMD} --label=System.Web.Services --timeout=5m make -w -C mcs/class/System.Web.Services run-test +${TESTCMD} --label=System.Runtime.SFS --timeout=5m make -w -C mcs/class/System.Runtime.Serialization.Formatters.Soap run-test +${TESTCMD} --label=System.Runtime.Remoting --timeout=5m make -w -C mcs/class/System.Runtime.Remoting run-test +${TESTCMD} --label=Cscompmgd --timeout=5m make -w -C mcs/class/Cscompmgd run-test +${TESTCMD} --label=Commons.Xml.Relaxng --timeout=5m make -w -C mcs/class/Commons.Xml.Relaxng run-test +${TESTCMD} --label=System.ServiceProcess --timeout=5m make -w -C mcs/class/System.ServiceProcess run-test +${TESTCMD} --label=I18N.CJK --timeout=5m make -w -C mcs/class/I18N/CJK run-test +${TESTCMD} --label=I18N.West --timeout=5m make -w -C mcs/class/I18N/West run-test +${TESTCMD} --label=I18N.MidEast --timeout=5m make -w -C mcs/class/I18N/MidEast run-test +${TESTCMD} --label=System.DirectoryServices --timeout=5m make -w -C mcs/class/System.DirectoryServices run-test +${TESTCMD} --label=Microsoft.Build.Engine --timeout=5m make -w -C mcs/class/Microsoft.Build.Engine run-test +${TESTCMD} --label=Microsoft.Build.Framework --timeout=5m make -w -C mcs/class/Microsoft.Build.Framework run-test +${TESTCMD} --label=Microsoft.Build.Tasks --timeout=5m make -w -C mcs/class/Microsoft.Build.Tasks run-test +${TESTCMD} --label=Microsoft.Build.Utilities --timeout=5m make -w -C mcs/class/Microsoft.Build.Utilities run-test +${TESTCMD} --label=Mono.C5 --timeout=5m make -w -C mcs/class/Mono.C5 run-test +${TESTCMD} --label=System.Configuration --timeout=5m make -w -C mcs/class/System.Configuration run-test +${TESTCMD} --label=System.Transactions --timeout=5m make -w -C mcs/class/System.Transactions run-test +${TESTCMD} --label=System.Web.Extensions --timeout=5m make -w -C mcs/class/System.Web.Extensions run-test +${TESTCMD} --label=System.Core --timeout=15m make -w -C mcs/class/System.Core run-test +${TESTCMD} --label=symbolicate --timeout=60m make -w -C mcs/tools/mono-symbolicate check +${TESTCMD} --label=System.Xml.Linq --timeout=5m make -w -C mcs/class/System.Xml.Linq run-test +${TESTCMD} --label=System.Data.DSE --timeout=5m make -w -C mcs/class/System.Data.DataSetExtensions run-test +${TESTCMD} --label=System.Web.Abstractions --timeout=5m make -w -C mcs/class/System.Web.Abstractions run-test +${TESTCMD} --label=System.Web.Routing --timeout=5m make -w -C mcs/class/System.Web.Routing run-test +${TESTCMD} --label=System.Runtime.Serialization --timeout=5m make -w -C mcs/class/System.Runtime.Serialization run-test +${TESTCMD} --label=System.IdentityModel --timeout=5m make -w -C mcs/class/System.IdentityModel run-test +${TESTCMD} --label=System.ServiceModel --timeout=15m make -w -C mcs/class/System.ServiceModel run-test +${TESTCMD} --label=System.ServiceModel.Web --timeout=5m make -w -C mcs/class/System.ServiceModel.Web run-test +${TESTCMD} --label=System.Web.Extensions-standalone --timeout=5m make -w -C mcs/class/System.Web.Extensions run-standalone-test +${TESTCMD} --label=System.ComponentModel.DataAnnotations --timeout=5m make -w -C mcs/class/System.ComponentModel.DataAnnotations run-test +${TESTCMD} --label=Mono.CodeContracts --timeout=5m make -w -C mcs/class/Mono.CodeContracts run-test +${TESTCMD} --label=System.Runtime.Caching --timeout=5m make -w -C mcs/class/System.Runtime.Caching run-test +${TESTCMD} --label=System.Data.Services --timeout=5m make -w -C mcs/class/System.Data.Services run-test +${TESTCMD} --label=System.Web.DynamicData --timeout=5m make -w -C mcs/class/System.Web.DynamicData run-test +${TESTCMD} --label=Mono.CSharp --timeout=5m make -w -C mcs/class/Mono.CSharp run-test +${TESTCMD} --label=WindowsBase --timeout=5m make -w -C mcs/class/WindowsBase run-test +${TESTCMD} --label=System.Numerics --timeout=5m make -w -C mcs/class/System.Numerics run-test +${TESTCMD} --label=System.Runtime.DurableInstancing --timeout=5m make -w -C mcs/class/System.Runtime.DurableInstancing run-test +${TESTCMD} --label=System.ServiceModel.Discovery --timeout=5m make -w -C mcs/class/System.ServiceModel.Discovery run-test +${TESTCMD} --label=System.Xaml --timeout=5m make -w -C mcs/class/System.Xaml run-test +${TESTCMD} --label=System.Net.Http --timeout=5m make -w -C mcs/class/System.Net.Http run-test +${TESTCMD} --label=System.Json --timeout=5m make -w -C mcs/class/System.Json run-test +${TESTCMD} --label=System.Threading.Tasks.Dataflow --timeout=5m make -w -C mcs/class/System.Threading.Tasks.Dataflow run-test +${TESTCMD} --label=Mono.Debugger.Soft --timeout=5m make -w -C mcs/class/Mono.Debugger.Soft run-test +${TESTCMD} --label=Microsoft.Build --timeout=5m make -w -C mcs/class/Microsoft.Build run-test +${TESTCMD} --label=monodoc --timeout=10m make -w -C mcs/tools/mdoc run-test +${TESTCMD} --label=Microsoft.Build-12 --timeout=10m make -w -C mcs/class/Microsoft.Build run-test PROFILE=xbuild_12 +${TESTCMD} --label=Microsoft.Build.Engine-12 --timeout=60m make -w -C mcs/class/Microsoft.Build.Engine run-test PROFILE=xbuild_12 +${TESTCMD} --label=Microsoft.Build.Framework-12 --timeout=60m make -w -C mcs/class/Microsoft.Build.Framework run-test PROFILE=xbuild_12 +${TESTCMD} --label=Microsoft.Build.Tasks-12 --timeout=60m make -w -C mcs/class/Microsoft.Build.Tasks run-test PROFILE=xbuild_12 +${TESTCMD} --label=Microsoft.Build.Utilities-12 --timeout=60m make -w -C mcs/class/Microsoft.Build.Utilities run-test PROFILE=xbuild_12 +${TESTCMD} --label=Microsoft.Build-14 --timeout=60m make -w -C mcs/class/Microsoft.Build run-test PROFILE=xbuild_14 +${TESTCMD} --label=Microsoft.Build.Engine-14 --timeout=60m make -w -C mcs/class/Microsoft.Build.Engine run-test PROFILE=xbuild_14 +${TESTCMD} --label=Microsoft.Build.Framework-14 --timeout=60m make -w -C mcs/class/Microsoft.Build.Framework run-test PROFILE=xbuild_14 +${TESTCMD} --label=Microsoft.Build.Tasks-14 --timeout=60m make -w -C mcs/class/Microsoft.Build.Tasks run-test PROFILE=xbuild_14 +${TESTCMD} --label=Microsoft.Build.Utilities-14 --timeout=60m make -w -C mcs/class/Microsoft.Build.Utilities run-test PROFILE=xbuild_14 +rm -fr /tmp/jenkins-temp-aspnet* diff --git a/scripts/ci/run-step.sh b/scripts/ci/run-step.sh new file mode 100755 index 00000000000..2e47f855184 --- /dev/null +++ b/scripts/ci/run-step.sh @@ -0,0 +1,56 @@ +#!/bin/bash -e +TIMEOUTCMD=`dirname "${BASH_SOURCE[0]}"`/babysitter +export MONO_BABYSITTER_LOG_FILE=babysitter_report.json_lines + +helptext () +{ + echo "run-step.sh {--label=LABEL} {--skip|--timeout=TIMEOUT [--fatal]} command to run with arguments" +} + +for i in "$@" +do +case $i in + --help) + helptext + exit 0 + ;; + --label=*) + LABEL="${i#*=}" + shift # past argument=value + ;; + --timeout=*) + TIMEOUT="${i#*=}" + shift # past argument=value + ;; + --fatal) + FATAL="true" + shift # past argument + ;; + --skip) + SKIP="true" + shift # past argument + ;; + *) + # unknown option, assume just part of cmdline + ;; +esac +done +if [ -n "${SKIP}" ] && [ -z "${LABEL}" ] + then helptext + exit 1 +fi +if [ -n "${SKIP}" ] + then echo -e "*** start: ${LABEL}\n*** end(0): ${LABEL}: \e[45mSkipped\e[0m" + exit 0 +fi +if [ -z "${LABEL}" ] || [ -z "${TIMEOUT}" ] + then helptext + exit 1 +fi +STARTTIME=`date +%s` +echo "*** start: ${LABEL}" +if [ -n "${FATAL}" ]; then + ${TIMEOUTCMD} --signal=ABRT --kill-after=60s ${TIMEOUT} $@ && echo -e "*** end($(echo $(date +%s) - ${STARTTIME} | bc)): ${LABEL}: \e[42mPassed\e[0m" || (echo -e "*** end($(echo $(date +%s) - ${STARTTIME} | bc)): ${LABEL}: \e[41mFailed\e[0m" && exit 1) +else + ${TIMEOUTCMD} --signal=ABRT --kill-after=60s ${TIMEOUT} $@ && echo -e "*** end($(echo $(date +%s) - ${STARTTIME} | bc)): ${LABEL}: \e[42mPassed\e[0m" || echo -e "*** end($(echo $(date +%s) - ${STARTTIME} | bc)): ${LABEL}: \e[43mUnstable\e[0m" +fi -- cgit v1.2.3 From a9113e917eb34b4caeac1b0c2cba24663b08e969 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20K=C3=B6plinger?= Date: Wed, 6 Apr 2016 22:34:52 +0200 Subject: [ci] Put windows msbuild command in quotes --- scripts/ci/run-jenkins.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/ci/run-jenkins.sh b/scripts/ci/run-jenkins.sh index a8a93ae26ef..2507bc9777f 100755 --- a/scripts/ci/run-jenkins.sh +++ b/scripts/ci/run-jenkins.sh @@ -12,8 +12,8 @@ if [[ ${label} == 'w64' ]]; then PLATFORM=x64; EXTRA_CONF_FLAGS="--host=i686-pc- ${TESTCMD} --label=configure --timeout=60m --fatal ./autogen.sh --with-monodroid --with-monotouch --with-monotouch_watch --with-monotouch_tv --with-xammac --with-mobile_static $EXTRA_CONF_FLAGS if [[ ${label} == w* ]]; then - ${TESTCMD} --label=make-msvc --timeout=60m --fatal /cygdrive/c/Program\ Files\ \(x86\)/MSBuild/12.0/Bin/MSBuild.exe /p:Platform=${PLATFORM} /p:Configuration=Release msvc/mono.sln - ${TESTCMD} --label=make-msvc-sgen --timeout=60m --fatal /cygdrive/c/Program\ Files\ \(x86\)/MSBuild/12.0/Bin/MSBuild.exe /p:Platform=${PLATFORM} /p:Configuration=Release_SGen msvc/mono.sln + ${TESTCMD} --label=make-msvc --timeout=60m --fatal "/cygdrive/c/Program\ Files\ \(x86\)/MSBuild/12.0/Bin/MSBuild.exe" /p:Platform=${PLATFORM} /p:Configuration=Release msvc/mono.sln + ${TESTCMD} --label=make-msvc-sgen --timeout=60m --fatal "/cygdrive/c/Program\ Files\ \(x86\)/MSBuild/12.0/Bin/MSBuild.exe" /p:Platform=${PLATFORM} /p:Configuration=Release_SGen msvc/mono.sln fi ${TESTCMD} --label=make --timeout=300m --fatal make -w V=1 if [[ -n "${ghprbPullId}" ]] && [[ ${label} == w* ]]; -- cgit v1.2.3 From 4f09d58203580d01355d12f73148d73bcbb22b1a Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Thu, 7 Apr 2016 15:28:59 +0100 Subject: [ci] Escape run-step.sh invocations, for paths with spaces in --- scripts/ci/run-step.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/ci/run-step.sh b/scripts/ci/run-step.sh index 2e47f855184..7067a3f8a58 100755 --- a/scripts/ci/run-step.sh +++ b/scripts/ci/run-step.sh @@ -50,7 +50,7 @@ fi STARTTIME=`date +%s` echo "*** start: ${LABEL}" if [ -n "${FATAL}" ]; then - ${TIMEOUTCMD} --signal=ABRT --kill-after=60s ${TIMEOUT} $@ && echo -e "*** end($(echo $(date +%s) - ${STARTTIME} | bc)): ${LABEL}: \e[42mPassed\e[0m" || (echo -e "*** end($(echo $(date +%s) - ${STARTTIME} | bc)): ${LABEL}: \e[41mFailed\e[0m" && exit 1) + ${TIMEOUTCMD} --signal=ABRT --kill-after=60s ${TIMEOUT} "$@" && echo -e "*** end($(echo $(date +%s) - ${STARTTIME} | bc)): ${LABEL}: \e[42mPassed\e[0m" || (echo -e "*** end($(echo $(date +%s) - ${STARTTIME} | bc)): ${LABEL}: \e[41mFailed\e[0m" && exit 1) else - ${TIMEOUTCMD} --signal=ABRT --kill-after=60s ${TIMEOUT} $@ && echo -e "*** end($(echo $(date +%s) - ${STARTTIME} | bc)): ${LABEL}: \e[42mPassed\e[0m" || echo -e "*** end($(echo $(date +%s) - ${STARTTIME} | bc)): ${LABEL}: \e[43mUnstable\e[0m" + ${TIMEOUTCMD} --signal=ABRT --kill-after=60s ${TIMEOUT} "$@" && echo -e "*** end($(echo $(date +%s) - ${STARTTIME} | bc)): ${LABEL}: \e[42mPassed\e[0m" || echo -e "*** end($(echo $(date +%s) - ${STARTTIME} | bc)): ${LABEL}: \e[43mUnstable\e[0m" fi -- cgit v1.2.3 From a4b24252d4053df6061d1ebdae4e3f859f8cf87e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20K=C3=B6plinger?= Date: Thu, 7 Apr 2016 16:57:54 +0200 Subject: [ci] Fall back to timeout command if babysitter doesn't work Some Jenkins workers (e.g. the s390x one) have an earlier python version installed where the babysitter doesn't work. Fall back to timeout in those cases. --- scripts/ci/run-step.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/ci/run-step.sh b/scripts/ci/run-step.sh index 7067a3f8a58..4ea5fbc4c3a 100755 --- a/scripts/ci/run-step.sh +++ b/scripts/ci/run-step.sh @@ -1,5 +1,9 @@ #!/bin/bash -e TIMEOUTCMD=`dirname "${BASH_SOURCE[0]}"`/babysitter +if ! ${TIMEOUTCMD} -h >/dev/null 2>&1; then + TIMEOUTCMD=timeout # fall back to timeout if babysitter doesn't work (e.g. python not installed or wrong version) +fi + export MONO_BABYSITTER_LOG_FILE=babysitter_report.json_lines helptext () -- cgit v1.2.3 From e7918c6fe4c546180aad5cf66c6fcfec2fb1d205 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20K=C3=B6plinger?= Date: Fri, 8 Apr 2016 00:52:33 +0200 Subject: [ci] Don't enable the mobile profiles on the community archs It only takes time and e.g. causes issues if e.g. AOT doesn't work. --- scripts/ci/run-jenkins.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/ci/run-jenkins.sh b/scripts/ci/run-jenkins.sh index 2507bc9777f..3a22f9182a7 100755 --- a/scripts/ci/run-jenkins.sh +++ b/scripts/ci/run-jenkins.sh @@ -9,7 +9,13 @@ if [[ ${label} == 'osx-amd64' ]]; then EXTRA_CONF_FLAGS="--with-libgdiplus=/Libr if [[ ${label} == 'w32' ]]; then PLATFORM=Win32; EXTRA_CONF_FLAGS="--host=i686-pc-mingw32"; export MONO_EXECUTABLE="`cygpath -u ${WORKSPACE}\\\msvc\\\Win32\\\bin\\\Release_SGen\\\mono-sgen.exe`";fi if [[ ${label} == 'w64' ]]; then PLATFORM=x64; EXTRA_CONF_FLAGS="--host=i686-pc-mingw32"; export MONO_EXECUTABLE="`cygpath -u ${WORKSPACE}\\\msvc\\\x64\\\bin\\\Release_SGen\\\mono-sgen.exe`"; fi -${TESTCMD} --label=configure --timeout=60m --fatal ./autogen.sh --with-monodroid --with-monotouch --with-monotouch_watch --with-monotouch_tv --with-xammac --with-mobile_static $EXTRA_CONF_FLAGS +if [[ ${label} != w* ]] && [[ ${label} != 'debian-ppc64el' ]] && [[ ${label} != 'centos-s390x' ]]; + then + EXTRA_CONF_FLAGS="$EXTRA_CONF_FLAGS --with-monodroid --with-monotouch --with-monotouch_watch --with-monotouch_tv --with-xammac --with-mobile_static" + # only enable the mobile profiles and mobile_static on the main architectures +fi + +${TESTCMD} --label=configure --timeout=60m --fatal ./autogen.sh $EXTRA_CONF_FLAGS if [[ ${label} == w* ]]; then ${TESTCMD} --label=make-msvc --timeout=60m --fatal "/cygdrive/c/Program\ Files\ \(x86\)/MSBuild/12.0/Bin/MSBuild.exe" /p:Platform=${PLATFORM} /p:Configuration=Release msvc/mono.sln -- cgit v1.2.3 From 5ce2d0d5e2174e95cc666c93f6450de436459768 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20K=C3=B6plinger?= Date: Fri, 8 Apr 2016 01:17:41 +0200 Subject: [ci] Remove quotes from windows msbuild command Turns out it wasn't the right fix. --- scripts/ci/run-jenkins.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/ci/run-jenkins.sh b/scripts/ci/run-jenkins.sh index 3a22f9182a7..7e999c540bf 100755 --- a/scripts/ci/run-jenkins.sh +++ b/scripts/ci/run-jenkins.sh @@ -18,8 +18,8 @@ fi ${TESTCMD} --label=configure --timeout=60m --fatal ./autogen.sh $EXTRA_CONF_FLAGS if [[ ${label} == w* ]]; then - ${TESTCMD} --label=make-msvc --timeout=60m --fatal "/cygdrive/c/Program\ Files\ \(x86\)/MSBuild/12.0/Bin/MSBuild.exe" /p:Platform=${PLATFORM} /p:Configuration=Release msvc/mono.sln - ${TESTCMD} --label=make-msvc-sgen --timeout=60m --fatal "/cygdrive/c/Program\ Files\ \(x86\)/MSBuild/12.0/Bin/MSBuild.exe" /p:Platform=${PLATFORM} /p:Configuration=Release_SGen msvc/mono.sln + ${TESTCMD} --label=make-msvc --timeout=60m --fatal /cygdrive/c/Program\ Files\ \(x86\)/MSBuild/12.0/Bin/MSBuild.exe /p:Platform=${PLATFORM} /p:Configuration=Release msvc/mono.sln + ${TESTCMD} --label=make-msvc-sgen --timeout=60m --fatal /cygdrive/c/Program\ Files\ \(x86\)/MSBuild/12.0/Bin/MSBuild.exe /p:Platform=${PLATFORM} /p:Configuration=Release_SGen msvc/mono.sln fi ${TESTCMD} --label=make --timeout=300m --fatal make -w V=1 if [[ -n "${ghprbPullId}" ]] && [[ ${label} == w* ]]; -- cgit v1.2.3 From 4444fe417207bfc580d0ac478c985ee1d6a84f86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20K=C3=B6plinger?= Date: Fri, 8 Apr 2016 14:45:26 +0200 Subject: [ci] Disable mobile profiles on all archs, they aren't wired up in 4.4 --- scripts/ci/run-jenkins.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/scripts/ci/run-jenkins.sh b/scripts/ci/run-jenkins.sh index 7e999c540bf..8e24b348092 100755 --- a/scripts/ci/run-jenkins.sh +++ b/scripts/ci/run-jenkins.sh @@ -9,12 +9,6 @@ if [[ ${label} == 'osx-amd64' ]]; then EXTRA_CONF_FLAGS="--with-libgdiplus=/Libr if [[ ${label} == 'w32' ]]; then PLATFORM=Win32; EXTRA_CONF_FLAGS="--host=i686-pc-mingw32"; export MONO_EXECUTABLE="`cygpath -u ${WORKSPACE}\\\msvc\\\Win32\\\bin\\\Release_SGen\\\mono-sgen.exe`";fi if [[ ${label} == 'w64' ]]; then PLATFORM=x64; EXTRA_CONF_FLAGS="--host=i686-pc-mingw32"; export MONO_EXECUTABLE="`cygpath -u ${WORKSPACE}\\\msvc\\\x64\\\bin\\\Release_SGen\\\mono-sgen.exe`"; fi -if [[ ${label} != w* ]] && [[ ${label} != 'debian-ppc64el' ]] && [[ ${label} != 'centos-s390x' ]]; - then - EXTRA_CONF_FLAGS="$EXTRA_CONF_FLAGS --with-monodroid --with-monotouch --with-monotouch_watch --with-monotouch_tv --with-xammac --with-mobile_static" - # only enable the mobile profiles and mobile_static on the main architectures -fi - ${TESTCMD} --label=configure --timeout=60m --fatal ./autogen.sh $EXTRA_CONF_FLAGS if [[ ${label} == w* ]]; then -- cgit v1.2.3