Commit 9426d302 authored by Zeyi (Rice) Fan's avatar Zeyi (Rice) Fan Committed by Facebook GitHub Bot

add retry flag to getdeps test

Summary: This commit adds a flag `--retry` to getdeps and teach it to run retry failed test. This allows us to still pass the tests when there are some flaky tests presents.

Reviewed By: wez

Differential Revision: D22291063

fbshipit-source-id: 572af48a52ceb4a9abbf530cc0154ded0120c0de
parent f9dec043
......@@ -701,6 +701,7 @@ class TestCmd(ProjectCmdBase):
schedule_type=args.schedule_type,
owner=args.test_owner,
test_filter=args.filter,
retry=args.retry,
)
install_dirs.append(inst_dir)
......@@ -711,6 +712,13 @@ class TestCmd(ProjectCmdBase):
)
parser.add_argument("--test-owner", help="Owner for testpilot")
parser.add_argument("--filter", help="Only run the tests matching the regex")
parser.add_argument(
"--retry",
type=int,
default=3,
help="Number of immediate retries for failed tests "
"(noop in continuous and testwarden runs)",
)
@cmd("generate-github-actions", "generate a GitHub actions configuration")
......
......@@ -58,7 +58,7 @@ class BuilderBase(object):
return [vcvarsall, "amd64", "&&"]
return []
def _run_cmd(self, cmd, cwd=None, env=None, use_cmd_prefix=True):
def _run_cmd(self, cmd, cwd=None, env=None, use_cmd_prefix=True, allow_fail=False):
if env:
e = self.env.copy()
e.update(env)
......@@ -72,7 +72,13 @@ class BuilderBase(object):
cmd = cmd_prefix + cmd
log_file = os.path.join(self.build_dir, "getdeps_build.log")
run_cmd(cmd=cmd, env=env, cwd=cwd or self.build_dir, log_file=log_file)
return run_cmd(
cmd=cmd,
env=env,
cwd=cwd or self.build_dir,
log_file=log_file,
allow_fail=allow_fail,
)
def build(self, install_dirs, reconfigure):
print("Building %s..." % self.manifest.name)
......@@ -94,7 +100,7 @@ class BuilderBase(object):
dep_dirs = self.get_dev_run_extra_path_dirs(install_dirs, dep_munger)
dep_munger.emit_dev_run_script(script_path, dep_dirs)
def run_tests(self, install_dirs, schedule_type, owner, test_filter):
def run_tests(self, install_dirs, schedule_type, owner, test_filter, retry):
""" Execute any tests that we know how to run. If they fail,
raise an exception. """
pass
......@@ -537,7 +543,7 @@ if __name__ == "__main__":
env=env,
)
def run_tests(self, install_dirs, schedule_type, owner, test_filter):
def run_tests(self, install_dirs, schedule_type, owner, test_filter, retry):
env = self._compute_env(install_dirs)
ctest = path_search(env, "ctest")
cmake = path_search(env, "cmake")
......@@ -606,6 +612,11 @@ if __name__ == "__main__":
)
return tests
if schedule_type == "continuous" or schedule_type == "testwarden":
# for continuous and testwarden runs, disabling retry can give up
# better signals for flaky tests.
retry = 0
testpilot = path_search(env, "testpilot")
if testpilot:
buck_test_info = list_tests()
......@@ -629,7 +640,7 @@ if __name__ == "__main__":
self.build_opts.fbsource_dir,
"--buck-test-info",
buck_test_info_name,
"--retry=3",
"--retry=%d" % retry,
"-j=%s" % str(self.build_opts.num_jobs),
"--test-config",
"platform=%s" % machine_suffix,
......@@ -692,7 +703,18 @@ if __name__ == "__main__":
args = [ctest, "--output-on-failure", "-j", str(self.build_opts.num_jobs)]
if test_filter:
args += ["-R", test_filter]
self._run_cmd(args, env=env, use_cmd_prefix=use_cmd_prefix)
count = 0
while count < retry:
retcode = self._run_cmd(
args, env=env, use_cmd_prefix=use_cmd_prefix, allow_fail=True
)
if retcode == 0:
break
if count == 0:
# Only add this option in the second run.
args += ["--rerun-failed"]
count += 1
class NinjaBootstrap(BuilderBase):
......@@ -1041,7 +1063,7 @@ incremental = false
self.run_cargo(install_dirs, "build")
self.recreate_dir(build_source_dir, os.path.join(self.inst_dir, "source"))
def run_tests(self, install_dirs, schedule_type, owner, test_filter):
def run_tests(self, install_dirs, schedule_type, owner, test_filter, retry):
if test_filter:
args = ["--", test_filter]
else:
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment