Commit 9426d302 authored by Zeyi (Rice) Fan's avatar Zeyi (Rice) Fan Committed by Facebook GitHub Bot

add retry flag to getdeps test

Summary: This commit adds a flag `--retry` to getdeps and teach it to run retry failed test. This allows us to still pass the tests when there are some flaky tests presents.

Reviewed By: wez

Differential Revision: D22291063

fbshipit-source-id: 572af48a52ceb4a9abbf530cc0154ded0120c0de
parent f9dec043
...@@ -701,6 +701,7 @@ class TestCmd(ProjectCmdBase): ...@@ -701,6 +701,7 @@ class TestCmd(ProjectCmdBase):
schedule_type=args.schedule_type, schedule_type=args.schedule_type,
owner=args.test_owner, owner=args.test_owner,
test_filter=args.filter, test_filter=args.filter,
retry=args.retry,
) )
install_dirs.append(inst_dir) install_dirs.append(inst_dir)
...@@ -711,6 +712,13 @@ class TestCmd(ProjectCmdBase): ...@@ -711,6 +712,13 @@ class TestCmd(ProjectCmdBase):
) )
parser.add_argument("--test-owner", help="Owner for testpilot") parser.add_argument("--test-owner", help="Owner for testpilot")
parser.add_argument("--filter", help="Only run the tests matching the regex") parser.add_argument("--filter", help="Only run the tests matching the regex")
parser.add_argument(
"--retry",
type=int,
default=3,
help="Number of immediate retries for failed tests "
"(noop in continuous and testwarden runs)",
)
@cmd("generate-github-actions", "generate a GitHub actions configuration") @cmd("generate-github-actions", "generate a GitHub actions configuration")
......
...@@ -58,7 +58,7 @@ class BuilderBase(object): ...@@ -58,7 +58,7 @@ class BuilderBase(object):
return [vcvarsall, "amd64", "&&"] return [vcvarsall, "amd64", "&&"]
return [] return []
def _run_cmd(self, cmd, cwd=None, env=None, use_cmd_prefix=True): def _run_cmd(self, cmd, cwd=None, env=None, use_cmd_prefix=True, allow_fail=False):
if env: if env:
e = self.env.copy() e = self.env.copy()
e.update(env) e.update(env)
...@@ -72,7 +72,13 @@ class BuilderBase(object): ...@@ -72,7 +72,13 @@ class BuilderBase(object):
cmd = cmd_prefix + cmd cmd = cmd_prefix + cmd
log_file = os.path.join(self.build_dir, "getdeps_build.log") log_file = os.path.join(self.build_dir, "getdeps_build.log")
run_cmd(cmd=cmd, env=env, cwd=cwd or self.build_dir, log_file=log_file) return run_cmd(
cmd=cmd,
env=env,
cwd=cwd or self.build_dir,
log_file=log_file,
allow_fail=allow_fail,
)
def build(self, install_dirs, reconfigure): def build(self, install_dirs, reconfigure):
print("Building %s..." % self.manifest.name) print("Building %s..." % self.manifest.name)
...@@ -94,7 +100,7 @@ class BuilderBase(object): ...@@ -94,7 +100,7 @@ class BuilderBase(object):
dep_dirs = self.get_dev_run_extra_path_dirs(install_dirs, dep_munger) dep_dirs = self.get_dev_run_extra_path_dirs(install_dirs, dep_munger)
dep_munger.emit_dev_run_script(script_path, dep_dirs) dep_munger.emit_dev_run_script(script_path, dep_dirs)
def run_tests(self, install_dirs, schedule_type, owner, test_filter): def run_tests(self, install_dirs, schedule_type, owner, test_filter, retry):
""" Execute any tests that we know how to run. If they fail, """ Execute any tests that we know how to run. If they fail,
raise an exception. """ raise an exception. """
pass pass
...@@ -537,7 +543,7 @@ if __name__ == "__main__": ...@@ -537,7 +543,7 @@ if __name__ == "__main__":
env=env, env=env,
) )
def run_tests(self, install_dirs, schedule_type, owner, test_filter): def run_tests(self, install_dirs, schedule_type, owner, test_filter, retry):
env = self._compute_env(install_dirs) env = self._compute_env(install_dirs)
ctest = path_search(env, "ctest") ctest = path_search(env, "ctest")
cmake = path_search(env, "cmake") cmake = path_search(env, "cmake")
...@@ -606,6 +612,11 @@ if __name__ == "__main__": ...@@ -606,6 +612,11 @@ if __name__ == "__main__":
) )
return tests return tests
if schedule_type == "continuous" or schedule_type == "testwarden":
# for continuous and testwarden runs, disabling retry can give up
# better signals for flaky tests.
retry = 0
testpilot = path_search(env, "testpilot") testpilot = path_search(env, "testpilot")
if testpilot: if testpilot:
buck_test_info = list_tests() buck_test_info = list_tests()
...@@ -629,7 +640,7 @@ if __name__ == "__main__": ...@@ -629,7 +640,7 @@ if __name__ == "__main__":
self.build_opts.fbsource_dir, self.build_opts.fbsource_dir,
"--buck-test-info", "--buck-test-info",
buck_test_info_name, buck_test_info_name,
"--retry=3", "--retry=%d" % retry,
"-j=%s" % str(self.build_opts.num_jobs), "-j=%s" % str(self.build_opts.num_jobs),
"--test-config", "--test-config",
"platform=%s" % machine_suffix, "platform=%s" % machine_suffix,
...@@ -692,7 +703,18 @@ if __name__ == "__main__": ...@@ -692,7 +703,18 @@ if __name__ == "__main__":
args = [ctest, "--output-on-failure", "-j", str(self.build_opts.num_jobs)] args = [ctest, "--output-on-failure", "-j", str(self.build_opts.num_jobs)]
if test_filter: if test_filter:
args += ["-R", test_filter] args += ["-R", test_filter]
self._run_cmd(args, env=env, use_cmd_prefix=use_cmd_prefix)
count = 0
while count < retry:
retcode = self._run_cmd(
args, env=env, use_cmd_prefix=use_cmd_prefix, allow_fail=True
)
if retcode == 0:
break
if count == 0:
# Only add this option in the second run.
args += ["--rerun-failed"]
count += 1
class NinjaBootstrap(BuilderBase): class NinjaBootstrap(BuilderBase):
...@@ -1041,7 +1063,7 @@ incremental = false ...@@ -1041,7 +1063,7 @@ incremental = false
self.run_cargo(install_dirs, "build") self.run_cargo(install_dirs, "build")
self.recreate_dir(build_source_dir, os.path.join(self.inst_dir, "source")) self.recreate_dir(build_source_dir, os.path.join(self.inst_dir, "source"))
def run_tests(self, install_dirs, schedule_type, owner, test_filter): def run_tests(self, install_dirs, schedule_type, owner, test_filter, retry):
if test_filter: if test_filter:
args = ["--", test_filter] args = ["--", test_filter]
else: else:
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment