Commit 36cd1d7e authored by Alex Hornby's avatar Alex Hornby Committed by Facebook GitHub Bot

add github actions for EdenFS on linux and fix Eden SCM Mac build (#106)

Summary:
Pull Request resolved: https://github.com/facebookexperimental/eden/pull/106

Pull Request resolved: https://github.com/facebookexperimental/eden/pull/107

Summary

* Add EdenFS builds on external CI now EdenSCM is good

* Mac builds on github actions by using brew for system dependencies

To make this work had to fix some path ordering issues with install directories for Linux and Mac, and generalise the homebrew path fixups we were doing for bison to all the used homebrew packages.

Previously Installed packages were being added after system paths, so our own installed thing might be ignored. On github these meant system python 3.9 was being used for hg tests rather than our specified 3.8 (this showed we have some test fails on python 3.9 with "SystemError: deallocated bytearray object has exported buffers", that are beyond the scope of this diff to fix)

Also needed to include the getdeps generated python into the generated edenscmdeps3.zip archive setup.py produces otherwise EdenFS tests failed to import thrift.Thrift

Eden tests are hanging when run externally about half way through, so disable them on github actions for now as this PR is already fairly large. They work when run locally on an internal devserver, so probably some bit of environment necessary is not defined in the test runner

Reviewed By: chadaustin

Differential Revision: D34116505

fbshipit-source-id: d0d628db5daabc28d0bd8997cd5c1bc885ed1e73
parent ebc15bae
......@@ -971,20 +971,26 @@ jobs:
out.write(" - name: Fix Git config\n")
out.write(" run: git config --system core.longpaths true\n")
projects = loader.manifests_in_dependency_order()
allow_sys_arg = ""
if (
build_opts.allow_system_packages
and build_opts.is_linux()
and build_opts.host_type.get_package_manager()
):
allow_sys_arg = " --allow-system-packages"
out.write(" - name: Install system deps\n")
sudo_arg = "sudo "
if build_opts.is_darwin():
# brew is installed as regular user
sudo_arg = ""
out.write(
f" run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive {manifest.name}\n"
f" run: {sudo_arg}python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive {manifest.name}\n"
)
projects = loader.manifests_in_dependency_order()
main_repo_url = manifest.get_repo_url(manifest_ctx)
has_same_repo_dep = False
for m in projects:
if m != manifest:
if m.name == "rust":
......@@ -995,19 +1001,27 @@ jobs:
out.write(" default: true\n")
out.write(" profile: minimal\n")
else:
out.write(" - name: Fetch %s\n" % m.name)
out.write(
f" run: {getdepscmd}{allow_sys_arg} fetch --no-tests {m.name}\n"
)
ctx = loader.ctx_gen.get_context(m.name)
if m.get_repo_url(ctx) != main_repo_url:
out.write(" - name: Fetch %s\n" % m.name)
out.write(
f" run: {getdepscmd}{allow_sys_arg} fetch --no-tests {m.name}\n"
)
for m in projects:
if m != manifest:
if m.name == "rust":
continue
else:
src_dir_arg = ""
ctx = loader.ctx_gen.get_context(m.name)
if main_repo_url and m.get_repo_url(ctx) == main_repo_url:
# Its in the same repo, so src-dir is also .
src_dir_arg = "--src-dir=. "
has_same_repo_dep = True
out.write(" - name: Build %s\n" % m.name)
out.write(
f" run: {getdepscmd}{allow_sys_arg} build --no-tests {m.name}\n"
f" run: {getdepscmd}{allow_sys_arg} build {src_dir_arg}--no-tests {m.name}\n"
)
out.write(" - name: Build %s\n" % manifest.name)
......@@ -1018,8 +1032,13 @@ jobs:
" --project-install-prefix %s:/usr/local" % manifest.name
)
# If we have dep from same repo, we already built it and don't want to rebuild it again
no_deps_arg = ""
if has_same_repo_dep:
no_deps_arg = "--no-deps "
out.write(
f" run: {getdepscmd}{allow_sys_arg} build --src-dir=. {manifest.name} {project_prefix}\n"
f" run: {getdepscmd}{allow_sys_arg} build {no_deps_arg}--src-dir=. {manifest.name} {project_prefix}\n"
)
out.write(" - name: Copy artifacts\n")
......@@ -1043,10 +1062,11 @@ jobs:
out.write(" name: %s\n" % manifest.name)
out.write(" path: _artifacts\n")
out.write(" - name: Test %s\n" % manifest.name)
out.write(
f" run: {getdepscmd}{allow_sys_arg} test --src-dir=. {manifest.name} {project_prefix}\n"
)
if manifest.get("github.actions", "run_tests", ctx=manifest_ctx) != "off":
out.write(" - name: Test %s\n" % manifest.name)
out.write(
f" run: {getdepscmd}{allow_sys_arg} test --src-dir=. {manifest.name} {project_prefix}\n"
)
def setup_project_cmd_parser(self, parser):
parser.add_argument(
......
This diff is collapsed.
This diff is collapsed.
......@@ -188,19 +188,6 @@ class SystemPackageFetcher(object):
else:
self.installed = False
# Hack to make openssl discovery with homebrew work. If openssl was
# built with autoconf we could use autoconf.envcmd.OPENSSL_ROOT_DIR
# from the manifest, but it isn't, so handle the special case here.
if (
self.installed
and self.host_type.is_darwin()
and self.manager == "homebrew"
and "openssl@1.1" in self.packages
):
candidate = homebrew_package_prefix("openssl@1.1")
if os.path.exists(candidate):
os.environ["OPENSSL_ROOT_DIR"] = candidate
return bool(self.installed)
def update(self):
......
......@@ -4,6 +4,7 @@
# LICENSE file in the root directory of this source tree.
import base64
import copy
import hashlib
import os
......@@ -178,6 +179,7 @@ class ManifestLoader(object):
deps = [manifest]
# The list of manifests in dependency order
dep_order = []
system_packages = {}
while len(deps) > 0:
m = deps.pop(0)
......@@ -212,6 +214,19 @@ class ManifestLoader(object):
# Its deps are done, so we can emit it
seen.add(m.name)
# Capture system packages as we may need to set PATHs to then later
if (
self.build_opts.allow_system_packages
and self.build_opts.host_type.get_package_manager()
):
packages = m.get_required_system_packages(ctx)
for pkg_type, v in packages.items():
merged = system_packages.get(pkg_type, [])
if v not in merged:
merged += v
system_packages[pkg_type] = merged
# A manifest depends on all system packages in it dependencies as well
m.resolved_system_packages = copy.copy(system_packages)
dep_order.append(m)
return dep_order
......
......@@ -77,6 +77,12 @@ SCHEMA = {
"manifests_to_build": OPTIONAL,
},
},
"github.actions": {
"optional_section": True,
"fields": {
"run_tests": OPTIONAL,
},
},
"cmake.defines": {"optional_section": True},
"autoconf.args": {"optional_section": True},
"autoconf.envcmd.LDFLAGS": {"optional_section": True},
......@@ -115,6 +121,7 @@ ALLOWED_EXPR_SECTIONS = [
"shipit.pathmap",
"shipit.strip",
"homebrew",
"github.actions",
]
......@@ -223,6 +230,7 @@ class ManifestParser(object):
self.fbsource_path = self.get("manifest", "fbsource_path")
self.shipit_project = self.get("manifest", "shipit_project")
self.shipit_fbcode_builder = self.get("manifest", "shipit_fbcode_builder")
self.resolved_system_packages = {}
if self.name != os.path.basename(file_name):
raise Exception(
......@@ -378,6 +386,9 @@ class ManifestParser(object):
return True
def get_repo_url(self, ctx):
return self.get("git", "repo_url", ctx=ctx)
def create_fetcher(self, build_options, ctx):
use_real_shipit = (
ShipitTransformerFetcher.available() and build_options.use_shipit
......@@ -409,7 +420,7 @@ class ManifestParser(object):
if package_fetcher.packages_are_installed():
return package_fetcher
repo_url = self.get("git", "repo_url", ctx=ctx)
repo_url = self.get_repo_url(ctx)
if repo_url:
rev = self.get("git", "rev")
depth = self.get("git", "depth")
......
......@@ -19,6 +19,8 @@ libboost-all-dev
[homebrew]
boost
# Boost cmake detection on homebrew adds this as requirement: https://github.com/Homebrew/homebrew-core/issues/67427#issuecomment-754187345
icu4c
[rpms.all(distro=centos_stream,distro_vers=8)]
boost169
......@@ -55,6 +57,7 @@ boost169-program-options
[rpms.not(all(distro=centos_stream,distro_vers=8))]
boost-devel
boost-static
[build]
builder = boost
......
[manifest]
name = bz2
[debs]
libbz2-dev
[homebrew]
bzip2
[rpms]
bzip2-devel
[download]
url = https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz
sha256 = ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269
[build.not(os=windows)]
builder = make
subdir = bzip2-1.0.8
[make.build_args.os=linux]
# python bz2 support on linux needs dynamic library
-f
Makefile-libbz2_so
[make.install_args]
install
[build.os=windows]
builder = nop
......@@ -7,6 +7,9 @@ shipit_fbcode_builder = true
[git]
repo_url = https://github.com/facebookexperimental/eden.git
[github.actions]
run_tests = off
[build]
builder = cmake
......@@ -37,11 +40,17 @@ sqlite3
[dependencies.os=darwin]
osxfuse
[dependencies.not(os=windows)]
# TODO: teach getdeps to compile curl on Windows.
# Enabling curl on Windows requires us to find a way to compile libcurl with
# msvc.
[dependencies.not(os=windows)]
libcurl
# Added so that OSS doesn't see system "python" which is python 2 on darwin and some linux
python
[dependencies.all(fb=off)]
# Outside Meta hg is not installed, or if it is, its not the one we want to test with
eden_scm
[shipit.pathmap.fb=on]
# for internal builds that use getdeps
......
[manifest]
name = ncurses
[debs]
libncurses-dev
[homebrew]
ncurses
[rpms]
ncurses-devel
[download]
url = https://ftp.gnu.org/pub/gnu/ncurses/ncurses-6.3.tar.gz
sha256 = 97fc51ac2b085d4cde31ef4d2c3122c21abc217e9090a43a30fc5ec21684e059
[build.not(os=windows)]
builder = autoconf
subdir = ncurses-6.3
[autoconf.args]
--without-cxx-binding
--without-ada
[autoconf.args.os=linux]
--enable-shared
--with-shared
[build.os=windows]
builder = nop
......@@ -6,6 +6,7 @@ libssl-dev
[homebrew]
openssl@1.1
# on homebrew need the matching curl and ca-
[rpms]
openssl
......
......@@ -36,3 +36,7 @@ libffi
[dependencies]
libffi
# eden tests expect the python bz2 support
bz2
# eden tests expect the python curses support
ncurses
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment