Commit 36cd1d7e authored by Alex Hornby's avatar Alex Hornby Committed by Facebook GitHub Bot

add github actions for EdenFS on linux and fix Eden SCM Mac build (#106)

Summary:
Pull Request resolved: https://github.com/facebookexperimental/eden/pull/106

Pull Request resolved: https://github.com/facebookexperimental/eden/pull/107

Summary

* Add EdenFS builds on external CI now EdenSCM is good

* Mac builds on github actions by using brew for system dependencies

To make this work had to fix some path ordering issues with install directories for Linux and Mac, and generalise the homebrew path fixups we were doing for bison to all the used homebrew packages.

Previously Installed packages were being added after system paths, so our own installed thing might be ignored. On github these meant system python 3.9 was being used for hg tests rather than our specified 3.8 (this showed we have some test fails on python 3.9 with "SystemError: deallocated bytearray object has exported buffers", that are beyond the scope of this diff to fix)

Also needed to include the getdeps generated python into the generated edenscmdeps3.zip archive setup.py produces otherwise EdenFS tests failed to import thrift.Thrift

Eden tests are hanging when run externally about half way through, so disable them on github actions for now as this PR is already fairly large. They work when run locally on an internal devserver, so probably some bit of environment necessary is not defined in the test runner

Reviewed By: chadaustin

Differential Revision: D34116505

fbshipit-source-id: d0d628db5daabc28d0bd8997cd5c1bc885ed1e73
parent ebc15bae
...@@ -971,20 +971,26 @@ jobs: ...@@ -971,20 +971,26 @@ jobs:
out.write(" - name: Fix Git config\n") out.write(" - name: Fix Git config\n")
out.write(" run: git config --system core.longpaths true\n") out.write(" run: git config --system core.longpaths true\n")
projects = loader.manifests_in_dependency_order()
allow_sys_arg = "" allow_sys_arg = ""
if ( if (
build_opts.allow_system_packages build_opts.allow_system_packages
and build_opts.is_linux()
and build_opts.host_type.get_package_manager() and build_opts.host_type.get_package_manager()
): ):
allow_sys_arg = " --allow-system-packages" allow_sys_arg = " --allow-system-packages"
out.write(" - name: Install system deps\n") out.write(" - name: Install system deps\n")
sudo_arg = "sudo "
if build_opts.is_darwin():
# brew is installed as regular user
sudo_arg = ""
out.write( out.write(
f" run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive {manifest.name}\n" f" run: {sudo_arg}python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive {manifest.name}\n"
) )
projects = loader.manifests_in_dependency_order()
main_repo_url = manifest.get_repo_url(manifest_ctx)
has_same_repo_dep = False
for m in projects: for m in projects:
if m != manifest: if m != manifest:
if m.name == "rust": if m.name == "rust":
...@@ -995,6 +1001,8 @@ jobs: ...@@ -995,6 +1001,8 @@ jobs:
out.write(" default: true\n") out.write(" default: true\n")
out.write(" profile: minimal\n") out.write(" profile: minimal\n")
else: else:
ctx = loader.ctx_gen.get_context(m.name)
if m.get_repo_url(ctx) != main_repo_url:
out.write(" - name: Fetch %s\n" % m.name) out.write(" - name: Fetch %s\n" % m.name)
out.write( out.write(
f" run: {getdepscmd}{allow_sys_arg} fetch --no-tests {m.name}\n" f" run: {getdepscmd}{allow_sys_arg} fetch --no-tests {m.name}\n"
...@@ -1005,9 +1013,15 @@ jobs: ...@@ -1005,9 +1013,15 @@ jobs:
if m.name == "rust": if m.name == "rust":
continue continue
else: else:
src_dir_arg = ""
ctx = loader.ctx_gen.get_context(m.name)
if main_repo_url and m.get_repo_url(ctx) == main_repo_url:
# Its in the same repo, so src-dir is also .
src_dir_arg = "--src-dir=. "
has_same_repo_dep = True
out.write(" - name: Build %s\n" % m.name) out.write(" - name: Build %s\n" % m.name)
out.write( out.write(
f" run: {getdepscmd}{allow_sys_arg} build --no-tests {m.name}\n" f" run: {getdepscmd}{allow_sys_arg} build {src_dir_arg}--no-tests {m.name}\n"
) )
out.write(" - name: Build %s\n" % manifest.name) out.write(" - name: Build %s\n" % manifest.name)
...@@ -1018,8 +1032,13 @@ jobs: ...@@ -1018,8 +1032,13 @@ jobs:
" --project-install-prefix %s:/usr/local" % manifest.name " --project-install-prefix %s:/usr/local" % manifest.name
) )
# If we have dep from same repo, we already built it and don't want to rebuild it again
no_deps_arg = ""
if has_same_repo_dep:
no_deps_arg = "--no-deps "
out.write( out.write(
f" run: {getdepscmd}{allow_sys_arg} build --src-dir=. {manifest.name} {project_prefix}\n" f" run: {getdepscmd}{allow_sys_arg} build {no_deps_arg}--src-dir=. {manifest.name} {project_prefix}\n"
) )
out.write(" - name: Copy artifacts\n") out.write(" - name: Copy artifacts\n")
...@@ -1043,6 +1062,7 @@ jobs: ...@@ -1043,6 +1062,7 @@ jobs:
out.write(" name: %s\n" % manifest.name) out.write(" name: %s\n" % manifest.name)
out.write(" path: _artifacts\n") out.write(" path: _artifacts\n")
if manifest.get("github.actions", "run_tests", ctx=manifest_ctx) != "off":
out.write(" - name: Test %s\n" % manifest.name) out.write(" - name: Test %s\n" % manifest.name)
out.write( out.write(
f" run: {getdepscmd}{allow_sys_arg} test --src-dir=. {manifest.name} {project_prefix}\n" f" run: {getdepscmd}{allow_sys_arg} test --src-dir=. {manifest.name} {project_prefix}\n"
......
This diff is collapsed.
This diff is collapsed.
...@@ -188,19 +188,6 @@ class SystemPackageFetcher(object): ...@@ -188,19 +188,6 @@ class SystemPackageFetcher(object):
else: else:
self.installed = False self.installed = False
# Hack to make openssl discovery with homebrew work. If openssl was
# built with autoconf we could use autoconf.envcmd.OPENSSL_ROOT_DIR
# from the manifest, but it isn't, so handle the special case here.
if (
self.installed
and self.host_type.is_darwin()
and self.manager == "homebrew"
and "openssl@1.1" in self.packages
):
candidate = homebrew_package_prefix("openssl@1.1")
if os.path.exists(candidate):
os.environ["OPENSSL_ROOT_DIR"] = candidate
return bool(self.installed) return bool(self.installed)
def update(self): def update(self):
......
...@@ -4,6 +4,7 @@ ...@@ -4,6 +4,7 @@
# LICENSE file in the root directory of this source tree. # LICENSE file in the root directory of this source tree.
import base64 import base64
import copy
import hashlib import hashlib
import os import os
...@@ -178,6 +179,7 @@ class ManifestLoader(object): ...@@ -178,6 +179,7 @@ class ManifestLoader(object):
deps = [manifest] deps = [manifest]
# The list of manifests in dependency order # The list of manifests in dependency order
dep_order = [] dep_order = []
system_packages = {}
while len(deps) > 0: while len(deps) > 0:
m = deps.pop(0) m = deps.pop(0)
...@@ -212,6 +214,19 @@ class ManifestLoader(object): ...@@ -212,6 +214,19 @@ class ManifestLoader(object):
# Its deps are done, so we can emit it # Its deps are done, so we can emit it
seen.add(m.name) seen.add(m.name)
# Capture system packages as we may need to set PATHs to then later
if (
self.build_opts.allow_system_packages
and self.build_opts.host_type.get_package_manager()
):
packages = m.get_required_system_packages(ctx)
for pkg_type, v in packages.items():
merged = system_packages.get(pkg_type, [])
if v not in merged:
merged += v
system_packages[pkg_type] = merged
# A manifest depends on all system packages in it dependencies as well
m.resolved_system_packages = copy.copy(system_packages)
dep_order.append(m) dep_order.append(m)
return dep_order return dep_order
......
...@@ -77,6 +77,12 @@ SCHEMA = { ...@@ -77,6 +77,12 @@ SCHEMA = {
"manifests_to_build": OPTIONAL, "manifests_to_build": OPTIONAL,
}, },
}, },
"github.actions": {
"optional_section": True,
"fields": {
"run_tests": OPTIONAL,
},
},
"cmake.defines": {"optional_section": True}, "cmake.defines": {"optional_section": True},
"autoconf.args": {"optional_section": True}, "autoconf.args": {"optional_section": True},
"autoconf.envcmd.LDFLAGS": {"optional_section": True}, "autoconf.envcmd.LDFLAGS": {"optional_section": True},
...@@ -115,6 +121,7 @@ ALLOWED_EXPR_SECTIONS = [ ...@@ -115,6 +121,7 @@ ALLOWED_EXPR_SECTIONS = [
"shipit.pathmap", "shipit.pathmap",
"shipit.strip", "shipit.strip",
"homebrew", "homebrew",
"github.actions",
] ]
...@@ -223,6 +230,7 @@ class ManifestParser(object): ...@@ -223,6 +230,7 @@ class ManifestParser(object):
self.fbsource_path = self.get("manifest", "fbsource_path") self.fbsource_path = self.get("manifest", "fbsource_path")
self.shipit_project = self.get("manifest", "shipit_project") self.shipit_project = self.get("manifest", "shipit_project")
self.shipit_fbcode_builder = self.get("manifest", "shipit_fbcode_builder") self.shipit_fbcode_builder = self.get("manifest", "shipit_fbcode_builder")
self.resolved_system_packages = {}
if self.name != os.path.basename(file_name): if self.name != os.path.basename(file_name):
raise Exception( raise Exception(
...@@ -378,6 +386,9 @@ class ManifestParser(object): ...@@ -378,6 +386,9 @@ class ManifestParser(object):
return True return True
def get_repo_url(self, ctx):
return self.get("git", "repo_url", ctx=ctx)
def create_fetcher(self, build_options, ctx): def create_fetcher(self, build_options, ctx):
use_real_shipit = ( use_real_shipit = (
ShipitTransformerFetcher.available() and build_options.use_shipit ShipitTransformerFetcher.available() and build_options.use_shipit
...@@ -409,7 +420,7 @@ class ManifestParser(object): ...@@ -409,7 +420,7 @@ class ManifestParser(object):
if package_fetcher.packages_are_installed(): if package_fetcher.packages_are_installed():
return package_fetcher return package_fetcher
repo_url = self.get("git", "repo_url", ctx=ctx) repo_url = self.get_repo_url(ctx)
if repo_url: if repo_url:
rev = self.get("git", "rev") rev = self.get("git", "rev")
depth = self.get("git", "depth") depth = self.get("git", "depth")
......
...@@ -19,6 +19,8 @@ libboost-all-dev ...@@ -19,6 +19,8 @@ libboost-all-dev
[homebrew] [homebrew]
boost boost
# Boost cmake detection on homebrew adds this as requirement: https://github.com/Homebrew/homebrew-core/issues/67427#issuecomment-754187345
icu4c
[rpms.all(distro=centos_stream,distro_vers=8)] [rpms.all(distro=centos_stream,distro_vers=8)]
boost169 boost169
...@@ -55,6 +57,7 @@ boost169-program-options ...@@ -55,6 +57,7 @@ boost169-program-options
[rpms.not(all(distro=centos_stream,distro_vers=8))] [rpms.not(all(distro=centos_stream,distro_vers=8))]
boost-devel boost-devel
boost-static
[build] [build]
builder = boost builder = boost
......
[manifest]
name = bz2
[debs]
libbz2-dev
[homebrew]
bzip2
[rpms]
bzip2-devel
[download]
url = https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz
sha256 = ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269
[build.not(os=windows)]
builder = make
subdir = bzip2-1.0.8
[make.build_args.os=linux]
# python bz2 support on linux needs dynamic library
-f
Makefile-libbz2_so
[make.install_args]
install
[build.os=windows]
builder = nop
...@@ -7,6 +7,9 @@ shipit_fbcode_builder = true ...@@ -7,6 +7,9 @@ shipit_fbcode_builder = true
[git] [git]
repo_url = https://github.com/facebookexperimental/eden.git repo_url = https://github.com/facebookexperimental/eden.git
[github.actions]
run_tests = off
[build] [build]
builder = cmake builder = cmake
...@@ -37,11 +40,17 @@ sqlite3 ...@@ -37,11 +40,17 @@ sqlite3
[dependencies.os=darwin] [dependencies.os=darwin]
osxfuse osxfuse
[dependencies.not(os=windows)]
# TODO: teach getdeps to compile curl on Windows. # TODO: teach getdeps to compile curl on Windows.
# Enabling curl on Windows requires us to find a way to compile libcurl with # Enabling curl on Windows requires us to find a way to compile libcurl with
# msvc. # msvc.
[dependencies.not(os=windows)]
libcurl libcurl
# Added so that OSS doesn't see system "python" which is python 2 on darwin and some linux
python
[dependencies.all(fb=off)]
# Outside Meta hg is not installed, or if it is, its not the one we want to test with
eden_scm
[shipit.pathmap.fb=on] [shipit.pathmap.fb=on]
# for internal builds that use getdeps # for internal builds that use getdeps
......
[manifest]
name = ncurses
[debs]
libncurses-dev
[homebrew]
ncurses
[rpms]
ncurses-devel
[download]
url = https://ftp.gnu.org/pub/gnu/ncurses/ncurses-6.3.tar.gz
sha256 = 97fc51ac2b085d4cde31ef4d2c3122c21abc217e9090a43a30fc5ec21684e059
[build.not(os=windows)]
builder = autoconf
subdir = ncurses-6.3
[autoconf.args]
--without-cxx-binding
--without-ada
[autoconf.args.os=linux]
--enable-shared
--with-shared
[build.os=windows]
builder = nop
...@@ -6,6 +6,7 @@ libssl-dev ...@@ -6,6 +6,7 @@ libssl-dev
[homebrew] [homebrew]
openssl@1.1 openssl@1.1
# on homebrew need the matching curl and ca-
[rpms] [rpms]
openssl openssl
......
...@@ -36,3 +36,7 @@ libffi ...@@ -36,3 +36,7 @@ libffi
[dependencies] [dependencies]
libffi libffi
# eden tests expect the python bz2 support
bz2
# eden tests expect the python curses support
ncurses
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment