Commit 36cd1d7e authored by Alex Hornby's avatar Alex Hornby Committed by Facebook GitHub Bot

add github actions for EdenFS on linux and fix Eden SCM Mac build (#106)

Summary:
Pull Request resolved: https://github.com/facebookexperimental/eden/pull/106

Pull Request resolved: https://github.com/facebookexperimental/eden/pull/107

Summary

* Add EdenFS builds on external CI now EdenSCM is good

* Mac builds on github actions by using brew for system dependencies

To make this work had to fix some path ordering issues with install directories for Linux and Mac, and generalise the homebrew path fixups we were doing for bison to all the used homebrew packages.

Previously Installed packages were being added after system paths, so our own installed thing might be ignored. On github these meant system python 3.9 was being used for hg tests rather than our specified 3.8 (this showed we have some test fails on python 3.9 with "SystemError: deallocated bytearray object has exported buffers", that are beyond the scope of this diff to fix)

Also needed to include the getdeps generated python into the generated edenscmdeps3.zip archive setup.py produces otherwise EdenFS tests failed to import thrift.Thrift

Eden tests are hanging when run externally about half way through, so disable them on github actions for now as this PR is already fairly large. They work when run locally on an internal devserver, so probably some bit of environment necessary is not defined in the test runner

Reviewed By: chadaustin

Differential Revision: D34116505

fbshipit-source-id: d0d628db5daabc28d0bd8997cd5c1bc885ed1e73
parent ebc15bae
......@@ -971,20 +971,26 @@ jobs:
out.write(" - name: Fix Git config\n")
out.write(" run: git config --system core.longpaths true\n")
projects = loader.manifests_in_dependency_order()
allow_sys_arg = ""
if (
build_opts.allow_system_packages
and build_opts.is_linux()
and build_opts.host_type.get_package_manager()
):
allow_sys_arg = " --allow-system-packages"
out.write(" - name: Install system deps\n")
sudo_arg = "sudo "
if build_opts.is_darwin():
# brew is installed as regular user
sudo_arg = ""
out.write(
f" run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive {manifest.name}\n"
f" run: {sudo_arg}python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive {manifest.name}\n"
)
projects = loader.manifests_in_dependency_order()
main_repo_url = manifest.get_repo_url(manifest_ctx)
has_same_repo_dep = False
for m in projects:
if m != manifest:
if m.name == "rust":
......@@ -995,6 +1001,8 @@ jobs:
out.write(" default: true\n")
out.write(" profile: minimal\n")
else:
ctx = loader.ctx_gen.get_context(m.name)
if m.get_repo_url(ctx) != main_repo_url:
out.write(" - name: Fetch %s\n" % m.name)
out.write(
f" run: {getdepscmd}{allow_sys_arg} fetch --no-tests {m.name}\n"
......@@ -1005,9 +1013,15 @@ jobs:
if m.name == "rust":
continue
else:
src_dir_arg = ""
ctx = loader.ctx_gen.get_context(m.name)
if main_repo_url and m.get_repo_url(ctx) == main_repo_url:
# Its in the same repo, so src-dir is also .
src_dir_arg = "--src-dir=. "
has_same_repo_dep = True
out.write(" - name: Build %s\n" % m.name)
out.write(
f" run: {getdepscmd}{allow_sys_arg} build --no-tests {m.name}\n"
f" run: {getdepscmd}{allow_sys_arg} build {src_dir_arg}--no-tests {m.name}\n"
)
out.write(" - name: Build %s\n" % manifest.name)
......@@ -1018,8 +1032,13 @@ jobs:
" --project-install-prefix %s:/usr/local" % manifest.name
)
# If we have dep from same repo, we already built it and don't want to rebuild it again
no_deps_arg = ""
if has_same_repo_dep:
no_deps_arg = "--no-deps "
out.write(
f" run: {getdepscmd}{allow_sys_arg} build --src-dir=. {manifest.name} {project_prefix}\n"
f" run: {getdepscmd}{allow_sys_arg} build {no_deps_arg}--src-dir=. {manifest.name} {project_prefix}\n"
)
out.write(" - name: Copy artifacts\n")
......@@ -1043,6 +1062,7 @@ jobs:
out.write(" name: %s\n" % manifest.name)
out.write(" path: _artifacts\n")
if manifest.get("github.actions", "run_tests", ctx=manifest_ctx) != "off":
out.write(" - name: Test %s\n" % manifest.name)
out.write(
f" run: {getdepscmd}{allow_sys_arg} test --src-dir=. {manifest.name} {project_prefix}\n"
......
......@@ -4,6 +4,7 @@
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
import glob
import json
import os
import shutil
......@@ -197,6 +198,14 @@ class MakeBuilder(BuilderBase):
install_cmd = [self._make_binary] + self.install_args + self._get_prefix()
self._run_cmd(install_cmd, env=env)
# bz2's Makefile doesn't install its .so properly
if self.manifest and self.manifest.name == "bz2":
libdir = os.path.join(self.inst_dir, "lib")
srcpattern = os.path.join(self.src_dir, "lib*.so.*")
print(f"copying to {libdir} from {srcpattern}")
for file in glob.glob(srcpattern):
shutil.copy(file, libdir)
def run_tests(
self, install_dirs, schedule_type, owner, test_filter, retry, no_testpilot
):
......@@ -1185,308 +1194,3 @@ install(FILES sqlite3.h sqlite3ext.h DESTINATION include)
],
env=env,
)
class CargoBuilder(BuilderBase):
def __init__(
self,
build_opts,
ctx,
manifest,
src_dir,
build_dir,
inst_dir,
build_doc,
workspace_dir,
manifests_to_build,
loader,
):
super(CargoBuilder, self).__init__(
build_opts, ctx, manifest, src_dir, build_dir, inst_dir
)
self.build_doc = build_doc
self.ws_dir = workspace_dir
self.manifests_to_build = manifests_to_build and manifests_to_build.split(",")
self.loader = loader
def run_cargo(self, install_dirs, operation, args=None):
args = args or []
env = self._compute_env(install_dirs)
# Enable using nightly features with stable compiler
env["RUSTC_BOOTSTRAP"] = "1"
env["LIBZ_SYS_STATIC"] = "1"
cmd = [
"cargo",
operation,
"--workspace",
"-j%s" % self.num_jobs,
] + args
self._run_cmd(cmd, cwd=self.workspace_dir(), env=env)
def build_source_dir(self):
return os.path.join(self.build_dir, "source")
def workspace_dir(self):
return os.path.join(self.build_source_dir(), self.ws_dir or "")
def manifest_dir(self, manifest):
return os.path.join(self.build_source_dir(), manifest)
def recreate_dir(self, src, dst):
if os.path.isdir(dst):
shutil.rmtree(dst)
shutil.copytree(src, dst)
def _build(self, install_dirs, reconfigure):
build_source_dir = self.build_source_dir()
self.recreate_dir(self.src_dir, build_source_dir)
dot_cargo_dir = os.path.join(build_source_dir, ".cargo")
if not os.path.isdir(dot_cargo_dir):
os.mkdir(dot_cargo_dir)
with open(os.path.join(dot_cargo_dir, "config"), "w+") as f:
f.write(
"""\
[build]
target-dir = '''{}'''
[net]
git-fetch-with-cli = true
[profile.dev]
debug = false
incremental = false
""".format(
self.build_dir.replace("\\", "\\\\")
)
)
if self.ws_dir is not None:
self._patchup_workspace()
try:
from .facebook.rust import vendored_crates
vendored_crates(self.build_opts, build_source_dir)
except ImportError:
# This FB internal module isn't shippped to github,
# so just rely on cargo downloading crates on it's own
pass
if self.manifests_to_build is None:
self.run_cargo(
install_dirs,
"build",
["--out-dir", os.path.join(self.inst_dir, "bin"), "-Zunstable-options"],
)
else:
for manifest in self.manifests_to_build:
self.run_cargo(
install_dirs,
"build",
[
"--out-dir",
os.path.join(self.inst_dir, "bin"),
"-Zunstable-options",
"--manifest-path",
self.manifest_dir(manifest),
],
)
self.recreate_dir(build_source_dir, os.path.join(self.inst_dir, "source"))
def run_tests(
self, install_dirs, schedule_type, owner, test_filter, retry, no_testpilot
):
if test_filter:
args = ["--", test_filter]
else:
args = []
if self.manifests_to_build is None:
self.run_cargo(install_dirs, "test", args)
if self.build_doc:
self.run_cargo(install_dirs, "doc", ["--no-deps"])
else:
for manifest in self.manifests_to_build:
margs = ["--manifest-path", self.manifest_dir(manifest)]
self.run_cargo(install_dirs, "test", args + margs)
if self.build_doc:
self.run_cargo(install_dirs, "doc", ["--no-deps"] + margs)
def _patchup_workspace(self):
"""
This method makes some assumptions about the state of the project and
its cargo dependendies:
1. Crates from cargo dependencies can be extracted from Cargo.toml files
using _extract_crates function. It is using a heuristic so check its
code to understand how it is done.
2. The extracted cargo dependencies crates can be found in the
dependency's install dir using _resolve_crate_to_path function
which again is using a heuristic.
Notice that many things might go wrong here. E.g. if someone depends
on another getdeps crate by writing in their Cargo.toml file:
my-rename-of-crate = { package = "crate", git = "..." }
they can count themselves lucky because the code will raise an
Exception. There migh be more cases where the code will silently pass
producing bad results.
"""
workspace_dir = self.workspace_dir()
config = self._resolve_config()
if config:
with open(os.path.join(workspace_dir, "Cargo.toml"), "r+") as f:
manifest_content = f.read()
if "[package]" not in manifest_content:
# A fake manifest has to be crated to change the virtual
# manifest into a non-virtual. The virtual manifests are limited
# in many ways and the inability to define patches on them is
# one. Check https://github.com/rust-lang/cargo/issues/4934 to
# see if it is resolved.
f.write(
"""
[package]
name = "fake_manifest_of_{}"
version = "0.0.0"
[lib]
path = "/dev/null"
""".format(
self.manifest.name
)
)
else:
f.write("\n")
f.write(config)
def _resolve_config(self):
"""
Returns a configuration to be put inside root Cargo.toml file which
patches the dependencies git code with local getdeps versions.
See https://doc.rust-lang.org/cargo/reference/manifest.html#the-patch-section
"""
dep_to_git = self._resolve_dep_to_git()
dep_to_crates = CargoBuilder._resolve_dep_to_crates(
self.build_source_dir(), dep_to_git
)
config = []
for name in sorted(dep_to_git.keys()):
git_conf = dep_to_git[name]
crates = sorted(dep_to_crates.get(name, []))
if not crates:
continue # nothing to patch, move along
crates_patches = [
'{} = {{ path = "{}" }}'.format(
crate,
CargoBuilder._resolve_crate_to_path(crate, git_conf).replace(
"\\", "\\\\"
),
)
for crate in crates
]
config.append(
'[patch."{0}"]\n'.format(git_conf["repo_url"])
+ "\n".join(crates_patches)
)
return "\n".join(config)
def _resolve_dep_to_git(self):
"""
For each direct dependency of the currently build manifest check if it
is also cargo-builded and if yes then extract it's git configs and
install dir
"""
dependencies = self.manifest.get_dependencies(self.ctx)
if not dependencies:
return []
dep_to_git = {}
for dep in dependencies:
dep_manifest = self.loader.load_manifest(dep)
dep_builder = dep_manifest.get("build", "builder", ctx=self.ctx)
if dep_builder not in ["cargo", "nop"] or dep == "rust":
# This is a direct dependency, but it is not build with cargo
# and it is not simply copying files with nop, so ignore it.
# The "rust" dependency is an exception since it contains the
# toolchain.
continue
git_conf = dep_manifest.get_section_as_dict("git", self.ctx)
if "repo_url" not in git_conf:
raise Exception(
"A cargo dependency requires git.repo_url to be defined."
)
source_dir = self.loader.get_project_install_dir(dep_manifest)
if dep_builder == "cargo":
source_dir = os.path.join(source_dir, "source")
git_conf["source_dir"] = source_dir
dep_to_git[dep] = git_conf
return dep_to_git
@staticmethod
def _resolve_dep_to_crates(build_source_dir, dep_to_git):
"""
This function traverse the build_source_dir in search of Cargo.toml
files, extracts the crate names from them using _extract_crates
function and returns a merged result containing crate names per
dependency name from all Cargo.toml files in the project.
"""
if not dep_to_git:
return {} # no deps, so don't waste time traversing files
dep_to_crates = {}
for root, _, files in os.walk(build_source_dir):
for f in files:
if f == "Cargo.toml":
more_dep_to_crates = CargoBuilder._extract_crates(
os.path.join(root, f), dep_to_git
)
for name, crates in more_dep_to_crates.items():
dep_to_crates.setdefault(name, set()).update(crates)
return dep_to_crates
@staticmethod
def _extract_crates(cargo_toml_file, dep_to_git):
"""
This functions reads content of provided cargo toml file and extracts
crate names per each dependency. The extraction is done by a heuristic
so it might be incorrect.
"""
deps_to_crates = {}
with open(cargo_toml_file, "r") as f:
for line in f.readlines():
if line.startswith("#") or "git = " not in line:
continue # filter out commented lines and ones without git deps
for name, conf in dep_to_git.items():
if 'git = "{}"'.format(conf["repo_url"]) in line:
pkg_template = ' package = "'
if pkg_template in line:
crate_name, _, _ = line.partition(pkg_template)[
2
].partition('"')
else:
crate_name, _, _ = line.partition("=")
deps_to_crates.setdefault(name, set()).add(crate_name.strip())
return deps_to_crates
@staticmethod
def _resolve_crate_to_path(crate, git_conf):
"""
Tries to find <crate> in git_conf["inst_dir"] by searching a [package]
keyword followed by name = "<crate>".
"""
source_dir = git_conf["source_dir"]
search_pattern = '[package]\nname = "{}"'.format(crate)
for root, _, files in os.walk(source_dir):
for fname in files:
if fname == "Cargo.toml":
with open(os.path.join(root, fname), "r") as f:
if search_pattern in f.read():
return root
raise Exception("Failed to found crate {} in path {}".format(crate, source_dir))
......@@ -13,7 +13,7 @@ import tempfile
from typing import Optional, Mapping
from .copytree import containing_repo_type
from .envfuncs import Env, add_path_entry
from .envfuncs import Env, add_flag, add_path_entry
from .fetcher import get_fbsource_repo_data, homebrew_package_prefix
from .manifest import ContextGenerator
from .platform import HostType, is_windows, get_available_ram
......@@ -103,6 +103,18 @@ class BuildOptions(object):
self.allow_system_packages = allow_system_packages
self.lfs_path = lfs_path
self.shared_libs = shared_libs
lib_path = None
if self.is_darwin():
lib_path = "DYLD_LIBRARY_PATH"
elif self.is_linux():
lib_path = "LD_LIBRARY_PATH"
elif self.is_windows():
lib_path = "PATH"
else:
lib_path = None
self.lib_path = lib_path
if vcvars_path is None and is_windows():
try:
......@@ -196,7 +208,9 @@ class BuildOptions(object):
}
)
def compute_env_for_install_dirs(self, install_dirs, env=None, manifest=None):
def compute_env_for_install_dirs(
self, install_dirs, env=None, manifest=None
): # noqa: C901
if env is not None:
env = env.copy()
else:
......@@ -236,11 +250,23 @@ class BuildOptions(object):
sdkroot = subprocess.check_output(["xcrun", "--show-sdk-path"])
env["SDKROOT"] = sdkroot.decode().strip()
# MacOS includes a version of bison so homebrew won't automatically add
# its own version to PATH. Find where the homebrew bison is and prepend
# it to PATH.
if self.is_darwin() and self.host_type.get_package_manager() == "homebrew":
add_homebrew_package_to_path(env, "bison")
if (
self.is_darwin()
and self.allow_system_packages
and self.host_type.get_package_manager() == "homebrew"
and manifest
and manifest.resolved_system_packages
):
# Homebrew packages may not be on the default PATHs
brew_packages = manifest.resolved_system_packages.get("homebrew", [])
for p in brew_packages:
found = self.add_homebrew_package_to_env(p, env)
# Try extra hard to find openssl, needed with homebrew on macOS
if found and p.startswith("openssl"):
candidate = homebrew_package_prefix("openssl@1.1")
if os.path.exists(candidate):
os.environ["OPENSSL_ROOT_DIR"] = candidate
env["OPENSSL_ROOT_DIR"] = os.environ["OPENSSL_ROOT_DIR"]
if self.fbsource_dir:
env["YARN_YARN_OFFLINE_MIRROR"] = os.path.join(
......@@ -261,47 +287,115 @@ class BuildOptions(object):
env["FBSOURCE_HASH"] = hash_data.hash
env["FBSOURCE_DATE"] = hash_data.date
lib_path = None
if self.is_darwin():
lib_path = "DYLD_LIBRARY_PATH"
elif self.is_linux():
lib_path = "LD_LIBRARY_PATH"
elif self.is_windows():
lib_path = "PATH"
else:
lib_path = None
# reverse as we are prepending to the PATHs
for d in reversed(install_dirs):
self.add_prefix_to_env(d, env, append=False)
for d in install_dirs:
bindir = os.path.join(d, "bin")
# Linux is always system openssl
system_openssl = self.is_linux()
# For other systems lets see if package is requested
if not system_openssl and manifest and manifest.resolved_system_packages:
for _pkg_type, pkgs in manifest.resolved_system_packages.items():
for p in pkgs:
if p.startswith("openssl") or p.startswith("libssl"):
system_openssl = True
break
# Let openssl know to pick up the system certs if present
if system_openssl or "OPENSSL_DIR" in env:
for system_ssl_cfg in ["/etc/pki/tls", "/etc/ssl"]:
if os.path.isdir(system_ssl_cfg):
cert_dir = system_ssl_cfg + "/certs"
if os.path.isdir(cert_dir):
env["SSL_CERT_DIR"] = cert_dir
cert_file = system_ssl_cfg + "/cert.pem"
if os.path.isfile(cert_file):
env["SSL_CERT_FILE"] = cert_file
return env
pkgconfig = os.path.join(d, "lib/pkgconfig")
def add_homebrew_package_to_env(self, package, env):
prefix = homebrew_package_prefix(package)
if prefix and os.path.exists(prefix):
return self.add_prefix_to_env(
prefix, env, append=False, add_library_path=True
)
return False
def add_prefix_to_env(
self, d, env, append=True, add_library_path=False
): # noqa: C901
bindir = os.path.join(d, "bin")
found = False
pkgconfig = os.path.join(d, "lib", "pkgconfig")
if os.path.exists(pkgconfig):
add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig)
found = True
add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig, append=append)
pkgconfig = os.path.join(d, "lib64/pkgconfig")
pkgconfig = os.path.join(d, "lib64", "pkgconfig")
if os.path.exists(pkgconfig):
add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig)
found = True
add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig, append=append)
add_path_entry(env, "CMAKE_PREFIX_PATH", d)
add_path_entry(env, "CMAKE_PREFIX_PATH", d, append=append)
# Tell the thrift compiler about includes it needs to consider
thriftdir = os.path.join(d, "include/thrift-files")
thriftdir = os.path.join(d, "include", "thrift-files")
if os.path.exists(thriftdir):
add_path_entry(env, "THRIFT_INCLUDE_PATH", thriftdir)
found = True
add_path_entry(env, "THRIFT_INCLUDE_PATH", thriftdir, append=append)
# module detection for python is old fashioned and needs flags
includedir = os.path.join(d, "include")
if os.path.exists(includedir):
found = True
ncursesincludedir = os.path.join(d, "include", "ncurses")
if os.path.exists(ncursesincludedir):
add_path_entry(env, "C_INCLUDE_PATH", ncursesincludedir, append=append)
add_flag(env, "CPPFLAGS", f"-I{includedir}", append=append)
add_flag(env, "CPPFLAGS", f"-I{ncursesincludedir}", append=append)
elif "/bz2-" in d:
add_flag(env, "CPPFLAGS", f"-I{includedir}", append=append)
# Map from FB python manifests to PYTHONPATH
pydir = os.path.join(d, "lib", "fb-py-libs")
if os.path.exists(pydir):
found = True
manifest_ext = ".manifest"
pymanifestfiles = [
f
for f in os.listdir(pydir)
if f.endswith(manifest_ext) and os.path.isfile(os.path.join(pydir, f))
]
for f in pymanifestfiles:
subdir = f[: -len(manifest_ext)]
add_path_entry(
env, "PYTHONPATH", os.path.join(pydir, subdir), append=append
)
# Allow resolving shared objects built earlier (eg: zstd
# doesn't include the full path to the dylib in its linkage
# so we need to give it an assist)
if lib_path:
if self.lib_path:
for lib in ["lib", "lib64"]:
libdir = os.path.join(d, lib)
if os.path.exists(libdir):
add_path_entry(env, lib_path, libdir)
found = True
add_path_entry(env, self.lib_path, libdir, append=append)
# module detection for python is old fashioned and needs flags
if "/ncurses-" in d:
add_flag(env, "LDFLAGS", f"-L{libdir}", append=append)
elif "/bz2-" in d:
add_flag(env, "LDFLAGS", f"-L{libdir}", append=append)
if add_library_path:
add_path_entry(env, "LIBRARY_PATH", libdir, append=append)
# Allow resolving binaries (eg: cmake, ninja) and dlls
# built by earlier steps
if os.path.exists(bindir):
add_path_entry(env, "PATH", bindir, append=False)
found = True
add_path_entry(env, "PATH", bindir, append=append)
# If rustc is present in the `bin` directory, set RUSTC to prevent
# cargo uses the rustc installed in the system.
......@@ -319,32 +413,15 @@ class BuildOptions(object):
env["RUSTC"] = rustc_path
env["RUSTDOC"] = rustdoc_path
openssl_include = os.path.join(d, "include/openssl")
openssl_include = os.path.join(d, "include", "openssl")
if os.path.isdir(openssl_include) and any(
os.path.isfile(os.path.join(d, "lib", libcrypto))
for libcrypto in ("libcrypto.lib", "libcrypto.so", "libcrypto.a")
):
# This must be the openssl library, let Rust know about it
env["OPENSSL_DIR"] = d
# And let openssl know to pick up the system certs if present
for system_ssl_cfg in ["/etc/pki/tls", "/etc/ssl"]:
if os.path.isdir(system_ssl_cfg):
cert_dir = system_ssl_cfg + "/certs"
if os.path.isdir(cert_dir):
env["SSL_CERT_DIR"] = cert_dir
cert_file = system_ssl_cfg + "/cert.pem"
if os.path.isfile(cert_file):
env["SSL_CERT_FILE"] = cert_file
# Try extra hard to find openssl, needed with homebrew on macOS
if (
self.is_darwin()
and "OPENSSL_DIR" not in env
and "OPENSSL_ROOT_DIR" in os.environ
):
env["OPENSSL_ROOT_DIR"] = os.environ["OPENSSL_ROOT_DIR"]
return env
return found
def list_win32_subst_letters():
......@@ -536,9 +613,3 @@ def setup_build_options(args, host_type=None):
facebook_internal=args.facebook_internal,
**build_args,
)
def add_homebrew_package_to_path(env, package):
prefix = homebrew_package_prefix(package)
if prefix and os.path.exists(os.path.join(prefix, "bin")):
add_path_entry(env, "PATH", os.path.join(prefix, "bin"), append=False)
......@@ -188,19 +188,6 @@ class SystemPackageFetcher(object):
else:
self.installed = False
# Hack to make openssl discovery with homebrew work. If openssl was
# built with autoconf we could use autoconf.envcmd.OPENSSL_ROOT_DIR
# from the manifest, but it isn't, so handle the special case here.
if (
self.installed
and self.host_type.is_darwin()
and self.manager == "homebrew"
and "openssl@1.1" in self.packages
):
candidate = homebrew_package_prefix("openssl@1.1")
if os.path.exists(candidate):
os.environ["OPENSSL_ROOT_DIR"] = candidate
return bool(self.installed)
def update(self):
......
......@@ -4,6 +4,7 @@
# LICENSE file in the root directory of this source tree.
import base64
import copy
import hashlib
import os
......@@ -178,6 +179,7 @@ class ManifestLoader(object):
deps = [manifest]
# The list of manifests in dependency order
dep_order = []
system_packages = {}
while len(deps) > 0:
m = deps.pop(0)
......@@ -212,6 +214,19 @@ class ManifestLoader(object):
# Its deps are done, so we can emit it
seen.add(m.name)
# Capture system packages as we may need to set PATHs to then later
if (
self.build_opts.allow_system_packages
and self.build_opts.host_type.get_package_manager()
):
packages = m.get_required_system_packages(ctx)
for pkg_type, v in packages.items():
merged = system_packages.get(pkg_type, [])
if v not in merged:
merged += v
system_packages[pkg_type] = merged
# A manifest depends on all system packages in it dependencies as well
m.resolved_system_packages = copy.copy(system_packages)
dep_order.append(m)
return dep_order
......
......@@ -77,6 +77,12 @@ SCHEMA = {
"manifests_to_build": OPTIONAL,
},
},
"github.actions": {
"optional_section": True,
"fields": {
"run_tests": OPTIONAL,
},
},
"cmake.defines": {"optional_section": True},
"autoconf.args": {"optional_section": True},
"autoconf.envcmd.LDFLAGS": {"optional_section": True},
......@@ -115,6 +121,7 @@ ALLOWED_EXPR_SECTIONS = [
"shipit.pathmap",
"shipit.strip",
"homebrew",
"github.actions",
]
......@@ -223,6 +230,7 @@ class ManifestParser(object):
self.fbsource_path = self.get("manifest", "fbsource_path")
self.shipit_project = self.get("manifest", "shipit_project")
self.shipit_fbcode_builder = self.get("manifest", "shipit_fbcode_builder")
self.resolved_system_packages = {}
if self.name != os.path.basename(file_name):
raise Exception(
......@@ -378,6 +386,9 @@ class ManifestParser(object):
return True
def get_repo_url(self, ctx):
return self.get("git", "repo_url", ctx=ctx)
def create_fetcher(self, build_options, ctx):
use_real_shipit = (
ShipitTransformerFetcher.available() and build_options.use_shipit
......@@ -409,7 +420,7 @@ class ManifestParser(object):
if package_fetcher.packages_are_installed():
return package_fetcher
repo_url = self.get("git", "repo_url", ctx=ctx)
repo_url = self.get_repo_url(ctx)
if repo_url:
rev = self.get("git", "rev")
depth = self.get("git", "depth")
......
......@@ -19,6 +19,8 @@ libboost-all-dev
[homebrew]
boost
# Boost cmake detection on homebrew adds this as requirement: https://github.com/Homebrew/homebrew-core/issues/67427#issuecomment-754187345
icu4c
[rpms.all(distro=centos_stream,distro_vers=8)]
boost169
......@@ -55,6 +57,7 @@ boost169-program-options
[rpms.not(all(distro=centos_stream,distro_vers=8))]
boost-devel
boost-static
[build]
builder = boost
......
[manifest]
name = bz2
[debs]
libbz2-dev
[homebrew]
bzip2
[rpms]
bzip2-devel
[download]
url = https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz
sha256 = ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269
[build.not(os=windows)]
builder = make
subdir = bzip2-1.0.8
[make.build_args.os=linux]
# python bz2 support on linux needs dynamic library
-f
Makefile-libbz2_so
[make.install_args]
install
[build.os=windows]
builder = nop
......@@ -7,6 +7,9 @@ shipit_fbcode_builder = true
[git]
repo_url = https://github.com/facebookexperimental/eden.git
[github.actions]
run_tests = off
[build]
builder = cmake
......@@ -37,11 +40,17 @@ sqlite3
[dependencies.os=darwin]
osxfuse
[dependencies.not(os=windows)]
# TODO: teach getdeps to compile curl on Windows.
# Enabling curl on Windows requires us to find a way to compile libcurl with
# msvc.
[dependencies.not(os=windows)]
libcurl
# Added so that OSS doesn't see system "python" which is python 2 on darwin and some linux
python
[dependencies.all(fb=off)]
# Outside Meta hg is not installed, or if it is, its not the one we want to test with
eden_scm
[shipit.pathmap.fb=on]
# for internal builds that use getdeps
......
[manifest]
name = ncurses
[debs]
libncurses-dev
[homebrew]
ncurses
[rpms]
ncurses-devel
[download]
url = https://ftp.gnu.org/pub/gnu/ncurses/ncurses-6.3.tar.gz
sha256 = 97fc51ac2b085d4cde31ef4d2c3122c21abc217e9090a43a30fc5ec21684e059
[build.not(os=windows)]
builder = autoconf
subdir = ncurses-6.3
[autoconf.args]
--without-cxx-binding
--without-ada
[autoconf.args.os=linux]
--enable-shared
--with-shared
[build.os=windows]
builder = nop
......@@ -6,6 +6,7 @@ libssl-dev
[homebrew]
openssl@1.1
# on homebrew need the matching curl and ca-
[rpms]
openssl
......
......@@ -36,3 +36,7 @@ libffi
[dependencies]
libffi
# eden tests expect the python bz2 support
bz2
# eden tests expect the python curses support
ncurses
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment