Commit 36cd1d7e authored by Alex Hornby's avatar Alex Hornby Committed by Facebook GitHub Bot

add github actions for EdenFS on linux and fix Eden SCM Mac build (#106)

Summary:
Pull Request resolved: https://github.com/facebookexperimental/eden/pull/106

Pull Request resolved: https://github.com/facebookexperimental/eden/pull/107

Summary

* Add EdenFS builds on external CI now EdenSCM is good

* Mac builds on github actions by using brew for system dependencies

To make this work had to fix some path ordering issues with install directories for Linux and Mac, and generalise the homebrew path fixups we were doing for bison to all the used homebrew packages.

Previously Installed packages were being added after system paths, so our own installed thing might be ignored. On github these meant system python 3.9 was being used for hg tests rather than our specified 3.8 (this showed we have some test fails on python 3.9 with "SystemError: deallocated bytearray object has exported buffers", that are beyond the scope of this diff to fix)

Also needed to include the getdeps generated python into the generated edenscmdeps3.zip archive setup.py produces otherwise EdenFS tests failed to import thrift.Thrift

Eden tests are hanging when run externally about half way through, so disable them on github actions for now as this PR is already fairly large. They work when run locally on an internal devserver, so probably some bit of environment necessary is not defined in the test runner

Reviewed By: chadaustin

Differential Revision: D34116505

fbshipit-source-id: d0d628db5daabc28d0bd8997cd5c1bc885ed1e73
parent ebc15bae
...@@ -971,20 +971,26 @@ jobs: ...@@ -971,20 +971,26 @@ jobs:
out.write(" - name: Fix Git config\n") out.write(" - name: Fix Git config\n")
out.write(" run: git config --system core.longpaths true\n") out.write(" run: git config --system core.longpaths true\n")
projects = loader.manifests_in_dependency_order()
allow_sys_arg = "" allow_sys_arg = ""
if ( if (
build_opts.allow_system_packages build_opts.allow_system_packages
and build_opts.is_linux()
and build_opts.host_type.get_package_manager() and build_opts.host_type.get_package_manager()
): ):
allow_sys_arg = " --allow-system-packages" allow_sys_arg = " --allow-system-packages"
out.write(" - name: Install system deps\n") out.write(" - name: Install system deps\n")
sudo_arg = "sudo "
if build_opts.is_darwin():
# brew is installed as regular user
sudo_arg = ""
out.write( out.write(
f" run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive {manifest.name}\n" f" run: {sudo_arg}python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive {manifest.name}\n"
) )
projects = loader.manifests_in_dependency_order()
main_repo_url = manifest.get_repo_url(manifest_ctx)
has_same_repo_dep = False
for m in projects: for m in projects:
if m != manifest: if m != manifest:
if m.name == "rust": if m.name == "rust":
...@@ -995,19 +1001,27 @@ jobs: ...@@ -995,19 +1001,27 @@ jobs:
out.write(" default: true\n") out.write(" default: true\n")
out.write(" profile: minimal\n") out.write(" profile: minimal\n")
else: else:
out.write(" - name: Fetch %s\n" % m.name) ctx = loader.ctx_gen.get_context(m.name)
out.write( if m.get_repo_url(ctx) != main_repo_url:
f" run: {getdepscmd}{allow_sys_arg} fetch --no-tests {m.name}\n" out.write(" - name: Fetch %s\n" % m.name)
) out.write(
f" run: {getdepscmd}{allow_sys_arg} fetch --no-tests {m.name}\n"
)
for m in projects: for m in projects:
if m != manifest: if m != manifest:
if m.name == "rust": if m.name == "rust":
continue continue
else: else:
src_dir_arg = ""
ctx = loader.ctx_gen.get_context(m.name)
if main_repo_url and m.get_repo_url(ctx) == main_repo_url:
# Its in the same repo, so src-dir is also .
src_dir_arg = "--src-dir=. "
has_same_repo_dep = True
out.write(" - name: Build %s\n" % m.name) out.write(" - name: Build %s\n" % m.name)
out.write( out.write(
f" run: {getdepscmd}{allow_sys_arg} build --no-tests {m.name}\n" f" run: {getdepscmd}{allow_sys_arg} build {src_dir_arg}--no-tests {m.name}\n"
) )
out.write(" - name: Build %s\n" % manifest.name) out.write(" - name: Build %s\n" % manifest.name)
...@@ -1018,8 +1032,13 @@ jobs: ...@@ -1018,8 +1032,13 @@ jobs:
" --project-install-prefix %s:/usr/local" % manifest.name " --project-install-prefix %s:/usr/local" % manifest.name
) )
# If we have dep from same repo, we already built it and don't want to rebuild it again
no_deps_arg = ""
if has_same_repo_dep:
no_deps_arg = "--no-deps "
out.write( out.write(
f" run: {getdepscmd}{allow_sys_arg} build --src-dir=. {manifest.name} {project_prefix}\n" f" run: {getdepscmd}{allow_sys_arg} build {no_deps_arg}--src-dir=. {manifest.name} {project_prefix}\n"
) )
out.write(" - name: Copy artifacts\n") out.write(" - name: Copy artifacts\n")
...@@ -1043,10 +1062,11 @@ jobs: ...@@ -1043,10 +1062,11 @@ jobs:
out.write(" name: %s\n" % manifest.name) out.write(" name: %s\n" % manifest.name)
out.write(" path: _artifacts\n") out.write(" path: _artifacts\n")
out.write(" - name: Test %s\n" % manifest.name) if manifest.get("github.actions", "run_tests", ctx=manifest_ctx) != "off":
out.write( out.write(" - name: Test %s\n" % manifest.name)
f" run: {getdepscmd}{allow_sys_arg} test --src-dir=. {manifest.name} {project_prefix}\n" out.write(
) f" run: {getdepscmd}{allow_sys_arg} test --src-dir=. {manifest.name} {project_prefix}\n"
)
def setup_project_cmd_parser(self, parser): def setup_project_cmd_parser(self, parser):
parser.add_argument( parser.add_argument(
......
...@@ -4,6 +4,7 @@ ...@@ -4,6 +4,7 @@
# This source code is licensed under the MIT license found in the # This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree. # LICENSE file in the root directory of this source tree.
import glob
import json import json
import os import os
import shutil import shutil
...@@ -197,6 +198,14 @@ class MakeBuilder(BuilderBase): ...@@ -197,6 +198,14 @@ class MakeBuilder(BuilderBase):
install_cmd = [self._make_binary] + self.install_args + self._get_prefix() install_cmd = [self._make_binary] + self.install_args + self._get_prefix()
self._run_cmd(install_cmd, env=env) self._run_cmd(install_cmd, env=env)
# bz2's Makefile doesn't install its .so properly
if self.manifest and self.manifest.name == "bz2":
libdir = os.path.join(self.inst_dir, "lib")
srcpattern = os.path.join(self.src_dir, "lib*.so.*")
print(f"copying to {libdir} from {srcpattern}")
for file in glob.glob(srcpattern):
shutil.copy(file, libdir)
def run_tests( def run_tests(
self, install_dirs, schedule_type, owner, test_filter, retry, no_testpilot self, install_dirs, schedule_type, owner, test_filter, retry, no_testpilot
): ):
...@@ -1185,308 +1194,3 @@ install(FILES sqlite3.h sqlite3ext.h DESTINATION include) ...@@ -1185,308 +1194,3 @@ install(FILES sqlite3.h sqlite3ext.h DESTINATION include)
], ],
env=env, env=env,
) )
class CargoBuilder(BuilderBase):
def __init__(
self,
build_opts,
ctx,
manifest,
src_dir,
build_dir,
inst_dir,
build_doc,
workspace_dir,
manifests_to_build,
loader,
):
super(CargoBuilder, self).__init__(
build_opts, ctx, manifest, src_dir, build_dir, inst_dir
)
self.build_doc = build_doc
self.ws_dir = workspace_dir
self.manifests_to_build = manifests_to_build and manifests_to_build.split(",")
self.loader = loader
def run_cargo(self, install_dirs, operation, args=None):
args = args or []
env = self._compute_env(install_dirs)
# Enable using nightly features with stable compiler
env["RUSTC_BOOTSTRAP"] = "1"
env["LIBZ_SYS_STATIC"] = "1"
cmd = [
"cargo",
operation,
"--workspace",
"-j%s" % self.num_jobs,
] + args
self._run_cmd(cmd, cwd=self.workspace_dir(), env=env)
def build_source_dir(self):
return os.path.join(self.build_dir, "source")
def workspace_dir(self):
return os.path.join(self.build_source_dir(), self.ws_dir or "")
def manifest_dir(self, manifest):
return os.path.join(self.build_source_dir(), manifest)
def recreate_dir(self, src, dst):
if os.path.isdir(dst):
shutil.rmtree(dst)
shutil.copytree(src, dst)
def _build(self, install_dirs, reconfigure):
build_source_dir = self.build_source_dir()
self.recreate_dir(self.src_dir, build_source_dir)
dot_cargo_dir = os.path.join(build_source_dir, ".cargo")
if not os.path.isdir(dot_cargo_dir):
os.mkdir(dot_cargo_dir)
with open(os.path.join(dot_cargo_dir, "config"), "w+") as f:
f.write(
"""\
[build]
target-dir = '''{}'''
[net]
git-fetch-with-cli = true
[profile.dev]
debug = false
incremental = false
""".format(
self.build_dir.replace("\\", "\\\\")
)
)
if self.ws_dir is not None:
self._patchup_workspace()
try:
from .facebook.rust import vendored_crates
vendored_crates(self.build_opts, build_source_dir)
except ImportError:
# This FB internal module isn't shippped to github,
# so just rely on cargo downloading crates on it's own
pass
if self.manifests_to_build is None:
self.run_cargo(
install_dirs,
"build",
["--out-dir", os.path.join(self.inst_dir, "bin"), "-Zunstable-options"],
)
else:
for manifest in self.manifests_to_build:
self.run_cargo(
install_dirs,
"build",
[
"--out-dir",
os.path.join(self.inst_dir, "bin"),
"-Zunstable-options",
"--manifest-path",
self.manifest_dir(manifest),
],
)
self.recreate_dir(build_source_dir, os.path.join(self.inst_dir, "source"))
def run_tests(
self, install_dirs, schedule_type, owner, test_filter, retry, no_testpilot
):
if test_filter:
args = ["--", test_filter]
else:
args = []
if self.manifests_to_build is None:
self.run_cargo(install_dirs, "test", args)
if self.build_doc:
self.run_cargo(install_dirs, "doc", ["--no-deps"])
else:
for manifest in self.manifests_to_build:
margs = ["--manifest-path", self.manifest_dir(manifest)]
self.run_cargo(install_dirs, "test", args + margs)
if self.build_doc:
self.run_cargo(install_dirs, "doc", ["--no-deps"] + margs)
def _patchup_workspace(self):
"""
This method makes some assumptions about the state of the project and
its cargo dependendies:
1. Crates from cargo dependencies can be extracted from Cargo.toml files
using _extract_crates function. It is using a heuristic so check its
code to understand how it is done.
2. The extracted cargo dependencies crates can be found in the
dependency's install dir using _resolve_crate_to_path function
which again is using a heuristic.
Notice that many things might go wrong here. E.g. if someone depends
on another getdeps crate by writing in their Cargo.toml file:
my-rename-of-crate = { package = "crate", git = "..." }
they can count themselves lucky because the code will raise an
Exception. There migh be more cases where the code will silently pass
producing bad results.
"""
workspace_dir = self.workspace_dir()
config = self._resolve_config()
if config:
with open(os.path.join(workspace_dir, "Cargo.toml"), "r+") as f:
manifest_content = f.read()
if "[package]" not in manifest_content:
# A fake manifest has to be crated to change the virtual
# manifest into a non-virtual. The virtual manifests are limited
# in many ways and the inability to define patches on them is
# one. Check https://github.com/rust-lang/cargo/issues/4934 to
# see if it is resolved.
f.write(
"""
[package]
name = "fake_manifest_of_{}"
version = "0.0.0"
[lib]
path = "/dev/null"
""".format(
self.manifest.name
)
)
else:
f.write("\n")
f.write(config)
def _resolve_config(self):
"""
Returns a configuration to be put inside root Cargo.toml file which
patches the dependencies git code with local getdeps versions.
See https://doc.rust-lang.org/cargo/reference/manifest.html#the-patch-section
"""
dep_to_git = self._resolve_dep_to_git()
dep_to_crates = CargoBuilder._resolve_dep_to_crates(
self.build_source_dir(), dep_to_git
)
config = []
for name in sorted(dep_to_git.keys()):
git_conf = dep_to_git[name]
crates = sorted(dep_to_crates.get(name, []))
if not crates:
continue # nothing to patch, move along
crates_patches = [
'{} = {{ path = "{}" }}'.format(
crate,
CargoBuilder._resolve_crate_to_path(crate, git_conf).replace(
"\\", "\\\\"
),
)
for crate in crates
]
config.append(
'[patch."{0}"]\n'.format(git_conf["repo_url"])
+ "\n".join(crates_patches)
)
return "\n".join(config)
def _resolve_dep_to_git(self):
"""
For each direct dependency of the currently build manifest check if it
is also cargo-builded and if yes then extract it's git configs and
install dir
"""
dependencies = self.manifest.get_dependencies(self.ctx)
if not dependencies:
return []
dep_to_git = {}
for dep in dependencies:
dep_manifest = self.loader.load_manifest(dep)
dep_builder = dep_manifest.get("build", "builder", ctx=self.ctx)
if dep_builder not in ["cargo", "nop"] or dep == "rust":
# This is a direct dependency, but it is not build with cargo
# and it is not simply copying files with nop, so ignore it.
# The "rust" dependency is an exception since it contains the
# toolchain.
continue
git_conf = dep_manifest.get_section_as_dict("git", self.ctx)
if "repo_url" not in git_conf:
raise Exception(
"A cargo dependency requires git.repo_url to be defined."
)
source_dir = self.loader.get_project_install_dir(dep_manifest)
if dep_builder == "cargo":
source_dir = os.path.join(source_dir, "source")
git_conf["source_dir"] = source_dir
dep_to_git[dep] = git_conf
return dep_to_git
@staticmethod
def _resolve_dep_to_crates(build_source_dir, dep_to_git):
"""
This function traverse the build_source_dir in search of Cargo.toml
files, extracts the crate names from them using _extract_crates
function and returns a merged result containing crate names per
dependency name from all Cargo.toml files in the project.
"""
if not dep_to_git:
return {} # no deps, so don't waste time traversing files
dep_to_crates = {}
for root, _, files in os.walk(build_source_dir):
for f in files:
if f == "Cargo.toml":
more_dep_to_crates = CargoBuilder._extract_crates(
os.path.join(root, f), dep_to_git
)
for name, crates in more_dep_to_crates.items():
dep_to_crates.setdefault(name, set()).update(crates)
return dep_to_crates
@staticmethod
def _extract_crates(cargo_toml_file, dep_to_git):
"""
This functions reads content of provided cargo toml file and extracts
crate names per each dependency. The extraction is done by a heuristic
so it might be incorrect.
"""
deps_to_crates = {}
with open(cargo_toml_file, "r") as f:
for line in f.readlines():
if line.startswith("#") or "git = " not in line:
continue # filter out commented lines and ones without git deps
for name, conf in dep_to_git.items():
if 'git = "{}"'.format(conf["repo_url"]) in line:
pkg_template = ' package = "'
if pkg_template in line:
crate_name, _, _ = line.partition(pkg_template)[
2
].partition('"')
else:
crate_name, _, _ = line.partition("=")
deps_to_crates.setdefault(name, set()).add(crate_name.strip())
return deps_to_crates
@staticmethod
def _resolve_crate_to_path(crate, git_conf):
"""
Tries to find <crate> in git_conf["inst_dir"] by searching a [package]
keyword followed by name = "<crate>".
"""
source_dir = git_conf["source_dir"]
search_pattern = '[package]\nname = "{}"'.format(crate)
for root, _, files in os.walk(source_dir):
for fname in files:
if fname == "Cargo.toml":
with open(os.path.join(root, fname), "r") as f:
if search_pattern in f.read():
return root
raise Exception("Failed to found crate {} in path {}".format(crate, source_dir))
...@@ -13,7 +13,7 @@ import tempfile ...@@ -13,7 +13,7 @@ import tempfile
from typing import Optional, Mapping from typing import Optional, Mapping
from .copytree import containing_repo_type from .copytree import containing_repo_type
from .envfuncs import Env, add_path_entry from .envfuncs import Env, add_flag, add_path_entry
from .fetcher import get_fbsource_repo_data, homebrew_package_prefix from .fetcher import get_fbsource_repo_data, homebrew_package_prefix
from .manifest import ContextGenerator from .manifest import ContextGenerator
from .platform import HostType, is_windows, get_available_ram from .platform import HostType, is_windows, get_available_ram
...@@ -103,6 +103,18 @@ class BuildOptions(object): ...@@ -103,6 +103,18 @@ class BuildOptions(object):
self.allow_system_packages = allow_system_packages self.allow_system_packages = allow_system_packages
self.lfs_path = lfs_path self.lfs_path = lfs_path
self.shared_libs = shared_libs self.shared_libs = shared_libs
lib_path = None
if self.is_darwin():
lib_path = "DYLD_LIBRARY_PATH"
elif self.is_linux():
lib_path = "LD_LIBRARY_PATH"
elif self.is_windows():
lib_path = "PATH"
else:
lib_path = None
self.lib_path = lib_path
if vcvars_path is None and is_windows(): if vcvars_path is None and is_windows():
try: try:
...@@ -196,7 +208,9 @@ class BuildOptions(object): ...@@ -196,7 +208,9 @@ class BuildOptions(object):
} }
) )
def compute_env_for_install_dirs(self, install_dirs, env=None, manifest=None): def compute_env_for_install_dirs(
self, install_dirs, env=None, manifest=None
): # noqa: C901
if env is not None: if env is not None:
env = env.copy() env = env.copy()
else: else:
...@@ -236,11 +250,23 @@ class BuildOptions(object): ...@@ -236,11 +250,23 @@ class BuildOptions(object):
sdkroot = subprocess.check_output(["xcrun", "--show-sdk-path"]) sdkroot = subprocess.check_output(["xcrun", "--show-sdk-path"])
env["SDKROOT"] = sdkroot.decode().strip() env["SDKROOT"] = sdkroot.decode().strip()
# MacOS includes a version of bison so homebrew won't automatically add if (
# its own version to PATH. Find where the homebrew bison is and prepend self.is_darwin()
# it to PATH. and self.allow_system_packages
if self.is_darwin() and self.host_type.get_package_manager() == "homebrew": and self.host_type.get_package_manager() == "homebrew"
add_homebrew_package_to_path(env, "bison") and manifest
and manifest.resolved_system_packages
):
# Homebrew packages may not be on the default PATHs
brew_packages = manifest.resolved_system_packages.get("homebrew", [])
for p in brew_packages:
found = self.add_homebrew_package_to_env(p, env)
# Try extra hard to find openssl, needed with homebrew on macOS
if found and p.startswith("openssl"):
candidate = homebrew_package_prefix("openssl@1.1")
if os.path.exists(candidate):
os.environ["OPENSSL_ROOT_DIR"] = candidate
env["OPENSSL_ROOT_DIR"] = os.environ["OPENSSL_ROOT_DIR"]
if self.fbsource_dir: if self.fbsource_dir:
env["YARN_YARN_OFFLINE_MIRROR"] = os.path.join( env["YARN_YARN_OFFLINE_MIRROR"] = os.path.join(
...@@ -261,90 +287,141 @@ class BuildOptions(object): ...@@ -261,90 +287,141 @@ class BuildOptions(object):
env["FBSOURCE_HASH"] = hash_data.hash env["FBSOURCE_HASH"] = hash_data.hash
env["FBSOURCE_DATE"] = hash_data.date env["FBSOURCE_DATE"] = hash_data.date
lib_path = None # reverse as we are prepending to the PATHs
if self.is_darwin(): for d in reversed(install_dirs):
lib_path = "DYLD_LIBRARY_PATH" self.add_prefix_to_env(d, env, append=False)
elif self.is_linux():
lib_path = "LD_LIBRARY_PATH" # Linux is always system openssl
elif self.is_windows(): system_openssl = self.is_linux()
lib_path = "PATH"
else: # For other systems lets see if package is requested
lib_path = None if not system_openssl and manifest and manifest.resolved_system_packages:
for _pkg_type, pkgs in manifest.resolved_system_packages.items():
for p in pkgs:
if p.startswith("openssl") or p.startswith("libssl"):
system_openssl = True
break
# Let openssl know to pick up the system certs if present
if system_openssl or "OPENSSL_DIR" in env:
for system_ssl_cfg in ["/etc/pki/tls", "/etc/ssl"]:
if os.path.isdir(system_ssl_cfg):
cert_dir = system_ssl_cfg + "/certs"
if os.path.isdir(cert_dir):
env["SSL_CERT_DIR"] = cert_dir
cert_file = system_ssl_cfg + "/cert.pem"
if os.path.isfile(cert_file):
env["SSL_CERT_FILE"] = cert_file
for d in install_dirs: return env
bindir = os.path.join(d, "bin")
def add_homebrew_package_to_env(self, package, env):
pkgconfig = os.path.join(d, "lib/pkgconfig") prefix = homebrew_package_prefix(package)
if os.path.exists(pkgconfig): if prefix and os.path.exists(prefix):
add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig) return self.add_prefix_to_env(
prefix, env, append=False, add_library_path=True
pkgconfig = os.path.join(d, "lib64/pkgconfig") )
if os.path.exists(pkgconfig): return False
add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig)
def add_prefix_to_env(
add_path_entry(env, "CMAKE_PREFIX_PATH", d) self, d, env, append=True, add_library_path=False
): # noqa: C901
# Tell the thrift compiler about includes it needs to consider bindir = os.path.join(d, "bin")
thriftdir = os.path.join(d, "include/thrift-files") found = False
if os.path.exists(thriftdir): pkgconfig = os.path.join(d, "lib", "pkgconfig")
add_path_entry(env, "THRIFT_INCLUDE_PATH", thriftdir) if os.path.exists(pkgconfig):
found = True
# Allow resolving shared objects built earlier (eg: zstd add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig, append=append)
# doesn't include the full path to the dylib in its linkage
# so we need to give it an assist) pkgconfig = os.path.join(d, "lib64", "pkgconfig")
if lib_path: if os.path.exists(pkgconfig):
for lib in ["lib", "lib64"]: found = True
libdir = os.path.join(d, lib) add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig, append=append)
if os.path.exists(libdir):
add_path_entry(env, lib_path, libdir) add_path_entry(env, "CMAKE_PREFIX_PATH", d, append=append)
# Allow resolving binaries (eg: cmake, ninja) and dlls # Tell the thrift compiler about includes it needs to consider
# built by earlier steps thriftdir = os.path.join(d, "include", "thrift-files")
if os.path.exists(bindir): if os.path.exists(thriftdir):
add_path_entry(env, "PATH", bindir, append=False) found = True
add_path_entry(env, "THRIFT_INCLUDE_PATH", thriftdir, append=append)
# If rustc is present in the `bin` directory, set RUSTC to prevent
# cargo uses the rustc installed in the system. # module detection for python is old fashioned and needs flags
if self.is_windows(): includedir = os.path.join(d, "include")
cargo_path = os.path.join(bindir, "cargo.exe") if os.path.exists(includedir):
rustc_path = os.path.join(bindir, "rustc.exe") found = True
rustdoc_path = os.path.join(bindir, "rustdoc.exe") ncursesincludedir = os.path.join(d, "include", "ncurses")
else: if os.path.exists(ncursesincludedir):
cargo_path = os.path.join(bindir, "cargo") add_path_entry(env, "C_INCLUDE_PATH", ncursesincludedir, append=append)
rustc_path = os.path.join(bindir, "rustc") add_flag(env, "CPPFLAGS", f"-I{includedir}", append=append)
rustdoc_path = os.path.join(bindir, "rustdoc") add_flag(env, "CPPFLAGS", f"-I{ncursesincludedir}", append=append)
elif "/bz2-" in d:
if os.path.isfile(rustc_path): add_flag(env, "CPPFLAGS", f"-I{includedir}", append=append)
env["CARGO_BIN"] = cargo_path
env["RUSTC"] = rustc_path # Map from FB python manifests to PYTHONPATH
env["RUSTDOC"] = rustdoc_path pydir = os.path.join(d, "lib", "fb-py-libs")
if os.path.exists(pydir):
openssl_include = os.path.join(d, "include/openssl") found = True
if os.path.isdir(openssl_include) and any( manifest_ext = ".manifest"
os.path.isfile(os.path.join(d, "lib", libcrypto)) pymanifestfiles = [
for libcrypto in ("libcrypto.lib", "libcrypto.so", "libcrypto.a") f
): for f in os.listdir(pydir)
# This must be the openssl library, let Rust know about it if f.endswith(manifest_ext) and os.path.isfile(os.path.join(pydir, f))
env["OPENSSL_DIR"] = d ]
# And let openssl know to pick up the system certs if present for f in pymanifestfiles:
for system_ssl_cfg in ["/etc/pki/tls", "/etc/ssl"]: subdir = f[: -len(manifest_ext)]
if os.path.isdir(system_ssl_cfg): add_path_entry(
cert_dir = system_ssl_cfg + "/certs" env, "PYTHONPATH", os.path.join(pydir, subdir), append=append
if os.path.isdir(cert_dir): )
env["SSL_CERT_DIR"] = cert_dir
cert_file = system_ssl_cfg + "/cert.pem" # Allow resolving shared objects built earlier (eg: zstd
if os.path.isfile(cert_file): # doesn't include the full path to the dylib in its linkage
env["SSL_CERT_FILE"] = cert_file # so we need to give it an assist)
if self.lib_path:
# Try extra hard to find openssl, needed with homebrew on macOS for lib in ["lib", "lib64"]:
if ( libdir = os.path.join(d, lib)
self.is_darwin() if os.path.exists(libdir):
and "OPENSSL_DIR" not in env found = True
and "OPENSSL_ROOT_DIR" in os.environ add_path_entry(env, self.lib_path, libdir, append=append)
# module detection for python is old fashioned and needs flags
if "/ncurses-" in d:
add_flag(env, "LDFLAGS", f"-L{libdir}", append=append)
elif "/bz2-" in d:
add_flag(env, "LDFLAGS", f"-L{libdir}", append=append)
if add_library_path:
add_path_entry(env, "LIBRARY_PATH", libdir, append=append)
# Allow resolving binaries (eg: cmake, ninja) and dlls
# built by earlier steps
if os.path.exists(bindir):
found = True
add_path_entry(env, "PATH", bindir, append=append)
# If rustc is present in the `bin` directory, set RUSTC to prevent
# cargo uses the rustc installed in the system.
if self.is_windows():
cargo_path = os.path.join(bindir, "cargo.exe")
rustc_path = os.path.join(bindir, "rustc.exe")
rustdoc_path = os.path.join(bindir, "rustdoc.exe")
else:
cargo_path = os.path.join(bindir, "cargo")
rustc_path = os.path.join(bindir, "rustc")
rustdoc_path = os.path.join(bindir, "rustdoc")
if os.path.isfile(rustc_path):
env["CARGO_BIN"] = cargo_path
env["RUSTC"] = rustc_path
env["RUSTDOC"] = rustdoc_path
openssl_include = os.path.join(d, "include", "openssl")
if os.path.isdir(openssl_include) and any(
os.path.isfile(os.path.join(d, "lib", libcrypto))
for libcrypto in ("libcrypto.lib", "libcrypto.so", "libcrypto.a")
): ):
env["OPENSSL_ROOT_DIR"] = os.environ["OPENSSL_ROOT_DIR"] # This must be the openssl library, let Rust know about it
env["OPENSSL_DIR"] = d
return env return found
def list_win32_subst_letters(): def list_win32_subst_letters():
...@@ -536,9 +613,3 @@ def setup_build_options(args, host_type=None): ...@@ -536,9 +613,3 @@ def setup_build_options(args, host_type=None):
facebook_internal=args.facebook_internal, facebook_internal=args.facebook_internal,
**build_args, **build_args,
) )
def add_homebrew_package_to_path(env, package):
prefix = homebrew_package_prefix(package)
if prefix and os.path.exists(os.path.join(prefix, "bin")):
add_path_entry(env, "PATH", os.path.join(prefix, "bin"), append=False)
...@@ -188,19 +188,6 @@ class SystemPackageFetcher(object): ...@@ -188,19 +188,6 @@ class SystemPackageFetcher(object):
else: else:
self.installed = False self.installed = False
# Hack to make openssl discovery with homebrew work. If openssl was
# built with autoconf we could use autoconf.envcmd.OPENSSL_ROOT_DIR
# from the manifest, but it isn't, so handle the special case here.
if (
self.installed
and self.host_type.is_darwin()
and self.manager == "homebrew"
and "openssl@1.1" in self.packages
):
candidate = homebrew_package_prefix("openssl@1.1")
if os.path.exists(candidate):
os.environ["OPENSSL_ROOT_DIR"] = candidate
return bool(self.installed) return bool(self.installed)
def update(self): def update(self):
......
...@@ -4,6 +4,7 @@ ...@@ -4,6 +4,7 @@
# LICENSE file in the root directory of this source tree. # LICENSE file in the root directory of this source tree.
import base64 import base64
import copy
import hashlib import hashlib
import os import os
...@@ -178,6 +179,7 @@ class ManifestLoader(object): ...@@ -178,6 +179,7 @@ class ManifestLoader(object):
deps = [manifest] deps = [manifest]
# The list of manifests in dependency order # The list of manifests in dependency order
dep_order = [] dep_order = []
system_packages = {}
while len(deps) > 0: while len(deps) > 0:
m = deps.pop(0) m = deps.pop(0)
...@@ -212,6 +214,19 @@ class ManifestLoader(object): ...@@ -212,6 +214,19 @@ class ManifestLoader(object):
# Its deps are done, so we can emit it # Its deps are done, so we can emit it
seen.add(m.name) seen.add(m.name)
# Capture system packages as we may need to set PATHs to then later
if (
self.build_opts.allow_system_packages
and self.build_opts.host_type.get_package_manager()
):
packages = m.get_required_system_packages(ctx)
for pkg_type, v in packages.items():
merged = system_packages.get(pkg_type, [])
if v not in merged:
merged += v
system_packages[pkg_type] = merged
# A manifest depends on all system packages in it dependencies as well
m.resolved_system_packages = copy.copy(system_packages)
dep_order.append(m) dep_order.append(m)
return dep_order return dep_order
......
...@@ -77,6 +77,12 @@ SCHEMA = { ...@@ -77,6 +77,12 @@ SCHEMA = {
"manifests_to_build": OPTIONAL, "manifests_to_build": OPTIONAL,
}, },
}, },
"github.actions": {
"optional_section": True,
"fields": {
"run_tests": OPTIONAL,
},
},
"cmake.defines": {"optional_section": True}, "cmake.defines": {"optional_section": True},
"autoconf.args": {"optional_section": True}, "autoconf.args": {"optional_section": True},
"autoconf.envcmd.LDFLAGS": {"optional_section": True}, "autoconf.envcmd.LDFLAGS": {"optional_section": True},
...@@ -115,6 +121,7 @@ ALLOWED_EXPR_SECTIONS = [ ...@@ -115,6 +121,7 @@ ALLOWED_EXPR_SECTIONS = [
"shipit.pathmap", "shipit.pathmap",
"shipit.strip", "shipit.strip",
"homebrew", "homebrew",
"github.actions",
] ]
...@@ -223,6 +230,7 @@ class ManifestParser(object): ...@@ -223,6 +230,7 @@ class ManifestParser(object):
self.fbsource_path = self.get("manifest", "fbsource_path") self.fbsource_path = self.get("manifest", "fbsource_path")
self.shipit_project = self.get("manifest", "shipit_project") self.shipit_project = self.get("manifest", "shipit_project")
self.shipit_fbcode_builder = self.get("manifest", "shipit_fbcode_builder") self.shipit_fbcode_builder = self.get("manifest", "shipit_fbcode_builder")
self.resolved_system_packages = {}
if self.name != os.path.basename(file_name): if self.name != os.path.basename(file_name):
raise Exception( raise Exception(
...@@ -378,6 +386,9 @@ class ManifestParser(object): ...@@ -378,6 +386,9 @@ class ManifestParser(object):
return True return True
def get_repo_url(self, ctx):
return self.get("git", "repo_url", ctx=ctx)
def create_fetcher(self, build_options, ctx): def create_fetcher(self, build_options, ctx):
use_real_shipit = ( use_real_shipit = (
ShipitTransformerFetcher.available() and build_options.use_shipit ShipitTransformerFetcher.available() and build_options.use_shipit
...@@ -409,7 +420,7 @@ class ManifestParser(object): ...@@ -409,7 +420,7 @@ class ManifestParser(object):
if package_fetcher.packages_are_installed(): if package_fetcher.packages_are_installed():
return package_fetcher return package_fetcher
repo_url = self.get("git", "repo_url", ctx=ctx) repo_url = self.get_repo_url(ctx)
if repo_url: if repo_url:
rev = self.get("git", "rev") rev = self.get("git", "rev")
depth = self.get("git", "depth") depth = self.get("git", "depth")
......
...@@ -19,6 +19,8 @@ libboost-all-dev ...@@ -19,6 +19,8 @@ libboost-all-dev
[homebrew] [homebrew]
boost boost
# Boost cmake detection on homebrew adds this as requirement: https://github.com/Homebrew/homebrew-core/issues/67427#issuecomment-754187345
icu4c
[rpms.all(distro=centos_stream,distro_vers=8)] [rpms.all(distro=centos_stream,distro_vers=8)]
boost169 boost169
...@@ -55,6 +57,7 @@ boost169-program-options ...@@ -55,6 +57,7 @@ boost169-program-options
[rpms.not(all(distro=centos_stream,distro_vers=8))] [rpms.not(all(distro=centos_stream,distro_vers=8))]
boost-devel boost-devel
boost-static
[build] [build]
builder = boost builder = boost
......
[manifest]
name = bz2
[debs]
libbz2-dev
[homebrew]
bzip2
[rpms]
bzip2-devel
[download]
url = https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz
sha256 = ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269
[build.not(os=windows)]
builder = make
subdir = bzip2-1.0.8
[make.build_args.os=linux]
# python bz2 support on linux needs dynamic library
-f
Makefile-libbz2_so
[make.install_args]
install
[build.os=windows]
builder = nop
...@@ -7,6 +7,9 @@ shipit_fbcode_builder = true ...@@ -7,6 +7,9 @@ shipit_fbcode_builder = true
[git] [git]
repo_url = https://github.com/facebookexperimental/eden.git repo_url = https://github.com/facebookexperimental/eden.git
[github.actions]
run_tests = off
[build] [build]
builder = cmake builder = cmake
...@@ -37,11 +40,17 @@ sqlite3 ...@@ -37,11 +40,17 @@ sqlite3
[dependencies.os=darwin] [dependencies.os=darwin]
osxfuse osxfuse
[dependencies.not(os=windows)]
# TODO: teach getdeps to compile curl on Windows. # TODO: teach getdeps to compile curl on Windows.
# Enabling curl on Windows requires us to find a way to compile libcurl with # Enabling curl on Windows requires us to find a way to compile libcurl with
# msvc. # msvc.
[dependencies.not(os=windows)]
libcurl libcurl
# Added so that OSS doesn't see system "python" which is python 2 on darwin and some linux
python
[dependencies.all(fb=off)]
# Outside Meta hg is not installed, or if it is, its not the one we want to test with
eden_scm
[shipit.pathmap.fb=on] [shipit.pathmap.fb=on]
# for internal builds that use getdeps # for internal builds that use getdeps
......
[manifest]
name = ncurses
[debs]
libncurses-dev
[homebrew]
ncurses
[rpms]
ncurses-devel
[download]
url = https://ftp.gnu.org/pub/gnu/ncurses/ncurses-6.3.tar.gz
sha256 = 97fc51ac2b085d4cde31ef4d2c3122c21abc217e9090a43a30fc5ec21684e059
[build.not(os=windows)]
builder = autoconf
subdir = ncurses-6.3
[autoconf.args]
--without-cxx-binding
--without-ada
[autoconf.args.os=linux]
--enable-shared
--with-shared
[build.os=windows]
builder = nop
...@@ -6,6 +6,7 @@ libssl-dev ...@@ -6,6 +6,7 @@ libssl-dev
[homebrew] [homebrew]
openssl@1.1 openssl@1.1
# on homebrew need the matching curl and ca-
[rpms] [rpms]
openssl openssl
......
...@@ -36,3 +36,7 @@ libffi ...@@ -36,3 +36,7 @@ libffi
[dependencies] [dependencies]
libffi libffi
# eden tests expect the python bz2 support
bz2
# eden tests expect the python curses support
ncurses
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment