Commit c2270f28 authored by Harvey Hunt's avatar Harvey Hunt Committed by Facebook GitHub Bot

getdeps: Add support for brew packages and define package dependencies

Summary:
In order to speed up build times on a mac, add homebrew support to
getdeps.

Homebrew packages can be declared in a manifest using the `homebrew` header.

Futher, ahornby has added manifest entries for homebrew packages which are
included in this diff and also included a change to use the correct version of
openssl. Without this openssl change, homebrew cmake configure finds an old
openssl 1.0.2 install.

This diff provides a 2x speed up for building folly:

Timings for clean getdeps folly build on mid-2018 2.9Ghz i9 6 core intel macbook pro with 32GB RAM:

With new homebrew system deps:
```
rm -rf /Users/ahornby/.scratch/UsersZahornbyZfbsource/fbcode_builder_getdeps/
time ./opensource/fbcode_builder/getdeps.py build --allow-system-packages folly
real	17m39.329s
user	76m10.317s
sys	5m50.163s
```

Without:
```
rm -rf /Users/ahornby/.scratch/UsersZahornbyZfbsource/fbcode_builder_getdeps/
time ./opensource/fbcode_builder/getdeps.py build folly
real	32m10.344s
user	105m53.448s
sys	15m57.858s
```

Reviewed By: ahornby

Differential Revision: D33842632

fbshipit-source-id: ac785d4a8dcfa31b77292bddd9e747022ac36e3b
parent e711c040
...@@ -78,7 +78,7 @@ if (ZLIB_FOUND) ...@@ -78,7 +78,7 @@ if (ZLIB_FOUND)
list(APPEND CMAKE_REQUIRED_LIBRARIES ${ZLIB_LIBRARIES}) list(APPEND CMAKE_REQUIRED_LIBRARIES ${ZLIB_LIBRARIES})
endif() endif()
find_package(OpenSSL MODULE REQUIRED) find_package(OpenSSL 1.1.1 MODULE REQUIRED)
list(APPEND FOLLY_LINK_LIBRARIES ${OPENSSL_LIBRARIES}) list(APPEND FOLLY_LINK_LIBRARIES ${OPENSSL_LIBRARIES})
list(APPEND FOLLY_INCLUDE_DIRECTORIES ${OPENSSL_INCLUDE_DIR}) list(APPEND FOLLY_INCLUDE_DIRECTORIES ${OPENSSL_INCLUDE_DIR})
list(APPEND CMAKE_REQUIRED_LIBRARIES ${OPENSSL_LIBRARIES}) list(APPEND CMAKE_REQUIRED_LIBRARIES ${OPENSSL_LIBRARIES})
......
...@@ -88,7 +88,7 @@ The settings for folly's cmake build are held in its getdeps manifest `build/fbc ...@@ -88,7 +88,7 @@ The settings for folly's cmake build are held in its getdeps manifest `build/fbc
### Dependencies ### Dependencies
If on Linux you can install system dependencies to save building them: If on Linux or MacOS (with homebrew installed) you can install system dependencies to save building them:
# Clone the repo # Clone the repo
git clone https://github.com/facebook/folly git clone https://github.com/facebook/folly
......
...@@ -415,6 +415,11 @@ class InstallSysDepsCmd(ProjectCmdBase): ...@@ -415,6 +415,11 @@ class InstallSysDepsCmd(ProjectCmdBase):
packages = sorted(set(all_packages["deb"])) packages = sorted(set(all_packages["deb"]))
if packages: if packages:
cmd_args = ["apt", "install", "-y"] + packages cmd_args = ["apt", "install", "-y"] + packages
elif manager == "homebrew":
packages = sorted(set(all_packages["homebrew"]))
if packages:
cmd_args = ["brew", "install"] + packages
else: else:
host_tuple = loader.build_opts.host_type.as_tuple_string() host_tuple = loader.build_opts.host_type.as_tuple_string()
print( print(
......
...@@ -14,7 +14,7 @@ from typing import Optional, Mapping ...@@ -14,7 +14,7 @@ from typing import Optional, Mapping
from .copytree import containing_repo_type from .copytree import containing_repo_type
from .envfuncs import Env, add_path_entry from .envfuncs import Env, add_path_entry
from .fetcher import get_fbsource_repo_data from .fetcher import get_fbsource_repo_data, homebrew_package_prefix
from .manifest import ContextGenerator from .manifest import ContextGenerator
from .platform import HostType, is_windows, get_available_ram from .platform import HostType, is_windows, get_available_ram
...@@ -206,6 +206,12 @@ class BuildOptions(object): ...@@ -206,6 +206,12 @@ class BuildOptions(object):
sdkroot = subprocess.check_output(["xcrun", "--show-sdk-path"]) sdkroot = subprocess.check_output(["xcrun", "--show-sdk-path"])
env["SDKROOT"] = sdkroot.decode().strip() env["SDKROOT"] = sdkroot.decode().strip()
# MacOS includes a version of bison so homebrew won't automatically add
# its own version to PATH. Find where the homebrew bison is and prepend
# it to PATH.
if self.is_darwin() and self.host_type.get_package_manager() == "homebrew":
add_homebrew_package_to_path(env, "bison")
if self.fbsource_dir: if self.fbsource_dir:
env["YARN_YARN_OFFLINE_MIRROR"] = os.path.join( env["YARN_YARN_OFFLINE_MIRROR"] = os.path.join(
self.fbsource_dir, "xplat/third-party/yarn/offline-mirror" self.fbsource_dir, "xplat/third-party/yarn/offline-mirror"
...@@ -300,6 +306,14 @@ class BuildOptions(object): ...@@ -300,6 +306,14 @@ class BuildOptions(object):
if os.path.isfile(cert_file): if os.path.isfile(cert_file):
env["SSL_CERT_FILE"] = cert_file env["SSL_CERT_FILE"] = cert_file
# Try extra hard to find openssl, needed with homebrew on macOS
if (
self.is_darwin()
and "OPENSSL_DIR" not in env
and "OPENSSL_ROOT_DIR" in os.environ
):
env["OPENSSL_ROOT_DIR"] = os.environ["OPENSSL_ROOT_DIR"]
return env return env
...@@ -491,3 +505,9 @@ def setup_build_options(args, host_type=None): ...@@ -491,3 +505,9 @@ def setup_build_options(args, host_type=None):
install_dir=args.install_prefix, install_dir=args.install_prefix,
**build_args **build_args
) )
def add_homebrew_package_to_path(env, package):
prefix = homebrew_package_prefix(package)
if prefix and os.path.exists(os.path.join(prefix, "bin")):
add_path_entry(env, "PATH", os.path.join(prefix, "bin"), append=False)
...@@ -158,6 +158,7 @@ class SystemPackageFetcher(object): ...@@ -158,6 +158,7 @@ class SystemPackageFetcher(object):
def __init__(self, build_options, packages): def __init__(self, build_options, packages):
self.manager = build_options.host_type.get_package_manager() self.manager = build_options.host_type.get_package_manager()
self.packages = packages.get(self.manager) self.packages = packages.get(self.manager)
self.host_type = build_options.host_type
if self.packages: if self.packages:
self.installed = None self.installed = None
else: else:
...@@ -172,6 +173,8 @@ class SystemPackageFetcher(object): ...@@ -172,6 +173,8 @@ class SystemPackageFetcher(object):
cmd = ["rpm", "-q"] + sorted(self.packages) cmd = ["rpm", "-q"] + sorted(self.packages)
elif self.manager == "deb": elif self.manager == "deb":
cmd = ["dpkg", "-s"] + sorted(self.packages) cmd = ["dpkg", "-s"] + sorted(self.packages)
elif self.manager == "homebrew":
cmd = ["brew", "ls", "--versions"] + sorted(self.packages)
if cmd: if cmd:
proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
...@@ -181,9 +184,23 @@ class SystemPackageFetcher(object): ...@@ -181,9 +184,23 @@ class SystemPackageFetcher(object):
else: else:
# Need all packages to be present to consider us installed # Need all packages to be present to consider us installed
self.installed = False self.installed = False
else: else:
self.installed = False self.installed = False
# Hack to make openssl discovery with homebrew work. If openssl was
# built with autoconf we could use autoconf.envcmd.OPENSSL_ROOT_DIR
# from the manifest, but it isn't, so handle the special case here.
if (
self.installed
and self.host_type.is_darwin()
and self.manager == "homebrew"
and "openssl@1.1" in self.packages
):
candidate = homebrew_package_prefix("openssl@1.1")
if os.path.exists(candidate):
os.environ["OPENSSL_ROOT_DIR"] = candidate
return bool(self.installed) return bool(self.installed)
def update(self): def update(self):
...@@ -789,3 +806,14 @@ class ArchiveFetcher(Fetcher): ...@@ -789,3 +806,14 @@ class ArchiveFetcher(Fetcher):
def get_src_dir(self): def get_src_dir(self):
return self.src_dir return self.src_dir
def homebrew_package_prefix(package):
cmd = ["brew", "--prefix", package]
try:
proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
except FileNotFoundError:
return
if proc.returncode == 0:
return proc.stdout.decode("utf-8").rstrip()
...@@ -81,6 +81,7 @@ SCHEMA = { ...@@ -81,6 +81,7 @@ SCHEMA = {
"autoconf.envcmd.LDFLAGS": {"optional_section": True}, "autoconf.envcmd.LDFLAGS": {"optional_section": True},
"rpms": {"optional_section": True}, "rpms": {"optional_section": True},
"debs": {"optional_section": True}, "debs": {"optional_section": True},
"homebrew": {"optional_section": True},
"preinstalled.env": {"optional_section": True}, "preinstalled.env": {"optional_section": True},
"bootstrap.args": {"optional_section": True}, "bootstrap.args": {"optional_section": True},
"b2.args": {"optional_section": True}, "b2.args": {"optional_section": True},
...@@ -112,6 +113,7 @@ ALLOWED_EXPR_SECTIONS = [ ...@@ -112,6 +113,7 @@ ALLOWED_EXPR_SECTIONS = [
"debs", "debs",
"shipit.pathmap", "shipit.pathmap",
"shipit.strip", "shipit.strip",
"homebrew",
] ]
...@@ -358,6 +360,7 @@ class ManifestParser(object): ...@@ -358,6 +360,7 @@ class ManifestParser(object):
return { return {
"rpm": self.get_section_as_args("rpms", ctx), "rpm": self.get_section_as_args("rpms", ctx),
"deb": self.get_section_as_args("debs", ctx), "deb": self.get_section_as_args("debs", ctx),
"homebrew": self.get_section_as_args("homebrew", ctx),
} }
def _is_satisfied_by_preinstalled_environment(self, ctx): def _is_satisfied_by_preinstalled_environment(self, ctx):
......
...@@ -193,8 +193,10 @@ class HostType(object): ...@@ -193,8 +193,10 @@ class HostType(object):
) )
def get_package_manager(self): def get_package_manager(self):
if not self.is_linux(): if not self.is_linux() and not self.is_darwin():
return None return None
if self.is_darwin():
return "homebrew"
if self.distro in ("fedora", "centos", "centos_stream"): if self.distro in ("fedora", "centos", "centos_stream"):
return "rpm" return "rpm"
if self.distro.startswith(("debian", "ubuntu")): if self.distro.startswith(("debian", "ubuntu")):
......
[manifest] [manifest]
name = autoconf name = autoconf
[rpms] [debs]
autoconf autoconf
[debs] [homebrew]
autoconf
[rpms]
autoconf autoconf
[download] [download]
......
[manifest] [manifest]
name = automake name = automake
[rpms] [homebrew]
automake automake
[debs] [debs]
automake automake
[rpms]
automake
[download] [download]
url = http://ftp.gnu.org/gnu/automake/automake-1.16.1.tar.gz url = http://ftp.gnu.org/gnu/automake/automake-1.16.1.tar.gz
sha256 = 608a97523f97db32f1f5d5615c98ca69326ced2054c9f82e65bade7fc4c9dea8 sha256 = 608a97523f97db32f1f5d5615c98ca69326ced2054c9f82e65bade7fc4c9dea8
......
[manifest] [manifest]
name = bison name = bison
[rpms] [debs]
bison bison
[debs] [homebrew]
bison
[rpms]
bison bison
[download.not(os=windows)] [download.not(os=windows)]
......
...@@ -15,6 +15,9 @@ BOOST_ROOT_1_78_0 ...@@ -15,6 +15,9 @@ BOOST_ROOT_1_78_0
[debs] [debs]
libboost-all-dev libboost-all-dev
[homebrew]
boost
[rpms.all(distro=centos_stream,distro_vers=8)] [rpms.all(distro=centos_stream,distro_vers=8)]
boost169 boost169
boost169-math boost169-math
......
[manifest] [manifest]
name = cmake name = cmake
[homebrew]
cmake
[rpms] [rpms]
cmake cmake
......
[manifest] [manifest]
name = cpptoml name = cpptoml
[homebrew]
cpptoml
[download] [download]
url = https://github.com/chadaustin/cpptoml/archive/refs/tags/v0.1.2.tar.gz url = https://github.com/chadaustin/cpptoml/archive/refs/tags/v0.1.2.tar.gz
sha256 = beda37e94f9746874436c8090c045fd80ae6f8a51f7c668c932a2b110a4fc277 sha256 = beda37e94f9746874436c8090c045fd80ae6f8a51f7c668c932a2b110a4fc277
......
...@@ -5,6 +5,9 @@ name = double-conversion ...@@ -5,6 +5,9 @@ name = double-conversion
url = https://github.com/google/double-conversion/archive/v3.1.4.tar.gz url = https://github.com/google/double-conversion/archive/v3.1.4.tar.gz
sha256 = 95004b65e43fefc6100f337a25da27bb99b9ef8d4071a36a33b5e83eb1f82021 sha256 = 95004b65e43fefc6100f337a25da27bb99b9ef8d4071a36a33b5e83eb1f82021
[homebrew]
double-conversion
[rpms] [rpms]
double-conversion double-conversion
double-conversion-devel double-conversion-devel
......
[manifest] [manifest]
name = flex name = flex
[rpms] [debs]
flex flex
[debs] [homebrew]
flex
[rpms]
flex flex
[download.not(os=windows)] [download.not(os=windows)]
......
[manifest] [manifest]
name = libevent name = libevent
[rpms]
libevent-devel
[debs] [debs]
libevent-dev libevent-dev
[homebrew]
libevent
[rpms]
libevent-devel
# Note that the CMakeLists.txt file is present only in # Note that the CMakeLists.txt file is present only in
# git repo and not in the release tarball, so take care # git repo and not in the release tarball, so take care
# to use the github generated source tarball rather than # to use the github generated source tarball rather than
......
[manifest] [manifest]
name = libffi name = libffi
[debs]
libffi-dev
[homebrew]
libffi
[rpms] [rpms]
libffi-devel libffi-devel
libffi libffi
[debs]
libffi-dev
[download] [download]
url = https://github.com/libffi/libffi/releases/download/v3.4.2/libffi-3.4.2.tar.gz url = https://github.com/libffi/libffi/releases/download/v3.4.2/libffi-3.4.2.tar.gz
sha256 = 540fb721619a6aba3bdeef7d940d8e9e0e6d2c193595bc243241b77ff9e93620 sha256 = 540fb721619a6aba3bdeef7d940d8e9e0e6d2c193595bc243241b77ff9e93620
......
[manifest] [manifest]
name = libgit2 name = libgit2
[homebrew]
libgit2
[rpms] [rpms]
libgit2-devel libgit2-devel
......
[manifest] [manifest]
name = libsodium name = libsodium
[debs]
libsodium-dev
[homebrew]
libsodium
[rpms] [rpms]
libsodium-devel libsodium-devel
libsodium-static libsodium-static
[debs]
libsodium-dev
[download.not(os=windows)] [download.not(os=windows)]
url = https://github.com/jedisct1/libsodium/releases/download/1.0.17/libsodium-1.0.17.tar.gz url = https://github.com/jedisct1/libsodium/releases/download/1.0.17/libsodium-1.0.17.tar.gz
sha256 = 0cc3dae33e642cc187b5ceb467e0ad0e1b51dcba577de1190e9ffa17766ac2b1 sha256 = 0cc3dae33e642cc187b5ceb467e0ad0e1b51dcba577de1190e9ffa17766ac2b1
......
[manifest] [manifest]
name = libtool name = libtool
[homebrew]
libtool
[rpms] [rpms]
libtool libtool
......
[manifest] [manifest]
name = libusb name = libusb
[debs]
libusb-1.0-0-dev
[homebrew]
libusb
[rpms] [rpms]
libusb-devel libusb-devel
libusb libusb
[debs]
libusb-1.0-0-dev
[download] [download]
url = https://github.com/libusb/libusb/releases/download/v1.0.22/libusb-1.0.22.tar.bz2 url = https://github.com/libusb/libusb/releases/download/v1.0.22/libusb-1.0.22.tar.bz2
sha256 = 75aeb9d59a4fdb800d329a545c2e6799f732362193b465ea198f2aa275518157 sha256 = 75aeb9d59a4fdb800d329a545c2e6799f732362193b465ea198f2aa275518157
......
[manifest] [manifest]
name = libzmq name = libzmq
[debs]
libzmq3-dev
[homebrew]
zeromq
[rpms] [rpms]
zeromq-devel zeromq-devel
zeromq zeromq
[debs]
libzmq3-dev
[download] [download]
url = https://github.com/zeromq/libzmq/releases/download/v4.3.1/zeromq-4.3.1.tar.gz url = https://github.com/zeromq/libzmq/releases/download/v4.3.1/zeromq-4.3.1.tar.gz
sha256 = bcbabe1e2c7d0eec4ed612e10b94b112dd5f06fcefa994a0c79a45d835cd21eb sha256 = bcbabe1e2c7d0eec4ed612e10b94b112dd5f06fcefa994a0c79a45d835cd21eb
......
[manifest] [manifest]
name = lz4 name = lz4
[homebrew]
lz4
[rpms] [rpms]
lz4-devel lz4-devel
lz4-static lz4-static
......
[manifest] [manifest]
name = lzo name = lzo
[rpms]
lzo-devel
[debs] [debs]
liblzo2-dev liblzo2-dev
[homebrew]
lzo
[rpms]
lzo-devel
[download] [download]
url = http://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz url = http://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz
sha256 = c0f892943208266f9b6543b3ae308fab6284c5c90e627931446fb49b4221a072 sha256 = c0f892943208266f9b6543b3ae308fab6284c5c90e627931446fb49b4221a072
......
[manifest] [manifest]
name = ninja name = ninja
[rpms] [debs]
ninja-build ninja-build
[debs] [homebrew]
ninja
[rpms]
ninja-build ninja-build
[download.os=windows] [download.os=windows]
......
[manifest] [manifest]
name = openssl name = openssl
[debs]
libssl-dev
[homebrew]
openssl@1.1
[rpms] [rpms]
openssl openssl
openssl-devel openssl-devel
openssl-libs openssl-libs
[debs]
libssl-dev
[download] [download]
url = https://www.openssl.org/source/openssl-1.1.1l.tar.gz url = https://www.openssl.org/source/openssl-1.1.1l.tar.gz
sha256 = 0b7a3e5e59c34827fe0c3a74b7ec8baef302b98fa80088d7f9153aa16fa76bd1 sha256 = 0b7a3e5e59c34827fe0c3a74b7ec8baef302b98fa80088d7f9153aa16fa76bd1
......
[manifest] [manifest]
name = pcre name = pcre
[homebrew]
pcre
[rpms] [rpms]
pcre-devel pcre-devel
pcre-static pcre-static
...@@ -15,4 +18,3 @@ sha256 = 0b8e7465dc5e98c757cc3650a20a7843ee4c3edf50aaf60bb33fd879690d2c73 ...@@ -15,4 +18,3 @@ sha256 = 0b8e7465dc5e98c757cc3650a20a7843ee4c3edf50aaf60bb33fd879690d2c73
[build] [build]
builder = cmake builder = cmake
subdir = pcre-8.43 subdir = pcre-8.43
[manifest] [manifest]
name = python name = python
[homebrew]
python@3.8
[rpms] [rpms]
python3 python3
python3-devel python3-devel
......
[manifest] [manifest]
name = re2 name = re2
[rpms] [homebrew]
re2 re2
re2-devel
[debs] [debs]
libre2-dev libre2-dev
[rpms]
re2
re2-devel
[download] [download]
url = https://github.com/google/re2/archive/2019-06-01.tar.gz url = https://github.com/google/re2/archive/2019-06-01.tar.gz
sha256 = 02b7d73126bd18e9fbfe5d6375a8bb13fadaf8e99e48cbb062e4500fc18e8e2e sha256 = 02b7d73126bd18e9fbfe5d6375a8bb13fadaf8e99e48cbb062e4500fc18e8e2e
......
[manifest] [manifest]
name = sqlite3 name = sqlite3
[debs]
libsqlite3-dev
[homebrew]
sqlite
[rpms] [rpms]
sqlite-devel sqlite-devel
sqlite-libs sqlite-libs
[debs]
libsqlite3-dev
[download] [download]
url = https://sqlite.org/2019/sqlite-amalgamation-3280000.zip url = https://sqlite.org/2019/sqlite-amalgamation-3280000.zip
sha256 = d02fc4e95cfef672b45052e221617a050b7f2e20103661cda88387349a9b1327 sha256 = d02fc4e95cfef672b45052e221617a050b7f2e20103661cda88387349a9b1327
......
[manifest] [manifest]
name = tree name = tree
[rpms] [debs]
tree tree
[debs] [homebrew]
tree
[rpms]
tree tree
[download.os=linux] [download.os=linux]
......
[manifest] [manifest]
name = zlib name = zlib
[debs]
zlib1g-dev
[homebrew]
zlib
[rpms] [rpms]
zlib-devel zlib-devel
zlib-static zlib-static
[debs]
zlib1g-dev
[download] [download]
url = http://www.zlib.net/zlib-1.2.11.tar.gz url = http://www.zlib.net/zlib-1.2.11.tar.gz
sha256 = c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1 sha256 = c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1
......
[manifest] [manifest]
name = zstd name = zstd
[homebrew]
zstd
[rpms] [rpms]
libzstd-devel libzstd-devel
libzstd libzstd
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment