fbcode_builder.py 18.7 KB
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536
#!/usr/bin/env python
# Copyright (c) Facebook, Inc. and its affiliates.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

"""

This is a small DSL to describe builds of Facebook's open-source projects
that are published to Github from a single internal repo, including projects
that depend on folly, wangle, proxygen, fbthrift, etc.

This file defines the interface of the DSL, and common utilieis, but you
will have to instantiate a specific builder, with specific options, in
order to get work done -- see e.g. make_docker_context.py.

== Design notes ==

Goals:

 - A simple declarative language for what needs to be checked out & built,
   how, in what order.

 - The same specification should work for external continuous integration
   builds (e.g. Travis + Docker) and for internal VM-based continuous
   integration builds.

 - One should be able to build without root, and to install to a prefix.

Non-goals:

 - General usefulness. The only point of this is to make it easier to build
   and test Facebook's open-source services.

Ideas for the future -- these may not be very good :)

 - Especially on Ubuntu 14.04 the current initial setup is inefficient:
   we add PPAs after having installed a bunch of packages -- this prompts
   reinstalls of large amounts of code.  We also `apt-get update` a few
   times.

 - A "shell script" builder. Like DockerFBCodeBuilder, but outputs a
   shell script that runs outside of a container. Or maybe even
   synchronously executes the shell commands, `make`-style.

 - A "Makefile" generator. That might make iterating on builds even quicker
   than what you can currently get with Docker build caching.

 - Generate a rebuild script that can be run e.g. inside the built Docker
   container by tagging certain steps with list-inheriting Python objects:
     * do change directories
     * do NOT `git clone` -- if we want to update code this should be a
       separate script that e.g. runs rebase on top of specific targets
       across all the repos.
     * do NOT install software (most / all setup can be skipped)
     * do NOT `autoreconf` or `configure`
     * do `make` and `cmake`

 - If we get non-Debian OSes, part of ccache setup should be factored out.
"""

import os
import re

from shell_quoting import path_join, shell_join, ShellQuoted


def _read_project_github_hashes():
    base_dir = "deps/github_hashes/"  # trailing slash used in regex below
    for dirname, _, files in os.walk(base_dir):
        for filename in files:
            path = os.path.join(dirname, filename)
            with open(path) as f:
                m_proj = re.match("^" + base_dir + "(.*)-rev\.txt$", path)
                if m_proj is None:
                    raise RuntimeError("Not a hash file? {0}".format(path))
                m_hash = re.match("^Subproject commit ([0-9a-f]+)\n$", f.read())
                if m_hash is None:
                    raise RuntimeError("No hash in {0}".format(path))
                yield m_proj.group(1), m_hash.group(1)


class FBCodeBuilder(object):
    def __init__(self, **kwargs):
        self._options_do_not_access = kwargs  # Use .option() instead.
        # This raises upon detecting options that are specified but unused,
        # because otherwise it is very easy to make a typo in option names.
        self.options_used = set()
        # Mark 'projects_dir' used even if the build installs no github
        # projects.  This is needed because driver programs like
        # `shell_builder.py` unconditionally set this for all builds.
        self._github_dir = self.option("projects_dir")
        self._github_hashes = dict(_read_project_github_hashes())

    def __repr__(self):
        return "{0}({1})".format(
            self.__class__.__name__,
            ", ".join(
                "{0}={1}".format(k, repr(v))
                for k, v in self._options_do_not_access.items()
            ),
        )

    def option(self, name, default=None):
        value = self._options_do_not_access.get(name, default)
        if value is None:
            raise RuntimeError("Option {0} is required".format(name))
        self.options_used.add(name)
        return value

    def has_option(self, name):
        return name in self._options_do_not_access

    def add_option(self, name, value):
        if name in self._options_do_not_access:
            raise RuntimeError("Option {0} already set".format(name))
        self._options_do_not_access[name] = value

    #
    # Abstract parts common to every installation flow
    #

    def render(self, steps):
        """

        Converts nested actions to your builder's expected output format.
        Typically takes the output of build().

        """
        res = self._render_impl(steps)  # Implementation-dependent
        # Now that the output is rendered, we expect all options to have
        # been used.
        unused_options = set(self._options_do_not_access)
        unused_options -= self.options_used
        if unused_options:
            raise RuntimeError(
                "Unused options: {0} -- please check if you made a typo "
                "in any of them. Those that are truly not useful should "
                "be not be set so that this typo detection can be useful.".format(
                    unused_options
                )
            )
        return res

    def build(self, steps):
        if not steps:
            raise RuntimeError(
                "Please ensure that the config you are passing " "contains steps"
            )
        return [self.setup(), self.diagnostics()] + steps

    def setup(self):
        "Your builder may want to install packages here."
        raise NotImplementedError

    def diagnostics(self):
        "Log some system diagnostics before/after setup for ease of debugging"
        # The builder's repr is not used in a command to avoid pointlessly
        # invalidating Docker's build cache.
        return self.step(
            "Diagnostics",
            [
                self.comment("Builder {0}".format(repr(self))),
                self.run(ShellQuoted("hostname")),
                self.run(ShellQuoted("cat /etc/issue || echo no /etc/issue")),
                self.run(ShellQuoted("g++ --version || echo g++ not installed")),
                self.run(ShellQuoted("cmake --version || echo cmake not installed")),
            ],
        )

    def step(self, name, actions):
        "A labeled collection of actions or other steps"
        raise NotImplementedError

    def run(self, shell_cmd):
        "Run this bash command"
        raise NotImplementedError

    def set_env(self, key, value):
        'Set the environment "key" to value "value"'
        raise NotImplementedError

    def workdir(self, dir):
        "Create this directory if it does not exist, and change into it"
        raise NotImplementedError

    def copy_local_repo(self, dir, dest_name):
        """
        Copy the local repo at `dir` into this step's `workdir()`, analog of:
          cp -r /path/to/folly folly
        """
        raise NotImplementedError

    def python_deps(self):
        return [
            "wheel",
            "cython==0.28.6",
        ]

    def debian_deps(self):
        return [
            "autoconf-archive",
            "bison",
            "build-essential",
            "cmake",
            "curl",
            "flex",
            "git",
            "gperf",
            "joe",
            "libboost-all-dev",
            "libcap-dev",
            "libdouble-conversion-dev",
            "libevent-dev",
            "libgflags-dev",
            "libgoogle-glog-dev",
            "libkrb5-dev",
            "libpcre3-dev",
            "libpthread-stubs0-dev",
            "libnuma-dev",
            "libsasl2-dev",
            "libsnappy-dev",
            "libsqlite3-dev",
            "libssl-dev",
            "libtool",
            "netcat-openbsd",
            "pkg-config",
            "sudo",
            "unzip",
            "wget",
            "python3-venv",
        ]

    #
    # Specific build helpers
    #

    def install_debian_deps(self):
        actions = [
            self.run(
                ShellQuoted("apt-get update && apt-get install -yq {deps}").format(
                    deps=shell_join(
                        " ", (ShellQuoted(dep) for dep in self.debian_deps())
                    )
                )
            ),
        ]
        gcc_version = self.option("gcc_version")

        # Make the selected GCC the default before building anything
        actions.extend(
            [
                self.run(
                    ShellQuoted("apt-get install -yq {c} {cpp}").format(
                        c=ShellQuoted("gcc-{v}").format(v=gcc_version),
                        cpp=ShellQuoted("g++-{v}").format(v=gcc_version),
                    )
                ),
                self.run(
                    ShellQuoted(
                        "update-alternatives --install /usr/bin/gcc gcc {c} 40 "
                        "--slave /usr/bin/g++ g++ {cpp}"
                    ).format(
                        c=ShellQuoted("/usr/bin/gcc-{v}").format(v=gcc_version),
                        cpp=ShellQuoted("/usr/bin/g++-{v}").format(v=gcc_version),
                    )
                ),
                self.run(ShellQuoted("update-alternatives --config gcc")),
            ]
        )

        actions.extend(self.debian_ccache_setup_steps())

        return self.step("Install packages for Debian-based OS", actions)

    def create_python_venv(self):
        actions = []
        if self.option("PYTHON_VENV", "OFF") == "ON":
            actions.append(
                self.run(
                    ShellQuoted("python3 -m venv {p}").format(
                        p=path_join(self.option("prefix"), "venv")
                    )
                )
            )
        return actions

    def python_venv(self):
        actions = []
        if self.option("PYTHON_VENV", "OFF") == "ON":
            actions.append(
                ShellQuoted("source {p}").format(
                    p=path_join(self.option("prefix"), "venv", "bin", "activate")
                )
            )

            actions.append(
                self.run(
                    ShellQuoted("python3 -m pip install {deps}").format(
                        deps=shell_join(
                            " ", (ShellQuoted(dep) for dep in self.python_deps())
                        )
                    )
                )
            )
        return actions

    def enable_rust_toolchain(self, toolchain="stable", is_bootstrap=True):
        choices = set(["stable", "beta", "nightly"])

        assert toolchain in choices, (
            "while enabling rust toolchain: {} is not in {}"
        ).format(toolchain, choices)

        rust_toolchain_opt = (toolchain, is_bootstrap)
        prev_opt = self.option("rust_toolchain", rust_toolchain_opt)
        assert prev_opt == rust_toolchain_opt, (
            "while enabling rust toolchain: previous toolchain already set to"
            " {}, but trying to set it to {} now"
        ).format(prev_opt, rust_toolchain_opt)

        self.add_option("rust_toolchain", rust_toolchain_opt)

    def rust_toolchain(self):
        actions = []
        if self.option("rust_toolchain", False):
            (toolchain, is_bootstrap) = self.option("rust_toolchain")
            rust_dir = path_join(self.option("prefix"), "rust")
            actions = [
                self.set_env("CARGO_HOME", rust_dir),
                self.set_env("RUSTUP_HOME", rust_dir),
                self.set_env("RUSTC_BOOTSTRAP", "1" if is_bootstrap else "0"),
                self.run(
                    ShellQuoted(
                        "curl -sSf https://build.travis-ci.com/files/rustup-init.sh"
                        " | sh -s --"
                        "   --default-toolchain={r} "
                        "   --profile=minimal"
                        "   --no-modify-path"
                        "   -y"
                    ).format(p=rust_dir, r=toolchain)
                ),
                self.set_env(
                    "PATH",
                    ShellQuoted("{p}:$PATH").format(p=path_join(rust_dir, "bin")),
                ),
                self.run(ShellQuoted("rustup update")),
                self.run(ShellQuoted("rustc --version")),
                self.run(ShellQuoted("rustup --version")),
                self.run(ShellQuoted("cargo --version")),
            ]
        return actions

    def debian_ccache_setup_steps(self):
        return []  # It's ok to ship a renderer without ccache support.

    def github_project_workdir(self, project, path):
        # Only check out a non-default branch if requested. This especially
        # makes sense when building from a local repo.
        git_hash = self.option(
            "{0}:git_hash".format(project),
            # Any repo that has a hash in deps/github_hashes defaults to
            # that, with the goal of making builds maximally consistent.
            self._github_hashes.get(project, ""),
        )
        maybe_change_branch = (
            [
                self.run(ShellQuoted("git checkout {hash}").format(hash=git_hash)),
            ]
            if git_hash
            else []
        )

        local_repo_dir = self.option("{0}:local_repo_dir".format(project), "")
        return self.step(
            "Check out {0}, workdir {1}".format(project, path),
            [
                self.workdir(self._github_dir),
                self.run(
                    ShellQuoted("git clone {opts} https://github.com/{p}").format(
                        p=project,
                        opts=ShellQuoted(
                            self.option("{}:git_clone_opts".format(project), "")
                        ),
                    )
                )
                if not local_repo_dir
                else self.copy_local_repo(local_repo_dir, os.path.basename(project)),
                self.workdir(
                    path_join(self._github_dir, os.path.basename(project), path),
                ),
            ]
            + maybe_change_branch,
        )

    def fb_github_project_workdir(self, project_and_path, github_org="facebook"):
        "This helper lets Facebook-internal CI special-cases FB projects"
        project, path = project_and_path.split("/", 1)
        return self.github_project_workdir(github_org + "/" + project, path)

    def _make_vars(self, make_vars):
        return shell_join(
            " ",
            (
                ShellQuoted("{k}={v}").format(k=k, v=v)
                for k, v in ({} if make_vars is None else make_vars).items()
            ),
        )

    def parallel_make(self, make_vars=None):
        return self.run(
            ShellQuoted("make -j {n} VERBOSE=1 {vars}").format(
                n=self.option("make_parallelism"),
                vars=self._make_vars(make_vars),
            )
        )

    def make_and_install(self, make_vars=None):
        return [
            self.parallel_make(make_vars),
            self.run(
                ShellQuoted("make install VERBOSE=1 {vars}").format(
                    vars=self._make_vars(make_vars),
                )
            ),
        ]

    def configure(self, name=None):
        autoconf_options = {}
        if name is not None:
            autoconf_options.update(
                self.option("{0}:autoconf_options".format(name), {})
            )
        return [
            self.run(
                ShellQuoted(
                    'LDFLAGS="$LDFLAGS -L"{p}"/lib -Wl,-rpath="{p}"/lib" '
                    'CFLAGS="$CFLAGS -I"{p}"/include" '
                    'CPPFLAGS="$CPPFLAGS -I"{p}"/include" '
                    "PY_PREFIX={p} "
                    "./configure --prefix={p} {args}"
                ).format(
                    p=self.option("prefix"),
                    args=shell_join(
                        " ",
                        (
                            ShellQuoted("{k}={v}").format(k=k, v=v)
                            for k, v in autoconf_options.items()
                        ),
                    ),
                )
            ),
        ]

    def autoconf_install(self, name):
        return self.step(
            "Build and install {0}".format(name),
            [
                self.run(ShellQuoted("autoreconf -ivf")),
            ]
            + self.configure()
            + self.make_and_install(),
        )

    def cmake_configure(self, name, cmake_path=".."):
        cmake_defines = {
            "BUILD_SHARED_LIBS": "ON",
            "CMAKE_INSTALL_PREFIX": self.option("prefix"),
        }

        # Hacks to add thriftpy3 support
        if "BUILD_THRIFT_PY3" in os.environ and "folly" in name:
            cmake_defines["PYTHON_EXTENSIONS"] = "True"

        if "BUILD_THRIFT_PY3" in os.environ and "fbthrift" in name:
            cmake_defines["thriftpy3"] = "ON"

        cmake_defines.update(self.option("{0}:cmake_defines".format(name), {}))
        return [
            self.run(
                ShellQuoted(
                    'CXXFLAGS="$CXXFLAGS -fPIC -isystem "{p}"/include" '
                    'CFLAGS="$CFLAGS -fPIC -isystem "{p}"/include" '
                    "cmake {args} {cmake_path}"
                ).format(
                    p=self.option("prefix"),
                    args=shell_join(
                        " ",
                        (
                            ShellQuoted("-D{k}={v}").format(k=k, v=v)
                            for k, v in cmake_defines.items()
                        ),
                    ),
                    cmake_path=cmake_path,
                )
            ),
        ]

    def cmake_install(self, name, cmake_path=".."):
        return self.step(
            "Build and install {0}".format(name),
            self.cmake_configure(name, cmake_path) + self.make_and_install(),
        )

    def cargo_build(self, name):
        return self.step(
            "Build {0}".format(name),
            [
                self.run(
                    ShellQuoted("cargo build -j {n}").format(
                        n=self.option("make_parallelism")
                    )
                )
            ],
        )

    def fb_github_autoconf_install(self, project_and_path, github_org="facebook"):
        return [
            self.fb_github_project_workdir(project_and_path, github_org),
            self.autoconf_install(project_and_path),
        ]

    def fb_github_cmake_install(
        self, project_and_path, cmake_path="..", github_org="facebook"
    ):
        return [
            self.fb_github_project_workdir(project_and_path, github_org),
            self.cmake_install(project_and_path, cmake_path),
        ]

    def fb_github_cargo_build(self, project_and_path, github_org="facebook"):
        return [
            self.fb_github_project_workdir(project_and_path, github_org),
            self.cargo_build(project_and_path),
        ]