Commit b2c3257c authored by Adam Simpkins's avatar Adam Simpkins Committed by Facebook Github Bot

add a new ManifestLoader class

Summary:
Add a new ManifestLoader class to handle loading manifests and computing
dependencies.

For now the main thing this class does is maintain the `manifest_by_name`
mapping.  In subsequent diffs we should be able to move some additional logic
into this class, which will help clean up the code and eliminate some redudant
work.  In particular, we can have this class cache project hashes, which will
avoid re-computing hashes over and over again for the same projects as we do
in many cases today.  We should also be able to save and re-use some of the
project dependency ordering computation in some cases as well.

Reviewed By: strager

Differential Revision: D16477400

fbshipit-source-id: f06f62f77d8443fccaa69fe4c1306e39c395b325
parent c0c0f5cf
...@@ -18,7 +18,7 @@ import sys ...@@ -18,7 +18,7 @@ import sys
from getdeps.buildopts import setup_build_options from getdeps.buildopts import setup_build_options
from getdeps.dyndeps import create_dyn_dep_munger from getdeps.dyndeps import create_dyn_dep_munger
from getdeps.errors import TransientFailure from getdeps.errors import TransientFailure
from getdeps.load import load_project, manifests_in_dependency_order from getdeps.load import ManifestLoader
from getdeps.manifest import ManifestParser from getdeps.manifest import ManifestParser
from getdeps.platform import HostType from getdeps.platform import HostType
from getdeps.subcmd import SubCmd, add_subcommands, cmd from getdeps.subcmd import SubCmd, add_subcommands, cmd
...@@ -84,14 +84,14 @@ class FetchCmd(SubCmd): ...@@ -84,14 +84,14 @@ class FetchCmd(SubCmd):
def run(self, args): def run(self, args):
opts = setup_build_options(args) opts = setup_build_options(args)
ctx_gen = opts.get_context_generator() loader = ManifestLoader(opts)
manifest = load_project(opts, args.project) manifest = loader.load_manifest(args.project)
if args.recursive: if args.recursive:
projects = manifests_in_dependency_order(opts, manifest, ctx_gen) projects = loader.manifests_in_dependency_order()
else: else:
projects = [manifest] projects = [manifest]
for m in projects: for m in projects:
fetcher = m.create_fetcher(opts, ctx_gen.get_context(m.name)) fetcher = m.create_fetcher(opts, loader.ctx_gen.get_context(m.name))
fetcher.update() fetcher.update()
...@@ -99,10 +99,10 @@ class FetchCmd(SubCmd): ...@@ -99,10 +99,10 @@ class FetchCmd(SubCmd):
class ListDepsCmd(SubCmd): class ListDepsCmd(SubCmd):
def run(self, args): def run(self, args):
opts = setup_build_options(args) opts = setup_build_options(args)
ctx_gen = opts.get_context_generator() loader = ManifestLoader(opts)
ctx_gen.set_value_for_project(args.project, "test", "on") loader.ctx_gen.set_value_for_project(args.project, "test", "on")
manifest = load_project(opts, args.project) loader.load_manifest(args.project)
for m in manifests_in_dependency_order(opts, manifest, ctx_gen): for m in loader.manifests_in_dependency_order():
print(m.name) print(m.name)
return 0 return 0
...@@ -142,11 +142,10 @@ class CleanCmd(SubCmd): ...@@ -142,11 +142,10 @@ class CleanCmd(SubCmd):
class ShowInstDirCmd(SubCmd): class ShowInstDirCmd(SubCmd):
def run(self, args): def run(self, args):
opts = setup_build_options(args) opts = setup_build_options(args)
ctx_gen = opts.get_context_generator() loader = ManifestLoader(opts)
ctx_gen.set_value_for_project(args.project, "test", "on") loader.ctx_gen.set_value_for_project(args.project, "test", "on")
manifest = load_project(opts, args.project) manifest = loader.load_manifest(args.project)
projects = manifests_in_dependency_order(opts, manifest, ctx_gen) projects = loader.manifests_in_dependency_order()
manifests_by_name = {m.name: m for m in projects}
if args.recursive: if args.recursive:
manifests = projects manifests = projects
...@@ -154,9 +153,11 @@ class ShowInstDirCmd(SubCmd): ...@@ -154,9 +153,11 @@ class ShowInstDirCmd(SubCmd):
manifests = [manifest] manifests = [manifest]
for m in manifests: for m in manifests:
ctx = ctx_gen.get_context(m.name) ctx = loader.ctx_gen.get_context(m.name)
fetcher = m.create_fetcher(opts, ctx) fetcher = m.create_fetcher(opts, ctx)
dirs = opts.compute_dirs(m, fetcher, manifests_by_name, ctx_gen) dirs = opts.compute_dirs(
m, fetcher, loader.manifests_by_name, loader.ctx_gen
)
inst_dir = dirs["inst_dir"] inst_dir = dirs["inst_dir"]
print(inst_dir) print(inst_dir)
...@@ -180,17 +181,17 @@ class ShowInstDirCmd(SubCmd): ...@@ -180,17 +181,17 @@ class ShowInstDirCmd(SubCmd):
class ShowSourceDirCmd(SubCmd): class ShowSourceDirCmd(SubCmd):
def run(self, args): def run(self, args):
opts = setup_build_options(args) opts = setup_build_options(args)
ctx_gen = opts.get_context_generator() loader = ManifestLoader(opts)
ctx_gen.set_value_for_project(args.project, "test", "on") loader.ctx_gen.set_value_for_project(args.project, "test", "on")
manifest = load_project(opts, args.project) manifest = loader.load_manifest(args.project)
if args.recursive: if args.recursive:
manifests = manifests_in_dependency_order(opts, manifest, ctx_gen) manifests = loader.manifests_in_dependency_order()
else: else:
manifests = [manifest] manifests = [manifest]
for m in manifests: for m in manifests:
fetcher = m.create_fetcher(opts, ctx_gen.get_context(m.name)) fetcher = m.create_fetcher(opts, loader.ctx_gen.get_context(m.name))
print(fetcher.get_src_dir()) print(fetcher.get_src_dir())
def setup_parser(self, parser): def setup_parser(self, parser):
...@@ -213,17 +214,18 @@ class ShowSourceDirCmd(SubCmd): ...@@ -213,17 +214,18 @@ class ShowSourceDirCmd(SubCmd):
class BuildCmd(SubCmd): class BuildCmd(SubCmd):
def run(self, args): def run(self, args):
opts = setup_build_options(args) opts = setup_build_options(args)
if args.clean:
clean_dirs(opts)
ctx_gen = opts.get_context_generator(facebook_internal=args.facebook_internal) ctx_gen = opts.get_context_generator(facebook_internal=args.facebook_internal)
if args.enable_tests: if args.enable_tests:
ctx_gen.set_value_for_project(args.project, "test", "on") ctx_gen.set_value_for_project(args.project, "test", "on")
manifest = load_project(opts, args.project) loader = ManifestLoader(opts, ctx_gen)
if args.clean:
clean_dirs(opts)
manifest = loader.load_manifest(args.project)
print("Building on %s" % ctx_gen.get_context(args.project)) print("Building on %s" % ctx_gen.get_context(args.project))
projects = manifests_in_dependency_order(opts, manifest, ctx_gen) projects = loader.manifests_in_dependency_order()
manifests_by_name = {m.name: m for m in projects}
# Accumulate the install directories so that the build steps # Accumulate the install directories so that the build steps
# can find their dep installation # can find their dep installation
...@@ -236,7 +238,7 @@ class BuildCmd(SubCmd): ...@@ -236,7 +238,7 @@ class BuildCmd(SubCmd):
if args.clean: if args.clean:
fetcher.clean() fetcher.clean()
dirs = opts.compute_dirs(m, fetcher, manifests_by_name, ctx_gen) dirs = opts.compute_dirs(m, fetcher, loader.manifests_by_name, ctx_gen)
build_dir = dirs["build_dir"] build_dir = dirs["build_dir"]
inst_dir = dirs["inst_dir"] inst_dir = dirs["inst_dir"]
...@@ -318,10 +320,10 @@ class FixupDeps(SubCmd): ...@@ -318,10 +320,10 @@ class FixupDeps(SubCmd):
if args.enable_tests: if args.enable_tests:
ctx_gen.set_value_for_project(args.project, "test", "on") ctx_gen.set_value_for_project(args.project, "test", "on")
manifest = load_project(opts, args.project) loader = ManifestLoader(opts, ctx_gen)
manifest = loader.load_manifest(args.project)
projects = manifests_in_dependency_order(opts, manifest, ctx_gen) projects = loader.manifests_in_dependency_order()
manifests_by_name = {m.name: m for m in projects}
# Accumulate the install directories so that the build steps # Accumulate the install directories so that the build steps
# can find their dep installation # can find their dep installation
...@@ -331,7 +333,7 @@ class FixupDeps(SubCmd): ...@@ -331,7 +333,7 @@ class FixupDeps(SubCmd):
ctx = ctx_gen.get_context(m.name) ctx = ctx_gen.get_context(m.name)
fetcher = m.create_fetcher(opts, ctx) fetcher = m.create_fetcher(opts, ctx)
dirs = opts.compute_dirs(m, fetcher, manifests_by_name, ctx_gen) dirs = opts.compute_dirs(m, fetcher, loader.manifests_by_name, ctx_gen)
inst_dir = dirs["inst_dir"] inst_dir = dirs["inst_dir"]
install_dirs.append(inst_dir) install_dirs.append(inst_dir)
...@@ -376,9 +378,9 @@ class TestCmd(SubCmd): ...@@ -376,9 +378,9 @@ class TestCmd(SubCmd):
else: else:
ctx_gen.set_value_for_project(args.project, "test", "on") ctx_gen.set_value_for_project(args.project, "test", "on")
manifest = load_project(opts, args.project) loader = ManifestLoader(opts, ctx_gen)
projects = manifests_in_dependency_order(opts, manifest, ctx_gen) manifest = loader.load_manifest(args.project)
manifests_by_name = {m.name: m for m in projects} projects = loader.manifests_in_dependency_order()
# Accumulate the install directories so that the test steps # Accumulate the install directories so that the test steps
# can find their dep installation # can find their dep installation
...@@ -388,7 +390,7 @@ class TestCmd(SubCmd): ...@@ -388,7 +390,7 @@ class TestCmd(SubCmd):
ctx = ctx_gen.get_context(m.name) ctx = ctx_gen.get_context(m.name)
fetcher = m.create_fetcher(opts, ctx) fetcher = m.create_fetcher(opts, ctx)
dirs = opts.compute_dirs(m, fetcher, manifests_by_name, ctx_gen) dirs = opts.compute_dirs(m, fetcher, loader.manifests_by_name, ctx_gen)
build_dir = dirs["build_dir"] build_dir = dirs["build_dir"]
inst_dir = dirs["inst_dir"] inst_dir = dirs["inst_dir"]
......
...@@ -83,68 +83,107 @@ def load_all_manifests(build_opts): ...@@ -83,68 +83,107 @@ def load_all_manifests(build_opts):
return LOADER.load_all(build_opts) return LOADER.load_all(build_opts)
def manifests_in_dependency_order(build_opts, manifest, ctx_gen): class ManifestLoader(object):
""" Given a manifest, expand its dependencies and return a list """ ManifestLoader stores information about project manifest relationships for a
of the manifest objects that would need to be built in the order given set of (build options + platform) configuration.
that they would need to be built. This does not evaluate whether
a build is needed; it just returns the list in the order specified The ManifestLoader class primarily serves as a location to cache project dependency
by the manifest files. """ relationships and project hash values for this build configuration.
# A dict to save loading a project multiple times """
manifests_by_name = {manifest.name: manifest}
# The list of deps that have been fully processed def __init__(self, build_opts, ctx_gen=None):
seen = set() self._loader = LOADER
# The list of deps which have yet to be evaluated. This self.build_opts = build_opts
# can potentially contain duplicates. if ctx_gen is None:
deps = [manifest] self.ctx_gen = self.build_opts.get_context_generator()
# The list of manifests in dependency order else:
dep_order = [] self.ctx_gen = ctx_gen
while len(deps) > 0: self.manifests_by_name = {}
m = deps.pop(0) self._loaded_all = False
if m.name in seen:
continue def load_manifest(self, name):
manifest = self.manifests_by_name.get(name)
# Consider its deps, if any. if manifest is None:
# We sort them for increased determinism; we'll produce manifest = self._loader.load_project(self.build_opts, name)
# a correct order even if they aren't sorted, but we prefer self.manifests_by_name[name] = manifest
# to produce the same order regardless of how they are listed return manifest
# in the project manifest files.
ctx = ctx_gen.get_context(m.name) def load_all_manifests(self):
dep_list = sorted(m.get_section_as_dict("dependencies", ctx).keys()) if not self._loaded_all:
builder = m.get("build", "builder", ctx=ctx) self.manifests_by_name = self._loader.load_all(self.build_opts)
if builder == "cmake": self._loaded_all = True
dep_list.append("cmake")
elif builder == "autoconf" and m.name not in ( return self.manifests_by_name
"autoconf",
"libtool", def manifests_in_dependency_order(self, manifest=None):
"automake", """ Compute all dependencies of the specified project. Returns a list of the
): dependencies plus the project itself, in topologically sorted order.
# they need libtool and its deps (automake, autoconf) so add
# those as deps (but obviously not if we're building those Each entry in the returned list only depends on projects that appear before it
# projects themselves) in the list.
dep_list.append("libtool")
If the input manifest is None, the dependencies for all currently loaded
dep_count = 0 projects will be computed. i.e., if you call load_all_manifests() followed by
for dep in dep_list: manifests_in_dependency_order() this will return a global dependency ordering of
# If we're not sure whether it is done, queue it up all projects. """
if dep not in seen: # The list of deps that have been fully processed
if dep not in manifests_by_name: seen = set()
dep = load_project(build_opts, dep) # The list of deps which have yet to be evaluated. This
manifests_by_name[dep.name] = dep # can potentially contain duplicates.
else: if manifest is None:
dep = manifests_by_name[dep] deps = list(self.manifests_by_name.values())
else:
deps.append(dep) assert manifest.name in self.manifests_by_name
dep_count += 1 deps = [manifest]
# The list of manifests in dependency order
if dep_count > 0: dep_order = []
# If we queued anything, re-queue this item, as it depends
# those new item(s) and their transitive deps. while len(deps) > 0:
deps.append(m) m = deps.pop(0)
continue if m.name in seen:
continue
# Its deps are done, so we can emit it
seen.add(m.name) # Consider its deps, if any.
dep_order.append(m) # We sort them for increased determinism; we'll produce
# a correct order even if they aren't sorted, but we prefer
return dep_order # to produce the same order regardless of how they are listed
# in the project manifest files.
ctx = self.ctx_gen.get_context(m.name)
dep_list = sorted(m.get_section_as_dict("dependencies", ctx).keys())
builder = m.get("build", "builder", ctx=ctx)
if builder == "cmake":
dep_list.append("cmake")
elif builder == "autoconf" and m.name not in (
"autoconf",
"libtool",
"automake",
):
# they need libtool and its deps (automake, autoconf) so add
# those as deps (but obviously not if we're building those
# projects themselves)
dep_list.append("libtool")
dep_count = 0
for dep_name in dep_list:
# If we're not sure whether it is done, queue it up
if dep_name not in seen:
dep = self.manifests_by_name.get(dep_name)
if dep is None:
dep = self._loader.load_project(self.build_opts, dep_name)
self.manifests_by_name[dep.name] = dep
deps.append(dep)
dep_count += 1
if dep_count > 0:
# If we queued anything, re-queue this item, as it depends
# those new item(s) and their transitive deps.
deps.append(m)
continue
# Its deps are done, so we can emit it
seen.add(m.name)
dep_order.append(m)
return dep_order
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment