add a new ManifestLoader class

Summary:
Add a new ManifestLoader class to handle loading manifests and computing
dependencies.

For now the main thing this class does is maintain the `manifest_by_name`
mapping.  In subsequent diffs we should be able to move some additional logic
into this class, which will help clean up the code and eliminate some redudant
work.  In particular, we can have this class cache project hashes, which will
avoid re-computing hashes over and over again for the same projects as we do
in many cases today.  We should also be able to save and re-use some of the
project dependency ordering computation in some cases as well.

Reviewed By: strager

Differential Revision: D16477400

fbshipit-source-id: f06f62f77d8443fccaa69fe4c1306e39c395b325

build/fbcode_builder/getdeps.py

@@ -18,7 +18,7 @@ import sys
 from getdeps.buildopts import setup_build_options
 from getdeps.dyndeps import create_dyn_dep_munger
 from getdeps.errors import TransientFailure
-from getdeps.load import load_project, manifests_in_dependency_order
+from getdeps.load import ManifestLoader
 from getdeps.manifest import ManifestParser
 from getdeps.platform import HostType
 from getdeps.subcmd import SubCmd, add_subcommands, cmd
@@ -84,14 +84,14 @@ class FetchCmd(SubCmd):
 
     def run(self, args):
         opts = setup_build_options(args)
-        ctx_gen = opts.get_context_generator()
-        manifest = load_project(opts, args.project)
+        loader = ManifestLoader(opts)
+        manifest = loader.load_manifest(args.project)
         if args.recursive:
-            projects = manifests_in_dependency_order(opts, manifest, ctx_gen)
+            projects = loader.manifests_in_dependency_order()
         else:
             projects = [manifest]
         for m in projects:
-            fetcher = m.create_fetcher(opts, ctx_gen.get_context(m.name))
+            fetcher = m.create_fetcher(opts, loader.ctx_gen.get_context(m.name))
             fetcher.update()
 
 
@@ -99,10 +99,10 @@ class FetchCmd(SubCmd):
 class ListDepsCmd(SubCmd):
     def run(self, args):
         opts = setup_build_options(args)
-        ctx_gen = opts.get_context_generator()
-        ctx_gen.set_value_for_project(args.project, "test", "on")
-        manifest = load_project(opts, args.project)
-        for m in manifests_in_dependency_order(opts, manifest, ctx_gen):
+        loader = ManifestLoader(opts)
+        loader.ctx_gen.set_value_for_project(args.project, "test", "on")
+        loader.load_manifest(args.project)
+        for m in loader.manifests_in_dependency_order():
             print(m.name)
         return 0
 
@@ -142,11 +142,10 @@ class CleanCmd(SubCmd):
 class ShowInstDirCmd(SubCmd):
     def run(self, args):
         opts = setup_build_options(args)
-        ctx_gen = opts.get_context_generator()
-        ctx_gen.set_value_for_project(args.project, "test", "on")
-        manifest = load_project(opts, args.project)
-        projects = manifests_in_dependency_order(opts, manifest, ctx_gen)
-        manifests_by_name = {m.name: m for m in projects}
+        loader = ManifestLoader(opts)
+        loader.ctx_gen.set_value_for_project(args.project, "test", "on")
+        manifest = loader.load_manifest(args.project)
+        projects = loader.manifests_in_dependency_order()
 
         if args.recursive:
             manifests = projects
@@ -154,9 +153,11 @@ class ShowInstDirCmd(SubCmd):
             manifests = [manifest]
 
         for m in manifests:
-            ctx = ctx_gen.get_context(m.name)
+            ctx = loader.ctx_gen.get_context(m.name)
             fetcher = m.create_fetcher(opts, ctx)
-            dirs = opts.compute_dirs(m, fetcher, manifests_by_name, ctx_gen)
+            dirs = opts.compute_dirs(
+                m, fetcher, loader.manifests_by_name, loader.ctx_gen
+            )
             inst_dir = dirs["inst_dir"]
             print(inst_dir)
 
@@ -180,17 +181,17 @@ class ShowInstDirCmd(SubCmd):
 class ShowSourceDirCmd(SubCmd):
     def run(self, args):
         opts = setup_build_options(args)
-        ctx_gen = opts.get_context_generator()
-        ctx_gen.set_value_for_project(args.project, "test", "on")
-        manifest = load_project(opts, args.project)
+        loader = ManifestLoader(opts)
+        loader.ctx_gen.set_value_for_project(args.project, "test", "on")
+        manifest = loader.load_manifest(args.project)
 
         if args.recursive:
-            manifests = manifests_in_dependency_order(opts, manifest, ctx_gen)
+            manifests = loader.manifests_in_dependency_order()
         else:
             manifests = [manifest]
 
         for m in manifests:
-            fetcher = m.create_fetcher(opts, ctx_gen.get_context(m.name))
+            fetcher = m.create_fetcher(opts, loader.ctx_gen.get_context(m.name))
             print(fetcher.get_src_dir())
 
     def setup_parser(self, parser):
@@ -213,17 +214,18 @@ class ShowSourceDirCmd(SubCmd):
 class BuildCmd(SubCmd):
     def run(self, args):
         opts = setup_build_options(args)
-        if args.clean:
-            clean_dirs(opts)
-
         ctx_gen = opts.get_context_generator(facebook_internal=args.facebook_internal)
         if args.enable_tests:
             ctx_gen.set_value_for_project(args.project, "test", "on")
-        manifest = load_project(opts, args.project)
+        loader = ManifestLoader(opts, ctx_gen)
+
+        if args.clean:
+            clean_dirs(opts)
+
+        manifest = loader.load_manifest(args.project)
 
         print("Building on %s" % ctx_gen.get_context(args.project))
-        projects = manifests_in_dependency_order(opts, manifest, ctx_gen)
-        manifests_by_name = {m.name: m for m in projects}
+        projects = loader.manifests_in_dependency_order()
 
         # Accumulate the install directories so that the build steps
         # can find their dep installation
@@ -236,7 +238,7 @@ class BuildCmd(SubCmd):
             if args.clean:
                 fetcher.clean()
 
-            dirs = opts.compute_dirs(m, fetcher, manifests_by_name, ctx_gen)
+            dirs = opts.compute_dirs(m, fetcher, loader.manifests_by_name, ctx_gen)
             build_dir = dirs["build_dir"]
             inst_dir = dirs["inst_dir"]
 
@@ -318,10 +320,10 @@ class FixupDeps(SubCmd):
         if args.enable_tests:
             ctx_gen.set_value_for_project(args.project, "test", "on")
 
-        manifest = load_project(opts, args.project)
+        loader = ManifestLoader(opts, ctx_gen)
+        manifest = loader.load_manifest(args.project)
 
-        projects = manifests_in_dependency_order(opts, manifest, ctx_gen)
-        manifests_by_name = {m.name: m for m in projects}
+        projects = loader.manifests_in_dependency_order()
 
         # Accumulate the install directories so that the build steps
         # can find their dep installation
@@ -331,7 +333,7 @@ class FixupDeps(SubCmd):
             ctx = ctx_gen.get_context(m.name)
             fetcher = m.create_fetcher(opts, ctx)
 
-            dirs = opts.compute_dirs(m, fetcher, manifests_by_name, ctx_gen)
+            dirs = opts.compute_dirs(m, fetcher, loader.manifests_by_name, ctx_gen)
             inst_dir = dirs["inst_dir"]
 
             install_dirs.append(inst_dir)
@@ -376,9 +378,9 @@ class TestCmd(SubCmd):
         else:
             ctx_gen.set_value_for_project(args.project, "test", "on")
 
-        manifest = load_project(opts, args.project)
-        projects = manifests_in_dependency_order(opts, manifest, ctx_gen)
-        manifests_by_name = {m.name: m for m in projects}
+        loader = ManifestLoader(opts, ctx_gen)
+        manifest = loader.load_manifest(args.project)
+        projects = loader.manifests_in_dependency_order()
 
         # Accumulate the install directories so that the test steps
         # can find their dep installation
@@ -388,7 +390,7 @@ class TestCmd(SubCmd):
             ctx = ctx_gen.get_context(m.name)
             fetcher = m.create_fetcher(opts, ctx)
 
-            dirs = opts.compute_dirs(m, fetcher, manifests_by_name, ctx_gen)
+            dirs = opts.compute_dirs(m, fetcher, loader.manifests_by_name, ctx_gen)
             build_dir = dirs["build_dir"]
             inst_dir = dirs["inst_dir"]
 

build/fbcode_builder/getdeps/load.py

@@ -83,18 +83,58 @@ def load_all_manifests(build_opts):
     return LOADER.load_all(build_opts)
 
 
-def manifests_in_dependency_order(build_opts, manifest, ctx_gen):
-    """ Given a manifest, expand its dependencies and return a list
-    of the manifest objects that would need to be built in the order
-    that they would need to be built.  This does not evaluate whether
-    a build is needed; it just returns the list in the order specified
-    by the manifest files. """
-    # A dict to save loading a project multiple times
-    manifests_by_name = {manifest.name: manifest}
+class ManifestLoader(object):
+    """ ManifestLoader stores information about project manifest relationships for a
+    given set of (build options + platform) configuration.
+
+    The ManifestLoader class primarily serves as a location to cache project dependency
+    relationships and project hash values for this build configuration.
+    """
+
+    def __init__(self, build_opts, ctx_gen=None):
+        self._loader = LOADER
+        self.build_opts = build_opts
+        if ctx_gen is None:
+            self.ctx_gen = self.build_opts.get_context_generator()
+        else:
+            self.ctx_gen = ctx_gen
+
+        self.manifests_by_name = {}
+        self._loaded_all = False
+
+    def load_manifest(self, name):
+        manifest = self.manifests_by_name.get(name)
+        if manifest is None:
+            manifest = self._loader.load_project(self.build_opts, name)
+            self.manifests_by_name[name] = manifest
+        return manifest
+
+    def load_all_manifests(self):
+        if not self._loaded_all:
+            self.manifests_by_name = self._loader.load_all(self.build_opts)
+            self._loaded_all = True
+
+        return self.manifests_by_name
+
+    def manifests_in_dependency_order(self, manifest=None):
+        """ Compute all dependencies of the specified project.  Returns a list of the
+        dependencies plus the project itself, in topologically sorted order.
+
+        Each entry in the returned list only depends on projects that appear before it
+        in the list.
+
+        If the input manifest is None, the dependencies for all currently loaded
+        projects will be computed.  i.e., if you call load_all_manifests() followed by
+        manifests_in_dependency_order() this will return a global dependency ordering of
+        all projects.  """
         # The list of deps that have been fully processed
         seen = set()
         # The list of deps which have yet to be evaluated.  This
         # can potentially contain duplicates.
+        if manifest is None:
+            deps = list(self.manifests_by_name.values())
+        else:
+            assert manifest.name in self.manifests_by_name
             deps = [manifest]
         # The list of manifests in dependency order
         dep_order = []
@@ -109,7 +149,7 @@ def manifests_in_dependency_order(build_opts, manifest, ctx_gen):
             # a correct order even if they aren't sorted, but we prefer
             # to produce the same order regardless of how they are listed
             # in the project manifest files.
-        ctx = ctx_gen.get_context(m.name)
+            ctx = self.ctx_gen.get_context(m.name)
             dep_list = sorted(m.get_section_as_dict("dependencies", ctx).keys())
             builder = m.get("build", "builder", ctx=ctx)
             if builder == "cmake":
@@ -125,14 +165,13 @@ def manifests_in_dependency_order(build_opts, manifest, ctx_gen):
                 dep_list.append("libtool")
 
             dep_count = 0
-        for dep in dep_list:
+            for dep_name in dep_list:
                 # If we're not sure whether it is done, queue it up
-            if dep not in seen:
-                if dep not in manifests_by_name:
-                    dep = load_project(build_opts, dep)
-                    manifests_by_name[dep.name] = dep
-                else:
-                    dep = manifests_by_name[dep]
+                if dep_name not in seen:
+                    dep = self.manifests_by_name.get(dep_name)
+                    if dep is None:
+                        dep = self._loader.load_project(self.build_opts, dep_name)
+                        self.manifests_by_name[dep.name] = dep
 
                     deps.append(dep)
                     dep_count += 1