Commit 3a777987 authored by Wez Furlong's avatar Wez Furlong Committed by Facebook Github Bot

fbcode_builder: getdeps: add LFS caching for ArchiveFetcher

Summary:
When running in FB infra, prefer to download from our local LFS
server rather than going out to the internet.

Fall back to a normal internet download if the LFS get fails for some reason.

Upload to LFS after successfully verifying the hash for the downloaded archive.

Add a subcommand that performs a fetch for all possible platforms so that it
is easier to ensure that the lfs-pointers file is up to date.

Reviewed By: simpkins

Differential Revision: D14978660

fbshipit-source-id: 240fc32fc7003d1e06c88b80d85054dae36e2f31
parent 13356782
......@@ -583,13 +583,16 @@ class ArchiveFetcher(Fetcher):
"%s: expected sha256 %s but got %s" % (self.url, self.sha256, digest)
)
def _download(self):
def _download_dir(self):
""" returns the download dir, creating it if it doesn't already exist """
download_dir = os.path.dirname(self.file_name)
if not os.path.exists(download_dir):
os.makedirs(download_dir)
return download_dir
def _download(self):
self._download_dir()
download_url_to_file_with_progress(self.url, self.file_name)
self._verify_hash()
def clean(self):
......
......@@ -329,9 +329,20 @@ class ManifestParser(object):
url = self.get("download", "url", ctx=ctx)
if url:
return ArchiveFetcher(
build_options, self, url, self.get("download", "sha256", ctx=ctx)
)
# We need to defer this import until now to avoid triggering
# a cycle when the facebook/__init__.py is loaded.
try:
from getdeps.facebook.lfs import LFSCachingArchiveFetcher
return LFSCachingArchiveFetcher(
build_options, self, url, self.get("download", "sha256", ctx=ctx)
)
except ImportError:
# This FB internal module isn't shippped to github,
# so just use its base class
return ArchiveFetcher(
build_options, self, url, self.get("download", "sha256", ctx=ctx)
)
raise KeyError(
"project %s has no fetcher configuration matching %r" % (self.name, ctx)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment