_downloadablefilesource.py: Support version tracking
diff --git a/buildstream/plugins/sources/_downloadablefilesource.py b/buildstream/plugins/sources/_downloadablefilesource.py
index dba2a73..c7a6229 100644
--- a/buildstream/plugins/sources/_downloadablefilesource.py
+++ b/buildstream/plugins/sources/_downloadablefilesource.py
@@ -4,30 +4,46 @@
import urllib.request
import urllib.error
import contextlib
+import re
import shutil
+from distutils.version import LooseVersion
+
from buildstream import Source, SourceError, Consistency
from buildstream import utils
class DownloadableFileSource(Source):
- COMMON_CONFIG_KEYS = Source.COMMON_CONFIG_KEYS + ['url', 'ref', 'etag']
+ COMMON_CONFIG_KEYS = Source.COMMON_CONFIG_KEYS + ['url', 'ref', 'etag', 'version']
def configure(self, node):
self.original_url = self.node_get_member(node, str, 'url')
+ self.version = self.node_get_member(node, str, 'version', '') or None
self.ref = self.node_get_member(node, str, 'ref', '') or None
self.etag = self.node_get_member(node, str, 'etag', '') or None
self.url = self.translate_url(self.original_url)
+ if '{version}' in os.path.basename(self.url):
+ self.url_format = self.url
+ if self.version:
+ self.alias_url = self.original_url.format(version=self.version)
+ self.url = self.url_format.format(version=self.version)
+ else:
+ self.alias_url = None
+ self.url = None
+ else:
+ self.alias_url = self.original_url
+ self.url_format = None
+
def preflight(self):
return
def get_unique_key(self):
- return [self.original_url, self.ref]
+ return [self.alias_url, self.ref]
def get_consistency(self):
- if self.ref is None:
+ if self.alias_url is None or self.ref is None:
return Consistency.INCONSISTENT
if os.path.isfile(self._get_mirror_file()):
@@ -37,31 +53,64 @@
return Consistency.RESOLVED
def get_ref(self):
- return (self.ref, self.etag)
+ return (self.ref, self.etag, self.version)
def set_ref(self, ref, node):
- self.ref, self.etag = ref
+ self.ref, self.etag, self.version = ref
- node['ref'] = self.ref
+ if self.version:
+ node['version'] = self.version
if self.etag:
node['etag'] = self.etag
+ node['ref'] = self.ref
def track(self):
# there is no 'track' field in the source to determine what/whether
# or not to update refs, because tracking a ref is always a conscious
# decision by the user.
- with self.timed_activity("Tracking {}".format(self.url),
+ with self.timed_activity("Tracking {}".format(self.url_format or self.url),
silent_nested=True):
+ if self.url_format:
+ try:
+ filename_format = os.path.basename(self.url_format)
+ index = filename_format.find('{version}')
+ escaped_prefix = re.escape('href="' + filename_format[:index])
+ escaped_suffix = re.escape(filename_format[index + len('{version}'):] + '"')
+ pattern = re.compile(escaped_prefix + r'([0-9.]+)' + escaped_suffix)
+ request = urllib.request.Request(self.url_format[:-len(filename_format)])
+ with contextlib.closing(urllib.request.urlopen(request)) as response:
+ info = response.info()
+ charset = info.get_content_charset()
+ listing = response.read().decode(charset or 'utf-8')
+
+ new_version = None
+ for match in re.findall(pattern, listing):
+ if new_version is None or LooseVersion(match) > LooseVersion(new_version):
+ new_version = match
+
+ except (urllib.error.URLError, urllib.error.ContentTooShortError, OSError) as e:
+ raise SourceError("{}: Error tracking {}: {}"
+ .format(self, self.url_format, e)) from e
+
+ if new_version is None:
+ raise SourceError("{}: Error tracking {}: Pattern not found"
+ .format(self, self.url_format))
+
+ self.alias_url = self.original_url.format(version=new_version)
+ self.url = self.url_format.format(version=new_version)
+ else:
+ new_version = None
+
new_ref, new_etag = self._ensure_mirror()
- if self.ref and self.ref != new_ref:
+ if self.ref and self.ref != new_ref and self.version == new_version:
detail = "When tracking, new ref differs from current ref:\n" \
+ " Tracked URL: {}\n".format(self.url) \
+ " Current ref: {}\n".format(self.ref) \
+ " New ref: {}\n".format(new_ref)
self.warn("Potential man-in-the-middle attack!", detail=detail)
- return (new_ref, new_etag)
+ return (new_ref, new_etag, new_version)
def fetch(self):
@@ -125,7 +174,7 @@
def _get_mirror_dir(self):
return os.path.join(self.get_mirror_directory(),
- utils.url_directory_name(self.original_url))
+ utils.url_directory_name(self.alias_url))
def _get_mirror_file(self, sha=None):
return os.path.join(self._get_mirror_dir(), sha or self.ref)
diff --git a/buildstream/plugins/sources/tar.py b/buildstream/plugins/sources/tar.py
index 284554f..3f020ae 100644
--- a/buildstream/plugins/sources/tar.py
+++ b/buildstream/plugins/sources/tar.py
@@ -71,7 +71,7 @@
def preflight(self):
self.host_lzip = None
- if self.url.endswith('.lz'):
+ if self.original_url.endswith('.lz'):
self.host_lzip = utils.get_host_tool('lzip')
def get_unique_key(self):
@@ -92,7 +92,7 @@
@contextmanager
def _get_tar(self):
- if self.url.endswith('.lz'):
+ if self.original_url.endswith('.lz'):
with self._run_lzip() as lzip_dec:
with tarfile.open(fileobj=lzip_dec, mode='r:') as tar:
yield tar
