1 files changed, 135 insertions, 0 deletions
diff --git a/lib/swupd/bundles.py b/lib/swupd/bundles.py
index 611f59d..78ffaa5 100644
--- a/lib/swupd/bundles.py
+++ b/lib/swupd/bundles.py
@@ -1,6 +1,10 @@
 import glob
+import re
 import subprocess
 import shutil
+import urllib.request
+import urllib.error
+from bb.utils import export_proxies
 from oe.package_manager import RpmPM
 from oe.package_manager import OpkgPM
 from oe.package_manager import DpkgPM
@@ -159,3 +163,134 @@ def copy_old_versions(d):
             output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
             if output:
                 bb.fatal('Unexpected output from the following command:\n%s\n%s' % (cmd, output))
+
+def download_manifests(content_url, version, component, to_dir):
+    """
+    Download one manifest file and recursively all manifests referenced by it.
+    Does not overwrite existing files. Unpacks on-the-fly using bsdtar
+    and thus is independent of the compression format, as long as bsdtar
+    recognizes it.
+    """
+    source = '%s/%d/Manifest.%s.tar' % (content_url, version, component)
+    target = os.path.join(to_dir, 'Manifest.%s' % component)
+    base_versions = set()
+    if not os.path.exists(target):
+        bb.debug(1, 'Downloading %s -> %s' % (source, target))
+        response = urllib.request.urlopen(source)
+        archive = response.read()
+        bb.utils.mkdirhier(to_dir)
+        with open(target + '.tar', 'wb') as tarfile:
+            tarfile.write(archive)
+        bsdtar = subprocess.Popen(['bsdtar', '-xf', '-', '-C', to_dir],
+                                  stdin=subprocess.PIPE,
+                                  stdout=subprocess.PIPE,
+                                  stderr=subprocess.STDOUT)
+        output, _ = bsdtar.communicate(archive)
+        if output or bsdtar.returncode:
+            bb.fatal('Unpacking %s with bsdtar failed:\n%s' % (source, output.decode('utf-8')))
+    with open(target) as f:
+        # Matches the header. We might be parsing Manifest.os-core from build
+        # 1000, but the actual Manifest could be from "version: 900", so get
+        # that as base version, too.
+        version_re = re.compile(r'^(?:previous|version):\s+(\d+)\n$')
+        # Matches the individual entries.
+        manifest_re = re.compile(r'^M.*\s(\d+)\s+(\S+)\n$')
+        for line in f.readlines():
+            m = manifest_re.match(line)
+            if m:
+                subversion = int(m.group(1))
+                submanifest = m.group(2)
+                download_manifests(content_url, subversion, submanifest, to_dir)
+                base_versions.add(subversion)
+            else:
+                m = version_re.match(line)
+                if m:
+                    base_versions.add(int(m.group(1)))
+    return base_versions
+
+def download_old_versions(d):
+    """
+    Download the necessary information from the update repo that is needed
+    to build updates in that update stream. This can run in parallel to
+    a normal build and thus is not on the critical path.
+    """
+
+    content_url = d.getVar('SWUPD_CONTENT_URL', True)
+    version_url = d.getVar('SWUPD_VERSION_URL', True)
+    current_format = int(d.getVar('SWUPD_FORMAT', True))
+    deploy_dir = d.getVar('DEPLOY_DIR_SWUPD', True)
+    www_dir = os.path.join(deploy_dir, 'www')
+
+    if not content_url or not version_url:
+        bb.warn('SWUPD_CONTENT_URL and/or SWUPD_VERSION_URL not set, skipping download of old versions for the initial build of a swupd update stream.')
+        return
+
+    # Avoid double // in path. At least twisted is sensitive to that.
+    content_url = content_url.rstrip('/')
+
+    # Set up env variables with proxy information for use in urllib.
+    export_proxies(d)
+
+    # Find latest version for each of the older formats.
+    # For now we ignore the released milestones and go
+    # directly to the URL with all builds. The information
+    # about milestones may be relevant for determining
+    # how format changes need to be handled.
+    latest_versions = {}
+    for format in range(3, current_format + 1):
+        try:
+            url = '%s/version/format%d/latest' % (content_url, format)
+            response = urllib.request.urlopen(url)
+            version = int(response.read())
+            latest_versions[format] = version
+            formatdir = os.path.join(www_dir, 'version', 'format%d' % format)
+            bb.utils.mkdirhier(formatdir)
+            with open(os.path.join(formatdir, 'latest'), 'w') as latest:
+                latest.write(str(version))
+        except urllib.error.HTTPError as http_error:
+            if http_error.code == 404:
+                bb.debug(1, '%s does not exist, skipping that format' % url)
+            else:
+                raise
+
+    # Now get the Manifests of the latest versions and the
+    # versions we are supposed to provide a delta for, as a starting point.
+    # In addition, we also need Manifests that provide files reused by
+    # these initial set of Manifests or get referenced by them.
+    #
+    # There's no integrity checking for the files. bsdtar is
+    # expected to detect corrupted archives and https is expected
+    # to protect against man-in-the-middle attacks.
+    pending_versions = set(latest_versions.values())
+    pending_versions.update([int(x) for x in d.getVar('SWUPD_DELTAPACK_VERSIONS', True).split()])
+    fetched_versions = set([0])
+    while pending_versions:
+        version = pending_versions.pop()
+        sub_versions = set()
+        sub_versions.update(download_manifests(content_url, version,
+                                               'MoM',
+                                               os.path.join(www_dir, str(version))))
+        sub_versions.update(download_manifests(content_url, version,
+                                               'full',
+                                               os.path.join(www_dir, str(version))))
+        fetched_versions.add(version)
+        pending_versions.update(sub_versions.difference(fetched_versions))
+
+    latest_version_file = os.path.join(deploy_dir, 'image', 'latest.version')
+    if not os.path.exists(latest_version_file):
+        # We located information about latest version from online www update repo.
+        # Now use that to determine what we are updating from. Doing this here
+        # instead of swupd-image.bbclass has the advantage that we can do some
+        # sanity checking very early in a build.
+        #
+        # Building a proper update makes swupd_create_fullfiles
+        # a lot faster because it allows reusing existing, unmodified files.
+        # Saves a lot of space, too, because the new Manifest files then merely
+        # point to the older version (no entry in ${DEPLOY_DIR_SWUPD}/www/${OS_VERSION}/files,
+        # not even a link).
+        if not latest_versions:
+            bb.fatal("%s does not exist and no information was found under SWUPD_CONTENT_URL %s, cannot proceed without information about the previous build. When building the initial version, unset SWUPD_VERSION_URL and SWUPD_CONTENT_URL to proceed." % (latest_version_file, content_url))
+        latest = sorted(latest_versions.values())[-1]
+        bb.debug(2, "Setting %d in latest.version file" % latest)
+        with open(latest_version_file, 'w') as f:
+            f.write(str(latest))