summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/fetch2
diff options
context:
space:
mode:
Diffstat (limited to 'bitbake/lib/bb/fetch2')
-rw-r--r--bitbake/lib/bb/fetch2/__init__.py202
-rw-r--r--bitbake/lib/bb/fetch2/crate.py32
-rw-r--r--bitbake/lib/bb/fetch2/gcp.py102
-rw-r--r--bitbake/lib/bb/fetch2/git.py198
-rw-r--r--bitbake/lib/bb/fetch2/gitsm.py27
-rw-r--r--bitbake/lib/bb/fetch2/hg.py1
-rw-r--r--bitbake/lib/bb/fetch2/local.py8
-rw-r--r--bitbake/lib/bb/fetch2/npm.py13
-rw-r--r--bitbake/lib/bb/fetch2/npmsw.py77
-rw-r--r--bitbake/lib/bb/fetch2/sftp.py2
-rw-r--r--bitbake/lib/bb/fetch2/ssh.py6
-rw-r--r--bitbake/lib/bb/fetch2/svn.py3
-rw-r--r--bitbake/lib/bb/fetch2/wget.py39
13 files changed, 561 insertions, 149 deletions
diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
index 0fb718b23e..5bf2c4b8cf 100644
--- a/bitbake/lib/bb/fetch2/__init__.py
+++ b/bitbake/lib/bb/fetch2/__init__.py
@@ -290,12 +290,12 @@ class URI(object):
def _param_str_split(self, string, elmdelim, kvdelim="="):
ret = collections.OrderedDict()
- for k, v in [x.split(kvdelim, 1) for x in string.split(elmdelim) if x]:
+ for k, v in [x.split(kvdelim, 1) if kvdelim in x else (x, None) for x in string.split(elmdelim) if x]:
ret[k] = v
return ret
def _param_str_join(self, dict_, elmdelim, kvdelim="="):
- return elmdelim.join([kvdelim.join([k, v]) for k, v in dict_.items()])
+ return elmdelim.join([kvdelim.join([k, v]) if v else k for k, v in dict_.items()])
@property
def hostport(self):
@@ -388,7 +388,7 @@ def decodeurl(url):
if s:
if not '=' in s:
raise MalformedUrl(url, "The URL: '%s' is invalid: parameter %s does not specify a value (missing '=')" % (url, s))
- s1, s2 = s.split('=')
+ s1, s2 = s.split('=', 1)
p[s1] = s2
return type, host, urllib.parse.unquote(path), user, pswd, p
@@ -469,6 +469,7 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None):
basename = os.path.basename(mirrortarball)
# Kill parameters, they make no sense for mirror tarballs
uri_decoded[5] = {}
+ uri_find_decoded[5] = {}
elif ud.localpath and ud.method.supports_checksum(ud):
basename = os.path.basename(ud.localpath)
if basename:
@@ -517,7 +518,7 @@ def fetcher_init(d):
else:
raise FetchError("Invalid SRCREV cache policy of: %s" % srcrev_policy)
- _checksum_cache.init_cache(d)
+ _checksum_cache.init_cache(d.getVar("BB_CACHEDIR"))
for m in methods:
if hasattr(m, "init"):
@@ -545,7 +546,7 @@ def mirror_from_string(data):
bb.warn('Invalid mirror data %s, should have paired members.' % data)
return list(zip(*[iter(mirrors)]*2))
-def verify_checksum(ud, d, precomputed={}):
+def verify_checksum(ud, d, precomputed={}, localpath=None, fatal_nochecksum=True):
"""
verify the MD5 and SHA256 checksum for downloaded src
@@ -559,17 +560,19 @@ def verify_checksum(ud, d, precomputed={}):
file against those in the recipe each time, rather than only after
downloading. See https://bugzilla.yoctoproject.org/show_bug.cgi?id=5571.
"""
-
if ud.ignore_checksums or not ud.method.supports_checksum(ud):
return {}
+ if localpath is None:
+ localpath = ud.localpath
+
def compute_checksum_info(checksum_id):
checksum_name = getattr(ud, "%s_name" % checksum_id)
if checksum_id in precomputed:
checksum_data = precomputed[checksum_id]
else:
- checksum_data = getattr(bb.utils, "%s_file" % checksum_id)(ud.localpath)
+ checksum_data = getattr(bb.utils, "%s_file" % checksum_id)(localpath)
checksum_expected = getattr(ud, "%s_expected" % checksum_id)
@@ -595,17 +598,13 @@ def verify_checksum(ud, d, precomputed={}):
checksum_lines = ["SRC_URI[%s] = \"%s\"" % (ci["name"], ci["data"])]
# If no checksum has been provided
- if ud.method.recommends_checksum(ud) and all(ci["expected"] is None for ci in checksum_infos):
+ if fatal_nochecksum and ud.method.recommends_checksum(ud) and all(ci["expected"] is None for ci in checksum_infos):
messages = []
strict = d.getVar("BB_STRICT_CHECKSUM") or "0"
# If strict checking enabled and neither sum defined, raise error
if strict == "1":
- messages.append("No checksum specified for '%s', please add at " \
- "least one to the recipe:" % ud.localpath)
- messages.extend(checksum_lines)
- logger.error("\n".join(messages))
- raise NoChecksumError("Missing SRC_URI checksum", ud.url)
+ raise NoChecksumError("\n".join(checksum_lines))
bb.event.fire(MissingChecksumEvent(ud.url, **checksum_event), d)
@@ -627,7 +626,7 @@ def verify_checksum(ud, d, precomputed={}):
for ci in checksum_infos:
if ci["expected"] and ci["expected"] != ci["data"]:
messages.append("File: '%s' has %s checksum '%s' when '%s' was " \
- "expected" % (ud.localpath, ci["id"], ci["data"], ci["expected"]))
+ "expected" % (localpath, ci["id"], ci["data"], ci["expected"]))
bad_checksum = ci["data"]
if bad_checksum:
@@ -745,13 +744,16 @@ def subprocess_setup():
# SIGPIPE errors are known issues with gzip/bash
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
-def get_autorev(d):
- # only not cache src rev in autorev case
+def mark_recipe_nocache(d):
if d.getVar('BB_SRCREV_POLICY') != "cache":
d.setVar('BB_DONT_CACHE', '1')
+
+def get_autorev(d):
+ mark_recipe_nocache(d)
+ d.setVar("__BBAUTOREV_SEEN", True)
return "AUTOINC"
-def get_srcrev(d, method_name='sortable_revision'):
+def _get_srcrev(d, method_name='sortable_revision'):
"""
Return the revision string, usually for use in the version string (PV) of the current package
Most packages usually only have one SCM so we just pass on the call.
@@ -765,13 +767,14 @@ def get_srcrev(d, method_name='sortable_revision'):
that fetcher provides a method with the given name and the same signature as sortable_revision.
"""
- d.setVar("__BBSEENSRCREV", "1")
+ d.setVar("__BBSRCREV_SEEN", "1")
recursion = d.getVar("__BBINSRCREV")
if recursion:
raise FetchError("There are recursive references in fetcher variables, likely through SRC_URI")
d.setVar("__BBINSRCREV", True)
scms = []
+ revs = []
fetcher = Fetch(d.getVar('SRC_URI').split(), d)
urldata = fetcher.ud
for u in urldata:
@@ -779,16 +782,19 @@ def get_srcrev(d, method_name='sortable_revision'):
scms.append(u)
if not scms:
- raise FetchError("SRCREV was used yet no valid SCM was found in SRC_URI")
+ d.delVar("__BBINSRCREV")
+ return "", revs
+
if len(scms) == 1 and len(urldata[scms[0]].names) == 1:
autoinc, rev = getattr(urldata[scms[0]].method, method_name)(urldata[scms[0]], d, urldata[scms[0]].names[0])
+ revs.append(rev)
if len(rev) > 10:
rev = rev[:10]
d.delVar("__BBINSRCREV")
if autoinc:
- return "AUTOINC+" + rev
- return rev
+ return "AUTOINC+" + rev, revs
+ return rev, revs
#
# Mutiple SCMs are in SRC_URI so we resort to SRCREV_FORMAT
@@ -804,6 +810,7 @@ def get_srcrev(d, method_name='sortable_revision'):
ud = urldata[scm]
for name in ud.names:
autoinc, rev = getattr(ud.method, method_name)(ud, d, name)
+ revs.append(rev)
seenautoinc = seenautoinc or autoinc
if len(rev) > 10:
rev = rev[:10]
@@ -821,7 +828,21 @@ def get_srcrev(d, method_name='sortable_revision'):
format = "AUTOINC+" + format
d.delVar("__BBINSRCREV")
- return format
+ return format, revs
+
+def get_hashvalue(d, method_name='sortable_revision'):
+ pkgv, revs = _get_srcrev(d, method_name=method_name)
+ return " ".join(revs)
+
+def get_pkgv_string(d, method_name='sortable_revision'):
+ pkgv, revs = _get_srcrev(d, method_name=method_name)
+ return pkgv
+
+def get_srcrev(d, method_name='sortable_revision'):
+ pkgv, revs = _get_srcrev(d, method_name=method_name)
+ if not pkgv:
+ raise FetchError("SRCREV was used yet no valid SCM was found in SRC_URI")
+ return pkgv
def localpath(url, d):
fetcher = bb.fetch2.Fetch([url], d)
@@ -847,10 +868,17 @@ FETCH_EXPORT_VARS = ['HOME', 'PATH',
'DBUS_SESSION_BUS_ADDRESS',
'P4CONFIG',
'SSL_CERT_FILE',
+ 'NODE_EXTRA_CA_CERTS',
'AWS_PROFILE',
'AWS_ACCESS_KEY_ID',
'AWS_SECRET_ACCESS_KEY',
- 'AWS_DEFAULT_REGION']
+ 'AWS_ROLE_ARN',
+ 'AWS_WEB_IDENTITY_TOKEN_FILE',
+ 'AWS_DEFAULT_REGION',
+ 'AWS_SESSION_TOKEN',
+ 'GIT_CACHE_PATH',
+ 'REMOTE_CONTAINERS_IPC',
+ 'SSL_CERT_DIR']
def get_fetcher_environment(d):
newenv = {}
@@ -915,7 +943,10 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None):
elif e.stderr:
output = "output:\n%s" % e.stderr
else:
- output = "no output"
+ if log:
+ output = "see logfile for output"
+ else:
+ output = "no output"
error_message = "Fetch command %s failed with exit code %s, %s" % (e.command, e.exitcode, output)
except bb.process.CmdError as e:
error_message = "Fetch command %s could not be run:\n%s" % (e.command, e.msg)
@@ -977,6 +1008,7 @@ def build_mirroruris(origud, mirrors, ld):
try:
newud = FetchData(newuri, ld)
+ newud.ignore_checksums = True
newud.setup_localpath(ld)
except bb.fetch2.BBFetchException as e:
logger.debug("Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url))
@@ -1086,7 +1118,8 @@ def try_mirror_url(fetch, origud, ud, ld, check = False):
logger.debug("Mirror fetch failure for url %s (original url: %s)" % (ud.url, origud.url))
logger.debug(str(e))
try:
- ud.method.clean(ud, ld)
+ if ud.method.cleanup_upon_failure():
+ ud.method.clean(ud, ld)
except UnboundLocalError:
pass
return False
@@ -1211,6 +1244,7 @@ def srcrev_internal_helper(ud, d, name):
if srcrev == "INVALID" or not srcrev:
raise FetchError("Please set a valid SRCREV for url %s (possible key names are %s, or use a ;rev=X URL parameter)" % (str(attempts), ud.url), ud.url)
if srcrev == "AUTOINC":
+ d.setVar("__BBAUTOREV_ACTED_UPON", True)
srcrev = ud.method.latest_revision(ud, d, name)
return srcrev
@@ -1227,7 +1261,7 @@ def get_checksum_file_list(d):
ud = fetch.ud[u]
if ud and isinstance(ud.method, local.Local):
found = False
- paths = ud.method.localpaths(ud, d)
+ paths = ud.method.localfile_searchpaths(ud, d)
for f in paths:
pth = ud.decodedurl
if os.path.exists(f):
@@ -1283,18 +1317,13 @@ class FetchData(object):
if checksum_name in self.parm:
checksum_expected = self.parm[checksum_name]
- elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3", "az"]:
+ elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3", "az", "crate", "gs"]:
checksum_expected = None
else:
checksum_expected = d.getVarFlag("SRC_URI", checksum_name)
setattr(self, "%s_expected" % checksum_id, checksum_expected)
- for checksum_id in CHECKSUM_LIST:
- configure_checksum(checksum_id)
-
- self.ignore_checksums = False
-
self.names = self.parm.get("name",'default').split(',')
self.method = None
@@ -1316,6 +1345,11 @@ class FetchData(object):
if hasattr(self.method, "urldata_init"):
self.method.urldata_init(self, d)
+ for checksum_id in CHECKSUM_LIST:
+ configure_checksum(checksum_id)
+
+ self.ignore_checksums = False
+
if "localpath" in self.parm:
# if user sets localpath for file, use it instead.
self.localpath = self.parm["localpath"]
@@ -1395,6 +1429,9 @@ class FetchMethod(object):
Is localpath something that can be represented by a checksum?
"""
+ # We cannot compute checksums for None
+ if urldata.localpath is None:
+ return False
# We cannot compute checksums for directories
if os.path.isdir(urldata.localpath):
return False
@@ -1407,6 +1444,12 @@ class FetchMethod(object):
"""
return False
+ def cleanup_upon_failure(self):
+ """
+ When a fetch fails, should clean() be called?
+ """
+ return True
+
def verify_donestamp(self, ud, d):
"""
Verify the donestamp file
@@ -1549,6 +1592,7 @@ class FetchMethod(object):
unpackdir = rootdir
if not unpack or not cmd:
+ urldata.unpack_tracer.unpack("file-copy", unpackdir)
# If file == dest, then avoid any copies, as we already put the file into dest!
dest = os.path.join(unpackdir, os.path.basename(file))
if file != dest and not (os.path.exists(dest) and os.path.samefile(file, dest)):
@@ -1563,6 +1607,8 @@ class FetchMethod(object):
destdir = urlpath.rsplit("/", 1)[0] + '/'
bb.utils.mkdirhier("%s/%s" % (unpackdir, destdir))
cmd = 'cp -fpPRH "%s" "%s"' % (file, destdir)
+ else:
+ urldata.unpack_tracer.unpack("archive-extract", unpackdir)
if not cmd:
return
@@ -1654,6 +1700,55 @@ class FetchMethod(object):
"""
return []
+
+class DummyUnpackTracer(object):
+ """
+ Abstract API definition for a class that traces unpacked source files back
+ to their respective upstream SRC_URI entries, for software composition
+ analysis, license compliance and detailed SBOM generation purposes.
+ User may load their own unpack tracer class (instead of the dummy
+ one) by setting the BB_UNPACK_TRACER_CLASS config parameter.
+ """
+ def start(self, unpackdir, urldata_dict, d):
+ """
+ Start tracing the core Fetch.unpack process, using an index to map
+ unpacked files to each SRC_URI entry.
+ This method is called by Fetch.unpack and it may receive nested calls by
+ gitsm and npmsw fetchers, that expand SRC_URI entries by adding implicit
+ URLs and by recursively calling Fetch.unpack from new (nested) Fetch
+ instances.
+ """
+ return
+ def start_url(self, url):
+ """Start tracing url unpack process.
+ This method is called by Fetch.unpack before the fetcher-specific unpack
+ method starts, and it may receive nested calls by gitsm and npmsw
+ fetchers.
+ """
+ return
+ def unpack(self, unpack_type, destdir):
+ """
+ Set unpack_type and destdir for current url.
+ This method is called by the fetcher-specific unpack method after url
+ tracing started.
+ """
+ return
+ def finish_url(self, url):
+ """Finish tracing url unpack process and update the file index.
+ This method is called by Fetch.unpack after the fetcher-specific unpack
+ method finished its job, and it may receive nested calls by gitsm
+ and npmsw fetchers.
+ """
+ return
+ def complete(self):
+ """
+ Finish tracing the Fetch.unpack process, and check if all nested
+ Fecth.unpack calls (if any) have been completed; if so, save collected
+ metadata.
+ """
+ return
+
+
class Fetch(object):
def __init__(self, urls, d, cache = True, localonly = False, connection_cache = None):
if localonly and cache:
@@ -1674,10 +1769,30 @@ class Fetch(object):
if key in urldata_cache:
self.ud = urldata_cache[key]
+ # the unpack_tracer object needs to be made available to possible nested
+ # Fetch instances (when those are created by gitsm and npmsw fetchers)
+ # so we set it as a global variable
+ global unpack_tracer
+ try:
+ unpack_tracer
+ except NameError:
+ class_path = d.getVar("BB_UNPACK_TRACER_CLASS")
+ if class_path:
+ # use user-defined unpack tracer class
+ import importlib
+ module_name, _, class_name = class_path.rpartition(".")
+ module = importlib.import_module(module_name)
+ class_ = getattr(module, class_name)
+ unpack_tracer = class_()
+ else:
+ # fall back to the dummy/abstract class
+ unpack_tracer = DummyUnpackTracer()
+
for url in urls:
if url not in self.ud:
try:
self.ud[url] = FetchData(url, d, localonly)
+ self.ud[url].unpack_tracer = unpack_tracer
except NonLocalMethod:
if localonly:
self.ud[url] = None
@@ -1716,6 +1831,7 @@ class Fetch(object):
network = self.d.getVar("BB_NO_NETWORK")
premirroronly = bb.utils.to_boolean(self.d.getVar("BB_FETCH_PREMIRRORONLY"))
+ checksum_missing_messages = []
for u in urls:
ud = self.ud[u]
ud.setup_localpath(self.d)
@@ -1727,7 +1843,6 @@ class Fetch(object):
try:
self.d.setVar("BB_NO_NETWORK", network)
-
if m.verify_donestamp(ud, self.d) and not m.need_update(ud, self.d):
done = True
elif m.try_premirror(ud, self.d):
@@ -1780,7 +1895,7 @@ class Fetch(object):
logger.debug(str(e))
firsterr = e
# Remove any incomplete fetch
- if not verified_stamp:
+ if not verified_stamp and m.cleanup_upon_failure():
m.clean(ud, self.d)
logger.debug("Trying MIRRORS")
mirrors = mirror_from_string(self.d.getVar('MIRRORS'))
@@ -1799,13 +1914,20 @@ class Fetch(object):
raise ChecksumError("Stale Error Detected")
except BBFetchException as e:
- if isinstance(e, ChecksumError):
+ if isinstance(e, NoChecksumError):
+ (message, _) = e.args
+ checksum_missing_messages.append(message)
+ continue
+ elif isinstance(e, ChecksumError):
logger.error("Checksum failure fetching %s" % u)
raise
finally:
if ud.lockfile:
bb.utils.unlockfile(lf)
+ if checksum_missing_messages:
+ logger.error("Missing SRC_URI checksum, please add those to the recipe: \n%s", "\n".join(checksum_missing_messages))
+ raise BBFetchException("There was some missing checksums in the recipe")
def checkstatus(self, urls=None):
"""
@@ -1836,7 +1958,7 @@ class Fetch(object):
ret = m.try_mirrors(self, ud, self.d, mirrors, True)
if not ret:
- raise FetchError("URL %s doesn't work" % u, u)
+ raise FetchError("URL doesn't work", u)
def unpack(self, root, urls=None):
"""
@@ -1846,6 +1968,8 @@ class Fetch(object):
if not urls:
urls = self.urls
+ unpack_tracer.start(root, self.ud, self.d)
+
for u in urls:
ud = self.ud[u]
ud.setup_localpath(self.d)
@@ -1853,11 +1977,15 @@ class Fetch(object):
if ud.lockfile:
lf = bb.utils.lockfile(ud.lockfile)
+ unpack_tracer.start_url(u)
ud.method.unpack(ud, root, self.d)
+ unpack_tracer.finish_url(u)
if ud.lockfile:
bb.utils.unlockfile(lf)
+ unpack_tracer.complete()
+
def clean(self, urls=None):
"""
Clean files that the fetcher gets or places
@@ -1959,6 +2087,7 @@ from . import npm
from . import npmsw
from . import az
from . import crate
+from . import gcp
methods.append(local.Local())
methods.append(wget.Wget())
@@ -1980,3 +2109,4 @@ methods.append(npm.Npm())
methods.append(npmsw.NpmShrinkWrap())
methods.append(az.Az())
methods.append(crate.Crate())
+methods.append(gcp.GCP())
diff --git a/bitbake/lib/bb/fetch2/crate.py b/bitbake/lib/bb/fetch2/crate.py
index f4ddc782a9..e611736f06 100644
--- a/bitbake/lib/bb/fetch2/crate.py
+++ b/bitbake/lib/bb/fetch2/crate.py
@@ -33,7 +33,7 @@ class Crate(Wget):
return ud.type in ['crate']
def recommends_checksum(self, urldata):
- return False
+ return True
def urldata_init(self, ud, d):
"""
@@ -56,22 +56,26 @@ class Crate(Wget):
if len(parts) < 5:
raise bb.fetch2.ParameterError("Invalid URL: Must be crate://HOST/NAME/VERSION", ud.url)
- # last field is version
- version = parts[len(parts) - 1]
+ # version is expected to be the last token
+ # but ignore possible url parameters which will be used
+ # by the top fetcher class
+ version = parts[-1].split(";")[0]
# second to last field is name
- name = parts[len(parts) - 2]
+ name = parts[-2]
# host (this is to allow custom crate registries to be specified
- host = '/'.join(parts[2:len(parts) - 2])
+ host = '/'.join(parts[2:-2])
# if using upstream just fix it up nicely
if host == 'crates.io':
host = 'crates.io/api/v1/crates'
ud.url = "https://%s/%s/%s/download" % (host, name, version)
+ ud.versionsurl = "https://%s/%s/versions" % (host, name)
ud.parm['downloadfilename'] = "%s-%s.crate" % (name, version)
- ud.parm['name'] = name
+ if 'name' not in ud.parm:
+ ud.parm['name'] = '%s-%s' % (name, version)
- logger.debug("Fetching %s to %s" % (ud.url, ud.parm['downloadfilename']))
+ logger.debug2("Fetching %s to %s" % (ud.url, ud.parm['downloadfilename']))
def unpack(self, ud, rootdir, d):
"""
@@ -95,11 +99,13 @@ class Crate(Wget):
save_cwd = os.getcwd()
os.chdir(rootdir)
- pn = d.getVar('BPN')
- if pn == ud.parm.get('name'):
+ bp = d.getVar('BP')
+ if bp == ud.parm.get('name'):
cmd = "tar -xz --no-same-owner -f %s" % thefile
+ ud.unpack_tracer.unpack("crate-extract", rootdir)
else:
cargo_bitbake = self._cargo_bitbake_path(rootdir)
+ ud.unpack_tracer.unpack("cargo-extract", cargo_bitbake)
cmd = "tar -xz --no-same-owner -f %s -C %s" % (thefile, cargo_bitbake)
@@ -134,3 +140,11 @@ class Crate(Wget):
mdpath = os.path.join(bbpath, cratepath, mdfile)
with open(mdpath, "w") as f:
json.dump(metadata, f)
+
+ def latest_versionstring(self, ud, d):
+ from functools import cmp_to_key
+ json_data = json.loads(self._fetch_index(ud.versionsurl, ud, d))
+ versions = [(0, i["num"], "") for i in json_data["versions"]]
+ versions = sorted(versions, key=cmp_to_key(bb.utils.vercmp))
+
+ return (versions[-1][1], "")
diff --git a/bitbake/lib/bb/fetch2/gcp.py b/bitbake/lib/bb/fetch2/gcp.py
new file mode 100644
index 0000000000..eb3e0c6a6b
--- /dev/null
+++ b/bitbake/lib/bb/fetch2/gcp.py
@@ -0,0 +1,102 @@
+"""
+BitBake 'Fetch' implementation for Google Cloup Platform Storage.
+
+Class for fetching files from Google Cloud Storage using the
+Google Cloud Storage Python Client. The GCS Python Client must
+be correctly installed, configured and authenticated prior to use.
+Additionally, gsutil must also be installed.
+
+"""
+
+# Copyright (C) 2023, Snap Inc.
+#
+# Based in part on bb.fetch2.s3:
+# Copyright (C) 2017 Andre McCurdy
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Based on functions from the base bb module, Copyright 2003 Holger Schurig
+
+import os
+import bb
+import urllib.parse, urllib.error
+from bb.fetch2 import FetchMethod
+from bb.fetch2 import FetchError
+from bb.fetch2 import logger
+from bb.fetch2 import runfetchcmd
+
+class GCP(FetchMethod):
+ """
+ Class to fetch urls via GCP's Python API.
+ """
+ def __init__(self):
+ self.gcp_client = None
+
+ def supports(self, ud, d):
+ """
+ Check to see if a given url can be fetched with GCP.
+ """
+ return ud.type in ['gs']
+
+ def recommends_checksum(self, urldata):
+ return True
+
+ def urldata_init(self, ud, d):
+ if 'downloadfilename' in ud.parm:
+ ud.basename = ud.parm['downloadfilename']
+ else:
+ ud.basename = os.path.basename(ud.path)
+
+ ud.localfile = d.expand(urllib.parse.unquote(ud.basename))
+ ud.basecmd = "gsutil stat"
+
+ def get_gcp_client(self):
+ from google.cloud import storage
+ self.gcp_client = storage.Client(project=None)
+
+ def download(self, ud, d):
+ """
+ Fetch urls using the GCP API.
+ Assumes localpath was called first.
+ """
+ logger.debug2(f"Trying to download gs://{ud.host}{ud.path} to {ud.localpath}")
+ if self.gcp_client is None:
+ self.get_gcp_client()
+
+ bb.fetch2.check_network_access(d, ud.basecmd, f"gs://{ud.host}{ud.path}")
+ runfetchcmd("%s %s" % (ud.basecmd, f"gs://{ud.host}{ud.path}"), d)
+
+ # Path sometimes has leading slash, so strip it
+ path = ud.path.lstrip("/")
+ blob = self.gcp_client.bucket(ud.host).blob(path)
+ blob.download_to_filename(ud.localpath)
+
+ # Additional sanity checks copied from the wget class (although there
+ # are no known issues which mean these are required, treat the GCP API
+ # tool with a little healthy suspicion).
+ if not os.path.exists(ud.localpath):
+ raise FetchError(f"The GCP API returned success for gs://{ud.host}{ud.path} but {ud.localpath} doesn't exist?!")
+
+ if os.path.getsize(ud.localpath) == 0:
+ os.remove(ud.localpath)
+ raise FetchError(f"The downloaded file for gs://{ud.host}{ud.path} resulted in a zero size file?! Deleting and failing since this isn't right.")
+
+ return True
+
+ def checkstatus(self, fetch, ud, d):
+ """
+ Check the status of a URL.
+ """
+ logger.debug2(f"Checking status of gs://{ud.host}{ud.path}")
+ if self.gcp_client is None:
+ self.get_gcp_client()
+
+ bb.fetch2.check_network_access(d, ud.basecmd, f"gs://{ud.host}{ud.path}")
+ runfetchcmd("%s %s" % (ud.basecmd, f"gs://{ud.host}{ud.path}"), d)
+
+ # Path sometimes has leading slash, so strip it
+ path = ud.path.lstrip("/")
+ if self.gcp_client.bucket(ud.host).blob(path).exists() == False:
+ raise FetchError(f"The GCP API reported that gs://{ud.host}{ud.path} does not exist")
+ else:
+ return True
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
index 4534bd7580..c7ff769fdf 100644
--- a/bitbake/lib/bb/fetch2/git.py
+++ b/bitbake/lib/bb/fetch2/git.py
@@ -44,13 +44,27 @@ Supported SRC_URI options are:
- nobranch
Don't check the SHA validation for branch. set this option for the recipe
- referring to commit which is valid in tag instead of branch.
+ referring to commit which is valid in any namespace (branch, tag, ...)
+ instead of branch.
The default is "0", set nobranch=1 if needed.
+- subpath
+ Limit the checkout to a specific subpath of the tree.
+ By default, checkout the whole tree, set subpath=<path> if needed
+
+- destsuffix
+ The name of the path in which to place the checkout.
+ By default, the path is git/, set destsuffix=<suffix> if needed
+
- usehead
For local git:// urls to use the current branch HEAD as the revision for use with
AUTOREV. Implies nobranch.
+- lfs
+ Enable the checkout to use LFS for large files. This will download all LFS files
+ in the download step, as the unpack step does not have network access.
+ The default is "1", set lfs=0 to skip.
+
"""
# Copyright (C) 2005 Richard Purdie
@@ -64,6 +78,7 @@ import fnmatch
import os
import re
import shlex
+import shutil
import subprocess
import tempfile
import bb
@@ -72,6 +87,7 @@ from contextlib import contextmanager
from bb.fetch2 import FetchMethod
from bb.fetch2 import runfetchcmd
from bb.fetch2 import logger
+from bb.fetch2 import trusted_network
sha1_re = re.compile(r'^[0-9a-f]{40}$')
@@ -134,6 +150,9 @@ class Git(FetchMethod):
def supports_checksum(self, urldata):
return False
+ def cleanup_upon_failure(self):
+ return False
+
def urldata_init(self, ud, d):
"""
init git specific variable within url data
@@ -243,7 +262,7 @@ class Git(FetchMethod):
for name in ud.names:
ud.unresolvedrev[name] = 'HEAD'
- ud.basecmd = d.getVar("FETCHCMD_git") or "git -c core.fsyncobjectfiles=0 -c gc.autoDetach=false -c core.pager=cat"
+ ud.basecmd = d.getVar("FETCHCMD_git") or "git -c gc.autoDetach=false -c core.pager=cat -c safe.bareRepository=all"
write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0"
ud.write_tarballs = write_tarballs != "0" or ud.rebaseable
@@ -258,7 +277,7 @@ class Git(FetchMethod):
ud.unresolvedrev[name] = ud.revisions[name]
ud.revisions[name] = self.latest_revision(ud, d, name)
- gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_'))
+ gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_').replace('(', '_').replace(')', '_'))
if gitsrcname.startswith('.'):
gitsrcname = gitsrcname[1:]
@@ -309,7 +328,10 @@ class Git(FetchMethod):
return ud.clonedir
def need_update(self, ud, d):
- return self.clonedir_need_update(ud, d) or self.shallow_tarball_need_update(ud) or self.tarball_need_update(ud)
+ return self.clonedir_need_update(ud, d) \
+ or self.shallow_tarball_need_update(ud) \
+ or self.tarball_need_update(ud) \
+ or self.lfs_need_update(ud, d)
def clonedir_need_update(self, ud, d):
if not os.path.exists(ud.clonedir):
@@ -321,6 +343,15 @@ class Git(FetchMethod):
return True
return False
+ def lfs_need_update(self, ud, d):
+ if self.clonedir_need_update(ud, d):
+ return True
+
+ for name in ud.names:
+ if not self._lfs_objects_downloaded(ud, d, name, ud.clonedir):
+ return True
+ return False
+
def clonedir_need_shallow_revs(self, ud, d):
for rev in ud.shallow_revs:
try:
@@ -340,6 +371,16 @@ class Git(FetchMethod):
# is not possible
if bb.utils.to_boolean(d.getVar("BB_FETCH_PREMIRRORONLY")):
return True
+ # If the url is not in trusted network, that is, BB_NO_NETWORK is set to 0
+ # and BB_ALLOWED_NETWORKS does not contain the host that ud.url uses, then
+ # we need to try premirrors first as using upstream is destined to fail.
+ if not trusted_network(d, ud.url):
+ return True
+ # the following check is to ensure incremental fetch in downloads, this is
+ # because the premirror might be old and does not contain the new rev required,
+ # and this will cause a total removal and new clone. So if we can reach to
+ # network, we prefer upstream over premirror, though the premirror might contain
+ # the new rev.
if os.path.exists(ud.clonedir):
return False
return True
@@ -360,15 +401,47 @@ class Git(FetchMethod):
else:
tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR'))
runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=tmpdir)
- fetch_cmd = "LANG=C %s fetch -f --progress %s " % (ud.basecmd, shlex.quote(tmpdir))
+ output = runfetchcmd("%s remote" % ud.basecmd, d, quiet=True, workdir=ud.clonedir)
+ if 'mirror' in output:
+ runfetchcmd("%s remote rm mirror" % ud.basecmd, d, workdir=ud.clonedir)
+ runfetchcmd("%s remote add --mirror=fetch mirror %s" % (ud.basecmd, tmpdir), d, workdir=ud.clonedir)
+ fetch_cmd = "LANG=C %s fetch -f --update-head-ok --progress mirror " % (ud.basecmd)
runfetchcmd(fetch_cmd, d, workdir=ud.clonedir)
repourl = self._get_repo_url(ud)
+ needs_clone = False
+ if os.path.exists(ud.clonedir):
+ # The directory may exist, but not be the top level of a bare git
+ # repository in which case it needs to be deleted and re-cloned.
+ try:
+ # Since clones can be bare, use --absolute-git-dir instead of --show-toplevel
+ output = runfetchcmd("LANG=C %s rev-parse --absolute-git-dir" % ud.basecmd, d, workdir=ud.clonedir)
+ toplevel = output.rstrip()
+
+ if not bb.utils.path_is_descendant(toplevel, ud.clonedir):
+ logger.warning("Top level directory '%s' is not a descendant of '%s'. Re-cloning", toplevel, ud.clonedir)
+ needs_clone = True
+ except bb.fetch2.FetchError as e:
+ logger.warning("Unable to get top level for %s (not a git directory?): %s", ud.clonedir, e)
+ needs_clone = True
+ except FileNotFoundError as e:
+ logger.warning("%s", e)
+ needs_clone = True
+
+ if needs_clone:
+ shutil.rmtree(ud.clonedir)
+ else:
+ needs_clone = True
+
# If the repo still doesn't exist, fallback to cloning it
- if not os.path.exists(ud.clonedir):
- # We do this since git will use a "-l" option automatically for local urls where possible
+ if needs_clone:
+ # We do this since git will use a "-l" option automatically for local urls where possible,
+ # but it doesn't work when git/objects is a symlink, only works when it is a directory.
if repourl.startswith("file://"):
- repourl = repourl[7:]
+ repourl_path = repourl[7:]
+ objects = os.path.join(repourl_path, 'objects')
+ if os.path.isdir(objects) and not os.path.islink(objects):
+ repourl = repourl_path
clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, shlex.quote(repourl), ud.clonedir)
if ud.proto.lower() != 'file':
bb.fetch2.check_network_access(d, clone_cmd, ud.url)
@@ -382,7 +455,11 @@ class Git(FetchMethod):
runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir)
runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=ud.clonedir)
- fetch_cmd = "LANG=C %s fetch -f --progress %s refs/*:refs/*" % (ud.basecmd, shlex.quote(repourl))
+
+ if ud.nobranch:
+ fetch_cmd = "LANG=C %s fetch -f --progress %s refs/*:refs/*" % (ud.basecmd, shlex.quote(repourl))
+ else:
+ fetch_cmd = "LANG=C %s fetch -f --progress %s refs/heads/*:refs/heads/* refs/tags/*:refs/tags/*" % (ud.basecmd, shlex.quote(repourl))
if ud.proto.lower() != 'file':
bb.fetch2.check_network_access(d, fetch_cmd, ud.url)
progresshandler = GitProgressHandler(d)
@@ -405,15 +482,14 @@ class Git(FetchMethod):
if missing_rev:
raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev)
- if self._contains_lfs(ud, d, ud.clonedir) and self._need_lfs(ud):
+ if self.lfs_need_update(ud, d):
# Unpack temporary working copy, use it to run 'git checkout' to force pre-fetching
# of all LFS blobs needed at the srcrev.
#
# It would be nice to just do this inline here by running 'git-lfs fetch'
# on the bare clonedir, but that operation requires a working copy on some
# releases of Git LFS.
- tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR'))
- try:
+ with tempfile.TemporaryDirectory(dir=d.getVar('DL_DIR')) as tmpdir:
# Do the checkout. This implicitly involves a Git LFS fetch.
Git.unpack(self, ud, tmpdir, d)
@@ -429,10 +505,8 @@ class Git(FetchMethod):
# Only do this if the unpack resulted in a .git/lfs directory being
# created; this only happens if at least one blob needed to be
# downloaded.
- if os.path.exists(os.path.join(tmpdir, "git", ".git", "lfs")):
- runfetchcmd("tar -cf - lfs | tar -xf - -C %s" % ud.clonedir, d, workdir="%s/git/.git" % tmpdir)
- finally:
- bb.utils.remove(tmpdir, recurse=True)
+ if os.path.exists(os.path.join(ud.destdir, ".git", "lfs")):
+ runfetchcmd("tar -cf - lfs | tar -xf - -C %s" % ud.clonedir, d, workdir="%s/.git" % ud.destdir)
def build_mirror_data(self, ud, d):
@@ -470,7 +544,7 @@ class Git(FetchMethod):
logger.info("Creating tarball of git repository")
with create_atomic(ud.fullmirror) as tfile:
- mtime = runfetchcmd("git log --all -1 --format=%cD", d,
+ mtime = runfetchcmd("{} log --all -1 --format=%cD".format(ud.basecmd), d,
quiet=True, workdir=ud.clonedir)
runfetchcmd("tar -czf %s --owner oe:0 --group oe:0 --mtime \"%s\" ."
% (tfile, mtime), d, workdir=ud.clonedir)
@@ -558,6 +632,8 @@ class Git(FetchMethod):
destdir = ud.destdir = os.path.join(destdir, destsuffix)
if os.path.exists(destdir):
bb.utils.prunedir(destdir)
+ if not ud.bareclone:
+ ud.unpack_tracer.unpack("git", destdir)
need_lfs = self._need_lfs(ud)
@@ -567,13 +643,12 @@ class Git(FetchMethod):
source_found = False
source_error = []
- if not source_found:
- clonedir_is_up_to_date = not self.clonedir_need_update(ud, d)
- if clonedir_is_up_to_date:
- runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, destdir), d)
- source_found = True
- else:
- source_error.append("clone directory not available or not up to date: " + ud.clonedir)
+ clonedir_is_up_to_date = not self.clonedir_need_update(ud, d)
+ if clonedir_is_up_to_date:
+ runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, destdir), d)
+ source_found = True
+ else:
+ source_error.append("clone directory not available or not up to date: " + ud.clonedir)
if not source_found:
if ud.shallow:
@@ -597,6 +672,8 @@ class Git(FetchMethod):
raise bb.fetch2.FetchError("Repository %s has LFS content, install git-lfs on host to download (or set lfs=0 to ignore it)" % (repourl))
elif not need_lfs:
bb.note("Repository %s has LFS content but it is not being fetched" % (repourl))
+ else:
+ runfetchcmd("%s lfs install --local" % ud.basecmd, d, workdir=destdir)
if not ud.nocheckout:
if subpath:
@@ -648,6 +725,35 @@ class Git(FetchMethod):
raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output))
return output.split()[0] != "0"
+ def _lfs_objects_downloaded(self, ud, d, name, wd):
+ """
+ Verifies whether the LFS objects for requested revisions have already been downloaded
+ """
+ # Bail out early if this repository doesn't use LFS
+ if not self._need_lfs(ud) or not self._contains_lfs(ud, d, wd):
+ return True
+
+ # The Git LFS specification specifies ([1]) the LFS folder layout so it should be safe to check for file
+ # existence.
+ # [1] https://github.com/git-lfs/git-lfs/blob/main/docs/spec.md#intercepting-git
+ cmd = "%s lfs ls-files -l %s" \
+ % (ud.basecmd, ud.revisions[name])
+ output = runfetchcmd(cmd, d, quiet=True, workdir=wd).rstrip()
+ # Do not do any further matching if no objects are managed by LFS
+ if not output:
+ return True
+
+ # Match all lines beginning with the hexadecimal OID
+ oid_regex = re.compile("^(([a-fA-F0-9]{2})([a-fA-F0-9]{2})[A-Fa-f0-9]+)")
+ for line in output.split("\n"):
+ oid = re.search(oid_regex, line)
+ if not oid:
+ bb.warn("git lfs ls-files output '%s' did not match expected format." % line)
+ if not os.path.exists(os.path.join(wd, "lfs", "objects", oid.group(2), oid.group(3), oid.group(1))):
+ return False
+
+ return True
+
def _need_lfs(self, ud):
return ud.parm.get("lfs", "1") == "1"
@@ -656,13 +762,11 @@ class Git(FetchMethod):
Check if the repository has 'lfs' (large file) content
"""
- if not ud.nobranch:
- branchname = ud.branches[ud.names[0]]
- else:
- branchname = "master"
-
- # The bare clonedir doesn't use the remote names; it has the branch immediately.
- if wd == ud.clonedir:
+ if ud.nobranch:
+ # If no branch is specified, use the current git commit
+ refname = self._build_revision(ud, d, ud.names[0])
+ elif wd == ud.clonedir:
+ # The bare clonedir doesn't use the remote names; it has the branch immediately.
refname = ud.branches[ud.names[0]]
else:
refname = "origin/%s" % ud.branches[ud.names[0]]
@@ -737,11 +841,11 @@ class Git(FetchMethod):
"""
Compute the HEAD revision for the url
"""
- if not d.getVar("__BBSEENSRCREV"):
+ if not d.getVar("__BBSRCREV_SEEN"):
raise bb.fetch2.FetchError("Recipe uses a floating tag/branch '%s' for repo '%s' without a fixed SRCREV yet doesn't call bb.fetch2.get_srcrev() (use SRCPV in PV for OE)." % (ud.unresolvedrev[name], ud.host+ud.path))
# Ensure we mark as not cached
- bb.fetch2.get_autorev(d)
+ bb.fetch2.mark_recipe_nocache(d)
output = self._lsremote(ud, d, "")
# Tags of the form ^{} may not work, need to fallback to other form
@@ -767,38 +871,42 @@ class Git(FetchMethod):
"""
pupver = ('', '')
- tagregex = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX') or r"(?P<pver>([0-9][\.|_]?)+)")
try:
output = self._lsremote(ud, d, "refs/tags/*")
except (bb.fetch2.FetchError, bb.fetch2.NetworkAccess) as e:
bb.note("Could not list remote: %s" % str(e))
return pupver
+ rev_tag_re = re.compile(r"([0-9a-f]{40})\s+refs/tags/(.*)")
+ pver_re = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX') or r"(?P<pver>([0-9][\.|_]?)+)")
+ nonrel_re = re.compile(r"(alpha|beta|rc|final)+")
+
verstring = ""
- revision = ""
for line in output.split("\n"):
if not line:
break
- tag_head = line.split("/")[-1]
+ m = rev_tag_re.match(line)
+ if not m:
+ continue
+
+ (revision, tag) = m.groups()
+
# Ignore non-released branches
- m = re.search(r"(alpha|beta|rc|final)+", tag_head)
- if m:
+ if nonrel_re.search(tag):
continue
# search for version in the line
- tag = tagregex.search(tag_head)
- if tag is None:
+ m = pver_re.search(tag)
+ if not m:
continue
- tag = tag.group('pver')
- tag = tag.replace("_", ".")
+ pver = m.group('pver').replace("_", ".")
- if verstring and bb.utils.vercmp(("0", tag, ""), ("0", verstring, "")) < 0:
+ if verstring and bb.utils.vercmp(("0", pver, ""), ("0", verstring, "")) < 0:
continue
- verstring = tag
- revision = line.split()[0]
+ verstring = pver
pupver = (verstring, revision)
return pupver
diff --git a/bitbake/lib/bb/fetch2/gitsm.py b/bitbake/lib/bb/fetch2/gitsm.py
index c1950e4819..f7f3af7212 100644
--- a/bitbake/lib/bb/fetch2/gitsm.py
+++ b/bitbake/lib/bb/fetch2/gitsm.py
@@ -90,7 +90,7 @@ class GitSM(Git):
# Convert relative to absolute uri based on parent uri
if uris[m].startswith('..') or uris[m].startswith('./'):
newud = copy.copy(ud)
- newud.path = os.path.realpath(os.path.join(newud.path, uris[m]))
+ newud.path = os.path.normpath(os.path.join(newud.path, uris[m]))
uris[m] = Git._get_repo_url(self, newud)
for module in submodules:
@@ -115,10 +115,21 @@ class GitSM(Git):
# This has to be a file reference
proto = "file"
url = "gitsm://" + uris[module]
+ if url.endswith("{}{}".format(ud.host, ud.path)):
+ raise bb.fetch2.FetchError("Submodule refers to the parent repository. This will cause deadlock situation in current version of Bitbake." \
+ "Consider using git fetcher instead.")
url += ';protocol=%s' % proto
url += ";name=%s" % module
url += ";subpath=%s" % module
+ url += ";nobranch=1"
+ url += ";lfs=%s" % self._need_lfs(ud)
+ # Note that adding "user=" here to give credentials to the
+ # submodule is not supported. Since using SRC_URI to give git://
+ # URL a password is not supported, one have to use one of the
+ # recommended way (eg. ~/.netrc or SSH config) which does specify
+ # the user (See comment in git.py).
+ # So, we will not take patches adding "user=" support here.
ld = d.createCopy()
# Not necessary to set SRC_URI, since we're passing the URI to
@@ -207,6 +218,10 @@ class GitSM(Git):
try:
newfetch = Fetch([url], d, cache=False)
+ # modpath is needed by unpack tracer to calculate submodule
+ # checkout dir
+ new_ud = newfetch.ud[url]
+ new_ud.modpath = modpath
newfetch.unpack(root=os.path.dirname(os.path.join(repo_conf, 'modules', module)))
except Exception as e:
logger.error('gitsm: submodule unpack failed: %s %s' % (type(e).__name__, str(e)))
@@ -232,10 +247,12 @@ class GitSM(Git):
ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d)
if not ud.bareclone and ret:
- # All submodules should already be downloaded and configured in the tree. This simply sets
- # up the configuration and checks out the files. The main project config should remain
- # unmodified, and no download from the internet should occur.
- runfetchcmd("%s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir)
+ # All submodules should already be downloaded and configured in the tree. This simply
+ # sets up the configuration and checks out the files. The main project config should
+ # remain unmodified, and no download from the internet should occur. As such, lfs smudge
+ # should also be skipped as these files were already smudged in the fetch stage if lfs
+ # was enabled.
+ runfetchcmd("GIT_LFS_SKIP_SMUDGE=1 %s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir)
def implicit_urldata(self, ud, d):
import shutil, subprocess, tempfile
diff --git a/bitbake/lib/bb/fetch2/hg.py b/bitbake/lib/bb/fetch2/hg.py
index 063e13008a..cbff8c490c 100644
--- a/bitbake/lib/bb/fetch2/hg.py
+++ b/bitbake/lib/bb/fetch2/hg.py
@@ -242,6 +242,7 @@ class Hg(FetchMethod):
revflag = "-r %s" % ud.revision
subdir = ud.parm.get("destsuffix", ud.module)
codir = "%s/%s" % (destdir, subdir)
+ ud.unpack_tracer.unpack("hg", codir)
scmdata = ud.parm.get("scmdata", "")
if scmdata != "nokeep":
diff --git a/bitbake/lib/bb/fetch2/local.py b/bitbake/lib/bb/fetch2/local.py
index 0bb987c644..7d7668110e 100644
--- a/bitbake/lib/bb/fetch2/local.py
+++ b/bitbake/lib/bb/fetch2/local.py
@@ -41,9 +41,9 @@ class Local(FetchMethod):
"""
Return the local filename of a given url assuming a successful fetch.
"""
- return self.localpaths(urldata, d)[-1]
+ return self.localfile_searchpaths(urldata, d)[-1]
- def localpaths(self, urldata, d):
+ def localfile_searchpaths(self, urldata, d):
"""
Return the local filename of a given url assuming a successful fetch.
"""
@@ -51,11 +51,13 @@ class Local(FetchMethod):
path = urldata.decodedurl
newpath = path
if path[0] == "/":
+ logger.debug2("Using absolute %s" % (path))
return [path]
filespath = d.getVar('FILESPATH')
if filespath:
logger.debug2("Searching for %s in paths:\n %s" % (path, "\n ".join(filespath.split(":"))))
newpath, hist = bb.utils.which(filespath, path, history=True)
+ logger.debug2("Using %s for %s" % (newpath, path))
searched.extend(hist)
return searched
@@ -72,7 +74,7 @@ class Local(FetchMethod):
filespath = d.getVar('FILESPATH')
if filespath:
locations = filespath.split(":")
- msg = "Unable to find file " + urldata.url + " anywhere. The paths that were searched were:\n " + "\n ".join(locations)
+ msg = "Unable to find file " + urldata.url + " anywhere to download to " + urldata.localpath + ". The paths that were searched were:\n " + "\n ".join(locations)
raise FetchError(msg)
return True
diff --git a/bitbake/lib/bb/fetch2/npm.py b/bitbake/lib/bb/fetch2/npm.py
index 8f7c10ac9b..15f3f19bc8 100644
--- a/bitbake/lib/bb/fetch2/npm.py
+++ b/bitbake/lib/bb/fetch2/npm.py
@@ -44,9 +44,12 @@ def npm_package(package):
"""Convert the npm package name to remove unsupported character"""
# Scoped package names (with the @) use the same naming convention
# as the 'npm pack' command.
- if package.startswith("@"):
- return re.sub("/", "-", package[1:])
- return package
+ name = re.sub("/", "-", package)
+ name = name.lower()
+ name = re.sub(r"[^\-a-z0-9]", "", name)
+ name = name.strip("-")
+ return name
+
def npm_filename(package, version):
"""Get the filename of a npm package"""
@@ -103,6 +106,7 @@ class NpmEnvironment(object):
"""Run npm command in a controlled environment"""
with tempfile.TemporaryDirectory() as tmpdir:
d = bb.data.createCopy(self.d)
+ d.setVar("PATH", d.getVar("PATH")) # PATH might contain $HOME - evaluate it before patching
d.setVar("HOME", tmpdir)
if not workdir:
@@ -156,7 +160,7 @@ class Npm(FetchMethod):
raise ParameterError("Invalid 'version' parameter", ud.url)
# Extract the 'registry' part of the url
- ud.registry = re.sub(r"^npm://", "http://", ud.url.split(";")[0])
+ ud.registry = re.sub(r"^npm://", "https://", ud.url.split(";")[0])
# Using the 'downloadfilename' parameter as local filename
# or the npm package name.
@@ -294,6 +298,7 @@ class Npm(FetchMethod):
destsuffix = ud.parm.get("destsuffix", "npm")
destdir = os.path.join(rootdir, destsuffix)
npm_unpack(ud.localpath, destdir, d)
+ ud.unpack_tracer.unpack("npm", destdir)
def clean(self, ud, d):
"""Clean any existing full or partial download"""
diff --git a/bitbake/lib/bb/fetch2/npmsw.py b/bitbake/lib/bb/fetch2/npmsw.py
index a8c4d3528f..b55e885d7b 100644
--- a/bitbake/lib/bb/fetch2/npmsw.py
+++ b/bitbake/lib/bb/fetch2/npmsw.py
@@ -41,8 +41,9 @@ def foreach_dependencies(shrinkwrap, callback=None, dev=False):
with:
name = the package name (string)
params = the package parameters (dictionary)
- deptree = the package dependency tree (array of strings)
+ destdir = the destination of the package (string)
"""
+ # For handling old style dependencies entries in shinkwrap files
def _walk_deps(deps, deptree):
for name in deps:
subtree = [*deptree, name]
@@ -52,9 +53,22 @@ def foreach_dependencies(shrinkwrap, callback=None, dev=False):
continue
elif deps[name].get("bundled", False):
continue
- callback(name, deps[name], subtree)
-
- _walk_deps(shrinkwrap.get("dependencies", {}), [])
+ destsubdirs = [os.path.join("node_modules", dep) for dep in subtree]
+ destsuffix = os.path.join(*destsubdirs)
+ callback(name, deps[name], destsuffix)
+
+ # packages entry means new style shrinkwrap file, else use dependencies
+ packages = shrinkwrap.get("packages", None)
+ if packages is not None:
+ for package in packages:
+ if package != "":
+ name = package.split('node_modules/')[-1]
+ package_infos = packages.get(package, {})
+ if dev == False and package_infos.get("dev", False):
+ continue
+ callback(name, package_infos, package)
+ else:
+ _walk_deps(shrinkwrap.get("dependencies", {}), [])
class NpmShrinkWrap(FetchMethod):
"""Class to fetch all package from a shrinkwrap file"""
@@ -75,12 +89,10 @@ class NpmShrinkWrap(FetchMethod):
# Resolve the dependencies
ud.deps = []
- def _resolve_dependency(name, params, deptree):
+ def _resolve_dependency(name, params, destsuffix):
url = None
localpath = None
extrapaths = []
- destsubdirs = [os.path.join("node_modules", dep) for dep in deptree]
- destsuffix = os.path.join(*destsubdirs)
unpack = True
integrity = params.get("integrity", None)
@@ -129,10 +141,28 @@ class NpmShrinkWrap(FetchMethod):
localpath = os.path.join(d.getVar("DL_DIR"), localfile)
+ # Handle local tarball and link sources
+ elif version.startswith("file"):
+ localpath = version[5:]
+ if not version.endswith(".tgz"):
+ unpack = False
+
# Handle git sources
- elif version.startswith("git"):
+ elif version.startswith(("git", "bitbucket","gist")) or (
+ not version.endswith((".tgz", ".tar", ".tar.gz"))
+ and not version.startswith((".", "@", "/"))
+ and "/" in version
+ ):
if version.startswith("github:"):
version = "git+https://github.com/" + version[len("github:"):]
+ elif version.startswith("gist:"):
+ version = "git+https://gist.github.com/" + version[len("gist:"):]
+ elif version.startswith("bitbucket:"):
+ version = "git+https://bitbucket.org/" + version[len("bitbucket:"):]
+ elif version.startswith("gitlab:"):
+ version = "git+https://gitlab.com/" + version[len("gitlab:"):]
+ elif not version.startswith(("git+","git:")):
+ version = "git+https://github.com/" + version
regex = re.compile(r"""
^
git\+
@@ -158,16 +188,12 @@ class NpmShrinkWrap(FetchMethod):
url = str(uri)
- # Handle local tarball and link sources
- elif version.startswith("file"):
- localpath = version[5:]
- if not version.endswith(".tgz"):
- unpack = False
-
else:
raise ParameterError("Unsupported dependency: %s" % name, ud.url)
+ # name is needed by unpack tracer for module mapping
ud.deps.append({
+ "name": name,
"url": url,
"localpath": localpath,
"extrapaths": extrapaths,
@@ -193,19 +219,23 @@ class NpmShrinkWrap(FetchMethod):
# This fetcher resolves multiple URIs from a shrinkwrap file and then
# forwards it to a proxy fetcher. The management of the donestamp file,
# the lockfile and the checksums are forwarded to the proxy fetcher.
- ud.proxy = Fetch([dep["url"] for dep in ud.deps if dep["url"]], data)
+ shrinkwrap_urls = [dep["url"] for dep in ud.deps if dep["url"]]
+ if shrinkwrap_urls:
+ ud.proxy = Fetch(shrinkwrap_urls, data)
ud.needdonestamp = False
@staticmethod
def _foreach_proxy_method(ud, handle):
returns = []
- for proxy_url in ud.proxy.urls:
- proxy_ud = ud.proxy.ud[proxy_url]
- proxy_d = ud.proxy.d
- proxy_ud.setup_localpath(proxy_d)
- lf = lockfile(proxy_ud.lockfile)
- returns.append(handle(proxy_ud.method, proxy_ud, proxy_d))
- unlockfile(lf)
+ #Check if there are dependencies before try to fetch them
+ if len(ud.deps) > 0:
+ for proxy_url in ud.proxy.urls:
+ proxy_ud = ud.proxy.ud[proxy_url]
+ proxy_d = ud.proxy.d
+ proxy_ud.setup_localpath(proxy_d)
+ lf = lockfile(proxy_ud.lockfile)
+ returns.append(handle(proxy_ud.method, proxy_ud, proxy_d))
+ unlockfile(lf)
return returns
def verify_donestamp(self, ud, d):
@@ -238,10 +268,11 @@ class NpmShrinkWrap(FetchMethod):
def unpack(self, ud, rootdir, d):
"""Unpack the downloaded dependencies"""
- destdir = d.getVar("S")
+ destdir = rootdir
destsuffix = ud.parm.get("destsuffix")
if destsuffix:
destdir = os.path.join(rootdir, destsuffix)
+ ud.unpack_tracer.unpack("npm-shrinkwrap", destdir)
bb.utils.mkdirhier(destdir)
bb.utils.copyfile(ud.shrinkwrap_file,
diff --git a/bitbake/lib/bb/fetch2/sftp.py b/bitbake/lib/bb/fetch2/sftp.py
index f87f292e5d..7884cce949 100644
--- a/bitbake/lib/bb/fetch2/sftp.py
+++ b/bitbake/lib/bb/fetch2/sftp.py
@@ -103,7 +103,7 @@ class SFTP(FetchMethod):
if path[:3] == '/~/':
path = path[3:]
- remote = '%s%s:%s' % (user, urlo.hostname, path)
+ remote = '"%s%s:%s"' % (user, urlo.hostname, path)
cmd = '%s %s %s %s' % (basecmd, port, remote, lpath)
diff --git a/bitbake/lib/bb/fetch2/ssh.py b/bitbake/lib/bb/fetch2/ssh.py
index 8d082b38c1..0cbb2a6f25 100644
--- a/bitbake/lib/bb/fetch2/ssh.py
+++ b/bitbake/lib/bb/fetch2/ssh.py
@@ -150,8 +150,6 @@ class SSH(FetchMethod):
)
check_network_access(d, cmd, urldata.url)
+ runfetchcmd(cmd, d)
- if runfetchcmd(cmd, d):
- return True
-
- return False
+ return True
diff --git a/bitbake/lib/bb/fetch2/svn.py b/bitbake/lib/bb/fetch2/svn.py
index d40e4d2909..0852108e7d 100644
--- a/bitbake/lib/bb/fetch2/svn.py
+++ b/bitbake/lib/bb/fetch2/svn.py
@@ -210,3 +210,6 @@ class Svn(FetchMethod):
def _build_revision(self, ud, d):
return ud.revision
+
+ def supports_checksum(self, urldata):
+ return False
diff --git a/bitbake/lib/bb/fetch2/wget.py b/bitbake/lib/bb/fetch2/wget.py
index b2b542e1dc..fbfa6938ac 100644
--- a/bitbake/lib/bb/fetch2/wget.py
+++ b/bitbake/lib/bb/fetch2/wget.py
@@ -26,7 +26,6 @@ from bb.fetch2 import FetchMethod
from bb.fetch2 import FetchError
from bb.fetch2 import logger
from bb.fetch2 import runfetchcmd
-from bb.utils import export_proxies
from bs4 import BeautifulSoup
from bs4 import SoupStrainer
@@ -88,7 +87,10 @@ class Wget(FetchMethod):
if not ud.localfile:
ud.localfile = d.expand(urllib.parse.unquote(ud.host + ud.path).replace("/", "."))
- self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -t 2 -T 30 --passive-ftp"
+ self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -t 2 -T 30"
+
+ if ud.type == 'ftp' or ud.type == 'ftps':
+ self.basecmd += " --passive-ftp"
if not self.check_certs(d):
self.basecmd += " --no-check-certificate"
@@ -132,6 +134,11 @@ class Wget(FetchMethod):
self._runwget(ud, d, fetchcmd, False)
+ # Try and verify any checksum now, meaning if it isn't correct, we don't remove the
+ # original file, which might be a race (imagine two recipes referencing the same
+ # source, one with an incorrect checksum)
+ bb.fetch2.verify_checksum(ud, d, localpath=localpath, fatal_nochecksum=False)
+
# Remove the ".tmp" and move the file into position atomically
# Our lock prevents multiple writers but mirroring code may grab incomplete files
os.rename(localpath, localpath[:-4])
@@ -336,7 +343,8 @@ class Wget(FetchMethod):
opener = urllib.request.build_opener(*handlers)
try:
- uri = ud.url.split(";")[0]
+ uri_base = ud.url.split(";")[0]
+ uri = "{}://{}{}".format(urllib.parse.urlparse(uri_base).scheme, ud.host, ud.path)
r = urllib.request.Request(uri)
r.get_method = lambda: "HEAD"
# Some servers (FusionForge, as used on Alioth) require that the
@@ -355,29 +363,22 @@ class Wget(FetchMethod):
try:
import netrc
- n = netrc.netrc()
- login, unused, password = n.authenticators(urllib.parse.urlparse(uri).hostname)
- add_basic_auth("%s:%s" % (login, password), r)
- except (TypeError, ImportError, IOError, netrc.NetrcParseError):
+ auth_data = netrc.netrc().authenticators(urllib.parse.urlparse(uri).hostname)
+ if auth_data:
+ login, _, password = auth_data
+ add_basic_auth("%s:%s" % (login, password), r)
+ except (FileNotFoundError, netrc.NetrcParseError):
pass
with opener.open(r, timeout=30) as response:
pass
- except urllib.error.URLError as e:
- if try_again:
- logger.debug2("checkstatus: trying again")
- return self.checkstatus(fetch, ud, d, False)
- else:
- # debug for now to avoid spamming the logs in e.g. remote sstate searches
- logger.debug2("checkstatus() urlopen failed: %s" % e)
- return False
- except ConnectionResetError as e:
+ except (urllib.error.URLError, ConnectionResetError, TimeoutError) as e:
if try_again:
logger.debug2("checkstatus: trying again")
return self.checkstatus(fetch, ud, d, False)
else:
# debug for now to avoid spamming the logs in e.g. remote sstate searches
- logger.debug2("checkstatus() urlopen failed: %s" % e)
+ logger.debug2("checkstatus() urlopen failed for %s: %s" % (uri,e))
return False
return True
@@ -639,10 +640,10 @@ class Wget(FetchMethod):
# search for version matches on folders inside the path, like:
# "5.7" in http://download.gnome.org/sources/${PN}/5.7/${PN}-${PV}.tar.gz
dirver_regex = re.compile(r"(?P<dirver>[^/]*(\d+\.)*\d+([-_]r\d+)*)/")
- m = dirver_regex.search(path)
+ m = dirver_regex.findall(path)
if m:
pn = d.getVar('PN')
- dirver = m.group('dirver')
+ dirver = m[-1][0]
dirver_pn_regex = re.compile(r"%s\d?" % (re.escape(pn)))
if not dirver_pn_regex.search(dirver):
generated by cgit 1.2.3-korg (git 2.39.0) at 2024-05-23 23:09:14 +0000