diff options
Diffstat (limited to 'scripts')
-rwxr-xr-x | scripts/buildhistory-diff | 5 | ||||
-rwxr-xr-x | scripts/contrib/convert-srcuri.py | 77 | ||||
-rw-r--r-- | scripts/lib/checklayer/cases/common.py | 2 | ||||
-rw-r--r-- | scripts/lib/devtool/deploy.py | 2 | ||||
-rw-r--r-- | scripts/lib/recipetool/create.py | 16 | ||||
-rw-r--r-- | scripts/lib/recipetool/create_buildsys.py | 3 | ||||
-rw-r--r-- | scripts/lib/scriptutils.py | 10 | ||||
-rw-r--r-- | scripts/lib/wic/engine.py | 6 | ||||
-rw-r--r-- | scripts/lib/wic/help.py | 4 | ||||
-rw-r--r-- | scripts/lib/wic/misc.py | 4 | ||||
-rw-r--r-- | scripts/lib/wic/pluginbase.py | 8 | ||||
-rw-r--r-- | scripts/lib/wic/plugins/imager/direct.py | 2 | ||||
-rwxr-xr-x | scripts/oe-pkgdata-browser | 2 | ||||
-rwxr-xr-x | scripts/runqemu | 3 | ||||
-rwxr-xr-x | scripts/runqemu-ifdown | 1 | ||||
-rwxr-xr-x | scripts/wic | 7 | ||||
-rwxr-xr-x | scripts/yocto-check-layer | 13 |
17 files changed, 135 insertions, 30 deletions
diff --git a/scripts/buildhistory-diff b/scripts/buildhistory-diff index 3bd40a2a1e..a6e785aa23 100755 --- a/scripts/buildhistory-diff +++ b/scripts/buildhistory-diff @@ -11,7 +11,6 @@ import sys import os import argparse -from distutils.version import LooseVersion # Ensure PythonGit is installed (buildhistory_analysis needs it) try: @@ -73,10 +72,6 @@ def main(): parser = get_args_parser() args = parser.parse_args() - if LooseVersion(git.__version__) < '0.3.1': - sys.stderr.write("Version of GitPython is too old, please install GitPython (python-git) 0.3.1 or later in order to use this script\n") - sys.exit(1) - if len(args.revisions) > 2: sys.stderr.write('Invalid argument(s) specified: %s\n\n' % ' '.join(args.revisions[2:])) parser.print_help() diff --git a/scripts/contrib/convert-srcuri.py b/scripts/contrib/convert-srcuri.py new file mode 100755 index 0000000000..587392334f --- /dev/null +++ b/scripts/contrib/convert-srcuri.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +# +# Conversion script to update SRC_URI to add branch to git urls +# +# Copyright (C) 2021 Richard Purdie +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import re +import os +import sys +import tempfile +import shutil +import mimetypes + +if len(sys.argv) < 2: + print("Please specify a directory to run the conversion script against.") + sys.exit(1) + +def processfile(fn): + def matchline(line): + if "MIRROR" in line or ".*" in line or "GNOME_GIT" in line: + return False + return True + print("processing file '%s'" % fn) + try: + if "distro_alias.inc" in fn or "linux-yocto-custom.bb" in fn: + return + fh, abs_path = tempfile.mkstemp() + modified = False + with os.fdopen(fh, 'w') as new_file: + with open(fn, "r") as old_file: + for line in old_file: + if ("git://" in line or "gitsm://" in line) and "branch=" not in line and matchline(line): + if line.endswith('"\n'): + line = line.replace('"\n', ';branch=master"\n') + elif re.search('\s*\\\\$', line): + line = re.sub('\s*\\\\$', ';branch=master \\\\', line) + modified = True + if ("git://" in line or "gitsm://" in line) and "github.com" in line and "protocol=https" not in line and matchline(line): + if "protocol=git" in line: + line = line.replace('protocol=git', 'protocol=https') + elif line.endswith('"\n'): + line = line.replace('"\n', ';protocol=https"\n') + elif re.search('\s*\\\\$', line): + line = re.sub('\s*\\\\$', ';protocol=https \\\\', line) + modified = True + new_file.write(line) + if modified: + shutil.copymode(fn, abs_path) + os.remove(fn) + shutil.move(abs_path, fn) + except UnicodeDecodeError: + pass + +ourname = os.path.basename(sys.argv[0]) +ourversion = "0.1" + +if os.path.isfile(sys.argv[1]): + processfile(sys.argv[1]) + sys.exit(0) + +for targetdir in sys.argv[1:]: + print("processing directory '%s'" % targetdir) + for root, dirs, files in os.walk(targetdir): + for name in files: + if name == ourname: + continue + fn = os.path.join(root, name) + if os.path.islink(fn): + continue + if "/.git/" in fn or fn.endswith(".html") or fn.endswith(".patch") or fn.endswith(".m4") or fn.endswith(".diff"): + continue + processfile(fn) + +print("All files processed with version %s" % ourversion) diff --git a/scripts/lib/checklayer/cases/common.py b/scripts/lib/checklayer/cases/common.py index b82304e361..4495f71b24 100644 --- a/scripts/lib/checklayer/cases/common.py +++ b/scripts/lib/checklayer/cases/common.py @@ -14,7 +14,7 @@ class CommonCheckLayer(OECheckLayerTestCase): # The top-level README file may have a suffix (like README.rst or README.txt). readme_files = glob.glob(os.path.join(self.tc.layer['path'], '[Rr][Ee][Aa][Dd][Mm][Ee]*')) self.assertTrue(len(readme_files) > 0, - msg="Layer doesn't contains README file.") + msg="Layer doesn't contain a README file.") # There might be more than one file matching the file pattern above # (for example, README.rst and README-COPYING.rst). The one with the shortest diff --git a/scripts/lib/devtool/deploy.py b/scripts/lib/devtool/deploy.py index 833322571f..e14a587417 100644 --- a/scripts/lib/devtool/deploy.py +++ b/scripts/lib/devtool/deploy.py @@ -170,7 +170,7 @@ def deploy(args, config, basepath, workspace): srcdir = recipe_outdir recipe_outdir = os.path.join(rd.getVar('WORKDIR'), 'devtool-deploy-target-stripped') if os.path.isdir(recipe_outdir): - bb.utils.remove(recipe_outdir, True) + exec_fakeroot(rd, "rm -rf %s" % recipe_outdir, shell=True) exec_fakeroot(rd, "cp -af %s %s" % (os.path.join(srcdir, '.'), recipe_outdir), shell=True) os.environ['PATH'] = ':'.join([os.environ['PATH'], rd.getVar('PATH') or '']) oe.package.strip_execs(args.recipename, recipe_outdir, rd.getVar('STRIP'), rd.getVar('libdir'), diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py index 566c75369a..b9f9c80367 100644 --- a/scripts/lib/recipetool/create.py +++ b/scripts/lib/recipetool/create.py @@ -366,7 +366,7 @@ def supports_srcrev(uri): def reformat_git_uri(uri): '''Convert any http[s]://....git URI into git://...;protocol=http[s]''' checkuri = uri.split(';', 1)[0] - if checkuri.endswith('.git') or '/git/' in checkuri or re.match('https?://github.com/[^/]+/[^/]+/?$', checkuri): + if checkuri.endswith('.git') or '/git/' in checkuri or re.match('https?://git(hub|lab).com/[^/]+/[^/]+/?$', checkuri): # Appends scheme if the scheme is missing if not '://' in uri: uri = 'git://' + uri @@ -435,7 +435,7 @@ def create_recipe(args): if args.binary: # Assume the archive contains the directory structure verbatim # so we need to extract to a subdirectory - fetchuri += ';subdir=${BP}' + fetchuri += ';subdir=${BPN}' srcuri = fetchuri rev_re = re.compile(';rev=([^;]+)') res = rev_re.search(srcuri) @@ -478,6 +478,9 @@ def create_recipe(args): storeTagName = params['tag'] params['nobranch'] = '1' del params['tag'] + # Assume 'master' branch if not set + if scheme in ['git', 'gitsm'] and 'branch' not in params and 'nobranch' not in params: + params['branch'] = 'master' fetchuri = bb.fetch2.encodeurl((scheme, network, path, user, passwd, params)) tmpparent = tinfoil.config_data.getVar('BASE_WORKDIR') @@ -527,10 +530,9 @@ def create_recipe(args): # Remove HEAD reference point and drop remote prefix get_branch = [x.split('/', 1)[1] for x in get_branch if not x.startswith('origin/HEAD')] if 'master' in get_branch: - # If it is master, we do not need to append 'branch=master' as this is default. # Even with the case where get_branch has multiple objects, if 'master' is one # of them, we should default take from 'master' - srcbranch = '' + srcbranch = 'master' elif len(get_branch) == 1: # If 'master' isn't in get_branch and get_branch contains only ONE object, then store result into 'srcbranch' srcbranch = get_branch[0] @@ -543,8 +545,8 @@ def create_recipe(args): # Since we might have a value in srcbranch, we need to # recontruct the srcuri to include 'branch' in params. scheme, network, path, user, passwd, params = bb.fetch2.decodeurl(srcuri) - if srcbranch: - params['branch'] = srcbranch + if scheme in ['git', 'gitsm']: + params['branch'] = srcbranch or 'master' if storeTagName and scheme in ['git', 'gitsm']: # Check srcrev using tag and check validity of the tag @@ -603,7 +605,7 @@ def create_recipe(args): splitline = line.split() if len(splitline) > 1: if splitline[0] == 'origin' and scriptutils.is_src_url(splitline[1]): - srcuri = reformat_git_uri(splitline[1]) + srcuri = reformat_git_uri(splitline[1]) + ';branch=master' srcsubdir = 'git' break diff --git a/scripts/lib/recipetool/create_buildsys.py b/scripts/lib/recipetool/create_buildsys.py index 35a97c9345..5015634476 100644 --- a/scripts/lib/recipetool/create_buildsys.py +++ b/scripts/lib/recipetool/create_buildsys.py @@ -545,7 +545,7 @@ class AutotoolsRecipeHandler(RecipeHandler): deps.append('zlib') elif keyword in ('AX_CHECK_OPENSSL', 'AX_LIB_CRYPTO'): deps.append('openssl') - elif keyword == 'AX_LIB_CURL': + elif keyword in ('AX_LIB_CURL', 'LIBCURL_CHECK_CONFIG'): deps.append('curl') elif keyword == 'AX_LIB_BEECRYPT': deps.append('beecrypt') @@ -624,6 +624,7 @@ class AutotoolsRecipeHandler(RecipeHandler): 'AX_CHECK_OPENSSL', 'AX_LIB_CRYPTO', 'AX_LIB_CURL', + 'LIBCURL_CHECK_CONFIG', 'AX_LIB_BEECRYPT', 'AX_LIB_EXPAT', 'AX_LIB_GCRYPT', diff --git a/scripts/lib/scriptutils.py b/scripts/lib/scriptutils.py index f92255d8dc..47a08194d0 100644 --- a/scripts/lib/scriptutils.py +++ b/scripts/lib/scriptutils.py @@ -18,7 +18,8 @@ import sys import tempfile import threading import importlib -from importlib import machinery +import importlib.machinery +import importlib.util class KeepAliveStreamHandler(logging.StreamHandler): def __init__(self, keepalive=True, **kwargs): @@ -82,7 +83,9 @@ def load_plugins(logger, plugins, pluginpath): logger.debug('Loading plugin %s' % name) spec = importlib.machinery.PathFinder.find_spec(name, path=[pluginpath] ) if spec: - return spec.loader.load_module() + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod def plugin_name(filename): return os.path.splitext(os.path.basename(filename))[0] @@ -215,7 +218,8 @@ def fetch_url(tinfoil, srcuri, srcrev, destdir, logger, preserve_tmp=False, mirr pathvars = ['T', 'RECIPE_SYSROOT', 'RECIPE_SYSROOT_NATIVE'] for pathvar in pathvars: path = rd.getVar(pathvar) - shutil.rmtree(path) + if os.path.exists(path): + shutil.rmtree(path) finally: if fetchrecipe: try: diff --git a/scripts/lib/wic/engine.py b/scripts/lib/wic/engine.py index 018815b966..674ccfc244 100644 --- a/scripts/lib/wic/engine.py +++ b/scripts/lib/wic/engine.py @@ -19,10 +19,10 @@ import os import tempfile import json import subprocess +import shutil import re from collections import namedtuple, OrderedDict -from distutils.spawn import find_executable from wic import WicError from wic.filemap import sparse_copy @@ -245,7 +245,7 @@ class Disk: for path in pathlist.split(':'): self.paths = "%s%s:%s" % (native_sysroot, path, self.paths) - self.parted = find_executable("parted", self.paths) + self.parted = shutil.which("parted", path=self.paths) if not self.parted: raise WicError("Can't find executable parted") @@ -283,7 +283,7 @@ class Disk: "resize2fs", "mkswap", "mkdosfs", "debugfs","blkid"): aname = "_%s" % name if aname not in self.__dict__: - setattr(self, aname, find_executable(name, self.paths)) + setattr(self, aname, shutil.which(name, path=self.paths)) if aname not in self.__dict__ or self.__dict__[aname] is None: raise WicError("Can't find executable '{}'".format(name)) return self.__dict__[aname] diff --git a/scripts/lib/wic/help.py b/scripts/lib/wic/help.py index bd3a2b97df..9c2d6991a9 100644 --- a/scripts/lib/wic/help.py +++ b/scripts/lib/wic/help.py @@ -840,8 +840,8 @@ DESCRIPTION meanings. The commands are based on the Fedora kickstart documentation but with modifications to reflect wic capabilities. - http://fedoraproject.org/wiki/Anaconda/Kickstart#part_or_partition - http://fedoraproject.org/wiki/Anaconda/Kickstart#bootloader + https://pykickstart.readthedocs.io/en/latest/kickstart-docs.html#part-or-partition + https://pykickstart.readthedocs.io/en/latest/kickstart-docs.html#bootloader Commands diff --git a/scripts/lib/wic/misc.py b/scripts/lib/wic/misc.py index 57c042c503..3e11822996 100644 --- a/scripts/lib/wic/misc.py +++ b/scripts/lib/wic/misc.py @@ -16,9 +16,9 @@ import logging import os import re import subprocess +import shutil from collections import defaultdict -from distutils import spawn from wic import WicError @@ -122,7 +122,7 @@ def find_executable(cmd, paths): if provided and "%s-native" % recipe in provided: return True - return spawn.find_executable(cmd, paths) + return shutil.which(cmd, path=paths) def exec_native_cmd(cmd_and_args, native_sysroot, pseudo=""): """ diff --git a/scripts/lib/wic/pluginbase.py b/scripts/lib/wic/pluginbase.py index d9b4e57747..b64568339b 100644 --- a/scripts/lib/wic/pluginbase.py +++ b/scripts/lib/wic/pluginbase.py @@ -9,9 +9,11 @@ __all__ = ['ImagerPlugin', 'SourcePlugin'] import os import logging +import types from collections import defaultdict -from importlib.machinery import SourceFileLoader +import importlib +import importlib.util from wic import WicError from wic.misc import get_bitbake_var @@ -54,7 +56,9 @@ class PluginMgr: mname = fname[:-3] mpath = os.path.join(ppath, fname) logger.debug("loading plugin module %s", mpath) - SourceFileLoader(mname, mpath).load_module() + spec = importlib.util.spec_from_file_location(mname, mpath) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) return PLUGINS.get(ptype) diff --git a/scripts/lib/wic/plugins/imager/direct.py b/scripts/lib/wic/plugins/imager/direct.py index ea709e8c54..2c5d5efe0e 100644 --- a/scripts/lib/wic/plugins/imager/direct.py +++ b/scripts/lib/wic/plugins/imager/direct.py @@ -258,6 +258,8 @@ class DirectPlugin(ImagerPlugin): if part.mountpoint == "/": if part.uuid: return "PARTUUID=%s" % part.uuid + elif part.label: + return "PARTLABEL=%s" % part.label else: suffix = 'p' if part.disk.startswith('mmcblk') else '' return "/dev/%s%s%-d" % (part.disk, suffix, part.realnum) diff --git a/scripts/oe-pkgdata-browser b/scripts/oe-pkgdata-browser index 8d223185a4..65a6ee956e 100755 --- a/scripts/oe-pkgdata-browser +++ b/scripts/oe-pkgdata-browser @@ -236,6 +236,8 @@ class PkgUi(): update_deps("RPROVIDES", "Provides: ", self.provides_label, clickable=False) def load_recipes(self): + if not os.path.exists(pkgdata): + sys.exit("Error: Please ensure %s exists by generating packages before using this tool." % pkgdata) for recipe in sorted(os.listdir(pkgdata)): if os.path.isfile(os.path.join(pkgdata, recipe)): self.recipe_iters[recipe] = self.recipe_store.append([recipe]) diff --git a/scripts/runqemu b/scripts/runqemu index f3527a4412..f9a1a0da32 100755 --- a/scripts/runqemu +++ b/scripts/runqemu @@ -1573,7 +1573,8 @@ def main(): def sigterm_handler(signum, frame): logger.info("SIGTERM received") - os.kill(config.qemupid, signal.SIGTERM) + if config.qemupid: + os.kill(config.qemupid, signal.SIGTERM) config.cleanup() # Deliberately ignore the return code of 'tput smam'. subprocess.call(["tput", "smam"]) diff --git a/scripts/runqemu-ifdown b/scripts/runqemu-ifdown index a104c37bf8..e0eb5344c6 100755 --- a/scripts/runqemu-ifdown +++ b/scripts/runqemu-ifdown @@ -64,3 +64,4 @@ n=$[ (`echo $TAP | sed 's/tap//'` * 2) + 1 ] dest=$[ (`echo $TAP | sed 's/tap//'` * 2) + 2 ] $IPTABLES -D POSTROUTING -t nat -j MASQUERADE -s 192.168.7.$n/32 $IPTABLES -D POSTROUTING -t nat -j MASQUERADE -s 192.168.7.$dest/32 +true diff --git a/scripts/wic b/scripts/wic index a741aed364..c0bc0e6fe8 100755 --- a/scripts/wic +++ b/scripts/wic @@ -22,9 +22,9 @@ import sys import argparse import logging import subprocess +import shutil from collections import namedtuple -from distutils import spawn # External modules scripts_path = os.path.dirname(os.path.realpath(__file__)) @@ -47,7 +47,7 @@ if os.environ.get('SDKTARGETSYSROOT'): break sdkroot = os.path.dirname(sdkroot) -bitbake_exe = spawn.find_executable('bitbake') +bitbake_exe = shutil.which('bitbake') if bitbake_exe: bitbake_path = scriptpath.add_bitbake_lib_path() import bb @@ -159,6 +159,9 @@ def wic_create_subcommand(options, usage_str): "(Use -e/--image-name to specify it)") native_sysroot = options.native_sysroot + if options.kernel_dir: + kernel_dir = options.kernel_dir + if not options.vars_dir and (not native_sysroot or not os.path.isdir(native_sysroot)): logger.info("Building wic-tools...\n") subprocess.check_call(["bitbake", "wic-tools"]) diff --git a/scripts/yocto-check-layer b/scripts/yocto-check-layer index 6975b09502..00a16d18fc 100755 --- a/scripts/yocto-check-layer +++ b/scripts/yocto-check-layer @@ -41,6 +41,12 @@ def test_layer(td, layer, test_software_layer_signatures): tc.loadTests(CASES_PATHS) return tc.runTests() +def dump_layer_debug(layer): + logger.debug("Found layer %s (%s)" % (layer["name"], layer["path"])) + collections = layer.get("collections", {}) + if collections: + logger.debug("%s collections: %s" % (layer["name"], ", ".join(collections))) + def main(): parser = argparse.ArgumentParser( description="Yocto Project layer checking tool", @@ -106,6 +112,13 @@ def main(): else: dep_layers = layers + logger.debug("Found additional layers:") + for l in additional_layers: + dump_layer_debug(l) + logger.debug("Found dependency layers:") + for l in dep_layers: + dump_layer_debug(l) + logger.info("Detected layers:") for layer in layers: if layer['type'] == LayerType.ERROR_BSP_DISTRO: |