summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--meta/classes/devtool-source.bbclass56
-rw-r--r--meta/lib/oe/recipeutils.py4
-rw-r--r--meta/lib/oe/utils.py24
-rw-r--r--meta/lib/oeqa/selftest/cases/devtool.py2
-rwxr-xr-xscripts/devtool63
-rw-r--r--scripts/lib/devtool/__init__.py14
-rw-r--r--scripts/lib/devtool/standard.py383
-rw-r--r--scripts/lib/devtool/upgrade.py1
8 files changed, 406 insertions, 141 deletions
diff --git a/meta/classes/devtool-source.bbclass b/meta/classes/devtool-source.bbclass
index 41900e651f0..90b50fb4d5d 100644
--- a/meta/classes/devtool-source.bbclass
+++ b/meta/classes/devtool-source.bbclass
@@ -69,7 +69,7 @@ python devtool_post_unpack() {
import shutil
sys.path.insert(0, os.path.join(d.getVar('COREBASE'), 'scripts', 'lib'))
import scriptutils
- from devtool import setup_git_repo
+ from devtool import setup_git_repo, find_git_repos
tempdir = d.getVar('DEVTOOL_TEMPDIR')
workdir = d.getVar('WORKDIR')
@@ -108,7 +108,26 @@ python devtool_post_unpack() {
shutil.copy2(cfg, workdir)
# Ignore local files with subdir={BP}
+ excludeitems = recipe_patches + list(local_files.keys())
+ pthvars = ['RECIPE_SYSROOT', 'RECIPE_SYSROOT_NATIVE', 'S']
+ for pthvar in pthvars:
+ relp = os.path.relpath(d.getVar(pthvar), d.getVar('WORKDIR'))
+ if not relp.startswith('..'):
+ excludeitems.append(relp.split(os.sep)[0])
+ extradirs = []
srcabspath = os.path.abspath(srcsubdir)
+ if srcabspath != os.path.abspath(workdir):
+ for pth in os.listdir(workdir):
+ if pth in excludeitems:
+ continue
+ wpth = os.path.join(workdir, pth)
+ if os.path.isdir(wpth) and os.listdir(wpth):
+ extradirs.append(wpth)
+
+ repos = find_git_repos(srcabspath)
+ extradirs.extend(repos)
+
+ # Ignore local files with subdir={BP}
local_files = [fname for fname in local_files if
os.path.exists(os.path.join(workdir, fname)) and
(srcabspath == workdir or not
@@ -151,11 +170,20 @@ python devtool_post_unpack() {
(stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srcsubdir)
initial_rev = stdout.rstrip()
+
+ initial_revs = {}
+ for extradir in extradirs:
+ setup_git_repo(extradir, d.getVar('PV'), devbranch, d=d)
+ (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=extradir)
+ initial_revs[extradir] = stdout.rstrip()
+
with open(os.path.join(tempdir, 'initial_rev'), 'w') as f:
f.write(initial_rev)
with open(os.path.join(tempdir, 'srcsubdir'), 'w') as f:
- f.write(srcsubdir)
+ f.write('%s\n' % srcsubdir)
+ for extradir in extradirs:
+ f.write('%s=%s\n' % (extradir, initial_revs[extradir]))
}
python devtool_pre_patch() {
@@ -166,18 +194,25 @@ python devtool_pre_patch() {
python devtool_post_patch() {
import shutil
tempdir = d.getVar('DEVTOOL_TEMPDIR')
+
+ srcdirs = []
with open(os.path.join(tempdir, 'srcsubdir'), 'r') as f:
- srcsubdir = f.read()
+ for line in f:
+ line = line.rstrip()
+ if line:
+ srcdirs.append(line.split('=')[0])
+ srcsubdir = srcdirs[0]
+
with open(os.path.join(tempdir, 'initial_rev'), 'r') as f:
initial_rev = f.read()
- def rm_patches():
- patches_dir = os.path.join(srcsubdir, 'patches')
+ def rm_patches(pth):
+ patches_dir = os.path.join(pth, 'patches')
if os.path.exists(patches_dir):
shutil.rmtree(patches_dir)
# Restore any "patches" directory that was actually part of the source tree
try:
- bb.process.run('git checkout -- patches', cwd=srcsubdir)
+ bb.process.run('git checkout -- patches', cwd=pth)
except bb.process.ExecutionError:
pass
@@ -201,7 +236,7 @@ python devtool_post_patch() {
localdata.setVar('OVERRIDES', ':'.join(no_overrides))
localdata.setVar('FILESOVERRIDES', ':'.join(no_overrides))
bb.build.exec_func('do_patch', localdata)
- rm_patches()
+ rm_patches(srcsubdir)
# Now we need to reconcile the dev branch with the no-overrides one
# (otherwise we'd likely be left with identical commits that have different hashes)
bb.process.run('git checkout %s' % devbranch, cwd=srcsubdir)
@@ -220,12 +255,15 @@ python devtool_post_patch() {
localdata.setVar('OVERRIDES', ':'.join(no_overrides + [override]))
localdata.setVar('FILESOVERRIDES', ':'.join(no_overrides + [override]))
bb.build.exec_func('do_patch', localdata)
- rm_patches()
+ rm_patches(srcsubdir)
# Now we need to reconcile the new branch with the no-overrides one
# (otherwise we'd likely be left with identical commits that have different hashes)
bb.process.run('git rebase devtool-no-overrides', cwd=srcsubdir)
bb.process.run('git checkout %s' % devbranch, cwd=srcsubdir)
- bb.process.run('git tag -f devtool-patched', cwd=srcsubdir)
+ for srcdir in srcdirs:
+ bb.process.run('git tag -f devtool-patched', cwd=srcdir)
+ if srcdir != srcsubdir:
+ rm_patches(srcdir)
}
python devtool_post_configure() {
diff --git a/meta/lib/oe/recipeutils.py b/meta/lib/oe/recipeutils.py
index 3594db31abf..de71d8d7bfc 100644
--- a/meta/lib/oe/recipeutils.py
+++ b/meta/lib/oe/recipeutils.py
@@ -506,10 +506,10 @@ def get_recipe_patches(d):
"""Get a list of the patches included in SRC_URI within a recipe."""
import oe.patch
patches = oe.patch.src_patches(d, expand=False)
- patchfiles = []
+ patchfiles = OrderedDict()
for patch in patches:
_, _, local, _, _, parm = bb.fetch.decodeurl(patch)
- patchfiles.append(local)
+ patchfiles[local] = parm
return patchfiles
diff --git a/meta/lib/oe/utils.py b/meta/lib/oe/utils.py
index 9a2187e36f8..6bc4bd5eecd 100644
--- a/meta/lib/oe/utils.py
+++ b/meta/lib/oe/utils.py
@@ -536,3 +536,27 @@ class ImageQAFailed(Exception):
def sh_quote(string):
import shlex
return shlex.quote(string)
+
+def is_path_under(path1, possible_parent):
+ """
+ Return True if a path is underneath another, False otherwise.
+ Multiple paths to test can be specified (as a list or tuple) in
+ which case all specified test paths must be under the parent to
+ return True.
+ """
+ def abs_path_trailing(pth):
+ pth_abs = os.path.abspath(pth)
+ if not pth_abs.endswith(os.sep):
+ pth_abs += os.sep
+ return pth_abs
+
+ possible_parent_abs = abs_path_trailing(possible_parent)
+ if isinstance(path1, str):
+ testpaths = [path1]
+ else:
+ testpaths = path1
+ for path in testpaths:
+ path_abs = abs_path_trailing(path)
+ if not path_abs.startswith(possible_parent_abs):
+ return False
+ return True
diff --git a/meta/lib/oeqa/selftest/cases/devtool.py b/meta/lib/oeqa/selftest/cases/devtool.py
index e9b466f4eb8..21eb0029aad 100644
--- a/meta/lib/oeqa/selftest/cases/devtool.py
+++ b/meta/lib/oeqa/selftest/cases/devtool.py
@@ -11,7 +11,7 @@ import fnmatch
import oeqa.utils.ftools as ftools
from oeqa.selftest.case import OESelftestTestCase
-from oeqa.utils.commands import runCmd, bitbake, get_bb_var, create_temp_layer
+from oeqa.utils.commands import runCmd, bitbake, get_bb_var, create_temp_layer, CommandError
from oeqa.utils.commands import get_bb_vars, runqemu, get_test_layer
oldmetapath = None
diff --git a/scripts/devtool b/scripts/devtool
index 8a4f41bc372..cfc26cfb855 100755
--- a/scripts/devtool
+++ b/scripts/devtool
@@ -15,6 +15,8 @@ import re
import configparser
import subprocess
import logging
+import json
+from collections import OrderedDict
basepath = ''
workspace = {}
@@ -99,32 +101,59 @@ def read_workspace():
if not context.fixed_setup:
_enable_workspace_layer(config.workspace_path, config, basepath)
+ def process_inline_json(strvalue, in_value):
+ if strvalue.count('{') == strvalue.count('}') and strvalue.count('[') == strvalue.count(']'):
+ pnvalues[in_value] = json.loads(strvalue, object_pairs_hook=OrderedDict)
+ return True
+ return False
+
logger.debug('Reading workspace in %s' % config.workspace_path)
externalsrc_re = re.compile(r'^EXTERNALSRC(_pn-([^ =]+))? *= *"([^"]*)"$')
for fn in glob.glob(os.path.join(config.workspace_path, 'appends', '*.bbappend')):
with open(fn, 'r') as f:
pnvalues = {}
+ in_value = None
+ strvalue = ''
for line in f:
- res = externalsrc_re.match(line.rstrip())
- if res:
- recipepn = os.path.splitext(os.path.basename(fn))[0].split('_')[0]
- pn = res.group(2) or recipepn
- # Find the recipe file within the workspace, if any
- bbfile = os.path.basename(fn).replace('.bbappend', '.bb').replace('%', '*')
- recipefile = glob.glob(os.path.join(config.workspace_path,
- 'recipes',
- recipepn,
- bbfile))
- if recipefile:
- recipefile = recipefile[0]
- pnvalues['srctree'] = res.group(3)
- pnvalues['bbappend'] = fn
- pnvalues['recipefile'] = recipefile
- elif line.startswith('# srctreebase: '):
- pnvalues['srctreebase'] = line.split(':', 1)[1].strip()
+ if in_value:
+ if not line.startswith('#'):
+ logger.error('Syntax error in %s - invalid in-line json' % fn)
+ sys.exit(1)
+ strvalue += line[1:]
+ if process_inline_json(strvalue, in_value):
+ in_value = None
+ else:
+ res = externalsrc_re.match(line.rstrip())
+ if res:
+ recipepn = os.path.splitext(os.path.basename(fn))[0].split('_')[0]
+ pn = res.group(2) or recipepn
+ # Find the recipe file within the workspace, if any
+ bbfile = os.path.basename(fn).replace('.bbappend', '.bb').replace('%', '*')
+ recipefile = glob.glob(os.path.join(config.workspace_path,
+ 'recipes',
+ recipepn,
+ bbfile))
+ if recipefile:
+ recipefile = recipefile[0]
+ pnvalues['srctree'] = res.group(3)
+ pnvalues['bbappend'] = fn
+ pnvalues['recipefile'] = recipefile
+ elif line.startswith('# srctreebase: '):
+ pnvalues['srctreebase'] = line.split(':', 1)[1].strip()
+ elif line.startswith('# srctreetop: '):
+ pnvalues['srctreetop'] = line.split(':', 1)[1].strip()
+ elif line.startswith('# extradirs: '):
+ strvalue = line.split(':', 1)[1].strip()
+ in_value = 'extradirs'
+ if process_inline_json(strvalue, in_value):
+ in_value = None
if pnvalues:
if not pnvalues.get('srctreebase', None):
pnvalues['srctreebase'] = pnvalues['srctree']
+ if not 'srctreetop' in pnvalues:
+ pnvalues['srctreetop'] = pnvalues['srctreebase']
+ if not 'extradirs' in pnvalues:
+ pnvalues['extradirs'] = []
logger.debug('Found recipe %s' % pnvalues)
workspace[pn] = pnvalues
diff --git a/scripts/lib/devtool/__init__.py b/scripts/lib/devtool/__init__.py
index 702db669de3..e5fa34a23ea 100644
--- a/scripts/lib/devtool/__init__.py
+++ b/scripts/lib/devtool/__init__.py
@@ -233,6 +233,20 @@ def setup_git_repo(repodir, version, devbranch, basetag='devtool-base', d=None):
bb.process.run('git checkout -b %s' % devbranch, cwd=repodir)
bb.process.run('git tag -f %s' % basetag, cwd=repodir)
+def find_git_repos(pth, toplevel=False):
+ """
+ Find git repositories under a path
+ """
+ repos = []
+ if toplevel and os.path.isdir(os.path.join(pth, '.git')):
+ repos.append(pth)
+ for root, dirs, _ in os.walk(pth):
+ for dfn in dirs:
+ dfp = os.path.join(root, dfn)
+ if os.path.isdir(os.path.join(dfp, '.git')) and dfp not in repos:
+ repos.append(dfp)
+ return repos
+
def recipe_to_append(recipefile, config, wildcard=False):
"""
Convert a recipe file to a bbappend file path within the workspace.
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index 554542ed4cf..f8a138e655f 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -19,8 +19,9 @@ import scriptutils
import errno
import glob
import filecmp
+import json
from collections import OrderedDict, namedtuple
-from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, use_external_build, setup_git_repo, recipe_to_append, get_bbclassextend_targets, update_unlockedsigs, check_prerelease_version, check_git_repo_dirty, check_git_repo_op, DevtoolError
+from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, use_external_build, setup_git_repo, recipe_to_append, get_bbclassextend_targets, update_unlockedsigs, check_prerelease_version, check_git_repo_dirty, check_git_repo_op, find_git_repos, DevtoolError
from devtool import parse_recipe
logger = logging.getLogger('devtool')
@@ -479,7 +480,7 @@ def symlink_oelocal_files_srctree(rd,srctree):
bb.process.run('git %s commit -m "Committing local file symlinks\n\n%s"' % (' '.join(useroptions), oe.patch.GitApplyTree.ignore_commit_prefix), cwd=srctree)
-ExtractSourceResult = namedtuple('ExtractSourceResult', ['initial_rev', 'srcsubdir'])
+ExtractSourceResult = namedtuple('ExtractSourceResult', ['initial_rev', 'srcsubdir', 'extradirs'])
def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, workspace, fixed_setup, d, tinfoil, no_overrides=False):
"""Extract sources of a recipe"""
@@ -598,15 +599,27 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works
if not res:
raise DevtoolError('Extracting source for %s failed' % pn)
+ tempworkdir = os.path.join(tempdir, 'workdir')
+ extra_revs = OrderedDict()
+ extradirs = []
try:
with open(os.path.join(tempdir, 'initial_rev'), 'r') as f:
initial_rev = f.read()
with open(os.path.join(tempdir, 'srcsubdir'), 'r') as f:
- srcsubdir = f.read()
+ srcsubdir = f.readline().rstrip()
+ for line in f:
+ line = line.rstrip()
+ if '=' in line:
+ linesplit = line.rstrip().split('=', 1)
+ extradir = linesplit[0]
+ extradirs.append(extradir)
+ extra_revs[os.path.relpath(extradir, tempworkdir)] = linesplit[1]
+
except FileNotFoundError as e:
raise DevtoolError('Something went wrong with source extraction - the devtool-source class was not active or did not function correctly:\n%s' % str(e))
- srcsubdir_rel = os.path.relpath(srcsubdir, os.path.join(tempdir, 'workdir'))
+
+ srcsubdir_rel = os.path.relpath(srcsubdir, tempworkdir)
# Check if work-shared is empty, if yes
# find source and copy to work-shared
@@ -652,12 +665,25 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works
logger.info('Adding local source files to srctree...')
shutil.move(tempdir_localdir, srcsubdir)
+ if extra_revs:
+ logger.warn('This recipe fetches more than one source tree. Each source tree will be tracked in a separate git repository.')
+ if not oe.utils.is_path_under(extradirs, srcsubdir):
+ # There's more than one source directory at the top level, we're going to need to create the parent
+ os.mkdir(srctree)
+ for extradir in extradirs:
+ shutil.move(extradir, srctree)
+ # Write a file to record which one is the main src in case we need to re-modify the same tree later
+ with open(os.path.join(srctree, '.devtool'), 'w') as f:
+ data = OrderedDict()
+ data['srcsubdir'] = os.path.basename(srcsubdir)
+ data['extradirs'] = extra_revs
+ json.dump(data, f, indent=' ')
shutil.move(srcsubdir, srctree)
symlink_oelocal_files_srctree(d,srctree)
if is_kernel_yocto:
logger.info('Copying kernel config to srctree')
- shutil.copy2(os.path.join(tempdir, '.config'), srctree)
+ shutil.copy2(os.path.join(tempdir, '.config'), os.path.join(srctree, srcsubdir_rel))
finally:
if appendbackup:
@@ -668,7 +694,7 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works
logger.info('Preserving temporary directory %s' % tempdir)
else:
shutil.rmtree(tempdir)
- return ExtractSourceResult(initial_rev, srcsubdir_rel)
+ return ExtractSourceResult(initial_rev, srcsubdir_rel, extra_revs)
def _add_md5(config, recipename, filename):
"""Record checksum of a file (or recursively for a directory) to the md5-file of the workspace"""
@@ -740,6 +766,32 @@ def get_staging_kbranch(srcdir):
staging_kbranch = "".join(branch.split('\n')[0])
return staging_kbranch
+def _get_initial_rev(srctree):
+ """
+ Given a source tree, get the initial revision and determine if it was prepared by devtool
+ """
+ initial_rev = None
+ # Check if it's a tree previously extracted by us. This is done
+ # by ensuring that devtool-base and args.branch (devtool) exist.
+ # The check_commits logic will cause an exception if either one
+ # of these doesn't exist
+ try:
+ (stdout, _) = bb.process.run('git branch --contains devtool-base', cwd=srctree)
+ bb.process.run('git rev-parse %s' % args.branch, cwd=srctree)
+ except bb.process.ExecutionError:
+ stdout = ''
+ if stdout:
+ was_devtool = True
+ for line in stdout.splitlines():
+ if line.startswith('*'):
+ (stdout, _) = bb.process.run('git rev-parse devtool-base', cwd=srctree)
+ initial_rev = stdout.rstrip()
+ if not initial_rev:
+ # Otherwise, just grab the head revision
+ (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree)
+ initial_rev = stdout.rstrip()
+ return initial_rev, was_devtool
+
def modify(args, config, basepath, workspace):
"""Entry point for the devtool 'modify' subcommand"""
import bb
@@ -841,33 +893,40 @@ def modify(args, config, basepath, workspace):
args.no_extract = True
torev = 'HEAD'
+ extradirs = OrderedDict()
+ srctreetop = None
+ extra_revs = {}
if not args.no_extract:
ret = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides)
initial_rev = ret.initial_rev
+ extradirs = ret.extradirs
+ if extradirs and not oe.utils.is_path_under(extradirs.keys(), ret.srcsubdir):
+ # Multiple source trees, we need to be inside the main one
+ srctreetop = srctree
+ srctree = os.path.join(srctree, ret.srcsubdir)
logger.info('Source tree extracted to %s' % srctree)
check_commits = True
else:
+ devtoolfile = os.path.join(srctree, '.devtool')
+ if os.path.exists(devtoolfile):
+ with open(devtoolfile, 'r') as f:
+ cfg = json.load(f, object_pairs_hook=OrderedDict)
+ srcsubdir = cfg.get('srcsubdir', '')
+ if srcsubdir:
+ srctreetop = srctree
+ srctree = os.path.abspath(os.path.join(srctree, srcsubdir))
+ extradirs.update(cfg.get('extradirs', {}))
+ else:
+ # Find any extra source trees
+ extradirpaths = find_git_repos(srctree)
+ for extradirpath in extradirpaths:
+ extradirs[os.path.relpath(extradirpath, srctree)], _ = _get_initial_rev(extradirpath)
+
if os.path.exists(os.path.join(srctree, '.git')):
- # Check if it's a tree previously extracted by us. This is done
- # by ensuring that devtool-base and args.branch (devtool) exist.
- # The check_commits logic will cause an exception if either one
- # of these doesn't exist
- try:
- (stdout, _) = bb.process.run('git branch --contains devtool-base', cwd=srctree)
- bb.process.run('git rev-parse %s' % args.branch, cwd=srctree)
- except bb.process.ExecutionError:
- stdout = ''
- if stdout:
- check_commits = True
+ # Check if it's a tree previously extracted by us
+ initial_rev, check_commits = _get_initial_rev(srctree)
+ if check_commits:
torev = 'devtool-patched'
- for line in stdout.splitlines():
- if line.startswith('*'):
- (stdout, _) = bb.process.run('git rev-parse devtool-base', cwd=srctree)
- initial_rev = stdout.rstrip()
- if not initial_rev:
- # Otherwise, just grab the head revision
- (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree)
- initial_rev = stdout.rstrip()
if initial_rev:
# Get list of commits since this revision
@@ -901,6 +960,23 @@ def modify(args, config, basepath, workspace):
# Need to grab this here in case the source is within a subdirectory
srctreebase = srctree
+ if extradirs:
+ extradirentries = []
+ for extradir, extrarev in extradirs.items():
+ extradirentry = OrderedDict()
+ if srctreetop:
+ # Several source trees at the top level
+ extradirentry['path'] = os.path.join(srctreetop, extradir)
+ else:
+ # All of the extra source trees are under the main one, so chop off
+ # the top level subdirectory
+ extradirentry['path'] = os.path.join(srctree, os.sep.join(extradir.split(os.sep)[1:]))
+ extradirentry['initial_rev'] = extrarev
+ # Get list of commits since this revision
+ (stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' % extrarev, cwd=extradirentry['path'])
+ extradirentry['commits'] = stdout.split()
+ extradirentries.append(extradirentry)
+
# Check that recipe isn't using a shared workdir
s = os.path.abspath(rd.getVar('S'))
workdir = os.path.abspath(rd.getVar('WORKDIR'))
@@ -941,6 +1017,23 @@ def modify(args, config, basepath, workspace):
' ln -sfT ${B}/.config ${S}/.config.new\n'
' fi\n'
'}\n')
+
+ if extradirs:
+ f.write('\n# srctreetop: %s\n' % srctreetop)
+ extradir_json = json.dumps(extradirentries, indent=' ')
+ prefix = '# extradirs:'
+ for line in extradir_json.splitlines():
+ f.write('%s %s\n' % (prefix, line))
+ prefix = '#'
+ f.write('\ndo_unpack[postfuncs] += "devtool_symlink_srctrees"\n')
+ f.write('devtool_symlink_srctrees () {\n')
+ # We could have done a loop in the function itself, but given there's
+ # usually only going to be a small number, there's not much point
+ for extradiritem in extradirentries:
+ if not oe.utils.is_path_under(extradiritem['path'], srctree):
+ f.write(' ln -sf %s ${WORKDIR}/\n' % extradiritem['path'])
+ f.write('}\n')
+
if initial_rev:
f.write('\n# initial_rev: %s\n' % initial_rev)
for commit in commits:
@@ -1155,7 +1248,7 @@ def rename(args, config, basepath, workspace):
return 0
-def _get_patchset_revs(srctree, recipe_path, initial_rev=None, force_patch_refresh=False):
+def _get_patchset_revs(recipename, workspace, srctree, initial_rev=None, force_patch_refresh=False):
"""Get initial and update rev of a recipe. These are the start point of the
whole patchset and start point for the patches to be re-generated/updated.
"""
@@ -1166,10 +1259,15 @@ def _get_patchset_revs(srctree, recipe_path, initial_rev=None, force_patch_refre
cwd=srctree)
branchname = stdout.rstrip()
+ append = workspace[recipename]['bbappend']
+ if not os.path.exists(append):
+ raise DevtoolError('unable to find workspace bbappend for recipe %s' %
+ recipename)
+
# Parse initial rev from recipe if not specified
commits = []
patches = []
- with open(recipe_path, 'r') as f:
+ with open(append, 'r') as f:
for line in f:
if line.startswith('# initial_rev:'):
if not initial_rev:
@@ -1179,6 +1277,15 @@ def _get_patchset_revs(srctree, recipe_path, initial_rev=None, force_patch_refre
elif line.startswith('# patches_%s:' % branchname):
patches = line.split(':')[-1].strip().split(',')
+ if srctree != workspace[recipename]['srctree']:
+ # Extra dir
+ for extradirentry in workspace[recipename]['extradirs']:
+ if srctree == extradirentry['path']:
+ commits = extradirentry['commits']
+ break
+ else:
+ raise Exception('Failed to find extradir entry for %s' % srctree)
+
update_rev = initial_rev
changed_revs = None
if initial_rev:
@@ -1259,7 +1366,7 @@ def _remove_source_files(append, files, destpath, no_report_remove=False, dry_ru
raise
-def _export_patches(srctree, rd, start_rev, destdir, changed_revs=None):
+def _export_patches(srctree, rd, start_rev, destdir, changed_revs=None, origsrcdir=None):
"""Export patches from srctree to given location.
Returns three-tuple of dicts:
1. updated - patches that already exist in SRCURI
@@ -1274,9 +1381,21 @@ def _export_patches(srctree, rd, start_rev, destdir, changed_revs=None):
added = OrderedDict()
seqpatch_re = re.compile('^([0-9]{4}-)?(.+)')
+ patches = oe.recipeutils.get_recipe_patches(rd)
+ if origsrcdir:
+ if not origsrcdir.endswith(os.sep):
+ origsrcdir = origsrcdir + os.sep
+ s = rd.getVar('S')
+ for patch, parms in list(patches.items()):
+ patchdir = os.path.abspath(os.path.join(s, parms.get('patchdir', '.')))
+ if not patchdir.endswith(os.sep):
+ patchdir = patchdir + os.sep
+ if not patchdir.startswith(origsrcdir):
+ del patches[patch]
+
existing_patches = dict((os.path.basename(path), path) for path in
- oe.recipeutils.get_recipe_patches(rd))
- logger.debug('Existing patches: %s' % existing_patches)
+ patches)
+ logger.debug('Existing patches (for %s): %s' % (srctree, existing_patches))
# Generate patches from Git, exclude local files directory
patch_pathspec = _git_exclude_path(srctree, 'oe-local-files')
@@ -1483,6 +1602,11 @@ def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wi
recipedir = os.path.basename(recipefile)
logger.info('Updating SRCREV in recipe %s%s' % (recipedir, dry_run_suffix))
+ # Work out what the extracted path would have been (so we can use it
+ # to filter patches when there are extra source directories)
+ s = rd.getVar('S')
+ origsrcdir = os.path.abspath(os.path.join(s, os.path.relpath(srctree, workspace[recipename]['srctree'])))
+
# Get HEAD revision
try:
stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree)
@@ -1511,7 +1635,8 @@ def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wi
patches_dir = tempfile.mkdtemp(dir=tempdir)
old_srcrev = rd.getVar('SRCREV') or ''
upd_p, new_p, del_p = _export_patches(srctree, rd, old_srcrev,
- patches_dir)
+ patches_dir,
+ origsrcdir=origsrcdir)
logger.debug('Patches: update %s, new %s, delete %s' % (dict(upd_p), dict(new_p), dict(del_p)))
# Remove deleted local files and "overlapping" patches
@@ -1557,6 +1682,10 @@ def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wi
if update_srcuri:
patchfields['SRC_URI'] = ' '.join(srcuri)
ret = oe.recipeutils.patch_recipe(rd, recipefile, patchfields, redirect_output=dry_run_outdir)
+ # Set the values into the datastore for the benefit of the next
+ # call (if extradirs are set)
+ for var, value in patchfields.items():
+ rd.setVar(var, value)
finally:
shutil.rmtree(tempdir)
if not 'git://' in orig_src_uri:
@@ -1574,12 +1703,8 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
recipefile = rd.getVar('FILE')
recipedir = os.path.dirname(recipefile)
- append = workspace[recipename]['bbappend']
- if not os.path.exists(append):
- raise DevtoolError('unable to find workspace bbappend for recipe %s' %
- recipename)
- initial_rev, update_rev, changed_revs, filter_patches = _get_patchset_revs(srctree, append, initial_rev, force_patch_refresh)
+ initial_rev, update_rev, changed_revs, filter_patches = _get_patchset_revs(recipename, workspace, srctree, initial_rev, force_patch_refresh)
if not initial_rev:
raise DevtoolError('Unable to find initial revision - please specify '
'it with --initial-rev')
@@ -1589,6 +1714,17 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
if not dl_dir.endswith('/'):
dl_dir += '/'
+ # Work out what the extracted path would have been (so we can use it
+ # to filter patches when there are extra source directories)
+ s = rd.getVar('S')
+ origsrcdir = os.path.abspath(os.path.join(s, os.path.relpath(srctree, workspace[recipename]['srctree'])))
+ # Determine patchdir for any new patches
+ patchdir = os.path.relpath(origsrcdir, s)
+ if patchdir != '.':
+ patchsuffix = ';patchdir=%s' % patchdir
+ else:
+ patchsuffix = ''
+
dry_run_suffix = ' (dry-run)' if dry_run_outdir else ''
tempdir = tempfile.mkdtemp(prefix='devtool')
@@ -1607,14 +1743,16 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
# Get all patches from source tree and check if any should be removed
all_patches_dir = tempfile.mkdtemp(dir=tempdir)
_, _, del_p = _export_patches(srctree, rd, initial_rev,
- all_patches_dir)
+ all_patches_dir,
+ origsrcdir=origsrcdir)
# Remove deleted local files and patches
remove_files = list(del_f.values()) + list(del_p.values())
# Get updated patches from source tree
patches_dir = tempfile.mkdtemp(dir=tempdir)
upd_p, new_p, _ = _export_patches(srctree, rd, update_rev,
- patches_dir, changed_revs)
+ patches_dir, changed_revs,
+ origsrcdir=origsrcdir)
logger.debug('Pre-filtering: update: %s, new: %s' % (dict(upd_p), dict(new_p)))
if filter_patches:
new_p = OrderedDict()
@@ -1644,7 +1782,7 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
removevalues=removevalues,
redirect_output=dry_run_outdir)
else:
- logger.info('No patches or local source files needed updating')
+ return False, None, []
else:
# Update existing files
files_dir = _determine_files_dir(rd)
@@ -1666,7 +1804,7 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
# replace the entry in SRC_URI with our local version
logger.info('Replacing remote patch %s with updated local version' % basepath)
path = os.path.join(files_dir, basepath)
- _replace_srcuri_entry(srcuri, basepath, 'file://%s' % basepath)
+ _replace_srcuri_entry(srcuri, basepath, 'file://%s%s' % (basepath, patchsuffix))
updaterecipe = True
else:
logger.info('Updating patch %s%s' % (basepath, dry_run_suffix))
@@ -1688,7 +1826,7 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
os.path.join(files_dir, basepath),
dry_run_outdir=dry_run_outdir,
base_outdir=recipedir)
- srcuri.append('file://%s' % basepath)
+ srcuri.append('file://%s%s' % (basepath, patchsuffix))
updaterecipe = True
# Update recipe, if needed
if _remove_file_entries(srcuri, remove_files)[0]:
@@ -1699,9 +1837,11 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
ret = oe.recipeutils.patch_recipe(rd, recipefile,
{'SRC_URI': ' '.join(srcuri)},
redirect_output=dry_run_outdir)
+ # Set the value into the datastore for the benefit of the next
+ # call (if extradirs are set)
+ rd.setVar('SRC_URI', ' '.join(srcuri))
elif not updatefiles:
# Neither patches nor recipe were updated
- logger.info('No patches or files need updating')
return False, None, []
finally:
shutil.rmtree(tempdir)
@@ -1732,70 +1872,84 @@ def _guess_recipe_update_mode(srctree, rdata):
return 'patch'
def _update_recipe(recipename, workspace, rd, mode, appendlayerdir, wildcard_version, no_remove, initial_rev, no_report_remove=False, dry_run_outdir=None, no_overrides=False, force_patch_refresh=False):
- srctree = workspace[recipename]['srctree']
+ main_srctree = workspace[recipename]['srctree']
if mode == 'auto':
- mode = _guess_recipe_update_mode(srctree, rd)
+ mode = _guess_recipe_update_mode(main_srctree, rd)
- override_branches = []
- mainbranch = None
- startbranch = None
- if not no_overrides:
- stdout, _ = bb.process.run('git branch', cwd=srctree)
- other_branches = []
- for line in stdout.splitlines():
- branchname = line[2:]
- if line.startswith('* '):
- startbranch = branchname
- if branchname.startswith(override_branch_prefix):
- override_branches.append(branchname)
- else:
- other_branches.append(branchname)
+ appendfile = workspace[recipename]['bbappend']
+ srctrees = OrderedDict([(main_srctree, initial_rev)])
+ for extradirentry in workspace[recipename]['extradirs']:
+ srctrees[extradirentry['path']] = extradirentry['initial_rev']
+
+ logger.debug('Considering source trees: %s' % srctrees)
+ if mode == 'auto':
+ mode = _guess_recipe_update_mode(main_srctree, rd)
+
+ crd = bb.data.createCopy(rd)
+ for srctree, src_initial_rev in srctrees.items():
+ override_branches = []
+ mainbranch = None
+ startbranch = None
+ if not no_overrides:
+ stdout, _ = bb.process.run('git branch', cwd=srctree)
+ other_branches = []
+ for line in stdout.splitlines():
+ branchname = line[2:]
+ if line.startswith('* '):
+ startbranch = branchname
+ if branchname.startswith(override_branch_prefix):
+ override_branches.append(branchname)
+ else:
+ other_branches.append(branchname)
+
+ if override_branches:
+ logger.debug('_update_recipe: override branches: %s' % override_branches)
+ logger.debug('_update_recipe: other branches: %s' % other_branches)
+ if startbranch.startswith(override_branch_prefix):
+ if len(other_branches) == 1:
+ mainbranch = other_branches[1]
+ else:
+ raise DevtoolError('Unable to determine main branch - please check out the main branch in source tree first')
+ else:
+ mainbranch = startbranch
+
+ checkedout = None
+ anyupdated = False
+ appendfile = None
+ allremoved = []
if override_branches:
- logger.debug('_update_recipe: override branches: %s' % override_branches)
- logger.debug('_update_recipe: other branches: %s' % other_branches)
- if startbranch.startswith(override_branch_prefix):
- if len(other_branches) == 1:
- mainbranch = other_branches[1]
+ logger.info('Handling main branch (%s)...' % mainbranch)
+ if startbranch != mainbranch:
+ bb.process.run('git checkout %s' % mainbranch, cwd=srctree)
+ checkedout = mainbranch
+ try:
+ branchlist = [mainbranch] + override_branches
+ for branch in branchlist:
+ if branch != mainbranch:
+ logger.info('Handling branch %s...' % branch)
+ override = branch[len(override_branch_prefix):]
+ crd.setVar('OVERRIDES', '%s:%s' % rd.getVar('OVERRIDES') + override)
+ bb.process.run('git checkout %s' % branch, cwd=srctree)
+ checkedout = branch
+
+ if mode == 'srcrev':
+ updated, appendf, removed = _update_recipe_srcrev(recipename, workspace, srctree, crd, appendlayerdir, wildcard_version, no_remove, no_report_remove, dry_run_outdir)
+ elif mode == 'patch':
+ updated, appendf, removed = _update_recipe_patch(recipename, workspace, srctree, crd, appendlayerdir, wildcard_version, no_remove, no_report_remove, src_initial_rev, dry_run_outdir, force_patch_refresh)
else:
- raise DevtoolError('Unable to determine main branch - please check out the main branch in source tree first')
- else:
- mainbranch = startbranch
+ raise DevtoolError('update_recipe: invalid mode %s' % mode)
+ if updated:
+ anyupdated = True
+ if appendf:
+ appendfile = appendf
+ allremoved.extend(removed)
+ finally:
+ if startbranch and checkedout != startbranch:
+ bb.process.run('git checkout %s' % startbranch, cwd=srctree)
- checkedout = None
- anyupdated = False
- appendfile = None
- allremoved = []
- if override_branches:
- logger.info('Handling main branch (%s)...' % mainbranch)
- if startbranch != mainbranch:
- bb.process.run('git checkout %s' % mainbranch, cwd=srctree)
- checkedout = mainbranch
- try:
- branchlist = [mainbranch] + override_branches
- for branch in branchlist:
- crd = bb.data.createCopy(rd)
- if branch != mainbranch:
- logger.info('Handling branch %s...' % branch)
- override = branch[len(override_branch_prefix):]
- crd.appendVar('OVERRIDES', ':%s' % override)
- bb.process.run('git checkout %s' % branch, cwd=srctree)
- checkedout = branch
-
- if mode == 'srcrev':
- updated, appendf, removed = _update_recipe_srcrev(recipename, workspace, srctree, crd, appendlayerdir, wildcard_version, no_remove, no_report_remove, dry_run_outdir)
- elif mode == 'patch':
- updated, appendf, removed = _update_recipe_patch(recipename, workspace, srctree, crd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev, dry_run_outdir, force_patch_refresh)
- else:
- raise DevtoolError('update_recipe: invalid mode %s' % mode)
- if updated:
- anyupdated = True
- if appendf:
- appendfile = appendf
- allremoved.extend(removed)
- finally:
- if startbranch and checkedout != startbranch:
- bb.process.run('git checkout %s' % startbranch, cwd=srctree)
+ if not anyupdated:
+ logger.info('No patches or files need updating')
return anyupdated, appendfile, allremoved
@@ -1925,21 +2079,21 @@ def _reset(recipes, no_clean, remove_work, config, basepath, workspace):
# We don't automatically create this dir next to appends, but the user can
preservedir(os.path.join(config.workspace_path, 'appends', pn))
- srctreebase = workspace[pn]['srctreebase']
- if os.path.isdir(srctreebase):
- if os.listdir(srctreebase):
+ srctreetop = workspace[pn]['srctreetop']
+ if os.path.isdir(srctreetop):
+ if os.listdir(srctreetop):
if remove_work:
logger.info('-r argument used on %s, removing source tree.'
' You will lose any unsaved work' %pn)
- shutil.rmtree(srctreebase)
+ shutil.rmtree(srctreetop)
else:
# We don't want to risk wiping out any work in progress
logger.info('Leaving source tree %s as-is; if you no '
'longer need it then please delete it manually'
- % srctreebase)
+ % srctreetop)
else:
# This is unlikely, but if it's empty we can just remove it
- os.rmdir(srctreebase)
+ os.rmdir(srctreetop)
clean_preferred_provider(pn, config.workspace_path)
@@ -2008,12 +2162,17 @@ def finish(args, config, basepath, workspace):
srctree = workspace[args.recipename]['srctree']
check_git_repo_op(srctree, [corebasedir])
- dirty = check_git_repo_dirty(srctree)
- if dirty:
- if args.force:
- logger.warning('Source tree is not clean, continuing as requested by -f/--force')
- else:
- raise DevtoolError('Source tree is not clean:\n\n%s\nEnsure you have committed your changes or use -f/--force if you are sure there\'s nothing that needs to be committed' % dirty)
+ srctrees = [srctree]
+ for extradirentry in workspace[args.recipename]['extradirs']:
+ srctrees.append(extradirentry['path'])
+ for pth in srctrees:
+ dirty = check_git_repo_dirty(pth)
+ if dirty:
+ if args.force:
+ logger.warning('Source tree %s is not clean, continuing as requested by -f/--force' % pth)
+ else:
+ raise DevtoolError('Source tree %s is not clean:\n\n%s\nEnsure you have committed your changes or use -f/--force if you are sure there\'s nothing that needs to be committed' % (pth, dirty))
+
no_clean = args.no_clean
remove_work=args.remove_work
diff --git a/scripts/lib/devtool/upgrade.py b/scripts/lib/devtool/upgrade.py
index 451fd19a4cf..94cba00a956 100644
--- a/scripts/lib/devtool/upgrade.py
+++ b/scripts/lib/devtool/upgrade.py
@@ -552,6 +552,7 @@ def upgrade(args, config, basepath, workspace):
current_ret = standard._extract_source(srctree, False, 'devtool-orig', False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides)
old_licenses = _extract_licenses(srctree, (rd.getVar('LIC_FILES_CHKSUM') or ""))
logger.info('Extracting upgraded version source...')
+ srctree = os.path.join(srctree, current_ret.srcsubdir)
rev2, md5, sha256, srcbranch, srcsubdir2 = _extract_new_source(args.version, srctree, args.no_patch,
args.srcrev, args.srcbranch, args.branch, args.keep_temp,
tinfoil, rd)
generated by cgit 1.2.3-korg (git 2.39.0) at 2024-05-01 07:53:30 +0000