diff options
Diffstat (limited to 'bitbake/lib/bb/build.py')
| -rw-r--r-- | bitbake/lib/bb/build.py | 179 |
1 files changed, 71 insertions, 108 deletions
diff --git a/bitbake/lib/bb/build.py b/bitbake/lib/bb/build.py index 65b7fc000d..44d08f5c55 100644 --- a/bitbake/lib/bb/build.py +++ b/bitbake/lib/bb/build.py @@ -20,10 +20,12 @@ import itertools import time import re import stat +import datetime import bb import bb.msg import bb.process import bb.progress +from io import StringIO from bb import data, event, utils bblogger = logging.getLogger('BitBake') @@ -176,7 +178,9 @@ class StdoutNoopContextManager: @property def name(self): - return sys.stdout.name + if "name" in dir(sys.stdout): + return sys.stdout.name + return "<mem>" def exec_func(func, d, dirs = None): @@ -295,9 +299,21 @@ def exec_func_python(func, d, runfile, cwd=None): lineno = int(d.getVarFlag(func, "lineno", False)) bb.methodpool.insert_method(func, text, fn, lineno - 1) + if verboseStdoutLogging: + sys.stdout.flush() + sys.stderr.flush() + currout = sys.stdout + currerr = sys.stderr + sys.stderr = sys.stdout = execio = StringIO() comp = utils.better_compile(code, func, "exec_func_python() autogenerated") utils.better_exec(comp, {"d": d}, code, "exec_func_python() autogenerated") finally: + if verboseStdoutLogging: + execio.flush() + logger.plain("%s" % execio.getvalue()) + sys.stdout = currout + sys.stderr = currerr + execio.close() # We want any stdout/stderr to be printed before any other log messages to make debugging # more accurate. In some cases we seem to lose stdout/stderr entirely in logging tests without this. sys.stdout.flush() @@ -440,7 +456,11 @@ exit $ret if fakerootcmd: cmd = [fakerootcmd, runfile] - if verboseStdoutLogging: + # We only want to output to logger via LogTee if stdout is sys.__stdout__ (which will either + # be real stdout or subprocess PIPE or similar). In other cases we are being run "recursively", + # ie. inside another function, in which case stdout is already being captured so we don't + # want to Tee here as output would be printed twice, and out of order. + if verboseStdoutLogging and sys.stdout == sys.__stdout__: logfile = LogTee(logger, StdoutNoopContextManager()) else: logfile = StdoutNoopContextManager() @@ -571,7 +591,6 @@ def _task_data(fn, task, d): localdata.setVar('BB_FILENAME', fn) localdata.setVar('OVERRIDES', 'task-%s:%s' % (task[3:].replace('_', '-'), d.getVar('OVERRIDES', False))) - localdata.finalize() bb.data.expandKeys(localdata) return localdata @@ -582,7 +601,7 @@ def _exec_task(fn, task, d, quieterr): running it with its own local metadata, and with some useful variables set. """ if not d.getVarFlag(task, 'task', False): - event.fire(TaskInvalid(task, d), d) + event.fire(TaskInvalid(task, fn, d), d) logger.error("No such task: %s" % task) return 1 @@ -618,7 +637,8 @@ def _exec_task(fn, task, d, quieterr): logorder = os.path.join(tempdir, 'log.task_order') try: with open(logorder, 'a') as logorderfile: - logorderfile.write('{0} ({1}): {2}\n'.format(task, os.getpid(), logbase)) + timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S.%f") + logorderfile.write('{0} {1} ({2}): {3}\n'.format(timestamp, task, os.getpid(), logbase)) except OSError: logger.exception("Opening log file '%s'", logorder) pass @@ -771,44 +791,7 @@ def exec_task(fn, task, d, profile = False): event.fire(failedevent, d) return 1 -def stamp_internal(taskname, d, file_name, baseonly=False, noextra=False): - """ - Internal stamp helper function - Makes sure the stamp directory exists - Returns the stamp path+filename - - In the bitbake core, d can be a CacheData and file_name will be set. - When called in task context, d will be a data store, file_name will not be set - """ - taskflagname = taskname - if taskname.endswith("_setscene") and taskname != "do_setscene": - taskflagname = taskname.replace("_setscene", "") - - if file_name: - stamp = d.stamp[file_name] - extrainfo = d.stamp_extrainfo[file_name].get(taskflagname) or "" - else: - stamp = d.getVar('STAMP') - file_name = d.getVar('BB_FILENAME') - extrainfo = d.getVarFlag(taskflagname, 'stamp-extra-info') or "" - - if baseonly: - return stamp - if noextra: - extrainfo = "" - - if not stamp: - return - - stamp = bb.parse.siggen.stampfile(stamp, file_name, taskname, extrainfo) - - stampdir = os.path.dirname(stamp) - if cached_mtime_noerror(stampdir) == 0: - bb.utils.mkdirhier(stampdir) - - return stamp - -def stamp_cleanmask_internal(taskname, d, file_name): +def _get_cleanmask(taskname, mcfn): """ Internal stamp helper function to generate stamp cleaning mask Returns the stamp path+filename @@ -816,31 +799,14 @@ def stamp_cleanmask_internal(taskname, d, file_name): In the bitbake core, d can be a CacheData and file_name will be set. When called in task context, d will be a data store, file_name will not be set """ - taskflagname = taskname - if taskname.endswith("_setscene") and taskname != "do_setscene": - taskflagname = taskname.replace("_setscene", "") - - if file_name: - stamp = d.stampclean[file_name] - extrainfo = d.stamp_extrainfo[file_name].get(taskflagname) or "" - else: - stamp = d.getVar('STAMPCLEAN') - file_name = d.getVar('BB_FILENAME') - extrainfo = d.getVarFlag(taskflagname, 'stamp-extra-info') or "" - - if not stamp: - return [] - - cleanmask = bb.parse.siggen.stampcleanmask(stamp, file_name, taskname, extrainfo) - - return [cleanmask, cleanmask.replace(taskflagname, taskflagname + "_setscene")] - -def make_stamp(task, d, file_name = None): - """ - Creates/updates a stamp for a given task - (d can be a data dict or dataCache) - """ - cleanmask = stamp_cleanmask_internal(task, d, file_name) + cleanmask = bb.parse.siggen.stampcleanmask_mcfn(taskname, mcfn) + taskflagname = taskname.replace("_setscene", "") + if cleanmask: + return [cleanmask, cleanmask.replace(taskflagname, taskflagname + "_setscene")] + return [] + +def clean_stamp_mcfn(task, mcfn): + cleanmask = _get_cleanmask(task, mcfn) for mask in cleanmask: for name in glob.glob(mask): # Preserve sigdata files in the stamps directory @@ -851,24 +817,45 @@ def make_stamp(task, d, file_name = None): continue os.unlink(name) - stamp = stamp_internal(task, d, file_name) +def clean_stamp(task, d): + mcfn = d.getVar('BB_FILENAME') + clean_stamp_mcfn(task, mcfn) + +def make_stamp_mcfn(task, mcfn): + + basestamp = bb.parse.siggen.stampfile_mcfn(task, mcfn) + + stampdir = os.path.dirname(basestamp) + if cached_mtime_noerror(stampdir) == 0: + bb.utils.mkdirhier(stampdir) + + clean_stamp_mcfn(task, mcfn) + # Remove the file and recreate to force timestamp # change on broken NFS filesystems - if stamp: - bb.utils.remove(stamp) - open(stamp, "w").close() + if basestamp: + bb.utils.remove(basestamp) + open(basestamp, "w").close() + +def make_stamp(task, d): + """ + Creates/updates a stamp for a given task + """ + mcfn = d.getVar('BB_FILENAME') + + make_stamp_mcfn(task, mcfn) # If we're in task context, write out a signature file for each task # as it completes - if not task.endswith("_setscene") and task != "do_setscene" and not file_name: - stampbase = stamp_internal(task, d, None, True) - file_name = d.getVar('BB_FILENAME') - bb.parse.siggen.dump_sigtask(file_name, task, stampbase, True) - -def find_stale_stamps(task, d, file_name=None): - current = stamp_internal(task, d, file_name) - current2 = stamp_internal(task + "_setscene", d, file_name) - cleanmask = stamp_cleanmask_internal(task, d, file_name) + if not task.endswith("_setscene"): + stampbase = bb.parse.siggen.stampfile_base(mcfn) + bb.parse.siggen.dump_sigtask(mcfn, task, stampbase, True) + + +def find_stale_stamps(task, mcfn): + current = bb.parse.siggen.stampfile_mcfn(task, mcfn) + current2 = bb.parse.siggen.stampfile_mcfn(task + "_setscene", mcfn) + cleanmask = _get_cleanmask(task, mcfn) found = [] for mask in cleanmask: for name in glob.glob(mask): @@ -882,38 +869,14 @@ def find_stale_stamps(task, d, file_name=None): found.append(name) return found -def del_stamp(task, d, file_name = None): - """ - Removes a stamp for a given task - (d can be a data dict or dataCache) - """ - stamp = stamp_internal(task, d, file_name) - bb.utils.remove(stamp) - -def write_taint(task, d, file_name = None): +def write_taint(task, d): """ Creates a "taint" file which will force the specified task and its dependents to be re-run the next time by influencing the value of its taskhash. - (d can be a data dict or dataCache) - """ - import uuid - if file_name: - taintfn = d.stamp[file_name] + '.' + task + '.taint' - else: - taintfn = d.getVar('STAMP') + '.' + task + '.taint' - bb.utils.mkdirhier(os.path.dirname(taintfn)) - # The specific content of the taint file is not really important, - # we just need it to be random, so a random UUID is used - with open(taintfn, 'w') as taintf: - taintf.write(str(uuid.uuid4())) - -def stampfile(taskname, d, file_name = None, noextra=False): - """ - Return the stamp for a given task - (d can be a data dict or dataCache) """ - return stamp_internal(taskname, d, file_name, noextra=noextra) + mcfn = d.getVar('BB_FILENAME') + bb.parse.siggen.invalidate_task(task, mcfn) def add_tasks(tasklist, d): task_deps = d.getVar('_task_deps', False) |