summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/main.py
blob: 29e391162e69c674796b3d29b4491915e7f46a82 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
#!/usr/bin/env python
# ex:ts=4:sw=4:sts=4:et
# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
#
# Copyright (C) 2003, 2004  Chris Larson
# Copyright (C) 2003, 2004  Phil Blundell
# Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
# Copyright (C) 2005        Holger Hans Peter Freyther
# Copyright (C) 2005        ROAD GmbH
# Copyright (C) 2006        Richard Purdie
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

import os
import sys
import logging
import optparse
import warnings
import fcntl

import bb
from bb import event
import bb.msg
from bb import cooker
from bb import ui
from bb import server
from bb import cookerdata

logger = logging.getLogger("BitBake")

class BBMainException(Exception):
    pass

def present_options(optionlist):
    if len(optionlist) > 1:
        return ' or '.join([', '.join(optionlist[:-1]), optionlist[-1]])
    else:
        return optionlist[0]

class BitbakeHelpFormatter(optparse.IndentedHelpFormatter):
    def format_option(self, option):
        # We need to do this here rather than in the text we supply to
        # add_option() because we don't want to call list_extension_modules()
        # on every execution (since it imports all of the modules)
        # Note also that we modify option.help rather than the returned text
        # - this is so that we don't have to re-format the text ourselves
        if option.dest == 'ui':
            valid_uis = list_extension_modules(bb.ui, 'main')
            option.help = option.help.replace('@CHOICES@', present_options(valid_uis))
        elif option.dest == 'servertype':
            valid_server_types = list_extension_modules(bb.server, 'BitBakeServer')
            option.help = option.help.replace('@CHOICES@', present_options(valid_server_types))

        return optparse.IndentedHelpFormatter.format_option(self, option)

def list_extension_modules(pkg, checkattr):
    """
    Lists extension modules in a specific Python package
    (e.g. UIs, servers). NOTE: Calling this function will import all of the
    submodules of the specified module in order to check for the specified
    attribute; this can have unusual side-effects. As a result, this should
    only be called when displaying help text or error messages.
    Parameters:
        pkg: previously imported Python package to list
        checkattr: attribute to look for in module to determine if it's valid
            as the type of extension you are looking for
    """
    import pkgutil
    pkgdir = os.path.dirname(pkg.__file__)

    modules = []
    for _, modulename, _ in pkgutil.iter_modules([pkgdir]):
        if os.path.isdir(os.path.join(pkgdir, modulename)):
            # ignore directories
            continue
        try:
            module = __import__(pkg.__name__, fromlist=[modulename])
        except:
            # If we can't import it, it's not valid
            continue
        module_if = getattr(module, modulename)
        if getattr(module_if, 'hidden_extension', False):
            continue
        if not checkattr or hasattr(module_if, checkattr):
            modules.append(modulename)
    return modules

def import_extension_module(pkg, modulename, checkattr):
    try:
        # Dynamically load the UI based on the ui name. Although we
        # suggest a fixed set this allows you to have flexibility in which
        # ones are available.
        module = __import__(pkg.__name__, fromlist=[modulename])
        return getattr(module, modulename)
    except AttributeError:
        modules = present_options(list_extension_modules(pkg, checkattr))
        raise BBMainException('FATAL: Unable to import extension module "%s" from %s. '
                              'Valid extension modules: %s' % (modulename, pkg.__name__, modules))

# Display bitbake/OE warnings via the BitBake.Warnings logger, ignoring others"""
warnlog = logging.getLogger("BitBake.Warnings")
_warnings_showwarning = warnings.showwarning
def _showwarning(message, category, filename, lineno, file=None, line=None):
    if file is not None:
        if _warnings_showwarning is not None:
            _warnings_showwarning(message, category, filename, lineno, file, line)
    else:
        s = warnings.formatwarning(message, category, filename, lineno)
        warnlog.warning(s)

warnings.showwarning = _showwarning
warnings.filterwarnings("ignore")
warnings.filterwarnings("default", module="(<string>$|(oe|bb)\.)")
warnings.filterwarnings("ignore", category=PendingDeprecationWarning)
warnings.filterwarnings("ignore", category=ImportWarning)
warnings.filterwarnings("ignore", category=DeprecationWarning, module="<string>$")
warnings.filterwarnings("ignore", message="With-statements now directly support multiple context managers")

class BitBakeConfigParameters(cookerdata.ConfigParameters):

    def parseCommandLine(self, argv=sys.argv):
        parser = optparse.OptionParser(
            formatter=BitbakeHelpFormatter(),
            version="BitBake Build Tool Core version %s" % bb.__version__,
            usage="""%prog [options] [recipename/target recipe:do_task ...]

    Executes the specified task (default is 'build') for a given set of target recipes (.bb files).
    It is assumed there is a conf/bblayers.conf available in cwd or in BBPATH which
    will provide the layer, BBFILES and other configuration information.""")

        parser.add_option("-b", "--buildfile", action="store", dest="buildfile", default=None,
                          help="Execute tasks from a specific .bb recipe directly. WARNING: Does "
                               "not handle any dependencies from other recipes.")

        parser.add_option("-k", "--continue", action="store_false", dest="abort", default=True,
                          help="Continue as much as possible after an error. While the target that "
                               "failed and anything depending on it cannot be built, as much as "
                               "possible will be built before stopping.")

        parser.add_option("-a", "--tryaltconfigs", action="store_true",
                          dest="tryaltconfigs", default=False,
                          help="Continue with builds by trying to use alternative providers "
                               "where possible.")

        parser.add_option("-f", "--force", action="store_true", dest="force", default=False,
                          help="Force the specified targets/task to run (invalidating any "
                               "existing stamp file).")

        parser.add_option("-c", "--cmd", action="store", dest="cmd",
                          help="Specify the task to execute. The exact options available "
                               "depend on the metadata. Some examples might be 'compile'"
                               " or 'populate_sysroot' or 'listtasks' may give a list of "
                               "the tasks available.")

        parser.add_option("-C", "--clear-stamp", action="store", dest="invalidate_stamp",
                          help="Invalidate the stamp for the specified task such as 'compile' "
                               "and then run the default task for the specified target(s).")

        parser.add_option("-r", "--read", action="append", dest="prefile", default=[],
                          help="Read the specified file before bitbake.conf.")

        parser.add_option("-R", "--postread", action="append", dest="postfile", default=[],
                          help="Read the specified file after bitbake.conf.")

        parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False,
                          help="Enable tracing of shell tasks (with 'set -x'). "
                               "Also print bb.note(...) messages to stdout (in "
                               "addition to writing them to ${T}/log.do_<task>).")

        parser.add_option("-D", "--debug", action="count", dest="debug", default=0,
                          help="Increase the debug level. You can specify this "
                               "more than once. -D sets the debug level to 1, "
                               "where only bb.debug(1, ...) messages are printed "
                               "to stdout; -DD sets the debug level to 2, where "
                               "both bb.debug(1, ...) and bb.debug(2, ...) "
                               "messages are printed; etc. Without -D, no debug "
                               "messages are printed. Note that -D only affects "
                               "output to stdout. All debug messages are written "
                               "to ${T}/log.do_taskname, regardless of the debug "
                               "level.")

        parser.add_option("-q", "--quiet", action="count", dest="quiet", default=0,
                          help="Output less log message data to the terminal. You can specify this more than once.")

        parser.add_option("-n", "--dry-run", action="store_true", dest="dry_run", default=False,
                          help="Don't execute, just go through the motions.")

        parser.add_option("-S", "--dump-signatures", action="append", dest="dump_signatures",
                          default=[], metavar="SIGNATURE_HANDLER",
                          help="Dump out the signature construction information, with no task "
                               "execution. The SIGNATURE_HANDLER parameter is passed to the "
                               "handler. Two common values are none and printdiff but the handler "
                               "may define more/less. none means only dump the signature, printdiff"
                               " means compare the dumped signature with the cached one.")

        parser.add_option("-p", "--parse-only", action="store_true",
                          dest="parse_only", default=False,
                          help="Quit after parsing the BB recipes.")

        parser.add_option("-s", "--show-versions", action="store_true",
                          dest="show_versions", default=False,
                          help="Show current and preferred versions of all recipes.")

        parser.add_option("-e", "--environment", action="store_true",
                          dest="show_environment", default=False,
                          help="Show the global or per-recipe environment complete with information"
                               " about where variables were set/changed.")

        parser.add_option("-g", "--graphviz", action="store_true", dest="dot_graph", default=False,
                          help="Save dependency tree information for the specified "
                               "targets in the dot syntax.")

        parser.add_option("-I", "--ignore-deps", action="append",
                          dest="extra_assume_provided", default=[],
                          help="Assume these dependencies don't exist and are already provided "
                               "(equivalent to ASSUME_PROVIDED). Useful to make dependency "
                               "graphs more appealing")

        parser.add_option("-l", "--log-domains", action="append", dest="debug_domains", default=[],
                          help="Show debug logging for the specified logging domains")

        parser.add_option("-P", "--profile", action="store_true", dest="profile", default=False,
                          help="Profile the command and save reports.")

        # @CHOICES@ is substituted out by BitbakeHelpFormatter above
        parser.add_option("-u", "--ui", action="store", dest="ui",
                          default=os.environ.get('BITBAKE_UI', 'knotty'),
                          help="The user interface to use (@CHOICES@ - default %default).")

        # @CHOICES@ is substituted out by BitbakeHelpFormatter above
        parser.add_option("-t", "--servertype", action="store", dest="servertype",
                          default=["process", "xmlrpc"]["BBSERVER" in os.environ],
                          help="Choose which server type to use (@CHOICES@ - default %default).")

        parser.add_option("", "--token", action="store", dest="xmlrpctoken",
                          default=os.environ.get("BBTOKEN"),
                          help="Specify the connection token to be used when connecting "
                               "to a remote server.")

        parser.add_option("", "--revisions-changed", action="store_true",
                          dest="revisions_changed", default=False,
                          help="Set the exit code depending on whether upstream floating "
                               "revisions have changed or not.")

        parser.add_option("", "--server-only", action="store_true",
                          dest="server_only", default=False,
                          help="Run bitbake without a UI, only starting a server "
                               "(cooker) process.")

        parser.add_option("", "--foreground", action="store_true",
                          help="Run bitbake server in foreground.")

        parser.add_option("-B", "--bind", action="store", dest="bind", default=False,
                          help="The name/address for the bitbake server to bind to.")

        parser.add_option("-T", "--idle-timeout", type=int,
                          default=int(os.environ.get("BBTIMEOUT", "0")),
                          help="Set timeout to unload bitbake server due to inactivity")

        parser.add_option("", "--no-setscene", action="store_true",
                          dest="nosetscene", default=False,
                          help="Do not run any setscene tasks. sstate will be ignored and "
                               "everything needed, built.")

        parser.add_option("", "--setscene-only", action="store_true",
                          dest="setsceneonly", default=False,
                          help="Only run setscene tasks, don't run any real tasks.")

        parser.add_option("", "--remote-server", action="store", dest="remote_server",
                          default=os.environ.get("BBSERVER"),
                          help="Connect to the specified server.")

        parser.add_option("-m", "--kill-server", action="store_true",
                          dest="kill_server", default=False,
                          help="Terminate the remote server.")

        parser.add_option("", "--observe-only", action="store_true",
                          dest="observe_only", default=False,
                          help="Connect to a server as an observing-only client.")

        parser.add_option("", "--status-only", action="store_true",
                          dest="status_only", default=False,
                          help="Check the status of the remote bitbake server.")

        parser.add_option("-w", "--write-log", action="store", dest="writeeventlog",
                          default=os.environ.get("BBEVENTLOG"),
                          help="Writes the event log of the build to a bitbake event json file. "
                               "Use '' (empty string) to assign the name automatically.")

        parser.add_option("", "--runall", action="store", dest="runall",
                          help="Run the specified task for all build targets and their dependencies.")

        options, targets = parser.parse_args(argv)

        if options.quiet and options.verbose:
            parser.error("options --quiet and --verbose are mutually exclusive")

        if options.quiet and options.debug:
            parser.error("options --quiet and --debug are mutually exclusive")

        # use configuration files from environment variables
        if "BBPRECONF" in os.environ:
            options.prefile.append(os.environ["BBPRECONF"])

        if "BBPOSTCONF" in os.environ:
            options.postfile.append(os.environ["BBPOSTCONF"])

        # fill in proper log name if not supplied
        if options.writeeventlog is not None and len(options.writeeventlog) == 0:
            from datetime import datetime
            eventlog = "bitbake_eventlog_%s.json" % datetime.now().strftime("%Y%m%d%H%M%S")
            options.writeeventlog = eventlog

        # if BBSERVER says to autodetect, let's do that
        if options.remote_server:
            port = -1
            if options.remote_server != 'autostart':
                host, port = options.remote_server.split(":", 2)
                port = int(port)
            # use automatic port if port set to -1, means read it from
            # the bitbake.lock file; this is a bit tricky, but we always expect
            # to be in the base of the build directory if we need to have a
            # chance to start the server later, anyway
            if port == -1:
                lock_location = "./bitbake.lock"
                # we try to read the address at all times; if the server is not started,
                # we'll try to start it after the first connect fails, below
                try:
                    lf = open(lock_location, 'r')
                    remotedef = lf.readline()
                    [host, port] = remotedef.split(":")
                    port = int(port)
                    lf.close()
                    options.remote_server = remotedef
                except Exception as e:
                    if options.remote_server != 'autostart':
                        raise BBMainException("Failed to read bitbake.lock (%s), invalid port" % str(e))

        return options, targets[1:]


def start_server(servermodule, configParams, configuration, features):
    server = servermodule.BitBakeServer()
    single_use = not configParams.server_only and os.getenv('BBSERVER') != 'autostart'
    if configParams.bind:
        (host, port) = configParams.bind.split(':')
        server.initServer((host, int(port)), single_use=single_use,
                          idle_timeout=configParams.idle_timeout)
        configuration.interface = [server.serverImpl.host, server.serverImpl.port]
    else:
        server.initServer(single_use=single_use)
        configuration.interface = []

    try:
        configuration.setServerRegIdleCallback(server.getServerIdleCB())

        cooker = bb.cooker.BBCooker(configuration, features)

        server.addcooker(cooker)
        server.saveConnectionDetails()
    except Exception as e:
        while hasattr(server, "event_queue"):
            import queue
            try:
                event = server.event_queue.get(block=False)
            except (queue.Empty, IOError):
                break
            if isinstance(event, logging.LogRecord):
                logger.handle(event)
        raise
    if not configParams.foreground:
        server.detach()
        cooker.shutdown()
    cooker.lock.close()
    return server


def bitbake_main(configParams, configuration):

    # Python multiprocessing requires /dev/shm on Linux
    if sys.platform.startswith('linux') and not os.access('/dev/shm', os.W_OK | os.X_OK):
        raise BBMainException("FATAL: /dev/shm does not exist or is not writable")

    # Unbuffer stdout to avoid log truncation in the event
    # of an unorderly exit as well as to provide timely
    # updates to log files for use with tail
    try:
        if sys.stdout.name == '<stdout>':
            # Reopen with O_SYNC (unbuffered)
            fl = fcntl.fcntl(sys.stdout.fileno(), fcntl.F_GETFL)
            fl |= os.O_SYNC
            fcntl.fcntl(sys.stdout.fileno(), fcntl.F_SETFL, fl)
    except:
        pass

    configuration.setConfigParameters(configParams)

    if configParams.server_only:
        if configParams.servertype != "xmlrpc":
            raise BBMainException("FATAL: If '--server-only' is defined, we must set the "
                                  "servertype as 'xmlrpc'.\n")
        if not configParams.bind:
            raise BBMainException("FATAL: The '--server-only' option requires a name/address "
                                  "to bind to with the -B option.\n")
        else:
            try:
                #Checking that the port is a number
                int(configParams.bind.split(":")[1])
            except (ValueError,IndexError):
                raise BBMainException(
                        "FATAL: Malformed host:port bind parameter")
        if configParams.remote_server:
            raise BBMainException("FATAL: The '--server-only' option conflicts with %s.\n" %
                                  ("the BBSERVER environment variable" if "BBSERVER" in os.environ \
                                   else "the '--remote-server' option"))

    elif configParams.foreground:
        raise BBMainException("FATAL: The '--foreground' option can only be used "
                              "with --server-only.\n")

    if configParams.bind and configParams.servertype != "xmlrpc":
        raise BBMainException("FATAL: If '-B' or '--bind' is defined, we must "
                              "set the servertype as 'xmlrpc'.\n")

    if configParams.remote_server and configParams.servertype != "xmlrpc":
        raise BBMainException("FATAL: If '--remote-server' is defined, we must "
                              "set the servertype as 'xmlrpc'.\n")

    if configParams.observe_only and (not configParams.remote_server or configParams.bind):
        raise BBMainException("FATAL: '--observe-only' can only be used by UI clients "
                              "connecting to a server.\n")

    if configParams.kill_server and not configParams.remote_server:
        raise BBMainException("FATAL: '--kill-server' can only be used to "
                              "terminate a remote server")

    if "BBDEBUG" in os.environ:
        level = int(os.environ["BBDEBUG"])
        if level > configuration.debug:
            configuration.debug = level

    bb.msg.init_msgconfig(configParams.verbose, configuration.debug,
                          configuration.debug_domains)

    server, server_connection, ui_module = setup_bitbake(configParams, configuration)
    if server_connection is None and configParams.kill_server:
        return 0

    if not configParams.server_only:
        if configParams.status_only:
            server_connection.terminate()
            return 0

        try:
            return ui_module.main(server_connection.connection, server_connection.events,
                                  configParams)
        finally:
            bb.event.ui_queue = []
            server_connection.terminate()
    else:
        print("Bitbake server address: %s, server port: %s" % (server.serverImpl.host,
                                                               server.serverImpl.port))
        if configParams.foreground:
            server.serverImpl.serve_forever()
        return 0

    return 1

def setup_bitbake(configParams, configuration, extrafeatures=None, setup_logging=True):
    # Ensure logging messages get sent to the UI as events
    handler = bb.event.LogHandler()
    if setup_logging and not configParams.status_only:
        # In status only mode there are no logs and no UI
        logger.addHandler(handler)

    # Clear away any spurious environment variables while we stoke up the cooker
    cleanedvars = bb.utils.clean_environment()

    if configParams.server_only:
        featureset = []
        ui_module = None
    else:
        ui_module = import_extension_module(bb.ui, configParams.ui, 'main')
        # Collect the feature set for the UI
        featureset = getattr(ui_module, "featureSet", [])

    if configParams.server_only:
        for param in ('prefile', 'postfile'):
            value = getattr(configParams, param)
            if value:
                setattr(configuration, "%s_server" % param, value)
                param = "%s_server" % param

    if extrafeatures:
        for feature in extrafeatures:
            if not feature in featureset:
                featureset.append(feature)

    servermodule = import_extension_module(bb.server,
                                            configParams.servertype,
                                            'BitBakeServer')
    if configParams.remote_server:
        if os.getenv('BBSERVER') == 'autostart':
            if configParams.remote_server == 'autostart' or \
               not servermodule.check_connection(configParams.remote_server, timeout=2):
                configParams.bind = 'localhost:0'
                srv = start_server(servermodule, configParams, configuration, featureset)
                configParams.remote_server = '%s:%d' % tuple(configuration.interface)
                bb.event.ui_queue = []
        # we start a stub server that is actually a XMLRPClient that connects to a real server
        from bb.server.xmlrpc import BitBakeXMLRPCClient
        server = servermodule.BitBakeXMLRPCClient(configParams.observe_only,
                                                  configParams.xmlrpctoken)
        server.saveConnectionDetails(configParams.remote_server)
    else:
        # we start a server with a given configuration
        server = start_server(servermodule, configParams, configuration, featureset)
        bb.event.ui_queue = []

    if configParams.server_only:
        server_connection = None
    else:
        try:
            server_connection = server.establishConnection(featureset)
        except Exception as e:
            bb.fatal("Could not connect to server %s: %s" % (configParams.remote_server, str(e)))

        if configParams.kill_server:
            server_connection.connection.terminateServer()
            bb.event.ui_queue = []
            return None, None, None

        server_connection.setupEventQueue()

        # Restore the environment in case the UI needs it
        for k in cleanedvars:
            os.environ[k] = cleanedvars[k]

        logger.removeHandler(handler)

    return server, server_connection, ui_module