summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--bitbake/lib/bb/cooker.py58
-rw-r--r--bitbake/lib/bb/cookerdata.py5
-rwxr-xr-xbitbake/lib/bb/main.py230
-rw-r--r--bitbake/lib/bb/server/__init__.py6
-rw-r--r--bitbake/lib/bb/server/process.py596
-rw-r--r--bitbake/lib/bb/server/xmlrpc.py492
-rw-r--r--bitbake/lib/bb/server/xmlrpcclient.py154
-rw-r--r--bitbake/lib/bb/server/xmlrpcserver.py158
-rw-r--r--bitbake/lib/bb/tinfoil.py2
-rw-r--r--bitbake/lib/prserv/serv.py4
10 files changed, 789 insertions, 916 deletions
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index b1311bb170..e27763ecab 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -215,19 +215,6 @@ class BBCooker:
self.configuration.server_register_idlecallback(_process_inotify_updates, [self.confignotifier, self.notifier])
- # Take a lock so only one copy of bitbake can run against a given build
- # directory at a time
- if not self.lockBitbake():
- bb.fatal("Only one copy of bitbake should be run against a build directory")
- try:
- self.lock.seek(0)
- self.lock.truncate()
- if len(configuration.interface) >= 2:
- self.lock.write("%s:%s\n" % (configuration.interface[0], configuration.interface[1]));
- self.lock.flush()
- except:
- pass
-
# TOSTOP must not be set or our children will hang when they output
try:
fd = sys.stdout.fileno()
@@ -1557,33 +1544,6 @@ class BBCooker:
def post_serve(self):
prserv.serv.auto_shutdown(self.data)
bb.event.fire(CookerExit(), self.data)
- lockfile = self.lock.name
- self.lock.close()
- self.lock = None
-
- while not self.lock:
- with bb.utils.timeout(3):
- self.lock = bb.utils.lockfile(lockfile, shared=False, retry=False, block=True)
- if not self.lock:
- # Some systems may not have lsof available
- procs = None
- try:
- procs = subprocess.check_output(["lsof", '-w', lockfile], stderr=subprocess.STDOUT)
- except OSError as e:
- if e.errno != errno.ENOENT:
- raise
- if procs is None:
- # Fall back to fuser if lsof is unavailable
- try:
- procs = subprocess.check_output(["fuser", '-v', lockfile], stderr=subprocess.STDOUT)
- except OSError as e:
- if e.errno != errno.ENOENT:
- raise
-
- msg = "Delaying shutdown due to active processes which appear to be holding bitbake.lock"
- if procs:
- msg += ":\n%s" % str(procs)
- print(msg)
def shutdown(self, force = False):
@@ -1605,23 +1565,7 @@ class BBCooker:
def clientComplete(self):
"""Called when the client is done using the server"""
- if self.configuration.server_only:
- self.finishcommand()
- else:
- self.shutdown(True)
-
- def lockBitbake(self):
- if not hasattr(self, 'lock'):
- self.lock = None
- if self.data:
- lockfile = self.data.expand("${TOPDIR}/bitbake.lock")
- if lockfile:
- self.lock = bb.utils.lockfile(lockfile, False, False)
- return self.lock
-
- def unlockBitbake(self):
- if hasattr(self, 'lock') and self.lock:
- bb.utils.unlockfile(self.lock)
+ self.finishcommand()
def server_main(cooker, func, *args):
cooker.pre_serve()
diff --git a/bitbake/lib/bb/cookerdata.py b/bitbake/lib/bb/cookerdata.py
index 6511dcbfad..d05abfe745 100644
--- a/bitbake/lib/bb/cookerdata.py
+++ b/bitbake/lib/bb/cookerdata.py
@@ -76,7 +76,7 @@ class ConfigParameters(object):
for o in ["abort", "tryaltconfigs", "force", "invalidate_stamp",
"verbose", "debug", "dry_run", "dump_signatures",
"debug_domains", "extra_assume_provided", "profile",
- "prefile", "postfile", "tracking"]:
+ "prefile", "postfile", "tracking", "server_timeout"]:
options[o] = getattr(self.options, o)
ret, error = server.runCommand(["updateConfig", options, environment, sys.argv])
@@ -144,7 +144,8 @@ class CookerConfiguration(object):
self.dump_signatures = []
self.dry_run = False
self.tracking = False
- self.interface = []
+ self.xmlrpcinterface = []
+ self.server_timeout = None
self.writeeventlog = False
self.server_only = False
self.limited_deps = False
diff --git a/bitbake/lib/bb/main.py b/bitbake/lib/bb/main.py
index 29e391162e..1edf56f41b 100755
--- a/bitbake/lib/bb/main.py
+++ b/bitbake/lib/bb/main.py
@@ -28,6 +28,8 @@ import logging
import optparse
import warnings
import fcntl
+import time
+import traceback
import bb
from bb import event
@@ -37,6 +39,9 @@ from bb import ui
from bb import server
from bb import cookerdata
+import bb.server.process
+import bb.server.xmlrpcclient
+
logger = logging.getLogger("BitBake")
class BBMainException(Exception):
@@ -58,9 +63,6 @@ class BitbakeHelpFormatter(optparse.IndentedHelpFormatter):
if option.dest == 'ui':
valid_uis = list_extension_modules(bb.ui, 'main')
option.help = option.help.replace('@CHOICES@', present_options(valid_uis))
- elif option.dest == 'servertype':
- valid_server_types = list_extension_modules(bb.server, 'BitBakeServer')
- option.help = option.help.replace('@CHOICES@', present_options(valid_server_types))
return optparse.IndentedHelpFormatter.format_option(self, option)
@@ -238,11 +240,6 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
default=os.environ.get('BITBAKE_UI', 'knotty'),
help="The user interface to use (@CHOICES@ - default %default).")
- # @CHOICES@ is substituted out by BitbakeHelpFormatter above
- parser.add_option("-t", "--servertype", action="store", dest="servertype",
- default=["process", "xmlrpc"]["BBSERVER" in os.environ],
- help="Choose which server type to use (@CHOICES@ - default %default).")
-
parser.add_option("", "--token", action="store", dest="xmlrpctoken",
default=os.environ.get("BBTOKEN"),
help="Specify the connection token to be used when connecting "
@@ -258,14 +255,11 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
help="Run bitbake without a UI, only starting a server "
"(cooker) process.")
- parser.add_option("", "--foreground", action="store_true",
- help="Run bitbake server in foreground.")
-
parser.add_option("-B", "--bind", action="store", dest="bind", default=False,
- help="The name/address for the bitbake server to bind to.")
+ help="The name/address for the bitbake xmlrpc server to bind to.")
- parser.add_option("-T", "--idle-timeout", type=int,
- default=int(os.environ.get("BBTIMEOUT", "0")),
+ parser.add_option("-T", "--idle-timeout", type=float, dest="server_timeout",
+ default=float(os.environ.get("BB_SERVER_TIMEOUT", 0)) or None,
help="Set timeout to unload bitbake server due to inactivity")
parser.add_option("", "--no-setscene", action="store_true",
@@ -283,7 +277,7 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
parser.add_option("-m", "--kill-server", action="store_true",
dest="kill_server", default=False,
- help="Terminate the remote server.")
+ help="Terminate the bitbake server.")
parser.add_option("", "--observe-only", action="store_true",
dest="observe_only", default=False,
@@ -322,70 +316,20 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
eventlog = "bitbake_eventlog_%s.json" % datetime.now().strftime("%Y%m%d%H%M%S")
options.writeeventlog = eventlog
- # if BBSERVER says to autodetect, let's do that
- if options.remote_server:
- port = -1
- if options.remote_server != 'autostart':
- host, port = options.remote_server.split(":", 2)
+ if options.bind:
+ try:
+ #Checking that the port is a number and is a ':' delimited value
+ (host, port) = options.bind.split(':')
port = int(port)
- # use automatic port if port set to -1, means read it from
- # the bitbake.lock file; this is a bit tricky, but we always expect
- # to be in the base of the build directory if we need to have a
- # chance to start the server later, anyway
- if port == -1:
- lock_location = "./bitbake.lock"
- # we try to read the address at all times; if the server is not started,
- # we'll try to start it after the first connect fails, below
- try:
- lf = open(lock_location, 'r')
- remotedef = lf.readline()
- [host, port] = remotedef.split(":")
- port = int(port)
- lf.close()
- options.remote_server = remotedef
- except Exception as e:
- if options.remote_server != 'autostart':
- raise BBMainException("Failed to read bitbake.lock (%s), invalid port" % str(e))
+ except (ValueError,IndexError):
+ raise BBMainException("FATAL: Malformed host:port bind parameter")
+ options.xmlrpcinterface = (host, port)
+ else:
+ options.xmlrpcinterface = (None, 0)
return options, targets[1:]
-def start_server(servermodule, configParams, configuration, features):
- server = servermodule.BitBakeServer()
- single_use = not configParams.server_only and os.getenv('BBSERVER') != 'autostart'
- if configParams.bind:
- (host, port) = configParams.bind.split(':')
- server.initServer((host, int(port)), single_use=single_use,
- idle_timeout=configParams.idle_timeout)
- configuration.interface = [server.serverImpl.host, server.serverImpl.port]
- else:
- server.initServer(single_use=single_use)
- configuration.interface = []
-
- try:
- configuration.setServerRegIdleCallback(server.getServerIdleCB())
-
- cooker = bb.cooker.BBCooker(configuration, features)
-
- server.addcooker(cooker)
- server.saveConnectionDetails()
- except Exception as e:
- while hasattr(server, "event_queue"):
- import queue
- try:
- event = server.event_queue.get(block=False)
- except (queue.Empty, IOError):
- break
- if isinstance(event, logging.LogRecord):
- logger.handle(event)
- raise
- if not configParams.foreground:
- server.detach()
- cooker.shutdown()
- cooker.lock.close()
- return server
-
-
def bitbake_main(configParams, configuration):
# Python multiprocessing requires /dev/shm on Linux
@@ -406,45 +350,15 @@ def bitbake_main(configParams, configuration):
configuration.setConfigParameters(configParams)
- if configParams.server_only:
- if configParams.servertype != "xmlrpc":
- raise BBMainException("FATAL: If '--server-only' is defined, we must set the "
- "servertype as 'xmlrpc'.\n")
- if not configParams.bind:
- raise BBMainException("FATAL: The '--server-only' option requires a name/address "
- "to bind to with the -B option.\n")
- else:
- try:
- #Checking that the port is a number
- int(configParams.bind.split(":")[1])
- except (ValueError,IndexError):
- raise BBMainException(
- "FATAL: Malformed host:port bind parameter")
- if configParams.remote_server:
+ if configParams.server_only and configParams.remote_server:
raise BBMainException("FATAL: The '--server-only' option conflicts with %s.\n" %
("the BBSERVER environment variable" if "BBSERVER" in os.environ \
else "the '--remote-server' option"))
- elif configParams.foreground:
- raise BBMainException("FATAL: The '--foreground' option can only be used "
- "with --server-only.\n")
-
- if configParams.bind and configParams.servertype != "xmlrpc":
- raise BBMainException("FATAL: If '-B' or '--bind' is defined, we must "
- "set the servertype as 'xmlrpc'.\n")
-
- if configParams.remote_server and configParams.servertype != "xmlrpc":
- raise BBMainException("FATAL: If '--remote-server' is defined, we must "
- "set the servertype as 'xmlrpc'.\n")
-
if configParams.observe_only and (not configParams.remote_server or configParams.bind):
raise BBMainException("FATAL: '--observe-only' can only be used by UI clients "
"connecting to a server.\n")
- if configParams.kill_server and not configParams.remote_server:
- raise BBMainException("FATAL: '--kill-server' can only be used to "
- "terminate a remote server")
-
if "BBDEBUG" in os.environ:
level = int(os.environ["BBDEBUG"])
if level > configuration.debug:
@@ -453,7 +367,7 @@ def bitbake_main(configParams, configuration):
bb.msg.init_msgconfig(configParams.verbose, configuration.debug,
configuration.debug_domains)
- server, server_connection, ui_module = setup_bitbake(configParams, configuration)
+ server_connection, ui_module = setup_bitbake(configParams, configuration)
if server_connection is None and configParams.kill_server:
return 0
@@ -463,16 +377,15 @@ def bitbake_main(configParams, configuration):
return 0
try:
+ for event in bb.event.ui_queue:
+ server_connection.events.queue_event(event)
+ bb.event.ui_queue = []
+
return ui_module.main(server_connection.connection, server_connection.events,
configParams)
finally:
- bb.event.ui_queue = []
server_connection.terminate()
else:
- print("Bitbake server address: %s, server port: %s" % (server.serverImpl.host,
- server.serverImpl.port))
- if configParams.foreground:
- server.serverImpl.serve_forever()
return 0
return 1
@@ -495,58 +408,69 @@ def setup_bitbake(configParams, configuration, extrafeatures=None, setup_logging
# Collect the feature set for the UI
featureset = getattr(ui_module, "featureSet", [])
- if configParams.server_only:
- for param in ('prefile', 'postfile'):
- value = getattr(configParams, param)
- if value:
- setattr(configuration, "%s_server" % param, value)
- param = "%s_server" % param
-
if extrafeatures:
for feature in extrafeatures:
if not feature in featureset:
featureset.append(feature)
- servermodule = import_extension_module(bb.server,
- configParams.servertype,
- 'BitBakeServer')
+ server_connection = None
+
if configParams.remote_server:
- if os.getenv('BBSERVER') == 'autostart':
- if configParams.remote_server == 'autostart' or \
- not servermodule.check_connection(configParams.remote_server, timeout=2):
- configParams.bind = 'localhost:0'
- srv = start_server(servermodule, configParams, configuration, featureset)
- configParams.remote_server = '%s:%d' % tuple(configuration.interface)
- bb.event.ui_queue = []
- # we start a stub server that is actually a XMLRPClient that connects to a real server
- from bb.server.xmlrpc import BitBakeXMLRPCClient
- server = servermodule.BitBakeXMLRPCClient(configParams.observe_only,
- configParams.xmlrpctoken)
- server.saveConnectionDetails(configParams.remote_server)
+ # Connect to a remote XMLRPC server
+ server_connection = bb.server.xmlrpcclient.connectXMLRPC(configParams.remote_server, featureset,
+ configParams.observe_only, configParams.xmlrpctoken)
else:
- # we start a server with a given configuration
- server = start_server(servermodule, configParams, configuration, featureset)
+ retries = 8
+ while retries:
+ try:
+ topdir, lock = lockBitbake()
+ sockname = topdir + "/bitbake.sock"
+ if lock:
+ # we start a server with a given configuration
+ logger.info("Starting bitbake server...")
+ server = bb.server.process.BitBakeServer(lock, sockname, configuration, featureset)
+ # The server will handle any events already in the queue
+ bb.event.ui_queue = []
+ else:
+ logger.info("Reconnecting to bitbake server...")
+ if not os.path.exists(sockname):
+ print("Previous bitbake instance shutting down?, waiting to retry...")
+ time.sleep(5)
+ raise bb.server.process.ProcessTimeout("Bitbake still shutting down as socket exists but no lock?")
+ if not configParams.server_only:
+ server_connection = bb.server.process.connectProcessServer(sockname, featureset)
+ if server_connection:
+ break
+ except (Exception, bb.server.process.ProcessTimeout) as e:
+ if not retries:
+ raise
+ retries -= 1
+ if isinstance(e, (bb.server.process.ProcessTimeout, BrokenPipeError)):
+ logger.info("Retrying server connection...")
+ else:
+ logger.info("Retrying server connection... (%s)" % traceback.format_exc())
+ if not retries:
+ bb.fatal("Unable to connect to bitbake server, or start one")
+ if retries < 5:
+ time.sleep(5)
+
+ if configParams.kill_server:
+ server_connection.connection.terminateServer()
+ server_connection.terminate()
bb.event.ui_queue = []
+ logger.info("Terminated bitbake server.")
+ return None, None
- if configParams.server_only:
- server_connection = None
- else:
- try:
- server_connection = server.establishConnection(featureset)
- except Exception as e:
- bb.fatal("Could not connect to server %s: %s" % (configParams.remote_server, str(e)))
-
- if configParams.kill_server:
- server_connection.connection.terminateServer()
- bb.event.ui_queue = []
- return None, None, None
+ # Restore the environment in case the UI needs it
+ for k in cleanedvars:
+ os.environ[k] = cleanedvars[k]
- server_connection.setupEventQueue()
+ logger.removeHandler(handler)
- # Restore the environment in case the UI needs it
- for k in cleanedvars:
- os.environ[k] = cleanedvars[k]
+ return server_connection, ui_module
- logger.removeHandler(handler)
+def lockBitbake():
+ topdir = bb.cookerdata.findTopdir()
+ lockfile = topdir + "/bitbake.lock"
+ return topdir, bb.utils.lockfile(lockfile, False, False)
- return server, server_connection, ui_module
diff --git a/bitbake/lib/bb/server/__init__.py b/bitbake/lib/bb/server/__init__.py
index 345691e40f..5a3fba968f 100644
--- a/bitbake/lib/bb/server/__init__.py
+++ b/bitbake/lib/bb/server/__init__.py
@@ -18,10 +18,4 @@
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-""" Base code for Bitbake server process
-
-Have a common base for that all Bitbake server classes ensures a consistent
-approach to the interface, and minimize risks associated with code duplication.
-
-"""
diff --git a/bitbake/lib/bb/server/process.py b/bitbake/lib/bb/server/process.py
index 48da7fe46c..6edb0213ad 100644
--- a/bitbake/lib/bb/server/process.py
+++ b/bitbake/lib/bb/server/process.py
@@ -22,128 +22,205 @@
import bb
import bb.event
-import itertools
import logging
import multiprocessing
+import threading
+import array
import os
-import signal
import sys
import time
import select
-from queue import Empty
-from multiprocessing import Event, Process, util, Queue, Pipe, queues, Manager
+import socket
+import subprocess
+import errno
+import bb.server.xmlrpcserver
+from bb import daemonize
+from multiprocessing import queues
logger = logging.getLogger('BitBake')
-class ServerCommunicator():
- def __init__(self, connection, event_handle, server):
- self.connection = connection
- self.event_handle = event_handle
- self.server = server
-
- def runCommand(self, command):
- # @todo try/except
- self.connection.send(command)
-
- if not self.server.is_alive():
- raise SystemExit
-
- while True:
- # don't let the user ctrl-c while we're waiting for a response
- try:
- for idx in range(0,4): # 0, 1, 2, 3
- if self.connection.poll(5):
- return self.connection.recv()
- else:
- bb.warn("Timeout while attempting to communicate with bitbake server")
- bb.fatal("Gave up; Too many tries: timeout while attempting to communicate with bitbake server")
- except KeyboardInterrupt:
- pass
-
- def getEventHandle(self):
- handle, error = self.runCommand(["getUIHandlerNum"])
- if error:
- logger.error("Unable to get UI Handler Number: %s" % error)
- raise BaseException(error)
+class ProcessTimeout(SystemExit):
+ pass
- return handle
-
-class EventAdapter():
- """
- Adapter to wrap our event queue since the caller (bb.event) expects to
- call a send() method, but our actual queue only has put()
- """
- def __init__(self, queue):
- self.queue = queue
-
- def send(self, event):
- try:
- self.queue.put(event)
- except Exception as err:
- print("EventAdapter puked: %s" % str(err))
-
-
-class ProcessServer(Process):
+class ProcessServer(multiprocessing.Process):
profile_filename = "profile.log"
profile_processed_filename = "profile.log.processed"
- def __init__(self, command_channel, event_queue, featurelist):
- self._idlefuns = {}
- Process.__init__(self)
- self.command_channel = command_channel
- self.event_queue = event_queue
- self.event = EventAdapter(event_queue)
- self.featurelist = featurelist
+ def __init__(self, lock, sock, sockname):
+ multiprocessing.Process.__init__(self)
+ self.command_channel = False
+ self.command_channel_reply = False
self.quit = False
self.heartbeat_seconds = 1 # default, BB_HEARTBEAT_EVENT will be checked once we have a datastore.
self.next_heartbeat = time.time()
- self.quitin, self.quitout = Pipe()
- self.event_handle = multiprocessing.Value("i")
+ self.event_handle = None
+ self.haveui = False
+ self.lastui = False
+ self.xmlrpc = False
+
+ self._idlefuns = {}
+
+ self.bitbake_lock = lock
+ self.sock = sock
+ self.sockname = sockname
+
+ def register_idle_function(self, function, data):
+ """Register a function to be called while the server is idle"""
+ assert hasattr(function, '__call__')
+ self._idlefuns[function] = data
def run(self):
- for event in bb.event.ui_queue:
- self.event_queue.put(event)
- self.event_handle.value = bb.event.register_UIHhandler(self, True)
+
+ if self.xmlrpcinterface[0]:
+ self.xmlrpc = bb.server.xmlrpcserver.BitBakeXMLRPCServer(self.xmlrpcinterface, self.cooker, self)
+
+ print("Bitbake XMLRPC server address: %s, server port: %s" % (self.xmlrpc.host, self.xmlrpc.port))
heartbeat_event = self.cooker.data.getVar('BB_HEARTBEAT_EVENT')
if heartbeat_event:
try:
self.heartbeat_seconds = float(heartbeat_event)
except:
- # Throwing an exception here causes bitbake to hang.
- # Just warn about the invalid setting and continue
bb.warn('Ignoring invalid BB_HEARTBEAT_EVENT=%s, must be a float specifying seconds.' % heartbeat_event)
+
+ self.timeout = self.server_timeout or self.cooker.data.getVar('BB_SERVER_TIMEOUT')
+ try:
+ if self.timeout:
+ self.timeout = float(self.timeout)
+ except:
+ bb.warn('Ignoring invalid BB_SERVER_TIMEOUT=%s, must be a float specifying seconds.' % self.timeout)
+
+
+ try:
+ self.bitbake_lock.seek(0)
+ self.bitbake_lock.truncate()
+ if self.xmlrpcinterface[0]:
+ self.bitbake_lock.write("%s %s:%s\n" % (os.getpid(), configuration.interface[0], configuration.interface[1]))
+ else:
+ self.bitbake_lock.write("%s\n" % (os.getpid()))
+ self.bitbake_lock.flush()
+ except:
+ pass
+
bb.cooker.server_main(self.cooker, self.main)
def main(self):
- # Ignore SIGINT within the server, as all SIGINT handling is done by
- # the UI and communicated to us
- self.quitin.close()
- signal.signal(signal.SIGINT, signal.SIG_IGN)
bb.utils.set_process_name("Cooker")
+
+ ready = []
+
+ self.controllersock = False
+ fds = [self.sock]
+ if self.xmlrpc:
+ fds.append(self.xmlrpc)
while not self.quit:
- try:
- if self.command_channel.poll():
- command = self.command_channel.recv()
- self.runCommand(command)
- if self.quitout.poll():
- self.quitout.recv()
+ if self.command_channel in ready:
+ command = self.command_channel.get()
+ if command[0] == "terminateServer":
self.quit = True
- try:
- self.runCommand(["stateForceShutdown"])
- except:
- pass
-
- self.idle_commands(.1, [self.command_channel, self.quitout])
- except Exception:
- logger.exception('Running command %s', command)
+ continue
+ try:
+ print("Running command %s" % command)
+ self.command_channel_reply.send(self.cooker.command.runCommand(command))
+ except Exception as e:
+ logger.exception('Exception in server main event loop running command %s (%s)' % (command, str(e)))
+
+ if self.xmlrpc in ready:
+ self.xmlrpc.handle_requests()
+ if self.sock in ready:
+ self.controllersock, address = self.sock.accept()
+ if self.haveui:
+ print("Dropping connection attempt as we have a UI %s" % (str(ready)))
+ self.controllersock.close()
+ else:
+ print("Accepting %s" % (str(ready)))
+ fds.append(self.controllersock)
+ if self.controllersock in ready:
+ try:
+ print("Connecting Client")
+ ui_fds = recvfds(self.controllersock, 3)
+
+ # Where to write events to
+ writer = ConnectionWriter(ui_fds[0])
+ self.event_handle = bb.event.register_UIHhandler(writer, True)
+ self.event_writer = writer
+
+ # Where to read commands from
+ reader = ConnectionReader(ui_fds[1])
+ fds.append(reader)
+ self.command_channel = reader
+
+ # Where to send command return values to
+ writer = ConnectionWriter(ui_fds[2])
+ self.command_channel_reply = writer
+
+ self.haveui = True
+
+ except EOFError:
+ print("Disconnecting Client")
+ fds.remove(self.controllersock)
+ fds.remove(self.command_channel)
+ bb.event.unregister_UIHhandler(self.event_handle, True)
+ self.command_channel_reply.writer.close()
+ self.event_writer.writer.close()
+ del self.event_writer
+ self.controllersock.close()
+ self.haveui = False
+ self.lastui = time.time()
+ self.cooker.clientComplete()
+ if self.timeout is None:
+ print("No timeout, exiting.")
+ self.quit = True
+ if not self.haveui and self.lastui and self.timeout and (self.lastui + self.timeout) < time.time():
+ print("Server timeout, exiting.")
+ self.quit = True
- self.event_queue.close()
- bb.event.unregister_UIHhandler(self.event_handle.value, True)
- self.command_channel.close()
- self.cooker.shutdown(True)
- self.quitout.close()
+ ready = self.idle_commands(.1, fds)
+
+ print("Exiting")
+ try:
+ self.cooker.shutdown(True)
+ except:
+ pass
+
+ # Remove the socket file so we don't get any more connections to avoid races
+ os.unlink(self.sockname)
+ self.sock.close()
+
+ # Finally release the lockfile but warn about other processes holding it open
+ lock = self.bitbake_lock
+ lockfile = lock.name
+ lock.close()
+ lock = None
+
+ while not lock:
+ with bb.utils.timeout(3):
+ lock = bb.utils.lockfile(lockfile, shared=False, retry=False, block=True)
+ if not lock:
+ # Some systems may not have lsof available
+ procs = None
+ try:
+ procs = subprocess.check_output(["lsof", '-w', lockfile], stderr=subprocess.STDOUT)
+ except OSError as e:
+ if e.errno != errno.ENOENT:
+ raise
+ if procs is None:
+ # Fall back to fuser if lsof is unavailable
+ try:
+ procs = subprocess.check_output(["fuser", '-v', lockfile], stderr=subprocess.STDOUT)
+ except OSError as e:
+ if e.errno != errno.ENOENT:
+ raise
+
+ msg = "Delaying shutdown due to active processes which appear to be holding bitbake.lock"
+ if procs:
+ msg += ":\n%s" % str(procs)
+ print(msg)
+ return
+ # We hold the lock so we can remove the file (hide stale pid data)
+ bb.utils.remove(lockfile)
+ bb.utils.unlockfile(lock)
def idle_commands(self, delay, fds=None):
nextsleep = delay
@@ -189,140 +266,253 @@ class ProcessServer(Process):
nextsleep = self.next_heartbeat - now
if nextsleep is not None:
+ if self.xmlrpc:
+ nextsleep = self.xmlrpc.get_timeout(nextsleep)
try:
- select.select(fds,[],[],nextsleep)
+ return select.select(fds,[],[],nextsleep)[0]
except InterruptedError:
- # ignore EINTR error, nextsleep only used for wait
- # certain time
- pass
+ # Ignore EINTR
+ return []
+ else:
+ return []
+
+
+class ServerCommunicator():
+ def __init__(self, connection, recv):
+ self.connection = connection
+ self.recv = recv
def runCommand(self, command):
- """
- Run a cooker command on the server
- """
- self.command_channel.send(self.cooker.command.runCommand(command))
- def stop(self):
- self.quitin.send("quit")
- self.quitin.close()
+ self.connection.send(command)
+ while True:
+ # don't let the user ctrl-c while we're waiting for a response
+ try:
+ for idx in range(0,4): # 0, 1, 2, 3
+ if self.recv.poll(1):
+ return self.recv.get()
+ else:
+ bb.warn("Timeout while attempting to communicate with bitbake server")
+ raise ProcessTimeout("Gave up; Too many tries: timeout while attempting to communicate with bitbake server")
+ except KeyboardInterrupt:
+ pass
+
+ def updateFeatureSet(self, featureset):
+ _, error = self.runCommand(["setFeatures", featureset])
+ if error:
+ logger.error("Unable to set the cooker to the correct featureset: %s" % error)
+ raise BaseException(error)
+
+ def getEventHandle(self):
+ handle, error = self.runCommand(["getUIHandlerNum"])
+ if error:
+ logger.error("Unable to get UI Handler Number: %s" % error)
+ raise BaseException(error)
- def addcooker(self, cooker):
- self.cooker = cooker
+ return handle
- def register_idle_function(self, function, data):
- """Register a function to be called while the server is idle"""
- assert hasattr(function, '__call__')
- self._idlefuns[function] = data
+ def terminateServer(self):
+ self.connection.send(['terminateServer'])
+ return
class BitBakeProcessServerConnection(object):
- def __init__(self, serverImpl, ui_channel, event_queue):
- self.procserver = serverImpl
- self.ui_channel = ui_channel
- self.event_queue = event_queue
- self.connection = ServerCommunicator(self.ui_channel, self.procserver.event_handle, self.procserver)
- self.events = self.event_queue
- self.terminated = False
-
- def sigterm_terminate(self):
- bb.error("UI received SIGTERM")
- self.terminate()
+ def __init__(self, ui_channel, recv, eq):
+ self.connection = ServerCommunicator(ui_channel, recv)
+ self.events = eq
def terminate(self):
- if self.terminated:
- return
- self.terminated = True
- def flushevents():
- while True:
- try:
- event = self.event_queue.get(block=False)
- except (Empty, IOError):
- break
- if isinstance(event, logging.LogRecord):
- logger.handle(event)
-
- self.procserver.stop()
-
- while self.procserver.is_alive():
- flushevents()
- self.procserver.join(0.1)
-
- self.ui_channel.close()
- self.event_queue.close()
- self.event_queue.setexit()
- # XXX: Call explicity close in _writer to avoid
- # fd leakage because isn't called on Queue.close()
- self.event_queue._writer.close()
-
- def setupEventQueue(self):
- pass
-
-# Wrap Queue to provide API which isn't server implementation specific
-class ProcessEventQueue(multiprocessing.queues.Queue):
- def __init__(self, maxsize):
- multiprocessing.queues.Queue.__init__(self, maxsize, ctx=multiprocessing.get_context())
- self.exit = False
- bb.utils.set_process_name("ProcessEQueue")
-
- def setexit(self):
- self.exit = True
-
- def waitEvent(self, timeout):
- if self.exit:
- return self.getEvent()
+ self.socket_connection.close()
+ return
+
+class BitBakeServer(object):
+ def __init__(self, lock, sockname, configuration, featureset):
+
+ self.configuration = configuration
+ self.featureset = featureset
+ self.sockname = sockname
+ self.bitbake_lock = lock
+
+ # Create server control socket
+ if os.path.exists(sockname):
+ os.unlink(sockname)
+
+ self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+ # AF_UNIX has path length issues so chdir here to workaround
+ cwd = os.getcwd()
try:
- if not self.server.is_alive():
- return self.getEvent()
- if timeout == 0:
- return self.get(False)
- return self.get(True, timeout)
- except Empty:
- return None
+ os.chdir(os.path.dirname(sockname))
+ self.sock.bind(os.path.basename(sockname))
+ finally:
+ os.chdir(cwd)
+ self.sock.listen(1)
+
+ os.set_inheritable(self.sock.fileno(), True)
+ bb.daemonize.createDaemon(self._startServer, "bitbake-cookerdaemon.log")
+ self.sock.close()
+ self.bitbake_lock.close()
+
+ def _startServer(self):
+ server = ProcessServer(self.bitbake_lock, self.sock, self.sockname)
+ self.configuration.setServerRegIdleCallback(server.register_idle_function)
+
+ # Copy prefile and postfile to _server variants
+ for param in ('prefile', 'postfile'):
+ value = getattr(self.configuration, param)
+ if value:
+ setattr(self.configuration, "%s_server" % param, value)
+
+ self.cooker = bb.cooker.BBCooker(self.configuration, self.featureset)
+ server.cooker = self.cooker
+ server.server_timeout = self.configuration.server_timeout
+ server.xmlrpcinterface = self.configuration.xmlrpcinterface
+ print("Started bitbake server pid %d" % os.getpid())
+ server.start()
+
+def connectProcessServer(sockname, featureset):
+ # Connect to socket
+ sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+ # AF_UNIX has path length issues so chdir here to workaround
+ cwd = os.getcwd()
+
+ try:
+ os.chdir(os.path.dirname(sockname))
+ sock.connect(os.path.basename(sockname))
+ finally:
+ os.chdir(cwd)
+
+ try:
+ # Send an fd for the remote to write events to
+ readfd, writefd = os.pipe()
+ eq = BBUIEventQueue(readfd)
+ # Send an fd for the remote to recieve commands from
+ readfd1, writefd1 = os.pipe()
+ command_chan = ConnectionWriter(writefd1)
+ # Send an fd for the remote to write commands results to
+ readfd2, writefd2 = os.pipe()
+ command_chan_recv = ConnectionReader(readfd2)
+
+ sendfds(sock, [writefd, readfd1, writefd2])
+
+ server_connection = BitBakeProcessServerConnection(command_chan, command_chan_recv, eq)
+
+ server_connection.connection.updateFeatureSet(featureset)
+
+ # Save sock so it doesn't get gc'd for the life of our connection
+ server_connection.socket_connection = sock
+ except:
+ sock.close()
+ raise
+
+ return server_connection
+
+def sendfds(sock, fds):
+ '''Send an array of fds over an AF_UNIX socket.'''
+ fds = array.array('i', fds)
+ msg = bytes([len(fds) % 256])
+ sock.sendmsg([msg], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, fds)])
+
+def recvfds(sock, size):
+ '''Receive an array of fds over an AF_UNIX socket.'''
+ a = array.array('i')
+ bytes_size = a.itemsize * size
+ msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_LEN(bytes_size))
+ if not msg and not ancdata:
+ raise EOFError
+ try:
+ if len(ancdata) != 1:
+ raise RuntimeError('received %d items of ancdata' %
+ len(ancdata))
+ cmsg_level, cmsg_type, cmsg_data = ancdata[0]
+ if (cmsg_level == socket.SOL_SOCKET and
+ cmsg_type == socket.SCM_RIGHTS):
+ if len(cmsg_data) % a.itemsize != 0:
+ raise ValueError
+ a.frombytes(cmsg_data)
+ assert len(a) % 256 == msg[0]
+ return list(a)
+ except (ValueError, IndexError):
+ pass
+ raise RuntimeError('Invalid data received')
+
+class BBUIEventQueue:
+ def __init__(self, readfd):
+
+ self.eventQueue = []
+ self.eventQueueLock = threading.Lock()
+ self.eventQueueNotify = threading.Event()
+
+ self.reader = ConnectionReader(readfd)
+
+ self.t = threading.Thread()
+ self.t.setDaemon(True)
+ self.t.run = self.startCallbackHandler
+ self.t.start()
def getEvent(self):
- try:
- if not self.server.is_alive():
- self.setexit()
- return self.get(False)
- except Empty:
- if self.exit:
- sys.exit(1)
+ self.eventQueueLock.acquire()
+
+ if len(self.eventQueue) == 0:
+ self.eventQueueLock.release()
return None
-class BitBakeServer(object):
- def initServer(self, single_use=True):
- # establish communication channels. We use bidirectional pipes for
- # ui <--> server command/response pairs
- # and a queue for server -> ui event notifications
- #
- self.ui_channel, self.server_channel = Pipe()
- self.event_queue = ProcessEventQueue(0)
- self.serverImpl = ProcessServer(self.server_channel, self.event_queue, None)
- self.event_queue.server = self.serverImpl
-
- def detach(self):
- self.serverImpl.start()
- return
+ item = self.eventQueue.pop(0)
- def establishConnection(self, featureset):
+ if len(self.eventQueue) == 0:
+ self.eventQueueNotify.clear()
- self.connection = BitBakeProcessServerConnection(self.serverImpl, self.ui_channel, self.event_queue)
+ self.eventQueueLock.release()
+ return item
- _, error = self.connection.connection.runCommand(["setFeatures", featureset])
- if error:
- logger.error("Unable to set the cooker to the correct featureset: %s" % error)
- raise BaseException(error)
- signal.signal(signal.SIGTERM, lambda i, s: self.connection.sigterm_terminate())
- return self.connection
+ def waitEvent(self, delay):
+ self.eventQueueNotify.wait(delay)
+ return self.getEvent()
- def addcooker(self, cooker):
- self.cooker = cooker
- self.serverImpl.addcooker(cooker)
+ def queue_event(self, event):
+ self.eventQueueLock.acquire()
+ self.eventQueue.append(event)
+ self.eventQueueNotify.set()
+ self.eventQueueLock.release()
- def getServerIdleCB(self):
- return self.serverImpl.register_idle_function
+ def send_event(self, event):
+ self.queue_event(pickle.loads(event))
- def saveConnectionDetails(self):
- return
+ def startCallbackHandler(self):
+ bb.utils.set_process_name("UIEventQueue")
+ while True:
+ self.reader.wait()
+ event = self.reader.get()
+ self.queue_event(event)
+
+class ConnectionReader(object):
+
+ def __init__(self, fd):
+ self.reader = multiprocessing.connection.Connection(fd, writable=False)
+ self.rlock = multiprocessing.Lock()
+
+ def wait(self, timeout=None):
+ return multiprocessing.connection.wait([self.reader], timeout)
+
+ def poll(self, timeout=None):
+ return self.reader.poll(timeout)
+
+ def get(self):
+ with self.rlock:
+ res = self.reader.recv_bytes()
+ return multiprocessing.reduction.ForkingPickler.loads(res)
+
+ def fileno(self):
+ return self.reader.fileno()
+
+class ConnectionWriter(object):
+
+ def __init__(self, fd):
+ self.writer = multiprocessing.connection.Connection(fd, readable=False)
+ self.wlock = multiprocessing.Lock()
+ # Why bb.event needs this I have no idea
+ self.event = self
+
+ def send(self, obj):
+ obj = multiprocessing.reduction.ForkingPickler.dumps(obj)
+ with self.wlock:
+ self.writer.send_bytes(obj)
- def endSession(self):
- self.connection.terminate()
diff --git a/bitbake/lib/bb/server/xmlrpc.py b/bitbake/lib/bb/server/xmlrpc.py
deleted file mode 100644
index 6874765136..0000000000
--- a/bitbake/lib/bb/server/xmlrpc.py
+++ /dev/null
@@ -1,492 +0,0 @@
-#
-# BitBake XMLRPC Server
-#
-# Copyright (C) 2006 - 2007 Michael 'Mickey' Lauer
-# Copyright (C) 2006 - 2008 Richard Purdie
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-"""
- This module implements an xmlrpc server for BitBake.
-
- Use this by deriving a class from BitBakeXMLRPCServer and then adding
- methods which you want to "export" via XMLRPC. If the methods have the
- prefix xmlrpc_, then registering those function will happen automatically,
- if not, you need to call register_function.
-
- Use register_idle_function() to add a function which the xmlrpc server
- calls from within server_forever when no requests are pending. Make sure
- that those functions are non-blocking or else you will introduce latency
- in the server's main loop.
-"""
-
-import os
-import sys
-
-import hashlib
-import time
-import socket
-import signal
-import threading
-import pickle
-import inspect
-import select
-import http.client
-import xmlrpc.client
-from xmlrpc.server import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
-
-import bb
-from bb import daemonize
-from bb.ui import uievent
-
-DEBUG = False
-
-class BBTransport(xmlrpc.client.Transport):
- def __init__(self, timeout):
- self.timeout = timeout
- self.connection_token = None
- xmlrpc.client.Transport.__init__(self)
-
- # Modified from default to pass timeout to HTTPConnection
- def make_connection(self, host):
- #return an existing connection if possible. This allows
- #HTTP/1.1 keep-alive.
- if self._connection and host == self._connection[0]:
- return self._connection[1]
-
- # create a HTTP connection object from a host descriptor
- chost, self._extra_headers, x509 = self.get_host_info(host)
- #store the host argument along with the connection object
- self._connection = host, http.client.HTTPConnection(chost, timeout=self.timeout)
- return self._connection[1]
-
- def set_connection_token(self, token):
- self.connection_token = token
-
- def send_content(self, h, body):
- if self.connection_token:
- h.putheader("Bitbake-token", self.connection_token)
- xmlrpc.client.Transport.send_content(self, h, body)
-
-def _create_server(host, port, timeout = 60):
- t = BBTransport(timeout)
- s = xmlrpc.client.ServerProxy("http://%s:%d/" % (host, port), transport=t, allow_none=True, use_builtin_types=True)
- return s, t
-
-def check_connection(remote, timeout):
- try:
- host, port = remote.split(":")
- port = int(port)
- except Exception as e:
- bb.warn("Failed to read remote definition (%s)" % str(e))
- raise e
-
- server, _transport = _create_server(host, port, timeout)
- try:
- ret, err = server.runCommand(['getVariable', 'TOPDIR'])
- if err or not ret:
- return False
- except ConnectionError:
- return False
- return True
-
-class BitBakeServerCommands():
-
- def __init__(self, server):
- self.server = server
- self.has_client = False
-
- def registerEventHandler(self, host, port):
- """
- Register a remote UI Event Handler
- """
- s, t = _create_server(host, port)
-
- # we don't allow connections if the cooker is running
- if (self.cooker.state in [bb.cooker.state.parsing, bb.cooker.state.running]):
- return None, "Cooker is busy: %s" % bb.cooker.state.get_name(self.cooker.state)
-
- self.event_handle = bb.event.register_UIHhandler(s, True)
- return self.event_handle, 'OK'
-
- def unregisterEventHandler(self, handlerNum):
- """
- Unregister a remote UI Event Handler
- """
- return bb.event.unregister_UIHhandler(handlerNum, True)
-
- def runCommand(self, command):
- """
- Run a cooker command on the server
- """
- return self.cooker.command.runCommand(command, self.server.readonly)
-
- def getEventHandle(self):
- return self.event_handle
-
- def terminateServer(self):
- """
- Trigger the server to quit
- """
- self.server.quit = True
- print("Server (cooker) exiting")
- return
-
- def addClient(self):
- if self.has_client:
- return None
- token = hashlib.md5(str(time.time()).encode("utf-8")).hexdigest()
- self.server.set_connection_token(token)
- self.has_client = True
- return token
-
- def removeClient(self):
- if self.has_client:
- self.server.set_connection_token(None)
- self.has_client = False
- if self.server.single_use:
- self.server.quit = True
-
-# This request handler checks if the request has a "Bitbake-token" header
-# field (this comes from the client side) and compares it with its internal
-# "Bitbake-token" field (this comes from the server). If the two are not
-# equal, it is assumed that a client is trying to connect to the server
-# while another client is connected to the server. In this case, a 503 error
-# ("service unavailable") is returned to the client.
-class BitBakeXMLRPCRequestHandler(SimpleXMLRPCRequestHandler):
- def __init__(self, request, client_address, server):
- self.server = server
- SimpleXMLRPCRequestHandler.__init__(self, request, client_address, server)
-
- def do_POST(self):
- try:
- remote_token = self.headers["Bitbake-token"]
- except:
- remote_token = None
- if remote_token != self.server.connection_token and remote_token != "observer":
- self.report_503()
- else:
- if remote_token == "observer":
- self.server.readonly = True
- else:
- self.server.readonly = False
- SimpleXMLRPCRequestHandler.do_POST(self)
-
- def report_503(self):
- self.send_response(503)
- response = 'No more client allowed'
- self.send_header("Content-type", "text/plain")
- self.send_header("Content-length", str(len(response)))
- self.end_headers()
- self.wfile.write(bytes(response, 'utf-8'))
-
-
-class XMLRPCProxyServer(object):
- """ not a real working server, but a stub for a proxy server connection
-
- """
- def __init__(self, host, port, use_builtin_types=True):
- self.host = host
- self.port = port
- self._idlefuns = {}
-
- def addcooker(self, cooker):
- self.cooker = cooker
-
- def register_idle_function(self, function, data):
- """Register a function to be called while the server is idle"""
- assert hasattr(function, '__call__')
- self._idlefuns[function] = data
-
-
-class XMLRPCServer(SimpleXMLRPCServer):
- # remove this when you're done with debugging
- # allow_reuse_address = True
-
- def __init__(self, interface, single_use=False, idle_timeout=0):
- """
- Constructor
- """
- self._idlefuns = {}
- self.single_use = single_use
- # Use auto port configuration
- if (interface[1] == -1):
- interface = (interface[0], 0)
- SimpleXMLRPCServer.__init__(self, interface,
- requestHandler=BitBakeXMLRPCRequestHandler,
- logRequests=False, allow_none=True)
- self.host, self.port = self.socket.getsockname()
- self.connection_token = None
- #self.register_introspection_functions()
- self.commands = BitBakeServerCommands(self)
- self.autoregister_all_functions(self.commands, "")
- self.interface = interface
- self.time = time.time()
- self.idle_timeout = idle_timeout
- if idle_timeout:
- self.register_idle_function(self.handle_idle_timeout, self)
- self.heartbeat_seconds = 1 # default, BB_HEARTBEAT_EVENT will be checked once we have a datastore.
- self.next_heartbeat = time.time()
-
- def addcooker(self, cooker):
- self.cooker = cooker
- self.commands.cooker = cooker
-
- def autoregister_all_functions(self, context, prefix):
- """
- Convenience method for registering all functions in the scope
- of this class that start with a common prefix
- """
- methodlist = inspect.getmembers(context, inspect.ismethod)
- for name, method in methodlist:
- if name.startswith(prefix):
- self.register_function(method, name[len(prefix):])
-
- def handle_idle_timeout(self, server, data, abort):
- if not abort:
- if time.time() - server.time > server.idle_timeout:
- server.quit = True
- print("Server idle timeout expired")
- return []
-
- def serve_forever(self):
- heartbeat_event = self.cooker.data.getVar('BB_HEARTBEAT_EVENT')
- if heartbeat_event:
- try:
- self.heartbeat_seconds = float(heartbeat_event)
- except:
- # Throwing an exception here causes bitbake to hang.
- # Just warn about the invalid setting and continue
- bb.warn('Ignoring invalid BB_HEARTBEAT_EVENT=%s, must be a float specifying seconds.' % heartbeat_event)
-
- # Start the actual XMLRPC server
- bb.cooker.server_main(self.cooker, self._serve_forever)
-
- def _serve_forever(self):
- """
- Serve Requests. Overloaded to honor a quit command
- """
- self.quit = False
- while not self.quit:
- fds = [self]
- nextsleep = 0.1
- for function, data in list(self._idlefuns.items()):
- retval = None
- try:
- retval = function(self, data, False)
- if retval is False:
- del self._idlefuns[function]
- elif retval is True:
- nextsleep = 0
- elif isinstance(retval, float):
- if (retval < nextsleep):
- nextsleep = retval
- else:
- fds = fds + retval
- except SystemExit:
- raise
- except:
- import traceback
- traceback.print_exc()
- if retval == None:
- # the function execute failed; delete it
- del self._idlefuns[function]
- pass
-
- socktimeout = self.socket.gettimeout() or nextsleep
- socktimeout = min(socktimeout, nextsleep)
- # Mirror what BaseServer handle_request would do
- try:
- fd_sets = select.select(fds, [], [], socktimeout)
- if fd_sets[0] and self in fd_sets[0]:
- if self.idle_timeout:
- self.time = time.time()
- self._handle_request_noblock()
- except IOError:
- # we ignore interrupted calls
- pass
-
- # Create new heartbeat event?
- now = time.time()
- if now >= self.next_heartbeat:
- # We might have missed heartbeats. Just trigger once in
- # that case and continue after the usual delay.
- self.next_heartbeat += self.heartbeat_seconds
- if self.next_heartbeat <= now:
- self.next_heartbeat = now + self.heartbeat_seconds
- heartbeat = bb.event.HeartbeatEvent(now)
- bb.event.fire(heartbeat, self.cooker.data)
- if nextsleep and now + nextsleep > self.next_heartbeat:
- # Shorten timeout so that we we wake up in time for
- # the heartbeat.
- nextsleep = self.next_heartbeat - now
-
- # Tell idle functions we're exiting
- for function, data in list(self._idlefuns.items()):
- try:
- retval = function(self, data, True)
- except:
- pass
- self.server_close()
- return
-
- def set_connection_token(self, token):
- self.connection_token = token
-
- def register_idle_function(self, function, data):
- """Register a function to be called while the server is idle"""
- assert hasattr(function, '__call__')
- self._idlefuns[function] = data
-
-
-class BitBakeXMLRPCServerConnection(object):
- def __init__(self, serverImpl, clientinfo=("localhost", 0), observer_only = False, featureset = None):
- self.connection, self.transport = _create_server(serverImpl.host, serverImpl.port)
- self.clientinfo = clientinfo
- self.serverImpl = serverImpl
- self.observer_only = observer_only
- if featureset:
- self.featureset = featureset
- else:
- self.featureset = []
-
- def connect(self, token = None):
- if token is None:
- if self.observer_only:
- token = "observer"
- else:
- token = self.connection.addClient()
-
- if token is None:
- return None
-
- self.transport.set_connection_token(token)
- return self
-
- def setupEventQueue(self):
- self.events = uievent.BBUIEventQueue(self.connection, self.clientinfo)
- for event in bb.event.ui_queue:
- self.events.queue_event(event)
-
- _, error = self.connection.runCommand(["setFeatures", self.featureset])
- if error:
- # disconnect the client, we can't make the setFeature work
- self.connection.removeClient()
- # no need to log it here, the error shall be sent to the client
- raise BaseException(error)
-
- def removeClient(self):
- if not self.observer_only:
- self.connection.removeClient()
-
- def terminate(self):
- # Don't wait for server indefinitely
- import socket
- socket.setdefaulttimeout(2)
- try:
- self.events.system_quit()
- except:
- pass
- try:
- self.connection.removeClient()
- except:
- pass
-
-class BitBakeServer(object):
- def initServer(self, interface = ("localhost", 0),
- single_use = False, idle_timeout=0):
- self.interface = interface
- self.serverImpl = XMLRPCServer(interface, single_use, idle_timeout)
-
- def detach(self):
- daemonize.createDaemon(self.serverImpl.serve_forever, "bitbake-cookerdaemon.log")
- del self.cooker
-
- def establishConnection(self, featureset):
- self.connection = BitBakeXMLRPCServerConnection(self.serverImpl, self.interface, False, featureset)
- return self.connection.connect()
-
- def set_connection_token(self, token):
- self.connection.transport.set_connection_token(token)
-
- def addcooker(self, cooker):
- self.cooker = cooker
- self.serverImpl.addcooker(cooker)
-
- def getServerIdleCB(self):
- return self.serverImpl.register_idle_function
-
- def saveConnectionDetails(self):
- return
-
- def endSession(self):
- self.connection.terminate()
-
-class BitBakeXMLRPCClient(object):
-
- def __init__(self, observer_only = False, token = None):
- self.token = token
-
- self.observer_only = observer_only
- # if we need extra caches, just tell the server to load them all
- pass
-
- def saveConnectionDetails(self, remote):
- self.remote = remote
-
- def establishConnection(self, featureset):
- # The format of "remote" must be "server:port"
- try:
- [host, port] = self.remote.split(":")
- port = int(port)
- except Exception as e:
- bb.warn("Failed to read remote definition (%s)" % str(e))
- raise e
-
- # We need our IP for the server connection. We get the IP
- # by trying to connect with the server
- try:
- s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
- s.connect((host, port))
- ip = s.getsockname()[0]
- s.close()
- except Exception as e:
- bb.warn("Could not create socket for %s:%s (%s)" % (host, port, str(e)))
- raise e
- try:
- self.serverImpl = XMLRPCProxyServer(host, port, use_builtin_types=True)
- self.connection = BitBakeXMLRPCServerConnection(self.serverImpl, (ip, 0), self.observer_only, featureset)
- return self.connection.connect(self.token)
- except Exception as e:
- bb.warn("Could not connect to server at %s:%s (%s)" % (host, port, str(e)))
- raise e
-
- def endSession(self):
- self.connection.removeClient()
-
- def initServer(self):
- self.serverImpl = None
- self.connection = None
- return
-
- def addcooker(self, cooker):
- self.cooker = cooker
- self.serverImpl.addcooker(cooker)
-
- def getServerIdleCB(self):
- return self.serverImpl.register_idle_function
-
- def detach(self):
- return
-
diff --git a/bitbake/lib/bb/server/xmlrpcclient.py b/bitbake/lib/bb/server/xmlrpcclient.py
new file mode 100644
index 0000000000..4661a9e5a0
--- /dev/null
+++ b/bitbake/lib/bb/server/xmlrpcclient.py
@@ -0,0 +1,154 @@
+#
+# BitBake XMLRPC Client Interface
+#
+# Copyright (C) 2006 - 2007 Michael 'Mickey' Lauer
+# Copyright (C) 2006 - 2008 Richard Purdie
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import os
+import sys
+
+import socket
+import http.client
+import xmlrpc.client
+
+import bb
+from bb.ui import uievent
+
+class BBTransport(xmlrpc.client.Transport):
+ def __init__(self, timeout):
+ self.timeout = timeout
+ self.connection_token = None
+ xmlrpc.client.Transport.__init__(self)
+
+ # Modified from default to pass timeout to HTTPConnection
+ def make_connection(self, host):
+ #return an existing connection if possible. This allows
+ #HTTP/1.1 keep-alive.
+ if self._connection and host == self._connection[0]:
+ return self._connection[1]
+
+ # create a HTTP connection object from a host descriptor
+ chost, self._extra_headers, x509 = self.get_host_info(host)
+ #store the host argument along with the connection object
+ self._connection = host, http.client.HTTPConnection(chost, timeout=self.timeout)
+ return self._connection[1]
+
+ def set_connection_token(self, token):
+ self.connection_token = token
+
+ def send_content(self, h, body):
+ if self.connection_token:
+ h.putheader("Bitbake-token", self.connection_token)
+ xmlrpc.client.Transport.send_content(self, h, body)
+
+def _create_server(host, port, timeout = 60):
+ t = BBTransport(timeout)
+ s = xmlrpc.client.ServerProxy("http://%s:%d/" % (host, port), transport=t, allow_none=True, use_builtin_types=True)
+ return s, t
+
+def check_connection(remote, timeout):
+ try:
+ host, port = remote.split(":")
+ port = int(port)
+ except Exception as e:
+ bb.warn("Failed to read remote definition (%s)" % str(e))
+ raise e
+
+ server, _transport = _create_server(host, port, timeout)
+ try:
+ ret, err = server.runCommand(['getVariable', 'TOPDIR'])
+ if err or not ret:
+ return False
+ except ConnectionError:
+ return False
+ return True
+
+class BitBakeXMLRPCServerConnection(object):
+ def __init__(self, host, port, clientinfo=("localhost", 0), observer_only = False, featureset = None):
+ self.connection, self.transport = _create_server(host, port)
+ self.clientinfo = clientinfo
+ self.observer_only = observer_only
+ if featureset:
+ self.featureset = featureset
+ else:
+ self.featureset = []
+
+ self.events = uievent.BBUIEventQueue(self.connection, self.clientinfo)
+
+ _, error = self.connection.runCommand(["setFeatures", self.featureset])
+ if error:
+ # disconnect the client, we can't make the setFeature work
+ self.connection.removeClient()
+ # no need to log it here, the error shall be sent to the client
+ raise BaseException(error)
+
+ def connect(self, token = None):
+ if token is None:
+ if self.observer_only:
+ token = "observer"
+ else:
+ token = self.connection.addClient()
+
+ if token is None:
+ return None
+
+ self.transport.set_connection_token(token)
+ return self
+
+ def removeClient(self):
+ if not self.observer_only:
+ self.connection.removeClient()
+
+ def terminate(self):
+ # Don't wait for server indefinitely
+ socket.setdefaulttimeout(2)
+ try:
+ self.events.system_quit()
+ except:
+ pass
+ try:
+ self.connection.removeClient()
+ except:
+ pass
+
+def connectXMLRPC(remote, featureset, observer_only = False, token = None):
+ # The format of "remote" must be "server:port"
+ try:
+ [host, port] = remote.split(":")
+ port = int(port)
+ except Exception as e:
+ bb.warn("Failed to parse remote definition %s (%s)" % (remote, str(e)))
+ raise e
+
+ # We need our IP for the server connection. We get the IP
+ # by trying to connect with the server
+ try:
+ s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+ s.connect((host, port))
+ ip = s.getsockname()[0]
+ s.close()
+ except Exception as e:
+ bb.warn("Could not create socket for %s:%s (%s)" % (host, port, str(e)))
+ raise e
+ try:
+ connection = BitBakeXMLRPCServerConnection(host, port, (ip, 0), observer_only, featureset)
+ return connection.connect(token)
+ except Exception as e:
+ bb.warn("Could not connect to server at %s:%s (%s)" % (host, port, str(e)))
+ raise e
+
+
+
diff --git a/bitbake/lib/bb/server/xmlrpcserver.py b/bitbake/lib/bb/server/xmlrpcserver.py
new file mode 100644
index 0000000000..875b1282e5
--- /dev/null
+++ b/bitbake/lib/bb/server/xmlrpcserver.py
@@ -0,0 +1,158 @@
+#
+# BitBake XMLRPC Server Interface
+#
+# Copyright (C) 2006 - 2007 Michael 'Mickey' Lauer
+# Copyright (C) 2006 - 2008 Richard Purdie
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import os
+import sys
+
+import hashlib
+import time
+import inspect
+from xmlrpc.server import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
+
+import bb
+
+# This request handler checks if the request has a "Bitbake-token" header
+# field (this comes from the client side) and compares it with its internal
+# "Bitbake-token" field (this comes from the server). If the two are not
+# equal, it is assumed that a client is trying to connect to the server
+# while another client is connected to the server. In this case, a 503 error
+# ("service unavailable") is returned to the client.
+class BitBakeXMLRPCRequestHandler(SimpleXMLRPCRequestHandler):
+ def __init__(self, request, client_address, server):
+ self.server = server
+ SimpleXMLRPCRequestHandler.__init__(self, request, client_address, server)
+
+ def do_POST(self):
+ try:
+ remote_token = self.headers["Bitbake-token"]
+ except:
+ remote_token = None
+ if 0 and remote_token != self.server.connection_token and remote_token != "observer":
+ self.report_503()
+ else:
+ if remote_token == "observer":
+ self.server.readonly = True
+ else:
+ self.server.readonly = False
+ SimpleXMLRPCRequestHandler.do_POST(self)
+
+ def report_503(self):
+ self.send_response(503)
+ response = 'No more client allowed'
+ self.send_header("Content-type", "text/plain")
+ self.send_header("Content-length", str(len(response)))
+ self.end_headers()
+ self.wfile.write(bytes(response, 'utf-8'))
+
+class BitBakeXMLRPCServer(SimpleXMLRPCServer):
+ # remove this when you're done with debugging
+ # allow_reuse_address = True
+
+ def __init__(self, interface, cooker, parent):
+ # Use auto port configuration
+ if (interface[1] == -1):
+ interface = (interface[0], 0)
+ SimpleXMLRPCServer.__init__(self, interface,
+ requestHandler=BitBakeXMLRPCRequestHandler,
+ logRequests=False, allow_none=True)
+ self.host, self.port = self.socket.getsockname()
+ self.interface = interface
+
+ self.connection_token = None
+ self.commands = BitBakeXMLRPCServerCommands(self)
+ self.register_functions(self.commands, "")
+
+ self.cooker = cooker
+ self.parent = parent
+
+
+ def register_functions(self, context, prefix):
+ """
+ Convenience method for registering all functions in the scope
+ of this class that start with a common prefix
+ """
+ methodlist = inspect.getmembers(context, inspect.ismethod)
+ for name, method in methodlist:
+ if name.startswith(prefix):
+ self.register_function(method, name[len(prefix):])
+
+ def get_timeout(self, delay):
+ socktimeout = self.socket.gettimeout() or delay
+ return min(socktimeout, delay)
+
+ def handle_requests(self):
+ self._handle_request_noblock()
+
+class BitBakeXMLRPCServerCommands():
+
+ def __init__(self, server):
+ self.server = server
+ self.has_client = False
+
+ def registerEventHandler(self, host, port):
+ """
+ Register a remote UI Event Handler
+ """
+ s, t = bb.server.xmlrpcclient._create_server(host, port)
+
+ # we don't allow connections if the cooker is running
+ if (self.server.cooker.state in [bb.cooker.state.parsing, bb.cooker.state.running]):
+ return None, "Cooker is busy: %s" % bb.cooker.state.get_name(self.server.cooker.state)
+
+ self.event_handle = bb.event.register_UIHhandler(s, True)
+ return self.event_handle, 'OK'
+
+ def unregisterEventHandler(self, handlerNum):
+ """
+ Unregister a remote UI Event Handler
+ """
+ ret = bb.event.unregister_UIHhandler(handlerNum, True)
+ self.event_handle = None
+ return ret
+
+ def runCommand(self, command):
+ """
+ Run a cooker command on the server
+ """
+ return self.server.cooker.command.runCommand(command, self.server.readonly)
+
+ def getEventHandle(self):
+ return self.event_handle
+
+ def terminateServer(self):
+ """
+ Trigger the server to quit
+ """
+ self.server.parent.quit = True
+ print("XMLRPC Server triggering exit")
+ return
+
+ def addClient(self):
+ if self.server.parent.haveui:
+ return None
+ token = hashlib.md5(str(time.time()).encode("utf-8")).hexdigest()
+ self.server.connection_token = token
+ self.server.parent.haveui = True
+ return token
+
+ def removeClient(self):
+ if self.server.parent.haveui:
+ self.server.connection_token = None
+ self.server.parent.haveui = False
+
diff --git a/bitbake/lib/bb/tinfoil.py b/bitbake/lib/bb/tinfoil.py
index fb0da62243..303ce02b00 100644
--- a/bitbake/lib/bb/tinfoil.py
+++ b/bitbake/lib/bb/tinfoil.py
@@ -243,7 +243,7 @@ class Tinfoil:
cookerconfig = CookerConfiguration()
cookerconfig.setConfigParameters(config_params)
- server, self.server_connection, ui_module = setup_bitbake(config_params,
+ self.server_connection, ui_module = setup_bitbake(config_params,
cookerconfig,
extrafeatures,
setup_logging=False)
diff --git a/bitbake/lib/prserv/serv.py b/bitbake/lib/prserv/serv.py
index a7efa58bc7..5f061c2623 100644
--- a/bitbake/lib/prserv/serv.py
+++ b/bitbake/lib/prserv/serv.py
@@ -6,7 +6,7 @@ import queue
import socket
import io
import sqlite3
-import bb.server.xmlrpc
+import bb.server.xmlrpcclient
import prserv
import prserv.db
import errno
@@ -300,7 +300,7 @@ class PRServerConnection(object):
host, port = singleton.getinfo()
self.host = host
self.port = port
- self.connection, self.transport = bb.server.xmlrpc._create_server(self.host, self.port)
+ self.connection, self.transport = bb.server.xmlrpcclient._create_server(self.host, self.port)
def terminate(self):
try: