summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/progress.py
blob: f54d1c76f8e6f99c69c654d3e9d913e39cffbce4 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
"""
BitBake progress handling code
"""

# Copyright (C) 2016 Intel Corporation
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

import sys
import re
import time
import inspect
import bb.event
import bb.build

class ProgressHandler(object):
    """
    Base class that can pretend to be a file object well enough to be
    used to build objects to intercept console output and determine the
    progress of some operation.
    """
    def __init__(self, d, outfile=None):
        self._progress = 0
        self._data = d
        self._lastevent = 0
        if outfile:
            self._outfile = outfile
        else:
            self._outfile = sys.stdout

    def _fire_progress(self, taskprogress, rate=None):
        """Internal function to fire the progress event"""
        bb.event.fire(bb.build.TaskProgress(taskprogress, rate), self._data)

    def write(self, string):
        self._outfile.write(string)

    def flush(self):
        self._outfile.flush()

    def update(self, progress, rate=None):
        ts = time.time()
        if progress > 100:
            progress = 100
        if progress != self._progress or self._lastevent + 1 < ts:
            self._fire_progress(progress, rate)
            self._lastevent = ts
            self._progress = progress

class LineFilterProgressHandler(ProgressHandler):
    """
    A ProgressHandler variant that provides the ability to filter out
    the lines if they contain progress information. Additionally, it
    filters out anything before the last line feed on a line. This can
    be used to keep the logs clean of output that we've only enabled for
    getting progress, assuming that that can be done on a per-line
    basis.
    """
    def __init__(self, d, outfile=None):
        self._linebuffer = ''
        super(LineFilterProgressHandler, self).__init__(d, outfile)

    def write(self, string):
        self._linebuffer += string
        while True:
            breakpos = self._linebuffer.find('\n') + 1
            if breakpos == 0:
                break
            line = self._linebuffer[:breakpos]
            self._linebuffer = self._linebuffer[breakpos:]
            # Drop any line feeds and anything that precedes them
            lbreakpos = line.rfind('\r') + 1
            if lbreakpos:
                line = line[lbreakpos:]
            if self.writeline(line):
                super(LineFilterProgressHandler, self).write(line)

    def writeline(self, line):
        return True

class BasicProgressHandler(ProgressHandler):
    def __init__(self, d, regex=r'(\d+)%', outfile=None):
        super(BasicProgressHandler, self).__init__(d, outfile)
        self._regex = re.compile(regex)
        # Send an initial progress event so the bar gets shown
        self._fire_progress(0)

    def write(self, string):
        percs = self._regex.findall(string)
        if percs:
            progress = int(percs[-1])
            self.update(progress)
        super(BasicProgressHandler, self).write(string)

class OutOfProgressHandler(ProgressHandler):
    def __init__(self, d, regex, outfile=None):
        super(OutOfProgressHandler, self).__init__(d, outfile)
        self._regex = re.compile(regex)
        # Send an initial progress event so the bar gets shown
        self._fire_progress(0)

    def write(self, string):
        nums = self._regex.findall(string)
        if nums:
            progress = (float(nums[-1][0]) / float(nums[-1][1])) * 100
            self.update(progress)
        super(OutOfProgressHandler, self).write(string)

class MultiStageProgressReporter(object):
    """
    Class which allows reporting progress without the caller
    having to know where they are in the overall sequence. Useful
    for tasks made up of python code spread across multiple
    classes / functions - the progress reporter object can
    be passed around or stored at the object level and calls
    to next_stage() and update() made whereever needed.
    """
    def __init__(self, d, stage_weights, debug=False):
        """
        Initialise the progress reporter.

        Parameters:
        * d: the datastore (needed for firing the events)
        * stage_weights: a list of weight values, one for each stage.
          The value is scaled internally so you only need to specify
          values relative to other values in the list, so if there
          are two stages and the first takes 2s and the second takes
          10s you would specify [2, 10] (or [1, 5], it doesn't matter).
        * debug: specify True (and ensure you call finish() at the end)
          in order to show a printout of the calculated stage weights
          based on timing each stage. Use this to determine what the
          weights should be when you're not sure.
        """
        self._data = d
        total = sum(stage_weights)
        self._stage_weights = [float(x)/total for x in stage_weights]
        self._stage = -1
        self._base_progress = 0
        # Send an initial progress event so the bar gets shown
        self._fire_progress(0)
        self._debug = debug
        self._finished = False
        if self._debug:
            self._last_time = time.time()
            self._stage_times = []
            self._stage_total = None
            self._callers = []

    def _fire_progress(self, taskprogress):
        bb.event.fire(bb.build.TaskProgress(taskprogress), self._data)

    def next_stage(self, stage_total=None):
        """
        Move to the next stage.
        Parameters:
        * stage_total: optional total for progress within the stage,
          see update() for details
        NOTE: you need to call this before the first stage.
        """
        self._stage += 1
        self._stage_total = stage_total
        if self._stage == 0:
            # First stage
            if self._debug:
                self._last_time = time.time()
        else:
            if self._stage < len(self._stage_weights):
                self._base_progress = sum(self._stage_weights[:self._stage]) * 100
                if self._debug:
                    currtime = time.time()
                    self._stage_times.append(currtime - self._last_time)
                    self._last_time = currtime
                    self._callers.append(inspect.getouterframes(inspect.currentframe())[1])
            elif not self._debug:
                bb.warn('ProgressReporter: current stage beyond declared number of stages')
                self._base_progress = 100
            self._fire_progress(self._base_progress)

    def update(self, stage_progress):
        """
        Update progress within the current stage.
        Parameters:
        * stage_progress: progress value within the stage. If stage_total
          was specified when next_stage() was last called, then this
          value is considered to be out of stage_total, otherwise it should
          be a percentage value from 0 to 100.
        """
        if self._stage_total:
            stage_progress = (float(stage_progress) / self._stage_total) * 100
        if self._stage < 0:
            bb.warn('ProgressReporter: update called before first call to next_stage()')
        elif self._stage < len(self._stage_weights):
            progress = self._base_progress + (stage_progress * self._stage_weights[self._stage])
        else:
            progress = self._base_progress
        if progress > 100:
            progress = 100
        self._fire_progress(progress)

    def finish(self):
        if self._finished:
            return
        self._finished = True
        if self._debug:
            import math
            self._stage_times.append(time.time() - self._last_time)
            mintime = max(min(self._stage_times), 0.01)
            self._callers.append(None)
            stage_weights = [int(math.ceil(x / mintime)) for x in self._stage_times]
            bb.warn('Stage weights: %s' % stage_weights)
            out = []
            for stage_weight, caller in zip(stage_weights, self._callers):
                if caller:
                    out.append('Up to %s:%d: %d' % (caller[1], caller[2], stage_weight))
                else:
                    out.append('Up to finish: %d' % stage_weight)
            bb.warn('Stage times:\n  %s' % '\n  '.join(out))

class MultiStageProcessProgressReporter(MultiStageProgressReporter):
    """
    Version of MultiStageProgressReporter intended for use with
    standalone processes (such as preparing the runqueue)
    """
    def __init__(self, d, processname, stage_weights, debug=False):
        self._processname = processname
        self._started = False
        MultiStageProgressReporter.__init__(self, d, stage_weights, debug)

    def start(self):
        if not self._started:
            bb.event.fire(bb.event.ProcessStarted(self._processname, 100), self._data)
            self._started = True

    def _fire_progress(self, taskprogress):
        if taskprogress == 0:
            self.start()
            return
        bb.event.fire(bb.event.ProcessProgress(self._processname, taskprogress), self._data)

    def finish(self):
        MultiStageProgressReporter.finish(self)
        bb.event.fire(bb.event.ProcessFinished(self._processname), self._data)

class DummyMultiStageProcessProgressReporter(MultiStageProgressReporter):
    """
    MultiStageProcessProgressReporter that takes the calls and does nothing
    with them (to avoid a bunch of "if progress_reporter:" checks)
    """
    def __init__(self):
        MultiStageProcessProgressReporter.__init__(self, "", None, [])

    def _fire_progress(self, taskprogress, rate=None):
        pass

    def start(self):
        pass

    def next_stage(self, stage_total=None):
        pass

    def update(self, stage_progress):
        pass

    def finish(self):
        pass