summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/toaster/toastergui/widgets.py
blob: db5c3aa00b7b75519eb2881a095acc7f4951707f (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
#
# ex:ts=4:sw=4:sts=4:et
# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
#
# BitBake Toaster Implementation
#
# Copyright (C) 2015        Intel Corporation
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

from django.views.generic import View, TemplateView
from django.views.decorators.cache import cache_control
from django.shortcuts import HttpResponse
from django.core.cache import cache
from django.core.paginator import Paginator, EmptyPage
from django.db.models import Q
from orm.models import Project, Build
from django.template import Context, Template
from django.template import VariableDoesNotExist
from django.template import TemplateSyntaxError
from django.core.serializers.json import DjangoJSONEncoder
from django.core.exceptions import FieldError
from django.utils import timezone
from toastergui.templatetags.projecttags import sectohms, get_tasks
from toastergui.templatetags.projecttags import json as template_json
from django.http import JsonResponse
from django.core.urlresolvers import reverse

import types
import json
import collections
import re
import os

from toastergui.tablefilter import TableFilterMap

try:
    from urllib import unquote_plus
except ImportError:
    from urllib.parse import unquote_plus

import logging
logger = logging.getLogger("toaster")


class NoFieldOrDataName(Exception):
    pass


class ToasterTable(TemplateView):
    def __init__(self, *args, **kwargs):
        super(ToasterTable, self).__init__()
        if 'template_name' in kwargs:
            self.template_name = kwargs['template_name']
        self.title = "Table"
        self.queryset = None
        self.columns = []

        # map from field names to Filter instances
        self.filter_map = TableFilterMap()

        self.total_count = 0
        self.static_context_extra = {}
        self.empty_state = "Sorry - no data found"
        self.default_orderby = ""

    # prevent HTTP caching of table data
    @cache_control(must_revalidate=True,
                   max_age=0, no_store=True, no_cache=True)
    def dispatch(self, *args, **kwargs):
        return super(ToasterTable, self).dispatch(*args, **kwargs)

    def get_context_data(self, **kwargs):
        context = super(ToasterTable, self).get_context_data(**kwargs)
        context['title'] = self.title
        context['table_name'] = type(self).__name__.lower()
        context['empty_state'] = self.empty_state

        # global variables
        context['project_enable'] = ('1' == os.environ.get('TOASTER_BUILDSERVER'))
        try:
            context['project_specific'] = ('1' == os.environ.get('TOASTER_PROJECTSPECIFIC'))
        except:
            context['project_specific'] = ''

        return context

    def get(self, request, *args, **kwargs):
        if request.GET.get('format', None) == 'json':

            self.setup_queryset(*args, **kwargs)
            # Put the project id into the context for the static_data_template
            if 'pid' in kwargs:
                self.static_context_extra['pid'] = kwargs['pid']

            cmd = request.GET.get('cmd', None)
            if cmd and 'filterinfo' in cmd:
                data = self.get_filter_info(request, **kwargs)
            else:
                # If no cmd is specified we give you the table data
                data = self.get_data(request, **kwargs)

            return HttpResponse(data, content_type="application/json")

        return super(ToasterTable, self).get(request, *args, **kwargs)

    def get_filter_info(self, request, **kwargs):
        self.setup_filters(**kwargs)

        search = request.GET.get("search", None)
        if search:
            self.apply_search(search)

        name = request.GET.get("name", None)
        table_filter = self.filter_map.get_filter(name)
        return json.dumps(table_filter.to_json(self.queryset),
                          indent=2,
                          cls=DjangoJSONEncoder)

    def setup_columns(self, *args, **kwargs):
        """ function to implement in the subclass which sets up
        the columns """
        pass

    def setup_filters(self, *args, **kwargs):
        """ function to implement in the subclass which sets up the
        filters """
        pass

    def setup_queryset(self, *args, **kwargs):
        """ function to implement in the subclass which sets up the
        queryset"""
        pass

    def add_filter(self, table_filter):
        """Add a filter to the table.

        Args:
            table_filter: Filter instance
        """
        self.filter_map.add_filter(table_filter.name, table_filter)

    def add_column(self, title="", help_text="",
                   orderable=False, hideable=True, hidden=False,
                   field_name="", filter_name=None, static_data_name=None,
                   static_data_template=None):
        """Add a column to the table.

        Args:
            title (str): Title for the table header
            help_text (str): Optional help text to describe the column
            orderable (bool): Whether the column can be ordered.
                We order on the field_name.
            hideable (bool): Whether the user can hide the column
            hidden (bool): Whether the column is default hidden
            field_name (str or list): field(s) required for this column's data
            static_data_name (str, optional): The column's main identifier
                which will replace the field_name.
            static_data_template(str, optional): The template to be rendered
                as data
        """

        self.columns.append({'title': title,
                             'help_text': help_text,
                             'orderable': orderable,
                             'hideable': hideable,
                             'hidden': hidden,
                             'field_name': field_name,
                             'filter_name': filter_name,
                             'static_data_name': static_data_name,
                             'static_data_template': static_data_template})

    def set_column_hidden(self, title, hidden):
        """
        Set the hidden state of the column to the value of hidden
        """
        for col in self.columns:
            if col['title'] == title:
                col['hidden'] = hidden
                break

    def set_column_hideable(self, title, hideable):
        """
        Set the hideable state of the column to the value of hideable
        """
        for col in self.columns:
            if col['title'] == title:
                col['hideable'] = hideable
                break

    def render_static_data(self, template, row):
        """Utility function to render the static data template"""

        context = {
          'extra': self.static_context_extra,
          'data': row,
        }

        context = Context(context)
        template = Template(template)

        return template.render(context)

    def apply_filter(self, filters, filter_value, **kwargs):
        """
        Apply a filter submitted in the querystring to the ToasterTable

        filters: (str) in the format:
          '<filter name>:<action name>'
        filter_value: (str) parameters to pass to the named filter

        <filter name> and <action name> are used to look up the correct filter
        in the ToasterTable's filter map; the <action params> are set on
        TableFilterAction* before its filter is applied and may modify the
        queryset returned by the filter
        """
        self.setup_filters(**kwargs)

        try:
            filter_name, action_name = filters.split(':')
            action_params = unquote_plus(filter_value)
        except ValueError:
            return

        if "all" in action_name:
            return

        try:
            table_filter = self.filter_map.get_filter(filter_name)
            action = table_filter.get_action(action_name)
            action.set_filter_params(action_params)
            self.queryset = action.filter(self.queryset)
        except KeyError:
            # pass it to the user - programming error here
            raise

    def apply_orderby(self, orderby):
        # Note that django will execute this when we try to retrieve the data
        self.queryset = self.queryset.order_by(orderby)

    def apply_search(self, search_term):
        """Creates a query based on the model's search_allowed_fields"""

        if not hasattr(self.queryset.model, 'search_allowed_fields'):
            raise Exception("Search fields aren't defined in the model %s"
                            % self.queryset.model)

        search_queries = None
        for st in search_term.split(" "):
            queries = None
            for field in self.queryset.model.search_allowed_fields:
                query = Q(**{field + '__icontains': st})
                if queries:
                    queries |= query
                else:
                    queries = query

            if search_queries:
                search_queries &= queries
            else:
                search_queries = queries

        self.queryset = self.queryset.filter(search_queries)

    def get_data(self, request, **kwargs):
        """
        Returns the data for the page requested with the specified
        parameters applied

        filters: filter and action name, e.g. "outcome:build_succeeded"
        filter_value: value to pass to the named filter+action, e.g. "on"
        (for a toggle filter) or "2015-12-11,2015-12-12"
        (for a date range filter)
        """

        page_num = request.GET.get("page", 1)
        limit = request.GET.get("limit", 10)
        search = request.GET.get("search", None)
        filters = request.GET.get("filter", None)
        filter_value = request.GET.get("filter_value", "on")
        orderby = request.GET.get("orderby", None)
        nocache = request.GET.get("nocache", None)

        # Make a unique cache name
        cache_name = self.__class__.__name__

        for key, val in request.GET.items():
            if key == 'nocache':
                continue
            cache_name = cache_name + str(key) + str(val)

        for key, val in kwargs.items():
            cache_name = cache_name + str(key) + str(val)

        # No special chars allowed in the cache name apart from dash
        cache_name = re.sub(r'[^A-Za-z0-9-]', "", cache_name)

        if nocache:
            cache.delete(cache_name)

        data = cache.get(cache_name)

        if data:
            logger.debug("Got cache data for table '%s'" % self.title)
            return data

        self.setup_columns(**kwargs)

        if search:
            self.apply_search(search)
        if filters:
            self.apply_filter(filters, filter_value, **kwargs)
        if orderby:
            self.apply_orderby(orderby)

        paginator = Paginator(self.queryset, limit)

        try:
            page = paginator.page(page_num)
        except EmptyPage:
            page = paginator.page(1)

        data = {
            'total': self.queryset.count(),
            'default_orderby': self.default_orderby,
            'columns': self.columns,
            'rows': [],
            'error': "ok",
        }

        try:
            for model_obj in page.object_list:
                # Use collection to maintain the order
                required_data = collections.OrderedDict()

                for col in self.columns:
                    field = col['field_name']
                    if not field:
                        field = col['static_data_name']
                    if not field:
                        raise NoFieldOrDataName("Must supply a field_name or"
                                                "static_data_name for column"
                                                "%s.%s" %
                                                (self.__class__.__name__, col)
                                                )

                    # Check if we need to process some static data
                    if "static_data_name" in col and col['static_data_name']:
                        # Overwrite the field_name with static_data_name
                        # so that this can be used as the html class name
                        col['field_name'] = col['static_data_name']

                        try:
                            # Render the template given
                            required_data[col['static_data_name']] = \
                                    self.render_static_data(
                                        col['static_data_template'], model_obj)
                        except (TemplateSyntaxError,
                                VariableDoesNotExist) as e:
                            logger.error("could not render template code"
                                         "%s %s %s",
                                         col['static_data_template'],
                                         e, self.__class__.__name__)
                            required_data[col['static_data_name']] =\
                                '<!--error-->'

                    else:
                        # Traverse to any foriegn key in the field
                        # e.g. recipe__layer_version__name
                        model_data = None

                        if "__" in field:
                            for subfield in field.split("__"):
                                if not model_data:
                                    # The first iteration is always going to
                                    # be on the actual model object instance.
                                    # Subsequent ones are on the result of
                                    # that. e.g. forieng key objects
                                    model_data = getattr(model_obj,
                                                         subfield)
                                else:
                                    model_data = getattr(model_data,
                                                         subfield)

                        else:
                            model_data = getattr(model_obj,
                                                 col['field_name'])

                        # We might have a model function as the field so
                        # call it to return the data needed
                        if isinstance(model_data, types.MethodType):
                            model_data = model_data()

                        required_data[col['field_name']] = model_data

                data['rows'].append(required_data)

        except FieldError:
            # pass  it to the user - programming-error here
            raise

        data = json.dumps(data, indent=2, cls=DjangoJSONEncoder)
        cache.set(cache_name, data, 60*30)

        return data


class ToasterTypeAhead(View):
    """ A typeahead mechanism to support the front end typeahead widgets """
    MAX_RESULTS = 6

    class MissingFieldsException(Exception):
        pass

    def __init__(self, *args, **kwargs):
        super(ToasterTypeAhead, self).__init__()

    def get(self, request, *args, **kwargs):
        def response(data):
            return HttpResponse(json.dumps(data,
                                           indent=2,
                                           cls=DjangoJSONEncoder),
                                content_type="application/json")

        error = "ok"

        search_term = request.GET.get("search", None)
        if search_term is None:
            # We got no search value so return empty reponse
            return response({'error': error, 'results': []})

        try:
            prj = Project.objects.get(pk=kwargs['pid'])
        except KeyError:
            prj = None

        results = self.apply_search(search_term,
                                    prj,
                                    request)[:ToasterTypeAhead.MAX_RESULTS]

        if len(results) > 0:
            try:
                self.validate_fields(results[0])
            except self.MissingFieldsException as e:
                error = e

        data = {'results': results,
                'error': error}

        return response(data)

    def validate_fields(self, result):
        if 'name' in result is False or 'detail' in result is False:
            raise self.MissingFieldsException(
                "name and detail are required fields")

    def apply_search(self, search_term, prj):
        """ Override this function to implement search. Return an array of
        dictionaries with a minium of a name and detail field"""
        pass


class MostRecentBuildsView(View):
    def _was_yesterday_or_earlier(self, completed_on):
        now = timezone.now()
        delta = now - completed_on

        if delta.days >= 1:
            return True

        return False

    def get(self, request, *args, **kwargs):
        """
        Returns a list of builds in JSON format.
        """
        project = None

        project_id = request.GET.get('project_id', None)
        if project_id:
            try:
                project = Project.objects.get(pk=project_id)
            except:
                # if project lookup fails, assume no project
                pass

        recent_build_objs = Build.get_recent(project)
        recent_builds = []

        for build_obj in recent_build_objs:
            dashboard_url = reverse('builddashboard', args=(build_obj.pk,))
            buildtime_url = reverse('buildtime', args=(build_obj.pk,))
            rebuild_url = \
                reverse('xhr_buildrequest', args=(build_obj.project.pk,))
            cancel_url = \
                reverse('xhr_buildrequest', args=(build_obj.project.pk,))

            build = {}
            build['id'] = build_obj.pk
            build['dashboard_url'] = dashboard_url

            buildrequest_id = None
            if hasattr(build_obj, 'buildrequest'):
                buildrequest_id = build_obj.buildrequest.pk
            build['buildrequest_id'] = buildrequest_id

            if build_obj.recipes_to_parse > 0:
                build['recipes_parsed_percentage'] = \
                    int((build_obj.recipes_parsed /
                         build_obj.recipes_to_parse) * 100)
            else:
                build['recipes_parsed_percentage'] = 0
            if build_obj.repos_to_clone > 0:
                build['repos_cloned_percentage'] = \
                    int((build_obj.repos_cloned /
                         build_obj.repos_to_clone) * 100)
            else:
                build['repos_cloned_percentage'] = 0

            build['progress_item'] = build_obj.progress_item

            tasks_complete_percentage = 0
            if build_obj.outcome in (Build.SUCCEEDED, Build.FAILED):
                tasks_complete_percentage = 100
            elif build_obj.outcome == Build.IN_PROGRESS:
                tasks_complete_percentage = build_obj.completeper()
            build['tasks_complete_percentage'] = tasks_complete_percentage

            build['state'] = build_obj.get_state()

            build['errors'] = build_obj.errors.count()
            build['dashboard_errors_url'] = dashboard_url + '#errors'

            build['warnings'] = build_obj.warnings.count()
            build['dashboard_warnings_url'] = dashboard_url + '#warnings'

            build['buildtime'] = sectohms(build_obj.timespent_seconds)
            build['buildtime_url'] = buildtime_url

            build['rebuild_url'] = rebuild_url
            build['cancel_url'] = cancel_url

            build['is_default_project_build'] = build_obj.project.is_default

            build['build_targets_json'] = \
                template_json(get_tasks(build_obj.target_set.all()))

            # convert completed_on time to user's timezone
            completed_on = timezone.localtime(build_obj.completed_on)

            completed_on_template = '%H:%M'
            if self._was_yesterday_or_earlier(completed_on):
                completed_on_template = '%d/%m/%Y ' + completed_on_template
            build['completed_on'] = completed_on.strftime(
                completed_on_template)

            targets = []
            target_objs = build_obj.get_sorted_target_list()
            for target_obj in target_objs:
                if target_obj.task:
                    targets.append(target_obj.target + ':' + target_obj.task)
                else:
                    targets.append(target_obj.target)
            build['targets'] = ' '.join(targets)

            # abbreviated form of the full target list
            abbreviated_targets = ''
            num_targets = len(targets)
            if num_targets > 0:
                abbreviated_targets = targets[0]
            if num_targets > 1:
                abbreviated_targets += (' +%s' % (num_targets - 1))
            build['targets_abbreviated'] = abbreviated_targets

            recent_builds.append(build)

        return JsonResponse(recent_builds, safe=False)