1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
|
# Base class to be used by all test cases defined in the suite
#
# Copyright (C) 2016 Intel Corporation
#
# SPDX-License-Identifier: GPL-2.0-only
import unittest
import logging
import json
import unidiff
from data import PatchTestInput
import mailbox
import collections
import sys
import os
import re
sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'pyparsing'))
logger = logging.getLogger('patchtest')
debug=logger.debug
info=logger.info
warn=logger.warn
error=logger.error
Commit = collections.namedtuple('Commit', ['author', 'subject', 'commit_message', 'shortlog', 'payload'])
class PatchtestOEError(Exception):
"""Exception for handling patchtest-oe errors"""
def __init__(self, message, exitcode=1):
super().__init__(message)
self.exitcode = exitcode
class Base(unittest.TestCase):
# if unit test fails, fail message will throw at least the following JSON: {"id": <testid>}
endcommit_messages_regex = re.compile(r'\(From \w+-\w+ rev:|(?<!\S)Signed-off-by|(?<!\S)---\n')
patchmetadata_regex = re.compile(r'-{3} \S+|\+{3} \S+|@{2} -\d+,\d+ \+\d+,\d+ @{2} \S+')
@staticmethod
def msg_to_commit(msg):
payload = msg.get_payload()
return Commit(subject=msg['subject'].replace('\n', ' ').replace(' ', ' '),
author=msg.get('From'),
shortlog=Base.shortlog(msg['subject']),
commit_message=Base.commit_message(payload),
payload=payload)
@staticmethod
def commit_message(payload):
commit_message = payload.__str__()
match = Base.endcommit_messages_regex.search(payload)
if match:
commit_message = payload[:match.start()]
return commit_message
@staticmethod
def shortlog(shlog):
# remove possible prefix (between brackets) before colon
start = shlog.find(']', 0, shlog.find(':'))
# remove also newlines and spaces at both sides
return shlog[start + 1:].replace('\n', '').strip()
@classmethod
def setUpClass(cls):
# General objects: mailbox.mbox and patchset
cls.mbox = mailbox.mbox(PatchTestInput.repo.patch)
# Patch may be malformed, so try parsing it
cls.unidiff_parse_error = ''
cls.patchset = None
try:
cls.patchset = unidiff.PatchSet.from_filename(PatchTestInput.repo.patch, encoding=u'UTF-8')
except unidiff.UnidiffParseError as upe:
cls.patchset = []
cls.unidiff_parse_error = str(upe)
# Easy to iterate list of commits
cls.commits = []
for msg in cls.mbox:
if msg['subject'] and msg.get_payload():
cls.commits.append(Base.msg_to_commit(msg))
cls.setUpClassLocal()
@classmethod
def tearDownClass(cls):
cls.tearDownClassLocal()
@classmethod
def setUpClassLocal(cls):
pass
@classmethod
def tearDownClassLocal(cls):
pass
def fail(self, issue, fix=None, commit=None, data=None):
""" Convert to a JSON string failure data"""
value = {'id': self.id(),
'issue': issue}
if fix:
value['fix'] = fix
if commit:
value['commit'] = {'subject': commit.subject,
'shortlog': commit.shortlog}
# extend return value with other useful info
if data:
value['data'] = data
return super(Base, self).fail(json.dumps(value))
def skip(self, issue, data=None):
""" Convert the skip string to JSON"""
value = {'id': self.id(),
'issue': issue}
# extend return value with other useful info
if data:
value['data'] = data
return super(Base, self).skipTest(json.dumps(value))
def shortid(self):
return self.id().split('.')[-1]
def __str__(self):
return json.dumps({'id': self.id()})
class Metadata(Base):
@classmethod
def setUpClassLocal(cls):
cls.tinfoil = cls.setup_tinfoil()
# get info about added/modified/remove recipes
cls.added, cls.modified, cls.removed = cls.get_metadata_stats(cls.patchset)
@classmethod
def tearDownClassLocal(cls):
cls.tinfoil.shutdown()
@classmethod
def setup_tinfoil(cls, config_only=False):
"""Initialize tinfoil api from bitbake"""
# import relevant libraries
try:
scripts_path = os.path.join(PatchTestInput.repodir, 'scripts', 'lib')
if scripts_path not in sys.path:
sys.path.insert(0, scripts_path)
import scriptpath
scriptpath.add_bitbake_lib_path()
import bb.tinfoil
except ImportError:
raise PatchtestOEError('Could not import tinfoil module')
orig_cwd = os.path.abspath(os.curdir)
# Load tinfoil
tinfoil = None
try:
builddir = os.environ.get('BUILDDIR')
if not builddir:
logger.warn('Bitbake environment not loaded?')
return tinfoil
os.chdir(builddir)
tinfoil = bb.tinfoil.Tinfoil()
tinfoil.prepare(config_only=config_only)
except bb.tinfoil.TinfoilUIException as te:
if tinfoil:
tinfoil.shutdown()
raise PatchtestOEError('Could not prepare properly tinfoil (TinfoilUIException)')
except Exception as e:
if tinfoil:
tinfoil.shutdown()
raise e
finally:
os.chdir(orig_cwd)
return tinfoil
@classmethod
def get_metadata_stats(cls, patchset):
"""Get lists of added, modified and removed metadata files"""
def find_pn(data, path):
"""Find the PN from data"""
pn = None
pn_native = None
for _path, _pn in data:
if path in _path:
if 'native' in _pn:
# store the native PN but look for the non-native one first
pn_native = _pn
else:
pn = _pn
break
else:
# sent the native PN if found previously
if pn_native:
return pn_native
# on renames (usually upgrades), we need to check (FILE) base names
# because the unidiff library does not provided the new filename, just the modified one
# and tinfoil datastore, once the patch is merged, will contain the new filename
path_basename = path.split('_')[0]
for _path, _pn in data:
_path_basename = _path.split('_')[0]
if path_basename == _path_basename:
pn = _pn
return pn
if not cls.tinfoil:
cls.tinfoil = cls.setup_tinfoil()
added_paths, modified_paths, removed_paths = [], [], []
added, modified, removed = [], [], []
# get metadata filename additions, modification and removals
for patch in patchset:
if patch.path.endswith('.bb') or patch.path.endswith('.bbappend') or patch.path.endswith('.inc'):
if patch.is_added_file:
added_paths.append(os.path.join(os.path.abspath(PatchTestInput.repodir), patch.path))
elif patch.is_modified_file:
modified_paths.append(os.path.join(os.path.abspath(PatchTestInput.repodir), patch.path))
elif patch.is_removed_file:
removed_paths.append(os.path.join(os.path.abspath(PatchTestInput.repodir), patch.path))
data = cls.tinfoil.cooker.recipecaches[''].pkg_fn.items()
added = [find_pn(data,path) for path in added_paths]
modified = [find_pn(data,path) for path in modified_paths]
removed = [find_pn(data,path) for path in removed_paths]
return [a for a in added if a], [m for m in modified if m], [r for r in removed if r]
|
