summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/providers.py
blob: e11a4637d16300fb7c5da645493fbd9b83366f5e (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
#
# Copyright (C) 2003, 2004  Chris Larson
# Copyright (C) 2003, 2004  Phil Blundell
# Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
# Copyright (C) 2005        Holger Hans Peter Freyther
# Copyright (C) 2005        ROAD GmbH
# Copyright (C) 2006        Richard Purdie
#
# SPDX-License-Identifier: GPL-2.0-only
#

import re
import logging
from bb import data, utils
from collections import defaultdict
import bb

logger = logging.getLogger("BitBake.Provider")

class NoProvider(bb.BBHandledException):
    """Exception raised when no provider of a build dependency can be found"""

class NoRProvider(bb.BBHandledException):
    """Exception raised when no provider of a runtime dependency can be found"""

class MultipleRProvider(bb.BBHandledException):
    """Exception raised when multiple providers of a runtime dependency can be found"""

def findProviders(cfgData, dataCache, pkg_pn = None):
    """
    Convenience function to get latest and preferred providers in pkg_pn
    """

    if not pkg_pn:
        pkg_pn = dataCache.pkg_pn

    # Need to ensure data store is expanded
    localdata = data.createCopy(cfgData)
    bb.data.expandKeys(localdata)

    required = {}
    preferred_versions = {}
    latest_versions = {}

    for pn in pkg_pn:
        (last_ver, last_file, pref_ver, pref_file, req) = findBestProvider(pn, localdata, dataCache, pkg_pn)
        preferred_versions[pn] = (pref_ver, pref_file)
        latest_versions[pn] = (last_ver, last_file)
        required[pn] = req

    return (latest_versions, preferred_versions, required)

def allProviders(dataCache):
    """
    Find all providers for each pn
    """
    all_providers = defaultdict(list)
    for (fn, pn) in dataCache.pkg_fn.items():
        ver = dataCache.pkg_pepvpr[fn]
        all_providers[pn].append((ver, fn))
    return all_providers

def sortPriorities(pn, dataCache, pkg_pn = None):
    """
    Reorder pkg_pn by file priority and default preference
    """

    if not pkg_pn:
        pkg_pn = dataCache.pkg_pn

    files = pkg_pn[pn]
    priorities = {}
    for f in files:
        priority = dataCache.bbfile_priority[f]
        preference = dataCache.pkg_dp[f]
        if priority not in priorities:
            priorities[priority] = {}
        if preference not in priorities[priority]:
            priorities[priority][preference] = []
        priorities[priority][preference].append(f)
    tmp_pn = []
    for pri in sorted(priorities):
        tmp_pref = []
        for pref in sorted(priorities[pri]):
            tmp_pref.extend(priorities[pri][pref])
        tmp_pn = [tmp_pref] + tmp_pn

    return tmp_pn

def versionVariableMatch(cfgData, keyword, pn):
    """
    Return the value of the <keyword>_VERSION variable if set.
    """

    # pn can contain '_', e.g. gcc-cross-x86_64 and an override cannot
    # hence we do this manually rather than use OVERRIDES
    ver = cfgData.getVar("%s_VERSION:pn-%s" % (keyword, pn))
    if not ver:
        ver = cfgData.getVar("%s_VERSION_%s" % (keyword, pn))
    if not ver:
        ver = cfgData.getVar("%s_VERSION" % keyword)

    return ver

def preferredVersionMatch(pe, pv, pr, preferred_e, preferred_v, preferred_r):
    """
    Check if the version pe,pv,pr is the preferred one.
    If there is preferred version defined and ends with '%', then pv has to start with that version after removing the '%'
    """
    if pr == preferred_r or preferred_r is None:
        if pe == preferred_e or preferred_e is None:
            if preferred_v == pv:
                return True
            if preferred_v is not None and preferred_v.endswith('%') and pv.startswith(preferred_v[:len(preferred_v)-1]):
                return True
    return False

def findPreferredProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
    """
    Find the first provider in pkg_pn with REQUIRED_VERSION or PREFERRED_VERSION set.
    """

    preferred_file = None
    preferred_ver = None
    required = False

    required_v = versionVariableMatch(cfgData, "REQUIRED", pn)
    preferred_v = versionVariableMatch(cfgData, "PREFERRED", pn)

    itemstr = ""
    if item:
        itemstr = " (for item %s)" % item

    if required_v is not None:
        if preferred_v is not None:
            logger.warning("REQUIRED_VERSION and PREFERRED_VERSION for package %s%s are both set using REQUIRED_VERSION %s", pn, itemstr, required_v)
        else:
            logger.debug("REQUIRED_VERSION is set for package %s%s", pn, itemstr)
        # REQUIRED_VERSION always takes precedence over PREFERRED_VERSION
        preferred_v = required_v
        required = True

    if preferred_v:
        m = re.match(r'(\d+:)*(.*)(_.*)*', preferred_v)
        if m:
            if m.group(1):
                preferred_e = m.group(1)[:-1]
            else:
                preferred_e = None
            preferred_v = m.group(2)
            if m.group(3):
                preferred_r = m.group(3)[1:]
            else:
                preferred_r = None
        else:
            preferred_e = None
            preferred_r = None

        for file_set in pkg_pn:
            for f in file_set:
                pe, pv, pr = dataCache.pkg_pepvpr[f]
                if preferredVersionMatch(pe, pv, pr, preferred_e, preferred_v, preferred_r):
                    preferred_file = f
                    preferred_ver = (pe, pv, pr)
                    break
            if preferred_file:
                break;
        if preferred_r:
            pv_str = '%s-%s' % (preferred_v, preferred_r)
        else:
            pv_str = preferred_v
        if not (preferred_e is None):
            pv_str = '%s:%s' % (preferred_e, pv_str)
        if preferred_file is None:
            if not required:
                logger.warning("preferred version %s of %s not available%s", pv_str, pn, itemstr)
            available_vers = []
            for file_set in pkg_pn:
                for f in file_set:
                    pe, pv, pr = dataCache.pkg_pepvpr[f]
                    ver_str = pv
                    if pe:
                        ver_str = "%s:%s" % (pe, ver_str)
                    if not ver_str in available_vers:
                        available_vers.append(ver_str)
            if available_vers:
                available_vers.sort()
                logger.warning("versions of %s available: %s", pn, ' '.join(available_vers))
            if required:
                logger.error("required version %s of %s not available%s", pv_str, pn, itemstr)
        else:
            if required:
                logger.debug("selecting %s as REQUIRED_VERSION %s of package %s%s", preferred_file, pv_str, pn, itemstr)
            else:
                logger.debug("selecting %s as PREFERRED_VERSION %s of package %s%s", preferred_file, pv_str, pn, itemstr)

    return (preferred_ver, preferred_file, required)

def findLatestProvider(pn, cfgData, dataCache, file_set):
    """
    Return the highest version of the providers in file_set.
    Take default preferences into account.
    """
    latest = None
    latest_p = 0
    latest_f = None
    for file_name in file_set:
        pe, pv, pr = dataCache.pkg_pepvpr[file_name]
        dp = dataCache.pkg_dp[file_name]

        if (latest is None) or ((latest_p == dp) and (utils.vercmp(latest, (pe, pv, pr)) < 0)) or (dp > latest_p):
            latest = (pe, pv, pr)
            latest_f = file_name
            latest_p = dp

    return (latest, latest_f)

def findBestProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
    """
    If there is a PREFERRED_VERSION, find the highest-priority bbfile
    providing that version.  If not, find the latest version provided by
    an bbfile in the highest-priority set.
    """

    sortpkg_pn = sortPriorities(pn, dataCache, pkg_pn)
    # Find the highest priority provider with a REQUIRED_VERSION or PREFERRED_VERSION set
    (preferred_ver, preferred_file, required) = findPreferredProvider(pn, cfgData, dataCache, sortpkg_pn, item)
    # Find the latest version of the highest priority provider
    (latest, latest_f) = findLatestProvider(pn, cfgData, dataCache, sortpkg_pn[0])

    if not required and preferred_file is None:
        preferred_file = latest_f
        preferred_ver = latest

    return (latest, latest_f, preferred_ver, preferred_file, required)

def _filterProviders(providers, item, cfgData, dataCache):
    """
    Take a list of providers and filter/reorder according to the
    environment variables
    """
    eligible = []
    preferred_versions = {}
    sortpkg_pn = {}

    # The order of providers depends on the order of the files on the disk
    # up to here. Sort pkg_pn to make dependency issues reproducible rather
    # than effectively random.
    providers.sort()

    # Collate providers by PN
    pkg_pn = {}
    for p in providers:
        pn = dataCache.pkg_fn[p]
        if pn not in pkg_pn:
            pkg_pn[pn] = []
        pkg_pn[pn].append(p)

    logger.debug("providers for %s are: %s", item, list(sorted(pkg_pn.keys())))

    # First add REQUIRED_VERSIONS or PREFERRED_VERSIONS
    for pn in sorted(pkg_pn):
        sortpkg_pn[pn] = sortPriorities(pn, dataCache, pkg_pn)
        preferred_ver, preferred_file, required = findPreferredProvider(pn, cfgData, dataCache, sortpkg_pn[pn], item)
        if required and preferred_file is None:
            return eligible
        preferred_versions[pn] = (preferred_ver, preferred_file)
        if preferred_versions[pn][1]:
            eligible.append(preferred_versions[pn][1])

    # Now add latest versions
    for pn in sorted(sortpkg_pn):
        if pn in preferred_versions and preferred_versions[pn][1]:
            continue
        preferred_versions[pn] = findLatestProvider(pn, cfgData, dataCache, sortpkg_pn[pn][0])
        eligible.append(preferred_versions[pn][1])

    if not eligible:
        return eligible

    # If pn == item, give it a slight default preference
    # This means PREFERRED_PROVIDER_foobar defaults to foobar if available
    for p in providers:
        pn = dataCache.pkg_fn[p]
        if pn != item:
            continue
        (newvers, fn) = preferred_versions[pn]
        if not fn in eligible:
            continue
        eligible.remove(fn)
        eligible = [fn] + eligible

    return eligible

def filterProviders(providers, item, cfgData, dataCache):
    """
    Take a list of providers and filter/reorder according to the
    environment variables
    Takes a "normal" target item
    """

    eligible = _filterProviders(providers, item, cfgData, dataCache)

    prefervar = cfgData.getVar('PREFERRED_PROVIDER_%s' % item)
    if prefervar:
        dataCache.preferred[item] = prefervar

    foundUnique = False
    if item in dataCache.preferred:
        for p in eligible:
            pn = dataCache.pkg_fn[p]
            if dataCache.preferred[item] == pn:
                logger.verbose("selecting %s to satisfy %s due to PREFERRED_PROVIDERS", pn, item)
                eligible.remove(p)
                eligible = [p] + eligible
                foundUnique = True
                break

    logger.debug("sorted providers for %s are: %s", item, eligible)

    return eligible, foundUnique

def filterProvidersRunTime(providers, item, cfgData, dataCache):
    """
    Take a list of providers and filter/reorder according to the
    environment variables
    Takes a "runtime" target item
    """

    eligible = _filterProviders(providers, item, cfgData, dataCache)

    # First try and match any PREFERRED_RPROVIDER entry
    prefervar = cfgData.getVar('PREFERRED_RPROVIDER_%s' % item)
    foundUnique = False
    if prefervar:
        for p in eligible:
            pn = dataCache.pkg_fn[p]
            if prefervar == pn:
                logger.verbose("selecting %s to satisfy %s due to PREFERRED_RPROVIDER", pn, item)
                eligible.remove(p)
                eligible = [p] + eligible
                foundUnique = True
                numberPreferred = 1
                break

    # If we didn't find an RPROVIDER entry, try and infer the provider from PREFERRED_PROVIDER entries
    # by looking through the provides of each eligible recipe and seeing if a PREFERRED_PROVIDER was set.
    # This is most useful for virtual/ entries rather than having a RPROVIDER per entry.
    if not foundUnique:
        # Should use dataCache.preferred here?
        preferred = []
        preferred_vars = []
        pns = {}
        for p in eligible:
            pns[dataCache.pkg_fn[p]] = p
        for p in eligible:
            pn = dataCache.pkg_fn[p]
            provides = dataCache.pn_provides[pn]
            for provide in provides:
                prefervar = cfgData.getVar('PREFERRED_PROVIDER_%s' % provide)
                #logger.debug("checking PREFERRED_PROVIDER_%s (value %s) against %s", provide, prefervar, pns.keys())
                if prefervar in pns and pns[prefervar] not in preferred:
                    var = "PREFERRED_PROVIDER_%s = %s" % (provide, prefervar)
                    logger.verbose("selecting %s to satisfy runtime %s due to %s", prefervar, item, var)
                    preferred_vars.append(var)
                    pref = pns[prefervar]
                    eligible.remove(pref)
                    eligible = [pref] + eligible
                    preferred.append(pref)
                    break

        numberPreferred = len(preferred)

    if numberPreferred > 1:
        logger.error("Trying to resolve runtime dependency %s resulted in conflicting PREFERRED_PROVIDER entries being found.\nThe providers found were: %s\nThe PREFERRED_PROVIDER entries resulting in this conflict were: %s. You could set PREFERRED_RPROVIDER_%s" % (item, preferred, preferred_vars, item))

    logger.debug("sorted runtime providers for %s are: %s", item, eligible)

    return eligible, numberPreferred

regexp_cache = {}

def getRuntimeProviders(dataCache, rdepend):
    """
    Return any providers of runtime dependency
    """
    rproviders = []

    if rdepend in dataCache.rproviders:
        rproviders += dataCache.rproviders[rdepend]

    if rdepend in dataCache.packages:
        rproviders += dataCache.packages[rdepend]

    if rproviders:
        return rproviders

    # Only search dynamic packages if we can't find anything in other variables
    for pat_key in dataCache.packages_dynamic:
        pattern = pat_key.replace(r'+', r"\+")
        if pattern in regexp_cache:
            regexp = regexp_cache[pattern]
        else:
            try:
                regexp = re.compile(pattern)
            except:
                logger.error("Error parsing regular expression '%s'", pattern)
                raise
            regexp_cache[pattern] = regexp
        if regexp.match(rdepend):
            rproviders += dataCache.packages_dynamic[pat_key]
            logger.debug("Assuming %s is a dynamic package, but it may not exist" % rdepend)

    return rproviders

def buildWorldTargetList(dataCache, task=None):
    """
    Build package list for "bitbake world"
    """
    if dataCache.world_target:
        return

    logger.debug("collating packages for \"world\"")
    for f in dataCache.possible_world:
        terminal = True
        pn = dataCache.pkg_fn[f]
        if task and task not in dataCache.task_deps[f]['tasks']:
            logger.debug2("World build skipping %s as task %s doesn't exist", f, task)
            terminal = False

        for p in dataCache.pn_provides[pn]:
            if p.startswith('virtual/'):
                logger.debug2("World build skipping %s due to %s provider starting with virtual/", f, p)
                terminal = False
                break
            for pf in dataCache.providers[p]:
                if dataCache.pkg_fn[pf] != pn:
                    logger.debug2("World build skipping %s due to both us and %s providing %s", f, pf, p)
                    terminal = False
                    break
        if terminal:
            dataCache.world_target.add(pn)