diff options
Diffstat (limited to 'scripts/lib/recipetool')
| -rw-r--r-- | scripts/lib/recipetool/append.py | 84 | ||||
| -rw-r--r-- | scripts/lib/recipetool/create.py | 366 | ||||
| -rw-r--r-- | scripts/lib/recipetool/create_buildsys.py | 43 | ||||
| -rw-r--r-- | scripts/lib/recipetool/create_buildsys_python.py | 1100 | ||||
| -rw-r--r-- | scripts/lib/recipetool/create_go.py | 777 | ||||
| -rw-r--r-- | scripts/lib/recipetool/create_kmod.py | 2 | ||||
| -rw-r--r-- | scripts/lib/recipetool/create_npm.py | 107 | ||||
| -rw-r--r-- | scripts/lib/recipetool/licenses.csv | 37 | ||||
| -rw-r--r-- | scripts/lib/recipetool/setvar.py | 1 |
9 files changed, 1966 insertions, 551 deletions
diff --git a/scripts/lib/recipetool/append.py b/scripts/lib/recipetool/append.py index e9d52bb67b..10945d6008 100644 --- a/scripts/lib/recipetool/append.py +++ b/scripts/lib/recipetool/append.py @@ -18,6 +18,7 @@ import shutil import scriptutils import errno from collections import defaultdict +import difflib logger = logging.getLogger('recipetool') @@ -49,7 +50,7 @@ def find_target_file(targetpath, d, pkglist=None): '/etc/group': '/etc/group should be managed through the useradd and extrausers classes', '/etc/shadow': '/etc/shadow should be managed through the useradd and extrausers classes', '/etc/gshadow': '/etc/gshadow should be managed through the useradd and extrausers classes', - '${sysconfdir}/hostname': '${sysconfdir}/hostname contents should be set by setting hostname_pn-base-files = "value" in configuration',} + '${sysconfdir}/hostname': '${sysconfdir}/hostname contents should be set by setting hostname:pn-base-files = "value" in configuration',} for pthspec, message in invalidtargets.items(): if fnmatch.fnmatchcase(targetpath, d.expand(pthspec)): @@ -72,15 +73,15 @@ def find_target_file(targetpath, d, pkglist=None): # This does assume that PN comes before other values, but that's a fairly safe assumption for line in f: if line.startswith('PN:'): - pn = line.split(':', 1)[1].strip() - elif line.startswith('FILES_INFO:'): - val = line.split(':', 1)[1].strip() + pn = line.split(': ', 1)[1].strip() + elif line.startswith('FILES_INFO'): + val = line.split(': ', 1)[1].strip() dictval = json.loads(val) for fullpth in dictval.keys(): if fnmatch.fnmatchcase(fullpth, targetpath): recipes[targetpath].append(pn) - elif line.startswith('pkg_preinst_') or line.startswith('pkg_postinst_'): - scriptval = line.split(':', 1)[1].strip().encode('utf-8').decode('unicode_escape') + elif line.startswith('pkg_preinst:') or line.startswith('pkg_postinst:'): + scriptval = line.split(': ', 1)[1].strip().encode('utf-8').decode('unicode_escape') if 'update-alternatives --install %s ' % targetpath in scriptval: recipes[targetpath].append('?%s' % pn) elif targetpath_re.search(scriptval): @@ -100,7 +101,7 @@ def determine_file_source(targetpath, rd): import oe.recipeutils # See if it's in do_install for the recipe - workdir = rd.getVar('WORKDIR') + unpackdir = rd.getVar('UNPACKDIR') src_uri = rd.getVar('SRC_URI') srcfile = '' modpatches = [] @@ -112,9 +113,9 @@ def determine_file_source(targetpath, rd): if not srcpath.startswith('/'): # Handle non-absolute path srcpath = os.path.abspath(os.path.join(rd.getVarFlag('do_install', 'dirs').split()[-1], srcpath)) - if srcpath.startswith(workdir): + if srcpath.startswith(unpackdir): # OK, now we have the source file name, look for it in SRC_URI - workdirfile = os.path.relpath(srcpath, workdir) + workdirfile = os.path.relpath(srcpath, unpackdir) # FIXME this is where we ought to have some code in the fetcher, because this is naive for item in src_uri.split(): localpath = bb.fetch2.localpath(item, rd) @@ -299,7 +300,10 @@ def appendfile(args): if st.st_mode & stat.S_IXUSR: perms = '0755' install = {args.newfile: (args.targetpath, perms)} - oe.recipeutils.bbappend_recipe(rd, args.destlayer, {args.newfile: sourcepath}, install, wildcardver=args.wildcard_version, machine=args.machine) + if sourcepath: + sourcepath = os.path.basename(sourcepath) + oe.recipeutils.bbappend_recipe(rd, args.destlayer, {args.newfile: {'newname' : sourcepath}}, install, wildcardver=args.wildcard_version, machine=args.machine) + tinfoil.modified_files() return 0 else: if alternative_pns: @@ -327,6 +331,7 @@ def appendsrc(args, files, rd, extralines=None): copyfiles = {} extralines = extralines or [] + params = [] for newfile, srcfile in files.items(): src_destdir = os.path.dirname(srcfile) if not args.use_workdir: @@ -337,25 +342,46 @@ def appendsrc(args, files, rd, extralines=None): src_destdir = os.path.join(os.path.relpath(srcdir, workdir), src_destdir) src_destdir = os.path.normpath(src_destdir) - source_uri = 'file://{0}'.format(os.path.basename(srcfile)) if src_destdir and src_destdir != '.': - source_uri += ';subdir={0}'.format(src_destdir) - - simple = bb.fetch.URI(source_uri) - simple.params = {} - simple_str = str(simple) - if simple_str in simplified: - existing = simplified[simple_str] - if source_uri != existing: - logger.warning('{0!r} is already in SRC_URI, with different parameters: {1!r}, not adding'.format(source_uri, existing)) - else: - logger.warning('{0!r} is already in SRC_URI, not adding'.format(source_uri)) + params.append({'subdir': src_destdir}) else: - extralines.append('SRC_URI += {0}'.format(source_uri)) - copyfiles[newfile] = srcfile - - oe.recipeutils.bbappend_recipe(rd, args.destlayer, copyfiles, None, wildcardver=args.wildcard_version, machine=args.machine, extralines=extralines) - + params.append({}) + + copyfiles[newfile] = {'newname' : os.path.basename(srcfile)} + + dry_run_output = None + dry_run_outdir = None + if args.dry_run: + import tempfile + dry_run_output = tempfile.TemporaryDirectory(prefix='devtool') + dry_run_outdir = dry_run_output.name + + appendfile, _ = oe.recipeutils.bbappend_recipe(rd, args.destlayer, copyfiles, None, wildcardver=args.wildcard_version, machine=args.machine, extralines=extralines, params=params, + redirect_output=dry_run_outdir, update_original_recipe=args.update_recipe) + if not appendfile: + return + if args.dry_run: + output = '' + appendfilename = os.path.basename(appendfile) + newappendfile = appendfile + if appendfile and os.path.exists(appendfile): + with open(appendfile, 'r') as f: + oldlines = f.readlines() + else: + appendfile = '/dev/null' + oldlines = [] + + with open(os.path.join(dry_run_outdir, appendfilename), 'r') as f: + newlines = f.readlines() + diff = difflib.unified_diff(oldlines, newlines, appendfile, newappendfile) + difflines = list(diff) + if difflines: + output += ''.join(difflines) + if output: + logger.info('Diff of changed files:\n%s' % output) + else: + logger.info('No changed files') + tinfoil.modified_files() def appendsrcfiles(parser, args): recipedata = _parse_recipe(args.recipe, tinfoil) @@ -435,6 +461,8 @@ def register_commands(subparsers): help='Create/update a bbappend to add or replace source files', description='Creates a bbappend (or updates an existing one) to add or replace the specified file in the recipe sources, either those in WORKDIR or those in the source tree. This command lets you specify multiple files with a destination directory, so cannot specify the destination filename. See the `appendsrcfile` command for the other behavior.') parser.add_argument('-D', '--destdir', help='Destination directory (relative to S or WORKDIR, defaults to ".")', default='', type=destination_path) + parser.add_argument('-u', '--update-recipe', help='Update recipe instead of creating (or updating) a bbapend file. DESTLAYER must contains the recipe to update', action='store_true') + parser.add_argument('-n', '--dry-run', help='Dry run mode', action='store_true') parser.add_argument('files', nargs='+', metavar='FILE', help='File(s) to be added to the recipe sources (WORKDIR or S)', type=existing_path) parser.set_defaults(func=lambda a: appendsrcfiles(parser, a), parserecipes=True) @@ -442,6 +470,8 @@ def register_commands(subparsers): parents=[common_src], help='Create/update a bbappend to add or replace a source file', description='Creates a bbappend (or updates an existing one) to add or replace the specified files in the recipe sources, either those in WORKDIR or those in the source tree. This command lets you specify the destination filename, not just destination directory, but only works for one file. See the `appendsrcfiles` command for the other behavior.') + parser.add_argument('-u', '--update-recipe', help='Update recipe instead of creating (or updating) a bbapend file. DESTLAYER must contains the recipe to update', action='store_true') + parser.add_argument('-n', '--dry-run', help='Dry run mode', action='store_true') parser.add_argument('file', metavar='FILE', help='File to be added to the recipe sources (WORKDIR or S)', type=existing_path) parser.add_argument('destfile', metavar='DESTFILE', nargs='?', help='Destination path (relative to S or WORKDIR, optional)', type=destination_path) parser.set_defaults(func=lambda a: appendsrcfile(parser, a), parserecipes=True) diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py index 8d78c5b6f9..8e9ff38db6 100644 --- a/scripts/lib/recipetool/create.py +++ b/scripts/lib/recipetool/create.py @@ -66,7 +66,7 @@ class RecipeHandler(object): libdir = d.getVar('libdir') base_libdir = d.getVar('base_libdir') libpaths = list(set([base_libdir, libdir])) - libname_re = re.compile('^lib(.+)\.so.*$') + libname_re = re.compile(r'^lib(.+)\.so.*$') pkglibmap = {} for lib, item in shlib_providers.items(): for path, pkg in item.items(): @@ -115,8 +115,8 @@ class RecipeHandler(object): for line in f: if line.startswith('PN:'): pn = line.split(':', 1)[-1].strip() - elif line.startswith('FILES_INFO:'): - val = line.split(':', 1)[1].strip() + elif line.startswith('FILES_INFO:%s:' % pkg): + val = line.split(': ', 1)[1].strip() dictval = json.loads(val) for fullpth in sorted(dictval): if fullpth.startswith(includedir) and fullpth.endswith('.h'): @@ -366,7 +366,7 @@ def supports_srcrev(uri): def reformat_git_uri(uri): '''Convert any http[s]://....git URI into git://...;protocol=http[s]''' checkuri = uri.split(';', 1)[0] - if checkuri.endswith('.git') or '/git/' in checkuri or re.match('https?://github.com/[^/]+/[^/]+/?$', checkuri): + if checkuri.endswith('.git') or '/git/' in checkuri or re.match('https?://git(hub|lab).com/[^/]+/[^/]+/?$', checkuri): # Appends scheme if the scheme is missing if not '://' in uri: uri = 'git://' + uri @@ -423,19 +423,50 @@ def create_recipe(args): storeTagName = '' pv_srcpv = False + handled = [] + classes = [] + + # Find all plugins that want to register handlers + logger.debug('Loading recipe handlers') + raw_handlers = [] + for plugin in plugins: + if hasattr(plugin, 'register_recipe_handlers'): + plugin.register_recipe_handlers(raw_handlers) + # Sort handlers by priority + handlers = [] + for i, handler in enumerate(raw_handlers): + if isinstance(handler, tuple): + handlers.append((handler[0], handler[1], i)) + else: + handlers.append((handler, 0, i)) + handlers.sort(key=lambda item: (item[1], -item[2]), reverse=True) + for handler, priority, _ in handlers: + logger.debug('Handler: %s (priority %d)' % (handler.__class__.__name__, priority)) + setattr(handler, '_devtool', args.devtool) + handlers = [item[0] for item in handlers] + + fetchuri = None + for handler in handlers: + if hasattr(handler, 'process_url'): + ret = handler.process_url(args, classes, handled, extravalues) + if 'url' in handled and ret: + fetchuri = ret + break + if os.path.isfile(source): source = 'file://%s' % os.path.abspath(source) if scriptutils.is_src_url(source): # Warn about github archive URLs - if re.match('https?://github.com/[^/]+/[^/]+/archive/.+(\.tar\..*|\.zip)$', source): + if re.match(r'https?://github.com/[^/]+/[^/]+/archive/.+(\.tar\..*|\.zip)$', source): logger.warning('github archive files are not guaranteed to be stable and may be re-generated over time. If the latter occurs, the checksums will likely change and the recipe will fail at do_fetch. It is recommended that you point to an actual commit or tag in the repository instead (using the repository URL in conjunction with the -S/--srcrev option).') # Fetch a URL - fetchuri = reformat_git_uri(urldefrag(source)[0]) + if not fetchuri: + fetchuri = reformat_git_uri(urldefrag(source)[0]) if args.binary: # Assume the archive contains the directory structure verbatim # so we need to extract to a subdirectory - fetchuri += ';subdir=${BP}' + fetchuri += ';subdir=${BPN}' srcuri = fetchuri rev_re = re.compile(';rev=([^;]+)') res = rev_re.search(srcuri) @@ -478,6 +509,9 @@ def create_recipe(args): storeTagName = params['tag'] params['nobranch'] = '1' del params['tag'] + # Assume 'master' branch if not set + if scheme in ['git', 'gitsm'] and 'branch' not in params and 'nobranch' not in params: + params['branch'] = 'master' fetchuri = bb.fetch2.encodeurl((scheme, network, path, user, passwd, params)) tmpparent = tinfoil.config_data.getVar('BASE_WORKDIR') @@ -527,10 +561,9 @@ def create_recipe(args): # Remove HEAD reference point and drop remote prefix get_branch = [x.split('/', 1)[1] for x in get_branch if not x.startswith('origin/HEAD')] if 'master' in get_branch: - # If it is master, we do not need to append 'branch=master' as this is default. # Even with the case where get_branch has multiple objects, if 'master' is one # of them, we should default take from 'master' - srcbranch = '' + srcbranch = 'master' elif len(get_branch) == 1: # If 'master' isn't in get_branch and get_branch contains only ONE object, then store result into 'srcbranch' srcbranch = get_branch[0] @@ -543,8 +576,8 @@ def create_recipe(args): # Since we might have a value in srcbranch, we need to # recontruct the srcuri to include 'branch' in params. scheme, network, path, user, passwd, params = bb.fetch2.decodeurl(srcuri) - if srcbranch: - params['branch'] = srcbranch + if scheme in ['git', 'gitsm']: + params['branch'] = srcbranch or 'master' if storeTagName and scheme in ['git', 'gitsm']: # Check srcrev using tag and check validity of the tag @@ -603,7 +636,7 @@ def create_recipe(args): splitline = line.split() if len(splitline) > 1: if splitline[0] == 'origin' and scriptutils.is_src_url(splitline[1]): - srcuri = reformat_git_uri(splitline[1]) + srcuri = reformat_git_uri(splitline[1]) + ';branch=master' srcsubdir = 'git' break @@ -636,8 +669,6 @@ def create_recipe(args): # We'll come back and replace this later in handle_license_vars() lines_before.append('##LICENSE_PLACEHOLDER##') - handled = [] - classes = [] # FIXME This is kind of a hack, we probably ought to be using bitbake to do this pn = None @@ -675,8 +706,10 @@ def create_recipe(args): if not srcuri: lines_before.append('# No information for SRC_URI yet (only an external source tree was specified)') lines_before.append('SRC_URI = "%s"' % srcuri) + shown_checksums = ["%ssum" % s for s in bb.fetch2.SHOWN_CHECKSUM_LIST] for key, value in sorted(checksums.items()): - lines_before.append('SRC_URI[%s] = "%s"' % (key, value)) + if key in shown_checksums: + lines_before.append('SRC_URI[%s] = "%s"' % (key, value)) if srcuri and supports_srcrev(srcuri): lines_before.append('') lines_before.append('# Modify these as desired') @@ -688,7 +721,7 @@ def create_recipe(args): srcpvprefix = 'svnr' else: srcpvprefix = scheme - lines_before.append('PV = "%s+%s${SRCPV}"' % (realpv or '1.0', srcpvprefix)) + lines_before.append('PV = "%s+%s"' % (realpv or '1.0', srcpvprefix)) pv_srcpv = True if not args.autorev and srcrev == '${AUTOREV}': if os.path.exists(os.path.join(srctree, '.git')): @@ -710,31 +743,12 @@ def create_recipe(args): lines_after.append('') if args.binary: - lines_after.append('INSANE_SKIP_${PN} += "already-stripped"') + lines_after.append('INSANE_SKIP:${PN} += "already-stripped"') lines_after.append('') if args.npm_dev: extravalues['NPM_INSTALL_DEV'] = 1 - # Find all plugins that want to register handlers - logger.debug('Loading recipe handlers') - raw_handlers = [] - for plugin in plugins: - if hasattr(plugin, 'register_recipe_handlers'): - plugin.register_recipe_handlers(raw_handlers) - # Sort handlers by priority - handlers = [] - for i, handler in enumerate(raw_handlers): - if isinstance(handler, tuple): - handlers.append((handler[0], handler[1], i)) - else: - handlers.append((handler, 0, i)) - handlers.sort(key=lambda item: (item[1], -item[2]), reverse=True) - for handler, priority, _ in handlers: - logger.debug('Handler: %s (priority %d)' % (handler.__class__.__name__, priority)) - setattr(handler, '_devtool', args.devtool) - handlers = [item[0] for item in handlers] - # Apply the handlers if args.binary: classes.append('bin_package') @@ -743,6 +757,10 @@ def create_recipe(args): for handler in handlers: handler.process(srctree_use, classes, lines_before, lines_after, handled, extravalues) + # native and nativesdk classes are special and must be inherited last + # If present, put them at the end of the classes list + classes.sort(key=lambda c: c in ("native", "nativesdk")) + extrafiles = extravalues.pop('extrafiles', {}) extra_pn = extravalues.pop('PN', None) extra_pv = extravalues.pop('PV', None) @@ -830,7 +848,7 @@ def create_recipe(args): elif line.startswith('PV = '): if realpv: # Replace the first part of the PV value - line = re.sub('"[^+]*\+', '"%s+' % realpv, line) + line = re.sub(r'"[^+]*\+', '"%s+' % realpv, line) lines_before.append(line) if args.also_native: @@ -867,8 +885,10 @@ def create_recipe(args): outlines.append('') outlines.extend(lines_after) + outlines = [ line.rstrip('\n') +"\n" for line in outlines] + if extravalues: - _, outlines = oe.recipeutils.patch_recipe_lines(outlines, extravalues, trailing_newline=False) + _, outlines = oe.recipeutils.patch_recipe_lines(outlines, extravalues, trailing_newline=True) if args.extract_to: scriptutils.git_convert_standalone_clone(srctree) @@ -884,7 +904,7 @@ def create_recipe(args): log_info_cond('Source extracted to %s' % args.extract_to, args.devtool) if outfile == '-': - sys.stdout.write('\n'.join(outlines) + '\n') + sys.stdout.write(''.join(outlines) + '\n') else: with open(outfile, 'w') as f: lastline = None @@ -892,9 +912,10 @@ def create_recipe(args): if not lastline and not line: # Skip extra blank lines continue - f.write('%s\n' % line) + f.write('%s' % line) lastline = line log_info_cond('Recipe %s has been created; further editing may be required to make it fully functional' % outfile, args.devtool) + tinfoil.modified_files() if tempsrc: if args.keep_temp: @@ -917,6 +938,22 @@ def split_value(value): else: return value +def fixup_license(value): + # Ensure licenses with OR starts and ends with brackets + if '|' in value: + return '(' + value + ')' + return value + +def tidy_licenses(value): + """Flat, split and sort licenses""" + from oe.license import flattened_licenses + def _choose(a, b): + str_a, str_b = sorted((" & ".join(a), " & ".join(b)), key=str.casefold) + return ["(%s | %s)" % (str_a, str_b)] + if not isinstance(value, str): + value = " & ".join(value) + return sorted(list(set(flattened_licenses(value, _choose))), key=str.casefold) + def handle_license_vars(srctree, lines_before, handled, extravalues, d): lichandled = [x for x in handled if x[0] == 'license'] if lichandled: @@ -930,10 +967,13 @@ def handle_license_vars(srctree, lines_before, handled, extravalues, d): lines = [] if licvalues: for licvalue in licvalues: - if not licvalue[0] in licenses: - licenses.append(licvalue[0]) + license = licvalue[0] + lics = tidy_licenses(fixup_license(license)) + lics = [lic for lic in lics if lic not in licenses] + if len(lics): + licenses.extend(lics) lic_files_chksum.append('file://%s;md5=%s' % (licvalue[1], licvalue[2])) - if licvalue[0] == 'Unknown': + if license == 'Unknown': lic_unknown.append(licvalue[1]) if lic_unknown: lines.append('#') @@ -942,9 +982,7 @@ def handle_license_vars(srctree, lines_before, handled, extravalues, d): for licfile in lic_unknown: lines.append('# %s' % licfile) - extra_license = split_value(extravalues.pop('LICENSE', [])) - if '&' in extra_license: - extra_license.remove('&') + extra_license = tidy_licenses(extravalues.pop('LICENSE', '')) if extra_license: if licenses == ['Unknown']: licenses = extra_license @@ -985,7 +1023,7 @@ def handle_license_vars(srctree, lines_before, handled, extravalues, d): lines.append('# instead of &. If there is any doubt, check the accompanying documentation') lines.append('# to determine which situation is applicable.') - lines.append('LICENSE = "%s"' % ' & '.join(licenses)) + lines.append('LICENSE = "%s"' % ' & '.join(sorted(licenses, key=str.casefold))) lines.append('LIC_FILES_CHKSUM = "%s"' % ' \\\n '.join(lic_files_chksum)) lines.append('') @@ -1002,118 +1040,170 @@ def handle_license_vars(srctree, lines_before, handled, extravalues, d): handled.append(('license', licvalues)) return licvalues -def get_license_md5sums(d, static_only=False): +def get_license_md5sums(d, static_only=False, linenumbers=False): import bb.utils + import csv md5sums = {} - if not static_only: + if not static_only and not linenumbers: # Gather md5sums of license files in common license dir commonlicdir = d.getVar('COMMON_LICENSE_DIR') for fn in os.listdir(commonlicdir): md5value = bb.utils.md5_file(os.path.join(commonlicdir, fn)) md5sums[md5value] = fn + # The following were extracted from common values in various recipes # (double checking the license against the license file itself, not just # the LICENSE value in the recipe) - md5sums['94d55d512a9ba36caa9b7df079bae19f'] = 'GPLv2' - md5sums['b234ee4d69f5fce4486a80fdaf4a4263'] = 'GPLv2' - md5sums['59530bdf33659b29e73d4adb9f9f6552'] = 'GPLv2' - md5sums['0636e73ff0215e8d672dc4c32c317bb3'] = 'GPLv2' - md5sums['eb723b61539feef013de476e68b5c50a'] = 'GPLv2' - md5sums['751419260aa954499f7abaabaa882bbe'] = 'GPLv2' - md5sums['393a5ca445f6965873eca0259a17f833'] = 'GPLv2' - md5sums['12f884d2ae1ff87c09e5b7ccc2c4ca7e'] = 'GPLv2' - md5sums['8ca43cbc842c2336e835926c2166c28b'] = 'GPLv2' - md5sums['ebb5c50ab7cab4baeffba14977030c07'] = 'GPLv2' - md5sums['c93c0550bd3173f4504b2cbd8991e50b'] = 'GPLv2' - md5sums['9ac2e7cff1ddaf48b6eab6028f23ef88'] = 'GPLv2' - md5sums['4325afd396febcb659c36b49533135d4'] = 'GPLv2' - md5sums['18810669f13b87348459e611d31ab760'] = 'GPLv2' - md5sums['d7810fab7487fb0aad327b76f1be7cd7'] = 'GPLv2' # the Linux kernel's COPYING file - md5sums['bbb461211a33b134d42ed5ee802b37ff'] = 'LGPLv2.1' - md5sums['7fbc338309ac38fefcd64b04bb903e34'] = 'LGPLv2.1' - md5sums['4fbd65380cdd255951079008b364516c'] = 'LGPLv2.1' - md5sums['2d5025d4aa3495befef8f17206a5b0a1'] = 'LGPLv2.1' - md5sums['fbc093901857fcd118f065f900982c24'] = 'LGPLv2.1' - md5sums['a6f89e2100d9b6cdffcea4f398e37343'] = 'LGPLv2.1' - md5sums['d8045f3b8f929c1cb29a1e3fd737b499'] = 'LGPLv2.1' - md5sums['fad9b3332be894bab9bc501572864b29'] = 'LGPLv2.1' - md5sums['3bf50002aefd002f49e7bb854063f7e7'] = 'LGPLv2' - md5sums['9f604d8a4f8e74f4f5140845a21b6674'] = 'LGPLv2' - md5sums['5f30f0716dfdd0d91eb439ebec522ec2'] = 'LGPLv2' - md5sums['55ca817ccb7d5b5b66355690e9abc605'] = 'LGPLv2' - md5sums['252890d9eee26aab7b432e8b8a616475'] = 'LGPLv2' - md5sums['3214f080875748938ba060314b4f727d'] = 'LGPLv2' - md5sums['db979804f025cf55aabec7129cb671ed'] = 'LGPLv2' - md5sums['d32239bcb673463ab874e80d47fae504'] = 'GPLv3' - md5sums['f27defe1e96c2e1ecd4e0c9be8967949'] = 'GPLv3' - md5sums['6a6a8e020838b23406c81b19c1d46df6'] = 'LGPLv3' - md5sums['3b83ef96387f14655fc854ddc3c6bd57'] = 'Apache-2.0' - md5sums['385c55653886acac3821999a3ccd17b3'] = 'Artistic-1.0 | GPL-2.0' # some perl modules - md5sums['54c7042be62e169199200bc6477f04d1'] = 'BSD-3-Clause' - md5sums['bfe1f75d606912a4111c90743d6c7325'] = 'MPL-1.1' + + # Read license md5sums from csv file + scripts_path = os.path.dirname(os.path.realpath(__file__)) + for path in (d.getVar('BBPATH').split(':') + + [os.path.join(scripts_path, '..', '..')]): + csv_path = os.path.join(path, 'lib', 'recipetool', 'licenses.csv') + if os.path.isfile(csv_path): + with open(csv_path, newline='') as csv_file: + fieldnames = ['md5sum', 'license', 'beginline', 'endline', 'md5'] + reader = csv.DictReader(csv_file, delimiter=',', fieldnames=fieldnames) + for row in reader: + if linenumbers: + md5sums[row['md5sum']] = ( + row['license'], row['beginline'], row['endline'], row['md5']) + else: + md5sums[row['md5sum']] = row['license'] + return md5sums -def crunch_license(licfile): +def crunch_known_licenses(d): ''' - Remove non-material text from a license file and then check - its md5sum against a known list. This works well for licenses - which contain a copyright statement, but is also a useful way - to handle people's insistence upon reformatting the license text - slightly (with no material difference to the text of the - license). + Calculate the MD5 checksums for the crunched versions of all common + licenses. Also add additional known checksums. ''' + + crunched_md5sums = {} - import oe.utils + # common licenses + crunched_md5sums['ad4e9d34a2e966dfe9837f18de03266d'] = 'GFDL-1.1-only' + crunched_md5sums['d014fb11a34eb67dc717fdcfc97e60ed'] = 'GFDL-1.2-only' + crunched_md5sums['e020ca655b06c112def28e597ab844f1'] = 'GFDL-1.3-only' - # Note: these are carefully constructed! - license_title_re = re.compile('^\(?(#+ *)?(The )?.{1,10} [Ll]icen[sc]e( \(.{1,10}\))?\)?:?$') - license_statement_re = re.compile('^(This (project|software) is( free software)? (released|licen[sc]ed)|(Released|Licen[cs]ed)) under the .{1,10} [Ll]icen[sc]e:?$') - copyright_re = re.compile('^(#+)? *Copyright .*$') - - crunched_md5sums = {} # The following two were gleaned from the "forever" npm package crunched_md5sums['0a97f8e4cbaf889d6fa51f84b89a79f6'] = 'ISC' - crunched_md5sums['eecf6429523cbc9693547cf2db790b5c'] = 'MIT' - # https://github.com/vasi/pixz/blob/master/LICENSE - crunched_md5sums['2f03392b40bbe663597b5bd3cc5ebdb9'] = 'BSD-2-Clause' # https://github.com/waffle-gl/waffle/blob/master/LICENSE.txt - crunched_md5sums['e72e5dfef0b1a4ca8a3d26a60587db66'] = 'BSD-2-Clause' + crunched_md5sums['50fab24ce589d69af8964fdbfe414c60'] = 'BSD-2-Clause' # https://github.com/spigwitmer/fakeds1963s/blob/master/LICENSE - crunched_md5sums['8be76ac6d191671f347ee4916baa637e'] = 'GPLv2' - # https://github.com/datto/dattobd/blob/master/COPYING - # http://git.savannah.gnu.org/cgit/freetype/freetype2.git/tree/docs/GPLv2.TXT - crunched_md5sums['1d65c5ad4bf6489f85f4812bf08ae73d'] = 'GPLv2' + crunched_md5sums['88a4355858a1433fea99fae34a44da88'] = 'GPL-2.0-only' # http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt - # http://git.neil.brown.name/?p=mdadm.git;a=blob;f=COPYING;h=d159169d1050894d3ea3b98e1c965c4058208fe1;hb=HEAD - crunched_md5sums['fb530f66a7a89ce920f0e912b5b66d4b'] = 'GPLv2' - # https://github.com/gkos/nrf24/blob/master/COPYING - crunched_md5sums['7b6aaa4daeafdfa6ed5443fd2684581b'] = 'GPLv2' - # https://github.com/josch09/resetusb/blob/master/COPYING - crunched_md5sums['8b8ac1d631a4d220342e83bcf1a1fbc3'] = 'GPLv3' + crunched_md5sums['063b5c3ebb5f3aa4c85a2ed18a31fbe7'] = 'GPL-2.0-only' # https://github.com/FFmpeg/FFmpeg/blob/master/COPYING.LGPLv2.1 - crunched_md5sums['2ea316ed973ae176e502e2297b574bb3'] = 'LGPLv2.1' + crunched_md5sums['7f5202f4d44ed15dcd4915f5210417d8'] = 'LGPL-2.1-only' # unixODBC-2.3.4 COPYING - crunched_md5sums['1daebd9491d1e8426900b4fa5a422814'] = 'LGPLv2.1' + crunched_md5sums['3debde09238a8c8e1f6a847e1ec9055b'] = 'LGPL-2.1-only' # https://github.com/FFmpeg/FFmpeg/blob/master/COPYING.LGPLv3 - crunched_md5sums['2ebfb3bb49b9a48a075cc1425e7f4129'] = 'LGPLv3' + crunched_md5sums['f90c613c51aa35da4d79dd55fc724ceb'] = 'LGPL-3.0-only' # https://raw.githubusercontent.com/eclipse/mosquitto/v1.4.14/epl-v10 crunched_md5sums['efe2cb9a35826992b9df68224e3c2628'] = 'EPL-1.0' - # https://raw.githubusercontent.com/eclipse/mosquitto/v1.4.14/edl-v10 - crunched_md5sums['0a9c78c0a398d1bbce4a166757d60387'] = 'EDL-1.0' + + # https://raw.githubusercontent.com/jquery/esprima/3.1.3/LICENSE.BSD + crunched_md5sums['80fa7b56a28e8c902e6af194003220a5'] = 'BSD-2-Clause' + # https://raw.githubusercontent.com/npm/npm-install-checks/master/LICENSE + crunched_md5sums['e659f77bfd9002659e112d0d3d59b2c1'] = 'BSD-2-Clause' + # https://raw.githubusercontent.com/silverwind/default-gateway/4.2.0/LICENSE + crunched_md5sums['4c641f2d995c47f5cb08bdb4b5b6ea05'] = 'BSD-2-Clause' + # https://raw.githubusercontent.com/tad-lispy/node-damerau-levenshtein/v1.0.5/LICENSE + crunched_md5sums['2b8c039b2b9a25f0feb4410c4542d346'] = 'BSD-2-Clause' + # https://raw.githubusercontent.com/terser/terser/v3.17.0/LICENSE + crunched_md5sums['8bd23871802951c9ad63855151204c2c'] = 'BSD-2-Clause' + # https://raw.githubusercontent.com/alexei/sprintf.js/1.0.3/LICENSE + crunched_md5sums['008c22318c8ea65928bf730ddd0273e3'] = 'BSD-3-Clause' + # https://raw.githubusercontent.com/Caligatio/jsSHA/v3.2.0/LICENSE + crunched_md5sums['0e46634a01bfef056892949acaea85b1'] = 'BSD-3-Clause' + # https://raw.githubusercontent.com/d3/d3-path/v1.0.9/LICENSE + crunched_md5sums['b5f72aef53d3b2b432702c30b0215666'] = 'BSD-3-Clause' + # https://raw.githubusercontent.com/feross/ieee754/v1.1.13/LICENSE + crunched_md5sums['a39327c997c20da0937955192d86232d'] = 'BSD-3-Clause' + # https://raw.githubusercontent.com/joyent/node-extsprintf/v1.3.0/LICENSE + crunched_md5sums['721f23a96ff4161ca3a5f071bbe18108'] = 'MIT' + # https://raw.githubusercontent.com/pvorb/clone/v0.2.0/LICENSE + crunched_md5sums['b376d29a53c9573006b9970709231431'] = 'MIT' + # https://raw.githubusercontent.com/andris9/encoding/v0.1.12/LICENSE + crunched_md5sums['85d8a977ee9d7c5ab4ac03c9b95431c4'] = 'MIT-0' + # https://raw.githubusercontent.com/faye/websocket-driver-node/0.7.3/LICENSE.md + crunched_md5sums['b66384e7137e41a9b1904ef4d39703b6'] = 'Apache-2.0' + # https://raw.githubusercontent.com/less/less.js/v4.1.1/LICENSE + crunched_md5sums['b27575459e02221ccef97ec0bfd457ae'] = 'Apache-2.0' + # https://raw.githubusercontent.com/microsoft/TypeScript/v3.5.3/LICENSE.txt + crunched_md5sums['a54a1a6a39e7f9dbb4a23a42f5c7fd1c'] = 'Apache-2.0' + # https://raw.githubusercontent.com/request/request/v2.87.0/LICENSE + crunched_md5sums['1034431802e57486b393d00c5d262b8a'] = 'Apache-2.0' + # https://raw.githubusercontent.com/dchest/tweetnacl-js/v0.14.5/LICENSE + crunched_md5sums['75605e6bdd564791ab698fca65c94a4f'] = 'Unlicense' + # https://raw.githubusercontent.com/stackgl/gl-mat3/v2.0.0/LICENSE.md + crunched_md5sums['75512892d6f59dddb6d1c7e191957e9c'] = 'Zlib' + + commonlicdir = d.getVar('COMMON_LICENSE_DIR') + for fn in sorted(os.listdir(commonlicdir)): + md5value, lictext = crunch_license(os.path.join(commonlicdir, fn)) + if md5value not in crunched_md5sums: + crunched_md5sums[md5value] = fn + elif fn != crunched_md5sums[md5value]: + bb.debug(2, "crunched_md5sums['%s'] is already set to '%s' rather than '%s'" % (md5value, crunched_md5sums[md5value], fn)) + else: + bb.debug(2, "crunched_md5sums['%s'] is already set to '%s'" % (md5value, crunched_md5sums[md5value])) + + return crunched_md5sums + +def crunch_license(licfile): + ''' + Remove non-material text from a license file and then calculate its + md5sum. This works well for licenses that contain a copyright statement, + but is also a useful way to handle people's insistence upon reformatting + the license text slightly (with no material difference to the text of the + license). + ''' + + import oe.utils + + # Note: these are carefully constructed! + license_title_re = re.compile(r'^#*\(? *(This is )?([Tt]he )?.{0,15} ?[Ll]icen[sc]e( \(.{1,10}\))?\)?[:\.]? ?#*$') + license_statement_re = re.compile(r'^((This (project|software)|.{1,10}) is( free software)? (released|licen[sc]ed)|(Released|Licen[cs]ed)) under the .{1,10} [Ll]icen[sc]e:?$') + copyright_re = re.compile(r'^ *[#\*]* *(Modified work |MIT LICENSED )?Copyright ?(\([cC]\))? .*$') + disclaimer_re = re.compile(r'^ *\*? ?All [Rr]ights [Rr]eserved\.$') + email_re = re.compile(r'^.*<[\w\.-]*@[\w\.\-]*>$') + header_re = re.compile(r'^(\/\**!?)? ?[\-=\*]* ?(\*\/)?$') + tag_re = re.compile(r'^ *@?\(?([Ll]icense|MIT)\)?$') + url_re = re.compile(r'^ *[#\*]* *https?:\/\/[\w\.\/\-]+$') + lictext = [] with open(licfile, 'r', errors='surrogateescape') as f: for line in f: # Drop opening statements if copyright_re.match(line): continue + elif disclaimer_re.match(line): + continue + elif email_re.match(line): + continue + elif header_re.match(line): + continue + elif tag_re.match(line): + continue + elif url_re.match(line): + continue elif license_title_re.match(line): continue elif license_statement_re.match(line): continue - # Squash spaces, and replace smart quotes, double quotes - # and backticks with single quotes + # Strip comment symbols + line = line.replace('*', '') \ + .replace('#', '') + # Unify spelling + line = line.replace('sub-license', 'sublicense') + # Squash spaces line = oe.utils.squashspaces(line.strip()) + # Replace smart quotes, double quotes and backticks with single quotes line = line.replace(u"\u2018", "'").replace(u"\u2019", "'").replace(u"\u201c","'").replace(u"\u201d", "'").replace('"', '\'').replace('`', '\'') + # Unify brackets + line = line.replace("{", "[").replace("}", "]") if line: lictext.append(line) @@ -1124,31 +1214,40 @@ def crunch_license(licfile): except UnicodeEncodeError: md5val = None lictext = '' - license = crunched_md5sums.get(md5val, None) - return license, md5val, lictext + return md5val, lictext def guess_license(srctree, d): import bb md5sums = get_license_md5sums(d) + crunched_md5sums = crunch_known_licenses(d) + licenses = [] licspecs = ['*LICEN[CS]E*', 'COPYING*', '*[Ll]icense*', 'LEGAL*', '[Ll]egal*', '*GPL*', 'README.lic*', 'COPYRIGHT*', '[Cc]opyright*', 'e[dp]l-v10'] + skip_extensions = (".html", ".js", ".json", ".svg", ".ts", ".go") licfiles = [] for root, dirs, files in os.walk(srctree): for fn in files: + if fn.endswith(skip_extensions): + continue for spec in licspecs: if fnmatch.fnmatch(fn, spec): fullpath = os.path.join(root, fn) if not fullpath in licfiles: licfiles.append(fullpath) - for licfile in licfiles: + for licfile in sorted(licfiles): md5value = bb.utils.md5_file(licfile) license = md5sums.get(md5value, None) if not license: - license, crunched_md5, lictext = crunch_license(licfile) - if not license: + crunched_md5, lictext = crunch_license(licfile) + license = crunched_md5sums.get(crunched_md5, None) + if lictext and not license: license = 'Unknown' - licenses.append((license, os.path.relpath(licfile, srctree), md5value)) + logger.info("Please add the following line for '%s' to a 'lib/recipetool/licenses.csv' " \ + "and replace `Unknown` with the license:\n" \ + "%s,Unknown" % (os.path.relpath(licfile, srctree), md5value)) + if license: + licenses.append((license, os.path.relpath(licfile, srctree), md5value)) # FIXME should we grab at least one source file with a license header and add that too? @@ -1162,6 +1261,7 @@ def split_pkg_licenses(licvalues, packages, outlines, fallback_licenses=None, pn """ pkglicenses = {pn: []} for license, licpath, _ in licvalues: + license = fixup_license(license) for pkgname, pkgpath in packages.items(): if licpath.startswith(pkgpath + '/'): if pkgname in pkglicenses: @@ -1174,11 +1274,14 @@ def split_pkg_licenses(licvalues, packages, outlines, fallback_licenses=None, pn pkglicenses[pn].append(license) outlicenses = {} for pkgname in packages: - license = ' '.join(list(set(pkglicenses.get(pkgname, ['Unknown'])))) or 'Unknown' - if license == 'Unknown' and pkgname in fallback_licenses: + # Assume AND operator between license files + license = ' & '.join(list(set(pkglicenses.get(pkgname, ['Unknown'])))) or 'Unknown' + if license == 'Unknown' and fallback_licenses and pkgname in fallback_licenses: license = fallback_licenses[pkgname] - outlines.append('LICENSE_%s = "%s"' % (pkgname, license)) - outlicenses[pkgname] = license.split() + licenses = tidy_licenses(license) + license = ' & '.join(licenses) + outlines.append('LICENSE:%s = "%s"' % (pkgname, license)) + outlicenses[pkgname] = licenses return outlicenses def read_pkgconfig_provides(d): @@ -1311,6 +1414,7 @@ def register_commands(subparsers): parser_create.add_argument('-B', '--srcbranch', help='Branch in source repository if fetching from an SCM such as git (default master)') parser_create.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') parser_create.add_argument('--npm-dev', action="store_true", help='For npm, also fetch devDependencies') + parser_create.add_argument('--no-pypi', action="store_true", help='Do not inherit pypi class') parser_create.add_argument('--devtool', action="store_true", help=argparse.SUPPRESS) parser_create.add_argument('--mirrors', action="store_true", help='Enable PREMIRRORS and MIRRORS for source tree fetching (disabled by default).') parser_create.set_defaults(func=create_recipe) diff --git a/scripts/lib/recipetool/create_buildsys.py b/scripts/lib/recipetool/create_buildsys.py index 35a97c9345..ec9d510e23 100644 --- a/scripts/lib/recipetool/create_buildsys.py +++ b/scripts/lib/recipetool/create_buildsys.py @@ -5,9 +5,9 @@ # SPDX-License-Identifier: GPL-2.0-only # +import os import re import logging -import glob from recipetool.create import RecipeHandler, validate_pv logger = logging.getLogger('recipetool') @@ -137,15 +137,15 @@ class CmakeRecipeHandler(RecipeHandler): deps = [] unmappedpkgs = [] - proj_re = re.compile('project\s*\(([^)]*)\)', re.IGNORECASE) - pkgcm_re = re.compile('pkg_check_modules\s*\(\s*[a-zA-Z0-9-_]+\s*(REQUIRED)?\s+([^)\s]+)\s*\)', re.IGNORECASE) - pkgsm_re = re.compile('pkg_search_module\s*\(\s*[a-zA-Z0-9-_]+\s*(REQUIRED)?((\s+[^)\s]+)+)\s*\)', re.IGNORECASE) - findpackage_re = re.compile('find_package\s*\(\s*([a-zA-Z0-9-_]+)\s*.*', re.IGNORECASE) - findlibrary_re = re.compile('find_library\s*\(\s*[a-zA-Z0-9-_]+\s*(NAMES\s+)?([a-zA-Z0-9-_ ]+)\s*.*') - checklib_re = re.compile('check_library_exists\s*\(\s*([^\s)]+)\s*.*', re.IGNORECASE) - include_re = re.compile('include\s*\(\s*([^)\s]*)\s*\)', re.IGNORECASE) - subdir_re = re.compile('add_subdirectory\s*\(\s*([^)\s]*)\s*([^)\s]*)\s*\)', re.IGNORECASE) - dep_re = re.compile('([^ ><=]+)( *[<>=]+ *[^ ><=]+)?') + proj_re = re.compile(r'project\s*\(([^)]*)\)', re.IGNORECASE) + pkgcm_re = re.compile(r'pkg_check_modules\s*\(\s*[a-zA-Z0-9-_]+\s*(REQUIRED)?\s+([^)\s]+)\s*\)', re.IGNORECASE) + pkgsm_re = re.compile(r'pkg_search_module\s*\(\s*[a-zA-Z0-9-_]+\s*(REQUIRED)?((\s+[^)\s]+)+)\s*\)', re.IGNORECASE) + findpackage_re = re.compile(r'find_package\s*\(\s*([a-zA-Z0-9-_]+)\s*.*', re.IGNORECASE) + findlibrary_re = re.compile(r'find_library\s*\(\s*[a-zA-Z0-9-_]+\s*(NAMES\s+)?([a-zA-Z0-9-_ ]+)\s*.*') + checklib_re = re.compile(r'check_library_exists\s*\(\s*([^\s)]+)\s*.*', re.IGNORECASE) + include_re = re.compile(r'include\s*\(\s*([^)\s]*)\s*\)', re.IGNORECASE) + subdir_re = re.compile(r'add_subdirectory\s*\(\s*([^)\s]*)\s*([^)\s]*)\s*\)', re.IGNORECASE) + dep_re = re.compile(r'([^ ><=]+)( *[<>=]+ *[^ ><=]+)?') def find_cmake_package(pkg): RecipeHandler.load_devel_filemap(tinfoil.config_data) @@ -423,16 +423,16 @@ class AutotoolsRecipeHandler(RecipeHandler): 'makeinfo': 'texinfo', } - pkg_re = re.compile('PKG_CHECK_MODULES\(\s*\[?[a-zA-Z0-9_]*\]?,\s*\[?([^,\]]*)\]?[),].*') - pkgce_re = re.compile('PKG_CHECK_EXISTS\(\s*\[?([^,\]]*)\]?[),].*') - lib_re = re.compile('AC_CHECK_LIB\(\s*\[?([^,\]]*)\]?,.*') - libx_re = re.compile('AX_CHECK_LIBRARY\(\s*\[?[^,\]]*\]?,\s*\[?([^,\]]*)\]?,\s*\[?([a-zA-Z0-9-]*)\]?,.*') - progs_re = re.compile('_PROGS?\(\s*\[?[a-zA-Z0-9_]*\]?,\s*\[?([^,\]]*)\]?[),].*') - dep_re = re.compile('([^ ><=]+)( [<>=]+ [^ ><=]+)?') - ac_init_re = re.compile('AC_INIT\(\s*([^,]+),\s*([^,]+)[,)].*') - am_init_re = re.compile('AM_INIT_AUTOMAKE\(\s*([^,]+),\s*([^,]+)[,)].*') - define_re = re.compile('\s*(m4_)?define\(\s*([^,]+),\s*([^,]+)\)') - version_re = re.compile('([0-9.]+)') + pkg_re = re.compile(r'PKG_CHECK_MODULES\(\s*\[?[a-zA-Z0-9_]*\]?,\s*\[?([^,\]]*)\]?[),].*') + pkgce_re = re.compile(r'PKG_CHECK_EXISTS\(\s*\[?([^,\]]*)\]?[),].*') + lib_re = re.compile(r'AC_CHECK_LIB\(\s*\[?([^,\]]*)\]?,.*') + libx_re = re.compile(r'AX_CHECK_LIBRARY\(\s*\[?[^,\]]*\]?,\s*\[?([^,\]]*)\]?,\s*\[?([a-zA-Z0-9-]*)\]?,.*') + progs_re = re.compile(r'_PROGS?\(\s*\[?[a-zA-Z0-9_]*\]?,\s*\[?([^,\]]*)\]?[),].*') + dep_re = re.compile(r'([^ ><=]+)( [<>=]+ [^ ><=]+)?') + ac_init_re = re.compile(r'AC_INIT\(\s*([^,]+),\s*([^,]+)[,)].*') + am_init_re = re.compile(r'AM_INIT_AUTOMAKE\(\s*([^,]+),\s*([^,]+)[,)].*') + define_re = re.compile(r'\s*(m4_)?define\(\s*([^,]+),\s*([^,]+)\)') + version_re = re.compile(r'([0-9.]+)') defines = {} def subst_defines(value): @@ -545,7 +545,7 @@ class AutotoolsRecipeHandler(RecipeHandler): deps.append('zlib') elif keyword in ('AX_CHECK_OPENSSL', 'AX_LIB_CRYPTO'): deps.append('openssl') - elif keyword == 'AX_LIB_CURL': + elif keyword in ('AX_LIB_CURL', 'LIBCURL_CHECK_CONFIG'): deps.append('curl') elif keyword == 'AX_LIB_BEECRYPT': deps.append('beecrypt') @@ -624,6 +624,7 @@ class AutotoolsRecipeHandler(RecipeHandler): 'AX_CHECK_OPENSSL', 'AX_LIB_CRYPTO', 'AX_LIB_CURL', + 'LIBCURL_CHECK_CONFIG', 'AX_LIB_BEECRYPT', 'AX_LIB_EXPAT', 'AX_LIB_GCRYPT', diff --git a/scripts/lib/recipetool/create_buildsys_python.py b/scripts/lib/recipetool/create_buildsys_python.py index adfa377956..a807dafae5 100644 --- a/scripts/lib/recipetool/create_buildsys_python.py +++ b/scripts/lib/recipetool/create_buildsys_python.py @@ -8,9 +8,9 @@ import ast import codecs import collections -import distutils.command.build_py +import setuptools.command.build_py import email -import imp +import importlib import glob import itertools import logging @@ -18,7 +18,11 @@ import os import re import sys import subprocess +import json +import urllib.request from recipetool.create import RecipeHandler +from urllib.parse import urldefrag +from recipetool.create import determine_from_url logger = logging.getLogger('recipetool') @@ -37,7 +41,334 @@ class PythonRecipeHandler(RecipeHandler): assume_provided = ['builtins', 'os.path'] # Assumes that the host python3 builtin_module_names is sane for target too assume_provided = assume_provided + list(sys.builtin_module_names) + excluded_fields = [] + + classifier_license_map = { + 'License :: OSI Approved :: Academic Free License (AFL)': 'AFL', + 'License :: OSI Approved :: Apache Software License': 'Apache', + 'License :: OSI Approved :: Apple Public Source License': 'APSL', + 'License :: OSI Approved :: Artistic License': 'Artistic', + 'License :: OSI Approved :: Attribution Assurance License': 'AAL', + 'License :: OSI Approved :: BSD License': 'BSD-3-Clause', + 'License :: OSI Approved :: Boost Software License 1.0 (BSL-1.0)': 'BSL-1.0', + 'License :: OSI Approved :: CEA CNRS Inria Logiciel Libre License, version 2.1 (CeCILL-2.1)': 'CECILL-2.1', + 'License :: OSI Approved :: Common Development and Distribution License 1.0 (CDDL-1.0)': 'CDDL-1.0', + 'License :: OSI Approved :: Common Public License': 'CPL', + 'License :: OSI Approved :: Eclipse Public License 1.0 (EPL-1.0)': 'EPL-1.0', + 'License :: OSI Approved :: Eclipse Public License 2.0 (EPL-2.0)': 'EPL-2.0', + 'License :: OSI Approved :: Eiffel Forum License': 'EFL', + 'License :: OSI Approved :: European Union Public Licence 1.0 (EUPL 1.0)': 'EUPL-1.0', + 'License :: OSI Approved :: European Union Public Licence 1.1 (EUPL 1.1)': 'EUPL-1.1', + 'License :: OSI Approved :: European Union Public Licence 1.2 (EUPL 1.2)': 'EUPL-1.2', + 'License :: OSI Approved :: GNU Affero General Public License v3': 'AGPL-3.0-only', + 'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)': 'AGPL-3.0-or-later', + 'License :: OSI Approved :: GNU Free Documentation License (FDL)': 'GFDL', + 'License :: OSI Approved :: GNU General Public License (GPL)': 'GPL', + 'License :: OSI Approved :: GNU General Public License v2 (GPLv2)': 'GPL-2.0-only', + 'License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)': 'GPL-2.0-or-later', + 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)': 'GPL-3.0-only', + 'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)': 'GPL-3.0-or-later', + 'License :: OSI Approved :: GNU Lesser General Public License v2 (LGPLv2)': 'LGPL-2.0-only', + 'License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+)': 'LGPL-2.0-or-later', + 'License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)': 'LGPL-3.0-only', + 'License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)': 'LGPL-3.0-or-later', + 'License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)': 'LGPL', + 'License :: OSI Approved :: Historical Permission Notice and Disclaimer (HPND)': 'HPND', + 'License :: OSI Approved :: IBM Public License': 'IPL', + 'License :: OSI Approved :: ISC License (ISCL)': 'ISC', + 'License :: OSI Approved :: Intel Open Source License': 'Intel', + 'License :: OSI Approved :: Jabber Open Source License': 'Jabber', + 'License :: OSI Approved :: MIT License': 'MIT', + 'License :: OSI Approved :: MIT No Attribution License (MIT-0)': 'MIT-0', + 'License :: OSI Approved :: MITRE Collaborative Virtual Workspace License (CVW)': 'CVWL', + 'License :: OSI Approved :: MirOS License (MirOS)': 'MirOS', + 'License :: OSI Approved :: Motosoto License': 'Motosoto', + 'License :: OSI Approved :: Mozilla Public License 1.0 (MPL)': 'MPL-1.0', + 'License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1)': 'MPL-1.1', + 'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)': 'MPL-2.0', + 'License :: OSI Approved :: Nethack General Public License': 'NGPL', + 'License :: OSI Approved :: Nokia Open Source License': 'Nokia', + 'License :: OSI Approved :: Open Group Test Suite License': 'OGTSL', + 'License :: OSI Approved :: Open Software License 3.0 (OSL-3.0)': 'OSL-3.0', + 'License :: OSI Approved :: PostgreSQL License': 'PostgreSQL', + 'License :: OSI Approved :: Python License (CNRI Python License)': 'CNRI-Python', + 'License :: OSI Approved :: Python Software Foundation License': 'PSF-2.0', + 'License :: OSI Approved :: Qt Public License (QPL)': 'QPL', + 'License :: OSI Approved :: Ricoh Source Code Public License': 'RSCPL', + 'License :: OSI Approved :: SIL Open Font License 1.1 (OFL-1.1)': 'OFL-1.1', + 'License :: OSI Approved :: Sleepycat License': 'Sleepycat', + 'License :: OSI Approved :: Sun Industry Standards Source License (SISSL)': 'SISSL', + 'License :: OSI Approved :: Sun Public License': 'SPL', + 'License :: OSI Approved :: The Unlicense (Unlicense)': 'Unlicense', + 'License :: OSI Approved :: Universal Permissive License (UPL)': 'UPL-1.0', + 'License :: OSI Approved :: University of Illinois/NCSA Open Source License': 'NCSA', + 'License :: OSI Approved :: Vovida Software License 1.0': 'VSL-1.0', + 'License :: OSI Approved :: W3C License': 'W3C', + 'License :: OSI Approved :: X.Net License': 'Xnet', + 'License :: OSI Approved :: Zope Public License': 'ZPL', + 'License :: OSI Approved :: zlib/libpng License': 'Zlib', + 'License :: Other/Proprietary License': 'Proprietary', + 'License :: Public Domain': 'PD', + } + + def __init__(self): + pass + + def process_url(self, args, classes, handled, extravalues): + """ + Convert any pypi url https://pypi.org/project/<package>/<version> into https://files.pythonhosted.org/packages/source/... + which corresponds to the archive location, and add pypi class + """ + + if 'url' in handled: + return None + + fetch_uri = None + source = args.source + required_version = args.version if args.version else None + match = re.match(r'https?://pypi.org/project/([^/]+)(?:/([^/]+))?/?$', urldefrag(source)[0]) + if match: + package = match.group(1) + version = match.group(2) if match.group(2) else required_version + + json_url = f"https://pypi.org/pypi/%s/json" % package + response = urllib.request.urlopen(json_url) + if response.status == 200: + data = json.loads(response.read()) + if not version: + # grab latest version + version = data["info"]["version"] + pypi_package = data["info"]["name"] + for release in reversed(data["releases"][version]): + if release["packagetype"] == "sdist": + fetch_uri = release["url"] + break + else: + logger.warning("Cannot handle pypi url %s: cannot fetch package information using %s", source, json_url) + return None + else: + match = re.match(r'^https?://files.pythonhosted.org/packages.*/(.*)-.*$', source) + if match: + fetch_uri = source + pypi_package = match.group(1) + _, version = determine_from_url(fetch_uri) + + if match and not args.no_pypi: + if required_version and version != required_version: + raise Exception("Version specified using --version/-V (%s) and version specified in the url (%s) do not match" % (required_version, version)) + # This is optionnal if BPN looks like "python-<pypi_package>" or "python3-<pypi_package>" (see pypi.bbclass) + # but at this point we cannot know because because user can specify the output name of the recipe on the command line + extravalues["PYPI_PACKAGE"] = pypi_package + # If the tarball extension is not 'tar.gz' (default value in pypi.bblcass) whe should set PYPI_PACKAGE_EXT in the recipe + pypi_package_ext = re.match(r'.*%s-%s\.(.*)$' % (pypi_package, version), fetch_uri) + if pypi_package_ext: + pypi_package_ext = pypi_package_ext.group(1) + if pypi_package_ext != "tar.gz": + extravalues["PYPI_PACKAGE_EXT"] = pypi_package_ext + + # Pypi class will handle S and SRC_URI variables, so remove them + # TODO: allow oe.recipeutils.patch_recipe_lines() to accept regexp so we can simplify the following to: + # extravalues['SRC_URI(?:\[.*?\])?'] = None + extravalues['S'] = None + extravalues['SRC_URI'] = None + + classes.append('pypi') + + handled.append('url') + return fetch_uri + + def handle_classifier_license(self, classifiers, existing_licenses=""): + + licenses = [] + for classifier in classifiers: + if classifier in self.classifier_license_map: + license = self.classifier_license_map[classifier] + if license == 'Apache' and 'Apache-2.0' in existing_licenses: + license = 'Apache-2.0' + elif license == 'GPL': + if 'GPL-2.0' in existing_licenses or 'GPLv2' in existing_licenses: + license = 'GPL-2.0' + elif 'GPL-3.0' in existing_licenses or 'GPLv3' in existing_licenses: + license = 'GPL-3.0' + elif license == 'LGPL': + if 'LGPL-2.1' in existing_licenses or 'LGPLv2.1' in existing_licenses: + license = 'LGPL-2.1' + elif 'LGPL-2.0' in existing_licenses or 'LGPLv2' in existing_licenses: + license = 'LGPL-2.0' + elif 'LGPL-3.0' in existing_licenses or 'LGPLv3' in existing_licenses: + license = 'LGPL-3.0' + licenses.append(license) + + if licenses: + return ' & '.join(licenses) + + return None + + def map_info_to_bbvar(self, info, extravalues): + + # Map PKG-INFO & setup.py fields to bitbake variables + for field, values in info.items(): + if field in self.excluded_fields: + continue + + if field not in self.bbvar_map: + continue + + if isinstance(values, str): + value = values + else: + value = ' '.join(str(v) for v in values if v) + + bbvar = self.bbvar_map[field] + if bbvar == "PN": + # by convention python recipes start with "python3-" + if not value.startswith('python'): + value = 'python3-' + value + + if bbvar not in extravalues and value: + extravalues[bbvar] = value + + def apply_info_replacements(self, info): + if not self.replacements: + return + + for variable, search, replace in self.replacements: + if variable not in info: + continue + + def replace_value(search, replace, value): + if replace is None: + if re.search(search, value): + return None + else: + new_value = re.sub(search, replace, value) + if value != new_value: + return new_value + return value + + value = info[variable] + if isinstance(value, str): + new_value = replace_value(search, replace, value) + if new_value is None: + del info[variable] + elif new_value != value: + info[variable] = new_value + elif hasattr(value, 'items'): + for dkey, dvalue in list(value.items()): + new_list = [] + for pos, a_value in enumerate(dvalue): + new_value = replace_value(search, replace, a_value) + if new_value is not None and new_value != value: + new_list.append(new_value) + + if value != new_list: + value[dkey] = new_list + else: + new_list = [] + for pos, a_value in enumerate(value): + new_value = replace_value(search, replace, a_value) + if new_value is not None and new_value != value: + new_list.append(new_value) + + if value != new_list: + info[variable] = new_list + + + def scan_python_dependencies(self, paths): + deps = set() + try: + dep_output = self.run_command(['pythondeps', '-d'] + paths) + except (OSError, subprocess.CalledProcessError): + pass + else: + for line in dep_output.splitlines(): + line = line.rstrip() + dep, filename = line.split('\t', 1) + if filename.endswith('/setup.py'): + continue + deps.add(dep) + + try: + provides_output = self.run_command(['pythondeps', '-p'] + paths) + except (OSError, subprocess.CalledProcessError): + pass + else: + provides_lines = (l.rstrip() for l in provides_output.splitlines()) + provides = set(l for l in provides_lines if l and l != 'setup') + deps -= provides + + return deps + + def parse_pkgdata_for_python_packages(self): + pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR') + + ldata = tinfoil.config_data.createCopy() + bb.parse.handle('classes-recipe/python3-dir.bbclass', ldata, True) + python_sitedir = ldata.getVar('PYTHON_SITEPACKAGES_DIR') + + dynload_dir = os.path.join(os.path.dirname(python_sitedir), 'lib-dynload') + python_dirs = [python_sitedir + os.sep, + os.path.join(os.path.dirname(python_sitedir), 'dist-packages') + os.sep, + os.path.dirname(python_sitedir) + os.sep] + packages = {} + for pkgdatafile in glob.glob('{}/runtime/*'.format(pkgdata_dir)): + files_info = None + with open(pkgdatafile, 'r') as f: + for line in f.readlines(): + field, value = line.split(': ', 1) + if field.startswith('FILES_INFO'): + files_info = ast.literal_eval(value) + break + else: + continue + + for fn in files_info: + for suffix in importlib.machinery.all_suffixes(): + if fn.endswith(suffix): + break + else: + continue + + if fn.startswith(dynload_dir + os.sep): + if '/.debug/' in fn: + continue + base = os.path.basename(fn) + provided = base.split('.', 1)[0] + packages[provided] = os.path.basename(pkgdatafile) + continue + + for python_dir in python_dirs: + if fn.startswith(python_dir): + relpath = fn[len(python_dir):] + relstart, _, relremaining = relpath.partition(os.sep) + if relstart.endswith('.egg'): + relpath = relremaining + base, _ = os.path.splitext(relpath) + + if '/.debug/' in base: + continue + if os.path.basename(base) == '__init__': + base = os.path.dirname(base) + base = base.replace(os.sep + os.sep, os.sep) + provided = base.replace(os.sep, '.') + packages[provided] = os.path.basename(pkgdatafile) + return packages + + @classmethod + def run_command(cls, cmd, **popenargs): + if 'stderr' not in popenargs: + popenargs['stderr'] = subprocess.STDOUT + try: + return subprocess.check_output(cmd, **popenargs).decode('utf-8') + except OSError as exc: + logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc) + raise + except subprocess.CalledProcessError as exc: + logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc.output) + raise + +class PythonSetupPyRecipeHandler(PythonRecipeHandler): bbvar_map = { 'Name': 'PN', 'Version': 'PV', @@ -45,9 +376,9 @@ class PythonRecipeHandler(RecipeHandler): 'Summary': 'SUMMARY', 'Description': 'DESCRIPTION', 'License': 'LICENSE', - 'Requires': 'RDEPENDS_${PN}', - 'Provides': 'RPROVIDES_${PN}', - 'Obsoletes': 'RREPLACES_${PN}', + 'Requires': 'RDEPENDS:${PN}', + 'Provides': 'RPROVIDES:${PN}', + 'Obsoletes': 'RREPLACES:${PN}', } # PN/PV are already set by recipetool core & desc can be extremely long excluded_fields = [ @@ -75,6 +406,7 @@ class PythonRecipeHandler(RecipeHandler): 'Supported-Platform', ] setuparg_multi_line_values = ['Description'] + replacements = [ ('License', r' +$', ''), ('License', r'^ +', ''), @@ -95,71 +427,161 @@ class PythonRecipeHandler(RecipeHandler): ('Install-requires', r'\[[^\]]+\]$', ''), ] - classifier_license_map = { - 'License :: OSI Approved :: Academic Free License (AFL)': 'AFL', - 'License :: OSI Approved :: Apache Software License': 'Apache', - 'License :: OSI Approved :: Apple Public Source License': 'APSL', - 'License :: OSI Approved :: Artistic License': 'Artistic', - 'License :: OSI Approved :: Attribution Assurance License': 'AAL', - 'License :: OSI Approved :: BSD License': 'BSD', - 'License :: OSI Approved :: Common Public License': 'CPL', - 'License :: OSI Approved :: Eiffel Forum License': 'EFL', - 'License :: OSI Approved :: European Union Public Licence 1.0 (EUPL 1.0)': 'EUPL-1.0', - 'License :: OSI Approved :: European Union Public Licence 1.1 (EUPL 1.1)': 'EUPL-1.1', - 'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)': 'AGPL-3.0+', - 'License :: OSI Approved :: GNU Affero General Public License v3': 'AGPL-3.0', - 'License :: OSI Approved :: GNU Free Documentation License (FDL)': 'GFDL', - 'License :: OSI Approved :: GNU General Public License (GPL)': 'GPL', - 'License :: OSI Approved :: GNU General Public License v2 (GPLv2)': 'GPL-2.0', - 'License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)': 'GPL-2.0+', - 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)': 'GPL-3.0', - 'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)': 'GPL-3.0+', - 'License :: OSI Approved :: GNU Lesser General Public License v2 (LGPLv2)': 'LGPL-2.0', - 'License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+)': 'LGPL-2.0+', - 'License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)': 'LGPL-3.0', - 'License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)': 'LGPL-3.0+', - 'License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)': 'LGPL', - 'License :: OSI Approved :: IBM Public License': 'IPL', - 'License :: OSI Approved :: ISC License (ISCL)': 'ISC', - 'License :: OSI Approved :: Intel Open Source License': 'Intel', - 'License :: OSI Approved :: Jabber Open Source License': 'Jabber', - 'License :: OSI Approved :: MIT License': 'MIT', - 'License :: OSI Approved :: MITRE Collaborative Virtual Workspace License (CVW)': 'CVWL', - 'License :: OSI Approved :: Motosoto License': 'Motosoto', - 'License :: OSI Approved :: Mozilla Public License 1.0 (MPL)': 'MPL-1.0', - 'License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1)': 'MPL-1.1', - 'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)': 'MPL-2.0', - 'License :: OSI Approved :: Nethack General Public License': 'NGPL', - 'License :: OSI Approved :: Nokia Open Source License': 'Nokia', - 'License :: OSI Approved :: Open Group Test Suite License': 'OGTSL', - 'License :: OSI Approved :: Python License (CNRI Python License)': 'CNRI-Python', - 'License :: OSI Approved :: Python Software Foundation License': 'PSF', - 'License :: OSI Approved :: Qt Public License (QPL)': 'QPL', - 'License :: OSI Approved :: Ricoh Source Code Public License': 'RSCPL', - 'License :: OSI Approved :: Sleepycat License': 'Sleepycat', - 'License :: OSI Approved :: Sun Industry Standards Source License (SISSL)': '-- Sun Industry Standards Source License (SISSL)', - 'License :: OSI Approved :: Sun Public License': 'SPL', - 'License :: OSI Approved :: University of Illinois/NCSA Open Source License': 'NCSA', - 'License :: OSI Approved :: Vovida Software License 1.0': 'VSL-1.0', - 'License :: OSI Approved :: W3C License': 'W3C', - 'License :: OSI Approved :: X.Net License': 'Xnet', - 'License :: OSI Approved :: Zope Public License': 'ZPL', - 'License :: OSI Approved :: zlib/libpng License': 'Zlib', - } - def __init__(self): pass + def parse_setup_py(self, setupscript='./setup.py'): + with codecs.open(setupscript) as f: + info, imported_modules, non_literals, extensions = gather_setup_info(f) + + def _map(key): + key = key.replace('_', '-') + key = key[0].upper() + key[1:] + if key in self.setup_parse_map: + key = self.setup_parse_map[key] + return key + + # Naive mapping of setup() arguments to PKG-INFO field names + for d in [info, non_literals]: + for key, value in list(d.items()): + if key is None: + continue + new_key = _map(key) + if new_key != key: + del d[key] + d[new_key] = value + + return info, 'setuptools' in imported_modules, non_literals, extensions + + def get_setup_args_info(self, setupscript='./setup.py'): + cmd = ['python3', setupscript] + info = {} + keys = set(self.bbvar_map.keys()) + keys |= set(self.setuparg_list_fields) + keys |= set(self.setuparg_multi_line_values) + grouped_keys = itertools.groupby(keys, lambda k: (k in self.setuparg_list_fields, k in self.setuparg_multi_line_values)) + for index, keys in grouped_keys: + if index == (True, False): + # Splitlines output for each arg as a list value + for key in keys: + arg = self.setuparg_map.get(key, key.lower()) + try: + arg_info = self.run_command(cmd + ['--' + arg], cwd=os.path.dirname(setupscript)) + except (OSError, subprocess.CalledProcessError): + pass + else: + info[key] = [l.rstrip() for l in arg_info.splitlines()] + elif index == (False, True): + # Entire output for each arg + for key in keys: + arg = self.setuparg_map.get(key, key.lower()) + try: + arg_info = self.run_command(cmd + ['--' + arg], cwd=os.path.dirname(setupscript)) + except (OSError, subprocess.CalledProcessError): + pass + else: + info[key] = arg_info + else: + info.update(self.get_setup_byline(list(keys), setupscript)) + return info + + def get_setup_byline(self, fields, setupscript='./setup.py'): + info = {} + + cmd = ['python3', setupscript] + cmd.extend('--' + self.setuparg_map.get(f, f.lower()) for f in fields) + try: + info_lines = self.run_command(cmd, cwd=os.path.dirname(setupscript)).splitlines() + except (OSError, subprocess.CalledProcessError): + pass + else: + if len(fields) != len(info_lines): + logger.error('Mismatch between setup.py output lines and number of fields') + sys.exit(1) + + for lineno, line in enumerate(info_lines): + line = line.rstrip() + info[fields[lineno]] = line + return info + + def get_pkginfo(self, pkginfo_fn): + msg = email.message_from_file(open(pkginfo_fn, 'r')) + msginfo = {} + for field in msg.keys(): + values = msg.get_all(field) + if len(values) == 1: + msginfo[field] = values[0] + else: + msginfo[field] = values + return msginfo + + def scan_setup_python_deps(self, srctree, setup_info, setup_non_literals): + if 'Package-dir' in setup_info: + package_dir = setup_info['Package-dir'] + else: + package_dir = {} + + dist = setuptools.Distribution() + + class PackageDir(setuptools.command.build_py.build_py): + def __init__(self, package_dir): + self.package_dir = package_dir + self.dist = dist + super().__init__(self.dist) + + pd = PackageDir(package_dir) + to_scan = [] + if not any(v in setup_non_literals for v in ['Py-modules', 'Scripts', 'Packages']): + if 'Py-modules' in setup_info: + for module in setup_info['Py-modules']: + try: + package, module = module.rsplit('.', 1) + except ValueError: + package, module = '.', module + module_path = os.path.join(pd.get_package_dir(package), module + '.py') + to_scan.append(module_path) + + if 'Packages' in setup_info: + for package in setup_info['Packages']: + to_scan.append(pd.get_package_dir(package)) + + if 'Scripts' in setup_info: + to_scan.extend(setup_info['Scripts']) + else: + logger.info("Scanning the entire source tree, as one or more of the following setup keywords are non-literal: py_modules, scripts, packages.") + + if not to_scan: + to_scan = ['.'] + + logger.info("Scanning paths for packages & dependencies: %s", ', '.join(to_scan)) + + provided_packages = self.parse_pkgdata_for_python_packages() + scanned_deps = self.scan_python_dependencies([os.path.join(srctree, p) for p in to_scan]) + mapped_deps, unmapped_deps = set(self.base_pkgdeps), set() + for dep in scanned_deps: + mapped = provided_packages.get(dep) + if mapped: + logger.debug('Mapped %s to %s' % (dep, mapped)) + mapped_deps.add(mapped) + else: + logger.debug('Could not map %s' % dep) + unmapped_deps.add(dep) + return mapped_deps, unmapped_deps + def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): + if 'buildsystem' in handled: return False + logger.debug("Trying setup.py parser") + # Check for non-zero size setup.py files setupfiles = RecipeHandler.checkfiles(srctree, ['setup.py']) for fn in setupfiles: if os.path.getsize(fn): break else: + logger.debug("No setup.py found") return False # setup.py is always parsed to get at certain required information, such as @@ -193,6 +615,18 @@ class PythonRecipeHandler(RecipeHandler): continue if line.startswith('['): + # PACKAGECONFIG must not contain expressions or whitespace + line = line.replace(" ", "") + line = line.replace(':', "") + line = line.replace('.', "-dot-") + line = line.replace('"', "") + line = line.replace('<', "-smaller-") + line = line.replace('>', "-bigger-") + line = line.replace('_', "-") + line = line.replace('(', "") + line = line.replace(')', "") + line = line.replace('!', "-not-") + line = line.replace('=', "-equals-") current_feature = line[1:-1] elif current_feature: extras_req[current_feature].append(line) @@ -226,51 +660,16 @@ class PythonRecipeHandler(RecipeHandler): if license_str: for i, line in enumerate(lines_before): - if line.startswith('LICENSE = '): + if line.startswith('##LICENSE_PLACEHOLDER##'): lines_before.insert(i, '# NOTE: License in setup.py/PKGINFO is: %s' % license_str) break if 'Classifier' in info: - existing_licenses = info.get('License', '') - licenses = [] - for classifier in info['Classifier']: - if classifier in self.classifier_license_map: - license = self.classifier_license_map[classifier] - if license == 'Apache' and 'Apache-2.0' in existing_licenses: - license = 'Apache-2.0' - elif license == 'GPL': - if 'GPL-2.0' in existing_licenses or 'GPLv2' in existing_licenses: - license = 'GPL-2.0' - elif 'GPL-3.0' in existing_licenses or 'GPLv3' in existing_licenses: - license = 'GPL-3.0' - elif license == 'LGPL': - if 'LGPL-2.1' in existing_licenses or 'LGPLv2.1' in existing_licenses: - license = 'LGPL-2.1' - elif 'LGPL-2.0' in existing_licenses or 'LGPLv2' in existing_licenses: - license = 'LGPL-2.0' - elif 'LGPL-3.0' in existing_licenses or 'LGPLv3' in existing_licenses: - license = 'LGPL-3.0' - licenses.append(license) - - if licenses: - info['License'] = ' & '.join(licenses) + license = self.handle_classifier_license(info['Classifier'], info.get('License', '')) + if license: + info['License'] = license - # Map PKG-INFO & setup.py fields to bitbake variables - for field, values in info.items(): - if field in self.excluded_fields: - continue - - if field not in self.bbvar_map: - continue - - if isinstance(values, str): - value = values - else: - value = ' '.join(str(v) for v in values if v) - - bbvar = self.bbvar_map[field] - if bbvar not in extravalues and value: - extravalues[bbvar] = value + self.map_info_to_bbvar(info, extravalues) mapped_deps, unmapped_deps = self.scan_setup_python_deps(srctree, setup_info, setup_non_literals) @@ -281,6 +680,7 @@ class PythonRecipeHandler(RecipeHandler): lines_after.append('# The following configs & dependencies are from setuptools extras_require.') lines_after.append('# These dependencies are optional, hence can be controlled via PACKAGECONFIG.') lines_after.append('# The upstream names may not correspond exactly to bitbake package names.') + lines_after.append('# The configs are might not correct, since PACKAGECONFIG does not support expressions as may used in requires.txt - they are just replaced by text.') lines_after.append('#') lines_after.append('# Uncomment this line to enable all the optional features.') lines_after.append('#PACKAGECONFIG ?= "{}"'.format(' '.join(k.lower() for k in extras_req))) @@ -301,7 +701,7 @@ class PythonRecipeHandler(RecipeHandler): inst_req_deps = ('python3-' + r.replace('.', '-').lower() for r in sorted(inst_reqs)) lines_after.append('# WARNING: the following rdepends are from setuptools install_requires. These') lines_after.append('# upstream names may not correspond exactly to bitbake package names.') - lines_after.append('RDEPENDS_${{PN}} += "{}"'.format(' '.join(inst_req_deps))) + lines_after.append('RDEPENDS:${{PN}} += "{}"'.format(' '.join(inst_req_deps))) if mapped_deps: name = info.get('Name') @@ -313,7 +713,7 @@ class PythonRecipeHandler(RecipeHandler): lines_after.append('') lines_after.append('# WARNING: the following rdepends are determined through basic analysis of the') lines_after.append('# python sources, and might not be 100% accurate.') - lines_after.append('RDEPENDS_${{PN}} += "{}"'.format(' '.join(sorted(mapped_deps)))) + lines_after.append('RDEPENDS:${{PN}} += "{}"'.format(' '.join(sorted(mapped_deps)))) unmapped_deps -= set(extensions) unmapped_deps -= set(self.assume_provided) @@ -326,275 +726,283 @@ class PythonRecipeHandler(RecipeHandler): handled.append('buildsystem') - def get_pkginfo(self, pkginfo_fn): - msg = email.message_from_file(open(pkginfo_fn, 'r')) - msginfo = {} - for field in msg.keys(): - values = msg.get_all(field) - if len(values) == 1: - msginfo[field] = values[0] - else: - msginfo[field] = values - return msginfo +class PythonPyprojectTomlRecipeHandler(PythonRecipeHandler): + """Base class to support PEP517 and PEP518 + + PEP517 https://peps.python.org/pep-0517/#source-trees + PEP518 https://peps.python.org/pep-0518/#build-system-table + """ + # bitbake currently supports the 4 following backends + build_backend_map = { + "setuptools.build_meta": "python_setuptools_build_meta", + "poetry.core.masonry.api": "python_poetry_core", + "flit_core.buildapi": "python_flit_core", + "hatchling.build": "python_hatchling", + "maturin": "python_maturin", + "mesonpy": "python_mesonpy", + } - def parse_setup_py(self, setupscript='./setup.py'): - with codecs.open(setupscript) as f: - info, imported_modules, non_literals, extensions = gather_setup_info(f) + # setuptools.build_meta and flit declare project metadata into the "project" section of pyproject.toml + # according to PEP-621: https://packaging.python.org/en/latest/specifications/declaring-project-metadata/#declaring-project-metadata + # while poetry uses the "tool.poetry" section according to its official documentation: https://python-poetry.org/docs/pyproject/ + # keys from "project" and "tool.poetry" sections are almost the same except for the HOMEPAGE which is "homepage" for tool.poetry + # and "Homepage" for "project" section. So keep both + bbvar_map = { + "name": "PN", + "version": "PV", + "Homepage": "HOMEPAGE", + "homepage": "HOMEPAGE", + "description": "SUMMARY", + "license": "LICENSE", + "dependencies": "RDEPENDS:${PN}", + "requires": "DEPENDS", + } - def _map(key): - key = key.replace('_', '-') - key = key[0].upper() + key[1:] - if key in self.setup_parse_map: - key = self.setup_parse_map[key] - return key + replacements = [ + ("license", r" +$", ""), + ("license", r"^ +", ""), + ("license", r" ", "-"), + ("license", r"^GNU-", ""), + ("license", r"-[Ll]icen[cs]e(,?-[Vv]ersion)?", ""), + ("license", r"^UNKNOWN$", ""), + # Remove currently unhandled version numbers from these variables + ("requires", r"\[[^\]]+\]$", ""), + ("requires", r"^([^><= ]+).*", r"\1"), + ("dependencies", r"\[[^\]]+\]$", ""), + ("dependencies", r"^([^><= ]+).*", r"\1"), + ] - # Naive mapping of setup() arguments to PKG-INFO field names - for d in [info, non_literals]: - for key, value in list(d.items()): - if key is None: - continue - new_key = _map(key) - if new_key != key: - del d[key] - d[new_key] = value + excluded_native_pkgdeps = [ + # already provided by python_setuptools_build_meta.bbclass + "python3-setuptools-native", + "python3-wheel-native", + # already provided by python_poetry_core.bbclass + "python3-poetry-core-native", + # already provided by python_flit_core.bbclass + "python3-flit-core-native", + # already provided by python_mesonpy + "python3-meson-python-native", + ] - return info, 'setuptools' in imported_modules, non_literals, extensions + # add here a list of known and often used packages and the corresponding bitbake package + known_deps_map = { + "setuptools": "python3-setuptools", + "wheel": "python3-wheel", + "poetry-core": "python3-poetry-core", + "flit_core": "python3-flit-core", + "setuptools-scm": "python3-setuptools-scm", + "hatchling": "python3-hatchling", + "hatch-vcs": "python3-hatch-vcs", + "meson-python" : "python3-meson-python", + } - def get_setup_args_info(self, setupscript='./setup.py'): - cmd = ['python3', setupscript] - info = {} - keys = set(self.bbvar_map.keys()) - keys |= set(self.setuparg_list_fields) - keys |= set(self.setuparg_multi_line_values) - grouped_keys = itertools.groupby(keys, lambda k: (k in self.setuparg_list_fields, k in self.setuparg_multi_line_values)) - for index, keys in grouped_keys: - if index == (True, False): - # Splitlines output for each arg as a list value - for key in keys: - arg = self.setuparg_map.get(key, key.lower()) - try: - arg_info = self.run_command(cmd + ['--' + arg], cwd=os.path.dirname(setupscript)) - except (OSError, subprocess.CalledProcessError): - pass - else: - info[key] = [l.rstrip() for l in arg_info.splitlines()] - elif index == (False, True): - # Entire output for each arg - for key in keys: - arg = self.setuparg_map.get(key, key.lower()) - try: - arg_info = self.run_command(cmd + ['--' + arg], cwd=os.path.dirname(setupscript)) - except (OSError, subprocess.CalledProcessError): - pass - else: - info[key] = arg_info - else: - info.update(self.get_setup_byline(list(keys), setupscript)) - return info + def __init__(self): + pass - def get_setup_byline(self, fields, setupscript='./setup.py'): + def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): info = {} + metadata = {} - cmd = ['python3', setupscript] - cmd.extend('--' + self.setuparg_map.get(f, f.lower()) for f in fields) - try: - info_lines = self.run_command(cmd, cwd=os.path.dirname(setupscript)).splitlines() - except (OSError, subprocess.CalledProcessError): - pass - else: - if len(fields) != len(info_lines): - logger.error('Mismatch between setup.py output lines and number of fields') - sys.exit(1) - - for lineno, line in enumerate(info_lines): - line = line.rstrip() - info[fields[lineno]] = line - return info - - def apply_info_replacements(self, info): - for variable, search, replace in self.replacements: - if variable not in info: - continue - - def replace_value(search, replace, value): - if replace is None: - if re.search(search, value): - return None - else: - new_value = re.sub(search, replace, value) - if value != new_value: - return new_value - return value - - value = info[variable] - if isinstance(value, str): - new_value = replace_value(search, replace, value) - if new_value is None: - del info[variable] - elif new_value != value: - info[variable] = new_value - elif hasattr(value, 'items'): - for dkey, dvalue in list(value.items()): - new_list = [] - for pos, a_value in enumerate(dvalue): - new_value = replace_value(search, replace, a_value) - if new_value is not None and new_value != value: - new_list.append(new_value) - - if value != new_list: - value[dkey] = new_list - else: - new_list = [] - for pos, a_value in enumerate(value): - new_value = replace_value(search, replace, a_value) - if new_value is not None and new_value != value: - new_list.append(new_value) - - if value != new_list: - info[variable] = new_list - - def scan_setup_python_deps(self, srctree, setup_info, setup_non_literals): - if 'Package-dir' in setup_info: - package_dir = setup_info['Package-dir'] - else: - package_dir = {} - - class PackageDir(distutils.command.build_py.build_py): - def __init__(self, package_dir): - self.package_dir = package_dir - - pd = PackageDir(package_dir) - to_scan = [] - if not any(v in setup_non_literals for v in ['Py-modules', 'Scripts', 'Packages']): - if 'Py-modules' in setup_info: - for module in setup_info['Py-modules']: - try: - package, module = module.rsplit('.', 1) - except ValueError: - package, module = '.', module - module_path = os.path.join(pd.get_package_dir(package), module + '.py') - to_scan.append(module_path) + if 'buildsystem' in handled: + return False - if 'Packages' in setup_info: - for package in setup_info['Packages']: - to_scan.append(pd.get_package_dir(package)) + logger.debug("Trying pyproject.toml parser") - if 'Scripts' in setup_info: - to_scan.extend(setup_info['Scripts']) + # Check for non-zero size setup.py files + setupfiles = RecipeHandler.checkfiles(srctree, ["pyproject.toml"]) + for fn in setupfiles: + if os.path.getsize(fn): + break else: - logger.info("Scanning the entire source tree, as one or more of the following setup keywords are non-literal: py_modules, scripts, packages.") - - if not to_scan: - to_scan = ['.'] - - logger.info("Scanning paths for packages & dependencies: %s", ', '.join(to_scan)) + logger.debug("No pyproject.toml found") + return False - provided_packages = self.parse_pkgdata_for_python_packages() - scanned_deps = self.scan_python_dependencies([os.path.join(srctree, p) for p in to_scan]) - mapped_deps, unmapped_deps = set(self.base_pkgdeps), set() - for dep in scanned_deps: - mapped = provided_packages.get(dep) - if mapped: - logger.debug('Mapped %s to %s' % (dep, mapped)) - mapped_deps.add(mapped) - else: - logger.debug('Could not map %s' % dep) - unmapped_deps.add(dep) - return mapped_deps, unmapped_deps + setupscript = os.path.join(srctree, "pyproject.toml") - def scan_python_dependencies(self, paths): - deps = set() try: - dep_output = self.run_command(['pythondeps', '-d'] + paths) - except (OSError, subprocess.CalledProcessError): - pass - else: - for line in dep_output.splitlines(): - line = line.rstrip() - dep, filename = line.split('\t', 1) - if filename.endswith('/setup.py'): - continue - deps.add(dep) + try: + import tomllib + except ImportError: + try: + import tomli as tomllib + except ImportError: + logger.error("Neither 'tomllib' nor 'tomli' could be imported, cannot scan pyproject.toml.") + return False + + try: + with open(setupscript, "rb") as f: + config = tomllib.load(f) + except Exception: + logger.exception("Failed to parse pyproject.toml") + return False + + build_backend = config["build-system"]["build-backend"] + if build_backend in self.build_backend_map: + classes.append(self.build_backend_map[build_backend]) + else: + logger.error( + "Unsupported build-backend: %s, cannot use pyproject.toml. Will try to use legacy setup.py" + % build_backend + ) + return False - try: - provides_output = self.run_command(['pythondeps', '-p'] + paths) - except (OSError, subprocess.CalledProcessError): - pass - else: - provides_lines = (l.rstrip() for l in provides_output.splitlines()) - provides = set(l for l in provides_lines if l and l != 'setup') - deps -= provides + licfile = "" - return deps + if build_backend == "poetry.core.masonry.api": + if "tool" in config and "poetry" in config["tool"]: + metadata = config["tool"]["poetry"] + else: + if "project" in config: + metadata = config["project"] + + if metadata: + for field, values in metadata.items(): + if field == "license": + # For setuptools.build_meta and flit, licence is a table + # but for poetry licence is a string + # for hatchling, both table (jsonschema) and string (iniconfig) have been used + if build_backend == "poetry.core.masonry.api": + value = values + else: + value = values.get("text", "") + if not value: + licfile = values.get("file", "") + continue + elif field == "dependencies" and build_backend == "poetry.core.masonry.api": + # For poetry backend, "dependencies" section looks like: + # [tool.poetry.dependencies] + # requests = "^2.13.0" + # requests = { version = "^2.13.0", source = "private" } + # See https://python-poetry.org/docs/master/pyproject/#dependencies-and-dependency-groups for more details + # This class doesn't handle versions anyway, so we just get the dependencies name here and construct a list + value = [] + for k in values.keys(): + value.append(k) + elif isinstance(values, dict): + for k, v in values.items(): + info[k] = v + continue + else: + value = values - def parse_pkgdata_for_python_packages(self): - suffixes = [t[0] for t in imp.get_suffixes()] - pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR') + info[field] = value - ldata = tinfoil.config_data.createCopy() - bb.parse.handle('classes/python3-dir.bbclass', ldata, True) - python_sitedir = ldata.getVar('PYTHON_SITEPACKAGES_DIR') + # Grab the license value before applying replacements + license_str = info.get("license", "").strip() - dynload_dir = os.path.join(os.path.dirname(python_sitedir), 'lib-dynload') - python_dirs = [python_sitedir + os.sep, - os.path.join(os.path.dirname(python_sitedir), 'dist-packages') + os.sep, - os.path.dirname(python_sitedir) + os.sep] - packages = {} - for pkgdatafile in glob.glob('{}/runtime/*'.format(pkgdata_dir)): - files_info = None - with open(pkgdatafile, 'r') as f: - for line in f.readlines(): - field, value = line.split(': ', 1) - if field == 'FILES_INFO': - files_info = ast.literal_eval(value) + if license_str: + for i, line in enumerate(lines_before): + if line.startswith("##LICENSE_PLACEHOLDER##"): + lines_before.insert( + i, "# NOTE: License in pyproject.toml is: %s" % license_str + ) break - else: - continue - for fn in files_info: - for suffix in suffixes: - if fn.endswith(suffix): - break - else: - continue + info["requires"] = config["build-system"]["requires"] + + self.apply_info_replacements(info) + + if "classifiers" in info: + license = self.handle_classifier_license( + info["classifiers"], info.get("license", "") + ) + if license: + if licfile: + lines = [] + md5value = bb.utils.md5_file(os.path.join(srctree, licfile)) + lines.append('LICENSE = "%s"' % license) + lines.append( + 'LIC_FILES_CHKSUM = "file://%s;md5=%s"' + % (licfile, md5value) + ) + lines.append("") + + # Replace the placeholder so we get the values in the right place in the recipe file + try: + pos = lines_before.index("##LICENSE_PLACEHOLDER##") + except ValueError: + pos = -1 + if pos == -1: + lines_before.extend(lines) + else: + lines_before[pos : pos + 1] = lines - if fn.startswith(dynload_dir + os.sep): - if '/.debug/' in fn: - continue - base = os.path.basename(fn) - provided = base.split('.', 1)[0] - packages[provided] = os.path.basename(pkgdatafile) - continue + handled.append(("license", [license, licfile, md5value])) + else: + info["license"] = license - for python_dir in python_dirs: - if fn.startswith(python_dir): - relpath = fn[len(python_dir):] - relstart, _, relremaining = relpath.partition(os.sep) - if relstart.endswith('.egg'): - relpath = relremaining - base, _ = os.path.splitext(relpath) + provided_packages = self.parse_pkgdata_for_python_packages() + provided_packages.update(self.known_deps_map) + native_mapped_deps, native_unmapped_deps = set(), set() + mapped_deps, unmapped_deps = set(), set() - if '/.debug/' in base: - continue - if os.path.basename(base) == '__init__': - base = os.path.dirname(base) - base = base.replace(os.sep + os.sep, os.sep) - provided = base.replace(os.sep, '.') - packages[provided] = os.path.basename(pkgdatafile) - return packages + if "requires" in info: + for require in info["requires"]: + mapped = provided_packages.get(require) - @classmethod - def run_command(cls, cmd, **popenargs): - if 'stderr' not in popenargs: - popenargs['stderr'] = subprocess.STDOUT - try: - return subprocess.check_output(cmd, **popenargs).decode('utf-8') - except OSError as exc: - logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc) - raise - except subprocess.CalledProcessError as exc: - logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc.output) - raise + if mapped: + logger.debug("Mapped %s to %s" % (require, mapped)) + native_mapped_deps.add(mapped) + else: + logger.debug("Could not map %s" % require) + native_unmapped_deps.add(require) + + info.pop("requires") + + if native_mapped_deps != set(): + native_mapped_deps = { + item + "-native" for item in native_mapped_deps + } + native_mapped_deps -= set(self.excluded_native_pkgdeps) + if native_mapped_deps != set(): + info["requires"] = " ".join(sorted(native_mapped_deps)) + + if native_unmapped_deps: + lines_after.append("") + lines_after.append( + "# WARNING: We were unable to map the following python package/module" + ) + lines_after.append( + "# dependencies to the bitbake packages which include them:" + ) + lines_after.extend( + "# {}".format(d) for d in sorted(native_unmapped_deps) + ) + + if "dependencies" in info: + for dependency in info["dependencies"]: + mapped = provided_packages.get(dependency) + if mapped: + logger.debug("Mapped %s to %s" % (dependency, mapped)) + mapped_deps.add(mapped) + else: + logger.debug("Could not map %s" % dependency) + unmapped_deps.add(dependency) + + info.pop("dependencies") + + if mapped_deps != set(): + if mapped_deps != set(): + info["dependencies"] = " ".join(sorted(mapped_deps)) + + if unmapped_deps: + lines_after.append("") + lines_after.append( + "# WARNING: We were unable to map the following python package/module" + ) + lines_after.append( + "# runtime dependencies to the bitbake packages which include them:" + ) + lines_after.extend( + "# {}".format(d) for d in sorted(unmapped_deps) + ) + + self.map_info_to_bbvar(info, extravalues) + + handled.append("buildsystem") + except Exception: + logger.exception("Failed to correctly handle pyproject.toml, falling back to another method") + return False def gather_setup_info(fileobj): @@ -710,5 +1118,7 @@ def has_non_literals(value): def register_recipe_handlers(handlers): - # We need to make sure this is ahead of the makefile fallback handler - handlers.append((PythonRecipeHandler(), 70)) + # We need to make sure these are ahead of the makefile fallback handler + # and the pyproject.toml handler ahead of the setup.py handler + handlers.append((PythonPyprojectTomlRecipeHandler(), 75)) + handlers.append((PythonSetupPyRecipeHandler(), 70)) diff --git a/scripts/lib/recipetool/create_go.py b/scripts/lib/recipetool/create_go.py new file mode 100644 index 0000000000..a85a2f2786 --- /dev/null +++ b/scripts/lib/recipetool/create_go.py @@ -0,0 +1,777 @@ +# Recipe creation tool - go support plugin +# +# The code is based on golang internals. See the afftected +# methods for further reference and information. +# +# Copyright (C) 2023 Weidmueller GmbH & Co KG +# Author: Lukas Funke <lukas.funke@weidmueller.com> +# +# SPDX-License-Identifier: GPL-2.0-only +# + + +from collections import namedtuple +from enum import Enum +from html.parser import HTMLParser +from recipetool.create import RecipeHandler, handle_license_vars +from recipetool.create import guess_license, tidy_licenses, fixup_license +from recipetool.create import determine_from_url +from urllib.error import URLError, HTTPError + +import bb.utils +import json +import logging +import os +import re +import subprocess +import sys +import shutil +import tempfile +import urllib.parse +import urllib.request + + +GoImport = namedtuple('GoImport', 'root vcs url suffix') +logger = logging.getLogger('recipetool') +CodeRepo = namedtuple( + 'CodeRepo', 'path codeRoot codeDir pathMajor pathPrefix pseudoMajor') + +tinfoil = None + +# Regular expression to parse pseudo semantic version +# see https://go.dev/ref/mod#pseudo-versions +re_pseudo_semver = re.compile( + r"^v[0-9]+\.(0\.0-|\d+\.\d+-([^+]*\.)?0\.)(?P<utc>\d{14})-(?P<commithash>[A-Za-z0-9]+)(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$") +# Regular expression to parse semantic version +re_semver = re.compile( + r"^v(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$") + + +def tinfoil_init(instance): + global tinfoil + tinfoil = instance + + +class GoRecipeHandler(RecipeHandler): + """Class to handle the go recipe creation""" + + @staticmethod + def __ensure_go(): + """Check if the 'go' command is available in the recipes""" + recipe = "go-native" + if not tinfoil.recipes_parsed: + tinfoil.parse_recipes() + try: + rd = tinfoil.parse_recipe(recipe) + except bb.providers.NoProvider: + bb.error( + "Nothing provides '%s' which is required for the build" % (recipe)) + bb.note( + "You will likely need to add a layer that provides '%s'" % (recipe)) + return None + + bindir = rd.getVar('STAGING_BINDIR_NATIVE') + gopath = os.path.join(bindir, 'go') + + if not os.path.exists(gopath): + tinfoil.build_targets(recipe, 'addto_recipe_sysroot') + + if not os.path.exists(gopath): + logger.error( + '%s required to process specified source, but %s did not seem to populate it' % 'go', recipe) + return None + + return bindir + + def __resolve_repository_static(self, modulepath): + """Resolve the repository in a static manner + + The method is based on the go implementation of + `repoRootFromVCSPaths` in + https://github.com/golang/go/blob/master/src/cmd/go/internal/vcs/vcs.go + """ + + url = urllib.parse.urlparse("https://" + modulepath) + req = urllib.request.Request(url.geturl()) + + try: + resp = urllib.request.urlopen(req) + # Some modulepath are just redirects to github (or some other vcs + # hoster). Therefore, we check if this modulepath redirects to + # somewhere else + if resp.geturl() != url.geturl(): + bb.debug(1, "%s is redirectred to %s" % + (url.geturl(), resp.geturl())) + url = urllib.parse.urlparse(resp.geturl()) + modulepath = url.netloc + url.path + + except URLError as url_err: + # This is probably because the module path + # contains the subdir and major path. Thus, + # we ignore this error for now + logger.debug( + 1, "Failed to fetch page from [%s]: %s" % (url, str(url_err))) + + host, _, _ = modulepath.partition('/') + + class vcs(Enum): + pathprefix = "pathprefix" + regexp = "regexp" + type = "type" + repo = "repo" + check = "check" + schemelessRepo = "schemelessRepo" + + # GitHub + vcsGitHub = {} + vcsGitHub[vcs.pathprefix] = "github.com" + vcsGitHub[vcs.regexp] = re.compile( + r'^(?P<root>github\.com/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(/(?P<suffix>[A-Za-z0-9_.\-]+))*$') + vcsGitHub[vcs.type] = "git" + vcsGitHub[vcs.repo] = "https://\\g<root>" + + # Bitbucket + vcsBitbucket = {} + vcsBitbucket[vcs.pathprefix] = "bitbucket.org" + vcsBitbucket[vcs.regexp] = re.compile( + r'^(?P<root>bitbucket\.org/(?P<bitname>[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))(/(?P<suffix>[A-Za-z0-9_.\-]+))*$') + vcsBitbucket[vcs.type] = "git" + vcsBitbucket[vcs.repo] = "https://\\g<root>" + + # IBM DevOps Services (JazzHub) + vcsIBMDevOps = {} + vcsIBMDevOps[vcs.pathprefix] = "hub.jazz.net/git" + vcsIBMDevOps[vcs.regexp] = re.compile( + r'^(?P<root>hub\.jazz\.net/git/[a-z0-9]+/[A-Za-z0-9_.\-]+)(/(?P<suffix>[A-Za-z0-9_.\-]+))*$') + vcsIBMDevOps[vcs.type] = "git" + vcsIBMDevOps[vcs.repo] = "https://\\g<root>" + + # Git at Apache + vcsApacheGit = {} + vcsApacheGit[vcs.pathprefix] = "git.apache.org" + vcsApacheGit[vcs.regexp] = re.compile( + r'^(?P<root>git\.apache\.org/[a-z0-9_.\-]+\.git)(/(?P<suffix>[A-Za-z0-9_.\-]+))*$') + vcsApacheGit[vcs.type] = "git" + vcsApacheGit[vcs.repo] = "https://\\g<root>" + + # Git at OpenStack + vcsOpenStackGit = {} + vcsOpenStackGit[vcs.pathprefix] = "git.openstack.org" + vcsOpenStackGit[vcs.regexp] = re.compile( + r'^(?P<root>git\.openstack\.org/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(\.git)?(/(?P<suffix>[A-Za-z0-9_.\-]+))*$') + vcsOpenStackGit[vcs.type] = "git" + vcsOpenStackGit[vcs.repo] = "https://\\g<root>" + + # chiselapp.com for fossil + vcsChiselapp = {} + vcsChiselapp[vcs.pathprefix] = "chiselapp.com" + vcsChiselapp[vcs.regexp] = re.compile( + r'^(?P<root>chiselapp\.com/user/[A-Za-z0-9]+/repository/[A-Za-z0-9_.\-]+)$') + vcsChiselapp[vcs.type] = "fossil" + vcsChiselapp[vcs.repo] = "https://\\g<root>" + + # General syntax for any server. + # Must be last. + vcsGeneralServer = {} + vcsGeneralServer[vcs.regexp] = re.compile( + "(?P<root>(?P<repo>([a-z0-9.\\-]+\\.)+[a-z0-9.\\-]+(:[0-9]+)?(/~?[A-Za-z0-9_.\\-]+)+?)\\.(?P<vcs>bzr|fossil|git|hg|svn))(/~?(?P<suffix>[A-Za-z0-9_.\\-]+))*$") + vcsGeneralServer[vcs.schemelessRepo] = True + + vcsPaths = [vcsGitHub, vcsBitbucket, vcsIBMDevOps, + vcsApacheGit, vcsOpenStackGit, vcsChiselapp, + vcsGeneralServer] + + if modulepath.startswith("example.net") or modulepath == "rsc.io": + logger.warning("Suspicious module path %s" % modulepath) + return None + if modulepath.startswith("http:") or modulepath.startswith("https:"): + logger.warning("Import path should not start with %s %s" % + ("http", "https")) + return None + + rootpath = None + vcstype = None + repourl = None + suffix = None + + for srv in vcsPaths: + m = srv[vcs.regexp].match(modulepath) + if vcs.pathprefix in srv: + if host == srv[vcs.pathprefix]: + rootpath = m.group('root') + vcstype = srv[vcs.type] + repourl = m.expand(srv[vcs.repo]) + suffix = m.group('suffix') + break + elif m and srv[vcs.schemelessRepo]: + rootpath = m.group('root') + vcstype = m[vcs.type] + repourl = m[vcs.repo] + suffix = m.group('suffix') + break + + return GoImport(rootpath, vcstype, repourl, suffix) + + def __resolve_repository_dynamic(self, modulepath): + """Resolve the repository root in a dynamic manner. + + The method is based on the go implementation of + `repoRootForImportDynamic` in + https://github.com/golang/go/blob/master/src/cmd/go/internal/vcs/vcs.go + """ + url = urllib.parse.urlparse("https://" + modulepath) + + class GoImportHTMLParser(HTMLParser): + + def __init__(self): + super().__init__() + self.__srv = {} + + def handle_starttag(self, tag, attrs): + if tag == 'meta' and list( + filter(lambda a: (a[0] == 'name' and a[1] == 'go-import'), attrs)): + content = list( + filter(lambda a: (a[0] == 'content'), attrs)) + if content: + srv = content[0][1].split() + self.__srv[srv[0]] = srv + + def go_import(self, modulepath): + if modulepath in self.__srv: + srv = self.__srv[modulepath] + return GoImport(srv[0], srv[1], srv[2], None) + return None + + url = url.geturl() + "?go-get=1" + req = urllib.request.Request(url) + + try: + body = urllib.request.urlopen(req).read() + except HTTPError as http_err: + logger.warning( + "Unclean status when fetching page from [%s]: %s", url, str(http_err)) + body = http_err.fp.read() + except URLError as url_err: + logger.warning( + "Failed to fetch page from [%s]: %s", url, str(url_err)) + return None + + parser = GoImportHTMLParser() + parser.feed(body.decode('utf-8')) + parser.close() + + return parser.go_import(modulepath) + + def __resolve_from_golang_proxy(self, modulepath, version): + """ + Resolves repository data from golang proxy + """ + url = urllib.parse.urlparse("https://proxy.golang.org/" + + modulepath + + "/@v/" + + version + + ".info") + + # Transform url to lower case, golang proxy doesn't like mixed case + req = urllib.request.Request(url.geturl().lower()) + + try: + resp = urllib.request.urlopen(req) + except URLError as url_err: + logger.warning( + "Failed to fetch page from [%s]: %s", url, str(url_err)) + return None + + golang_proxy_res = resp.read().decode('utf-8') + modinfo = json.loads(golang_proxy_res) + + if modinfo and 'Origin' in modinfo: + origin = modinfo['Origin'] + _root_url = urllib.parse.urlparse(origin['URL']) + + # We normalize the repo URL since we don't want the scheme in it + _subdir = origin['Subdir'] if 'Subdir' in origin else None + _root, _, _ = self.__split_path_version(modulepath) + if _subdir: + _root = _root[:-len(_subdir)].strip('/') + + _commit = origin['Hash'] + _vcs = origin['VCS'] + return (GoImport(_root, _vcs, _root_url.geturl(), None), _commit) + + return None + + def __resolve_repository(self, modulepath): + """ + Resolves src uri from go module-path + """ + repodata = self.__resolve_repository_static(modulepath) + if not repodata or not repodata.url: + repodata = self.__resolve_repository_dynamic(modulepath) + if not repodata or not repodata.url: + logger.error( + "Could not resolve repository for module path '%s'" % modulepath) + # There is no way to recover from this + sys.exit(14) + if repodata: + logger.debug(1, "Resolved download path for import '%s' => %s" % ( + modulepath, repodata.url)) + return repodata + + def __split_path_version(self, path): + i = len(path) + dot = False + for j in range(i, 0, -1): + if path[j - 1] < '0' or path[j - 1] > '9': + break + if path[j - 1] == '.': + dot = True + break + i = j - 1 + + if i <= 1 or i == len( + path) or path[i - 1] != 'v' or path[i - 2] != '/': + return path, "", True + + prefix, pathMajor = path[:i - 2], path[i - 2:] + if dot or len( + pathMajor) <= 2 or pathMajor[2] == '0' or pathMajor == "/v1": + return path, "", False + + return prefix, pathMajor, True + + def __get_path_major(self, pathMajor): + if not pathMajor: + return "" + + if pathMajor[0] != '/' and pathMajor[0] != '.': + logger.error( + "pathMajor suffix %s passed to PathMajorPrefix lacks separator", pathMajor) + + if pathMajor.startswith(".v") and pathMajor.endswith("-unstable"): + pathMajor = pathMajor[:len("-unstable") - 2] + + return pathMajor[1:] + + def __build_coderepo(self, repo, path): + codedir = "" + pathprefix, pathMajor, _ = self.__split_path_version(path) + if repo.root == path: + pathprefix = path + elif path.startswith(repo.root): + codedir = pathprefix[len(repo.root):].strip('/') + + pseudoMajor = self.__get_path_major(pathMajor) + + logger.debug("root='%s', codedir='%s', prefix='%s', pathMajor='%s', pseudoMajor='%s'", + repo.root, codedir, pathprefix, pathMajor, pseudoMajor) + + return CodeRepo(path, repo.root, codedir, + pathMajor, pathprefix, pseudoMajor) + + def __resolve_version(self, repo, path, version): + hash = None + coderoot = self.__build_coderepo(repo, path) + + def vcs_fetch_all(): + tmpdir = tempfile.mkdtemp() + clone_cmd = "%s clone --bare %s %s" % ('git', repo.url, tmpdir) + bb.process.run(clone_cmd) + log_cmd = "git log --all --pretty='%H %d' --decorate=short" + output, _ = bb.process.run( + log_cmd, shell=True, stderr=subprocess.PIPE, cwd=tmpdir) + bb.utils.prunedir(tmpdir) + return output.strip().split('\n') + + def vcs_fetch_remote(tag): + # add * to grab ^{} + refs = {} + ls_remote_cmd = "git ls-remote -q --tags {} {}*".format( + repo.url, tag) + output, _ = bb.process.run(ls_remote_cmd) + output = output.strip().split('\n') + for line in output: + f = line.split(maxsplit=1) + if len(f) != 2: + continue + + for prefix in ["HEAD", "refs/heads/", "refs/tags/"]: + if f[1].startswith(prefix): + refs[f[1][len(prefix):]] = f[0] + + for key, hash in refs.items(): + if key.endswith(r"^{}"): + refs[key.strip(r"^{}")] = hash + + return refs[tag] + + m_pseudo_semver = re_pseudo_semver.match(version) + + if m_pseudo_semver: + remote_refs = vcs_fetch_all() + short_commit = m_pseudo_semver.group('commithash') + for l in remote_refs: + r = l.split(maxsplit=1) + sha1 = r[0] if len(r) else None + if not sha1: + logger.error( + "Ups: could not resolve abbref commit for %s" % short_commit) + + elif sha1.startswith(short_commit): + hash = sha1 + break + else: + m_semver = re_semver.match(version) + if m_semver: + + def get_sha1_remote(re): + rsha1 = None + for line in remote_refs: + # Split lines of the following format: + # 22e90d9b964610628c10f673ca5f85b8c2a2ca9a (tag: sometag) + lineparts = line.split(maxsplit=1) + sha1 = lineparts[0] if len(lineparts) else None + refstring = lineparts[1] if len( + lineparts) == 2 else None + if refstring: + # Normalize tag string and split in case of multiple + # regs e.g. (tag: speech/v1.10.0, tag: orchestration/v1.5.0 ...) + refs = refstring.strip('(), ').split(',') + for ref in refs: + if re.match(ref.strip()): + rsha1 = sha1 + return rsha1 + + semver = "v" + m_semver.group('major') + "."\ + + m_semver.group('minor') + "."\ + + m_semver.group('patch') \ + + (("-" + m_semver.group('prerelease')) + if m_semver.group('prerelease') else "") + + tag = os.path.join( + coderoot.codeDir, semver) if coderoot.codeDir else semver + + # probe tag using 'ls-remote', which is faster than fetching + # complete history + hash = vcs_fetch_remote(tag) + if not hash: + # backup: fetch complete history + remote_refs = vcs_fetch_all() + hash = get_sha1_remote( + re.compile(fr"(tag:|HEAD ->) ({tag})")) + + logger.debug( + "Resolving commit for tag '%s' -> '%s'", tag, hash) + return hash + + def __generate_srcuri_inline_fcn(self, path, version, replaces=None): + """Generate SRC_URI functions for go imports""" + + logger.info("Resolving repository for module %s", path) + # First try to resolve repo and commit from golang proxy + # Most info is already there and we don't have to go through the + # repository or even perform the version resolve magic + golang_proxy_info = self.__resolve_from_golang_proxy(path, version) + if golang_proxy_info: + repo = golang_proxy_info[0] + commit = golang_proxy_info[1] + else: + # Fallback + # Resolve repository by 'hand' + repo = self.__resolve_repository(path) + commit = self.__resolve_version(repo, path, version) + + url = urllib.parse.urlparse(repo.url) + repo_url = url.netloc + url.path + + coderoot = self.__build_coderepo(repo, path) + + inline_fcn = "${@go_src_uri(" + inline_fcn += f"'{repo_url}','{version}'" + if repo_url != path: + inline_fcn += f",path='{path}'" + if coderoot.codeDir: + inline_fcn += f",subdir='{coderoot.codeDir}'" + if repo.vcs != 'git': + inline_fcn += f",vcs='{repo.vcs}'" + if replaces: + inline_fcn += f",replaces='{replaces}'" + if coderoot.pathMajor: + inline_fcn += f",pathmajor='{coderoot.pathMajor}'" + inline_fcn += ")}" + + return inline_fcn, commit + + def __go_handle_dependencies(self, go_mod, srctree, localfilesdir, extravalues, d): + + import re + src_uris = [] + src_revs = [] + + def generate_src_rev(path, version, commithash): + src_rev = f"# {path}@{version} => {commithash}\n" + # Ups...maybe someone manipulated the source repository and the + # version or commit could not be resolved. This is a sign of + # a) the supply chain was manipulated (bad) + # b) the implementation for the version resolving didn't work + # anymore (less bad) + if not commithash: + src_rev += f"#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n" + src_rev += f"#!!! Could not resolve version !!!\n" + src_rev += f"#!!! Possible supply chain attack !!!\n" + src_rev += f"#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n" + src_rev += f"SRCREV_{path.replace('/', '.')} = \"{commithash}\"" + + return src_rev + + # we first go over replacement list, because we are essentialy + # interested only in the replaced path + if go_mod['Replace']: + for replacement in go_mod['Replace']: + oldpath = replacement['Old']['Path'] + path = replacement['New']['Path'] + version = '' + if 'Version' in replacement['New']: + version = replacement['New']['Version'] + + if os.path.exists(os.path.join(srctree, path)): + # the module refers to the local path, remove it from requirement list + # because it's a local module + go_mod['Require'][:] = [v for v in go_mod['Require'] if v.get('Path') != oldpath] + else: + # Replace the path and the version, so we don't iterate replacement list anymore + for require in go_mod['Require']: + if require['Path'] == oldpath: + require.update({'Path': path, 'Version': version}) + break + + for require in go_mod['Require']: + path = require['Path'] + version = require['Version'] + + inline_fcn, commithash = self.__generate_srcuri_inline_fcn( + path, version) + src_uris.append(inline_fcn) + src_revs.append(generate_src_rev(path, version, commithash)) + + # strip version part from module URL /vXX + baseurl = re.sub(r'/v(\d+)$', '', go_mod['Module']['Path']) + pn, _ = determine_from_url(baseurl) + go_mods_basename = "%s-modules.inc" % pn + + go_mods_filename = os.path.join(localfilesdir, go_mods_basename) + with open(go_mods_filename, "w") as f: + # We introduce this indirection to make the tests a little easier + f.write("SRC_URI += \"${GO_DEPENDENCIES_SRC_URI}\"\n") + f.write("GO_DEPENDENCIES_SRC_URI = \"\\\n") + for uri in src_uris: + f.write(" " + uri + " \\\n") + f.write("\"\n\n") + for rev in src_revs: + f.write(rev + "\n") + + extravalues['extrafiles'][go_mods_basename] = go_mods_filename + + def __go_run_cmd(self, cmd, cwd, d): + return bb.process.run(cmd, env=dict(os.environ, PATH=d.getVar('PATH')), + shell=True, cwd=cwd) + + def __go_native_version(self, d): + stdout, _ = self.__go_run_cmd("go version", None, d) + m = re.match(r".*\sgo((\d+).(\d+).(\d+))\s([\w\/]*)", stdout) + major = int(m.group(2)) + minor = int(m.group(3)) + patch = int(m.group(4)) + + return major, minor, patch + + def __go_mod_patch(self, srctree, localfilesdir, extravalues, d): + + patchfilename = "go.mod.patch" + go_native_version_major, go_native_version_minor, _ = self.__go_native_version( + d) + self.__go_run_cmd("go mod tidy -go=%d.%d" % + (go_native_version_major, go_native_version_minor), srctree, d) + stdout, _ = self.__go_run_cmd("go mod edit -json", srctree, d) + + # Create patch in order to upgrade go version + self.__go_run_cmd("git diff go.mod > %s" % (patchfilename), srctree, d) + # Restore original state + self.__go_run_cmd("git checkout HEAD go.mod go.sum", srctree, d) + + go_mod = json.loads(stdout) + tmpfile = os.path.join(localfilesdir, patchfilename) + shutil.move(os.path.join(srctree, patchfilename), tmpfile) + + extravalues['extrafiles'][patchfilename] = tmpfile + + return go_mod, patchfilename + + def __go_mod_vendor(self, go_mod, srctree, localfilesdir, extravalues, d): + # Perform vendoring to retrieve the correct modules.txt + tmp_vendor_dir = tempfile.mkdtemp() + + # -v causes to go to print modules.txt to stderr + _, stderr = self.__go_run_cmd( + "go mod vendor -v -o %s" % (tmp_vendor_dir), srctree, d) + + modules_txt_basename = "modules.txt" + modules_txt_filename = os.path.join(localfilesdir, modules_txt_basename) + with open(modules_txt_filename, "w") as f: + f.write(stderr) + + extravalues['extrafiles'][modules_txt_basename] = modules_txt_filename + + licenses = [] + lic_files_chksum = [] + licvalues = guess_license(tmp_vendor_dir, d) + shutil.rmtree(tmp_vendor_dir) + + if licvalues: + for licvalue in licvalues: + license = licvalue[0] + lics = tidy_licenses(fixup_license(license)) + lics = [lic for lic in lics if lic not in licenses] + if len(lics): + licenses.extend(lics) + lic_files_chksum.append( + 'file://src/${GO_IMPORT}/vendor/%s;md5=%s' % (licvalue[1], licvalue[2])) + + # strip version part from module URL /vXX + baseurl = re.sub(r'/v(\d+)$', '', go_mod['Module']['Path']) + pn, _ = determine_from_url(baseurl) + licenses_basename = "%s-licenses.inc" % pn + + licenses_filename = os.path.join(localfilesdir, licenses_basename) + with open(licenses_filename, "w") as f: + f.write("GO_MOD_LICENSES = \"%s\"\n\n" % + ' & '.join(sorted(licenses, key=str.casefold))) + # We introduce this indirection to make the tests a little easier + f.write("LIC_FILES_CHKSUM += \"${VENDORED_LIC_FILES_CHKSUM}\"\n") + f.write("VENDORED_LIC_FILES_CHKSUM = \"\\\n") + for lic in lic_files_chksum: + f.write(" " + lic + " \\\n") + f.write("\"\n") + + extravalues['extrafiles'][licenses_basename] = licenses_filename + + def process(self, srctree, classes, lines_before, + lines_after, handled, extravalues): + + if 'buildsystem' in handled: + return False + + files = RecipeHandler.checkfiles(srctree, ['go.mod']) + if not files: + return False + + d = bb.data.createCopy(tinfoil.config_data) + go_bindir = self.__ensure_go() + if not go_bindir: + sys.exit(14) + + d.prependVar('PATH', '%s:' % go_bindir) + handled.append('buildsystem') + classes.append("go-vendor") + + stdout, _ = self.__go_run_cmd("go mod edit -json", srctree, d) + + go_mod = json.loads(stdout) + go_import = go_mod['Module']['Path'] + go_version_match = re.match("([0-9]+).([0-9]+)", go_mod['Go']) + go_version_major = int(go_version_match.group(1)) + go_version_minor = int(go_version_match.group(2)) + src_uris = [] + + localfilesdir = tempfile.mkdtemp(prefix='recipetool-go-') + extravalues.setdefault('extrafiles', {}) + + # Use an explicit name determined from the module name because it + # might differ from the actual URL for replaced modules + # strip version part from module URL /vXX + baseurl = re.sub(r'/v(\d+)$', '', go_mod['Module']['Path']) + pn, _ = determine_from_url(baseurl) + + # go.mod files with version < 1.17 may not include all indirect + # dependencies. Thus, we have to upgrade the go version. + if go_version_major == 1 and go_version_minor < 17: + logger.warning( + "go.mod files generated by Go < 1.17 might have incomplete indirect dependencies.") + go_mod, patchfilename = self.__go_mod_patch(srctree, localfilesdir, + extravalues, d) + src_uris.append( + "file://%s;patchdir=src/${GO_IMPORT}" % (patchfilename)) + + # Check whether the module is vendored. If so, we have nothing to do. + # Otherwise we gather all dependencies and add them to the recipe + if not os.path.exists(os.path.join(srctree, "vendor")): + + # Write additional $BPN-modules.inc file + self.__go_mod_vendor(go_mod, srctree, localfilesdir, extravalues, d) + lines_before.append("LICENSE += \" & ${GO_MOD_LICENSES}\"") + lines_before.append("require %s-licenses.inc" % (pn)) + + self.__rewrite_src_uri(lines_before, ["file://modules.txt"]) + + self.__go_handle_dependencies(go_mod, srctree, localfilesdir, extravalues, d) + lines_before.append("require %s-modules.inc" % (pn)) + + # Do generic license handling + handle_license_vars(srctree, lines_before, handled, extravalues, d) + self.__rewrite_lic_uri(lines_before) + + lines_before.append("GO_IMPORT = \"{}\"".format(baseurl)) + lines_before.append("SRCREV_FORMAT = \"${BPN}\"") + + def __update_lines_before(self, updated, newlines, lines_before): + if updated: + del lines_before[:] + for line in newlines: + # Hack to avoid newlines that edit_metadata inserts + if line.endswith('\n'): + line = line[:-1] + lines_before.append(line) + return updated + + def __rewrite_lic_uri(self, lines_before): + + def varfunc(varname, origvalue, op, newlines): + if varname == 'LIC_FILES_CHKSUM': + new_licenses = [] + licenses = origvalue.split('\\') + for license in licenses: + if not license: + logger.warning("No license file was detected for the main module!") + # the license list of the main recipe must be empty + # this can happen for example in case of CLOSED license + # Fall through to complete recipe generation + continue + license = license.strip() + uri, chksum = license.split(';', 1) + url = urllib.parse.urlparse(uri) + new_uri = os.path.join( + url.scheme + "://", "src", "${GO_IMPORT}", url.netloc + url.path) + ";" + chksum + new_licenses.append(new_uri) + + return new_licenses, None, -1, True + return origvalue, None, 0, True + + updated, newlines = bb.utils.edit_metadata( + lines_before, ['LIC_FILES_CHKSUM'], varfunc) + return self.__update_lines_before(updated, newlines, lines_before) + + def __rewrite_src_uri(self, lines_before, additional_uris = []): + + def varfunc(varname, origvalue, op, newlines): + if varname == 'SRC_URI': + src_uri = ["git://${GO_IMPORT};destsuffix=git/src/${GO_IMPORT};nobranch=1;name=${BPN};protocol=https"] + src_uri.extend(additional_uris) + return src_uri, None, -1, True + return origvalue, None, 0, True + + updated, newlines = bb.utils.edit_metadata(lines_before, ['SRC_URI'], varfunc) + return self.__update_lines_before(updated, newlines, lines_before) + + +def register_recipe_handlers(handlers): + handlers.append((GoRecipeHandler(), 60)) diff --git a/scripts/lib/recipetool/create_kmod.py b/scripts/lib/recipetool/create_kmod.py index 85b5c48e53..cc00106961 100644 --- a/scripts/lib/recipetool/create_kmod.py +++ b/scripts/lib/recipetool/create_kmod.py @@ -113,7 +113,7 @@ class KernelModuleRecipeHandler(RecipeHandler): kdirpath, _ = check_target(compile_lines, install=False) if manual_install or not install_lines: - lines_after.append('EXTRA_OEMAKE_append_task-install = " -C ${STAGING_KERNEL_DIR} M=${S}"') + lines_after.append('EXTRA_OEMAKE:append:task-install = " -C ${STAGING_KERNEL_DIR} M=${S}"') elif install_target and install_target != 'modules_install': lines_after.append('MODULES_INSTALL_TARGET = "install"') diff --git a/scripts/lib/recipetool/create_npm.py b/scripts/lib/recipetool/create_npm.py index 579b7ae48a..113a89f6a6 100644 --- a/scripts/lib/recipetool/create_npm.py +++ b/scripts/lib/recipetool/create_npm.py @@ -6,16 +6,20 @@ """Recipe creation tool - npm module support plugin""" import json +import logging import os import re import sys import tempfile import bb from bb.fetch2.npm import NpmEnvironment +from bb.fetch2.npm import npm_package from bb.fetch2.npmsw import foreach_dependencies from recipetool.create import RecipeHandler +from recipetool.create import get_license_md5sums from recipetool.create import guess_license from recipetool.create import split_pkg_licenses +logger = logging.getLogger('recipetool') TINFOIL = None @@ -28,15 +32,6 @@ class NpmRecipeHandler(RecipeHandler): """Class to handle the npm recipe creation""" @staticmethod - def _npm_name(name): - """Generate a Yocto friendly npm name""" - name = re.sub("/", "-", name) - name = name.lower() - name = re.sub(r"[^\-a-z0-9]", "", name) - name = name.strip("-") - return name - - @staticmethod def _get_registry(lines): """Get the registry value from the 'npm://registry' url""" registry = None @@ -118,23 +113,32 @@ class NpmRecipeHandler(RecipeHandler): licfiles = [] packages = {} - def _licfiles_append(licfile): - """Append 'licfile' to the license files list""" - licfilepath = os.path.join(srctree, licfile) - licmd5 = bb.utils.md5_file(licfilepath) - licfiles.append("file://%s;md5=%s" % (licfile, licmd5)) - # Handle the parent package - _licfiles_append("package.json") packages["${PN}"] = "" + def _licfiles_append_fallback_readme_files(destdir): + """Append README files as fallback to license files if a license files is missing""" + + fallback = True + readmes = [] + basedir = os.path.join(srctree, destdir) + for fn in os.listdir(basedir): + upper = fn.upper() + if upper.startswith("README"): + fullpath = os.path.join(basedir, fn) + readmes.append(fullpath) + if upper.startswith("COPYING") or "LICENCE" in upper or "LICENSE" in upper: + fallback = False + if fallback: + for readme in readmes: + licfiles.append(os.path.relpath(readme, srctree)) + # Handle the dependencies - def _handle_dependency(name, params, deptree): - suffix = "-".join([self._npm_name(dep) for dep in deptree]) - destdirs = [os.path.join("node_modules", dep) for dep in deptree] - destdir = os.path.join(*destdirs) - _licfiles_append(os.path.join(destdir, "package.json")) - packages["${PN}-" + suffix] = destdir + def _handle_dependency(name, params, destdir): + deptree = destdir.split('node_modules/') + suffix = "-".join([npm_package(dep) for dep in deptree]) + packages["${PN}" + suffix] = destdir + _licfiles_append_fallback_readme_files(destdir) with open(shrinkwrap_file, "r") as f: shrinkwrap = json.load(f) @@ -142,6 +146,23 @@ class NpmRecipeHandler(RecipeHandler): foreach_dependencies(shrinkwrap, _handle_dependency, dev) return licfiles, packages + + # Handle the peer dependencies + def _handle_peer_dependency(self, shrinkwrap_file): + """Check if package has peer dependencies and show warning if it is the case""" + with open(shrinkwrap_file, "r") as f: + shrinkwrap = json.load(f) + + packages = shrinkwrap.get("packages", {}) + peer_deps = packages.get("", {}).get("peerDependencies", {}) + + for peer_dep in peer_deps: + peer_dep_yocto_name = npm_package(peer_dep) + bb.warn(peer_dep + " is a peer dependencie of the actual package. " + + "Please add this peer dependencie to the RDEPENDS variable as %s and generate its recipe with devtool" + % peer_dep_yocto_name) + + def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): """Handle the npm recipe creation""" @@ -160,7 +181,7 @@ class NpmRecipeHandler(RecipeHandler): if "name" not in data or "version" not in data: return False - extravalues["PN"] = self._npm_name(data["name"]) + extravalues["PN"] = npm_package(data["name"]) extravalues["PV"] = data["version"] if "description" in data: @@ -204,6 +225,9 @@ class NpmRecipeHandler(RecipeHandler): self._run_npm_install(d, srctree, registry, dev) shrinkwrap_file = self._generate_shrinkwrap(d, srctree, dev) + with open(shrinkwrap_file, "r") as f: + shrinkwrap = json.load(f) + if os.path.exists(lock_copy): bb.utils.movefile(lock_copy, lock_file) @@ -226,7 +250,8 @@ class NpmRecipeHandler(RecipeHandler): value = origvalue.replace("version=" + data["version"], "version=${PV}") value = value.replace("version=latest", "version=${PV}") values = [line.strip() for line in value.strip('\n').splitlines()] - values.append(url_recipe) + if "dependencies" in shrinkwrap.get("packages", {}).get("", {}): + values.append(url_recipe) return values, None, 4, False (_, newlines) = bb.utils.edit_metadata(lines_before, ["SRC_URI"], _handle_srcuri) @@ -242,12 +267,42 @@ class NpmRecipeHandler(RecipeHandler): bb.note("Handling licences ...") (licfiles, packages) = self._handle_licenses(srctree, shrinkwrap_file, dev) - extravalues["LIC_FILES_CHKSUM"] = licfiles - split_pkg_licenses(guess_license(srctree, d), packages, lines_after, []) + + def _guess_odd_license(licfiles): + import bb + + md5sums = get_license_md5sums(d, linenumbers=True) + + chksums = [] + licenses = [] + for licfile in licfiles: + f = os.path.join(srctree, licfile) + md5value = bb.utils.md5_file(f) + (license, beginline, endline, md5) = md5sums.get(md5value, + (None, "", "", "")) + if not license: + license = "Unknown" + logger.info("Please add the following line for '%s' to a " + "'lib/recipetool/licenses.csv' and replace `Unknown`, " + "`X`, `Y` and `MD5` with the license, begin line, " + "end line and partial MD5 checksum:\n" \ + "%s,Unknown,X,Y,MD5" % (licfile, md5value)) + chksums.append("file://%s%s%s;md5=%s" % (licfile, + ";beginline=%s" % (beginline) if beginline else "", + ";endline=%s" % (endline) if endline else "", + md5 if md5 else md5value)) + licenses.append((license, licfile, md5value)) + return (licenses, chksums) + + (licenses, extravalues["LIC_FILES_CHKSUM"]) = _guess_odd_license(licfiles) + split_pkg_licenses([*licenses, *guess_license(srctree, d)], packages, lines_after) classes.append("npm") handled.append("buildsystem") + # Check if package has peer dependencies and inform the user + self._handle_peer_dependency(shrinkwrap_file) + return True def register_recipe_handlers(handlers): diff --git a/scripts/lib/recipetool/licenses.csv b/scripts/lib/recipetool/licenses.csv new file mode 100644 index 0000000000..80851111b3 --- /dev/null +++ b/scripts/lib/recipetool/licenses.csv @@ -0,0 +1,37 @@ +0636e73ff0215e8d672dc4c32c317bb3,GPL-2.0-only +12f884d2ae1ff87c09e5b7ccc2c4ca7e,GPL-2.0-only +18810669f13b87348459e611d31ab760,GPL-2.0-only +252890d9eee26aab7b432e8b8a616475,LGPL-2.0-only +2d5025d4aa3495befef8f17206a5b0a1,LGPL-2.1-only +3214f080875748938ba060314b4f727d,LGPL-2.0-only +385c55653886acac3821999a3ccd17b3,Artistic-1.0 | GPL-2.0-only +393a5ca445f6965873eca0259a17f833,GPL-2.0-only +3b83ef96387f14655fc854ddc3c6bd57,Apache-2.0 +3bf50002aefd002f49e7bb854063f7e7,LGPL-2.0-only +4325afd396febcb659c36b49533135d4,GPL-2.0-only +4fbd65380cdd255951079008b364516c,LGPL-2.1-only +54c7042be62e169199200bc6477f04d1,BSD-3-Clause +55ca817ccb7d5b5b66355690e9abc605,LGPL-2.0-only +59530bdf33659b29e73d4adb9f9f6552,GPL-2.0-only +5f30f0716dfdd0d91eb439ebec522ec2,LGPL-2.0-only +6a6a8e020838b23406c81b19c1d46df6,LGPL-3.0-only +751419260aa954499f7abaabaa882bbe,GPL-2.0-only +7fbc338309ac38fefcd64b04bb903e34,LGPL-2.1-only +8ca43cbc842c2336e835926c2166c28b,GPL-2.0-only +94d55d512a9ba36caa9b7df079bae19f,GPL-2.0-only +9ac2e7cff1ddaf48b6eab6028f23ef88,GPL-2.0-only +9f604d8a4f8e74f4f5140845a21b6674,LGPL-2.0-only +a6f89e2100d9b6cdffcea4f398e37343,LGPL-2.1-only +b234ee4d69f5fce4486a80fdaf4a4263,GPL-2.0-only +bbb461211a33b134d42ed5ee802b37ff,LGPL-2.1-only +bfe1f75d606912a4111c90743d6c7325,MPL-1.1-only +c93c0550bd3173f4504b2cbd8991e50b,GPL-2.0-only +d32239bcb673463ab874e80d47fae504,GPL-3.0-only +d7810fab7487fb0aad327b76f1be7cd7,GPL-2.0-only +d8045f3b8f929c1cb29a1e3fd737b499,LGPL-2.1-only +db979804f025cf55aabec7129cb671ed,LGPL-2.0-only +eb723b61539feef013de476e68b5c50a,GPL-2.0-only +ebb5c50ab7cab4baeffba14977030c07,GPL-2.0-only +f27defe1e96c2e1ecd4e0c9be8967949,GPL-3.0-only +fad9b3332be894bab9bc501572864b29,LGPL-2.1-only +fbc093901857fcd118f065f900982c24,LGPL-2.1-only diff --git a/scripts/lib/recipetool/setvar.py b/scripts/lib/recipetool/setvar.py index f8e2ee75fb..b5ad335cae 100644 --- a/scripts/lib/recipetool/setvar.py +++ b/scripts/lib/recipetool/setvar.py @@ -49,6 +49,7 @@ def setvar(args): for patch in patches: for line in patch: sys.stdout.write(line) + tinfoil.modified_files() return 0 |