#!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0-only # # kernel fragment manipulation utility # # Copyright (C) 2021 Bruce Ashfield # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import sys import os import glob from collections import OrderedDict from pathlib import Path import textwrap import contextlib import shutil import tempfile import argparse import re import subprocess import pathlib @contextlib.contextmanager def cd(newdir, cleanup=lambda: True): prevdir = os.getcwd() os.chdir(os.path.expanduser(newdir)) try: yield finally: os.chdir(prevdir) cleanup() @contextlib.contextmanager def tempdir(): dirpath = tempfile.mkdtemp() def cleanup(): # todo: make this contingent on a flag if not save_temps: shutil.rmtree(dirpath) pass with cd(dirpath, cleanup): yield dirpath def split_option( config_option_str ): option = config_option_str.rstrip( '\n' ) opt = None val = None try: m = re.match( r"(CONFIG_[^= ]+)=([^ ]+.*)", option) opt = m.group(1) val = m.group(2) except: if re.search( "^#\s*CONFIG_", option ): # print( "option is a is not set!!! %s" % option ) m = re.match(r"# (CONFIG_[^ ]+) is not set", option ) if m: opt = m.group(1) val = "n" else: # this is an invalid option opt = "invalid option format" val = option # a fully commented line, is not an option and val .. elif re.search(r"^#.*$", option ): opt = None val = None elif re.search( r".*= *", option): # space after equals opt = "invalid option format" val = option elif re.search( r" *=", option): # space before equals opt = "invalid option format" val = option return opt,val def config_queue_read( config_queue_file ): if verbose: print( "[INFO]: reading config.queue from: %s" % config_queue_file ) p = Path(config_queue_file ) frag_dir = p.parent # frag dict is a dictionary indexed by fragment path + name, that points # to a dictionary of config options -> values frag_dict = OrderedDict() # option dict is a diectionary indexed by the option name. it points to # a diectionary of fragment name and the value the fragment set it to. option_dict = OrderedDict() issues_dict = OrderedDict() issues_dict["duplicated_option"] = OrderedDict() issues_dict["redefined_option"] = OrderedDict() issues_dict["malformated_option"] = OrderedDict() non_hw_class_dict = OrderedDict() hw_class_dict = OrderedDict() try: p.resolve(True) except: return frag_dict,option_dict,issues_dict,hw_class_dict,non_hw_class_dict # There are two passes through the queue. # # First pass: # - the fragments and how they are included for classification. # "kconf include" .. # - broad classification instructions (.kcf files) # # Second pass: # - special files and their classification, this allows us to # override / reclassify something that was "kconf included" # a certain way. # first pass start with open(config_queue_file) as fp: for cnt, line in enumerate(fp): fragment = line.split( '#' ) frag_name = fragment[0].rstrip() frag_type = fragment[1].rstrip().lstrip() # print("Line {}: {}".format(cnt, line)) # print("{} Fragment: {}/{}".format(frag_type,frag_dir,frag_name)) frag_dict[frag_name] = OrderedDict() frag_path = Path( str(frag_dir) + "/" + frag_name.rstrip() ) frag_full_path = frag_path.resolve() frag_dirname = frag_path.parent.resolve() non_hardware_classification = Path( str(frag_dirname) + "/non-hardware.kcf" ).resolve() hardware_classification = Path( str(frag_dirname) + "/hardware.kcf" ).resolve() if non_hardware_classification.exists() and use_classifiers: if verbose: print( "[DBG]: classification found: %s" % non_hardware_classification ) with open( str(non_hardware_classification) ) as classification_file: for cline in classification_file: kconfig_start = Path( ksrc + "/" + cline.rstrip() ) if kconfig_start.exists(): with open( str(kconfig_start) ) as kfile: for kline in kfile: m = re.match( r"^(menu).*config (\w*)", kline ) if m: non_hw_class_dict[m.group(2)] = classification_file.name # being in both h/w and non-hardware leads us to results where we # can't easily silence a warning. So if we've added it to h/w we need # remove it from non-hardware. If we do it in both the hardware and # non hardware cases, the end result is that the last classification # wins, which is what we want. try: del hw_class_dict[m.group(2)] except: pass m = re.match( r"^config (\w*)", kline ) if m: non_hw_class_dict[m.group(1)] = classification_file.name # being in both h/w and non-hardware leads us to results where we # can't easily silence a warning. So if we've added it to h/w we need # remove it from non-hardware. If we do it in both the hardware and # non hardware cases, the end result is that the last classification # wins, which is what we want. try: del hw_class_dict[m.group(1)] except: pass # this is jut too slow, but keeping it for reference. # try: # ck = kconfiglib.Kconfig( cline.rstrip(), warn=False ) # except: # ck = None # if ck: # #print( "=================== %s" % cline.rstrip() ) # for c in ck.unique_defined_syms: # #print( "c: %s" % c.name ) # non_hw_class_dict[c.name] = classification_file # #print( ck.unique_defined_syms ) else: if verbose: print( "[WARNING]: %s does not exist, but was a classifier" % kconfig_start ) if hardware_classification.exists() and use_classifiers: if verbose: print( "[DBG]: hardware classification found: %s" % hardware_classification ) with open( str(hardware_classification) ) as classification_file: for cline in classification_file: kconfig_start = Path( ksrc + "/" + cline.rstrip() ) if kconfig_start.exists(): with open( str(kconfig_start) ) as kfile: for kline in kfile: m = re.match( r"^(menu).*config (\w*)", kline ) if m: hw_class_dict[m.group(2)] = classification_file.name # being in both h/w and non-hardware leads us to results where we # can't easily silence a warning. So if we've added it to h/w we need # remove it from non-hardware. If we do it in both the hardware and # non hardware cases, the end result is that the last classification # wins, which is what we want. try: del non_hw_class_dict[m.group(2)] except: pass m = re.match( r"^config (\w*)", kline ) if m: hw_class_dict[m.group(1).rstrip()] = classification_file.name # being in both h/w and non-hardware leads us to results where we # can't easily silence a warning. So if we've added it to h/w we need # remove it from non-hardware. If we do it in both the hardware and # non hardware cases, the end result is that the last classification # wins, which is what we want. try: del non_hw_class_dict[m.group(1)] except: pass else: if verbose: print( "[WARNING]: %s does not exist, but was a classifier" % kconfig_start ) with open( str(frag_path) ) as config_frag: for cline in config_frag: c,value = split_option( cline ) o_noprefix = "" if c: o_noprefix = re.sub( "^CONFIG_", "", c ) if c == "invalid option format": if frag_name in issues_dict["malformated_option"]: issues_dict["malformated_option"][frag_name].append( value ) else: issues_dict["malformated_option"][frag_name] = [ value ] elif c: if frag_type == "hardware": if use_classifiers: # print( " hw classification for: %s,%s" % (o_noprefix,str(frag_path))) hw_class_dict[o_noprefix] = str(frag_path) # we can't be both hardware and software, so we should be deleting # one or the other if in both try: del non_hw_class_dict[o_noprefix.rstrip()] except: pass elif frag_type == "non-hardware": if use_classifiers: # we can't be both hardware and software, so we should be deleting # one or the other if in both non_hw_class_dict[o_noprefix] = str(frag_path) try: # print( "deleting %s from hw_class, since it is also software" % o_noprefix.rstrip() ) del hw_class_dict[o_noprefix.rstrip()] except: pass if not c in frag_dict[frag_name]: frag_dict[frag_name][c] = value else: if frag_name in issues_dict["duplicated_option"]: issues_dict["duplicated_option"][frag_name].append( c ) else: issues_dict["duplicated_option"][frag_name] = [ c ] if not c in option_dict: option_dict[c] = OrderedDict() option_dict[c][frag_name] = value # 2nd pass start with open(config_queue_file) as fp: for cnt, line in enumerate(fp): fragment = line.split( '#' ) frag_name = fragment[0].rstrip() frag_type = fragment[1].rstrip().lstrip() # print("Line {}: {}".format(cnt, line)) # print("{} Fragment: {}/{}".format(frag_type,frag_dir,frag_name)) frag_path = Path( str(frag_dir) + "/" + frag_name.rstrip() ) frag_full_path = frag_path.resolve() frag_dirname = frag_path.parent.resolve() hardware_cfg = Path( str(frag_dirname) + "/hardware.cfg" ).resolve() non_hardware_cfg = Path( str(frag_dirname) + "/non-hardware.cfg" ).resolve() if hardware_cfg.exists() and use_classifiers: with open( str(hardware_cfg) ) as classification_file: for cline in classification_file: m = re.match( r"(CONFIG_[^= ]+)", cline ) if m: o_noprefix = re.sub( "^CONFIG_", "", m.group(1) ) hw_class_dict[o_noprefix.rstrip()] = str(hardware_cfg.resolve()) # being in both h/w and non-hardware leads us to results where we # can't easily silence a warning. So if we've added it to h/w we need # remove it from non-hardware. If we do it in both the hardware and # non hardware cases, the end result is that the last classification # wins, which is what we want. try: del non_hw_class_dict[o_noprefix.rstrip()] except: pass if non_hardware_cfg.exists() and use_classifiers: with open( str(non_hardware_cfg) ) as classification_file: for cline in classification_file: m = re.match( r"(CONFIG_[^= ]+)", cline ) if m: o_noprefix = re.sub( "^CONFIG_", "", m.group(1) ) non_hw_class_dict[o_noprefix.rstrip()] = str(non_hardware_cfg.resolve()) # being in both h/w and non-hardware leads us to results where we # can't easily silence a warning. So if we've added it to h/w we need # remove it from non-hardware. If we do it in both the hardware and # non hardware cases, the end result is that the last classification # wins, which is what we want. try: del hw_class_dict[o_noprefix.rstrip()] except Exception as e: pass for o in option_dict: fragments_defining_option = option_dict[o] if len( fragments_defining_option ) > 1: issues_dict["redefined_option"][o] = OrderedDict() for k in fragments_defining_option: issues_dict["redefined_option"][o][k] = fragments_defining_option[k] # val = fragments_defining_option[k] # print( " - {}: {} ({})".format(o,k,val) ) # this needs to just be captured in a class, so we can return one thing, # versus the growing list return frag_dict,option_dict,issues_dict,hw_class_dict,non_hw_class_dict def create_bsp_template( name, hardware_dict, policy_dict, outdir ): if verbose: print( "[INFO]: creating bsp template for: %s" % name ) if not os.path.exists( "{}/kernel-cache/bsp/{}".format( outdir,name ) ): os.makedirs( "{}/kernel-cache/bsp/{}".format( outdir,name ) ) bsp_file = "{}/kernel-cache/bsp/{}/{}.scc".format(outdir,name,name) hw_cfg_file = "{}/kernel-cache/bsp/{}/{}-hw.cfg".format(outdir,name,name) policy_cfg_file = "{}/kernel-cache/bsp/{}/{}-policy.cfg".format(outdir,name,name) with open( bsp_file, "w" ) as w: w.write("""\ # SPDX-License-Identifier: MIT define KMACHINE {} define KTYPE standard include ktypes/base/base.scc kconf hardware {}-hw.cfg # kconf non-hardware {}-policy.cfg """.format(name,name,name) ) with open( hw_cfg_file, "w" ) as w: for i,v in hardware_dict.items(): if i: w.write( "CONFIG_%s=%s\n" % (i,v)) with open( policy_cfg_file, "w" ) as w: for i,v in policy_dict.items(): if i: w.write( "CONFIG_%s=%s\n" % (i,v)) print( "BSP template created: %s" % bsp_file ) print( " Hardware options: %s" % hw_cfg_file ) print( " Policy options: %s" % policy_cfg_file ) def create_option_classifiers( ksrc, outdir ): if verbose: print( "[INFO]: creating option classifiers in %s" % outdir ) try: env = os.environ.copy() if verbose: print( "[NOTE]: running: kgit-create-buckets -v hardware nonhardware" ) analysis = subprocess.check_output([ 'kgit-create-buckets', '-v', '-v', 'hardware', 'nonhardware'], stderr=subprocess.STDOUT, cwd=ksrc, env=env ).decode('utf-8') except subprocess.CalledProcessError as e: print( "[ERROR]: creation failed: %s" % e.output.decode('utf-8')) if not os.path.exists( "{}/kernel-cache/ktypes/base".format( outdir ) ): os.makedirs( "{}/kernel-cache/ktypes/base".format( outdir) ) shutil.copy( "/tmp/hw_bucket.txt.sorted", "{}/kernel-cache/ktypes/base/hardware.kcf".format(outdir) ) shutil.copy( "/tmp/non_hw_bucket.txt.sorted", "{}/kernel-cache/ktypes/base/non-hardware.kcf".format(outdir) ) with open( "{}/kernel-cache/ktypes/base/base.scc".format(outdir), "w" ) as w: w.write("""\ kconf hardware basehw.cfg kconf non-hardware basenonhw.cfg """ ) with open( "{}/kernel-cache/ktypes/base/basehw.cfg".format(outdir), "w" ) as w: w.write("""\ # placeholder for hardware options """ ) with open( "{}/kernel-cache/ktypes/base/basenonhw.cfg".format(outdir), "w" ) as w: w.write("""\ # placeholder for non-hardware options """ ) if verbose: print( "[INFO]: option classifiers creation done" ) return "kernel-cache" pathname = os.path.dirname(sys.argv[0]) global verbose verbose = '' use_classifiers = True ksrc = '' parser = argparse.ArgumentParser( description="kernel fragment generation/manipulation/query", formatter_class=argparse.RawDescriptionHelpFormatter, epilog=textwrap.dedent('''\ Overview: kgit-config can be used to perform initial processing on a defconfig to determine what options are "hardware" and are hence suitable to be placed in a BSP, while leaving non-hardware/policy options to the fragments contained within a kernel-cache repository. It can also query a kernel-cache directory for details about which fragments provide an option, and where they are included (which is helpful when completing initial BSP fragments. Examples: # process a defconfig and output hardware options suitable for a BSP % kgit-config create --ksrc ~/poky-kernel/linux-yocto.git/ --defconfig ~/poky-kernel/linux-yocto.git/arch/x86/configs/x86_64_defconfig --kmeta ~/poky-kernel/kernel-cache # query a set of fragments for details on options % kgit-config query --kmeta ~/poky-kernel/kernel-cache CONFIG_DEBUG_FS [INFO]: looking for ['CONFIG_DEBUG_FS'] [INFO]: cfg files that have options matching regex: CONFIG_DEBUG_FS [INFO]: 7 cfg files have option: CONFIG_DEBUG_FS cfg file: cfg/fs/debugfs.cfg included by: /home/bruce/poky-kernel/kernel-cache/cfg/fs/debugfs.scc cfg file: cfg/debug/irq/debug-generic-irq-debugfs.cfg included by: /home/bruce/poky-kernel/kernel-cache/cfg/debug/irq/debug-generic-irq-debugfs.scc cfg file: cfg/debug/irq/debug-irq-domain.cfg included by: /home/bruce/poky-kernel/kernel-cache/cfg/debug/irq/debug-irq-domain.scc cfg file: cfg/debug/printk/debug-dynamic-debug.cfg included by: /home/bruce/poky-kernel/kernel-cache/cfg/debug/printk/debug-dynamic-debug.scc cfg file: features/systemtap/systemtap.cfg included by: /home/bruce/poky-kernel/kernel-cache/features/systemtap/systemtap.scc cfg file: bsp/beaglebone/beaglebone-non_hardware.cfg included by: /home/bruce/poky-kernel/kernel-cache/bsp/beaglebone/beaglebone.scc cfg file: bsp/intel-x86/intel-x86-acpi.cfg included by: /home/bruce/poky-kernel/kernel-cache/bsp/intel-x86/intel-x86.scc ''')) parser.add_argument("-v", action='store_true', dest="verbose", help="verbose") parser.add_argument("-s", "--ksrc", help="path to the kernel source") parser.add_argument("--create", action='store_true', help="Create a minimal BSP hardware config") parser.add_argument("--query", action='store_true', help="run a query against kernel meta data" ) parser.add_argument("-k", "--kmeta", help="path to the kernel meta data") parser.add_argument("-d", "--defconfig", help="path to the kernel defconfig") parser.add_argument("-e", "--entrypoint", help="fragment to use as an entrypoint for classification (default is ktypes/standard/standard.scc") parser.add_argument( "--savetemps", action='store_true', help="don't delete temporary working directory" ) parser.add_argument("-o", "--outdir", help="path to the output directory (defaults to '.')") parser.add_argument( "--bsp", help="create a BSP file with the provided name" ) # parser.add_argument("--strict", action='store_true', # help="When checking or processing, strictly apply ARCH and other config settings (i.e. no global checking will be done)" ) parser.add_argument('args', help=" (create or query)", nargs=argparse.REMAINDER) in_args = sys.argv if len(in_args) == 1: parser.print_help() sys.exit(1) create_flag = False query_flag = False save_temps = False if in_args: # we'll pass this to parse_known_args below, and when the default # of sys.argv is passed argv[0] is dropped. So we drop it now to # make sure things are parsed properly p_args = in_args[1:] if re.search( "^create$", in_args[1] ): # drop the command, so the argparser will pick up our dashed # items. p_args = in_args[2:] create_flag = True if re.search( "^query$", in_args[1] ): # drop the command, so the argparser will pick up our dashed # items. p_args = in_args[2:] query_flag = True args, unknown_args = parser.parse_known_args( p_args ) if create_flag: args.create = True if query_flag: args.query = True if args.ksrc: ksrc=args.ksrc outdir="." if args.outdir: outdir=args.outdir outdir = pathlib.Path(outdir).resolve() if args.savetemps: save_temps = True if args.verbose: verbose = True if args.create: if not ksrc: print( "[ERROR]: Kernel source directory not provided" ) sys.exit(1) # todo: convert to absolute and make sure it exists if not args.kmeta: if verbose: print( "[INFO]: no meta-data provided, creating option framework based on kernel tree: %s" % ksrc ) kmeta = create_option_classifiers( ksrc, outdir ) kmeta = pathlib.Path(kmeta).resolve() # creation will have made this entry point args.entrypoint = "ktypes/base/base.scc" args.kmeta = kmeta # todo: convert to absolute and make sure it exists if args.defconfig: defconfig_path = pathlib.Path(args.defconfig).exists() else: print( "[ERROR]: kernel defconfig required for minimal BSP creation" ) sys.exit(1) if verbose: print( "[INFO]: Creating minimal h/w BSP config" ) # make a temp directory to hold our work with tempdir() as dirpath: if save_temps and verbose: print( "[INFO]: --savetemps was passed, will not remove: %s" % dirpath ) try: entry_point = "ktypes/standard/standard.scc" if args.entrypoint: entry_point = args.entrypoint if verbose: print( "[NOTE]: running: " + 'scc' + ' --force' + ' -I{}'.format( args.kmeta ) + ' -o {}:cfg'.format( dirpath ) + ' {}/{}'.format(args.kmeta,entry_point) ) env = os.environ.copy() analysis = subprocess.check_output([ 'scc', '--force', '-I', '{}'.format( args.kmeta ), '-o', '{}:cfg'.format( dirpath ), '{}/{}'.format(args.kmeta,entry_point)], cwd=dirpath, env=env ).decode('utf-8') except subprocess.CalledProcessError as e: print( "[ERROR]: creation failed: %s" % e.output.decode('utf-8')) sys.exit(1) frag_dict,option_dict,issues_dict,hw_dict,non_hw_dict = \ config_queue_read( dirpath + "/config.queue" ) defconfig_hw_options = {} defconfig_nonhw_options = {} # loop through the defconfig, and remove all the non hardware options ? with open( str(args.defconfig) ) as defconfig: for cline in defconfig: c,value = split_option( cline ) o_noprefix = "" if c: o_noprefix = re.sub( "^CONFIG_", "", c ) if c == "invalid option format": # print( "invalid option: %s" % cline ) continue #if verbose: # print( "defconfig: %s --> %s" % (c,value)) try: if non_hw_dict[o_noprefix]: #print( "config %s is non-hardware, dropping" % (c)) defconfig_nonhw_options[o_noprefix] = value pass except: # it's hardware #print( " hardware option %s" % o_noprefix) defconfig_hw_options[o_noprefix] = value if args.bsp: create_bsp_template( args.bsp, defconfig_hw_options, defconfig_nonhw_options, outdir ) sys.exit(0) print( "[INFO]: BSP h/w configuration values:" ) for i,v in defconfig_hw_options.items(): if i: print( "CONFIG_%s=%s" % (i,v)) # look at the non-hardware options, and see if a fragment references # that option, we can suggest it as a feature print( "\n[INFO]: BSP policy options and fragments that provide them:" ) for i,v in defconfig_nonhw_options.items(): try: fragments_defining_option = option_dict["CONFIG_" + i] print( "CONFIG_%s=%s (%s)" % (i,v,list(fragments_defining_option.keys()) ) ) except: print( "CONFIG_%s=%s (no fragment)" % (i,v ) ) sys.exit(0) if args.query: # todo: convert to absolute and make sure it exists if not args.kmeta: print( "[ERROR]: meta data required for config query" ) sys.exit(1) if args.args[0] == "query": command = args.args[1:] else: command = args.args # make a temp directory to hold our work with tempdir() as dirpath: if save_temps: print( "[INFO]: --savetemps was passed, will not remove: %s" % dirpath ) possible_scc_files = [] possible_cfg_files = [] # list all the features for root, dirs, files in os.walk(args.kmeta): for file in files: if(file.endswith(".scc")): possible_scc_files.append(os.path.join(root,file)) if(file.endswith(".cfg")): possible_cfg_files.append(os.path.join(root,file)) if not command: feature_scc_files = {} for p in possible_scc_files: feature_scc = False with open(p) as f: for cnt,l in enumerate(f): if 'KFEATURE_DESCRIPTION' in l: d = l.split()[2:] d = ' '.join([str(elem) for elem in d]) full_path = p p = re.sub( args.kmeta + '/', "", full_path ) feature_scc_files[p] = { 'path' : full_path, 'description' : d } feature_scc = True if not feature_scc: pass print( "[INFO]: available feature scc files (%s)" % args.kmeta) for p in feature_scc_files: print( " %s: %s" % (p,feature_scc_files[p]['description']) ) else: print( "[INFO]: looking for %s" % command ) oregex = ' '.join([str(elem) for elem in command]) matching_cfg_files = {} for p in possible_cfg_files: matching_cfg = False with open(p) as f: for cnt,l in enumerate(f): if re.search( oregex, l ): full_path = p pp = re.sub( args.kmeta + '/', "", full_path ) matching_cfg_files[pp] = { 'path' : full_path, 'option' : re.sub( '\n', '', l) } matching_cfg = True if matching_cfg_files: print( "[INFO]: cfg files that have options matching regex: %s" % oregex ) sccs_that_include_matching_cfg = {} for p in matching_cfg_files: pregex = os.path.basename(p) match_found = False for sp in possible_scc_files: with open(sp) as f: for cnt,l in enumerate(f): if re.search( "[ /]" + pregex + "$", l ): full_path = sp pp = re.sub( args.kmeta + '/', "", full_path ) try: sccs_that_include_matching_cfg[p].append( full_path ) except: sccs_that_include_matching_cfg[p] = [ full_path ] match_found = True if not match_found: sccs_that_include_matching_cfg[p] = [ "Standalone" ] if matching_cfg_files: print( "[INFO]: %s cfg files have option: %s" % (len(matching_cfg_files),oregex)) for p in matching_cfg_files: print( " cfg file: %s" % p ) try: istring = ' '.join([str(elem) for elem in list(sccs_that_include_matching_cfg[p])]) print( " included by: %s" % istring) #for m in sccs_that_include_matching_cfg[p]: # print( " %s" % m ) except: pass else: print( "no matching cfg files with option %s found" % oregex )