diff options
| -rw-r--r-- | classes/dosocs-host.bbclass | 262 | ||||
| -rw-r--r-- | classes/dosocs.bbclass | 322 | ||||
| -rw-r--r-- | classes/fossdriver-host.bbclass | 219 | ||||
| -rw-r--r-- | classes/spdx-common.bbclass | 224 | ||||
| -rw-r--r-- | recipes-devtools/fossology/fossology-nomos-native_git.bb | 5 |
5 files changed, 322 insertions, 710 deletions
diff --git a/classes/dosocs-host.bbclass b/classes/dosocs-host.bbclass deleted file mode 100644 index a6ed691..0000000 --- a/classes/dosocs-host.bbclass +++ /dev/null @@ -1,262 +0,0 @@ -# This class integrates real-time license scanning, generation of SPDX standard -# output and verifiying license info during the building process. -# It is a combination of efforts from the OE-Core, SPDX and DoSOCSv2 projects. -# -# For more information on DoSOCSv2: -# https://github.com/DoSOCSv2 -# -# For more information on SPDX: -# http://www.spdx.org -# -# Note: -# 1) Make sure DoSOCSv2 has beed installed in your host -# 2) By default,spdx files will be output to the path which is defined as[SPDX_DEPLOY_DIR] -# in ./meta/conf/spdx-dosocs.conf. - -SPDXSSTATEDIR = "${WORKDIR}/spdx_sstate_dir" -LICENSELISTVERSION = "2.6" -CREATOR_TOOL = "meta-spdxscanner" -# If ${S} isn't actually the top-level source directory, set SPDX_S to point at -# the real top-level directory. - -SPDX_S ?= "${S}" - -python do_spdx () { - import os, sys - import json - - import shutil - - pn = d.getVar('PN') - workdir_tmp = d.getVar('WORKDIR') - - ## It's no necessary to get spdx files for *-native - if pn.find("-native") != -1 or pn.find("binutils-cross") != -1: - return None - - # Forcibly expand the sysroot paths as we're about to change WORKDIR - d.setVar('RECIPE_SYSROOT', d.getVar('RECIPE_SYSROOT')) - d.setVar('RECIPE_SYSROOT_NATIVE', d.getVar('RECIPE_SYSROOT_NATIVE')) - - ## gcc and kernel is too big to get spdx file. - if ('gcc') in d.getVar('PN', True): - #invoke_dosocs2("/yocto/work002/fnst/leimh/community/gcc-7.3.0/","/yocto/work001/gcc-7.3.spdx",(d.getVar('WORKDIR', True) or "")) - return None - if bb.data.inherits_class('kernel', d): - #invoke_dosocs2("/yocto/work002/fnst/leimh/community/linux-4.14.44","/yocto/work001/linux-4.14.44.spdx",(d.getVar('WORKDIR', True) or "")) - return None - - bb.note('Archiving the configured source...') - # "gcc-source-${PV}" recipes don't have "do_configure" - # task, so we need to run "do_preconfigure" instead - if pn.startswith("gcc-source-"): - d.setVar('WORKDIR', d.getVar('ARCHIVER_WORKDIR')) - bb.build.exec_func('do_preconfigure', d) - - # Change the WORKDIR to make do_configure run in another dir. - d.setVar('WORKDIR', d.getVar('SPDX_TEMP_DIR')) - #if bb.data.inherits_class('kernel-yocto', d): - # bb.build.exec_func('do_kernel_configme', d) - #if bb.data.inherits_class('cmake', d): - # bb.build.exec_func('do_generate_toolchain_file', d) - bb.build.exec_func('do_unpack', d) - - d.setVar('WORKDIR', workdir_tmp) - info = {} - info['workdir'] = (d.getVar('WORKDIR', True) or "") - info['pn'] = (d.getVar( 'PN', True ) or "") - info['pv'] = (d.getVar( 'PV', True ) or "") - info['package_download_location'] = (d.getVar( 'SRC_URI', True ) or "") - if info['package_download_location'] != "": - info['package_download_location'] = info['package_download_location'].split()[0] - info['spdx_version'] = (d.getVar('SPDX_VERSION', True) or '') - info['data_license'] = (d.getVar('DATA_LICENSE', True) or '') - info['creator'] = {} - info['creator']['Tool'] = (d.getVar('CREATOR_TOOL', True) or '') - info['license_list_version'] = (d.getVar('LICENSELISTVERSION', True) or '') - info['package_homepage'] = (d.getVar('HOMEPAGE', True) or "") - info['package_summary'] = (d.getVar('SUMMARY', True) or "") - info['package_summary'] = info['package_summary'].replace("\n","") - info['package_summary'] = info['package_summary'].replace("'"," ") - info['package_contains'] = (d.getVar('CONTAINED', True) or "") - info['package_static_link'] = (d.getVar('STATIC_LINK', True) or "") - - spdx_sstate_dir = (d.getVar('SPDXSSTATEDIR', True) or "") - manifest_dir = (d.getVar('SPDX_DEPLOY_DIR', True) or "") - info['outfile'] = os.path.join(manifest_dir, info['pn'] + "-" + info['pv'] + ".spdx" ) - sstatefile = os.path.join(spdx_sstate_dir, - info['pn'] + "-" + info['pv'] + ".spdx" ) - - ## get everything from cache. use it to decide if - ## something needs to be rerun - if not os.path.exists( spdx_sstate_dir ): - bb.utils.mkdirhier( spdx_sstate_dir ) - - d.setVar('WORKDIR', d.getVar('SPDX_TEMP_DIR', True)) - info['sourcedir'] = (d.getVar('SPDX_S', True) or "") - cur_ver_code = get_ver_code( info['sourcedir'] ).split()[0] - cache_cur = False - if os.path.exists( sstatefile ): - ## cache for this package exists. read it in - cached_spdx = get_cached_spdx( sstatefile ) - if cached_spdx: - cached_spdx = cached_spdx.split()[0] - if (cached_spdx == cur_ver_code): - bb.warn(info['pn'] + "'s ver code same as cache's. do nothing") - cache_cur = True - create_manifest(info,sstatefile) - if not cache_cur: - git_path = "%s/.git" % info['sourcedir'] - if os.path.exists(git_path): - remove_dir_tree(git_path) - - ## Get spdx file - #bb.warn(' run_dosocs2 ...... ') - invoke_dosocs2(info['sourcedir'],sstatefile,info['workdir']) - if get_cached_spdx( sstatefile ) != None: - write_cached_spdx( info,sstatefile,cur_ver_code ) - ## CREATE MANIFEST(write to outfile ) - create_manifest(info,sstatefile) - else: - bb.warn('Can\'t get the spdx file ' + info['pn'] + '. Please check your dosocs2.') - d.setVar('WORKDIR', info['workdir']) -} - -addtask spdx after do_patch before do_configure - -def invoke_dosocs2( OSS_src_dir, spdx_file, workdir): - import subprocess - import string - import json - import codecs - - - dosocs2_cmd = "/usr/local/bin/dosocs2" - dosocs2_oneshot_cmd = dosocs2_cmd + " oneshot " + OSS_src_dir - print(dosocs2_oneshot_cmd) - try: - dosocs2_output = subprocess.check_output(dosocs2_oneshot_cmd, - stderr=subprocess.STDOUT, - shell=True) - except subprocess.CalledProcessError as e: - bb.warn("Could not invoke dosocs2 oneshot Command " - "'%s' returned %d:\n%s" % (dosocs2_oneshot_cmd, e.returncode, e.output)) - return None - dosocs2_output = dosocs2_output.decode('utf-8') - - f = codecs.open(spdx_file,'w','utf-8') - f.write(dosocs2_output) - -def create_manifest(info,sstatefile): - import shutil - shutil.copyfile(sstatefile,info['outfile']) - -def get_cached_spdx( sstatefile ): - import subprocess - - if not os.path.exists( sstatefile ): - return None - - try: - output = subprocess.check_output(['grep', "PackageVerificationCode", sstatefile]) - except subprocess.CalledProcessError as e: - bb.error("Index creation command '%s' failed with return code %d:\n%s" % (e.cmd, e.returncode, e.output)) - return None - cached_spdx_info=output.decode('utf-8').split(': ') - return cached_spdx_info[1] - -## Add necessary information into spdx file -def write_cached_spdx( info,sstatefile, ver_code ): - import subprocess - - def sed_replace(dest_sed_cmd,key_word,replace_info): - dest_sed_cmd = dest_sed_cmd + "-e 's#^" + key_word + ".*#" + \ - key_word + replace_info + "#' " - return dest_sed_cmd - - def sed_insert(dest_sed_cmd,key_word,new_line): - dest_sed_cmd = dest_sed_cmd + "-e '/^" + key_word \ - + r"/a\\" + new_line + "' " - return dest_sed_cmd - - ## Document level information - sed_cmd = r"sed -i -e 's#\r$##g' " - spdx_DocumentComment = "<text>SPDX for " + info['pn'] + " version " \ - + info['pv'] + "</text>" - sed_cmd = sed_replace(sed_cmd,"DocumentComment",spdx_DocumentComment) - - ## Creator information - sed_cmd = sed_replace(sed_cmd,"Creator: ",info['creator']['Tool']) - sed_cmd = sed_replace(sed_cmd,"LicenseListVersion: ",info['license_list_version']) - - ## Package level information - sed_cmd = sed_replace(sed_cmd,"PackageName: ",info['pn']) - sed_cmd = sed_insert(sed_cmd,"PackageName: ", "PackageVersion: " + info['pv']) - sed_cmd = sed_replace(sed_cmd,"PackageDownloadLocation: ",info['package_download_location']) - sed_cmd = sed_replace(sed_cmd,"PackageHomePage: ",info['package_homepage']) - sed_cmd = sed_replace(sed_cmd,"PackageSummary: ","<text>" + info['package_summary'] + "</text>") - sed_cmd = sed_insert(sed_cmd,"PackageVerificationCode: ",ver_code) - sed_cmd = sed_replace(sed_cmd,"PackageDescription: ", - "<text>" + info['pn'] + " version " + info['pv'] + "</text>") - for contain in info['package_contains'].split( ): - sed_cmd = sed_insert(sed_cmd,"PackageComment:"," \\n\\n## Relationships\\nRelationship: " + info['pn'] + " CONTAINS " + contain) - for static_link in info['package_static_link'].split( ): - sed_cmd = sed_insert(sed_cmd,"PackageComment:"," \\n\\n## Relationships\\nRelationship: " + info['pn'] + " STATIC_LINK " + static_link) - sed_cmd = sed_cmd + sstatefile - - subprocess.call("%s" % sed_cmd, shell=True) - -def remove_dir_tree( dir_name ): - import shutil - try: - shutil.rmtree( dir_name ) - except: - pass - -def remove_file( file_name ): - try: - os.remove( file_name ) - except OSError as e: - pass - -def list_files( dir ): - for root, subFolders, files in os.walk( dir ): - for f in files: - rel_root = os.path.relpath( root, dir ) - yield rel_root, f - return - -def hash_file( file_name ): - """ - Return the hex string representation of the SHA1 checksum of the filename - """ - try: - import hashlib - except ImportError: - return None - - sha1 = hashlib.sha1() - with open( file_name, "rb" ) as f: - for line in f: - sha1.update(line) - return sha1.hexdigest() - -def hash_string( data ): - import hashlib - sha1 = hashlib.sha1() - sha1.update( data.encode('utf-8') ) - return sha1.hexdigest() - -def get_ver_code( dirname ): - chksums = [] - for f_dir, f in list_files( dirname ): - try: - stats = os.stat(os.path.join(dirname,f_dir,f)) - except OSError as e: - bb.warn( "Stat failed" + str(e) + "\n") - continue - chksums.append(hash_file(os.path.join(dirname,f_dir,f))) - ver_code_string = ''.join( chksums ).lower() - ver_code = hash_string( ver_code_string ) - return ver_code - diff --git a/classes/dosocs.bbclass b/classes/dosocs.bbclass index 755e826..659496c 100644 --- a/classes/dosocs.bbclass +++ b/classes/dosocs.bbclass @@ -9,54 +9,52 @@ # http://www.spdx.org # # Note: -# 1) Make sure DoSOCSv2 has beed installed in your host +# 1) Make sure fossdriver has beed installed in your host # 2) By default,spdx files will be output to the path which is defined as[SPDX_DEPLOY_DIR] -# in ./meta/conf/spdx-dosocs.conf. - -PYTHON_INHERIT = "${@bb.utils.contains('PN', '-native', '', 'python3-dir', d)}" -PYTHON_INHERIT .= "${@bb.utils.contains('PACKAGECONFIG', 'python3', 'python3native', '', d)}" - -inherit ${PYTHON_INHERIT} python3-dir - -PYTHON = "${@bb.utils.contains('PN', '-native', '${STAGING_BINDIR_NATIVE}/${PYTHON_PN}-native/${PYTHON_PN}', '', d)}" -EXTRANATIVEPATH += "${PYTHON_PN}-native" - -# python-config and other scripts are using distutils modules -# which we patch to access these variables -export STAGING_INCDIR -export STAGING_LIBDIR - -# autoconf macros will use their internal default preference otherwise -export PYTHON - -do_spdx[depends] += "python3-dosocs2-init-native:do_dosocs2_init" -do_spdx[depends] += "python3-dosocs2-native:do_populate_sysroot" - -SPDXSSTATEDIR = "${WORKDIR}/spdx_sstate_dir" +# By default, SPDX_DEPLOY_DIR is tmp/deploy +# +inherit spdx-common -# If ${S} isn't actually the top-level source directory, set SPDX_S to point at -# the real top-level directory. +SPDXEPENDENCY += "python3-dosocs2-init-native:do_dosocs2_init" +SPDXEPENDENCY += "python3-dosocs2-native:do_populate_sysroot" -SPDX_S ?= "${S}" +LICENSELISTVERSION = "2.6" +CREATOR_TOOL = "dosocs.bbclass in meta-spdxscanner" python do_spdx () { - import os, sys - import json - - pn = d.getVar("PN") - depends = d.getVar("DEPENDS") - ## It's no necessary to get spdx files for *-native - if pn.find("-native") == -1 and pn.find("binutils-cross") == -1: - PYTHON = "${STAGING_BINDIR_NATIVE}/${PYTHON_PN}-native/${PYTHON_PN}" - os.environ['PYTHON'] = PYTHON - depends = "%s python3-dosocs2-init-native" % depends - d.setVar("DEPENDS", depends) - else: - return None + import os, sys, json, shutil - ## gcc and kernel is too big to get spdx file. - if ('gcc' or 'linux-yocto') in d.getVar('PN', True): - return None + pn = d.getVar('PN') + assume_provided = (d.getVar("ASSUME_PROVIDED") or "").split() + if pn in assume_provided: + for p in d.getVar("PROVIDES").split(): + if p != pn: + pn = p + break + + # glibc-locale: do_fetch, do_unpack and do_patch tasks have been deleted, + # so avoid archiving source here. + if pn.startswith('glibc-locale'): + return + if (d.getVar('BPN') == "linux-yocto"): + return + if (d.getVar('PN') == "libtool-cross"): + return + if (d.getVar('PN') == "libgcc-initial"): + return + if (d.getVar('PN') == "shadow-sysroot"): + return + + + # We just archive gcc-source for all the gcc related recipes + if d.getVar('BPN') in ['gcc', 'libgcc']: + bb.debug(1, 'spdx: There is bug in scan of %s is, do nothing' % pn) + return + + spdx_outdir = d.getVar('SPDX_OUTDIR') + spdx_workdir = d.getVar('SPDX_WORKDIR') + spdx_temp_dir = os.path.join(spdx_workdir, "temp") + temp_dir = os.path.join(d.getVar('WORKDIR'), "temp") info = {} info['workdir'] = (d.getVar('WORKDIR', True) or "") @@ -76,98 +74,56 @@ python do_spdx () { info['package_summary'] = info['package_summary'].replace("'"," ") info['package_contains'] = (d.getVar('CONTAINED', True) or "") info['package_static_link'] = (d.getVar('STATIC_LINK', True) or "") + info['modified'] = "false" + srcuri = d.getVar("SRC_URI", False).split() + length = len("file://") + for item in srcuri: + if item.startswith("file://"): + item = item[length:] + if item.endswith(".patch") or item.endswith(".diff"): + info['modified'] = "true" - spdx_sstate_dir = (d.getVar('SPDXSSTATEDIR', True) or "") manifest_dir = (d.getVar('SPDX_DEPLOY_DIR', True) or "") + if not os.path.exists( manifest_dir ): + bb.utils.mkdirhier( manifest_dir ) info['outfile'] = os.path.join(manifest_dir, info['pn'] + "-" + info['pv'] + ".spdx" ) - sstatefile = os.path.join(spdx_sstate_dir, - info['pn'] + "-" + info['pv'] + ".spdx" ) - - ## get everything from cache. use it to decide if - ## something needs to be rerun - if not os.path.exists( spdx_sstate_dir ): - bb.utils.mkdirhier( spdx_sstate_dir ) + sstatefile = os.path.join(spdx_outdir, info['pn'] + "-" + info['pv'] + ".spdx" ) - d.setVar('WORKDIR', d.getVar('SPDX_TEMP_DIR', True)) - info['sourcedir'] = (d.getVar('SPDX_S', True) or "") - cur_ver_code = get_ver_code( info['sourcedir'] ).split()[0] - cache_cur = False + # if spdx has been exist + if os.path.exists(info['outfile']): + bb.note(info['pn'] + "spdx file has been exist, do nothing") + return if os.path.exists( sstatefile ): - ## cache for this package exists. read it in - cached_spdx = get_cached_spdx( sstatefile ) - if cached_spdx: - cached_spdx = cached_spdx.split()[0] - if (cached_spdx == cur_ver_code): - bb.warn(info['pn'] + "'s ver code same as cache's. do nothing") - cache_cur = True - create_manifest(info,sstatefile) - if not cache_cur: - git_path = "%s/.git" % info['sourcedir'] - if os.path.exists(git_path): - remove_dir_tree(git_path) - - ## Get spdx file - #bb.warn(' run_dosocs2 ...... ') - invoke_dosocs2(info['sourcedir'],sstatefile) - if get_cached_spdx( sstatefile ) != None: - write_cached_spdx( info,sstatefile,cur_ver_code ) - ## CREATE MANIFEST(write to outfile ) - create_manifest(info,sstatefile) - else: - bb.warn('Can\'t get the spdx file ' + info['pn'] + '. Please check your dosocs2.') - d.setVar('WORKDIR', info['workdir']) -} -#python () { -# deps = ' python3-dosocs2-native:do_dosocs2_init' -# d.appendVarFlag('do_spdx', 'depends', deps) -#} - -## Get the src after do_patch. -python do_get_spdx_s() { - import shutil - - pn = d.getVar('PN') - ## It's no necessary to get spdx files for *-native - if d.getVar('PN', True) == d.getVar('BPN', True) + "-native": - return None - - ## gcc and kernel is too big to get spdx file. - if ('gcc' or 'linux-yocto') in d.getVar('PN', True): - return None - - # Forcibly expand the sysroot paths as we're about to change WORKDIR - d.setVar('RECIPE_SYSROOT', d.getVar('RECIPE_SYSROOT')) - d.setVar('RECIPE_SYSROOT_NATIVE', d.getVar('RECIPE_SYSROOT_NATIVE')) - - bb.note('Archiving the configured source...') - pn = d.getVar('PN') - # "gcc-source-${PV}" recipes don't have "do_configure" - # task, so we need to run "do_preconfigure" instead - if pn.startswith("gcc-source-"): - d.setVar('WORKDIR', d.getVar('ARCHIVER_WORKDIR')) - bb.build.exec_func('do_preconfigure', d) - - # Change the WORKDIR to make do_configure run in another dir. - d.setVar('WORKDIR', d.getVar('SPDX_TEMP_DIR')) - #if bb.data.inherits_class('kernel-yocto', d): - # bb.build.exec_func('do_kernel_configme', d) - #if bb.data.inherits_class('cmake', d): - # bb.build.exec_func('do_generate_toolchain_file', d) - bb.build.exec_func('do_unpack', d) -} - -python () { - pn = d.getVar("PN") - depends = d.getVar("DEPENDS") - - if pn.find("-native") == -1 and pn.find("binutils-cross") == -1: - depends = "%s python3-dosocs2-native" % depends - d.setVar("DEPENDS", depends) - bb.build.addtask('do_get_spdx_s','do_configure','do_patch', d) - bb.build.addtask('do_spdx','do_package', 'do_get_spdx_s', d) + bb.note(info['pn'] + "spdx file has been exist, do nothing") + create_manifest(info,sstatefile) + return + + spdx_get_src(d) + + bb.note('SPDX: Archiving the patched source...') + if os.path.isdir(spdx_temp_dir): + for f_dir, f in list_files(spdx_temp_dir): + temp_file = os.path.join(spdx_temp_dir,f_dir,f) + shutil.copy(temp_file, temp_dir) + shutil.rmtree(spdx_temp_dir) + if not os.path.exists(spdx_outdir): + bb.utils.mkdirhier(spdx_outdir) + cur_ver_code = get_ver_code(spdx_workdir).split()[0] + ## Get spdx file + bb.note(' run dosocsv2 ...... ') + d.setVar('WORKDIR', d.getVar('SPDX_WORKDIR', True)) + info['sourcedir'] = spdx_workdir + git_path = "%s/.git" % info['sourcedir'] + if os.path.exists(git_path): + remove_dir_tree(git_path) + invoke_dosocs2(info['sourcedir'],sstatefile) + if get_cached_spdx(sstatefile) != None: + write_cached_spdx( info,sstatefile,cur_ver_code ) + ## CREATE MANIFEST(write to outfile ) + create_manifest(info,sstatefile) + else: + bb.warn('Can\'t get the spdx file ' + info['pn'] + '. Please check your.') } -#addtask get_spdx_s after do_patch before do_configure -#addtask spdx after do_get_spdx_s before do_package def invoke_dosocs2( OSS_src_dir, spdx_file): import subprocess @@ -191,112 +147,4 @@ def invoke_dosocs2( OSS_src_dir, spdx_file): f = codecs.open(spdx_file,'w','utf-8') f.write(dosocs2_output) -def create_manifest(info,sstatefile): - import shutil - shutil.copyfile(sstatefile,info['outfile']) - -def get_cached_spdx( sstatefile ): - import subprocess - - if not os.path.exists( sstatefile ): - return None - - try: - output = subprocess.check_output(['grep', "PackageVerificationCode", sstatefile]) - except subprocess.CalledProcessError as e: - bb.error("Index creation command '%s' failed with return code %d:\n%s" % (e.cmd, e.returncode, e.output)) - return None - cached_spdx_info=output.decode('utf-8').split(': ') - return cached_spdx_info[1] - -## Add necessary information into spdx file -def write_cached_spdx( info,sstatefile, ver_code ): - import subprocess - - def sed_replace(dest_sed_cmd,key_word,replace_info): - dest_sed_cmd = dest_sed_cmd + "-e 's#^" + key_word + ".*#" + \ - key_word + replace_info + "#' " - return dest_sed_cmd - - def sed_insert(dest_sed_cmd,key_word,new_line): - dest_sed_cmd = dest_sed_cmd + "-e '/^" + key_word \ - + r"/a\\" + new_line + "' " - return dest_sed_cmd - - ## Document level information - sed_cmd = r"sed -i -e 's#\r$##g' " - spdx_DocumentComment = "<text>SPDX for " + info['pn'] + " version " \ - + info['pv'] + "</text>" - sed_cmd = sed_replace(sed_cmd,"DocumentComment",spdx_DocumentComment) - - ## Package level information - sed_cmd = sed_replace(sed_cmd,"PackageName: ",info['pn']) - sed_cmd = sed_insert(sed_cmd,"PackageVersion: ",info['pv']) - sed_cmd = sed_replace(sed_cmd,"PackageDownloadLocation: ",info['package_download_location']) - sed_cmd = sed_replace(sed_cmd,"PackageChecksum: ","PackageHomePage: " + info['package_homepage']) - sed_cmd = sed_replace(sed_cmd,"PackageSummary: ","<text>" + info['package_summary'] + "</text>") - sed_cmd = sed_replace(sed_cmd,"PackageVerificationCode: ",ver_code) - sed_cmd = sed_replace(sed_cmd,"PackageDescription: ", - "<text>" + info['pn'] + " version " + info['pv'] + "</text>") - for contain in info['package_contains'].split( ): - sed_cmd = sed_insert(sed_cmd,"PackageComment:"," \\n\\n## Relationships\\nRelationship: " + info['pn'] + " CONTAINS " + contain) - for static_link in info['package_static_link'].split( ): - sed_cmd = sed_insert(sed_cmd,"PackageComment:"," \\n\\n## Relationships\\nRelationship: " + info['pn'] + " STATIC_LINK " + static_link) - sed_cmd = sed_cmd + sstatefile - - subprocess.call("%s" % sed_cmd, shell=True) - -def remove_dir_tree( dir_name ): - import shutil - try: - shutil.rmtree( dir_name ) - except: - pass - -def remove_file( file_name ): - try: - os.remove( file_name ) - except OSError as e: - pass - -def list_files( dir ): - for root, subFolders, files in os.walk( dir ): - for f in files: - rel_root = os.path.relpath( root, dir ) - yield rel_root, f - return - -def hash_file( file_name ): - """ - Return the hex string representation of the SHA1 checksum of the filename - """ - try: - import hashlib - except ImportError: - return None - - sha1 = hashlib.sha1() - with open( file_name, "rb" ) as f: - for line in f: - sha1.update(line) - return sha1.hexdigest() - -def hash_string( data ): - import hashlib - sha1 = hashlib.sha1() - sha1.update( data.encode('utf-8') ) - return sha1.hexdigest() - -def get_ver_code( dirname ): - chksums = [] - for f_dir, f in list_files( dirname ): - try: - stats = os.stat(os.path.join(dirname,f_dir,f)) - except OSError as e: - bb.warn( "Stat failed" + str(e) + "\n") - continue - chksums.append(hash_file(os.path.join(dirname,f_dir,f))) - ver_code_string = ''.join( chksums ).lower() - ver_code = hash_string( ver_code_string ) - return ver_code - +EXPORT_FUNCTIONS do_spdx diff --git a/classes/fossdriver-host.bbclass b/classes/fossdriver-host.bbclass index a279eab..29181f9 100644 --- a/classes/fossdriver-host.bbclass +++ b/classes/fossdriver-host.bbclass @@ -1,40 +1,21 @@ # This class integrates real-time license scanning, generation of SPDX standard # output and verifiying license info during the building process. -# It is a combination of efforts from the OE-Core, SPDX and DoSOCSv2 projects. +# It is a combination of efforts from the OE-Core, SPDX and fossdriver projects. # -# For more information on DoSOCSv2: -# https://github.com/DoSOCSv2 +# For more information on fossdriver: +# https://github.com/fossology/fossdriver # # For more information on SPDX: # http://www.spdx.org # # Note: -# 1) Make sure fossdriver has beed installed in your host -# 2) By default,spdx files will be output to the path which is defined as[SPDX_DEPLOY_DIR] -# in ./meta/conf/spdx-dosocs.conf. +# 1) Make sure fossology 3.5.0(https://hub.docker.com/r/fossology/fossology/) has beed started on your host +# 2) spdx files will be output to the path which is defined as[SPDX_DEPLOY_DIR]. +# By default, SPDX_DEPLOY_DIR is tmp/deploy/ +# -SPDXEPENDENCY += "${PATCHTOOL}-native:do_populate_sysroot" -SPDXEPENDENCY += " wget-native:do_populate_sysroot" -SPDXEPENDENCY += " subversion-native:do_populate_sysroot" -SPDXEPENDENCY += " git-native:do_populate_sysroot" -SPDXEPENDENCY += " lz4-native:do_populate_sysroot" -SPDXEPENDENCY += " lzip-native:do_populate_sysroot" -SPDXEPENDENCY += " xz-native:do_populate_sysroot" -SPDXEPENDENCY += " unzip-native:do_populate_sysroot" -SPDXEPENDENCY += " xz-native:do_populate_sysroot" -SPDXEPENDENCY += " nodejs-native:do_populate_sysroot" -SPDXEPENDENCY += " quilt-native:do_populate_sysroot" -SPDXEPENDENCY += " tar-native:do_populate_sysroot" - -SPDX_TOPDIR ?= "${WORKDIR}/spdx_sstate_dir" -SPDX_OUTDIR = "${SPDX_TOPDIR}/${TARGET_SYS}/${PF}/" -SPDX_WORKDIR = "${WORKDIR}/spdx_temp/" - -do_spdx[dirs] = "${WORKDIR}" - -LICENSELISTVERSION = "2.6" -CREATOR_TOOL = "meta-spdxscanner" +CREATOR_TOOL = "fossdriver-host.bbclass in meta-spdxscanner" # If ${S} isn't actually the top-level source directory, set SPDX_S to point at # the real top-level directory. @@ -103,6 +84,9 @@ python do_spdx () { info['modified'] = "true" manifest_dir = (d.getVar('SPDX_DEPLOY_DIR', True) or "") + if not os.path.exists( manifest_dir ): + bb.utils.mkdirhier( manifest_dir ) + info['outfile'] = os.path.join(manifest_dir, info['pn'] + "-" + info['pv'] + ".spdx" ) sstatefile = os.path.join(spdx_outdir, info['pn'] + "-" + info['pv'] + ".spdx" ) @@ -144,68 +128,6 @@ python do_spdx () { bb.warn('Can\'t get the spdx file ' + info['pn'] + '. Please check your.') } -addtask do_spdx before do_unpack after do_fetch - -def spdx_create_tarball(d, srcdir, suffix, ar_outdir): - """ - create the tarball from srcdir - """ - import tarfile, shutil - # Make sure we are only creating a single tarball for gcc sources - #if (d.getVar('SRC_URI') == ""): - # return - - # For the kernel archive, srcdir may just be a link to the - # work-shared location. Use os.path.realpath to make sure - # that we archive the actual directory and not just the link. - srcdir = os.path.realpath(srcdir) - - bb.utils.mkdirhier(ar_outdir) - if suffix: - filename = '%s-%s.tar.gz' % (d.getVar('PF'), suffix) - else: - filename = '%s.tar.gz' % d.getVar('PF') - tarname = os.path.join(ar_outdir, filename) - - bb.note('Creating %s' % tarname) - tar = tarfile.open(tarname, 'w:gz') - tar.add(srcdir, arcname=os.path.basename(srcdir)) - tar.close() - shutil.rmtree(srcdir) - return tarname - -# Run do_unpack and do_patch -def spdx_get_src(d): - import shutil - spdx_workdir = d.getVar('SPDX_WORKDIR') - spdx_sysroot_native = d.getVar('STAGING_DIR_NATIVE') - pn = d.getVar('PN') - - # We just archive gcc-source for all the gcc related recipes - if d.getVar('BPN') in ['gcc', 'libgcc']: - bb.debug(1, 'spdx: There is bug in scan of %s is, do nothing' % pn) - return - - # The kernel class functions require it to be on work-shared, so we dont change WORKDIR - if not is_work_shared(d): - # Change the WORKDIR to make do_unpack do_patch run in another dir. - d.setVar('WORKDIR', spdx_workdir) - # Restore the original path to recipe's native sysroot (it's relative to WORKDIR). - d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native) - - # The changed 'WORKDIR' also caused 'B' changed, create dir 'B' for the - # possibly requiring of the following tasks (such as some recipes's - # do_patch required 'B' existed). - bb.utils.mkdirhier(d.getVar('B')) - - bb.build.exec_func('do_unpack', d) - - # Make sure gcc and kernel sources are patched only once - if not (d.getVar('SRC_URI') == "" or is_work_shared(d)): - bb.build.exec_func('do_patch', d) - # Some userland has no source. - if not os.path.exists( spdx_workdir ): - bb.utils.mkdirhier(spdx_workdir) def invoke_fossdriver(tar_file, spdx_file): import os @@ -270,123 +192,4 @@ def invoke_fossdriver(tar_file, spdx_file): bb.warn("%s SPDXTV failed, Please check your fossology server." % tar_file) return False -def create_manifest(info,sstatefile): - import shutil - shutil.copyfile(sstatefile,info['outfile']) - -def get_cached_spdx( sstatefile ): - import subprocess - - if not os.path.exists( sstatefile ): - return None - - try: - output = subprocess.check_output(['grep', "PackageVerificationCode", sstatefile]) - except subprocess.CalledProcessError as e: - bb.error("Index creation command '%s' failed with return code %d:\n%s" % (e.cmd, e.returncode, e.output)) - return None - cached_spdx_info=output.decode('utf-8').split(': ') - return cached_spdx_info[1] - -## Add necessary information into spdx file -def write_cached_spdx( info,sstatefile, ver_code ): - import subprocess - - def sed_replace(dest_sed_cmd,key_word,replace_info): - dest_sed_cmd = dest_sed_cmd + "-e 's#^" + key_word + ".*#" + \ - key_word + replace_info + "#' " - return dest_sed_cmd - - def sed_insert(dest_sed_cmd,key_word,new_line): - dest_sed_cmd = dest_sed_cmd + "-e '/^" + key_word \ - + r"/a\\" + new_line + "' " - return dest_sed_cmd - - ## Document level information - sed_cmd = r"sed -i -e 's#\r$##g' " - spdx_DocumentComment = "<text>SPDX for " + info['pn'] + " version " \ - + info['pv'] + "</text>" - sed_cmd = sed_replace(sed_cmd,"DocumentComment",spdx_DocumentComment) - - ## Creator information - sed_cmd = sed_replace(sed_cmd,"Creator: ",info['creator']['Tool']) - - ## Package level information - sed_cmd = sed_replace(sed_cmd, "PackageName: ", info['pn']) - sed_cmd = sed_insert(sed_cmd, "PackageName: ", "PackageVersion: " + info['pv']) - sed_cmd = sed_replace(sed_cmd, "PackageDownloadLocation: ",info['package_download_location']) - sed_cmd = sed_insert(sed_cmd, "PackageDownloadLocation: ", "PackageHomePage: " + info['package_homepage']) - sed_cmd = sed_insert(sed_cmd, "PackageDownloadLocation: ", "PackageSummary: " + "<text>" + info['package_summary'] + "</text>") - sed_cmd = sed_insert(sed_cmd, "PackageDownloadLocation: ", "modification record : " + info['modified']) - sed_cmd = sed_replace(sed_cmd, "PackageVerificationCode: ",ver_code) - sed_cmd = sed_insert(sed_cmd, "PackageVerificationCode: ", "PackageDescription: " + - "<text>" + info['pn'] + " version " + info['pv'] + "</text>") - for contain in info['package_contains'].split( ): - sed_cmd = sed_insert(sed_cmd, "PackageComment:"," \\n\\n## Relationships\\nRelationship: " + info['pn'] + " CONTAINS " + contain) - for static_link in info['package_static_link'].split( ): - sed_cmd = sed_insert(sed_cmd, "PackageComment:"," \\n\\n## Relationships\\nRelationship: " + info['pn'] + " STATIC_LINK " + static_link) - sed_cmd = sed_cmd + sstatefile - - subprocess.call("%s" % sed_cmd, shell=True) - -def is_work_shared(d): - pn = d.getVar('PN') - return bb.data.inherits_class('kernel', d) or pn.startswith('gcc-source') - -def remove_dir_tree(dir_name): - import shutil - try: - shutil.rmtree(dir_name) - except: - pass - -def remove_file(file_name): - try: - os.remove(file_name) - except OSError as e: - pass - -def list_files(dir ): - for root, subFolders, files in os.walk(dir): - for f in files: - rel_root = os.path.relpath(root, dir) - yield rel_root, f - return - -def hash_file(file_name): - """ - Return the hex string representation of the SHA1 checksum of the filename - """ - try: - import hashlib - except ImportError: - return None - - sha1 = hashlib.sha1() - with open( file_name, "rb" ) as f: - for line in f: - sha1.update(line) - return sha1.hexdigest() - -def hash_string(data): - import hashlib - sha1 = hashlib.sha1() - sha1.update(data.encode('utf-8')) - return sha1.hexdigest() - -def get_ver_code(dirname): - chksums = [] - for f_dir, f in list_files(dirname): - try: - stats = os.stat(os.path.join(dirname,f_dir,f)) - except OSError as e: - bb.warn( "Stat failed" + str(e) + "\n") - continue - chksums.append(hash_file(os.path.join(dirname,f_dir,f))) - ver_code_string = ''.join(chksums).lower() - ver_code = hash_string(ver_code_string) - return ver_code - -do_spdx[depends] = "${SPDXEPENDENCY}" - EXPORT_FUNCTIONS do_spdx diff --git a/classes/spdx-common.bbclass b/classes/spdx-common.bbclass new file mode 100644 index 0000000..762437c --- /dev/null +++ b/classes/spdx-common.bbclass @@ -0,0 +1,224 @@ +# This class integrates real-time license scanning, generation of SPDX standard +# output and verifiying license info during the building process. +# It is a combination of efforts from the OE-Core, SPDX and DoSOCSv2 projects. +# +# For more information on DoSOCSv2: +# https://github.com/DoSOCSv2 +# +# For more information on SPDX: +# http://www.spdx.org +# +# Note: +# 1) Make sure fossdriver has beed installed in your host +# 2) By default,spdx files will be output to the path which is defined as[SPDX_DEPLOY_DIR] +# in ./meta/conf/spdx-dosocs.conf. + + +SPDXEPENDENCY += "${PATCHTOOL}-native:do_populate_sysroot" +SPDXEPENDENCY += " wget-native:do_populate_sysroot" +SPDXEPENDENCY += " subversion-native:do_populate_sysroot" +SPDXEPENDENCY += " git-native:do_populate_sysroot" +SPDXEPENDENCY += " lz4-native:do_populate_sysroot" +SPDXEPENDENCY += " lzip-native:do_populate_sysroot" +SPDXEPENDENCY += " xz-native:do_populate_sysroot" +SPDXEPENDENCY += " unzip-native:do_populate_sysroot" +SPDXEPENDENCY += " xz-native:do_populate_sysroot" +SPDXEPENDENCY += " nodejs-native:do_populate_sysroot" +SPDXEPENDENCY += " quilt-native:do_populate_sysroot" +SPDXEPENDENCY += " tar-native:do_populate_sysroot" + +SPDX_DEPLOY_DIR ??= "${DEPLOY_DIR}/spdx" +SPDX_TOPDIR ?= "${WORKDIR}/spdx_sstate_dir" +SPDX_OUTDIR = "${SPDX_TOPDIR}/${TARGET_SYS}/${PF}/" +SPDX_WORKDIR = "${WORKDIR}/spdx_temp/" + +do_spdx[dirs] = "${WORKDIR}" + +LICENSELISTVERSION = "2.6" + +# If ${S} isn't actually the top-level source directory, set SPDX_S to point at +# the real top-level directory. +SPDX_S ?= "${S}" + +addtask do_spdx before do_unpack after do_fetch + +def spdx_create_tarball(d, srcdir, suffix, ar_outdir): + """ + create the tarball from srcdir + """ + import tarfile, shutil + # Make sure we are only creating a single tarball for gcc sources + #if (d.getVar('SRC_URI') == ""): + # return + + # For the kernel archive, srcdir may just be a link to the + # work-shared location. Use os.path.realpath to make sure + # that we archive the actual directory and not just the link. + srcdir = os.path.realpath(srcdir) + + bb.utils.mkdirhier(ar_outdir) + if suffix: + filename = '%s-%s.tar.gz' % (d.getVar('PF'), suffix) + else: + filename = '%s.tar.gz' % d.getVar('PF') + tarname = os.path.join(ar_outdir, filename) + + bb.note('Creating %s' % tarname) + tar = tarfile.open(tarname, 'w:gz') + tar.add(srcdir, arcname=os.path.basename(srcdir)) + tar.close() + shutil.rmtree(srcdir) + return tarname + +# Run do_unpack and do_patch +def spdx_get_src(d): + import shutil + spdx_workdir = d.getVar('SPDX_WORKDIR') + spdx_sysroot_native = d.getVar('STAGING_DIR_NATIVE') + pn = d.getVar('PN') + + # We just archive gcc-source for all the gcc related recipes + if d.getVar('BPN') in ['gcc', 'libgcc']: + bb.debug(1, 'spdx: There is bug in scan of %s is, do nothing' % pn) + return + + # The kernel class functions require it to be on work-shared, so we dont change WORKDIR + if not is_work_shared(d): + # Change the WORKDIR to make do_unpack do_patch run in another dir. + d.setVar('WORKDIR', spdx_workdir) + # Restore the original path to recipe's native sysroot (it's relative to WORKDIR). + d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native) + + # The changed 'WORKDIR' also caused 'B' changed, create dir 'B' for the + # possibly requiring of the following tasks (such as some recipes's + # do_patch required 'B' existed). + bb.utils.mkdirhier(d.getVar('B')) + + bb.build.exec_func('do_unpack', d) + + # Make sure gcc and kernel sources are patched only once + if not (d.getVar('SRC_URI') == "" or is_work_shared(d)): + bb.build.exec_func('do_patch', d) + # Some userland has no source. + if not os.path.exists( spdx_workdir ): + bb.utils.mkdirhier(spdx_workdir) + +def create_manifest(info,sstatefile): + import shutil + shutil.copyfile(sstatefile,info['outfile']) + +def get_cached_spdx( sstatefile ): + import subprocess + + if not os.path.exists( sstatefile ): + return None + + try: + output = subprocess.check_output(['grep', "PackageVerificationCode", sstatefile]) + except subprocess.CalledProcessError as e: + bb.error("Index creation command '%s' failed with return code %d:\n%s" % (e.cmd, e.returncode, e.output)) + return None + cached_spdx_info=output.decode('utf-8').split(': ') + return cached_spdx_info[1] + +## Add necessary information into spdx file +def write_cached_spdx( info,sstatefile, ver_code ): + import subprocess + + def sed_replace(dest_sed_cmd,key_word,replace_info): + dest_sed_cmd = dest_sed_cmd + "-e 's#^" + key_word + ".*#" + \ + key_word + replace_info + "#' " + return dest_sed_cmd + + def sed_insert(dest_sed_cmd,key_word,new_line): + dest_sed_cmd = dest_sed_cmd + "-e '/^" + key_word \ + + r"/a\\" + new_line + "' " + return dest_sed_cmd + + ## Document level information + sed_cmd = r"sed -i -e 's#\r$##g' " + spdx_DocumentComment = "<text>SPDX for " + info['pn'] + " version " \ + + info['pv'] + "</text>" + sed_cmd = sed_replace(sed_cmd,"DocumentComment",spdx_DocumentComment) + + ## Creator information + sed_cmd = sed_replace(sed_cmd,"Creator: ",info['creator']['Tool']) + + ## Package level information + sed_cmd = sed_replace(sed_cmd, "PackageName: ", info['pn']) + sed_cmd = sed_insert(sed_cmd, "PackageName: ", "PackageVersion: " + info['pv']) + sed_cmd = sed_replace(sed_cmd, "PackageDownloadLocation: ",info['package_download_location']) + sed_cmd = sed_insert(sed_cmd, "PackageDownloadLocation: ", "PackageHomePage: " + info['package_homepage']) + sed_cmd = sed_insert(sed_cmd, "PackageDownloadLocation: ", "PackageSummary: " + "<text>" + info['package_summary'] + "</text>") + sed_cmd = sed_insert(sed_cmd, "PackageDownloadLocation: ", "modification record : " + info['modified']) + sed_cmd = sed_replace(sed_cmd, "PackageVerificationCode: ",ver_code) + sed_cmd = sed_insert(sed_cmd, "PackageVerificationCode: ", "PackageDescription: " + + "<text>" + info['pn'] + " version " + info['pv'] + "</text>") + for contain in info['package_contains'].split( ): + sed_cmd = sed_insert(sed_cmd, "PackageComment:"," \\n\\n## Relationships\\nRelationship: " + info['pn'] + " CONTAINS " + contain) + for static_link in info['package_static_link'].split( ): + sed_cmd = sed_insert(sed_cmd, "PackageComment:"," \\n\\n## Relationships\\nRelationship: " + info['pn'] + " STATIC_LINK " + static_link) + sed_cmd = sed_cmd + sstatefile + + subprocess.call("%s" % sed_cmd, shell=True) + +def is_work_shared(d): + pn = d.getVar('PN') + return bb.data.inherits_class('kernel', d) or pn.startswith('gcc-source') + +def remove_dir_tree(dir_name): + import shutil + try: + shutil.rmtree(dir_name) + except: + pass + +def remove_file(file_name): + try: + os.remove(file_name) + except OSError as e: + pass + +def list_files(dir ): + for root, subFolders, files in os.walk(dir): + for f in files: + rel_root = os.path.relpath(root, dir) + yield rel_root, f + return + +def hash_file(file_name): + """ + Return the hex string representation of the SHA1 checksum of the filename + """ + try: + import hashlib + except ImportError: + return None + + sha1 = hashlib.sha1() + with open( file_name, "rb" ) as f: + for line in f: + sha1.update(line) + return sha1.hexdigest() + +def hash_string(data): + import hashlib + sha1 = hashlib.sha1() + sha1.update(data.encode('utf-8')) + return sha1.hexdigest() + +def get_ver_code(dirname): + chksums = [] + for f_dir, f in list_files(dirname): + try: + stats = os.stat(os.path.join(dirname,f_dir,f)) + except OSError as e: + bb.warn( "Stat failed" + str(e) + "\n") + continue + chksums.append(hash_file(os.path.join(dirname,f_dir,f))) + ver_code_string = ''.join(chksums).lower() + ver_code = hash_string(ver_code_string) + return ver_code + +do_spdx[depends] = "${SPDXEPENDENCY}" + diff --git a/recipes-devtools/fossology/fossology-nomos-native_git.bb b/recipes-devtools/fossology/fossology-nomos-native_git.bb index e034a34..d7d2fd3 100644 --- a/recipes-devtools/fossology/fossology-nomos-native_git.bb +++ b/recipes-devtools/fossology/fossology-nomos-native_git.bb @@ -4,12 +4,11 @@ SECTION = "devel/python" LICENSE = "GPLv2" LIC_FILES_CHKSUM = "file://GenCodeCopyright;md5=7296ec131dbd040718b64fb843d63048" -DEPENDS = "glib-2.0-native libpcre-native" +DEPENDS = "glib-2.0-native libpcre-native json-c-native" -SRCREV = "34467fd530b832f08c095936a72c22c40fa13278" +SRCREV = "bd076cff2c235502055ecc020d44783ce5907051" BRANCH = "release/${PV}/master" SRC_URI = "git://github.com/fossology/fossology.git \ - file://0001-Add-rpath-to-fix-error-as-following.patch \ " S = "${WORKDIR}/git/src/nomos/agent" |