aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLei Maohui <leimaohui@cn.fujitsu.com>2016-11-23 16:58:37 +0900
committerLei Maohui <leimaohui@cn.fujitsu.com>2016-11-23 16:58:37 +0900
commit304faaa80aef77412301891b311cf2ca83484adc (patch)
tree5308696c36bdc13d89c762390afaa095e174a739
parent08d506ff55acec446823cceafecbf73bacd6f3b9 (diff)
downloadmeta-spdxscanner-304faaa80aef77412301891b311cf2ca83484adc.tar.gz
meta-spdxscanner-304faaa80aef77412301891b311cf2ca83484adc.tar.bz2
meta-spdxscanner-304faaa80aef77412301891b311cf2ca83484adc.zip
Add spdx-dosocs.bbclass
Change spdx create tool from fossology-spdx to dosocs2. With this bbclass, you can: 1. Get spdx files compliance with SPDX specification 2.0. 3. Get spdx files in less time, especially for huge software. Signed-off-by: Lei Maohui <leimaohui@cn.fujitsu.com>
-rw-r--r--classes/spdx-dosocs.bbclass252
1 files changed, 252 insertions, 0 deletions
diff --git a/classes/spdx-dosocs.bbclass b/classes/spdx-dosocs.bbclass
new file mode 100644
index 0000000..78a878e
--- /dev/null
+++ b/classes/spdx-dosocs.bbclass
@@ -0,0 +1,252 @@
+# This class integrates real-time license scanning, generation of SPDX standard
+# output and verifiying license info during the building process.
+# It is a combination of efforts from the OE-Core, SPDX and DoSOCSv2 projects.
+#
+# For more information on DoSOCSv2:
+# https://github.com/DoSOCSv2
+#
+# For more information on SPDX:
+# http://www.spdx.org
+#
+# Note:
+# 1) Make sure DoSOCSv2 has beed installed in your host
+# 2) By default,spdx files will be output to the path which is defined as[SPDX_MANIFEST_DIR]
+# in ./meta/conf/spdx-dosocs.conf.
+
+SPDXOUTPUTDIR = "${WORKDIR}/spdx_output_dir"
+SPDXSSTATEDIR = "${WORKDIR}/spdx_sstate_dir"
+
+# If ${S} isn't actually the top-level source directory, set SPDX_S to point at
+# the real top-level directory.
+
+SPDX_S ?= "${S}"
+
+python do_spdx () {
+ import os, sys
+ import json
+
+ ## It's no necessary to get spdx files for *-native
+ if d.getVar('PN', True) == d.getVar('BPN', True) + "-native":
+ return None
+
+ ## gcc is too big to get spdx file.
+ if 'gcc' in d.getVar('PN', True):
+ return None
+
+ info = {}
+ info['workdir'] = (d.getVar('WORKDIR', True) or "")
+ info['pn'] = (d.getVar( 'PN', True ) or "")
+ info['pv'] = (d.getVar( 'PV', True ) or "")
+ info['package_download_location'] = (d.getVar( 'SRC_URI', True ) or "")
+ if info['package_download_location'] != "":
+ info['package_download_location'] = info['package_download_location'].split()[0]
+ info['spdx_version'] = (d.getVar('SPDX_VERSION', True) or '')
+ info['data_license'] = (d.getVar('DATA_LICENSE', True) or '')
+ info['creator'] = {}
+ info['creator']['Tool'] = (d.getVar('CREATOR_TOOL', True) or '')
+ info['license_list_version'] = (d.getVar('LICENSELISTVERSION', True) or '')
+ info['package_homepage'] = (d.getVar('HOMEPAGE', True) or "")
+ info['package_summary'] = (d.getVar('SUMMARY', True) or "")
+ info['package_summary'] = info['package_summary'].replace("\n","")
+ info['package_summary'] = info['package_summary'].replace("'"," ")
+
+ spdx_sstate_dir = (d.getVar('SPDXSSTATEDIR', True) or "")
+ manifest_dir = (d.getVar('SPDX_MANIFEST_DIR', True) or "")
+ info['outfile'] = os.path.join(manifest_dir, info['pn'] + "-" + info['pv'] + ".spdx" )
+ sstatefile = os.path.join(spdx_sstate_dir,
+ info['pn'] + "-" + info['pv'] + ".spdx" )
+
+ ## get everything from cache. use it to decide if
+ ## something needs to be rerun
+ if not os.path.exists( spdx_sstate_dir ):
+ bb.utils.mkdirhier( spdx_sstate_dir )
+
+ d.setVar('WORKDIR', d.getVar('SPDX_TEMP_DIR', True))
+ info['sourcedir'] = (d.getVar('SPDX_S', True) or "")
+ cur_ver_code = get_ver_code( info['sourcedir'] ).split()[0]
+ cache_cur = False
+ if os.path.exists( sstatefile ):
+ ## cache for this package exists. read it in
+ cached_spdx = get_cached_spdx( sstatefile )
+ if cached_spdx:
+ cached_spdx = cached_spdx.split()[0]
+ if (cached_spdx == cur_ver_code):
+ bb.warn(info['pn'] + "'s ver code same as cache's. do nothing")
+ cache_cur = True
+ create_manifest(info,sstatefile)
+ if not cache_cur:
+ ## setup dosocs2 command
+ dosocs2_command = "dosocs2 oneshot %s" % info['sourcedir']
+ ## no necessary to scan the git directory.
+ git_path = "%s/.git" % info['sourcedir']
+ if os.path.exists(git_path):
+ remove_dir_tree(git_path)
+
+ ## Get spdx file
+ run_dosocs2(dosocs2_command,sstatefile)
+ if get_cached_spdx( sstatefile ) != None:
+ write_cached_spdx( info,sstatefile,cur_ver_code )
+ ## CREATE MANIFEST(write to outfile )
+ create_manifest(info,sstatefile)
+ else:
+ bb.warn('Can\'t get the spdx file ' + info['pn'] + '. Please check your dosocs2.')
+ d.setVar('WORKDIR', info['workdir'])
+}
+## Get the src after do_patch.
+python do_get_spdx_s() {
+
+ ## It's no necessary to get spdx files for *-native
+ if d.getVar('PN', True) == d.getVar('BPN', True) + "-native":
+ return None
+
+ ## gcc is too big to get spdx file.
+ if 'gcc' in d.getVar('PN', True):
+ return None
+
+ ## Change the WORKDIR to make do_unpack do_patch run in another dir.
+ d.setVar('WORKDIR', d.getVar('SPDX_TEMP_DIR', True))
+ ## The changed 'WORKDIR' also casued 'B' changed, create dir 'B' for the
+ ## possibly requiring of the following tasks (such as some recipes's
+ ## do_patch required 'B' existed).
+ bb.utils.mkdirhier(d.getVar('B', True))
+
+ ## The kernel source is ready after do_validate_branches
+ if bb.data.inherits_class('kernel-yocto', d):
+ bb.build.exec_func('do_unpack', d)
+ bb.build.exec_func('do_kernel_checkout', d)
+ bb.build.exec_func('do_validate_branches', d)
+ else:
+ bb.build.exec_func('do_unpack', d)
+ ## The S of the gcc source is work-share
+ flag = d.getVarFlag('do_unpack', 'stamp-base', True)
+ if flag:
+ d.setVar('S', d.getVar('WORKDIR', True) + "/gcc-" + d.getVar('PV', True))
+ bb.build.exec_func('do_patch', d)
+}
+
+addtask get_spdx_s after do_patch before do_configure
+addtask spdx after do_get_spdx_s before do_configure
+
+def create_manifest(info,sstatefile):
+ import shutil
+ shutil.copyfile(sstatefile,info['outfile'])
+
+def get_cached_spdx( sstatefile ):
+ import subprocess
+
+ if not os.path.exists( sstatefile ):
+ return None
+
+ try:
+ output = subprocess.check_output(['grep', "PackageVerificationCode", sstatefile])
+ except subprocess.CalledProcessError as e:
+ bb.error("Index creation command '%s' failed with return code %d:\n%s" % (e.cmd, e.returncode, e.output))
+ return None
+ cached_spdx_info=output.decode('utf-8').split(': ')
+ return cached_spdx_info[1]
+
+## Add necessary information into spdx file
+def write_cached_spdx( info,sstatefile, ver_code ):
+ import subprocess
+
+ def sed_replace(dest_sed_cmd,key_word,replace_info):
+ dest_sed_cmd = dest_sed_cmd + "-e 's#^" + key_word + ".*#" + \
+ key_word + replace_info + "#' "
+ return dest_sed_cmd
+
+ def sed_insert(dest_sed_cmd,key_word,new_line):
+ dest_sed_cmd = dest_sed_cmd + "-e '/^" + key_word \
+ + r"/a\\" + new_line + "' "
+ return dest_sed_cmd
+
+ ## Document level information
+ sed_cmd = r"sed -i -e 's#\r$##g' "
+ spdx_DocumentComment = "<text>SPDX for " + info['pn'] + " version " \
+ + info['pv'] + "</text>"
+ sed_cmd = sed_replace(sed_cmd,"DocumentComment",spdx_DocumentComment)
+
+ ## Creator information
+ sed_cmd = sed_insert(sed_cmd,"CreatorComment: ","LicenseListVersion: " + info['license_list_version'])
+
+ ## Package level information
+ sed_cmd = sed_replace(sed_cmd,"PackageName: ",info['pn'])
+ sed_cmd = sed_replace(sed_cmd,"PackageVersion: ",info['pv'])
+ sed_cmd = sed_replace(sed_cmd,"PackageDownloadLocation: ",info['package_download_location'])
+ sed_cmd = sed_insert(sed_cmd,"PackageChecksum: ","PackageHomePage: " + info['package_homepage'])
+ sed_cmd = sed_replace(sed_cmd,"PackageSummary: ","<text>" + info['package_summary'] + "</text>")
+ sed_cmd = sed_replace(sed_cmd,"PackageVerificationCode: ",ver_code)
+ sed_cmd = sed_replace(sed_cmd,"PackageDescription: ",
+ "<text>" + info['pn'] + " version " + info['pv'] + "</text>")
+ sed_cmd = sed_cmd + sstatefile
+
+ subprocess.call("%s" % sed_cmd, shell=True)
+
+def remove_dir_tree( dir_name ):
+ import shutil
+ try:
+ shutil.rmtree( dir_name )
+ except:
+ pass
+
+def remove_file( file_name ):
+ try:
+ os.remove( file_name )
+ except OSError as e:
+ pass
+
+def list_files( dir ):
+ for root, subFolders, files in os.walk( dir ):
+ for f in files:
+ rel_root = os.path.relpath( root, dir )
+ yield rel_root, f
+ return
+
+def hash_file( file_name ):
+ """
+ Return the hex string representation of the SHA1 checksum of the filename
+ """
+ try:
+ import hashlib
+ except ImportError:
+ return None
+
+ sha1 = hashlib.sha1()
+ with open( file_name, "rb" ) as f:
+ for line in f:
+ sha1.update(line)
+ return sha1.hexdigest()
+
+def hash_string( data ):
+ import hashlib
+ sha1 = hashlib.sha1()
+ sha1.update( data.encode('utf-8') )
+ return sha1.hexdigest()
+
+def run_dosocs2( dosocs2_command, spdx_file ):
+ import subprocess, codecs
+ import string, re
+
+ p = subprocess.Popen(dosocs2_command.split(),
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ dosocs2_output, dosocs2_error = p.communicate()
+ if p.returncode != 0:
+ return None
+
+ dosocs2_output = dosocs2_output.decode('utf-8')
+
+ f = codecs.open(spdx_file,'w','utf-8')
+ f.write(dosocs2_output)
+
+def get_ver_code( dirname ):
+ chksums = []
+ for f_dir, f in list_files( dirname ):
+ try:
+ stats = os.stat(os.path.join(dirname,f_dir,f))
+ except OSError as e:
+ bb.warn( "Stat failed" + str(e) + "\n")
+ continue
+ chksums.append(hash_file(os.path.join(dirname,f_dir,f)))
+ ver_code_string = ''.join( chksums ).lower()
+ ver_code = hash_string( ver_code_string )
+ return ver_code
+