diff options
Diffstat (limited to 'classes/spdx-common.bbclass')
| -rw-r--r-- | classes/spdx-common.bbclass | 181 |
1 files changed, 162 insertions, 19 deletions
diff --git a/classes/spdx-common.bbclass b/classes/spdx-common.bbclass index cb97c5b..1a5a5f4 100644 --- a/classes/spdx-common.bbclass +++ b/classes/spdx-common.bbclass @@ -1,5 +1,6 @@ # This class supplys common functions. +HOSTTOOLS += "xz" SPDX_DEPLOY_DIR ??= "${DEPLOY_DIR}/spdx" SPDX_TOPDIR ?= "${WORKDIR}/spdx_sstate_dir" SPDX_OUTDIR ?= "${SPDX_TOPDIR}/${TARGET_SYS}/${PF}/" @@ -31,9 +32,15 @@ def excluded_package(d, pn): if p != pn: pn = p break - if d.getVar('BPN') in ['gcc', 'libgcc']: - #bb.debug(1, 'spdx: There is a bug in the scan of %s, skip it.' % pn) + # We just archive gcc-source for all the gcc related recipes + if d.getVar('BPN') in ['gcc', 'libgcc'] \ + and not pn.startswith('gcc-source'): + bb.debug(1, 'archiver: %s is excluded, covered by gcc-source' % pn) return True + # TARGET_SYS in ARCHIVER_ARCH will break the stamp for gcc-source in multiconfig + if pn.startswith('gcc-source'): + d.setVar('ARCHIVER_ARCH', "allarch") + # The following: do_fetch, do_unpack and do_patch tasks have been deleted, # so avoid archiving do_spdx here. # -native is for the host aka during the build @@ -65,16 +72,10 @@ def get_tar_name(d, suffix): get the name of tarball """ - # Make sure we are only creating a single tarball for gcc sources - #if (d.getVar('SRC_URI') == ""): - # return - # For the kernel archive, srcdir may just be a link to the - # work-shared location. Use os.path.realpath to make sure - # that we archive the actual directory and not just the link. if suffix: - filename = '%s-%s.tar.gz' % (d.getVar('PF'), suffix) + filename = '%s-%s.tar.xz' % (d.getVar('PF'), suffix) else: - filename = '%s.tar.gz' % d.getVar('PF') + filename = '%s.tar.xz' % d.getVar('PF') return filename @@ -84,22 +85,29 @@ def spdx_create_tarball(d, srcdir, suffix, ar_outdir): """ import tarfile, shutil - # Make sure we are only creating a single tarball for gcc sources - #if (d.getVar('SRC_URI') == ""): - # return - # For the kernel archive, srcdir may just be a link to the - # work-shared location. Use os.path.realpath to make sure - # that we archive the actual directory and not just the link. srcdir = os.path.realpath(srcdir) bb.utils.mkdirhier(ar_outdir) filename = get_tar_name(d, suffix) tarname = os.path.join(ar_outdir, filename) bb.note('Creating %s' % tarname) - tar = tarfile.open(tarname, 'w:gz') + tar = tarfile.open(tarname, 'w:xz') tar.add(srcdir, arcname=os.path.basename(srcdir), filter=exclude_useless_paths) tar.close() shutil.rmtree(srcdir) + + info = {} + info['pn'] = (d.getVar( 'PN') or "") + info['pv'] = (d.getVar( 'PKGV') or "") + info['pr'] = (d.getVar( 'PR') or "") + + if d.getVar('SAVE_SPDX_ACHIVE'): + manifest_dir = (d.getVar('SPDX_DEPLOY_DIR') or "") + if not os.path.exists( manifest_dir ): + bb.utils.mkdirhier( manifest_dir ) + info['outfile'] = os.path.join(manifest_dir, filename) + create_manifest(info,tarname) + return tarname def get_tarball_name(d, srcdir, suffix, ar_outdir): @@ -142,6 +150,11 @@ def spdx_get_src(d): bb.utils.mkdirhier(src_dir) if bb.data.inherits_class('kernel',d): share_src = d.getVar('STAGING_KERNEL_DIR') + if pn.startswith('gcc-source'): + gcc_source_path = d.getVar('TMPDIR') + "/work-shared" + gcc_pv = d.getVar('PV') + gcc_pr = d.getVar('PR') + share_src = gcc_source_path + "/gcc-" + gcc_pv + "-" + gcc_pr + "/gcc-" + gcc_pv + "/" cmd_copy_share = "cp -rf " + share_src + "/* " + src_dir + "/" cmd_copy_kernel_result = os.popen(cmd_copy_share).read() bb.note("cmd_copy_kernel_result = " + cmd_copy_kernel_result) @@ -228,9 +241,15 @@ def find_infoinlicensefile(sstatefile): continue license = line_spdx.split(": ")[1] license = license.split("\n")[0] + bb.note("file path = " + file_path) file_path = file_path.split("\n")[0] + bb.note("file path = " + file_path) path_list = file_path.split('/') - if len(file_path.split('/')) < 5: + if len(file_path.split('/')) < 3: + file_path_simple = file_path.split('/',1)[1] + elif len(file_path.split('/')) < 4: + file_path_simple = file_path.split('/',2)[2] + elif len(file_path.split('/')) < 5: file_path_simple = file_path.split('/',3)[3] else: file_path_simple = file_path.split('/',4)[4] @@ -248,6 +267,7 @@ def find_infoinlicensefile(sstatefile): ## Add necessary information into spdx file def write_cached_spdx( info,sstatefile, ver_code ): import subprocess + import re infoinlicensefile="" @@ -256,11 +276,22 @@ def write_cached_spdx( info,sstatefile, ver_code ): key_word + replace_info + "#' " return dest_sed_cmd + def sed_replace_aline(dest_sed_cmd,origin_line,dest_line): + dest_sed_cmd = dest_sed_cmd + "-e 's#^" + origin_line + ".*#" + \ + dest_line + "#' " + return dest_sed_cmd + def sed_insert(dest_sed_cmd,key_word,new_line): dest_sed_cmd = dest_sed_cmd + "-e '/^" + key_word \ + r"/a\\" + new_line + "' " return dest_sed_cmd + def sed_insert_front(dest_sed_cmd,key_word,new_line): + dest_sed_cmd = dest_sed_cmd + "-e '/^" + key_word \ + + r"/i\\" + new_line + "' " + return dest_sed_cmd + + ## Delet ^M in doc format subprocess.call("sed -i -e 's#\r##g' %s" % sstatefile, shell=True) @@ -278,7 +309,10 @@ def write_cached_spdx( info,sstatefile, ver_code ): sed_cmd = sed_replace(sed_cmd,"Creator: Tool: ",info['creator']['Tool']) ## Package level information + sed_cmd = sed_replace_aline(sed_cmd, "SPDXVersion: SPDX-2.2", "SPDXVersion: SPDX-2.3") sed_cmd = sed_replace(sed_cmd, "PackageName: ", info['pn']) + sed_cmd = sed_replace_aline(sed_cmd, "SPDXID: SPDXRef-", "SPDXID: SPDXRef-" + info['pkg_spdx_id']) + sed_cmd = sed_replace(sed_cmd, "Relationship: SPDXRef-DOCUMENT DESCRIBES SPDXRef-", info['pkg_spdx_id']) sed_cmd = sed_insert(sed_cmd, "PackageName: ", "PackageVersion: " + info['pv']) sed_cmd = sed_replace(sed_cmd, "PackageDownloadLocation: ",info['package_download_location']) sed_cmd = sed_insert(sed_cmd, "PackageDownloadLocation: ", "PackageHomePage: " + info['package_homepage']) @@ -291,6 +325,13 @@ def write_cached_spdx( info,sstatefile, ver_code ): sed_cmd = sed_insert(sed_cmd, "PackageComment:"," \\n\\n## Relationships\\nRelationship: " + info['pn'] + " CONTAINS " + contain) for static_link in info['package_static_link'].split( ): sed_cmd = sed_insert(sed_cmd, "PackageComment:"," \\n\\n## Relationships\\nRelationship: " + info['pn'] + " STATIC_LINK " + static_link) + sed_cmd = sed_insert(sed_cmd, "PackageVerificationCode: ", "BuiltDate: " + info['build_time']) + sed_cmd = sed_insert(sed_cmd, "PackageVerificationCode: ", "ReleaseDate: " + info['release_date']) + sed_cmd = sed_insert(sed_cmd, "PackageVerificationCode: ", "PrimaryPackagePurpose: " + info['purpose']) + depends = info['depends_on'] + for depend in re.split(r'\s*[,\s\n\r]\s*', depends): + sed_cmd = sed_insert_front(sed_cmd, "PackageCopyrightText: ", "Relationship: SPDXRef-" + info['pn'] + " DEPENDS_ON SPDXRef-" + depend) + bb.note("sed_cmd = " + sed_cmd) sed_cmd = sed_cmd + sstatefile subprocess.call("%s" % sed_cmd, shell=True) @@ -301,6 +342,9 @@ def write_cached_spdx( info,sstatefile, ver_code ): sed_cmd = sed_insert(sed_cmd, "ModificationRecord: ", oneline_infoinlicensefile) sed_cmd = sed_cmd + sstatefile subprocess.call("%s" % sed_cmd, shell=True) + + with open(sstatefile, encoding="utf-8", mode="a") as file: + file.write(info['external_refs']) def is_work_shared(d): pn = d.getVar('PN') @@ -353,7 +397,7 @@ def get_ver_code(dirname): try: stats = os.stat(os.path.join(dirname,f_dir,f)) except OSError as e: - bb.warn( "Stat failed" + str(e) + "\n") + bb.note( "Stat failed" + str(e) + "\n") continue chksums.append(hash_file(os.path.join(dirname,f_dir,f))) ver_code_string = ''.join(chksums).lower() @@ -382,5 +426,104 @@ python do_spdx_creat_tarball(){ } # For scancode-tk.bbclass, just python do_spdx_get_src(){ + import shutil + + spdx_outdir = d.getVar('SPDX_OUTDIR') + + spdx_workdir = d.getVar('SPDX_WORKDIR') + spdx_temp_dir = os.path.join(spdx_workdir, "temp") + temp_dir = os.path.join(d.getVar('WORKDIR'), "temp") + bb.utils.mkdirhier(spdx_workdir) + spdx_get_src(d) + + if os.path.isdir(spdx_temp_dir): + for f_dir, f in list_files(spdx_temp_dir): + temp_file = os.path.join(spdx_temp_dir,f_dir,f) + shutil.copy(temp_file, temp_dir) + bb.note("temp_dir = " + spdx_temp_dir) } + +#For SPDX2.3 +def get_external_refs(d): + from oe.cve_check import get_patched_cves + external_refs = "##------------------------- \n" + external_refs += "## Security Information \n" + external_refs += "##------------------------- \n" + external_refs += "\"externalRefs\" : [" + unpatched_cves = [] + nvd_link = "https://nvd.nist.gov/vuln/detail/" + with bb.utils.fileslocked([d.getVar("CVE_CHECK_DB_FILE_LOCK")], shared=True): + if os.path.exists(d.getVar("CVE_CHECK_DB_FILE")): + try: + patched_cves = get_patched_cves(d) + except FileNotFoundError: + bb.fatal("Failure in searching patches") + ignored, patched, unpatched, status = check_cves(d, patched_cves) + if patched or unpatched or (d.getVar("CVE_CHECK_COVERAGE") == "1" and status): + cve_data = get_cve_info(d, patched + unpatched + ignored) + #cve_write_data(d, patched, unpatched, ignored, cve_data, status) + else: + bb.note("No CVE database found, skipping CVE check") + return " " + if not patched+unpatched+ignored: + return " " + + for cve in sorted(cve_data): + is_patched = cve in patched + is_ignored = cve in ignored + + status = "unpatched" + if is_ignored: + status = "ignored" + elif is_patched: + status = "fix" + else: + # default value of status is Unpatched + unpatched_cves.append(cve) + external_refs += "{\n" + external_refs += "\"referenceCategory\" : \"SECURITY\",\n" + external_refs += "\"referenceLocator\" : \"https://nvd.nist.gov/vuln/detail/%s\",\n" % cve + external_refs += "\"referenceType\" : \"%s\"\n" % status + external_refs += "}," + + external_refs += "]" + #bb.warn("external_refs = " + external_refs) + return external_refs + +def get_pkgpurpose(d): + section = d.getVar("SECTION") + if section in "libs": + return "LIBRARY" + else: + return "APPLICATION " + +def get_build_date(d): + from datetime import datetime, timezone + + build_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + return build_time + +def get_depends_on(d): + import re + + depends = re.split(r'\s*[\s]\s*',d.getVar("DEPENDS")) + depends_spdx = "" + for depend in depends: + bb.note("depend = " + depend) + if depend.endswith("-native"): + bb.note("Don't show *-native in depends relationship.\n") + else: + depends_spdx += depend + "," + depends_spdx = depends_spdx.strip(',') + return depends_spdx + + +def get_spdxid_pkg(d): + if d.getVar("PROVIDES"): + pid = d.getVar("PROVIDES") + else: + pid = d.getVar("PN") + bb.note("SPDX ID of pkg = " + pid) + return pid + |