diff options
Diffstat (limited to 'classes')
-rw-r--r-- | classes/blackduck-upload.bbclass | 18 | ||||
-rw-r--r-- | classes/fossology-python.bbclass | 56 | ||||
-rw-r--r-- | classes/spdx-common.bbclass | 27 |
3 files changed, 57 insertions, 44 deletions
diff --git a/classes/blackduck-upload.bbclass b/classes/blackduck-upload.bbclass index cad3ad5..8ce637a 100644 --- a/classes/blackduck-upload.bbclass +++ b/classes/blackduck-upload.bbclass @@ -17,13 +17,14 @@ inherit copyleft_filter inherit spdx-common do_upload[dirs] = "${SPDX_TOPDIR}" +do_bd_upload[network] = "1" WAIT_TIME ?= "20" python () { - + pn = d.getVar('PN') #If not for target, won't creat spdx. - if bb.data.inherits_class('nopackages', d): + if bb.data.inherits_class('nopackages', d) and not pn.startswith('gcc-source'): return pn = d.getVar('PN') @@ -46,7 +47,8 @@ python () { return # We just archive gcc-source for all the gcc related recipes - if d.getVar('BPN') in ['gcc', 'libgcc']: + if d.getVar('BPN') in ['gcc', 'libgcc'] \ + and not pn.startswith('gcc-source'): bb.debug(1, 'spdx: There is bug in scan of %s is, do nothing' % pn) return @@ -64,9 +66,9 @@ python () { def hasTask(task): return bool(d.getVarFlag(task, "task", False)) and not bool(d.getVarFlag(task, "noexec", False)) - if d.getVar('PACKAGES'): + if d.getVar('PACKAGES') or pn.startswith('gcc-source'): # Some recipes do not have any packaging tasks - if hasTask("do_package_write_rpm") or hasTask("do_package_write_ipk") or hasTask("do_package_write_deb"): + if hasTask("do_package_write_rpm") or hasTask("do_package_write_ipk") or hasTask("do_package_write_deb") or pn.startswith('gcc-source'): d.appendVarFlag('do_bd_upload', 'depends', ' synopsys-native:do_populate_sysroot') d.appendVarFlag('do_bd_upload', 'depends', ' %s:do_spdx_creat_tarball' % pn) d.appendVarFlag('do_synopsys_detect', 'depends', ' %s:do_bd_upload' % pn) @@ -77,7 +79,9 @@ python () { python do_bd_upload(){ import logging, shutil,time - if bb.data.inherits_class('nopackages', d): + pn = d.getVar( 'PN') + #If not for target, won't creat spdx. + if bb.data.inherits_class('nopackages', d) and not pn.startswith('gcc-source'): return logger = logging.getLogger() @@ -169,7 +173,7 @@ do_synopsys_detect () { echo "Upload OSS to blackduck server." } addtask do_spdx_creat_tarball after do_patch -addtask do_bd_upload after do_patch +addtask do_bd_upload after do_spdx_creat_tarball addtask do_synopsys_detect do_build[recrdeptask] += "do_synopsys_detect" do_populate_sdk[recrdeptask] += "do_synopsys_detect" diff --git a/classes/fossology-python.bbclass b/classes/fossology-python.bbclass index 4e4a2e6..72b6c22 100644 --- a/classes/fossology-python.bbclass +++ b/classes/fossology-python.bbclass @@ -37,11 +37,11 @@ python () { create_folder_lock = Lock() + pn = d.getVar('PN') #If not for target, won't creat spdx. - if bb.data.inherits_class('nopackages', d): + if bb.data.inherits_class('nopackages', d) and not pn.startswith('gcc-source'): return - pn = d.getVar('PN') assume_provided = (d.getVar("ASSUME_PROVIDED") or "").split() if pn in assume_provided: for p in d.getVar("PROVIDES").split(): @@ -61,8 +61,9 @@ python () { return # We just archive gcc-source for all the gcc related recipes - if d.getVar('BPN') in ['gcc', 'libgcc']: - bb.debug(1, 'spdx: There is bug in scan of %s is, do nothing' % pn) + if d.getVar('BPN') in ['gcc', 'libgcc'] \ + and not pn.startswith('gcc-source'): + bb.debug(1, 'archiver: %s is excluded, covered by gcc-source' % pn) return spdx_outdir = d.getVar('SPDX_OUTDIR') @@ -70,13 +71,15 @@ python () { info = {} info['pn'] = (d.getVar( 'PN') or "") info['pv'] = (d.getVar( 'PV') or "") + info['pr'] = (d.getVar( 'PR') or "") + spdx_name = info['pn'] + "-" + info['pv'] + "-" + info['pr'] + ".spdx" manifest_dir = (d.getVar('SPDX_DEPLOY_DIR') or "") if not os.path.exists( manifest_dir ): bb.utils.mkdirhier( manifest_dir ) - info['outfile'] = os.path.join(manifest_dir, info['pn'] + "-" + info['pv'] + ".spdx" ) - sstatefile = os.path.join(spdx_outdir, info['pn'] + "-" + info['pv'] + ".spdx" ) + info['outfile'] = os.path.join(manifest_dir, spdx_name ) + sstatefile = os.path.join(spdx_outdir, spdx_name ) if os.path.exists(info['outfile']): bb.note(info['pn'] + "spdx file has been exist, do nothing") return @@ -88,9 +91,9 @@ python () { def hasTask(task): return bool(d.getVarFlag(task, "task", False)) and not bool(d.getVarFlag(task, "noexec", False)) - if d.getVar('PACKAGES'): + if d.getVar('PACKAGES') or pn.startswith('gcc-source'): # Some recipes do not have any packaging tasks - if hasTask("do_package_write_rpm") or hasTask("do_package_write_ipk") or hasTask("do_package_write_deb"): + if hasTask("do_package_write_rpm") or hasTask("do_package_write_ipk") or hasTask("do_package_write_deb") or pn.startswith('gcc-source'): d.appendVarFlag('do_foss_upload', 'depends', ' %s:do_spdx_creat_tarball' % pn) d.appendVarFlag('do_schedule_jobs', 'depends', ' %s:do_foss_upload' % pn) d.appendVarFlag('do_get_report', 'depends', ' %s:do_schedule_jobs' % pn) @@ -198,8 +201,7 @@ def create_folder(d, foss, token, folder_name): create_folder_lock.release() if folder.name != folder_name: bb.error("Folder %s couldn't be created" % folder_name) - else: - return folder + return folder python do_schedule_jobs(){ import os @@ -207,8 +209,9 @@ python do_schedule_jobs(){ import time import logging + pn = d.getVar( 'PN') #If not for target, won't creat spdx. - if bb.data.inherits_class('nopackages', d): + if bb.data.inherits_class('nopackages', d) and not pn.startswith('gcc-source'): return logger = logging.getLogger() @@ -226,14 +229,17 @@ python do_schedule_jobs(){ info['workdir'] = (d.getVar('WORKDIR') or "") info['pn'] = (d.getVar( 'PN') or "") info['pv'] = (d.getVar( 'PV') or "") + info['pr'] = (d.getVar( 'PR') or "") + spdx_name = info['pn'] + "-" + info['pv'] + "-" + info['pr'] + ".spdx" + manifest_dir = (d.getVar('SPDX_DEPLOY_DIR') or "") if not os.path.exists( manifest_dir ): bb.utils.mkdirhier( manifest_dir ) spdx_outdir = d.getVar('SPDX_OUTDIR') - info['outfile'] = os.path.join(manifest_dir, info['pn'] + "-" + info['pv'] + ".spdx" ) - sstatefile = os.path.join(spdx_outdir, info['pn'] + "-" + info['pv'] + ".spdx" ) + info['outfile'] = os.path.join(manifest_dir, spdx_name ) + sstatefile = os.path.join(spdx_outdir, spdx_name) if os.path.exists(info['outfile']): bb.note(info['pn'] + "spdx file has been exist, do nothing") return @@ -245,7 +251,6 @@ python do_schedule_jobs(){ fossology_server = d.getVar('FOSSOLOGY_SERVER') token = d.getVar('TOKEN') foss = Fossology(fossology_server, token, "fossy") - pn = d.getVar('PN') if d.getVar('FOLDER_NAME', False): folder_name = d.getVar('FOLDER_NAME') @@ -349,8 +354,9 @@ python do_get_report(){ report_id = None report = None + pn = d.getVar('PN') #If not for target, won't creat spdx. - if bb.data.inherits_class('nopackages', d): + if bb.data.inherits_class('nopackages', d) and not pn.startswith('gcc-source'): return logger = logging.getLogger() @@ -362,7 +368,6 @@ python do_get_report(){ fossology_server = d.getVar('FOSSOLOGY_SERVER') token = d.getVar('TOKEN') foss = Fossology(fossology_server, token, "fossy") - pn = d.getVar('PN') if d.getVar('FOLDER_NAME', False): folder_name = d.getVar('FOLDER_NAME') @@ -383,12 +388,15 @@ python do_get_report(){ info['workdir'] = (d.getVar('WORKDIR') or "") info['pn'] = (d.getVar( 'PN') or "") info['pv'] = (d.getVar( 'PV') or "") + info['pr'] = (d.getVar( 'PR') or "") + spdx_name = info['pn'] + "-" + info['pv'] + "-" + info['pr'] + ".spdx" + info['package_download_location'] = (d.getVar( 'SRC_URI') or "") if info['package_download_location'] != "": info['package_download_location'] = info['package_download_location'].split()[0] info['spdx_version'] = (d.getVar('SPDX_VERSION') or '') - info['outfile'] = os.path.join(manifest_dir, info['pn'] + "-" + info['pv'] + ".spdx" ) - spdx_file = os.path.join(spdx_outdir, info['pn'] + "-" + info['pv'] + ".spdx" ) + info['outfile'] = os.path.join(manifest_dir, spdx_name ) + spdx_file = os.path.join(spdx_outdir, spdx_name ) if os.path.exists(info['outfile']): bb.note(info['pn'] + "spdx file has been exist, do nothing") return @@ -434,7 +442,7 @@ python do_get_report(){ while i < 20: i += 1 try: - report = foss.download_report(report_id) + report, name = foss.download_report(report_id, wait_time=wait_time*2) except TryAgain: bb.warn("SPDX file is still not ready, try again.") time.sleep(wait_time) @@ -446,10 +454,12 @@ python do_get_report(){ bb.error("Fail to download report.") break - report = str(report).lstrip("('") - report = report.rstrip("')") - with open(spdx_file, "w+") as file: - file.write(report) + with open(spdx_file, "wb") as file: + written = file.write(report) + assert written == len(report) + logger.info( + f"Report written to file: report_name {name} written to {spdx_file}" + ) file.close() subprocess.call(r"sed -i -e 's#\\n#\n#g' %s" % spdx_file, shell=True) diff --git a/classes/spdx-common.bbclass b/classes/spdx-common.bbclass index cb97c5b..a845f7f 100644 --- a/classes/spdx-common.bbclass +++ b/classes/spdx-common.bbclass @@ -31,9 +31,15 @@ def excluded_package(d, pn): if p != pn: pn = p break - if d.getVar('BPN') in ['gcc', 'libgcc']: - #bb.debug(1, 'spdx: There is a bug in the scan of %s, skip it.' % pn) + # We just archive gcc-source for all the gcc related recipes + if d.getVar('BPN') in ['gcc', 'libgcc'] \ + and not pn.startswith('gcc-source'): + bb.debug(1, 'archiver: %s is excluded, covered by gcc-source' % pn) return True + # TARGET_SYS in ARCHIVER_ARCH will break the stamp for gcc-source in multiconfig + if pn.startswith('gcc-source'): + d.setVar('ARCHIVER_ARCH', "allarch") + # The following: do_fetch, do_unpack and do_patch tasks have been deleted, # so avoid archiving do_spdx here. # -native is for the host aka during the build @@ -65,12 +71,6 @@ def get_tar_name(d, suffix): get the name of tarball """ - # Make sure we are only creating a single tarball for gcc sources - #if (d.getVar('SRC_URI') == ""): - # return - # For the kernel archive, srcdir may just be a link to the - # work-shared location. Use os.path.realpath to make sure - # that we archive the actual directory and not just the link. if suffix: filename = '%s-%s.tar.gz' % (d.getVar('PF'), suffix) else: @@ -84,12 +84,6 @@ def spdx_create_tarball(d, srcdir, suffix, ar_outdir): """ import tarfile, shutil - # Make sure we are only creating a single tarball for gcc sources - #if (d.getVar('SRC_URI') == ""): - # return - # For the kernel archive, srcdir may just be a link to the - # work-shared location. Use os.path.realpath to make sure - # that we archive the actual directory and not just the link. srcdir = os.path.realpath(srcdir) bb.utils.mkdirhier(ar_outdir) @@ -142,6 +136,11 @@ def spdx_get_src(d): bb.utils.mkdirhier(src_dir) if bb.data.inherits_class('kernel',d): share_src = d.getVar('STAGING_KERNEL_DIR') + if pn.startswith('gcc-source'): + gcc_source_path = d.getVar('TMPDIR') + "/work-shared" + gcc_pv = d.getVar('PV') + gcc_pr = d.getVar('PR') + share_src = gcc_source_path + "/gcc-" + gcc_pv + "-" + gcc_pr + "/gcc-" + gcc_pv + "/" cmd_copy_share = "cp -rf " + share_src + "/* " + src_dir + "/" cmd_copy_kernel_result = os.popen(cmd_copy_share).read() bb.note("cmd_copy_kernel_result = " + cmd_copy_kernel_result) |