diff options
Diffstat (limited to 'classes/fossology-python.bbclass')
| -rw-r--r-- | classes/fossology-python.bbclass | 127 |
1 files changed, 92 insertions, 35 deletions
diff --git a/classes/fossology-python.bbclass b/classes/fossology-python.bbclass index 4e4a2e6..524c5f6 100644 --- a/classes/fossology-python.bbclass +++ b/classes/fossology-python.bbclass @@ -37,11 +37,11 @@ python () { create_folder_lock = Lock() + pn = d.getVar('PN') #If not for target, won't creat spdx. - if bb.data.inherits_class('nopackages', d): + if bb.data.inherits_class('nopackages', d) and not pn.startswith('gcc-source'): return - pn = d.getVar('PN') assume_provided = (d.getVar("ASSUME_PROVIDED") or "").split() if pn in assume_provided: for p in d.getVar("PROVIDES").split(): @@ -61,22 +61,43 @@ python () { return # We just archive gcc-source for all the gcc related recipes - if d.getVar('BPN') in ['gcc', 'libgcc']: - bb.debug(1, 'spdx: There is bug in scan of %s is, do nothing' % pn) + if d.getVar('BPN') in ['gcc', 'libgcc'] \ + and not pn.startswith('gcc-source'): + bb.debug(1, 'archiver: %s is excluded, covered by gcc-source' % pn) return - spdx_outdir = d.getVar('SPDX_OUTDIR') - - info = {} - info['pn'] = (d.getVar( 'PN') or "") - info['pv'] = (d.getVar( 'PV') or "") + def hasTask(task): + return bool(d.getVarFlag(task, "task", False)) and not bool(d.getVarFlag(task, "noexec", False)) manifest_dir = (d.getVar('SPDX_DEPLOY_DIR') or "") if not os.path.exists( manifest_dir ): bb.utils.mkdirhier( manifest_dir ) - info['outfile'] = os.path.join(manifest_dir, info['pn'] + "-" + info['pv'] + ".spdx" ) - sstatefile = os.path.join(spdx_outdir, info['pn'] + "-" + info['pv'] + ".spdx" ) + info = {} + info['pn'] = (d.getVar( 'PN') or "") + info['pv'] = (d.getVar( 'PKGV') or "").replace('-', '+') + info['pr'] = (d.getVar( 'PR') or "") + + if (d.getVar('BPN') == "perf"): + info['pv'] = d.getVar("KERNEL_VERSION").split("-")[0] + + if 'AUTOINC' in info['pv']: + info['pv'] = info['pv'].replace("AUTOINC", "0") + + if d.getVar('SAVE_SPDX_ACHIVE'): + if d.getVar('PACKAGES') or pn.startswith('gcc-source'): + # Some recipes do not have any packaging tasks + if hasTask("do_package_write_rpm") or hasTask("do_package_write_ipk") or hasTask("do_package_write_deb") or pn.startswith('gcc-source'): + d.appendVarFlag('do_spdx', 'depends', ' %s:do_spdx_creat_tarball' % pn) + + spdx_outdir = d.getVar('SPDX_OUTDIR') + if pn.startswith('gcc-source'): + spdx_name = "gcc-" + info['pv'] + "-" + info['pr'] + ".spdx" + else: + spdx_name = info['pn'] + "-" + info['pv'] + "-" + info['pr'] + ".spdx" + + info['outfile'] = os.path.join(manifest_dir, spdx_name ) + sstatefile = os.path.join(spdx_outdir, spdx_name ) if os.path.exists(info['outfile']): bb.note(info['pn'] + "spdx file has been exist, do nothing") return @@ -84,13 +105,10 @@ python () { bb.note(info['pn'] + "spdx file has been exist, do nothing") create_manifest(info,sstatefile) return - - def hasTask(task): - return bool(d.getVarFlag(task, "task", False)) and not bool(d.getVarFlag(task, "noexec", False)) - if d.getVar('PACKAGES'): + if d.getVar('PACKAGES') or pn.startswith('gcc-source'): # Some recipes do not have any packaging tasks - if hasTask("do_package_write_rpm") or hasTask("do_package_write_ipk") or hasTask("do_package_write_deb"): + if hasTask("do_package_write_rpm") or hasTask("do_package_write_ipk") or hasTask("do_package_write_deb") or pn.startswith('gcc-source'): d.appendVarFlag('do_foss_upload', 'depends', ' %s:do_spdx_creat_tarball' % pn) d.appendVarFlag('do_schedule_jobs', 'depends', ' %s:do_foss_upload' % pn) d.appendVarFlag('do_get_report', 'depends', ' %s:do_schedule_jobs' % pn) @@ -137,8 +155,18 @@ python do_foss_upload(){ bb.warn(pn + " has already been uploaded, don't upload again.") } def get_upload(d, folder, foss): + from fossology.exceptions import FossologyApiError + filename = get_upload_file_name(d) - upload_list, _ = foss.list_uploads(page_size=1, all_pages=True) + try: + upload_list, _ = foss.list_uploads(page_size=1, all_pages=True) + except FossologyApiError as error: + time.sleep(10) + try: + upload_list, _ = foss.list_uploads(page_size=1, all_pages=True) + except FossologyApiError as error: + bb.error(error.message) + upload = None bb.note("Check tarball: %s ,has been uploaded?" % filename) for upload in upload_list: @@ -198,8 +226,7 @@ def create_folder(d, foss, token, folder_name): create_folder_lock.release() if folder.name != folder_name: bb.error("Folder %s couldn't be created" % folder_name) - else: - return folder + return folder python do_schedule_jobs(){ import os @@ -207,8 +234,9 @@ python do_schedule_jobs(){ import time import logging + pn = d.getVar( 'PN') #If not for target, won't creat spdx. - if bb.data.inherits_class('nopackages', d): + if bb.data.inherits_class('nopackages', d) and not pn.startswith('gcc-source'): return logger = logging.getLogger() @@ -225,15 +253,25 @@ python do_schedule_jobs(){ info = {} info['workdir'] = (d.getVar('WORKDIR') or "") info['pn'] = (d.getVar( 'PN') or "") - info['pv'] = (d.getVar( 'PV') or "") + info['pv'] = (d.getVar( 'PKGV') or "").replace('-', '+') + info['pr'] = (d.getVar( 'PR') or "") + if (d.getVar('BPN') == "perf"): + info['pv'] = d.getVar("KERNEL_VERSION").split("-")[0] + if 'AUTOINC' in info['pv']: + info['pv'] = info['pv'].replace("AUTOINC", "0") + + if pn.startswith('gcc-source'): + spdx_name = "gcc-" + info['pv'] + "-" + info['pr'] + ".spdx" + else: + spdx_name = info['pn'] + "-" + info['pv'] + "-" + info['pr'] + ".spdx" manifest_dir = (d.getVar('SPDX_DEPLOY_DIR') or "") if not os.path.exists( manifest_dir ): bb.utils.mkdirhier( manifest_dir ) spdx_outdir = d.getVar('SPDX_OUTDIR') - info['outfile'] = os.path.join(manifest_dir, info['pn'] + "-" + info['pv'] + ".spdx" ) - sstatefile = os.path.join(spdx_outdir, info['pn'] + "-" + info['pv'] + ".spdx" ) + info['outfile'] = os.path.join(manifest_dir, spdx_name ) + sstatefile = os.path.join(spdx_outdir, spdx_name) if os.path.exists(info['outfile']): bb.note(info['pn'] + "spdx file has been exist, do nothing") return @@ -245,7 +283,6 @@ python do_schedule_jobs(){ fossology_server = d.getVar('FOSSOLOGY_SERVER') token = d.getVar('TOKEN') foss = Fossology(fossology_server, token, "fossy") - pn = d.getVar('PN') if d.getVar('FOLDER_NAME', False): folder_name = d.getVar('FOLDER_NAME') @@ -349,8 +386,9 @@ python do_get_report(){ report_id = None report = None + pn = d.getVar('PN') #If not for target, won't creat spdx. - if bb.data.inherits_class('nopackages', d): + if bb.data.inherits_class('nopackages', d) and not pn.startswith('gcc-source'): return logger = logging.getLogger() @@ -362,7 +400,6 @@ python do_get_report(){ fossology_server = d.getVar('FOSSOLOGY_SERVER') token = d.getVar('TOKEN') foss = Fossology(fossology_server, token, "fossy") - pn = d.getVar('PN') if d.getVar('FOLDER_NAME', False): folder_name = d.getVar('FOLDER_NAME') @@ -380,15 +417,27 @@ python do_get_report(){ cur_ver_code = get_ver_code(spdx_workdir).split()[0] info = {} + info['workdir'] = (d.getVar('WORKDIR') or "") info['pn'] = (d.getVar( 'PN') or "") - info['pv'] = (d.getVar( 'PV') or "") + info['pv'] = (d.getVar( 'PKGV') or "").replace('-', '+') + info['pr'] = (d.getVar( 'PR') or "") + if (d.getVar('BPN') == "perf"): + info['pv'] = d.getVar("KERNEL_VERSION").split("-")[0] + if 'AUTOINC' in info['pv']: + info['pv'] = info['pv'].replace("AUTOINC", "0") + + if pn.startswith('gcc-source'): + spdx_name = "gcc-" + info['pv'] + "-" + info['pr'] + ".spdx" + else: + spdx_name = info['pn'] + "-" + info['pv'] + "-" + info['pr'] + ".spdx" + info['package_download_location'] = (d.getVar( 'SRC_URI') or "") if info['package_download_location'] != "": info['package_download_location'] = info['package_download_location'].split()[0] info['spdx_version'] = (d.getVar('SPDX_VERSION') or '') - info['outfile'] = os.path.join(manifest_dir, info['pn'] + "-" + info['pv'] + ".spdx" ) - spdx_file = os.path.join(spdx_outdir, info['pn'] + "-" + info['pv'] + ".spdx" ) + info['outfile'] = os.path.join(manifest_dir, spdx_name ) + spdx_file = os.path.join(spdx_outdir, spdx_name ) if os.path.exists(info['outfile']): bb.note(info['pn'] + "spdx file has been exist, do nothing") return @@ -407,6 +456,12 @@ python do_get_report(){ info['package_contains'] = (d.getVar('CONTAINED') or "") info['package_static_link'] = (d.getVar('STATIC_LINK') or "") info['modified'] = "false" + info['external_refs'] = get_external_refs(d) + info['purpose'] = get_pkgpurpose(d) + info['release_date'] = (d.getVar('REALASE_DATE') or "") + info['build_time'] = get_build_date(d) + info['depends_on'] = get_depends_on(d) + info['pkg_spdx_id'] = get_spdxid_pkg(d) srcuri = d.getVar("SRC_URI", False).split() length = len("file://") for item in srcuri: @@ -434,7 +489,7 @@ python do_get_report(){ while i < 20: i += 1 try: - report = foss.download_report(report_id) + report, name = foss.download_report(report_id, wait_time=wait_time*2) except TryAgain: bb.warn("SPDX file is still not ready, try again.") time.sleep(wait_time) @@ -446,10 +501,12 @@ python do_get_report(){ bb.error("Fail to download report.") break - report = str(report).lstrip("('") - report = report.rstrip("')") - with open(spdx_file, "w+") as file: - file.write(report) + with open(spdx_file, "wb") as file: + written = file.write(report) + assert written == len(report) + logger.info( + f"Report written to file: report_name {name} written to {spdx_file}" + ) file.close() subprocess.call(r"sed -i -e 's#\\n#\n#g' %s" % spdx_file, shell=True) @@ -496,4 +553,4 @@ addtask do_get_report after do_schedule_jobs addtask do_spdx do_build[recrdeptask] += "do_spdx" do_populate_sdk[recrdeptask] += "do_spdx" - +do_get_report[depends] = "cve-update-nvd2-native:do_fetch" |