aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJan-Simon Moeller <jsmoeller@linuxfoundation.org>2020-04-01 14:49:35 +0200
committerJan-Simon Moeller <jsmoeller@linuxfoundation.org>2020-04-07 21:26:27 +0200
commit8f2f26bcd6f8b21659265080751618ed39a9ae6a (patch)
tree5d6b5a498c7b174bc156c0ce4cc063703439fe45
parente302035685bcf26b0fdeb056eeb3600785a66362 (diff)
downloadmeta-spdxscanner-devel/jsmoeller/phased.tar.gz
meta-spdxscanner-devel/jsmoeller/phased.tar.bz2
meta-spdxscanner-devel/jsmoeller/phased.zip
Add WIP phased variant of fossology uploaderdevel/jsmoeller/phased
This is unfinished but lets you upload with --runall=spdxupload and trigger the analyse run with --runall=spdxanalyse and get the report with --runall=spdxreport Or run all of above using --runall=spdx Signed-off-by: Jan-Simon Moeller <jsmoeller@linuxfoundation.org>
-rw-r--r--classes/fossology-rest-phased.bbclass661
-rw-r--r--classes/spdx-common.bbclass41
2 files changed, 700 insertions, 2 deletions
diff --git a/classes/fossology-rest-phased.bbclass b/classes/fossology-rest-phased.bbclass
new file mode 100644
index 0000000..89b34c7
--- /dev/null
+++ b/classes/fossology-rest-phased.bbclass
@@ -0,0 +1,661 @@
+# This class integrates real-time license scanning, generation of SPDX standard
+# output and verifiying license info during the building process.
+# It is a combination of efforts from the OE-Core, SPDX and DoSOCSv2 projects.
+#
+# For more information on DoSOCSv2:
+# https://github.com/DoSOCSv2
+#
+# For more information on SPDX:
+# http://www.spdx.org
+#
+# Note:
+# 1) Make sure fossdriver has beed installed in your host
+# 2) By default,spdx files will be output to the path which is defined as[SPDX_DEPLOY_DIR]
+# in ./meta/conf/spdx-dosocs.conf.
+inherit spdx-common
+FOSSOLOGY_SERVER ?= "http://127.0.0.1:8081/repo"
+FOSSOLOGY_REUPLOAD ??= "0"
+
+#upload OSS into No.1 folder of fossology
+FOSSOLOGY_FOLDER_ID ??= "1"
+FOSSOLOGY_FOLDER_NAME ??= "${DISTRO_CODENAME}-${DISTRO_VERSION}"
+
+FOSSOLOGY_EXCLUDE_PACKAGES ??= "glibc-locale libtool-cross libgcc-initial shadow-sysroot"
+FOSSOLOGY_EXCLUDE_NATIVE ??= "1"
+FOSSOLOGY_EXCLUDE_SDK ??= "1"
+# translate to common variables
+SPDX_EXCLUDE_PACKAGES := "${FOSSOLOGY_EXCLUDE_PACKAGES}"
+SPDX_EXCLUDE_NATIVE := "${FOSSOLOGY_EXCLUDE_NATIVE}"
+SPDX_EXCLUDE_SDK := "${FOSSOLOGY_EXCLUDE_SDK}"
+
+
+HOSTTOOLS_NONFATAL += "curl quilt"
+
+CREATOR_TOOL = "fossology-rest-phased.bbclass in meta-spdxscanner"
+
+# If ${S} isn't actually the top-level source directory, set SPDX_S to point at
+# the real top-level directory.
+SPDX_S ?= "${S}"
+
+def populate_info(d, info):
+ info['workdir'] = (d.getVar('WORKDIR', True) or "")
+ info['pn'] = (d.getVar( 'PN', True ) or "")
+ info['pv'] = (d.getVar( 'PV', True ) or "")
+ info['package_download_location'] = (d.getVar( 'SRC_URI', True ) or "")
+ if info['package_download_location'] != "":
+ info['package_download_location'] = info['package_download_location'].split()[0]
+ info['spdx_version'] = (d.getVar('SPDX_VERSION', True) or '')
+ info['data_license'] = (d.getVar('DATA_LICENSE', True) or '')
+ info['creator'] = {}
+ info['creator']['Tool'] = (d.getVar('CREATOR_TOOL', True) or '')
+ info['license_list_version'] = (d.getVar('LICENSELISTVERSION', True) or '')
+ info['package_homepage'] = (d.getVar('HOMEPAGE', True) or "")
+ info['package_summary'] = (d.getVar('SUMMARY', True) or "")
+ info['package_summary'] = info['package_summary'].replace("\n","")
+ info['package_summary'] = info['package_summary'].replace("'"," ")
+ info['package_contains'] = (d.getVar('CONTAINED', True) or "")
+ info['package_static_link'] = (d.getVar('STATIC_LINK', True) or "")
+ info['modified'] = "false"
+ info['token'] = (d.getVar('FOSSOLOGY_TOKEN', True) or "")
+ info['manifest_dir'] = (d.getVar('SPDX_DEPLOY_DIR', True) or "")
+ info['srcuri'] = d.getVar("SRC_URI", False).split()
+ length = len("file://")
+ for item in info['srcuri']:
+ if item.startswith("file://"):
+ item = item[length:]
+ if item.endswith(".patch") or item.endswith(".diff"):
+ info['modified'] = "true"
+ info['spdx_outdir'] = d.getVar('SPDX_OUTDIR')
+ info['spdx_workdir'] = d.getVar('SPDX_WORKDIR')
+ info['spdx_temp_dir'] = os.path.join(info['spdx_workdir'], "temp")
+ info['temp_dir'] = os.path.join(d.getVar('WORKDIR'), "temp")
+ info['outfile'] = os.path.join(info['manifest_dir'], info['pn'] + "-" + info['pv'] + ".spdx" )
+ info['sstatefile'] = os.path.join(info['spdx_outdir'], info['pn'] + "-" + info['pv'] + ".spdx" )
+
+def prepare_folders(info):
+ if not os.path.exists( info['manifest_dir'] ):
+ bb.utils.mkdirhier( info['manifest_dir'] )
+ if not os.path.exists(info['spdx_outdir']):
+ bb.utils.mkdirhier(info['spdx_outdir'])
+
+def use_cached_files(info):
+ # if spdx sstate or report already exist
+ if os.path.exists(info['outfile']):
+ bb.warn(info['pn'] + "The spdx report file already exists, do nothing.")
+ return
+ if os.path.exists( info['sstatefile'] ):
+ bb.warn(info['pn'] + "The spdx sstate file already exists, do nothing.")
+ create_manifest(info, info['sstatefile'])
+ return
+
+def fossologyupload(d, bb, info):
+ import os, sys, shutil
+
+ pn = d.getVar('PN', True)
+ # exclude packages not necessary (-native, nativesdk-) or on blacklist
+ if excluded_package(d, pn):
+ bb.note("spdx: fossologyanalyze: excluding "+ pn)
+ return
+
+ # retrieve the folder id
+ info['folder_id'] = get_folder_id(d)
+
+ # get source
+ spdx_get_src(d)
+
+ bb.note('spdx: Archiving the patched source...')
+ if os.path.isdir(info['spdx_temp_dir']):
+ for f_dir, f in list_files(info['spdx_temp_dir']):
+ temp_file = os.path.join(info['spdx_temp_dir'],f_dir,f)
+ shutil.copy(temp_file, info['temp_dir'])
+ shutil.rmtree(info['spdx_temp_dir'])
+ d.setVar('WORKDIR', info['spdx_workdir'])
+ info['sourcedir'] = info['spdx_workdir']
+ git_path = "%s/git/.git" % info['sourcedir']
+ if os.path.exists(git_path):
+ remove_dir_tree(git_path)
+
+ tar_name = spdx_create_tarball(d, d.getVar('WORKDIR'), 'patched', info['spdx_outdir'])
+
+ ## get everything from cache. use it to decide if
+ ## something needs to be rerun
+ if not os.path.exists(info['spdx_outdir']):
+ bb.utils.mkdirhier(info['spdx_outdir'])
+ cur_ver_code = get_ver_code(info['spdx_workdir']).split()[0]
+ # upload archive
+ if invoke_rest_api_upload(d, tar_name, info['outfile'], info['folder_id']) == False:
+ bb.error(info['pn'] + ": Upload failed, please check fossology server.")
+ return
+
+
+def fossologyanalyse(d, bb, info):
+ import os, sys, shutil
+
+ pn = d.getVar('PN', True)
+ # exclude packages not necessary (-native, nativesdk-) or on blacklist
+ if excluded_package(d, pn):
+ bb.note("spdx: fossologyanalyze: excluding "+ pn)
+ return
+
+
+ # retrieve the folder id
+ info['folder_id'] = get_folder_id(d)
+
+
+ # TODO: put into common func or info[]
+ filename = '%s-%s.tar.gz' % (d.getVar('PF'), 'patched')
+ tar_name = os.path.join(info['spdx_outdir'], filename)
+
+ # invoce the analysis
+ if invoke_rest_api_analysis(d, tar_name, info['sstatefile'], info['folder_id']) == False:
+ bb.error("spdx: " + info['pn'] + ": Analysis trigger failed, please check fossology server.")
+ return
+
+
+def fossologytrigger(d, bb, info):
+ import os, sys, shutil
+
+ pn = d.getVar('PN', True)
+ # exclude packages not necessary (-native, nativesdk-) or on blacklist
+ if excluded_package(d, pn):
+ bb.note("spdx: fossologyanalyze: excluding "+ pn)
+ return
+
+ # setup variables
+ info = {}
+ populate_info(d, info)
+
+ # retrieve the folder id
+ info['folder_id'] = get_folder_id(d)
+
+ # TODO: put into common func or info[]
+ filename = '%s-%s.tar.gz' % (d.getVar('PF'), 'patched')
+ tar_name = os.path.join(info['spdx_outdir'], filename)
+
+ report_id = invoke_rest_api_triggerreport(d, tar_name, info['outfile'], info['folder_id'])
+ if report_id == False:
+ bb.error("spdx:" + info['pn'] + ": trigger_spdx failed.")
+ return False
+ else:
+ info['report_id'] = report_id
+ d.setVar('spdx_report_id', info['report_id'])
+ return True
+
+
+def fossologyreport(d, bb, info):
+ import os, sys, shutil
+
+ pn = d.getVar('PN', True)
+ # exclude packages not necessary (-native, nativesdk-) or on blacklist
+ if excluded_package(d, pn):
+ bb.note("spdx: fossologyanalyze: excluding "+ pn)
+ return
+
+ if not info['folder_id']:
+ # retrieve the folder id
+ info['folder_id'] = get_folder_id(d)
+ if not 'report_id' in info:
+ x = d.getVar("spdx_report_id")
+ if not x:
+ bb.error("spdx: no report_id")
+ return False
+ info['report_id'] = x
+
+ # TODO: put into common func or info[]
+ filename = '%s-%s.tar.gz' % (d.getVar('PF'), 'patched')
+ tar_name = os.path.join(info['spdx_outdir'], filename)
+
+ bb.warn("outfile: " + str(info['outfile']))
+
+ if invoke_rest_api_getresult(d, tar_name, info['outfile'], info['folder_id'], info['report_id']) == False:
+ bb.error("spdx: " + info['pn'] + ": get_spdx failed.")
+ return
+
+
+def fossologywaitextracted(d, bb, info):
+ import time
+ delaytime = 120
+ i = 0
+
+ pn = d.getVar('PN', True)
+ # exclude packages not necessary (-native, nativesdk-) or on blacklist
+ if excluded_package(d, pn):
+ bb.note("spdx: fossologyanalyze: excluding "+ pn)
+ return
+
+ # setup variables
+ info = {}
+ populate_info(d, info)
+
+ # retrieve the folder id
+ info['folder_id'] = get_folder_id(d)
+
+ # TODO: put into common func or info[]
+ filename = '%s-%s.tar.gz' % (d.getVar('PF'), 'patched')
+ tar_name = os.path.join(info['spdx_outdir'], filename)
+
+ time.sleep(10)
+ while i < 24:
+ upload_id = has_upload(d, tar_name, info['folder_id'])
+ bb.note("spdx: upload_id :" + str(upload_id))
+ if upload_id:
+ return
+ i += 1
+ time.sleep(delaytime)
+
+
+addtask do_spdxupload
+python do_spdxupload() {
+ # setup variables
+ info = {}
+ populate_info(d, info)
+
+ fossologyupload(d, bb, info)
+}
+
+addtask do_spdxanalyse
+python do_spdxanalyse() {
+ # setup variables
+ info = {}
+ populate_info(d, info)
+ fossologywaitextracted(d, bb, info)
+ fossologyanalyse(d, bb, info)
+}
+
+addtask do_spdxreport
+python do_spdxreport(){
+ import time
+ # setup variables
+ info = {}
+ populate_info(d, info)
+
+ if fossologytrigger(d, bb, info):
+ fossologyreport(d, bb, info)
+}
+
+python do_spdx() {
+
+ pn = d.getVar('PN')
+ info = {}
+ populate_info(d, info)
+
+ fossologyupload(d, bb, info)
+
+ fossologywaitextracted(d, bb, info)
+
+ fossologyanalyse(d, bb, info)
+
+ if fossologytrigger(d, bb, info):
+ fossologyreport(d, bb, info)
+
+}
+
+
+def get_folder_id_by_name(d, folder_name):
+ import os
+ import subprocess
+ import json
+
+ server_url = (d.getVar('FOSSOLOGY_SERVER', True) or "")
+ if server_url == "":
+ bb.note("Please set fossology server URL by setting FOSSOLOGY_SERVER!\n")
+ raise OSError(errno.ENOENT, "No setting of FOSSOLOGY_SERVER")
+
+ token = (d.getVar('FOSSOLOGY_TOKEN', True) or "")
+ if token == "":
+ bb.note("Please set token of fossology server by setting FOSSOLOGY_TOKEN!\n")
+ raise OSError(errno.ENOENT, "No setting of FOSSOLOGY_TOKEN comes from fossology server.")
+
+ rest_api_cmd = "curl -k -s -S -X GET " + server_url + "/api/v1/folders" \
+ + " -H \"Authorization: Bearer " + token + "\"" \
+ + " --noproxy 127.0.0.1"
+ bb.note("Invoke rest_api_cmd = " + rest_api_cmd )
+ try:
+ all_folder = subprocess.check_output(rest_api_cmd, stderr=subprocess.STDOUT, shell=True)
+ except subprocess.CalledProcessError as e:
+ bb.error(d.getVar('PN', True) + ": Get folder list failed: \n%s" % e.output.decode("utf-8"))
+ return False
+ all_folder = str(all_folder, encoding = "utf-8")
+ all_folder = json.loads(all_folder)
+ if len(all_folder) == 0:
+ bb.note("Can not get folder list.")
+ return False
+ bb.note("all_folder[0][name] = " + all_folder[0]["name"])
+ for i in range(0, len(all_folder)):
+ if all_folder[i]["name"] == folder_name:
+ bb.note("Found " + folder_name + "on fossology server.")
+ return all_folder[i]["id"]
+ return False
+
+def create_folder(d, folder_name):
+ import os
+ import subprocess
+
+ server_url = (d.getVar('FOSSOLOGY_SERVER', True) or "")
+ if server_url == "":
+ bb.note("Please set fossology server URL by setting FOSSOLOGY_SERVER!\n")
+ raise OSError(errno.ENOENT, "No setting of FOSSOLOGY_SERVER")
+
+ token = (d.getVar('FOSSOLOGY_TOKEN', True) or "")
+ if token == "":
+ bb.note("Please set token of fossology server by setting FOSSOLOGY_TOKEN!\n")
+ raise OSError(errno.ENOENT, "No setting of FOSSOLOGY_TOKEN comes from fossology server.")
+
+ rest_api_cmd = "curl -k -s -S -X POST " + server_url + "/api/v1/folders" \
+ + " -H \'parentFolder: 1\'" \
+ + " -H \'folderName: " + folder_name + "\'" \
+ + " -H \"Authorization: Bearer " + token + "\"" \
+ + " --noproxy 127.0.0.1"
+ bb.note("Invoke rest_api_cmd = " + rest_api_cmd)
+ try:
+ add_folder = subprocess.check_output(rest_api_cmd, stderr=subprocess.STDOUT, shell=True)
+ except subprocess.CalledProcessError as e:
+ bb.error(d.getVar('PN', True) + ": Added folder failed: \n%s" % e.output.decode("utf-8"))
+ return False
+
+ add_folder = str(add_folder, encoding = "utf-8")
+ add_folder = eval(add_folder)
+ if str(add_folder["code"]) == "201":
+ bb.note("add_folder = " + folder_name)
+ return add_folder["message"]
+ elif str(add_folder["code"]) == "200":
+ bb.note("Folder : " + folder_name + "has been created.")
+ return get_folder_id_by_name(d, folder_name)
+ else:
+ bb.error(d.getVar('PN', True) + ": Added folder failed, please check your fossology server.")
+ return False
+
+def get_folder_id(d):
+ if d.getVar('FOSSOLOGY_FOLDER_NAME', False):
+ folder_name = d.getVar('FOSSOLOGY_FOLDER_NAME')
+ folder_id = create_folder(d, folder_name)
+ else:
+ folder_id = (d.getVar('FOSSOLOGY_FOLDER_ID', True) or "1")
+ bb.note("Folder Id = " + str(folder_id))
+ return str(folder_id)
+
+def has_upload(d, tar_file, folder_id):
+ import os
+ import subprocess
+
+ (work_dir, file_name) = os.path.split(tar_file)
+
+ server_url = (d.getVar('FOSSOLOGY_SERVER', True) or "")
+ if server_url == "":
+ bb.note("Please set fossology server URL by setting FOSSOLOGY_SERVER!\n")
+ raise OSError(errno.ENOENT, "No setting of FOSSOLOGY_SERVER")
+
+ token = (d.getVar('FOSSOLOGY_TOKEN', True) or "")
+ if token == "":
+ bb.note("Please set token of fossology server by setting FOSSOLOGY_TOKEN!\n")
+ raise OSError(errno.ENOENT, "No setting of TOKEN comes from fossology server.")
+
+ rest_api_cmd = "curl -k -s -S -X GET " + server_url + "/api/v1/uploads" \
+ + " -H \"Authorization: Bearer " + token + "\"" \
+ + " --noproxy 127.0.0.1"
+ bb.note("Invoke rest_api_cmd = " + rest_api_cmd )
+ try:
+ upload_output = subprocess.check_output(rest_api_cmd, stderr=subprocess.STDOUT, shell=True)
+ except subprocess.CalledProcessError as e:
+ bb.error("curl failed: \n%s" % e.output.decode("utf-8"))
+ return False
+
+ upload_output = str(upload_output, encoding = "utf-8")
+ upload_output = eval(upload_output)
+ if len(upload_output) == 0:
+ bb.note("The upload of fossology is 0.")
+ return False
+ bb.note("upload_output[0][uploadname] = " + upload_output[0]["uploadname"])
+ for i in range(0, len(upload_output)):
+ if upload_output[i]["uploadname"] == file_name and str(upload_output[i]["folderid"]) == str(folder_id):
+ bb.note("Found " + file_name + " on fossology server \"Software Repository\" folder. Will skip upload.")
+ return upload_output[i]["id"]
+ return False
+
+def upload(d, tar_file, folder):
+ import os
+ import subprocess
+ import time
+ delaytime = 50
+ i = 0
+
+ server_url = (d.getVar('FOSSOLOGY_SERVER', True) or "")
+ if server_url == "":
+ bb.note("Please set fossology server URL by setting FOSSOLOGY_SERVER!\n")
+ raise OSError(errno.ENOENT, "No setting of FOSSOLOGY_SERVER")
+
+ token = (d.getVar('FOSSOLOGY_TOKEN', True) or "")
+ if token == "":
+ bb.note("Please set token of fossology server by setting FOSSOLOGY_TOKEN!\n")
+ raise OSError(errno.ENOENT, "No setting of TOKEN comes from fossology server.")
+
+ rest_api_cmd = "curl -k -s -S -X POST " + server_url + "/api/v1/uploads" \
+ + " -H \"folderId: " + folder + "\"" \
+ + " -H \"Authorization: Bearer " + token + "\"" \
+ + " -H \'uploadDescription: created by REST\'" \
+ + " -H \'public: public\'" \
+ + " -H \'Content-Type: multipart/form-data\'" \
+ + " -F \'fileInput=@\"" + tar_file + "\";type=application/octet-stream\'" \
+ + " --noproxy 127.0.0.1"
+ bb.note("Upload : Invoke rest_api_cmd = " + rest_api_cmd )
+ while i < 10:
+ try:
+ upload = subprocess.check_output(rest_api_cmd, stderr=subprocess.STDOUT, shell=True)
+ except subprocess.CalledProcessError as e:
+ bb.error(d.getVar('PN', True) + ": Upload failed: \n%s" % e.output.decode("utf-8"))
+ return False
+ upload = str(upload, encoding = "utf-8")
+ if not upload:
+ return False
+ if "ERROR" in str(upload):
+ return False
+ upload = eval(upload)
+ if str(upload["code"]) == "201":
+ time.sleep(delaytime)
+ return upload["message"]
+ i += 1
+ time.sleep(delaytime)
+ bb.error("spdx: " + d.getVar('PN', True) + ": Upload failed, please check your fossology server connection.")
+ bb.error(str(upload))
+ return False
+
+def analysis(d, folder_id, upload_id):
+ import os
+ import subprocess
+ delaytime = 50
+ i = 0
+
+ server_url = (d.getVar('FOSSOLOGY_SERVER', True) or "")
+ if server_url == "":
+ bb.note("Please set fossology server URL by setting FOSSOLOGY_SERVER!\n")
+ raise OSError(errno.ENOENT, "No setting of FOSSOLOGY_SERVER")
+
+ token = (d.getVar('FOSSOLOGY_TOKEN', True) or "")
+ if token == "":
+ bb.note("Please set token of fossology server by setting FOSSOLOGY_TOKEN!\n")
+ raise OSError(errno.ENOENT, "No setting of FOSSOLOGY_TOKEN comes from fossology server.")
+
+ rest_api_cmd = "curl -k -s -S -X POST " + server_url + "/api/v1/jobs" \
+ + " -H \"folderId: " + str(folder_id) + "\"" \
+ + " -H \"uploadId: " + str(upload_id) + "\"" \
+ + " -H \"Authorization: Bearer " + token + "\"" \
+ + " -H \'Content-Type: application/json\'" \
+ + " --data \'{\"analysis\": {\"bucket\": true,\"copyright_email_author\": true,\"ecc\": true, \"keyword\": true,\"mime\": true,\"monk\": true,\"nomos\": true,\"package\": true},\"decider\": {\"nomos_monk\": true,\"bulk_reused\": true,\"new_scanner\": true}}\'" \
+ + " --noproxy 127.0.0.1"
+ bb.note("Analysis : Invoke rest_api_cmd = " + rest_api_cmd )
+ try:
+ analysis = subprocess.check_output(rest_api_cmd, stderr=subprocess.STDOUT, shell=True)
+ except subprocess.CalledProcessError as e:
+ bb.error("Analysis failed: \n%s" % e.output.decode("utf-8"))
+ return False
+ analysis = str(analysis, encoding = "utf-8")
+ analysis = eval(analysis)
+ if str(analysis["code"]) == "201":
+ return analysis["message"]
+ elif str(analysis["code"]) == "404":
+ bb.error("spdx: " + d.getVar('PN', True) + ": cannot trigger analysis as extraction failed.")
+ return False
+ else:
+ return False
+
+def trigger(d, folder_id, upload_id):
+ import os
+ import subprocess
+ delaytime = 50
+ i = 0
+
+ server_url = (d.getVar('FOSSOLOGY_SERVER', True) or "")
+ if server_url == "":
+ bb.note("Please set fossology server URL by setting FOSSOLOGY_SERVER!\n")
+ raise OSError(errno.ENOENT, "No setting of FOSSOLOGY_SERVER")
+
+ token = (d.getVar('FOSSOLOGY_TOKEN', True) or "")
+ if token == "":
+ bb.note("Please set token of fossology server by setting FOSSOLOGY_TOKEN!\n")
+ raise OSError(errno.ENOENT, "No setting of FOSSOLOGY_TOKEN comes from fossology server.")
+
+ rest_api_cmd = "curl -k -s -S -X GET " + server_url + "/api/v1/report" \
+ + " -H \"Authorization: Bearer " + token + "\"" \
+ + " -H \"uploadId: " + str(upload_id) + "\"" \
+ + " -H \'reportFormat: spdx2tv\'" \
+ + " --noproxy 127.0.0.1"
+ bb.note("trigger : Invoke rest_api_cmd = " + rest_api_cmd )
+ while i < 10:
+ time.sleep(10)
+ try:
+ trigger = subprocess.check_output(rest_api_cmd, stderr=subprocess.STDOUT, shell=True)
+ except subprocess.CalledProcessError as e:
+ bb.error(d.getVar('PN', True) + ": Trigger failed: \n%s" % e.output.decode("utf-8"))
+ return False
+ trigger = str(trigger, encoding = "utf-8")
+ trigger = eval(trigger)
+ if str(trigger["code"]) == "201":
+ return trigger["message"].split("/")[-1]
+ i += 1
+ time.sleep(delaytime * 2)
+ bb.note("spdx: " + d.getVar('PN', True) + ": Trigger failed, will try again.")
+ bb.note("spdx: " + d.getVar('PN', True) + ": Trigger failed, please check your fossology server.")
+ return False
+
+def get_spdx(d, report_id, spdx_file):
+ import os
+ import subprocess
+ import time
+ delaytime = 10
+ complete = False
+ i = 0
+
+ # Wait 20 seconds for the report to be generated on the server
+ time.sleep(20)
+
+ server_url = (d.getVar('FOSSOLOGY_SERVER', True) or "")
+ if server_url == "":
+ bb.note("Please set fossology server URL by setting FOSSOLOGY_SERVER!\n")
+ raise OSError(errno.ENOENT, "No setting of FOSSOLOGY_SERVER")
+
+ token = (d.getVar('FOSSOLOGY_TOKEN', True) or "")
+ if token == "":
+ bb.note("Please set token of fossology server by setting FOSSOLOGY_TOKEN!\n")
+ raise OSError(errno.ENOENT, "No setting of FOSSOLOGY_TOKEN comes from fossology server.")
+ rest_api_cmd = "curl -k -s -S -X GET " + server_url + "/api/v1/report/" + report_id \
+ + " -H \'accept: text/plain\'" \
+ + " -H \"Authorization: Bearer " + token + "\"" \
+ + " --noproxy 127.0.0.1"
+ bb.note("get_spdx : Invoke rest_api_cmd = " + rest_api_cmd )
+ while i < 24:
+ file = open(spdx_file,'wt')
+ try:
+ p = subprocess.Popen(rest_api_cmd, shell=True, universal_newlines=True, stdout=file).wait()
+ except subprocess.CalledProcessError as e:
+ bb.error("Get spdx failed: \n%s. Please check fossology server." % e.output.decode("utf-8"))
+ file.close()
+ os.remove(spdx_file)
+ return False
+ file.flush()
+ file.close()
+ time.sleep(2)
+ file = open(spdx_file,'r+')
+ first_line = file.readline()
+ bb.note("SPDX_FILE first line: " + str(first_line))
+ if "SPDXVersion" in first_line:
+ complete = True
+ file.close()
+
+ if complete == True:
+ return True
+ else:
+ bb.note("spdx: " + d.getVar('PN', True) + ": Get the first line is " + first_line + ". Will try again.")
+ i += 1
+ time.sleep(delaytime)
+ delaytime = delaytime + 10
+ file.close()
+ bb.error("spdx: " + d.getVar('PN', True) + ": SPDX report could not be downloaded.")
+
+def invoke_rest_api_upload(d, tar_file, spdx_file, folder_id):
+ import os
+ import time
+ i = 0
+ bb.note("invoke fossology REST API : tar_file = %s " % tar_file)
+ upload_id = has_upload(d, tar_file, folder_id)
+ if upload_id == False:
+ bb.note("This OSS has not been scanned. So upload it to fossology server.")
+ upload_id = upload(d, tar_file, folder_id)
+ if upload_id == False:
+ return False
+ return True
+
+def invoke_rest_api_analysis(d, tar_file, spdx_file, folder_id):
+ upload_id = has_upload(d, tar_file, folder_id)
+ if upload_id == False:
+ if d.getVar('FOSSOLOGY_REUPLOAD') == "1":
+ bb.note("Reupload.")
+ upload_id = upload(d, tar_file, folder_id)
+ if upload_id == False:
+ return False
+ else:
+ bb.note("This OSS has not been uploaded. Skip it.")
+ return False
+ if analysis(d, folder_id, upload_id) == False:
+ bb.note("Analysis failed.")
+ return False
+ bb.note("Analysis success.")
+ return True
+
+def invoke_rest_api_triggerreport(d, tar_file, spdx_file, folder_id):
+ import time
+ i = 0
+
+ upload_id = has_upload(d, tar_file, folder_id)
+ if upload_id == False:
+ bb.error("Could not find the file on the fossology server!")
+ return False
+
+ while i < 3:
+ i += 1
+ report_id = trigger(d, folder_id, upload_id)
+ bb.note("Report_id: " + str(report_id))
+ if report_id:
+ return report_id
+ time.sleep(30)
+
+ bb.error("Could not trigger the report generation for " + d.getVar('PN', True))
+ return False
+
+def invoke_rest_api_getresult(d, tar_file, spdx_file, folder_id, report_id):
+ i = 0
+
+ upload_id = has_upload(d, tar_file, folder_id)
+ if upload_id == False:
+ bb.error("No upload of this software found on the fossology server!")
+ return False
+
+ while i < 3:
+ i += 1
+ spdx2tv = get_spdx(d, report_id, spdx_file)
+ if spdx2tv == False:
+ bb.note("spdx : " + d.getVar('PN', True) + ": get_spdx failed. Will try again!")
+ else:
+ return True
+
+ bb.error("spdx: get_spdx of " + d.getVar('PN', True) + "failed. Please confirm!")
+ return False
diff --git a/classes/spdx-common.bbclass b/classes/spdx-common.bbclass
index c074da4..9fe40f6 100644
--- a/classes/spdx-common.bbclass
+++ b/classes/spdx-common.bbclass
@@ -15,8 +15,13 @@ SPDXEPENDENCY += " tar-native:do_populate_sysroot"
SPDX_DEPLOY_DIR ??= "${DEPLOY_DIR}/spdx"
SPDX_TOPDIR ?= "${WORKDIR}/spdx_sstate_dir"
-SPDX_OUTDIR = "${SPDX_TOPDIR}/${TARGET_SYS}/${PF}/"
-SPDX_WORKDIR = "${WORKDIR}/spdx_temp/"
+SPDX_OUTDIR ?= "${SPDX_TOPDIR}/${TARGET_SYS}/${PF}/"
+SPDX_WORKDIR ?= "${WORKDIR}/spdx_temp/"
+
+SPDX_EXCLUDE_NATIVE ??= "1"
+SPDX_EXCLUDE_SDK ??= "1"
+SPDX_EXCLUDE_PACKAGES ??= ""
+
do_spdx[dirs] = "${WORKDIR}"
@@ -28,6 +33,38 @@ SPDX_S ?= "${S}"
addtask do_spdx before do_configure after do_patch
+# Exclude package based on variables.
+# SPDX_EXCLUDE_NATIVE ??= "1"
+# SPDX_EXCLUDE_SDK ??= "1"
+# SPDX_EXCLUDE_PACKAGES ??= ""
+def excluded_package(d, pn):
+ assume_provided = (d.getVar("ASSUME_PROVIDED") or "").split()
+ if pn in assume_provided:
+ for p in d.getVar("PROVIDES").split():
+ if p != pn:
+ pn = p
+ break
+ if d.getVar('BPN') in ['gcc', 'libgcc']:
+ #bb.debug(1, 'spdx: There is a bug in the scan of %s, skip it.' % pn)
+ return True
+ # The following: do_fetch, do_unpack and do_patch tasks have been deleted,
+ # so avoid archiving do_spdx here.
+ # -native is for the host aka during the build
+ if pn.endswith('-native') and d.getVar("SPDX_EXCLUDE_NATIVE") == "1":
+ return True
+ # nativesdk- is for the developer SDK
+ if pn.startswith('nativesdk-') and d.getVar("SPDX_EXCLUDE_SDK") == "1":
+ return True
+ # packagegroups have no files to scan
+ if pn.startswith('packagegroup'):
+ return True
+ if pn.startswith('glibc-locale'):
+ return True
+ for p in d.getVar("SPDX_EXCLUDE_PACKAGES").split():
+ if p in pn:
+ return True
+ return False
+
def exclude_useless_paths(tarinfo):
if tarinfo.isdir():
if tarinfo.name.endswith('/temp') or tarinfo.name.endswith('/patches') or tarinfo.name.endswith('/.pc'):