aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLei Maohui <leimaohui@cn.fujitsu.com>2019-09-09 15:43:38 +0900
committerLei Maohui <leimaohui@cn.fujitsu.com>2019-09-09 15:43:38 +0900
commitc9bc9cd0421f145f406fc1359b0ba4be44ec0694 (patch)
tree0446bf56db7d4c29d63e2a3139c860f1bf794a35
parentdf8887905da4324ed133027f984e5d766cdd97d0 (diff)
downloadmeta-spdxscanner-c9bc9cd0421f145f406fc1359b0ba4be44ec0694.tar.gz
meta-spdxscanner-c9bc9cd0421f145f406fc1359b0ba4be44ec0694.tar.bz2
meta-spdxscanner-c9bc9cd0421f145f406fc1359b0ba4be44ec0694.zip
Reduce failures during multitask of do_spdx.
Signed-off-by: Lei Maohui <leimaohui@cn.fujitsu.com>
-rw-r--r--classes/fossology-rest.bbclass111
1 files changed, 63 insertions, 48 deletions
diff --git a/classes/fossology-rest.bbclass b/classes/fossology-rest.bbclass
index 44d7968..3097e18 100644
--- a/classes/fossology-rest.bbclass
+++ b/classes/fossology-rest.bbclass
@@ -126,7 +126,7 @@ python do_spdx () {
bb.warn(info['pn'] + "has no source, do nothing")
return
folder_id = (d.getVar('FOLDER_ID', True) or "")
- if invoke_rest_api(d, tar_name,sstatefile, folder_id) == False:
+ if invoke_rest_api(d, tar_name, sstatefile, folder_id) == False:
bb.warn("info['pn']: Get spdx file fail, please check your fossology.")
return False
if get_cached_spdx(sstatefile) != None:
@@ -153,7 +153,9 @@ def has_upload(d, tar_file, folder):
bb.note("Please set token of fossology server by setting TOKEN!\n" + srcPath)
raise OSError(errno.ENOENT, "No setting of TOKEN comes from fossology server.")
- rest_api_cmd = "curl -k -s -S -X GET " + server_url + "/api/v1/uploads" + " -H \"Authorization: Bearer " + token + "\"" + " --noproxy 127.0.0.1"
+ rest_api_cmd = "curl -k -s -S -X GET " + server_url + "/api/v1/uploads" \
+ + " -H \"Authorization: Bearer " + token + "\"" \
+ + " --noproxy 127.0.0.1"
bb.note("Invoke rest_api_cmd = " + rest_api_cmd )
try:
@@ -177,7 +179,7 @@ def has_upload(d, tar_file, folder):
bb.note(str(len(upload_output)))
for i in range(0, len(upload_output)):
if upload_output[i]["uploadname"] == file_name:
- if os.path.getsize(tar_file) == upload_output[i]["filesize"] and upload_output[i]["foldername"] == "Software Repository":
+ if str(os.path.getsize(tar_file)) == str(upload_output[i]["filesize"]) and str(upload_output[i]["folderid"]) == str(folder_id):
bb.warn("Find " + file_name + "in fossology server \"Software Repository\" folder. So, will not upload again.")
return upload_output[i]["id"]
return False
@@ -204,7 +206,7 @@ def upload(d, tar_file, folder):
+ " -H \'uploadDescription: created by REST\'" \
+ " -H \'public: public\'" \
+ " -H \'Content-Type: multipart/form-data\'" \
- + " -F \'fileInput=@\"" + tar_file + "\";type=application/octet-stream\'"
+ + " -F \'fileInput=@\"" + tar_file + "\";type=application/octet-stream\'" \
+ " --noproxy 127.0.0.1"
bb.note("Upload : Invoke rest_api_cmd = " + rest_api_cmd )
while i < 10:
@@ -245,23 +247,28 @@ def analysis(d, folder_id, upload_id):
+ " -H \"uploadId: " + str(upload_id) + "\"" \
+ " -H \"Authorization: Bearer " + token + "\"" \
+ " -H \'Content-Type: application/json\'" \
- + " --data \'{\"analysis\": {\"bucket\": true,\"copyright_email_author\": true,\"ecc\": true, \"keyword\": true,\"mime\": true,\"monk\": true,\"nomos\": true,\"package\": true},\"decider\": {\"nomos_monk\": true,\"bulk_reused\": true,\"new_scanner\": true}}\'"
+ + " --data \'{\"analysis\": {\"bucket\": true,\"copyright_email_author\": true,\"ecc\": true, \"keyword\": true,\"mime\": true,\"monk\": true,\"nomos\": true,\"package\": true},\"decider\": {\"nomos_monk\": true,\"bulk_reused\": true,\"new_scanner\": true}}\'" \
+ " --noproxy 127.0.0.1"
bb.note("Analysis : Invoke rest_api_cmd = " + rest_api_cmd )
while i < 10:
- time.sleep(delaytime)
try:
+ time.sleep(delaytime)
analysis = subprocess.check_output(rest_api_cmd, stderr=subprocess.STDOUT, shell=True)
except subprocess.CalledProcessError as e:
bb.error("Analysis failed: \n%s" % e.output.decode("utf-8"))
return False
-
+ time.sleep(delaytime)
analysis = str(analysis, encoding = "utf-8")
bb.note("analysis = ")
bb.note(analysis)
analysis = eval(analysis)
if str(analysis["code"]) == "201":
return analysis["message"]
+ elif str(analysis["code"]) == "404":
+ bb.warn("analysis is still not complete.")
+ time.sleep(delaytime)
+ else:
+ return False
i += 1
bb.warn("Analysis is fail, will try again.")
bb.warn("Analysis is fail, please check your fossology server.")
@@ -286,7 +293,7 @@ def trigger(d, folder_id, upload_id):
rest_api_cmd = "curl -k -s -S -X GET " + server_url + "/api/v1/report" \
+ " -H \"Authorization: Bearer " + token + "\"" \
+ " -H \"uploadId: " + str(upload_id) + "\"" \
- + " -H \'reportFormat: spdx2tv\'"
+ + " -H \'reportFormat: spdx2tv\'" \
+ " --noproxy 127.0.0.1"
bb.note("trigger : Invoke rest_api_cmd = " + rest_api_cmd )
while i < 10:
@@ -296,7 +303,7 @@ def trigger(d, folder_id, upload_id):
except subprocess.CalledProcessError as e:
bb.error("Trigger failed: \n%s" % e.output.decode("utf-8"))
return False
-
+ time.sleep(delaytime)
trigger = str(trigger, encoding = "utf-8")
trigger = eval(trigger)
bb.note("trigger id = ")
@@ -304,6 +311,7 @@ def trigger(d, folder_id, upload_id):
if str(trigger["code"]) == "201":
return trigger["message"].split("/")[-1]
i += 1
+ time.sleep(delaytime)
bb.warn("Trigger is fail, will try again.")
bb.warn("Trigger is fail, please check your fossology server.")
return False
@@ -312,7 +320,7 @@ def get_spdx(d, report_id, spdx_file):
import os
import subprocess
import time
- delaytime = 100
+ delaytime = 50
empty = True
i = 0
@@ -327,47 +335,48 @@ def get_spdx(d, report_id, spdx_file):
raise OSError(errno.ENOENT, "No setting of TOKEN comes from fossology server.")
rest_api_cmd = "curl -k -s -S -X GET " + server_url + "/api/v1/report/" + report_id \
+ " -H \'accept: text/plain\'" \
- + " -H \"Authorization: Bearer " + token + "\""
+ + " -H \"Authorization: Bearer " + token + "\"" \
+ " --noproxy 127.0.0.1"
bb.note("get_spdx : Invoke rest_api_cmd = " + rest_api_cmd )
- try:
- while i < 10:
- time.sleep(delaytime)
- file = open(spdx_file,'wt')
+ while i < 10:
+ time.sleep(delaytime)
+ file = open(spdx_file,'wt')
+ try:
p = subprocess.Popen(rest_api_cmd, shell=True, universal_newlines=True, stdout=file)
- ret_code = p.wait()
- file.flush()
- file.close()
- file = open(spdx_file,'r+')
- first_line = file.readline()
- if "SPDXVersion" in first_line:
+ except subprocess.CalledProcessError as e:
+ bb.error("Get spdx failed: \n%s" % e.output.decode("utf-8"))
+ return False
+ ret_code = p.wait()
+ file.flush()
+ time.sleep(delaytime)
+ file.close()
+ file = open(spdx_file,'r+')
+ first_line = file.readline()
+ if "SPDXVersion" in first_line:
+ line = file.readline()
+ while line:
+ if "LicenseID:" in line:
+ empty = False
+ break
line = file.readline()
- while line:
- if "LicenseID:" in line:
- empty = False
- break
- line = file.readline()
- if empty == True:
- bb.warn("This is an empty spdx file. Please confirm")
- file.close()
- return
+ file.close()
+ if empty == True:
+ bb.warn("Hasn't get license info.")
+ return False
else:
- bb.warn("Get spdx failed, will try again.")
- file.close()
- os.remove(spdx_file)
- i += 1
- except subprocess.CalledProcessError as e:
- bb.error("Get spdx failed: \n%s" % e.output.decode("utf-8"))
- return False
-
- bb.warn("Get spdx failed, please check your fossology server.")
-
- file.close()
-
+ return True
+ else:
+ bb.warn("Get the first line is " + first_line)
+ bb.warn("spdx is not correct, will try again.")
+ file.close()
+ os.remove(spdx_file)
+ i += 1
+ bb.warn("Get spdx failed, Please check your fossology server.")
def invoke_rest_api(d, tar_file, spdx_file, folder_id):
import os
import time
+ i = 0
bb.note("invoke fossology REST API : tar_file = %s " % tar_file)
upload_id = has_upload(d, tar_file, folder_id)
@@ -379,9 +388,15 @@ def invoke_rest_api(d, tar_file, spdx_file, folder_id):
if analysis(d, folder_id, upload_id) == False:
return False
- report_id = trigger(d, folder_id, upload_id)
- if report_id == False:
- return False
- if get_spdx(d, report_id, spdx_file) == False:
- return False
- return True
+ while i < 3:
+ report_id = trigger(d, folder_id, upload_id)
+ if report_id == False:
+ return False
+ spdx2tv = get_spdx(d, report_id, spdx_file)
+ if spdx2tv == False:
+ bb.warn("get_spdx is unnormal. Will try again!")
+ else:
+ return True
+
+ print("get_spdx of %s is unnormal. Please check your fossology server!")
+ return False