aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLei Maohui <leimaohui@cn.fujitsu.com>2019-12-10 10:36:13 +0900
committerLei Maohui <leimaohui@cn.fujitsu.com>2019-12-10 10:36:13 +0900
commitc0ee3ee9d8c5da9654cd3c148a996c7dc14b5ebe (patch)
tree597b7f428070e28af1ac14091023b4dc900212ef
parentd70146c3b74d44daeeff43487d2de3233bfd145f (diff)
downloadmeta-spdxscanner-c0ee3ee9d8c5da9654cd3c148a996c7dc14b5ebe.tar.gz
meta-spdxscanner-c0ee3ee9d8c5da9654cd3c148a996c7dc14b5ebe.tar.bz2
meta-spdxscanner-c0ee3ee9d8c5da9654cd3c148a996c7dc14b5ebe.zip
Aim to support old dosocsv2, many old recipes have to been added.
These old recipes are conflicting with newest ones. So move dosocsv2 into a separate branch. Please make sure that recipes in this branch are not not conflict with your YP project before you using it. Signed-off-by: Lei Maohui <leimaohui@cn.fujitsu.com>
-rw-r--r--classes/fossdriver-host.bbclass209
-rw-r--r--classes/fossology-rest.bbclass407
-rw-r--r--classes/scancode-tk.bbclass139
-rw-r--r--recipes-devtools/scancode-toolkit/scancode-toolkit-native_3.1.1.bb40
4 files changed, 0 insertions, 795 deletions
diff --git a/classes/fossdriver-host.bbclass b/classes/fossdriver-host.bbclass
deleted file mode 100644
index 0b168a6..0000000
--- a/classes/fossdriver-host.bbclass
+++ /dev/null
@@ -1,209 +0,0 @@
-# This class integrates real-time license scanning, generation of SPDX standard
-# output and verifiying license info during the building process.
-# It is a combination of efforts from the OE-Core, SPDX and fossology projects.
-#
-# For more information on fossology REST API:
-# https://www.fossology.org/get-started/basic-rest-api-calls/
-#
-# For more information on SPDX:
-# http://www.spdx.org
-#
-# Note:
-# 1) Make sure fossology (after 3.5.0)(https://hub.docker.com/r/fossology/fossology/) has beed started on your host
-# 2) spdx files will be output to the path which is defined as[SPDX_DEPLOY_DIR].
-# By default, SPDX_DEPLOY_DIR is tmp/deploy/
-# 3) Added TOKEN has been set in conf/local.conf
-#
-
-inherit spdx-common
-
-CREATOR_TOOL = "fossdriver-host.bbclass in meta-spdxscanner"
-
-# If ${S} isn't actually the top-level source directory, set SPDX_S to point at
-# the real top-level directory.
-SPDX_S ?= "${S}"
-
-python do_spdx () {
- import os, sys, json, shutil
-
- pn = d.getVar('PN')
- assume_provided = (d.getVar("ASSUME_PROVIDED") or "").split()
- if pn in assume_provided:
- for p in d.getVar("PROVIDES").split():
- if p != pn:
- pn = p
- break
-
- # glibc-locale: do_fetch, do_unpack and do_patch tasks have been deleted,
- # so avoid archiving source here.
- if pn.startswith('glibc-locale'):
- return
- if (d.getVar('PN') == "libtool-cross"):
- return
- if (d.getVar('PN') == "libgcc-initial"):
- return
- if (d.getVar('PN') == "shadow-sysroot"):
- return
-
- if d.getVar('BPN') in ['gcc', 'libgcc']:
- bb.debug(1, 'spdx: There is bug in scan of %s is, do nothing' % pn)
- return
-
- # We just archive gcc-source for all the gcc related recipes
- if d.getVar('BPN') in ['gcc', 'libgcc']:
- bb.debug(1, 'spdx: There is bug in scan of %s is, do nothing' % pn)
- return
-
- spdx_outdir = d.getVar('SPDX_OUTDIR')
- spdx_workdir = d.getVar('SPDX_WORKDIR')
- spdx_temp_dir = os.path.join(spdx_workdir, "temp")
- temp_dir = os.path.join(d.getVar('WORKDIR'), "temp")
-
- info = {}
- info['workdir'] = (d.getVar('WORKDIR', True) or "")
- info['pn'] = (d.getVar( 'PN', True ) or "")
- info['pv'] = (d.getVar( 'PV', True ) or "")
- info['package_download_location'] = (d.getVar( 'SRC_URI', True ) or "")
- if info['package_download_location'] != "":
- info['package_download_location'] = info['package_download_location'].split()[0]
- info['spdx_version'] = (d.getVar('SPDX_VERSION', True) or '')
- info['data_license'] = (d.getVar('DATA_LICENSE', True) or '')
- info['creator'] = {}
- info['creator']['Tool'] = (d.getVar('CREATOR_TOOL', True) or '')
- info['license_list_version'] = (d.getVar('LICENSELISTVERSION', True) or '')
- info['package_homepage'] = (d.getVar('HOMEPAGE', True) or "")
- info['package_summary'] = (d.getVar('SUMMARY', True) or "")
- info['package_summary'] = info['package_summary'].replace("\n","")
- info['package_summary'] = info['package_summary'].replace("'"," ")
- info['package_contains'] = (d.getVar('CONTAINED', True) or "")
- info['package_static_link'] = (d.getVar('STATIC_LINK', True) or "")
- info['modified'] = "false"
- srcuri = d.getVar("SRC_URI", False).split()
- length = len("file://")
- for item in srcuri:
- if item.startswith("file://"):
- item = item[length:]
- if item.endswith(".patch") or item.endswith(".diff"):
- info['modified'] = "true"
-
- manifest_dir = (d.getVar('SPDX_DEPLOY_DIR', True) or "")
- if not os.path.exists( manifest_dir ):
- bb.utils.mkdirhier( manifest_dir )
-
- info['outfile'] = os.path.join(manifest_dir, info['pn'] + "-" + info['pv'] + ".spdx" )
- sstatefile = os.path.join(spdx_outdir, info['pn'] + "-" + info['pv'] + ".spdx" )
-
- # if spdx has been exist
- if os.path.exists(info['outfile']):
- bb.note(info['pn'] + "spdx file has been exist, do nothing")
- return
- if os.path.exists( sstatefile ):
- bb.note(info['pn'] + "spdx file has been exist, do nothing")
- create_manifest(info,sstatefile)
- return
-
- spdx_get_src(d)
-
- bb.note('SPDX: Archiving the patched source...')
- if os.path.isdir(spdx_temp_dir):
- for f_dir, f in list_files(spdx_temp_dir):
- temp_file = os.path.join(spdx_temp_dir,f_dir,f)
- shutil.copy(temp_file, temp_dir)
-
- d.setVar('WORKDIR', spdx_workdir)
- info['sourcedir'] = spdx_workdir
- git_path = "%s/git/.git" % info['sourcedir']
- if os.path.exists(git_path):
- remove_dir_tree(git_path)
- tar_name = spdx_create_tarball(d, d.getVar('WORKDIR'), 'patched', spdx_outdir)
- ## get everything from cache. use it to decide if
- ## something needs to be rerun
- if not os.path.exists(spdx_outdir):
- bb.utils.mkdirhier(spdx_outdir)
- cur_ver_code = get_ver_code(spdx_workdir).split()[0]
- ## Get spdx file
- bb.note(' run fossdriver ...... ')
- if not os.path.isfile(tar_name):
- bb.warn(info['pn'] + "has no source, do nothing")
- return
- invoke_fossdriver(tar_name,sstatefile)
- if get_cached_spdx(sstatefile) != None:
- write_cached_spdx( info,sstatefile,cur_ver_code )
- ## CREATE MANIFEST(write to outfile )
- create_manifest(info,sstatefile)
- else:
- bb.warn('Can\'t get the spdx file ' + info['pn'] + '. Please check your.')
- remove_file(tar_name)
-}
-
-
-def invoke_fossdriver(tar_file, spdx_file):
- import os
- import time
- delaytime = 20
-
- import logging
-
- logger = logging.getLogger()
- logger.setLevel(logging.INFO)
- logging.basicConfig(level=logging.INFO)
-
- (work_dir, tar_file) = os.path.split(tar_file)
- os.chdir(work_dir)
-
- from fossdriver.config import FossConfig
- from fossdriver.server import FossServer
- from fossdriver.tasks import (CreateFolder, Upload, Scanners, Copyright, Reuse, BulkTextMatch, SPDXTV)
- if 'http_proxy' in os.environ:
- del os.environ['http_proxy']
- config = FossConfig()
- configPath = os.path.join(os.path.expanduser('~'),".fossdriverrc")
- config.configure(configPath)
- server = FossServer(config)
- server.Login()
- bb.note("invoke_fossdriver : tar_file = %s " % tar_file)
- if (Reuse(server, tar_file, "Software Repository", tar_file, "Software Repository").run() != True):
- bb.note("This OSS has not been scanned. So upload it to fossology server.")
- i = 0
- while i < 5:
- if (Upload(server, tar_file, "Software Repository").run() != True):
- bb.warn("%s Upload failed, try again!" % tar_file)
- time.sleep(delaytime)
- i += 1
- else:
- i = 0
- while i < 10:
- if (Scanners(server, tar_file, "Software Repository").run() != True):
- bb.warn("%s Scanners failed, try again!" % tar_file)
- time.sleep(delaytime)
- i+= 1
- else:
- i = 0
- while i < 10:
- Copyright(server, tar_file, "Software Repository").run()
- if (SPDXTV(server, tar_file, "Software Repository", spdx_file).run() == False):
- bb.warn("%s SPDXTV failed, try again!" % tar_file)
- time.sleep(delaytime)
- i += 1
- else:
- return True
- bb.warn("%s SPDXTV failed, Please check your fossology server." % tar_file)
- return False
- bb.warn("%s Scanners failed, Please check your fossology server." % tar_file)
- return False
- bb.warn("%s Upload fail.Please check your fossology server." % tar_file)
- return False
- else:
- i = 0
- while i < 10:
- if (SPDXTV(server, tar_file, "Software Repository", spdx_file).run() == False):
- time.sleep(1)
- bb.warn("%s SPDXTV failed, try again!" % tar_file)
- i += 1
- time.sleep(delaytime)
- else:
- return True
- bb.warn("%s SPDXTV failed, Please check your fossology server." % tar_file)
- return False
-
-EXPORT_FUNCTIONS do_spdx
diff --git a/classes/fossology-rest.bbclass b/classes/fossology-rest.bbclass
deleted file mode 100644
index 0dd5b6f..0000000
--- a/classes/fossology-rest.bbclass
+++ /dev/null
@@ -1,407 +0,0 @@
-# This class integrates real-time license scanning, generation of SPDX standard
-# output and verifiying license info during the building process.
-# It is a combination of efforts from the OE-Core, SPDX and DoSOCSv2 projects.
-#
-# For more information on DoSOCSv2:
-# https://github.com/DoSOCSv2
-#
-# For more information on SPDX:
-# http://www.spdx.org
-#
-# Note:
-# 1) Make sure fossdriver has beed installed in your host
-# 2) By default,spdx files will be output to the path which is defined as[SPDX_DEPLOY_DIR]
-# in ./meta/conf/spdx-dosocs.conf.
-inherit spdx-common
-FOSSOLOGY_SERVER ?= "http://127.0.0.1:8081/repo"
-
-#upload OSS into No.1 folder of fossology
-FOLDER_ID = "1"
-
-HOSTTOOLS_NONFATAL += "curl"
-
-CREATOR_TOOL = "fossology-rest.bbclass in meta-spdxscanner"
-
-# If ${S} isn't actually the top-level source directory, set SPDX_S to point at
-# the real top-level directory.
-SPDX_S ?= "${S}"
-
-python do_spdx () {
- import os, sys, shutil
-
- pn = d.getVar('PN')
- assume_provided = (d.getVar("ASSUME_PROVIDED") or "").split()
- if pn in assume_provided:
- for p in d.getVar("PROVIDES").split():
- if p != pn:
- pn = p
- break
- if d.getVar('BPN') in ['gcc', 'libgcc']:
- bb.debug(1, 'spdx: There is bug in scan of %s is, do nothing' % pn)
- return
-
- # The following: do_fetch, do_unpack and do_patch tasks have been deleted,
- # so avoid archiving do_spdx here.
- if pn.startswith('glibc-locale'):
- return
- if (d.getVar('PN') == "libtool-cross"):
- return
- if (d.getVar('PN') == "libgcc-initial"):
- return
- if (d.getVar('PN') == "shadow-sysroot"):
- return
-
- spdx_outdir = d.getVar('SPDX_OUTDIR')
- spdx_workdir = d.getVar('SPDX_WORKDIR')
- spdx_temp_dir = os.path.join(spdx_workdir, "temp")
- temp_dir = os.path.join(d.getVar('WORKDIR'), "temp")
-
- info = {}
- info['workdir'] = (d.getVar('WORKDIR', True) or "")
- info['pn'] = (d.getVar( 'PN', True ) or "")
- info['pv'] = (d.getVar( 'PV', True ) or "")
- info['package_download_location'] = (d.getVar( 'SRC_URI', True ) or "")
- if info['package_download_location'] != "":
- info['package_download_location'] = info['package_download_location'].split()[0]
- info['spdx_version'] = (d.getVar('SPDX_VERSION', True) or '')
- info['data_license'] = (d.getVar('DATA_LICENSE', True) or '')
- info['creator'] = {}
- info['creator']['Tool'] = (d.getVar('CREATOR_TOOL', True) or '')
- info['license_list_version'] = (d.getVar('LICENSELISTVERSION', True) or '')
- info['package_homepage'] = (d.getVar('HOMEPAGE', True) or "")
- info['package_summary'] = (d.getVar('SUMMARY', True) or "")
- info['package_summary'] = info['package_summary'].replace("\n","")
- info['package_summary'] = info['package_summary'].replace("'"," ")
- info['package_contains'] = (d.getVar('CONTAINED', True) or "")
- info['package_static_link'] = (d.getVar('STATIC_LINK', True) or "")
- info['modified'] = "false"
- info['token'] = (d.getVar('TOKEN', True) or "")
-
- srcuri = d.getVar("SRC_URI", False).split()
- length = len("file://")
- for item in srcuri:
- if item.startswith("file://"):
- item = item[length:]
- if item.endswith(".patch") or item.endswith(".diff"):
- info['modified'] = "true"
-
- manifest_dir = (d.getVar('SPDX_DEPLOY_DIR', True) or "")
- if not os.path.exists( manifest_dir ):
- bb.utils.mkdirhier( manifest_dir )
-
- info['outfile'] = os.path.join(manifest_dir, info['pn'] + "-" + info['pv'] + ".spdx" )
- sstatefile = os.path.join(spdx_outdir, info['pn'] + "-" + info['pv'] + ".spdx" )
-
- # if spdx has been exist
- if os.path.exists(info['outfile']):
- bb.note(info['pn'] + "spdx file has been exist, do nothing")
- return
- if os.path.exists( sstatefile ):
- bb.note(info['pn'] + "spdx file has been exist, do nothing")
- create_manifest(info,sstatefile)
- return
-
- spdx_get_src(d)
-
- bb.note('SPDX: Archiving the patched source...')
- if os.path.isdir(spdx_temp_dir):
- for f_dir, f in list_files(spdx_temp_dir):
- temp_file = os.path.join(spdx_temp_dir,f_dir,f)
- shutil.copy(temp_file, temp_dir)
- # shutil.rmtree(spdx_temp_dir)
- d.setVar('WORKDIR', spdx_workdir)
- info['sourcedir'] = spdx_workdir
- git_path = "%s/git/.git" % info['sourcedir']
- if os.path.exists(git_path):
- remove_dir_tree(git_path)
- tar_name = spdx_create_tarball(d, d.getVar('WORKDIR'), 'patched', spdx_outdir)
-
- ## get everything from cache. use it to decide if
- ## something needs to be rerun
- if not os.path.exists(spdx_outdir):
- bb.utils.mkdirhier(spdx_outdir)
- cur_ver_code = get_ver_code(spdx_workdir).split()[0]
- ## Get spdx file
- bb.note(' run fossology rest api ...... ')
- if not os.path.isfile(tar_name):
- bb.warn(info['pn'] + "has no source, do nothing")
- return
- folder_id = (d.getVar('FOLDER_ID', True) or "")
- if invoke_rest_api(d, tar_name, sstatefile, folder_id) == False:
- bb.warn(info['pn'] + ": Get spdx file fail, please check your fossology.")
- remove_file(tar_name)
- return False
- if get_cached_spdx(sstatefile) != None:
- write_cached_spdx( info,sstatefile,cur_ver_code )
- ## CREATE MANIFEST(write to outfile )
- create_manifest(info,sstatefile)
- else:
- bb.warn(info['pn'] + ': Can\'t get the spdx file ' + '. Please check your.')
- remove_file(tar_name)
-}
-
-def has_upload(d, tar_file, folder_id):
- import os
- import subprocess
-
- (work_dir, file_name) = os.path.split(tar_file)
-
- server_url = (d.getVar('FOSSOLOGY_SERVER', True) or "")
- if server_url == "":
- bb.note("Please set fossology server URL by setting FOSSOLOGY_SERVER!\n")
- raise OSError(errno.ENOENT, "No setting of FOSSOLOGY_SERVER")
-
- token = (d.getVar('TOKEN', True) or "")
- if token == "":
- bb.note("Please set token of fossology server by setting TOKEN!\n" + srcPath)
- raise OSError(errno.ENOENT, "No setting of TOKEN comes from fossology server.")
-
- rest_api_cmd = "curl -k -s -S -X GET " + server_url + "/api/v1/uploads" \
- + " -H \"Authorization: Bearer " + token + "\"" \
- + " --noproxy 127.0.0.1"
- bb.note("Invoke rest_api_cmd = " + rest_api_cmd )
-
- try:
- upload_output = subprocess.check_output(rest_api_cmd, stderr=subprocess.STDOUT, shell=True)
- except subprocess.CalledProcessError as e:
- bb.error("curl failed: \n%s" % e.output.decode("utf-8"))
- return False
-
- upload_output = str(upload_output, encoding = "utf-8")
- upload_output = eval(upload_output)
- bb.note("upload_output = ")
- print(upload_output)
- bb.note("len of upload_output = ")
- bb.note(str(len(upload_output)))
- if len(upload_output) == 0:
- bb.note("The upload of fossology is 0.")
- return False
- bb.note("upload_output[0][uploadname] = ")
- bb.note(upload_output[0]["uploadname"])
- bb.note("len of upload_output = ")
- bb.note(str(len(upload_output)))
- for i in range(0, len(upload_output)):
- if upload_output[i]["uploadname"] == file_name:
- if str(os.path.getsize(tar_file)) == str(upload_output[i]["filesize"]) and str(upload_output[i]["folderid"]) == str(folder_id):
- bb.warn("Find " + file_name + "in fossology server \"Software Repository\" folder. So, will not upload again.")
- return upload_output[i]["id"]
- return False
-
-def upload(d, tar_file, folder):
- import os
- import subprocess
- delaytime = 50
- i = 0
-
- server_url = (d.getVar('FOSSOLOGY_SERVER', True) or "")
- if server_url == "":
- bb.note("Please set fossology server URL by setting FOSSOLOGY_SERVER!\n")
- raise OSError(errno.ENOENT, "No setting of FOSSOLOGY_SERVER")
-
- token = (d.getVar('TOKEN', True) or "")
- if token == "":
- bb.note("Please set token of fossology server by setting TOKEN!\n" + srcPath)
- raise OSError(errno.ENOENT, "No setting of TOKEN comes from fossology server.")
-
- rest_api_cmd = "curl -k -s -S -X POST " + server_url + "/api/v1/uploads" \
- + " -H \"folderId: " + folder + "\"" \
- + " -H \"Authorization: Bearer " + token + "\"" \
- + " -H \'uploadDescription: created by REST\'" \
- + " -H \'public: public\'" \
- + " -H \'Content-Type: multipart/form-data\'" \
- + " -F \'fileInput=@\"" + tar_file + "\";type=application/octet-stream\'" \
- + " --noproxy 127.0.0.1"
- bb.note("Upload : Invoke rest_api_cmd = " + rest_api_cmd )
- while i < 10:
- time.sleep(delaytime)
- try:
- upload = subprocess.check_output(rest_api_cmd, stderr=subprocess.STDOUT, shell=True)
- except subprocess.CalledProcessError as e:
- bb.error(d.getVar('PN', True) + ": Upload failed: \n%s" % e.output.decode("utf-8"))
- return False
- upload = str(upload, encoding = "utf-8")
- bb.note("Upload = ")
- bb.note(upload)
- upload = eval(upload)
- if str(upload["code"]) == "201":
- return upload["message"]
- i += 1
- bb.warn(d.getVar('PN', True) + ": Upload is fail, please check your fossology server.")
- return False
-
-def analysis(d, folder_id, upload_id):
- import os
- import subprocess
- delaytime = 50
- i = 0
-
- server_url = (d.getVar('FOSSOLOGY_SERVER', True) or "")
- if server_url == "":
- bb.note("Please set fossology server URL by setting FOSSOLOGY_SERVER!\n")
- raise OSError(errno.ENOENT, "No setting of FOSSOLOGY_SERVER")
-
- token = (d.getVar('TOKEN', True) or "")
- if token == "":
- bb.note("Please set token of fossology server by setting TOKEN!\n" + srcPath)
- raise OSError(errno.ENOENT, "No setting of TOKEN comes from fossology server.")
-
- rest_api_cmd = "curl -k -s -S -X POST " + server_url + "/api/v1/jobs" \
- + " -H \"folderId: " + str(folder_id) + "\"" \
- + " -H \"uploadId: " + str(upload_id) + "\"" \
- + " -H \"Authorization: Bearer " + token + "\"" \
- + " -H \'Content-Type: application/json\'" \
- + " --data \'{\"analysis\": {\"bucket\": true,\"copyright_email_author\": true,\"ecc\": true, \"keyword\": true,\"mime\": true,\"monk\": true,\"nomos\": true,\"package\": true},\"decider\": {\"nomos_monk\": true,\"bulk_reused\": true,\"new_scanner\": true}}\'" \
- + " --noproxy 127.0.0.1"
- bb.note("Analysis : Invoke rest_api_cmd = " + rest_api_cmd )
- while i < 10:
- try:
- time.sleep(delaytime)
- analysis = subprocess.check_output(rest_api_cmd, stderr=subprocess.STDOUT, shell=True)
- except subprocess.CalledProcessError as e:
- bb.error("Analysis failed: \n%s" % e.output.decode("utf-8"))
- return False
- time.sleep(delaytime)
- analysis = str(analysis, encoding = "utf-8")
- bb.note("analysis = ")
- bb.note(analysis)
- analysis = eval(analysis)
- if str(analysis["code"]) == "201":
- return analysis["message"]
- elif str(analysis["code"]) == "404":
- bb.warn(d.getVar('PN', True) + ": analysis is still not complete.")
- time.sleep(delaytime*2)
- else:
- return False
- i += 1
- bb.warn(d.getVar('PN', True) + ": Analysis is fail, will try again.")
- bb.warn(d.getVar('PN', True) + ": Analysis is fail, please check your fossology server.")
- return False
-
-def trigger(d, folder_id, upload_id):
- import os
- import subprocess
- delaytime = 50
- i = 0
-
- server_url = (d.getVar('FOSSOLOGY_SERVER', True) or "")
- if server_url == "":
- bb.note("Please set fossology server URL by setting FOSSOLOGY_SERVER!\n")
- raise OSError(errno.ENOENT, "No setting of FOSSOLOGY_SERVER")
-
- token = (d.getVar('TOKEN', True) or "")
- if token == "":
- bb.note("Please set token of fossology server by setting TOKEN!\n" + srcPath)
- raise OSError(errno.ENOENT, "No setting of TOKEN comes from fossology server.")
-
- rest_api_cmd = "curl -k -s -S -X GET " + server_url + "/api/v1/report" \
- + " -H \"Authorization: Bearer " + token + "\"" \
- + " -H \"uploadId: " + str(upload_id) + "\"" \
- + " -H \'reportFormat: spdx2tv\'" \
- + " --noproxy 127.0.0.1"
- bb.note("trigger : Invoke rest_api_cmd = " + rest_api_cmd )
- while i < 10:
- time.sleep(delaytime)
- try:
- trigger = subprocess.check_output(rest_api_cmd, stderr=subprocess.STDOUT, shell=True)
- except subprocess.CalledProcessError as e:
- bb.error(d.getVar('PN', True) + ": Trigger failed: \n%s" % e.output.decode("utf-8"))
- return False
- time.sleep(delaytime)
- trigger = str(trigger, encoding = "utf-8")
- trigger = eval(trigger)
- bb.note("trigger id = ")
- bb.note(str(trigger["message"]))
- if str(trigger["code"]) == "201":
- return trigger["message"].split("/")[-1]
- i += 1
- time.sleep(delaytime * 2)
- bb.warn(d.getVar('PN', True) + ": Trigger is fail, will try again.")
- bb.warn(d.getVar('PN', True) + ": Trigger is fail, please check your fossology server.")
- return False
-
-def get_spdx(d, report_id, spdx_file):
- import os
- import subprocess
- import time
- delaytime = 50
- empty = True
- i = 0
-
- server_url = (d.getVar('FOSSOLOGY_SERVER', True) or "")
- if server_url == "":
- bb.note("Please set fossology server URL by setting FOSSOLOGY_SERVER!\n")
- raise OSError(errno.ENOENT, "No setting of FOSSOLOGY_SERVER")
-
- token = (d.getVar('TOKEN', True) or "")
- if token == "":
- bb.note("Please set token of fossology server by setting TOKEN!\n" + srcPath)
- raise OSError(errno.ENOENT, "No setting of TOKEN comes from fossology server.")
- rest_api_cmd = "curl -k -s -S -X GET " + server_url + "/api/v1/report/" + report_id \
- + " -H \'accept: text/plain\'" \
- + " -H \"Authorization: Bearer " + token + "\"" \
- + " --noproxy 127.0.0.1"
- bb.note("get_spdx : Invoke rest_api_cmd = " + rest_api_cmd )
- while i < 3:
- time.sleep(delaytime)
- file = open(spdx_file,'wt')
- try:
- p = subprocess.Popen(rest_api_cmd, shell=True, universal_newlines=True, stdout=file)
- except subprocess.CalledProcessError as e:
- bb.error("Get spdx failed: \n%s" % e.output.decode("utf-8"))
- return False
- ret_code = p.wait()
- file.flush()
- time.sleep(delaytime)
- file.close()
- file = open(spdx_file,'r+')
- first_line = file.readline()
- if "SPDXVersion" in first_line:
- line = file.readline()
- while line:
- if "LicenseID:" in line:
- empty = False
- break
- line = file.readline()
- file.close()
- if empty == True:
- bb.warn("Hasn't get license info.")
- return False
- else:
- return True
- else:
- bb.warn(d.getVar('PN', True) + ": Get the first line is " + first_line)
- bb.warn(d.getVar('PN', True) + ": spdx is not correct, will try again.")
- file.close()
- os.remove(spdx_file)
- i += 1
- time.sleep(delaytime*2)
- bb.warn(d.getVar('PN', True) + ": Get spdx failed, Please check your fossology server.")
-
-def invoke_rest_api(d, tar_file, spdx_file, folder_id):
- import os
- import time
- i = 0
-
- bb.note("invoke fossology REST API : tar_file = %s " % tar_file)
- upload_id = has_upload(d, tar_file, folder_id)
- if upload_id == False:
- bb.note("This OSS has not been scanned. So upload it to fossology server.")
- upload_id = upload(d, tar_file, folder_id)
- if upload_id == False:
- return False
-
- if analysis(d, folder_id, upload_id) == False:
- return False
- while i < 10:
- i += 1
- report_id = trigger(d, folder_id, upload_id)
- if report_id == False:
- return False
- spdx2tv = get_spdx(d, report_id, spdx_file)
- if spdx2tv == False:
- bb.warn(d.getVar('PN', True) + ": get_spdx is unnormal. Will try again!")
- else:
- return True
-
- bb.warn("get_spdx of %s is unnormal. Please confirm!")
- return False
diff --git a/classes/scancode-tk.bbclass b/classes/scancode-tk.bbclass
deleted file mode 100644
index 9fbbb48..0000000
--- a/classes/scancode-tk.bbclass
+++ /dev/null
@@ -1,139 +0,0 @@
-# This class integrates real-time license scanning, generation of SPDX standard
-# output and verifiying license info during the building process.
-# It is a combination of efforts from the OE-Core, SPDX and ScanCode projects.
-#
-# For more information on ScanCode:
-# https://github.com/nexB/scancode-toolkit
-#
-# For more information on SPDX:
-# http://www.spdx.org
-#
-# Note:
-# 1) By default,spdx files will be output to the path which is defined as[SPDX_DEPLOY_DIR]
-# 2) By default, SPDX_DEPLOY_DIR is tmp/deploy
-#
-
-inherit spdx-common
-
-SPDXEPENDENCY += "scancode-toolkit-native:do_populate_sysroot"
-
-CREATOR_TOOL = "cancode.bbclass in meta-spdxscanner"
-
-python do_spdx(){
- import os, sys, json, shutil
- pn = d.getVar('PN')
- assume_provided = (d.getVar("ASSUME_PROVIDED") or "").split()
- if pn in assume_provided:
- for p in d.getVar("PROVIDES").split():
- if p != pn:
- pn = p
- break
- if d.getVar('BPN') in ['gcc', 'libgcc']:
- bb.debug(1, 'spdx: There is bug in scan of %s is, do nothing' % pn)
- return
- # glibc-locale: do_fetch, do_unpack and do_patch tasks have been deleted,
- # so avoid archiving source here.
- if pn.startswith('glibc-locale'):
- return
- if (d.getVar('PN') == "libtool-cross"):
- return
- if (d.getVar('PN') == "libgcc-initial"):
- return
- if (d.getVar('PN') == "shadow-sysroot"):
- return
-
- spdx_outdir = d.getVar('SPDX_OUTDIR')
- spdx_workdir = d.getVar('SPDX_WORKDIR')
- spdx_temp_dir = os.path.join(spdx_workdir, "temp")
- temp_dir = os.path.join(d.getVar('WORKDIR'), "temp")
-
- info = {}
- info['workdir'] = (d.getVar('WORKDIR', True) or "")
- info['pn'] = (d.getVar( 'PN', True ) or "")
- info['pv'] = (d.getVar( 'PV', True ) or "")
- info['package_download_location'] = (d.getVar( 'SRC_URI', True ) or "")
- if info['package_download_location'] != "":
- info['package_download_location'] = info['package_download_location'].split()[0]
- info['spdx_version'] = (d.getVar('SPDX_VERSION', True) or '')
- info['data_license'] = (d.getVar('DATA_LICENSE', True) or '')
- info['creator'] = {}
- info['creator']['Tool'] = (d.getVar('CREATOR_TOOL', True) or '')
- info['license_list_version'] = (d.getVar('LICENSELISTVERSION', True) or '')
- info['package_homepage'] = (d.getVar('HOMEPAGE', True) or "")
- info['package_summary'] = (d.getVar('SUMMARY', True) or "")
- info['package_summary'] = info['package_summary'].replace("\n","")
- info['package_summary'] = info['package_summary'].replace("'"," ")
- info['package_contains'] = (d.getVar('CONTAINED', True) or "")
- info['package_static_link'] = (d.getVar('STATIC_LINK', True) or "")
- info['modified'] = "false"
- srcuri = d.getVar("SRC_URI", False).split()
- length = len("file://")
- for item in srcuri:
- if item.startswith("file://"):
- item = item[length:]
- if item.endswith(".patch") or item.endswith(".diff"):
- info['modified'] = "true"
-
- manifest_dir = (d.getVar('SPDX_DEPLOY_DIR', True) or "")
- if not os.path.exists( manifest_dir ):
- bb.utils.mkdirhier( manifest_dir )
- info['outfile'] = os.path.join(manifest_dir, info['pn'] + "-" + info['pv'] + ".spdx" )
- sstatefile = os.path.join(spdx_outdir, info['pn'] + "-" + info['pv'] + ".spdx" )
- # if spdx has been exist
- if os.path.exists(info['outfile']):
- bb.note(info['pn'] + "spdx file has been exist, do nothing")
- return
- if os.path.exists( sstatefile ):
- bb.note(info['pn'] + "spdx file has been exist, do nothing")
- create_manifest(info,sstatefile)
- return
- spdx_get_src(d)
-
- bb.note('SPDX: Archiving the patched source...')
- if os.path.isdir(spdx_temp_dir):
- for f_dir, f in list_files(spdx_temp_dir):
- temp_file = os.path.join(spdx_temp_dir,f_dir,f)
- shutil.copy(temp_file, temp_dir)
- #shutil.rmtree(spdx_temp_dir)
- if not os.path.exists(spdx_outdir):
- bb.utils.mkdirhier(spdx_outdir)
- cur_ver_code = get_ver_code(spdx_workdir).split()[0]
- ## Get spdx file
- bb.note(' run scanCode ...... ')
- d.setVar('WORKDIR', d.getVar('SPDX_WORKDIR', True))
- info['sourcedir'] = spdx_workdir
- git_path = "%s/git/.git" % info['sourcedir']
- if os.path.exists(git_path):
- remove_dir_tree(git_path)
- invoke_scancode(info['sourcedir'],sstatefile)
- bb.warn("source dir = " + info['sourcedir'])
- if get_cached_spdx(sstatefile) != None:
- write_cached_spdx( info,sstatefile,cur_ver_code )
- ## CREATE MANIFEST(write to outfile )
- create_manifest(info,sstatefile)
- else:
- bb.warn('Can\'t get the spdx file ' + info['pn'] + '. Please check your.')
-}
-
-def invoke_scancode( OSS_src_dir, spdx_file):
- import subprocess
- import string
- import json
- import codecs
- import logging
-
- logger = logging.getLogger()
- logger.setLevel(logging.INFO)
- logging.basicConfig(level=logging.INFO)
-
- path = os.getenv('PATH')
- scancode_cmd = bb.utils.which(os.getenv('PATH'), "scancode")
- scancode_cmd = scancode_cmd + " -lpci --spdx-tv " + spdx_file + " " + OSS_src_dir
- print(scancode_cmd)
- try:
- subprocess.check_output(scancode_cmd,
- stderr=subprocess.STDOUT,
- shell=True)
- except subprocess.CalledProcessError as e:
- bb.fatal("Could not invoke scancode Command "
- "'%s' returned %d:\n%s" % (scancode_cmd, e.returncode, e.output))
diff --git a/recipes-devtools/scancode-toolkit/scancode-toolkit-native_3.1.1.bb b/recipes-devtools/scancode-toolkit/scancode-toolkit-native_3.1.1.bb
deleted file mode 100644
index 8f6f466..0000000
--- a/recipes-devtools/scancode-toolkit/scancode-toolkit-native_3.1.1.bb
+++ /dev/null
@@ -1,40 +0,0 @@
-SUMMARY = "ScanCode toolkit"
-DESCRIPTION = "A typical software project often reuses hundreds of third-party \
-packages. License and origin information is not always easy to find and not \
-normalized: ScanCode discovers and normalizes this data for you."
-HOMEPAGE = "https://github.com/nexB/scancode-toolkit"
-SECTION = "devel"
-
-LICENSE = "Apache-2.0"
-LIC_FILES_CHKSUM = "file://NOTICE;md5=8aedb84647f637c585e71f8f2e96e5c8"
-
-EXTRANATIVEPATH_remove = "python-native"
-
-inherit setuptools pypi distutils native
-
-DEPENDS = "python-setuptools-native xz-native zlib-native libxml2-native \
- libxslt-native bzip2-native \
- "
-
-SRC_URI = "git://github.com/nexB/scancode-toolkit;branch=master \
- "
-
-SRCREV = "1af5ac8449cbb1ce98a0b461a6d9a5ad42a5d248"
-
-
-S = "${WORKDIR}/git"
-
-do_configure(){
- ./scancode --help
-}
-
-do_install_append(){
- install -d ${D}${bindir}/bin
- install -d ${D}${bindir}/include
- install -d ${D}${bindir}/local
-
- install ${S}/scancode ${D}${bindir}/
- install ${S}/bin/* ${D}${bindir}/bin/
- mv ${S}/include/* ${D}${bindir}/include/
-}
-