aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLei Maohui <leimaohui@cn.fujitsu.com>2019-11-18 15:33:45 +0900
committerLei Maohui <leimaohui@cn.fujitsu.com>2019-11-18 15:33:45 +0900
commit90257f0281df3c9006c2ffa8b66c657a5e387983 (patch)
treecfa33c9b2798bc54b09011ed22e3d64a908c3802
parent0a49da9a436206b188dbbd26e045ea401333b886 (diff)
downloadmeta-spdxscanner-90257f0281df3c9006c2ffa8b66c657a5e387983.tar.gz
meta-spdxscanner-90257f0281df3c9006c2ffa8b66c657a5e387983.tar.bz2
meta-spdxscanner-90257f0281df3c9006c2ffa8b66c657a5e387983.zip
Added scancode-toolkit support into meta-spdxscanner.
Signed-off-by: Lei Maohui <leimaohui@cn.fujitsu.com>
-rw-r--r--README.md24
-rw-r--r--classes/scancode.bbclass148
-rw-r--r--recipes-devtools/scancode-toolkit/scancode-toolkit-native_3.1.1.bb78
3 files changed, 246 insertions, 4 deletions
diff --git a/README.md b/README.md
index 8704042..f2efb19 100644
--- a/README.md
+++ b/README.md
@@ -2,8 +2,9 @@
meta-spdxscanner supports the following SPDX create tools.
1. fossology REST API (Can work with fossology after 3.5.0)
-2. fossdriver (Can work with fossology 3.5.0)
-3. DoSOCSv2 (Scanner comes from fossology 3.4.0)
+2. fossdriver (Can work with fossology)
+3. cancode-toolkit
+4. DoSOCSv2 (Scanner comes from fossology 3.4.0)
# This layer supplys invoking scanners as following:
@@ -13,7 +14,10 @@ meta-spdxscanner supports the following SPDX create tools.
2. fossdriver
- openembedded-core
-3. DoSOCSv2
+3. scancode-toolkit
+- openembedded-core
+
+4. DoSOCSv2
- openembedded-core
- meta-openembedded/meta-oe
- meta-openembedded/meta-python
@@ -47,7 +51,19 @@ Note
Please reference to https://hub.docker.com/r/fossology/fossology/ and https://github.com/fossology/fossdriver.
- Please use meta-spdxscanner/classes/nopackages.bbclass instead of oe-core. Because there is no necessary to create spdx files for *-native.
-3. dosocs.bbclass
+3. scancode.bbclass
+- inherit the folowing class in your conf/local.conf for all of recipes or
+ in some recipes which you want.
+
+```
+ INHERIT += "scancode"
+```
+Note
+- If you want to use scancode.bbclass, There is no need to install anything on your host.
+- To aviod loop dependence,please use meta-spdxscanner/classes/nopackages.bbclass instead the file comes from oe-core.
+
+
+4. dosocs.bbclass
- inherit the folowing class in your conf/local.conf for all of recipes or
in some recipes which you want.
diff --git a/classes/scancode.bbclass b/classes/scancode.bbclass
new file mode 100644
index 0000000..4c362ce
--- /dev/null
+++ b/classes/scancode.bbclass
@@ -0,0 +1,148 @@
+# This class integrates real-time license scanning, generation of SPDX standard
+# output and verifiying license info during the building process.
+# It is a combination of efforts from the OE-Core, SPDX and ScanCode projects.
+#
+# For more information on ScanCode:
+# https://github.com/nexB/scancode-toolkit
+#
+# For more information on SPDX:
+# http://www.spdx.org
+#
+# Note:
+# 1) By default,spdx files will be output to the path which is defined as[SPDX_DEPLOY_DIR]
+# 2) By default, SPDX_DEPLOY_DIR is tmp/deploy
+#
+inherit spdx-common
+
+SPDXEPENDENCY += "scancode-toolkit-native:do_populate_sysroot"
+
+CREATOR_TOOL = "cancode.bbclass in meta-spdxscanner"
+
+python do_spdx () {
+ import os, sys, json, shutil
+
+ pn = d.getVar('PN')
+ assume_provided = (d.getVar("ASSUME_PROVIDED") or "").split()
+ if pn in assume_provided:
+ for p in d.getVar("PROVIDES").split():
+ if p != pn:
+ pn = p
+ break
+
+ # glibc-locale: do_fetch, do_unpack and do_patch tasks have been deleted,
+ # so avoid archiving source here.
+ if pn.startswith('glibc-locale'):
+ return
+ if (d.getVar('BPN') == "linux-yocto"):
+ return
+ if (d.getVar('PN') == "libtool-cross"):
+ return
+ if (d.getVar('PN') == "libgcc-initial"):
+ return
+ if (d.getVar('PN') == "shadow-sysroot"):
+ return
+
+
+ # We just archive gcc-source for all the gcc related recipes
+ if d.getVar('BPN') in ['gcc', 'libgcc']:
+ bb.debug(1, 'spdx: There is bug in scan of %s is, do nothing' % pn)
+ return
+
+ spdx_outdir = d.getVar('SPDX_OUTDIR')
+ spdx_workdir = d.getVar('SPDX_WORKDIR')
+ spdx_temp_dir = os.path.join(spdx_workdir, "temp")
+ temp_dir = os.path.join(d.getVar('WORKDIR'), "temp")
+
+ info = {}
+ info['workdir'] = (d.getVar('WORKDIR', True) or "")
+ info['pn'] = (d.getVar( 'PN', True ) or "")
+ info['pv'] = (d.getVar( 'PV', True ) or "")
+ info['package_download_location'] = (d.getVar( 'SRC_URI', True ) or "")
+ if info['package_download_location'] != "":
+ info['package_download_location'] = info['package_download_location'].split()[0]
+ info['spdx_version'] = (d.getVar('SPDX_VERSION', True) or '')
+ info['data_license'] = (d.getVar('DATA_LICENSE', True) or '')
+ info['creator'] = {}
+ info['creator']['Tool'] = (d.getVar('CREATOR_TOOL', True) or '')
+ info['license_list_version'] = (d.getVar('LICENSELISTVERSION', True) or '')
+ info['package_homepage'] = (d.getVar('HOMEPAGE', True) or "")
+ info['package_summary'] = (d.getVar('SUMMARY', True) or "")
+ info['package_summary'] = info['package_summary'].replace("\n","")
+ info['package_summary'] = info['package_summary'].replace("'"," ")
+ info['package_contains'] = (d.getVar('CONTAINED', True) or "")
+ info['package_static_link'] = (d.getVar('STATIC_LINK', True) or "")
+ info['modified'] = "false"
+ srcuri = d.getVar("SRC_URI", False).split()
+ length = len("file://")
+ for item in srcuri:
+ if item.startswith("file://"):
+ item = item[length:]
+ if item.endswith(".patch") or item.endswith(".diff"):
+ info['modified'] = "true"
+
+ manifest_dir = (d.getVar('SPDX_DEPLOY_DIR', True) or "")
+ if not os.path.exists( manifest_dir ):
+ bb.utils.mkdirhier( manifest_dir )
+ info['outfile'] = os.path.join(manifest_dir, info['pn'] + "-" + info['pv'] + ".spdx" )
+ sstatefile = os.path.join(spdx_outdir, info['pn'] + "-" + info['pv'] + ".spdx" )
+
+ # if spdx has been exist
+ if os.path.exists(info['outfile']):
+ bb.note(info['pn'] + "spdx file has been exist, do nothing")
+ return
+ if os.path.exists( sstatefile ):
+ bb.note(info['pn'] + "spdx file has been exist, do nothing")
+ create_manifest(info,sstatefile)
+ return
+
+ spdx_get_src(d)
+
+ bb.note('SPDX: Archiving the patched source...')
+ if os.path.isdir(spdx_temp_dir):
+ for f_dir, f in list_files(spdx_temp_dir):
+ temp_file = os.path.join(spdx_temp_dir,f_dir,f)
+ shutil.copy(temp_file, temp_dir)
+ shutil.rmtree(spdx_temp_dir)
+ if not os.path.exists(spdx_outdir):
+ bb.utils.mkdirhier(spdx_outdir)
+ cur_ver_code = get_ver_code(spdx_workdir).split()[0]
+ ## Get spdx file
+ bb.note(' run ScanCode ...... ')
+ d.setVar('WORKDIR', d.getVar('SPDX_WORKDIR', True))
+ info['sourcedir'] = spdx_workdir
+ git_path = "%s/.git" % info['sourcedir']
+ if os.path.exists(git_path):
+ remove_dir_tree(git_path)
+ invoke_scancode(info['sourcedir'],sstatefile)
+ if get_cached_spdx(sstatefile) != None:
+ write_cached_spdx( info,sstatefile,cur_ver_code )
+ ## CREATE MANIFEST(write to outfile )
+ create_manifest(info,sstatefile)
+ else:
+ bb.warn('Can\'t get the spdx file ' + info['pn'] + '. Please check your.')
+}
+
+def invoke_scancode( OSS_src_dir, spdx_file):
+ import subprocess
+ import string
+ import json
+ import codecs
+ import logging
+
+ logger = logging.getLogger()
+ logger.setLevel(logging.INFO)
+ logging.basicConfig(level=logging.INFO)
+
+ path = os.getenv('PATH')
+ scancode_cmd = bb.utils.which(os.getenv('PATH'), "scancode")
+ scancode_cmd = scancode_cmd + " -lpci --spdx-tv " + spdx_file + " " + OSS_src_dir
+ print(scancode_cmd)
+ try:
+ subprocess.check_output(scancode_cmd,
+ stderr=subprocess.STDOUT,
+ shell=True)
+ except subprocess.CalledProcessError as e:
+ bb.fatal("Could not invoke scancode Command "
+ "'%s' returned %d:\n%s" % (scancode_cmd, e.returncode, e.output))
+
+EXPORT_FUNCTIONS do_spdx
diff --git a/recipes-devtools/scancode-toolkit/scancode-toolkit-native_3.1.1.bb b/recipes-devtools/scancode-toolkit/scancode-toolkit-native_3.1.1.bb
new file mode 100644
index 0000000..b7024ac
--- /dev/null
+++ b/recipes-devtools/scancode-toolkit/scancode-toolkit-native_3.1.1.bb
@@ -0,0 +1,78 @@
+SUMMARY = "ScanCode toolkit"
+DESCRIPTION = "A typical software project often reuses hundreds of third-party \
+packages. License and origin information is not always easy to find and not \
+normalized: ScanCode discovers and normalizes this data for you."
+HOMEPAGE = "https://github.com/nexB/scancode-toolkit"
+SECTION = "devel"
+
+LICENSE = "Apache-2.0"
+LIC_FILES_CHKSUM = "file://NOTICE;md5=8aedb84647f637c585e71f8f2e96e5c8"
+
+EXTRANATIVEPATH_remove = "python-native"
+
+inherit setuptools pypi distutils native
+
+DEPENDS = "python-setuptools-native xz-native zlib-native libxml2-native \
+ libxslt-native bzip2-native \
+ "
+
+SRC_URI = "git://github.com/nexB/scancode-toolkit;branch=master \
+ "
+
+SRCREV = "1af5ac8449cbb1ce98a0b461a6d9a5ad42a5d248"
+
+
+S = "${WORKDIR}/git"
+
+do_configure(){
+ ./scancode --help
+}
+
+do_install_append(){
+ install -d ${D}${bindir}/bin
+ install -d ${D}${bindir}/include
+ install -d ${D}${bindir}/local
+
+ install ${S}/scancode ${D}${bindir}/
+ install ${S}/bin/* ${D}${bindir}/bin/
+ mv ${S}/include/* ${D}${bindir}/include/
+ #ln -sf ${S}/apache-2.0.LICENSE ${D}${bindir}/local/apache-2.0.LICENSE
+ #ln -sf ${S}/appveyor.yml ${D}${bindir}/local/appveyor.yml
+ #ln -sf ${S}/AUTHORS.rst ${D}${bindir}/local/AUTHORS.rst
+ #ln -sf ${S}/azure-pipelines.yml ${D}${bindir}/local/azure-pipelines.yml
+ #ln -sf ${S}/bin/ ${D}${bindir}/local/bin
+ #ln -sf ${S}/.bumpversion.cfg ${D}${bindir}/local/.bumpversion.cfg
+ #ln -sf ${S}/cc0-1.0.LICENSE ${D}${bindir}/local/cc0-1.0.LICENSE
+ #ln -sf ${S}/CHANGELOG.rst ${D}${bindir}/local/CHANGELOG.rst
+ #ln -sf ${S}/.cirrus.yml ${D}${bindir}/local/.cirrus.yml
+ #ln -sf ${S}/codecov.yml ${D}${bindir}/local/codecov.yml
+ #ln -sf ${S}/CODE_OF_CONDUCT.rst ${D}${bindir}/local/CODE_OF_CONDUCT.rst
+ #ln -sf ${S}/configure ${D}${bindir}/local/configure
+ #ln -sf ${S}/configure.bat ${D}${bindir}/local/configure.bat
+ #ln -sf ${S}/conftest.py ${D}${bindir}/local/conftest.py
+ #ln -sf ${S}/CONTRIBUTING.rst ${D}${bindir}/local/CONTRIBUTING.rst
+ #ln -sf ${S}/.coveragerc ${D}${bindir}/local/.coveragerc
+ ##ln -sf ${S}/docs/ ${D}${bindir}/local/docs
+ #ln -sf ${S}/etc/ ${D}${bindir}/local/etc
+ #ln -sf ${S}/extractcode ${D}${bindir}/local/extractcode
+ #ln -sf ${S}/extractcode.bat ${D}${bindir}/local/extractcode.bat
+ #ln -sf ${S}/include/ ${D}${bindir}/local/include
+ #ln -sf ${S}/ISSUE_TEMPLATE.md ${D}${bindir}/local/ISSUE_TEMPLATE.md
+ #ln -sf ${S}/lib/ ${D}${bindir}/local/lib
+ #ln -sf ${S}/MANIFEST.in ${D}${bindir}/local/MANIFEST.in
+ #ln -sf ${S}/NOTICE ${D}${bindir}/local/NOTICE
+ #ln -sf ${S}/plugins/ ${D}${bindir}/local/plugins
+ #ln -sf ${S}/plugins-builtin/ ${D}${bindir}/local/plugins-builtin
+ #ln -sf ${S}/README.rst ${D}${bindir}/local/README.rst
+ #ln -sf ${S}/samples/ ${D}${bindir}/local/samples
+ #ln -sf ${S}/scancode ${D}${bindir}/local/scancode
+ #ln -sf ${S}/scancode.bat ${D}${bindir}/local/scancode.bat
+ #ln -sf ${S}/scancode-toolkit.ABOUT ${D}${bindir}/local/scancode-toolkit.ABOUT
+ #ln -sf ${S}/setup.cfg ${D}${bindir}/local/setup.cfg
+ #ln -sf ${S}/setup.py ${D}${bindir}/local/setup.py
+ #ln -sf ${S}/src/ ${D}${bindir}/local/src
+ #ln -sf ${S}/tests ${D}${bindir}/local/tests
+ #ln -sf ${S}/thirdparty/ ${D}${bindir}/local/thirdparty
+ #ln -sf ${S}/.travis.yml ${D}${bindir}/local/.travis.yml
+}
+