aboutsummaryrefslogtreecommitdiffstats
path: root/classes/fossdriver-host.bbclass
blob: cb04333f9beeb9a6518a876a1af7d2610afdd969 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
# This class integrates real-time license scanning, generation of SPDX standard
# output and verifiying license info during the building process.
# It is a combination of efforts from the OE-Core, SPDX and fossdriver projects.
#
# For more information on fossdriver:
#   https://github.com/fossology/fossdriver
#
# For more information on SPDX:
#   http://www.spdx.org
#
# Note:
# 1) Make sure fossology 3.5.0(https://hub.docker.com/r/fossology/fossology/) has beed started on your host
# 2) spdx files will be output to the path which is defined as[SPDX_DEPLOY_DIR].
#    By default, SPDX_DEPLOY_DIR is tmp/deploy/
# 
inherit spdx-common

CREATOR_TOOL = "fossdriver-host.bbclass in meta-spdxscanner"

# If ${S} isn't actually the top-level source directory, set SPDX_S to point at
# the real top-level directory.
SPDX_S ?= "${S}"

python do_spdx () {
    import os, sys, json, shutil

    pn = d.getVar('PN')
    assume_provided = (d.getVar("ASSUME_PROVIDED") or "").split()
    if pn in assume_provided:
        for p in d.getVar("PROVIDES").split():
            if p != pn:
                pn = p
                break

    # glibc-locale: do_fetch, do_unpack and do_patch tasks have been deleted,
    # so avoid archiving source here.
    if pn.startswith('glibc-locale'):
        return
    if (d.getVar('BPN') == "linux-yocto"):
        return
    if (d.getVar('PN') == "libtool-cross"):
        return
    if (d.getVar('PN') == "libgcc-initial"):
        return
    if (d.getVar('PN') == "shadow-sysroot"):
        return


    # We just archive gcc-source for all the gcc related recipes
    if d.getVar('BPN') in ['gcc', 'libgcc']:
        bb.debug(1, 'spdx: There is bug in scan of %s is, do nothing' % pn)
        return

    spdx_outdir = d.getVar('SPDX_OUTDIR')
    spdx_workdir = d.getVar('SPDX_WORKDIR')
    spdx_temp_dir = os.path.join(spdx_workdir, "temp")
    temp_dir = os.path.join(d.getVar('WORKDIR'), "temp")
    
    info = {} 
    info['workdir'] = (d.getVar('WORKDIR', True) or "")
    info['pn'] = (d.getVar( 'PN', True ) or "")
    info['pv'] = (d.getVar( 'PV', True ) or "")
    info['package_download_location'] = (d.getVar( 'SRC_URI', True ) or "")
    if info['package_download_location'] != "":
        info['package_download_location'] = info['package_download_location'].split()[0]
    info['spdx_version'] = (d.getVar('SPDX_VERSION', True) or '')
    info['data_license'] = (d.getVar('DATA_LICENSE', True) or '')
    info['creator'] = {}
    info['creator']['Tool'] = (d.getVar('CREATOR_TOOL', True) or '')
    info['license_list_version'] = (d.getVar('LICENSELISTVERSION', True) or '')
    info['package_homepage'] = (d.getVar('HOMEPAGE', True) or "")
    info['package_summary'] = (d.getVar('SUMMARY', True) or "")
    info['package_summary'] = info['package_summary'].replace("\n","")
    info['package_summary'] = info['package_summary'].replace("'"," ")
    info['package_contains'] = (d.getVar('CONTAINED', True) or "")
    info['package_static_link'] = (d.getVar('STATIC_LINK', True) or "")
    info['modified'] = "false"
    srcuri = d.getVar("SRC_URI", False).split()
    length = len("file://")
    for item in srcuri:
        if item.startswith("file://"):
            item = item[length:]
            if item.endswith(".patch") or item.endswith(".diff"):
                info['modified'] = "true"

    manifest_dir = (d.getVar('SPDX_DEPLOY_DIR', True) or "")
    if not os.path.exists( manifest_dir ):
        bb.utils.mkdirhier( manifest_dir )

    info['outfile'] = os.path.join(manifest_dir, info['pn'] + "-" + info['pv'] + ".spdx" )
    sstatefile = os.path.join(spdx_outdir, info['pn'] + "-" + info['pv'] + ".spdx" )
    
    # if spdx has been exist
    if os.path.exists(info['outfile']):
        bb.note(info['pn'] + "spdx file has been exist, do nothing")
        return
    if os.path.exists( sstatefile ):
        bb.note(info['pn'] + "spdx file has been exist, do nothing")
        create_manifest(info,sstatefile)
        return

    spdx_get_src(d)

    bb.note('SPDX: Archiving the patched source...')
    if os.path.isdir(spdx_temp_dir):
        for f_dir, f in list_files(spdx_temp_dir):
            temp_file = os.path.join(spdx_temp_dir,f_dir,f)
            shutil.copy(temp_file, temp_dir)
        shutil.rmtree(spdx_temp_dir)
    d.setVar('WORKDIR', spdx_workdir)
    tar_name = spdx_create_tarball(d, d.getVar('WORKDIR'), 'patched', spdx_outdir)
    ## get everything from cache.  use it to decide if 
    ## something needs to be rerun
    if not os.path.exists(spdx_outdir):
        bb.utils.mkdirhier(spdx_outdir)
    cur_ver_code = get_ver_code(spdx_workdir).split()[0] 
    ## Get spdx file
    bb.note(' run fossdriver ...... ')
    if not os.path.isfile(tar_name):
        bb.warn(info['pn'] + "has no source, do nothing")
        return
    invoke_fossdriver(tar_name,sstatefile)
    if get_cached_spdx(sstatefile) != None:
        write_cached_spdx( info,sstatefile,cur_ver_code )
        ## CREATE MANIFEST(write to outfile )
        create_manifest(info,sstatefile)
    else:
        bb.warn('Can\'t get the spdx file ' + info['pn'] + '. Please check your.')
}


def invoke_fossdriver(tar_file, spdx_file):
    import os
    import time
    delaytime = 20
    
    (work_dir, tar_file) = os.path.split(tar_file)
    os.chdir(work_dir)

    from fossdriver.config import FossConfig
    from fossdriver.server import FossServer
    from fossdriver.tasks import (CreateFolder, Upload, Scanners, Copyright, Reuse, BulkTextMatch, SPDXTV)
    if 'http_proxy' in os.environ:
        del os.environ['http_proxy']
    config = FossConfig()
    configPath = os.path.join(os.path.expanduser('~'),".fossdriverrc")
    config.configure(configPath)
    server = FossServer(config)
    server.Login()
    bb.note("invoke_fossdriver : tar_file = %s " % tar_file)
    if (Reuse(server, tar_file, "Software Repository", tar_file, "Software Repository").run()  != True):
        bb.note("This OSS has not been scanned. So upload it to fossology server.")
        i = 0
        while i < 5:
            if (Upload(server, tar_file, "Software Repository").run() != True):
                bb.warn("%s Upload failed, try again!" %  tar_file)
                time.sleep(delaytime)
                i += 1
            else:
                i = 0
                while i < 10:                
                    if (Scanners(server, tar_file, "Software Repository").run() != True):
                        bb.warn("%s scanner failed, try again!" % tar_file)
                        time.sleep(delaytime)
                        i+= 1
                    else:
                        i = 0
                        while i < 10:
                            Copyright(server, tar_file, "Software Repository").run()
                            if (SPDXTV(server, tar_file, "Software Repository", spdx_file).run() == False):
                                bb.warn("%s SPDXTV failed, try again!" % tar_file)
                                time.sleep(delaytime)
                                i += 1
                            else:
                                return True
                        bb.warn("%s SPDXTV failed, Please check your fossology server." % tar_file)
                        return False
                bb.warn("%s Scanners failed, Please check your fossology server." % tar_file)
                return False
        bb.warn("%s  Upload fail.Please check your fossology server." % tar_file)
        return False
    else:
        i = 0
        while i < 10:
            if (SPDXTV(server, tar_file, "Software Repository", spdx_file).run() == False):
                time.sleep(1)
                bb.warn("%s SPDXTV failed, try again!" % tar_file)
                i += 1
                time.sleep(delaytime)
            else:
                return True
        bb.warn("%s SPDXTV failed, Please check your fossology server." % tar_file)
        return False

EXPORT_FUNCTIONS do_spdx