summaryrefslogtreecommitdiffstats
path: root/scripts/lib/recipetool/create_npm.py
blob: 39429ebad3cc0561ff64b4a4aad640aee4570504 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
# Recipe creation tool - node.js NPM module support plugin
#
# Copyright (C) 2016 Intel Corporation
#
# SPDX-License-Identifier: GPL-2.0-only
#

import os
import sys
import logging
import subprocess
import tempfile
import shutil
import json
from recipetool.create import RecipeHandler, split_pkg_licenses, handle_license_vars

logger = logging.getLogger('recipetool')


tinfoil = None

def tinfoil_init(instance):
    global tinfoil
    tinfoil = instance


class NpmRecipeHandler(RecipeHandler):
    lockdownpath = None

    def _ensure_npm(self, fixed_setup=False):
        if not tinfoil.recipes_parsed:
            tinfoil.parse_recipes()
        try:
            rd = tinfoil.parse_recipe('nodejs-native')
        except bb.providers.NoProvider:
            if fixed_setup:
                msg = 'nodejs-native is required for npm but is not available within this SDK'
            else:
                msg = 'nodejs-native is required for npm but is not available - you will likely need to add a layer that provides nodejs'
            logger.error(msg)
            return None
        bindir = rd.getVar('STAGING_BINDIR_NATIVE')
        npmpath = os.path.join(bindir, 'npm')
        if not os.path.exists(npmpath):
            tinfoil.build_targets('nodejs-native', 'addto_recipe_sysroot')
            if not os.path.exists(npmpath):
                logger.error('npm required to process specified source, but nodejs-native did not seem to populate it')
                return None
        return bindir

    def _handle_license(self, data):
        '''
        Handle the license value from an npm package.json file
        '''
        license = None
        if 'license' in data:
            license = data['license']
            if isinstance(license, dict):
                license = license.get('type', None)
            if license:
                if 'OR' in license:
                    license = license.replace('OR', '|')
                    license = license.replace('AND', '&')
                    license = license.replace(' ', '_')
                    if not license[0] == '(':
                        license = '(' + license + ')'
                else:
                    license = license.replace('AND', '&')
                    if license[0] == '(':
                        license = license[1:]
                    if license[-1] == ')':
                        license = license[:-1]
                license = license.replace('MIT/X11', 'MIT')
                license = license.replace('Public Domain', 'PD')
                license = license.replace('SEE LICENSE IN EULA',
                                          'SEE-LICENSE-IN-EULA')
        return license

    def _shrinkwrap(self, srctree, localfilesdir, extravalues, lines_before, d):
        try:
            runenv = dict(os.environ, PATH=d.getVar('PATH'))
            bb.process.run('npm shrinkwrap', cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True)
        except bb.process.ExecutionError as e:
            logger.warning('npm shrinkwrap failed:\n%s' % e.stdout)
            return

        tmpfile = os.path.join(localfilesdir, 'npm-shrinkwrap.json')
        shutil.move(os.path.join(srctree, 'npm-shrinkwrap.json'), tmpfile)
        extravalues.setdefault('extrafiles', {})
        extravalues['extrafiles']['npm-shrinkwrap.json'] = tmpfile
        lines_before.append('NPM_SHRINKWRAP := "${THISDIR}/${PN}/npm-shrinkwrap.json"')

    def _lockdown(self, srctree, localfilesdir, extravalues, lines_before, d):
        runenv = dict(os.environ, PATH=d.getVar('PATH'))
        if not NpmRecipeHandler.lockdownpath:
            NpmRecipeHandler.lockdownpath = tempfile.mkdtemp('recipetool-npm-lockdown')
            bb.process.run('npm install lockdown --prefix %s' % NpmRecipeHandler.lockdownpath,
                           cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True)
        relockbin = os.path.join(NpmRecipeHandler.lockdownpath, 'node_modules', 'lockdown', 'relock.js')
        if not os.path.exists(relockbin):
            logger.warning('Could not find relock.js within lockdown directory; skipping lockdown')
            return
        try:
            bb.process.run('node %s' % relockbin, cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True)
        except bb.process.ExecutionError as e:
            logger.warning('lockdown-relock failed:\n%s' % e.stdout)
            return

        tmpfile = os.path.join(localfilesdir, 'lockdown.json')
        shutil.move(os.path.join(srctree, 'lockdown.json'), tmpfile)
        extravalues.setdefault('extrafiles', {})
        extravalues['extrafiles']['lockdown.json'] = tmpfile
        lines_before.append('NPM_LOCKDOWN := "${THISDIR}/${PN}/lockdown.json"')

    def _handle_dependencies(self, d, deps, optdeps, devdeps, lines_before, srctree):
        import scriptutils
        # If this isn't a single module we need to get the dependencies
        # and add them to SRC_URI
        def varfunc(varname, origvalue, op, newlines):
            if varname == 'SRC_URI':
                if not origvalue.startswith('npm://'):
                    src_uri = origvalue.split()
                    deplist = {}
                    for dep, depver in optdeps.items():
                        depdata = self.get_npm_data(dep, depver, d)
                        if self.check_npm_optional_dependency(depdata):
                            deplist[dep] = depdata
                    for dep, depver in devdeps.items():
                        depdata = self.get_npm_data(dep, depver, d)
                        if self.check_npm_optional_dependency(depdata):
                            deplist[dep] = depdata
                    for dep, depver in deps.items():
                        depdata = self.get_npm_data(dep, depver, d)
                        deplist[dep] = depdata

                    extra_urls = []
                    for dep, depdata in deplist.items():
                        version = depdata.get('version', None)
                        if version:
                            url = 'npm://registry.npmjs.org;name=%s;version=%s;subdir=node_modules/%s' % (dep, version, dep)
                            extra_urls.append(url)
                    if extra_urls:
                        scriptutils.fetch_url(tinfoil, ' '.join(extra_urls), None, srctree, logger)
                        src_uri.extend(extra_urls)
                        return src_uri, None, -1, True
            return origvalue, None, 0, True
        updated, newlines = bb.utils.edit_metadata(lines_before, ['SRC_URI'], varfunc)
        if updated:
            del lines_before[:]
            for line in newlines:
                # Hack to avoid newlines that edit_metadata inserts
                if line.endswith('\n'):
                    line = line[:-1]
                lines_before.append(line)
        return updated

    def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
        import bb.utils
        import oe.package
        from collections import OrderedDict

        if 'buildsystem' in handled:
            return False

        def read_package_json(fn):
            with open(fn, 'r', errors='surrogateescape') as f:
                return json.loads(f.read())

        files = RecipeHandler.checkfiles(srctree, ['package.json'])
        if files:
            d = bb.data.createCopy(tinfoil.config_data)
            npm_bindir = self._ensure_npm()
            if not npm_bindir:
                sys.exit(14)
            d.prependVar('PATH', '%s:' % npm_bindir)

            data = read_package_json(files[0])
            if 'name' in data and 'version' in data:
                extravalues['PN'] = data['name']
                extravalues['PV'] = data['version']
                classes.append('npm')
                handled.append('buildsystem')
                if 'description' in data:
                    extravalues['SUMMARY'] = data['description']
                if 'homepage' in data:
                    extravalues['HOMEPAGE'] = data['homepage']

                fetchdev = extravalues['fetchdev'] or None
                deps, optdeps, devdeps = self.get_npm_package_dependencies(data, fetchdev)
                self._handle_dependencies(d, deps, optdeps, devdeps, lines_before, srctree)

                # Shrinkwrap
                localfilesdir = tempfile.mkdtemp(prefix='recipetool-npm')
                self._shrinkwrap(srctree, localfilesdir, extravalues, lines_before, d)

                # Lockdown
                self._lockdown(srctree, localfilesdir, extravalues, lines_before, d)

                # Split each npm module out to is own package
                npmpackages = oe.package.npm_split_package_dirs(srctree)
                licvalues = None
                for item in handled:
                    if isinstance(item, tuple):
                        if item[0] == 'license':
                            licvalues = item[1]
                            break
                if not licvalues:
                    licvalues = handle_license_vars(srctree, lines_before, handled, extravalues, d)
                if licvalues:
                    # Augment the license list with information we have in the packages
                    licenses = {}
                    license = self._handle_license(data)
                    if license:
                        licenses['${PN}'] = license
                    for pkgname, pkgitem in npmpackages.items():
                        _, pdata = pkgitem
                        license = self._handle_license(pdata)
                        if license:
                            licenses[pkgname] = license
                    # Now write out the package-specific license values
                    # We need to strip out the json data dicts for this since split_pkg_licenses
                    # isn't expecting it
                    packages = OrderedDict((x,y[0]) for x,y in npmpackages.items())
                    packages['${PN}'] = ''
                    pkglicenses = split_pkg_licenses(licvalues, packages, lines_after, licenses)
                    all_licenses = list(set([item.replace('_', ' ') for pkglicense in pkglicenses.values() for item in pkglicense]))
                    if '&' in all_licenses:
                        all_licenses.remove('&')
                    extravalues['LICENSE'] = ' & '.join(all_licenses)

                # Need to move S setting after inherit npm
                for i, line in enumerate(lines_before):
                    if line.startswith('S ='):
                        lines_before.pop(i)
                        lines_after.insert(0, '# Must be set after inherit npm since that itself sets S')
                        lines_after.insert(1, line)
                        break

                return True

        return False

    # FIXME this is duplicated from lib/bb/fetch2/npm.py
    def _parse_view(self, output):
        '''
        Parse the output of npm view --json; the last JSON result
        is assumed to be the one that we're interested in.
        '''
        pdata = None
        outdeps = {}
        datalines = []
        bracelevel = 0
        for line in output.splitlines():
            if bracelevel:
                datalines.append(line)
            elif '{' in line:
                datalines = []
                datalines.append(line)
            bracelevel = bracelevel + line.count('{') - line.count('}')
        if datalines:
            pdata = json.loads('\n'.join(datalines))
        return pdata

    # FIXME this is effectively duplicated from lib/bb/fetch2/npm.py
    # (split out from _getdependencies())
    def get_npm_data(self, pkg, version, d):
        import bb.fetch2
        pkgfullname = pkg
        if version != '*' and not '/' in version:
            pkgfullname += "@'%s'" % version
        logger.debug(2, "Calling getdeps on %s" % pkg)
        runenv = dict(os.environ, PATH=d.getVar('PATH'))
        fetchcmd = "npm view %s --json" % pkgfullname
        output, _ = bb.process.run(fetchcmd, stderr=subprocess.STDOUT, env=runenv, shell=True)
        data = self._parse_view(output)
        return data

    # FIXME this is effectively duplicated from lib/bb/fetch2/npm.py
    # (split out from _getdependencies())
    def get_npm_package_dependencies(self, pdata, fetchdev):
        dependencies = pdata.get('dependencies', {})
        optionalDependencies = pdata.get('optionalDependencies', {})
        dependencies.update(optionalDependencies)
        if fetchdev:
            devDependencies = pdata.get('devDependencies', {})
            dependencies.update(devDependencies)
        else:
            devDependencies = {}
        depsfound = {}
        optdepsfound = {}
        devdepsfound = {}
        for dep in dependencies:
            if dep in optionalDependencies:
                optdepsfound[dep] = dependencies[dep]
            elif dep in devDependencies:
                devdepsfound[dep] = dependencies[dep]
            else:
                depsfound[dep] = dependencies[dep]
        return depsfound, optdepsfound, devdepsfound

    # FIXME this is effectively duplicated from lib/bb/fetch2/npm.py
    # (split out from _getdependencies())
    def check_npm_optional_dependency(self, pdata):
        pkg_os = pdata.get('os', None)
        if pkg_os:
            if not isinstance(pkg_os, list):
                pkg_os = [pkg_os]
            blacklist = False
            for item in pkg_os:
                if item.startswith('!'):
                    blacklist = True
                    break
            if (not blacklist and 'linux' not in pkg_os) or '!linux' in pkg_os:
                pkg = pdata.get('name', 'Unnamed package')
                logger.debug(2, "Skipping %s since it's incompatible with Linux" % pkg)
                return False
        return True


def register_recipe_handlers(handlers):
    handlers.append((NpmRecipeHandler(), 60))