summaryrefslogtreecommitdiffstats
path: root/scripts/lib/devtool/deploy.py
blob: e14a5874177e71bc1dfb73f1210adf38747a7abd (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
# Development tool - deploy/undeploy command plugin
#
# Copyright (C) 2014-2016 Intel Corporation
#
# SPDX-License-Identifier: GPL-2.0-only
#
"""Devtool plugin containing the deploy subcommands"""

import logging
import os
import shutil
import subprocess
import tempfile

import bb.utils
import argparse_oe
import oe.types

from devtool import exec_fakeroot, setup_tinfoil, check_workspace_recipe, DevtoolError

logger = logging.getLogger('devtool')

deploylist_path = '/.devtool'

def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=False, nopreserve=False, nocheckspace=False):
    """
    Prepare a shell script for running on the target to
    deploy/undeploy files. We have to be careful what we put in this
    script - only commands that are likely to be available on the
    target are suitable (the target might be constrained, e.g. using
    busybox rather than bash with coreutils).
    """
    lines = []
    lines.append('#!/bin/sh')
    lines.append('set -e')
    if undeployall:
        # Yes, I know this is crude - but it does work
        lines.append('for entry in %s/*.list; do' % deploylist_path)
        lines.append('[ ! -f $entry ] && exit')
        lines.append('set `basename $entry | sed "s/.list//"`')
    if dryrun:
        if not deploy:
            lines.append('echo "Previously deployed files for $1:"')
    lines.append('manifest="%s/$1.list"' % deploylist_path)
    lines.append('preservedir="%s/$1.preserve"' % deploylist_path)
    lines.append('if [ -f $manifest ] ; then')
    # Read manifest in reverse and delete files / remove empty dirs
    lines.append('    sed \'1!G;h;$!d\' $manifest | while read file')
    lines.append('    do')
    if dryrun:
        lines.append('        if [ ! -d $file ] ; then')
        lines.append('            echo $file')
        lines.append('        fi')
    else:
        lines.append('        if [ -d $file ] ; then')
        # Avoid deleting a preserved directory in case it has special perms
        lines.append('            if [ ! -d $preservedir/$file ] ; then')
        lines.append('                rmdir $file > /dev/null 2>&1 || true')
        lines.append('            fi')
        lines.append('        else')
        lines.append('            rm -f $file')
        lines.append('        fi')
    lines.append('    done')
    if not dryrun:
        lines.append('    rm $manifest')
    if not deploy and not dryrun:
        # May as well remove all traces
        lines.append('    rmdir `dirname $manifest` > /dev/null 2>&1 || true')
    lines.append('fi')

    if deploy:
        if not nocheckspace:
            # Check for available space
            # FIXME This doesn't take into account files spread across multiple
            # partitions, but doing that is non-trivial
            # Find the part of the destination path that exists
            lines.append('checkpath="$2"')
            lines.append('while [ "$checkpath" != "/" ] && [ ! -e $checkpath ]')
            lines.append('do')
            lines.append('    checkpath=`dirname "$checkpath"`')
            lines.append('done')
            lines.append(r'freespace=$(df -P $checkpath | sed -nre "s/^(\S+\s+){3}([0-9]+).*/\2/p")')
            # First line of the file is the total space
            lines.append('total=`head -n1 $3`')
            lines.append('if [ $total -gt $freespace ] ; then')
            lines.append('    echo "ERROR: insufficient space on target (available ${freespace}, needed ${total})"')
            lines.append('    exit 1')
            lines.append('fi')
        if not nopreserve:
            # Preserve any files that exist. Note that this will add to the
            # preserved list with successive deployments if the list of files
            # deployed changes, but because we've deleted any previously
            # deployed files at this point it will never preserve anything
            # that was deployed, only files that existed prior to any deploying
            # (which makes the most sense)
            lines.append('cat $3 | sed "1d" | while read file fsize')
            lines.append('do')
            lines.append('    if [ -e $file ] ; then')
            lines.append('    dest="$preservedir/$file"')
            lines.append('    mkdir -p `dirname $dest`')
            lines.append('    mv $file $dest')
            lines.append('    fi')
            lines.append('done')
            lines.append('rm $3')
        lines.append('mkdir -p `dirname $manifest`')
        lines.append('mkdir -p $2')
        if verbose:
            lines.append('    tar xv -C $2 -f - | tee $manifest')
        else:
            lines.append('    tar xv -C $2 -f - > $manifest')
        lines.append('sed -i "s!^./!$2!" $manifest')
    elif not dryrun:
        # Put any preserved files back
        lines.append('if [ -d $preservedir ] ; then')
        lines.append('    cd $preservedir')
        # find from busybox might not have -exec, so we don't use that
        lines.append('    find . -type f | while read file')
        lines.append('    do')
        lines.append('        mv $file /$file')
        lines.append('    done')
        lines.append('    cd /')
        lines.append('    rm -rf $preservedir')
        lines.append('fi')

    if undeployall:
        if not dryrun:
            lines.append('echo "NOTE: Successfully undeployed $1"')
        lines.append('done')

    # Delete the script itself
    lines.append('rm $0')
    lines.append('')

    return '\n'.join(lines)



def deploy(args, config, basepath, workspace):
    """Entry point for the devtool 'deploy' subcommand"""
    import math
    import oe.recipeutils
    import oe.package

    check_workspace_recipe(workspace, args.recipename, checksrc=False)

    try:
        host, destdir = args.target.split(':')
    except ValueError:
        destdir = '/'
    else:
        args.target = host
    if not destdir.endswith('/'):
        destdir += '/'

    tinfoil = setup_tinfoil(basepath=basepath)
    try:
        try:
            rd = tinfoil.parse_recipe(args.recipename)
        except Exception as e:
            raise DevtoolError('Exception parsing recipe %s: %s' %
                            (args.recipename, e))
        recipe_outdir = rd.getVar('D')
        if not os.path.exists(recipe_outdir) or not os.listdir(recipe_outdir):
            raise DevtoolError('No files to deploy - have you built the %s '
                            'recipe? If so, the install step has not installed '
                            'any files.' % args.recipename)

        if args.strip and not args.dry_run:
            # Fakeroot copy to new destination
            srcdir = recipe_outdir
            recipe_outdir = os.path.join(rd.getVar('WORKDIR'), 'devtool-deploy-target-stripped')
            if os.path.isdir(recipe_outdir):
                exec_fakeroot(rd, "rm -rf %s" % recipe_outdir, shell=True)
            exec_fakeroot(rd, "cp -af %s %s" % (os.path.join(srcdir, '.'), recipe_outdir), shell=True)
            os.environ['PATH'] = ':'.join([os.environ['PATH'], rd.getVar('PATH') or ''])
            oe.package.strip_execs(args.recipename, recipe_outdir, rd.getVar('STRIP'), rd.getVar('libdir'),
                        rd.getVar('base_libdir'), rd)

        filelist = []
        inodes = set({})
        ftotalsize = 0
        for root, _, files in os.walk(recipe_outdir):
            for fn in files:
                fstat = os.lstat(os.path.join(root, fn))
                # Get the size in kiB (since we'll be comparing it to the output of du -k)
                # MUST use lstat() here not stat() or getfilesize() since we don't want to
                # dereference symlinks
                if fstat.st_ino in inodes:
                    fsize = 0
                else:
                    fsize = int(math.ceil(float(fstat.st_size)/1024))
                inodes.add(fstat.st_ino)
                ftotalsize += fsize
                # The path as it would appear on the target
                fpath = os.path.join(destdir, os.path.relpath(root, recipe_outdir), fn)
                filelist.append((fpath, fsize))

        if args.dry_run:
            print('Files to be deployed for %s on target %s:' % (args.recipename, args.target))
            for item, _ in filelist:
                print('  %s' % item)
            return 0

        extraoptions = ''
        if args.no_host_check:
            extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
        if not args.show_status:
            extraoptions += ' -q'

        scp_sshexec = ''
        ssh_sshexec = 'ssh'
        if args.ssh_exec:
            scp_sshexec = "-S %s" % args.ssh_exec
            ssh_sshexec = args.ssh_exec
        scp_port = ''
        ssh_port = ''
        if args.port:
            scp_port = "-P %s" % args.port
            ssh_port = "-p %s" % args.port

        if args.key:
            extraoptions += ' -i %s' % args.key

        # In order to delete previously deployed files and have the manifest file on
        # the target, we write out a shell script and then copy it to the target
        # so we can then run it (piping tar output to it).
        # (We cannot use scp here, because it doesn't preserve symlinks.)
        tmpdir = tempfile.mkdtemp(prefix='devtool')
        try:
            tmpscript = '/tmp/devtool_deploy.sh'
            tmpfilelist = os.path.join(os.path.dirname(tmpscript), 'devtool_deploy.list')
            shellscript = _prepare_remote_script(deploy=True,
                                                verbose=args.show_status,
                                                nopreserve=args.no_preserve,
                                                nocheckspace=args.no_check_space)
            # Write out the script to a file
            with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f:
                f.write(shellscript)
            # Write out the file list
            with open(os.path.join(tmpdir, os.path.basename(tmpfilelist)), 'w') as f:
                f.write('%d\n' % ftotalsize)
                for fpath, fsize in filelist:
                    f.write('%s %d\n' % (fpath, fsize))
            # Copy them to the target
            ret = subprocess.call("scp %s %s %s %s/* %s:%s" % (scp_sshexec, scp_port, extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True)
            if ret != 0:
                raise DevtoolError('Failed to copy script to %s - rerun with -s to '
                                'get a complete error message' % args.target)
        finally:
            shutil.rmtree(tmpdir)

        # Now run the script
        ret = exec_fakeroot(rd, 'tar cf - . | %s  %s %s %s \'sh %s %s %s %s\'' % (ssh_sshexec, ssh_port, extraoptions, args.target, tmpscript, args.recipename, destdir, tmpfilelist), cwd=recipe_outdir, shell=True)
        if ret != 0:
            raise DevtoolError('Deploy failed - rerun with -s to get a complete '
                            'error message')

        logger.info('Successfully deployed %s' % recipe_outdir)

        files_list = []
        for root, _, files in os.walk(recipe_outdir):
            for filename in files:
                filename = os.path.relpath(os.path.join(root, filename), recipe_outdir)
                files_list.append(os.path.join(destdir, filename))
    finally:
        tinfoil.shutdown()

    return 0

def undeploy(args, config, basepath, workspace):
    """Entry point for the devtool 'undeploy' subcommand"""
    if args.all and args.recipename:
        raise argparse_oe.ArgumentUsageError('Cannot specify -a/--all with a recipe name', 'undeploy-target')
    elif not args.recipename and not args.all:
        raise argparse_oe.ArgumentUsageError('If you don\'t specify a recipe, you must specify -a/--all', 'undeploy-target')

    extraoptions = ''
    if args.no_host_check:
        extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
    if not args.show_status:
        extraoptions += ' -q'

    scp_sshexec = ''
    ssh_sshexec = 'ssh'
    if args.ssh_exec:
        scp_sshexec = "-S %s" % args.ssh_exec
        ssh_sshexec = args.ssh_exec
    scp_port = ''
    ssh_port = ''
    if args.port:
        scp_port = "-P %s" % args.port
        ssh_port = "-p %s" % args.port

    args.target = args.target.split(':')[0]

    tmpdir = tempfile.mkdtemp(prefix='devtool')
    try:
        tmpscript = '/tmp/devtool_undeploy.sh'
        shellscript = _prepare_remote_script(deploy=False, dryrun=args.dry_run, undeployall=args.all)
        # Write out the script to a file
        with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f:
            f.write(shellscript)
        # Copy it to the target
        ret = subprocess.call("scp %s %s %s %s/* %s:%s" % (scp_sshexec, scp_port, extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True)
        if ret != 0:
            raise DevtoolError('Failed to copy script to %s - rerun with -s to '
                                'get a complete error message' % args.target)
    finally:
        shutil.rmtree(tmpdir)

    # Now run the script
    ret = subprocess.call('%s %s %s %s \'sh %s %s\'' % (ssh_sshexec, ssh_port, extraoptions, args.target, tmpscript, args.recipename), shell=True)
    if ret != 0:
        raise DevtoolError('Undeploy failed - rerun with -s to get a complete '
                           'error message')

    if not args.all and not args.dry_run:
        logger.info('Successfully undeployed %s' % args.recipename)
    return 0


def register_commands(subparsers, context):
    """Register devtool subcommands from the deploy plugin"""

    parser_deploy = subparsers.add_parser('deploy-target',
                                          help='Deploy recipe output files to live target machine',
                                          description='Deploys a recipe\'s build output (i.e. the output of the do_install task) to a live target machine over ssh. By default, any existing files will be preserved instead of being overwritten and will be restored if you run devtool undeploy-target. Note: this only deploys the recipe itself and not any runtime dependencies, so it is assumed that those have been installed on the target beforehand.',
                                          group='testbuild')
    parser_deploy.add_argument('recipename', help='Recipe to deploy')
    parser_deploy.add_argument('target', help='Live target machine running an ssh server: user@hostname[:destdir]')
    parser_deploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true')
    parser_deploy.add_argument('-s', '--show-status', help='Show progress/status output', action='store_true')
    parser_deploy.add_argument('-n', '--dry-run', help='List files to be deployed only', action='store_true')
    parser_deploy.add_argument('-p', '--no-preserve', help='Do not preserve existing files', action='store_true')
    parser_deploy.add_argument('--no-check-space', help='Do not check for available space before deploying', action='store_true')
    parser_deploy.add_argument('-e', '--ssh-exec', help='Executable to use in place of ssh')
    parser_deploy.add_argument('-P', '--port', help='Specify port to use for connection to the target')
    parser_deploy.add_argument('-I', '--key',
                               help='Specify ssh private key for connection to the target')

    strip_opts = parser_deploy.add_mutually_exclusive_group(required=False)
    strip_opts.add_argument('-S', '--strip',
                               help='Strip executables prior to deploying (default: %(default)s). '
                                    'The default value of this option can be controlled by setting the strip option in the [Deploy] section to True or False.',
                               default=oe.types.boolean(context.config.get('Deploy', 'strip', default='0')),
                               action='store_true')
    strip_opts.add_argument('--no-strip', help='Do not strip executables prior to deploy', dest='strip', action='store_false')

    parser_deploy.set_defaults(func=deploy)

    parser_undeploy = subparsers.add_parser('undeploy-target',
                                            help='Undeploy recipe output files in live target machine',
                                            description='Un-deploys recipe output files previously deployed to a live target machine by devtool deploy-target.',
                                            group='testbuild')
    parser_undeploy.add_argument('recipename', help='Recipe to undeploy (if not using -a/--all)', nargs='?')
    parser_undeploy.add_argument('target', help='Live target machine running an ssh server: user@hostname')
    parser_undeploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true')
    parser_undeploy.add_argument('-s', '--show-status', help='Show progress/status output', action='store_true')
    parser_undeploy.add_argument('-a', '--all', help='Undeploy all recipes deployed on the target', action='store_true')
    parser_undeploy.add_argument('-n', '--dry-run', help='List files to be undeployed only', action='store_true')
    parser_undeploy.add_argument('-e', '--ssh-exec', help='Executable to use in place of ssh')
    parser_undeploy.add_argument('-P', '--port', help='Specify port to use for connection to the target')
    parser_undeploy.add_argument('-I', '--key',
                               help='Specify ssh private key for connection to the target')

    parser_undeploy.set_defaults(func=undeploy)