diff options
| -rwxr-xr-x | bin/acme/patcher.json | 16 | ||||
| -rwxr-xr-x | bin/acme/patcher/inplace/bin/srt | 302 | ||||
| -rwxr-xr-x | bin/common/srtool_patcher.py | 190 |
3 files changed, 470 insertions, 38 deletions
diff --git a/bin/acme/patcher.json b/bin/acme/patcher.json index 365990fa..03332d0e 100755 --- a/bin/acme/patcher.json +++ b/bin/acme/patcher.json @@ -12,9 +12,21 @@ }, { "original" : "bin/srt", - "custom" : "bin/srt", + "custom" : "bin/acme/patcher/inplace/bin/srt", "patch" : "", - "options" : "DISABLE" + "options" : "INPLACE DISABLE" + } + ], + "documentation" : [ + { + "help_original" : "the location of the original mainline file", + "help_custom" : "the location of the derived and customized file", + "help_original_INPLACE" : "In the INPLACE mode, this is the location of the mainline file that has been customized", + "help_custom_INPLACE" : "In the INPLACE mode, this is the stash location of the customized file", + "help_patch" : "optional location of extracted patch file, default is '$patcher_dir/$filename.patch'", + "help_options" : "When empty, indicates the default workflow of a custom file in custom app directory derived from a mainline template file (e.g. bin/common/srtool_jira_template.py)", + "help_options_INPLACE" : "Add the 'INPLACE' key if the file is patched in place in the mainline code", + "help_options_DISABLE" : "Add the 'DISABLE' key to make this mapping inactive" } ] } diff --git a/bin/acme/patcher/inplace/bin/srt b/bin/acme/patcher/inplace/bin/srt new file mode 100755 index 00000000..eaa7cd31 --- /dev/null +++ b/bin/acme/patcher/inplace/bin/srt @@ -0,0 +1,302 @@ +#!/bin/bash + +# SRTool - shell script to start "Security Response Tool" + +# Copyright (C) 2013-2015 Intel Corp. +# Copyright (C) 2018 Wind River Systems + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see http://www.gnu.org/licenses/. + +### ACME_EXTENSION_BEGIN ### +echo "Welcome to the SRTool, ACME Edition!" +echo "" +### ACME_EXTENSION_END ### + +HELP=" +Usage: source srt start|stop [webport=<address:port>] + Optional arguments: + [webport] Set the SRTool server port (default: localhost:8000) + [noautoupdate] Disable the auto update server +" + +databaseCheck() +{ + retval=0 + # you can always add a superuser later via + # ../srt/manage.py createsuperuser --username=<ME> + $MANAGE migrate --noinput || retval=1 + + if [ $retval -eq 1 ] ; then + echo "Failed migrations, aborting system start" 1>&2 + return $retval + fi + $MANAGE checksettings --traceback || retval=1 + + if [ $retval -eq 1 ]; then + printf "\nError while checking settings; aborting\n" + return $retval + fi + + return $retval +} + +get_srt_env_settings() { + mainapp="yp" + # Apply all shell settings except default app 'yp' + # Only look in directories with proper 'datasource.json' files + for envscript in $(find ./bin -name "datasource.json") ; do + envscript=${envscript/datasource.json/srtool_env.sh} + if [ -f "$envscript" -a "$envscript" = "${envscript/bin\/yp/}" ] ; then + . $envscript + fi + done + # if no main app, default to 'yp' + if [ -z "$SRT_MAIN_APP" ] ; then + . ./bin/yp/srtool_env.sh + fi + echo "SRT_MAIN_APP=$SRT_MAIN_APP" +} + +webserverKillAll() +{ + local pidfile + for pidfile in ${SRT_BASE_DIR}/.srtmain.pid ; do + if [ -f ${pidfile} ] ; then + pid=`cat ${pidfile}` + while kill -0 $pid 2>/dev/null; do + echo "KILL:$pid" + kill -SIGTERM -9 $pid 2>/dev/null + sleep 1 + done + rm ${pidfile} + fi + done + + # Stop the Update app + if [ 0 -eq $no_auto_update ] ; then + ./bin/common/srtool_update.py --cron-stop + fi +} + +webserverStartAll() +{ + # do not start if srtmain points to a valid process + if ! cat "${SRT_BASE_DIR}/.srtmain.pid" 2>/dev/null | xargs -I{} kill -0 {} ; then + retval=1 + rm "${SRT_BASE_DIR}/.srtmain.pid" + fi + + retval=0 + + # check the database + databaseCheck || return 1 + + echo "Starting SRTool webserver..." + + $MANAGE runserver --noreload "$ADDR_PORT" \ + </dev/null >>${SRT_BASE_DIR}/srt_web.log 2>&1 \ + & echo $! >${SRT_BASE_DIR}/.srtmain.pid + + sleep 1 + + if ! cat "${SRT_BASE_DIR}/.srtmain.pid" | xargs -I{} kill -0 {} ; then + retval=1 + rm "${SRT_BASE_DIR}/.srtmain.pid" + echo "SRTool webserver NOT STARTED" + else + echo "SRTool webserver started at http://$ADDR_PORT" + fi + + # Start the Update app + if [ 0 -eq $no_auto_update ] ; then + ./bin/common/srtool_update.py --cron-start > /dev/null 2>&1 & + echo "SRTool update service started at PID $!" + fi + return $retval +} + +INSTOPSYSTEM=0 + +# define the stop command +stop_system() +{ + # prevent reentry + if [ $INSTOPSYSTEM -eq 1 ] ; then return; fi + INSTOPSYSTEM=1 + webserverKillAll + # unset exported variables + unset SRT_BASE_DIR + trap - SIGHUP + #trap - SIGCHLD + INSTOPSYSTEM=0 +} + +verify_prereq() { + # Quick check for Python3 + if [ -z "$(which python3)" ] ; then + echo "ERROR: missing 'python3' host package" + return 2 + fi + if [ -z "$(which sqlite3)" ] ; then + echo "ERROR: missing 'sqlite3' host package" + return 2 + fi + + # Verify Django version + reqfile=$(python3 -c "import os; print(os.path.realpath('$SRT_BASE_DIR/bin/srtool-requirements.txt'))") + exp='s/Django\([><=]\+\)\([^,]\+\),\([><=]\+\)\(.\+\)/' + # expand version parts to 2 digits to support 1.10.x > 1.8 + # (note:helper functions hard to insert in-line) + exp=$exp'import sys,django;' + # Allow for development versions like '2.2.dev20181217100344' + exp=$exp'version=["%02d" % int(n) for n in django.get_version().replace("dev","").split(".")];' + exp=$exp'vmin=["%02d" % int(n) for n in "\2".split(".")];' + exp=$exp'vmax=["%02d" % int(n) for n in "\4".split(".")];' + exp=$exp'sys.exit(not (version \1 vmin and version \3 vmax))' + exp=$exp'/p' + if ! sed -n "$exp" $reqfile | python3 - ; then + req=`grep ^Django $reqfile` + echo "This program needs $req" + echo "Please install with pip3 install -r $reqfile" + return 2 + fi + + return 0 +} + +# read command line parameters +if [ -n "$BASH_SOURCE" ] ; then + SRT=${BASH_SOURCE} +elif [ -n "$ZSH_NAME" ] ; then + SRT=${(%):-%x} +else + SRT=$0 +fi + +# set up base paths and definitions +export SRT_BASE_DIR=$(dirname $SRT) +SRT_BASE_DIR=$(readlink -f $SRT_BASE_DIR) +SRT_BASE_DIR=$(dirname $SRT_BASE_DIR) +MANAGE="python3 $SRT_BASE_DIR/lib/manage.py" + +# Fetch the datasource environent settings +get_srt_env_settings + +# insure basic directories are present +mkdir -p $SRT_BASE_DIR/data +mkdir -p $SRT_BASE_DIR/data/cache +mkdir -p $SRT_BASE_DIR/update_logs +touch $SRT_BASE_DIR/update_logs/master_log.txt + +ADDR_PORT="localhost:8000" +unset CMD +manage_cmd="" +if [ "1" = "$SRT_SKIP_AUTOUPDATE" ] ; then + no_auto_update=1 +else + no_auto_update=0 +fi +for param in $*; do + case $param in + start ) + CMD=$param + ;; + stop ) + CMD=$param + ;; + manage ) + CMD=$param + ;; + webport=*) + ADDR_PORT="${param#*=}" + # Split the addr:port string + ADDR=`echo $ADDR_PORT | cut -f 1 -d ':'` + PORT=`echo $ADDR_PORT | cut -f 2 -d ':'` + # If only a port has been specified then set address to localhost. + if [ $ADDR = $PORT ] ; then + ADDR_PORT="localhost:$PORT" + fi + ;; + noautoupdate ) + no_auto_update=1 + ;; + --help) + echo "$HELP" + exit 0 + ;; + *) + if [ "manage" == "$CMD" ] ; then + cd $SRT_BASE_DIR/lib + manage_cmd="$manage_cmd $param" + else + echo "$HELP" + exit 1 + fi + ;; + + esac +done + +verify_prereq || exit 1 + +# this defines the dir SRTool will use for +# 1) the sqlite db if that is being used. +# 2) pid's we need to clean up on exit/shutdown + +# Determine the action. If specified by arguments, fine, if not, toggle it +if [ "$CMD" = "start" ] ; then + if [ -n "$BBSERVER" ]; then + echo " SRT is already running. Exiting..." + exit 1 + fi +elif [ "$CMD" = "" ]; then + echo "No command specified" + echo "$HELP" + exit 1 +fi + +echo "The system will $CMD." + +# Execute the commands +case $CMD in + start ) + # check if addr:port is not in use + if [ "$CMD" == 'start' ] ; then + $MANAGE checksocket "$ADDR_PORT" || exit 1 + fi + + if ! webserverStartAll; then + echo "Failed ${CMD}." + exit 4 + fi + # create working directories for srtool + mkdir -p $SRT_BASE_DIR/update_logs + mkdir -p $SRT_BASE_DIR/backups + mkdir -p $SRT_BASE_DIR/reports + # set fail safe stop system on terminal exit + trap stop_system SIGHUP + echo "Successful ${CMD}." + exit 0 + ;; + stop ) + stop_system + echo "Successful ${CMD}." + ;; + manage ) + $MANAGE $manage_cmd + ;; + + +esac + diff --git a/bin/common/srtool_patcher.py b/bin/common/srtool_patcher.py index ccf93060..bda941fc 100755 --- a/bin/common/srtool_patcher.py +++ b/bin/common/srtool_patcher.py @@ -59,12 +59,17 @@ import json # Setup: verbose = False +INPLACE_TAG = 'INPLACE' +DISABLE_TAG = 'DISABLE' ################################# # extract_custom_patch # +# Extract a clean version of the file without the the custom sections, +# and generate a patch file of those sections +# -def extract_custom_patch(custom_file,clean_file,patch_file,label,patcher_dir): +def extract_custom_patch(custom_file,clean_file,patch_file,label,patcher_dir,options): tag_begin = "%s_EXTENSION_BEGIN" % label tag_end = "%s_EXTENSION_END" % label ret = 0 @@ -108,18 +113,32 @@ def extract_custom_patch(custom_file,clean_file,patch_file,label,patcher_dir): # Did we end cleanly? if state != "find": print("ERROR: START not STOPPED (%s)" % state) - os.system("diff -u %s %s > %s" % (clean_file,custom_file,patch_file)) - print("Custom File: %s" % (custom_file)) - print("Clean File: %s" % (clean_file)) - print("Patch File: %s" % (patch_file)) + if INPLACE_TAG in options: + cmd = "git diff %s > %s" % (custom_file,patch_file) + print(cmd) + os.system(cmd) + print("In-place File: %s" % (clean_file)) + else: + os.system("diff -u %s %s > %s" % (clean_file,custom_file,patch_file)) + print("Custom File: %s" % (custom_file)) + print("Clean File: %s" % (clean_file)) + print("Patch File: %s" % (patch_file)) return(ret,clean_file,patch_file) ################################# # merge_original # +# Merge any changes in the original upstream file into the +# custom file and reapply the custom patches, so that general upstream +# changes be be included in the local custom file +# + +def merge_original(custom_file,original_file,patch_file,label,patcher_dir,options): + # Skip if in-place + if INPLACE_TAG in options: + return 0 -def merge_original(custom_file,original_file,patch_file,label,patcher_dir): - ret,clean_file,patch_file = extract_custom_patch(custom_file,'',patch_file,label,patcher_dir) + ret,clean_file,patch_file = extract_custom_patch(custom_file,'',patch_file,label,patcher_dir,options) custom_saved = os.path.join(os.path.dirname(patch_file),os.path.basename(custom_file) + '.saved') if 0 == ret: print("* Preserving custom file as '%s'" % custom_saved) @@ -143,9 +162,16 @@ def merge_original(custom_file,original_file,patch_file,label,patcher_dir): ################################# # merge_custom # +# Merge any changes in the normal non-custom sections of the custom file back into +# original file, so that general fixes can be shared upstream +# + +def merge_custom(custom_file,original_file,patch_file,label,patcher_dir,options): + # Skip if in-place + if INPLACE_TAG in options: + return 0 -def merge_custom(custom_file,original_file,patch_file,label,patcher_dir): - ret,clean_file,patch_file = extract_custom_patch(custom_file,'',patch_file,label,patcher_dir) + ret,clean_file,patch_file = extract_custom_patch(custom_file,'',patch_file,label,patcher_dir,options) if 0 == ret: print("== Copying clean version of custom file to '%s' ===" % original_file) cmd = "cp -f %s %s" % (clean_file, original_file) @@ -156,14 +182,102 @@ def merge_custom(custom_file,original_file,patch_file,label,patcher_dir): ################################# # diff_original # +# Show how the general sections of the custom file compare to the +# original file, so that the user can (a) see might be shared upstream, +# and/or (b) insure the custom sections are correct and complete +# -def diff_original(custom_file,original_file,patch_file,label,patcher_dir): - ret,clean_file,patch_file = extract_custom_patch(custom_file,'',patch_file,label,patcher_dir) +def diff_original(custom_file,original_file,patch_file,label,patcher_dir,options): + # Skip if in-place + if INPLACE_TAG in options: + return 0 + + ret,clean_file,patch_file = extract_custom_patch(custom_file,'',patch_file,label,patcher_dir,options) if 0 == ret: print("== DIFF from '%s' to clean version of custom file ===" % original_file) os.system("diff -u %s %s" % (original_file,clean_file)) return(ret) + +################################# +# clean_inplace +# +# Stash the customized general file and replace it with a clean version +# without the custom sections, so that general fixes can be shared upstream +# + +def clean_inplace(custom_file,original_file,patch_file,label,patcher_dir,options): + print("FOO1:%s,%s" % (INPLACE_TAG,options)) + # Skip if not in-place + if not INPLACE_TAG in options: + return 0 + + print("FOO2") + # Only continue if 'original' file has custom patches, in case this + # command is run multiple times by accident + cmd = "grep %s %s" % (label,original_file) + ret = os.system(cmd) + if 0 != ret: + print("ERROR:SKIP: original file '%s' does not have custom tags '%s'" % (original_file,label)) + return ret + + print("* Backup customized in-place file") + try: + os.makedirs(os.path.dirname(custom_file)) + except: + pass + cmd = "cp --force %s %s" % (original_file,custom_file) + print(cmd) + os.system(cmd) + + print("* Extract patch file (via 'git diff')") + clean_file = custom_file + '.clean' + patch_file = custom_file + '.patch' + ret,clean_file,patch_file = extract_custom_patch(original_file,clean_file,patch_file,label,'',options) + if 0 != ret: + return(ret) + + print("* Copy cleaned version back to in-place location") + cmd = "cp --force %s %s" % (clean_file,original_file) + print(cmd) + ret = os.system(cmd) + return(ret) + +################################# +# patch_inplace +# +# Re-apply the custom patches on top of the general file, so that general fixes can be adopted +# from upstream together with the customized patches applied on top +# + +def patch_inplace(custom_file,original_file,patch_file,label,patcher_dir,options): + # Skip if not in-place + if not INPLACE_TAG in options: + return 0 + + ret = 0 + patch_file = custom_file + '.patch' + if not os.path.isfile(patch_file): + # No patch file found, do a simple copy-over of custom file + if not os.path.isfile(custom_file): + print("ERROR:SKIP: neither the patch file '%s' nor the custom file '%s' can be found" % (patch_file,custom_file)) + return 1 + print("* No patch file found, copying custom file over in-place file") + cmd = "cp -f %s %s" % (custom_file,original_file) + print(cmd) + ret = os.system(cmd) + else: + print("* Patching the in-place file with custom patch") + cmd = "patch %s %s" % (original_file,patch_file) + print(cmd) + ret = os.system(cmd) + if 0 != ret: + print("* ERROR: Merge failed, restoring previous custom file") + cmd = "cp %s %s" % (custom_saved,custom_file) + print(cmd) + os.system(cmd) + return(ret) + ################################# # load_json_list # @@ -182,7 +296,7 @@ def load_json_list(json_file): patcher_dir = dct['patcher_dir'] if 'patch_set' in dct: for patch in dct['patch_set']: - if 'DISABLE' in patch['options']: + if DISABLE_TAG in patch['options']: continue file_list.append([patch['custom'],patch['original'],patch['patch'],patch['options']]) return file_list,patcher_dir,label @@ -197,13 +311,17 @@ def main(argv): # setup parser = argparse.ArgumentParser(description='srtool_sanity_test.py: SRTool common sanity tests') - parser.add_argument('--merge-original', '-O', action='store_const', const='merge_original', dest='command', help='Copy the (updated) original file, merge the custom patches') - parser.add_argument('--merge-custom', '-C', action='store_const', const='merge_custom', dest='command', help='Copy the (updated) file to the original, without custom patches') + parser.add_argument('--merge-original', '-o', action='store_const', const='merge_original', dest='command', help='Copy the (updated) original file, merge the custom patches') + parser.add_argument('--merge-custom', '-c', action='store_const', const='merge_custom', dest='command', help='Copy the (updated) file to the original, without custom patches') - parser.add_argument('--custom', '-c', help='Custom file') - parser.add_argument('--original', '-o', help='Original file') - parser.add_argument('--label', '-l', help='Custom label tag (default="CUSTOM")') - parser.add_argument('--json', '-j', help='Use JSON file for file list') + parser.add_argument('--clean-inplace', '-i', action='store_const', const='clean_inplace', dest='command', help='Stash the customized file, leaving the clean mainline version inline') + parser.add_argument('--patch-inplace', '-I', action='store_const', const='patch_inplace', dest='command', help='Restore the customized file in place of the mainline version inline') + + parser.add_argument('--json', '-J', help='Use JSON file for file list') + parser.add_argument('--custom', '-C', help='Custom file') + parser.add_argument('--original', '-O', help='Original file') + parser.add_argument('--label', '-L', help='Custom label tag (default="CUSTOM")') + parser.add_argument('--options', '-P', help='Options, for example "INPLACE", "DISABLE"') parser.add_argument('--extract-custom-patch', '-e', action='store_const', const='extract_custom_patch', dest='command', help='Extract a patch of the custom content') parser.add_argument('--patch', '-p', help='Patch file') @@ -224,40 +342,40 @@ def main(argv): validate_file(args.json,"JSON") file_list,patcher_dir,label = load_json_list(args.json) else: - custom_file = '' - if args.custom: - custom_file = args.custom - original_file = '' - if args.original: - original_file = args.original - patch_file = '' - if args.patch: - patch_file = args.patch - options = '' + original_file = args.original if args.original else '' + custom_file = args.custom if args.custom else '' + patch_file = args.patch if args.patch else '' + label = args.label if args.label else 'CUSTOM' + options = args.options if args.options else '' file_list.append([custom_file,original_file,patch_file,options]) - label = 'CUSTOM' - if args.label: - label = args.label patcher_dir = os.path.join(os.path.dirname(custom_file),'patcher') ret = 0 for custom_file,original_file,patch_file,options in file_list: - #print("PATCH_FILE:%s,%s,%s,%s,%s" % (custom_file,original_file,patch_file,options,patcher_dir)) + #print("PATCH_FILE:%s,%s,%s,%s,%s,%s" % (custom_file,original_file,patch_file,options,patcher_dir,options)) if 'merge_original' == args.command: validate_file(custom_file,'Custom') validate_file(original_file,'Original') - ret = merge_original(custom_file,original_file,patch_file,label,patcher_dir) + ret = merge_original(custom_file,original_file,patch_file,label,patcher_dir,options) elif 'merge_custom' == args.command: validate_file(custom_file,'Custom') validate_file(original_file,'Original') - ret = merge_custom(custom_file,original_file,patch_file,label,patcher_dir) + ret = merge_custom(custom_file,original_file,patch_file,label,patcher_dir,options) + elif 'clean_inplace' == args.command: + validate_file(custom_file,'Custom') + validate_file(original_file,'Original') + ret = clean_inplace(custom_file,original_file,patch_file,label,patcher_dir,options) + elif 'patch_inplace' == args.command: + validate_file(custom_file,'Custom') + validate_file(original_file,'Original') + ret = patch_inplace(custom_file,original_file,patch_file,label,patcher_dir,options) elif 'extract_custom_patch' == args.command: validate_file(custom_file,'Custom') - ret,clean_file,patch_file = extract_custom_patch(custom_file,'',patch_file,label,patcher_dir) + ret,clean_file,patch_file = extract_custom_patch(custom_file,'',patch_file,label,patcher_dir,options) elif 'diff_original' == args.command: validate_file(custom_file,'Custom') validate_file(original_file,'Original') - ret = diff_original(custom_file,original_file,patch_file,label,patcher_dir) + ret = diff_original(custom_file,original_file,patch_file,label,patcher_dir,options) else: print("Command not found '%s'" % args.command) ret = 1 |