diff options
Diffstat (limited to 'bin/common/srtool_backup.py')
| -rwxr-xr-x | bin/common/srtool_backup.py | 216 |
1 files changed, 208 insertions, 8 deletions
diff --git a/bin/common/srtool_backup.py b/bin/common/srtool_backup.py index b37e2d08..f065e2f2 100755 --- a/bin/common/srtool_backup.py +++ b/bin/common/srtool_backup.py @@ -5,7 +5,7 @@ # # Security Response Tool Commandline Tool # -# Copyright (C) 2018-2019 Wind River Systems +# Copyright (C) 2018-2021 Wind River Systems # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -27,7 +27,6 @@ import os import sys import argparse -import sqlite3 import json from datetime import datetime, date @@ -35,11 +34,17 @@ from datetime import datetime, date dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) sys.path.insert(0, dir_path) from common.srt_schema import ORM +from common.srtool_sql import * # Setup: verbose = False +force_update = False srtDbName = 'srt.sqlite' +BACKUP_DIR = 'backups' +BACKUP_PREFIX = 'backup_' +BACKUP_LOG_DIR = 'update_logs' +BACKUP_LOG_PREFIX = 'backup_log_' ################################# # Common routines @@ -57,20 +62,35 @@ def _log(msg): f1.close() ################################# +# Set backup database stamp file +# + +def backup_stamp(backup_dir): + if not os.path.isdir(backup_dir): + print("ERROR: no such directory '%s'" % backup_dir) + exit(1) + statinfo = os.stat(os.path.join(backup_dir, 'srt.sqlite')) + mod_timestamp = datetime.fromtimestamp(statinfo.st_mtime) + stamp_str = mod_timestamp.strftime('%Y-%m-%d %H:%M:%S | %A, %B %d %Y') + with open(os.path.join(backup_dir,'timestamp.txt'), 'w') as file: + file.write('%s\n' % stamp_str) + print("* Set Timestamp:%s" % mod_timestamp.strftime('%Y-%m-%d|%H:%M:%S|%A, %B %d %Y')) + +################################# # Backup the database and data files # def backup_db(is_daily): today = datetime.today() weeknum = today.strftime("%W") - weekday = today.isoweekday() + weekday = today.strftime("%A") #today.isoweekday() year = today.strftime("%Y") # Where are we backing up to if is_daily: - backup_dir = os.path.join(script_pathname, "backups/backup_%s" % (weekday)) + backup_dir = os.path.join(script_pathname, "%s/%s%s" % (BACKUP_DIR,BACKUP_PREFIX,weekday)) else: - backup_dir = os.path.join(script_pathname, "backups/backup_%s_%s" % (year,weeknum)) + backup_dir = os.path.join(script_pathname, "%s/%s%s_%s" % (BACKUP_DIR,BACKUP_PREFIX,year,weeknum)) # Make sure directory exists try: os.makedirs(backup_dir) @@ -82,36 +102,199 @@ def backup_db(is_daily): print("*** Backup dir='%s' ***" % backup_dir) print("* Copy database") - cmd = 'cp %s %s' % (os.path.join(script_pathname,srtDbName),os.path.join(script_pathname,backup_dir)) + cmd = 'cp -p %s %s' % (os.path.join(script_pathname,srtDbName),backup_dir) print(cmd) os.system(cmd) # Copy data but skip cache dir (no deep copy) print("* Copy data files") - cmd = 'cp %s/data/* %s/data' % (script_pathname,os.path.join(script_pathname,backup_dir)) + cmd = 'cp -p %s/data/* %s/data' % (script_pathname,backup_dir) print(cmd) os.system(cmd) # Copy attachments print("* Copy attachment files") - cmd = 'cp -r %s/downloads %s' % (script_pathname,os.path.join(script_pathname,backup_dir)) + cmd = 'cp -r -p %s/downloads %s' % (script_pathname,backup_dir) print(cmd) os.system(cmd) + # Set stamp file + backup_stamp(backup_dir) + + # Additional remote backup directory copy + conn = SQL_CONNECT() + cur = conn.cursor() + sql = '''SELECT * FROM orm_srtsetting WHERE name=?''' + setting = SQL_EXECUTE(cur, sql, ('SRT_REMOTE_BACKUP_PATH',)).fetchone() + print("* Check for remote backup (%s) " % str(setting)) + cur.close() + conn.close() + if setting and setting[ORM.SRTSETTING_VALUE].strip(): + remote_backup_dir = setting[ORM.SRTSETTING_VALUE] + if is_daily: + remote_backup_dir = os.path.join(script_pathname, "%s/%s%s" % (remote_backup_dir,BACKUP_PREFIX,weekday)) + else: + remote_backup_dir = os.path.join(script_pathname, "%s/%s%s_%s" % (remote_backup_dir,BACKUP_PREFIX,year,weeknum)) + + print("* Safety copy to remote location '%s'" % remote_backup_dir) + try: + os.makedirs(remote_backup_dir) + except: + # If exists, clean it out + os.system("rm -rf %s/*" % (remote_backup_dir)) + + # Is path a failure? + if not os.path.isdir(remote_backup_dir): + print("ERROR: no such remote backup path '%s'" % remote_backup_dir) + return(-1) + + # Copy the backup directory to the remote location + cmd = 'cp -r -p %s %s' % (backup_dir,remote_backup_dir) + print(cmd) + os.system(cmd) + + os.system('bash -c "echo \"BACKUP:`date`:%s\" >> backup_db.log"' % cmd) + + + +####################################################################### +# backup_logs: +# +# Back up the logs to a daily wheel, and reset the logs +# to empty for the next day. +# If end of week, concatenate the respective daily logs into a +# week-of-year log +# + +def backup_logs(): + today = datetime.today() + date_str = today.strftime("%Y/%m/%d") + weeknum = today.strftime("%W") # 00, 01, ... , 53 + weekday = today.strftime("%A") # Sunday, Monday, ... , Saturday + weekday_num = today.strftime("%w") # 0, 1, ..., 6 + year = today.strftime("%Y") + log_name_list = ('srt_web.log','srt_update.log','srt_dbg.log','update_logs/master_log.txt') + + # Perform the daily back up + backup_day_dir = os.path.join(script_pathname, "%s/%s%s" % (BACKUP_LOG_DIR,BACKUP_LOG_PREFIX,weekday)) + # Make sure directory exists + try: + os.makedirs(backup_day_dir) + except: + # If exists, clean it out + os.system("rm -rf %s/*" % (backup_day_dir)) + pass + os.makedirs(os.path.join(backup_day_dir,'data')) + + print("*** Backup dir[%s]='%s' ***" % (weekday_num,backup_day_dir)) + print("* Copy logs") + for logname in log_name_list: + logname_base = os.path.basename(logname) + # Backup log file + cmd = 'echo "=== Backup: %s ===" > %s' % (date_str,os.path.join(backup_day_dir,logname_base)) + print(cmd) + os.system(cmd) + cmd = 'cat %s | grep -v "/srtgui/mostrecentjobs?format=json" >> %s' % (os.path.join(script_pathname,logname),os.path.join(backup_day_dir,logname_base)) + print(cmd) + os.system(cmd) + # Reset log file to empty + cmd = 'echo "" > %s' % (os.path.join(script_pathname,logname)) + print(cmd) + os.system(cmd) + + # If week's end, perform the weekly backup + if (6 == weekday) or force_update: + backup_week_dir = os.path.join(script_pathname, "%s/%s%s_%s" % (BACKUP_LOG_DIR,BACKUP_LOG_PREFIX,year,weeknum)) + if not os.path.isdir(backup_week_dir): + os.makedirs(backup_week_dir) + day_names = ['Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday'] + concat = '>' + for weekday_num,weekday in enumerate(day_names): + backup_day_dir = os.path.join(script_pathname, "%s/%s%s" % (BACKUP_LOG_DIR,BACKUP_LOG_PREFIX,weekday)) + if os.path.isdir(backup_day_dir): + # Concatinate the respective logs + for logname in log_name_list: + logname_base = os.path.basename(logname) + cmd = 'cat %s %s %s' % (os.path.join(backup_day_dir,logname_base),concat,os.path.join(backup_week_dir,logname_base)) + print(cmd) + os.system(cmd) + concat = '>>' + +####################################################################### +# list +# + +def backup_list(): + def sort_key(elem): + return elem[0]+elem[2] + + stamps = [] + for directory in os.listdir(os.path.join(script_pathname, 'backups')): + prefix = '1Week' if not directory[len(BACKUP_PREFIX)].isalpha() else '2Day' + directory = os.path.join(script_pathname, 'backups', directory) + with open(os.path.join(directory,'timestamp.txt'), 'r') as file: + line = file.read().strip() + #print("DIR=%s,%s" % (directory,line)) + stamps.append([prefix, directory, line]) + + # Add the current database (now) + prefix = '3Now' + directory = script_pathname + statinfo = os.stat(os.path.join(directory, 'srt.sqlite')) + mod_timestamp = datetime.fromtimestamp(statinfo.st_mtime) + stamp_str = mod_timestamp.strftime('%Y-%m-%d %H:%M:%S | %A, %B %d %Y') + stamps.append([prefix, directory, stamp_str]) + + # Sort my time and return + stamps.sort(key=sort_key) + return stamps + +def list(db_list=False): + stamps = backup_list() + for stamp in stamps: + # Insert a separator between the date and the time + stamp[2] = stamp[2].replace(' ','|',1) + if db_list: + print("%s|%s|%s" % (stamp[0][1:],os.path.basename(stamp[1]),stamp[2].replace(' | ','|'))) + else: + snap_date,snap_time,snap_day = stamp[2].split('|') + print("%-4s,%-16s,%s" % (stamp[0][1:],os.path.basename(stamp[1]),stamp[2].replace(' | ','|'))) + +################################# +# Init stamps +# + +def init_stamps(): + stamps = backup_list() + for stamp in stamps: + stamp_prefix, stamp_directory, stamp_line = stamp + backup_stamp(stamp_directory) + ################################# # main loop # + def main(argv): global verbose global cmd_skip global cmd_count + global force_update # setup parser = argparse.ArgumentParser(description='srtool_backup.py: backup the SRTool database') parser.add_argument('--backup-db', '-b', action='store_const', const='backup', dest='command', help='Backup the database, save to year_weeknum dir') parser.add_argument('--backup-db-daily', '-d', action='store_const', const='backup-daily', dest='command', help='Backup the database, save to weekday dir') + parser.add_argument('--backup-logs', '-B', action='store_const', const='backup_logs', dest='command', help='Backup the logs, save to year_weeknum and weekday dir') + + parser.add_argument('--init-stamps', '-I', action='store_const', const='init-stamps', dest='command', help='Initialize the backup directory timestamps') + parser.add_argument('--init-dir-stamp', '-D', dest='init_dir_stamp', help='Initialize a specific backup directory timestamp') + + parser.add_argument('--list-backups', '-l', action='store_const', const='list', dest='command', help='List the backup directory timestamps') + parser.add_argument('--list-backups-db', '-L', action='store_const', const='list-db', dest='command', help='Dump the backup directory timestamps') + parser.add_argument('--force', '-f', action='store_true', dest='force', help='Force the update') + parser.add_argument('--update-skip-history', '-H', action='store_true', dest='update_skip_history', help='Skip history updates') parser.add_argument('--verbose', '-v', action='store_true', dest='verbose', help='Debugging: verbose output') parser.add_argument('--skip', dest='skip', help='Debugging: skip record count') parser.add_argument('--count', dest='count', help='Debugging: short run record count') @@ -121,6 +304,7 @@ def main(argv): master_log = open(os.path.join(script_pathname, "update_logs/master_log.txt"), "a") verbose = args.verbose + force_update = args.force cmd_skip = 0 if None != args.skip: cmd_skip = int(args.skip) @@ -136,6 +320,22 @@ def main(argv): except Exception as e: print ("DATABASE BACKUP FAILED ... %s" % e) master_log.write("SRTOOL:%s:DATABASE BACKUP:\t\t\t\t...\t\t\tFAILED ... %s\n" % (date.today(), e)) + + elif 'backup_logs' == args.command: + try: + backup_logs() + except Exception as e: + print ("Log BACKUP FAILED ... %s" % e) + master_log.write("SRTOOL:%s:LOGS BACKUP:\t\t\t\t...\t\t\tFAILED ... %s\n" % (date.today(), e)) + elif 'list' == args.command: + list() + elif 'list-db' == args.command: + list(True) + elif 'init-stamps' == args.command: + init_stamps() + elif args.init_dir_stamp: + backup_stamp(args.init_dir_stamp) + else: print("Command not found") master_log.close() |