#!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0-only """ Utility to create opkg compatible indexes """ from __future__ import absolute_import from __future__ import print_function import argparse import sys import os import posixpath import re import subprocess import opkg def to_morgue(filename, pkg_dir, verbose): """ Move files to morgue folder """ morgue_dir = pkg_dir + "/morgue" if verbose: sys.stderr.write("Moving " + filename + " to morgue\n") if not os.path.exists(morgue_dir): os.mkdir(morgue_dir) if os.path.exists(pkg_dir + "/" + filename): os.rename(pkg_dir + "/" + filename, morgue_dir + "/" + filename) if os.path.exists(pkg_dir + "/" + filename + ".asc"): os.rename(pkg_dir + "/" + filename + ".asc", morgue_dir + "/" + filename + ".asc") def to_locale(filename, locale, pkg_dir, locales_dir, verbose): """ Move file to locale_dir""" locale_dir = pkg_dir + '/' + locales_dir + '/' + locale + "/" if verbose: sys.stderr.write("Moving " + filename + " to " + locale_dir + "\n") if not os.path.exists(locale_dir): os.mkdir(locale_dir) os.rename(pkg_dir + "/" + filename, locale_dir + "/" + filename) if os.path.exists(pkg_dir + "/" + filename + ".asc"): os.rename(pkg_dir + "/" + filename + ".asc", locale_dir + "/" + filename + ".asc") def main(): """ Script entry point """ stamplist_filename = "Packages.stamps" parser = argparse.ArgumentParser(description='Opkg index creation tool') parser.add_argument('-s', dest='opt_s', default=0, action="store_true", help='Old simulation mode') parser.add_argument('-m', dest='opt_m', action="store_true", help='Archive old packages') parser.add_argument('-a', dest='opt_a', action='store_true', help='Add version information') parser.add_argument('-f', dest='opt_f', action='store_true', help='Include user-defined fields') parser.add_argument('-l', dest='filelist_filename', default=None, help='Packages filelist name') parser.add_argument('-p', dest='packages_filename', default=None, help='Package index filename') parser.add_argument('-r', dest='old_filename', help='Old Package index filename') parser.add_argument('-L', dest='locales_dir', help='Locales dirname') parser.add_argument('-v', dest='verbose', action="store_true", default=0, help='Verbose output') parser.add_argument('--checksum', action='append', dest='checksum', choices=['md5', 'sha256'], help='Select checksum type (default is md5)') parser.add_argument('packagesdir', help='Directory to be indexed') args = parser.parse_args() opt_s = args.opt_s packages_filename = args.packages_filename filelist_filename = args.filelist_filename verbose = args.verbose opt_m = args.opt_m old_filename = args.old_filename locales_dir = args.locales_dir opt_a = args.opt_a opt_f = args.opt_f checksum = args.checksum if args.checksum else ['md5'] pkg_dir = args.packagesdir if packages_filename: stamplist_filename = packages_filename + ".stamps" packages = opkg.Packages() old_pkg_hash = {} if packages_filename and not old_filename and os.path.exists(packages_filename): old_filename = packages_filename pkgs_stamps = {} if old_filename: if verbose: sys.stderr.write("Reading package list from " + old_filename + "\n") old_packages = opkg.Packages() old_packages.read_packages_file(old_filename, opt_f) for k in list(old_packages.packages.keys()): pkg = old_packages.packages[k] old_pkg_hash[pkg.filename] = pkg try: with open(stamplist_filename, "r") as stamplist_filename_hdl: for line in stamplist_filename_hdl: line = line.strip() stamp, filename = line.split(" ", 1) pkgs_stamps[filename] = int(stamp) except IOError: pass if verbose: sys.stderr.write("Reading in all the package info from %s\n" % (pkg_dir, )) files = [] opkg_extensions = ['.ipk', '.opk', '.deb'] for dirpath, _, filenames in os.walk(pkg_dir): for filename in filenames: ext = os.path.splitext(filename)[1] if ext in opkg_extensions: files.append(os.path.join(dirpath, filename)) files.sort() for abspath in files: try: filename = os.path.relpath(abspath, pkg_dir) pkg = None stat = os.stat(abspath) if filename in old_pkg_hash: if filename in pkgs_stamps and int(stat.st_ctime) == pkgs_stamps[filename]: if verbose: sys.stderr.write("Found %s in Packages\n" % (filename,)) pkg = old_pkg_hash[filename] else: sys.stderr.write("Found %s in Packages, but ctime differs - re-reading\n" % (filename,)) if not pkg: if verbose: sys.stderr.write("Reading info for package %s\n" % (filename,)) pkg = opkg.Package(abspath, relpath=pkg_dir, all_fields=opt_f) if opt_a: pkg_key = ("%s:%s:%s" % (pkg.package, pkg.architecture, pkg.version)) else: pkg_key = ("%s:%s" % (pkg.package, pkg.architecture)) if pkg_key in packages.packages: old_filename = packages.packages[pkg_key].filename else: old_filename = "" ret = packages.add_package(pkg, opt_a) pkgs_stamps[filename] = stat.st_ctime if ret == 0: if old_filename: # old package was displaced by newer if opt_m: to_morgue(old_filename, pkg_dir, verbose) if opt_s: print(("%s/%s" % (pkg_dir, old_filename))) else: if opt_m: to_morgue(filename, pkg_dir, verbose) if opt_s: print(filename) except OSError as ex: sys.stderr.write("Package %s disappeared on us!\n(%s)\n" % (filename, ex)) continue except IOError as ex: sys.stderr.write("Package %s disappeared on us!\n(%s)\n" % (filename, ex)) continue pkgs_stamps_file = open(stamplist_filename, "w") for filename in list(pkgs_stamps.keys()): pkgs_stamps_file.write("%d %s\n" % (pkgs_stamps[filename], filename)) pkgs_stamps_file.close() if opt_s: sys.exit(0) if verbose: sys.stderr.write("Generating Packages file\n") if packages_filename: tmp_packages_filename = ("%s.%d" % (packages_filename, os.getpid())) pkgs_file = open(tmp_packages_filename, "w") names = list(packages.packages.keys()) names.sort() for name in names: try: pkg = packages.packages[name] if locales_dir and pkg.depends: depends = pkg.depends.split(',') locale = None for depend in depends: match = re.match('.*virtual-locale-([a-zA-Z]+).*', depend) match_by_pkg = re.match('locale-base-([a-zA-Z]+)([-+])?.*', pkg.package) if match: locale = match.group(1) if match_by_pkg: locale = match_by_pkg.group(1) if locale: to_locale(pkg.filename, locale, pkg_dir, locales_dir, verbose) continue if verbose: sys.stderr.write("Writing info for package %s\n" % (pkg.package,)) if packages_filename: pkgs_file.write(pkg.print(checksum)) else: print(pkg.print(checksum)) except OSError as ex: sys.stderr.write("Package %s disappeared on us!\n(%s)\n" % (name, ex)) continue except IOError as ex: sys.stderr.write("Package %s disappeared on us!\n(%s)\n" % (name, ex)) continue if packages_filename: pkgs_file.close() gzip_filename = ("%s.gz" % packages_filename) tmp_gzip_filename = ("%s.%d" % (gzip_filename, os.getpid())) gzip_cmd = "gzip -9c < %s > %s" % (tmp_packages_filename, tmp_gzip_filename) subprocess.call(gzip_cmd, shell=True) os.rename(tmp_packages_filename, packages_filename) os.rename(tmp_gzip_filename, gzip_filename) if filelist_filename: if verbose: sys.stderr.write("Generate Packages.filelist file\n") files = {} names = list(packages.packages.keys()) names.sort() for name in names: try: if verbose: sys.stderr.write("Reading filelist for package '%s'\n" % name) # sys.stderr.write("Package for name '%s':\n'%s'\n" % (name, packages[name])) file_list = packages[name].get_file_list_dir(pkg_dir) # sys.stderr.write("Filelist for package '%s': '%s'\n" % (name, fnlist)) except OSError as ex: sys.stderr.write("Package %s disappeared on us!\n(%s)\n" % (name, ex)) continue except IOError as ex: sys.stderr.write("Package %s disappeared on us!\n(%s)\n" % (name, ex)) continue for filepath in file_list: (_, filename) = os.path.split(filepath) if not filename: continue if filename not in files: files[filename] = name+':'+filepath else: files[filename] = files[filename] + ',' + name+':'+filepath tmp_filelist_filename = ("%s.%d" % (filelist_filename, os.getpid())) with open(tmp_filelist_filename, "w") as tmp_filelist_filename_hdl: names = list(files.keys()) names.sort() for name in names: tmp_filelist_filename_hdl.write("%s %s\n" % (name, files[name])) if posixpath.exists(filelist_filename): os.unlink(filelist_filename) os.rename(tmp_filelist_filename, filelist_filename) if __name__ == "__main__": sys.exit(main())