#!/usr/bin/env python """ Utility to create opkg compatible indexes """ from __future__ import absolute_import from __future__ import print_function import getopt import sys import os import posixpath import re import subprocess import opkg def usage(): """ Print help """ sys.stderr.write("%s [-h] [-s] [-m] [-a] [-f] [-l Packages.filelist] [-p Packages]" " [-r Packages.old] [-L localesdir] [-v] packagesdir\n" % (sys.argv[0],)) sys.exit(-1) def to_morgue(filename, pkg_dir, verbose): """ Move files to morgue folder """ morgue_dir = pkg_dir + "/morgue" if verbose: sys.stderr.write("Moving " + filename + " to morgue\n") if not os.path.exists(morgue_dir): os.mkdir(morgue_dir) if os.path.exists(pkg_dir + "/" + filename): os.rename(pkg_dir + "/" + filename, morgue_dir + "/" + filename) if os.path.exists(pkg_dir + "/" + filename + ".asc"): os.rename(pkg_dir + "/" + filename + ".asc", morgue_dir + "/" + filename + ".asc") def to_locale(filename, locale, pkg_dir, locales_dir, verbose): """ Move file to locale_dir""" locale_dir = pkg_dir + '/' + locales_dir + '/' + locale + "/" if verbose: sys.stderr.write("Moving " + filename + " to " + locale_dir + "\n") if not os.path.exists(locale_dir): os.mkdir(locale_dir) os.rename(pkg_dir + "/" + filename, locale_dir + "/" + filename) if os.path.exists(pkg_dir + "/" + filename + ".asc"): os.rename(pkg_dir + "/" + filename + ".asc", locale_dir + "/" + filename + ".asc") def main(): """ Script entry point """ verbose = 0 locales_dir = None old_filename = None packages_filename = None filelist_filename = None stamplist_filename = "Packages.stamps" opt_s = 0 opt_m = 0 opt_a = 0 opt_f = 0 (opts, remaining_args) = getopt.getopt(sys.argv[1:], "hl:p:vsmr:L:af") for (optkey, optval) in opts: if optkey == '-h': usage() if optkey == '-s': opt_s = 1 if optkey == '-p': packages_filename = optval stamplist_filename = optval + ".stamps" if optkey == '-l': filelist_filename = optval if optkey == '-v': verbose = 1 if optkey == '-m': opt_m = 1 if optkey == '-r': old_filename = optval if optkey == '-L': locales_dir = optval if optkey == '-a': opt_a = 1 if optkey == '-f': opt_f = 1 if not remaining_args: usage() pkg_dir = remaining_args[0] packages = opkg.Packages() old_pkg_hash = {} if packages_filename and not old_filename and os.path.exists(packages_filename): old_filename = packages_filename pkgs_stamps = {} if old_filename: if verbose: sys.stderr.write("Reading package list from " + old_filename + "\n") old_packages = opkg.Packages() old_packages.read_packages_file(old_filename, opt_f) for k in list(old_packages.packages.keys()): pkg = old_packages.packages[k] old_pkg_hash[pkg.filename] = pkg try: with open(stamplist_filename, "r") as stamplist_filename_hdl: for line in stamplist_filename_hdl: line = line.strip() stamp, filename = line.split(" ", 1) pkgs_stamps[filename] = int(stamp) except IOError: pass if verbose: sys.stderr.write("Reading in all the package info from %s\n" % (pkg_dir, )) files = [] opkg_extensions = ['.ipk', '.opk', '.deb'] for dirpath, _, filenames in os.walk(pkg_dir): for filename in filenames: ext = os.path.splitext(filename)[1] if ext in opkg_extensions: files.append(os.path.join(dirpath, filename)) files.sort() for abspath in files: try: filename = os.path.relpath(abspath, pkg_dir) pkg = None stat = os.stat(abspath) if filename in old_pkg_hash: if filename in pkgs_stamps and int(stat.st_ctime) == pkgs_stamps[filename]: if verbose: sys.stderr.write("Found %s in Packages\n" % (filename,)) pkg = old_pkg_hash[filename] else: sys.stderr.write("Found %s in Packages, but ctime differs - re-reading\n" % (filename,)) if not pkg: if verbose: sys.stderr.write("Reading info for package %s\n" % (filename,)) pkg = opkg.Package(abspath, relpath=pkg_dir, all_fields=opt_f) if opt_a: pkg_key = ("%s:%s:%s" % (pkg.package, pkg.architecture, pkg.version)) else: pkg_key = ("%s:%s" % (pkg.package, pkg.architecture)) if pkg_key in packages.packages: old_filename = packages.packages[pkg_key].filename else: old_filename = "" ret = packages.add_package(pkg, opt_a) pkgs_stamps[filename] = stat.st_ctime if ret == 0: if old_filename: # old package was displaced by newer if opt_m: to_morgue(old_filename, pkg_dir, verbose) if opt_s: print(("%s/%s" % (pkg_dir, old_filename))) else: if opt_m: to_morgue(filename, pkg_dir, verbose) if opt_s: print(filename) except OSError as ex: sys.stderr.write("Package %s disappeared on us!\n(%s)\n" % (filename, ex)) continue except IOError as ex: sys.stderr.write("Package %s disappeared on us!\n(%s)\n" % (filename, ex)) continue pkgs_stamps_file = open(stamplist_filename, "w") for filename in list(pkgs_stamps.keys()): pkgs_stamps_file.write("%d %s\n" % (pkgs_stamps[filename], filename)) pkgs_stamps_file.close() if opt_s: sys.exit(0) if verbose: sys.stderr.write("Generating Packages file\n") if packages_filename: tmp_packages_filename = ("%s.%d" % (packages_filename, os.getpid())) pkgs_file = open(tmp_packages_filename, "w") names = list(packages.packages.keys()) names.sort() for name in names: try: pkg = packages.packages[name] if locales_dir and pkg.depends: depends = pkg.depends.split(',') locale = None for depend in depends: match = re.match('.*virtual-locale-([a-zA-Z]+).*', depend) match_by_pkg = re.match('locale-base-([a-zA-Z]+)([-+])?.*', pkg.package) if match: locale = match.group(1) if match_by_pkg: locale = match_by_pkg.group(1) if locale: to_locale(pkg.filename, locale, pkg_dir, locales_dir, verbose) continue if verbose: sys.stderr.write("Writing info for package %s\n" % (pkg.package,)) if packages_filename: pkgs_file.write(str(pkg)) else: print(pkg) except OSError as ex: sys.stderr.write("Package %s disappeared on us!\n(%s)\n" % (name, ex)) continue except IOError as ex: sys.stderr.write("Package %s disappeared on us!\n(%s)\n" % (name, ex)) continue if packages_filename: pkgs_file.close() gzip_filename = ("%s.gz" % packages_filename) tmp_gzip_filename = ("%s.%d" % (gzip_filename, os.getpid())) gzip_cmd = "gzip -9c < %s > %s" % (tmp_packages_filename, tmp_gzip_filename) subprocess.call(gzip_cmd, shell=True) os.rename(tmp_packages_filename, packages_filename) os.rename(tmp_gzip_filename, gzip_filename) if filelist_filename: if verbose: sys.stderr.write("Generate Packages.filelist file\n") files = {} names = list(packages.packages.keys()) names.sort() for name in names: try: if verbose: sys.stderr.write("Reading filelist for package '%s'\n" % name) # sys.stderr.write("Package for name '%s':\n'%s'\n" % (name, packages[name])) file_list = packages[name].get_file_list_dir(pkg_dir) # sys.stderr.write("Filelist for package '%s': '%s'\n" % (name, fnlist)) except OSError as ex: sys.stderr.write("Package %s disappeared on us!\n(%s)\n" % (name, ex)) continue except IOError as ex: sys.stderr.write("Package %s disappeared on us!\n(%s)\n" % (name, ex)) continue for filepath in file_list: (_, filename) = os.path.split(filepath) if not filename: continue if filename not in files: files[filename] = name+':'+filepath else: files[filename] = files[filename] + ',' + name+':'+filepath tmp_filelist_filename = ("%s.%d" % (filelist_filename, os.getpid())) with open(tmp_filelist_filename, "w") as tmp_filelist_filename_hdl: names = list(files.keys()) names.sort() for name in names: tmp_filelist_filename_hdl.write("%s %s\n" % (name, files[name])) if posixpath.exists(filelist_filename): os.unlink(filelist_filename) os.rename(tmp_filelist_filename, filelist_filename) if __name__ == "__main__": sys.exit(main())