diff options
Diffstat (limited to 'opkg-make-index')
| -rwxr-xr-x | opkg-make-index | 477 |
1 files changed, 247 insertions, 230 deletions
diff --git a/opkg-make-index b/opkg-make-index index db7bf64..6f010a2 100755 --- a/opkg-make-index +++ b/opkg-make-index @@ -1,247 +1,264 @@ #!/usr/bin/env python +""" + Utility to create opkg compatible indexes +""" from __future__ import absolute_import from __future__ import print_function -import sys, os, posixpath -import subprocess -import opkg import getopt +import sys +import os +import posixpath import re +import subprocess -verbose = 0 +import opkg def usage(): - sys.stderr.write("%s [-h] [-s] [-m] [-a] [-f] [-l Packages.filelist] [-p Packages] [-r Packages.old] [-L localesdir] [-v] packagesdir\n" % (sys.argv[0],)) - sys.exit(-1) - -def to_morgue(filename): - morgue_dir = pkg_dir + "/morgue" - if verbose: - sys.stderr.write ("Moving " + filename + " to morgue\n") - if not os.path.exists(morgue_dir): - os.mkdir(morgue_dir) - if os.path.exists(pkg_dir + "/" + filename): - os.rename(pkg_dir + "/" + filename, morgue_dir + "/" + filename) - if os.path.exists(pkg_dir + "/" + filename + ".asc"): - os.rename(pkg_dir + "/" + filename + ".asc", morgue_dir + "/" + filename + ".asc") - -locales_dir = None -def to_locale(filename, locale): - locale_dir = pkg_dir + '/' + locales_dir + '/' + locale + "/" - if verbose: - sys.stderr.write ("Moving " + filename + " to " + locale_dir + "\n") - if not os.path.exists(locale_dir): - os.mkdir(locale_dir) - os.rename(pkg_dir + "/" + filename, locale_dir + "/" + filename) - if os.path.exists(pkg_dir + "/" + filename + ".asc"): - os.rename(pkg_dir + "/" + filename + ".asc", locale_dir + "/" + filename + ".asc") - -old_filename = None -packages_filename = None -filelist_filename = None -stamplist_filename = "Packages.stamps" -opt_s = 0 -opt_m = 0 -opt_a = 0 -opt_f = 0 -(opts, remaining_args) = getopt.getopt(sys.argv[1:], "hl:p:vsmr:L:af") -for (optkey, optval) in opts: - if optkey == '-h': - usage() - if optkey == '-s': - opt_s = 1 - if optkey == '-p': - packages_filename = optval - stamplist_filename = optval + ".stamps" - if optkey == '-l': - filelist_filename = optval - if optkey == '-v': - verbose = 1 - if optkey == '-m': - opt_m = 1 - if optkey == '-r': - old_filename = optval - if optkey == '-L': - locales_dir = optval - if optkey == '-a': - opt_a = 1 - if optkey == '-f': - opt_f = 1 - -if ( not remaining_args ): - usage() - -pkg_dir=remaining_args[0] - -packages = opkg.Packages() - -old_pkg_hash = {} -if packages_filename and not old_filename and os.path.exists(packages_filename): - old_filename = packages_filename - -pkgsStamps = {} -if old_filename: - if (verbose): - sys.stderr.write("Reading package list from " + old_filename + "\n") - old_packages = opkg.Packages() - old_packages.read_packages_file(old_filename, opt_f) - for k in list(old_packages.packages.keys()): - p = old_packages.packages[k] - old_pkg_hash[p.filename] = p - try: - f = open(stamplist_filename, "r") - for l in f: - l = l.strip() - s, f = l.split(" ", 1) - pkgsStamps[f] = int(s) - except IOError: - pass - -if (verbose): - sys.stderr.write("Reading in all the package info from %s\n" % (pkg_dir, )) - -files=[] -opkg_extensions=['.ipk','.opk','.deb'] -for dirpath, dirnames, filenames in os.walk(pkg_dir): - for f in filenames: - ext = os.path.splitext(f)[1] - if ext in opkg_extensions: - files.append(os.path.join(dirpath, f)) - -files.sort() -for abspath in files: - try: - filename = os.path.relpath(abspath, pkg_dir) - pkg = None - fnameStat = os.stat(abspath) - if filename in old_pkg_hash: - if filename in pkgsStamps and int(fnameStat.st_ctime) == pkgsStamps[filename]: - if (verbose): - sys.stderr.write("Found %s in Packages\n" % (filename,)) - pkg = old_pkg_hash[filename] - else: - sys.stderr.write("Found %s in Packages, but ctime differs - re-reading\n" % (filename,)) - - if not pkg: - if (verbose): - sys.stderr.write("Reading info for package %s\n" % (filename,)) - pkg = opkg.Package(abspath, relpath=pkg_dir, all_fields=opt_f) - - if opt_a: - pkg_key = ("%s:%s:%s" % (pkg.package, pkg.architecture, pkg.version)) - else: - pkg_key = ("%s:%s" % (pkg.package, pkg.architecture)) - - if (pkg_key in packages.packages): - old_filename = packages.packages[pkg_key].filename - else: - old_filename = "" - s = packages.add_package(pkg, opt_a) - pkgsStamps[filename] = fnameStat.st_ctime - if s == 0: - if old_filename: - # old package was displaced by newer - if opt_m: - to_morgue(old_filename) - if opt_s: - print(("%s/%s" % (pkg_dir, old_filename))) - else: - if opt_m: - to_morgue(filename) - if opt_s: - print(filename) - except OSError as e: - sys.stderr.write("Package %s disappeared on us!\n(%s)\n" % (filename, e)) - continue - except IOError as e: - sys.stderr.write("Package %s disappeared on us!\n(%s)\n" % (filename, e)) - continue - -pkgsStampsFile = open(stamplist_filename, "w") -for f in list(pkgsStamps.keys()): - pkgsStampsFile.write("%d %s\n" % (pkgsStamps[f], f)) -pkgsStampsFile.close() - -if opt_s: - sys.exit(0) - -if verbose: - sys.stderr.write("Generating Packages file\n") -if packages_filename: - tmp_packages_filename = ("%s.%d" % (packages_filename, os.getpid())) - pkgsFile = open(tmp_packages_filename, "w") -names = list(packages.packages.keys()) -names.sort() -for name in names: - try: - pkg = packages.packages[name] - if locales_dir and pkg.depends: - depends = pkg.depends.split(',') - locale = None - for d in depends: - m = re.match('.*virtual-locale-([a-zA-Z]+).*', d) - mp = re.match('locale-base-([a-zA-Z]+)([-+])?.*', pkg.package) - if m: - locale = m.group(1) - if mp: - locale = mp.group(1) - if locale: - to_locale(pkg.filename, locale) - continue - if (verbose): - sys.stderr.write("Writing info for package %s\n" % (pkg.package,)) - if packages_filename: - pkgsFile.write(str(pkg)) - else: - print(pkg) - except OSError as e: - sys.stderr.write("Package %s disappeared on us!\n(%s)\n" % (name, e)) - continue - except IOError as e: - sys.stderr.write("Package %s disappeared on us!\n(%s)\n" % (name, e)) - continue - -if packages_filename: - pkgsFile.close() - gzip_filename = ("%s.gz" % packages_filename) - tmp_gzip_filename = ("%s.%d" % (gzip_filename, os.getpid())) - gzip_cmd = "gzip -9c < %s > %s" % (tmp_packages_filename, tmp_gzip_filename) - subprocess.call(gzip_cmd, shell=True) - os.rename(tmp_packages_filename, packages_filename) - os.rename(tmp_gzip_filename, gzip_filename) - -if filelist_filename: + """ Print help """ + sys.stderr.write("%s [-h] [-s] [-m] [-a] [-f] [-l Packages.filelist] [-p Packages]" + " [-r Packages.old] [-L localesdir] [-v] packagesdir\n" % (sys.argv[0],)) + sys.exit(-1) + +def to_morgue(filename, pkg_dir, verbose): + """ Move files to morgue folder """ + morgue_dir = pkg_dir + "/morgue" + if verbose: + sys.stderr.write("Moving " + filename + " to morgue\n") + if not os.path.exists(morgue_dir): + os.mkdir(morgue_dir) + if os.path.exists(pkg_dir + "/" + filename): + os.rename(pkg_dir + "/" + filename, morgue_dir + "/" + filename) + if os.path.exists(pkg_dir + "/" + filename + ".asc"): + os.rename(pkg_dir + "/" + filename + ".asc", morgue_dir + "/" + filename + ".asc") + +def to_locale(filename, locale, pkg_dir, locales_dir, verbose): + """ Move file to locale_dir""" + locale_dir = pkg_dir + '/' + locales_dir + '/' + locale + "/" + if verbose: + sys.stderr.write("Moving " + filename + " to " + locale_dir + "\n") + if not os.path.exists(locale_dir): + os.mkdir(locale_dir) + os.rename(pkg_dir + "/" + filename, locale_dir + "/" + filename) + if os.path.exists(pkg_dir + "/" + filename + ".asc"): + os.rename(pkg_dir + "/" + filename + ".asc", locale_dir + "/" + filename + ".asc") + +def main(): + """ Script entry point """ + verbose = 0 + locales_dir = None + old_filename = None + packages_filename = None + filelist_filename = None + stamplist_filename = "Packages.stamps" + opt_s = 0 + opt_m = 0 + opt_a = 0 + opt_f = 0 + (opts, remaining_args) = getopt.getopt(sys.argv[1:], "hl:p:vsmr:L:af") + for (optkey, optval) in opts: + if optkey == '-h': + usage() + if optkey == '-s': + opt_s = 1 + if optkey == '-p': + packages_filename = optval + stamplist_filename = optval + ".stamps" + if optkey == '-l': + filelist_filename = optval + if optkey == '-v': + verbose = 1 + if optkey == '-m': + opt_m = 1 + if optkey == '-r': + old_filename = optval + if optkey == '-L': + locales_dir = optval + if optkey == '-a': + opt_a = 1 + if optkey == '-f': + opt_f = 1 + + if not remaining_args: + usage() + + pkg_dir = remaining_args[0] + + packages = opkg.Packages() + + old_pkg_hash = {} + if packages_filename and not old_filename and os.path.exists(packages_filename): + old_filename = packages_filename + + pkgs_stamps = {} + if old_filename: + if verbose: + sys.stderr.write("Reading package list from " + old_filename + "\n") + old_packages = opkg.Packages() + old_packages.read_packages_file(old_filename, opt_f) + for k in list(old_packages.packages.keys()): + pkg = old_packages.packages[k] + old_pkg_hash[pkg.filename] = pkg + try: + with open(stamplist_filename, "r") as stamplist_filename_hdl: + for line in stamplist_filename_hdl: + line = line.strip() + stamp, filename = line.split(" ", 1) + pkgs_stamps[filename] = int(stamp) + except IOError: + pass + + if verbose: + sys.stderr.write("Reading in all the package info from %s\n" % (pkg_dir, )) + + files = [] + opkg_extensions = ['.ipk', '.opk', '.deb'] + for dirpath, _, filenames in os.walk(pkg_dir): + for filename in filenames: + ext = os.path.splitext(filename)[1] + if ext in opkg_extensions: + files.append(os.path.join(dirpath, filename)) + + files.sort() + for abspath in files: + try: + filename = os.path.relpath(abspath, pkg_dir) + pkg = None + stat = os.stat(abspath) + if filename in old_pkg_hash: + if filename in pkgs_stamps and int(stat.st_ctime) == pkgs_stamps[filename]: + if verbose: + sys.stderr.write("Found %s in Packages\n" % (filename,)) + pkg = old_pkg_hash[filename] + else: + sys.stderr.write("Found %s in Packages, but ctime differs - re-reading\n" + % (filename,)) + + if not pkg: + if verbose: + sys.stderr.write("Reading info for package %s\n" % (filename,)) + pkg = opkg.Package(abspath, relpath=pkg_dir, all_fields=opt_f) + + if opt_a: + pkg_key = ("%s:%s:%s" % (pkg.package, pkg.architecture, pkg.version)) + else: + pkg_key = ("%s:%s" % (pkg.package, pkg.architecture)) + + if pkg_key in packages.packages: + old_filename = packages.packages[pkg_key].filename + else: + old_filename = "" + ret = packages.add_package(pkg, opt_a) + pkgs_stamps[filename] = stat.st_ctime + if ret == 0: + if old_filename: + # old package was displaced by newer + if opt_m: + to_morgue(old_filename, pkg_dir, verbose) + if opt_s: + print(("%s/%s" % (pkg_dir, old_filename))) + else: + if opt_m: + to_morgue(filename, pkg_dir, verbose) + if opt_s: + print(filename) + except OSError as ex: + sys.stderr.write("Package %s disappeared on us!\n(%s)\n" % (filename, ex)) + continue + except IOError as ex: + sys.stderr.write("Package %s disappeared on us!\n(%s)\n" % (filename, ex)) + continue + + pkgs_stamps_file = open(stamplist_filename, "w") + for filename in list(pkgs_stamps.keys()): + pkgs_stamps_file.write("%d %s\n" % (pkgs_stamps[filename], filename)) + pkgs_stamps_file.close() + + if opt_s: + sys.exit(0) + if verbose: - sys.stderr.write("Generate Packages.filelist file\n") - files = {} + sys.stderr.write("Generating Packages file\n") + if packages_filename: + tmp_packages_filename = ("%s.%d" % (packages_filename, os.getpid())) + pkgs_file = open(tmp_packages_filename, "w") names = list(packages.packages.keys()) names.sort() for name in names: try: + pkg = packages.packages[name] + if locales_dir and pkg.depends: + depends = pkg.depends.split(',') + locale = None + for depend in depends: + match = re.match('.*virtual-locale-([a-zA-Z]+).*', depend) + match_by_pkg = re.match('locale-base-([a-zA-Z]+)([-+])?.*', pkg.package) + if match: + locale = match.group(1) + if match_by_pkg: + locale = match_by_pkg.group(1) + if locale: + to_locale(pkg.filename, locale, pkg_dir, locales_dir, verbose) + continue if verbose: - sys.stderr.write("Reading filelist for package '%s'\n" % name) -# sys.stderr.write("Package for name '%s':\n'%s'\n" % (name, packages[name])) - fnlist = packages[name].get_file_list_dir(pkg_dir) -# sys.stderr.write("Filelist for package '%s': '%s'\n" % (name, fnlist)) - except OSError as e: - sys.stderr.write("Package %s disappeared on us!\n(%s)\n" % (name, e)) + sys.stderr.write("Writing info for package %s\n" % (pkg.package,)) + if packages_filename: + pkgs_file.write(str(pkg)) + else: + print(pkg) + except OSError as ex: + sys.stderr.write("Package %s disappeared on us!\n(%s)\n" % (name, ex)) continue - except IOError as e: - sys.stderr.write("Package %s disappeared on us!\n(%s)\n" % (name, e)) + except IOError as ex: + sys.stderr.write("Package %s disappeared on us!\n(%s)\n" % (name, ex)) continue - for fn in fnlist: - (h,t) = os.path.split(fn) - if not t: continue - if t not in files: files[t] = name+':'+fn - else: files[t] = files[t] + ',' + name+':'+fn - - tmp_filelist_filename = ("%s.%d" % (filelist_filename, os.getpid())) - f = open(tmp_filelist_filename, "w") - names = list(files.keys()) - names.sort() - for name in names: - f.write("%s %s\n" % (name, files[name])) - f.close() - if posixpath.exists(filelist_filename): - os.unlink(filelist_filename) - os.rename(tmp_filelist_filename, filelist_filename) + + if packages_filename: + pkgs_file.close() + gzip_filename = ("%s.gz" % packages_filename) + tmp_gzip_filename = ("%s.%d" % (gzip_filename, os.getpid())) + gzip_cmd = "gzip -9c < %s > %s" % (tmp_packages_filename, tmp_gzip_filename) + subprocess.call(gzip_cmd, shell=True) + os.rename(tmp_packages_filename, packages_filename) + os.rename(tmp_gzip_filename, gzip_filename) + + if filelist_filename: + if verbose: + sys.stderr.write("Generate Packages.filelist file\n") + files = {} + names = list(packages.packages.keys()) + names.sort() + for name in names: + try: + if verbose: + sys.stderr.write("Reading filelist for package '%s'\n" % name) +# sys.stderr.write("Package for name '%s':\n'%s'\n" % (name, packages[name])) + file_list = packages[name].get_file_list_dir(pkg_dir) +# sys.stderr.write("Filelist for package '%s': '%s'\n" % (name, fnlist)) + except OSError as ex: + sys.stderr.write("Package %s disappeared on us!\n(%s)\n" % (name, ex)) + continue + except IOError as ex: + sys.stderr.write("Package %s disappeared on us!\n(%s)\n" % (name, ex)) + continue + for filepath in file_list: + (_, filename) = os.path.split(filepath) + if not filename: + continue + if filename not in files: + files[filename] = name+':'+filepath + else: + files[filename] = files[filename] + ',' + name+':'+filepath + + tmp_filelist_filename = ("%s.%d" % (filelist_filename, os.getpid())) + with open(tmp_filelist_filename, "w") as tmp_filelist_filename_hdl: + names = list(files.keys()) + names.sort() + for name in names: + tmp_filelist_filename_hdl.write("%s %s\n" % (name, files[name])) + if posixpath.exists(filelist_filename): + os.unlink(filelist_filename) + os.rename(tmp_filelist_filename, filelist_filename) + +if __name__ == "__main__": + sys.exit(main()) |