From 0a983d1368af8709c2ebe37dca34adde575cabdc Mon Sep 17 00:00:00 2001 From: Hongxu Jia Date: Mon, 23 Jul 2018 17:46:13 +0800 Subject: [PATCH 07/65] dnfpayload.py: customize for OE Upstream-Status: Inappropriate [oe specific] Signed-off-by: Hongxu Jia --- pyanaconda/payload/dnfpayload.py | 360 ++++++++++++++++++++++++++++++--------- 1 file changed, 281 insertions(+), 79 deletions(-) diff --git a/pyanaconda/payload/dnfpayload.py b/pyanaconda/payload/dnfpayload.py index 664a8f6..afff294 100644 --- a/pyanaconda/payload/dnfpayload.py +++ b/pyanaconda/payload/dnfpayload.py @@ -18,6 +18,7 @@ # Red Hat, Inc. # import os +import os.path from blivet.size import Size from pykickstart.constants import GROUP_ALL, GROUP_DEFAULT, KS_MISSING_IGNORE @@ -45,6 +46,11 @@ import time import threading from requests.exceptions import RequestException +import inspect +if __name__ == "__main__": + from pyanaconda import anaconda_logging + anaconda_logging.init() + anaconda_logging.logger.setupVirtio() from pyanaconda.anaconda_loggers import get_packaging_logger, get_dnf_logger log = get_packaging_logger() @@ -56,6 +62,7 @@ import dnf.repo import dnf.callback import dnf.conf.parser import dnf.conf.substitutions +import dnf.const import rpm import librepo @@ -227,7 +234,17 @@ class PayloadRPMDisplay(dnf.callback.TransactionProgress): """Report an error that occurred during the transaction. Message is a string which describes the error. """ - self._queue.put(('error', message)) + token = "error" + if message.startswith("Non-fatal"): + token = "warn" + self._queue.put((token, message)) + log.info("error: %s" % message) + + def scriptout(self, msgs): + if msgs is None: + return + + log.info("scriptout: %s" % msgs) class DownloadProgress(dnf.callback.DownloadProgress): @@ -297,6 +314,12 @@ class DNFPayload(payload.PackagePayload): self._updates_enabled = True self._configure() + self.requirements.add_packages(['base-files', 'base-passwd', 'shadow'], reason="basic packages") + # Support grub-mkconfig + self.requirements.add_packages(['sed', 'coreutils'], reason="Support grub-mkconfig") + # Support create new user + self.requirements.add_packages(['shadow'], reason="Support create new user") + # Protect access to _base.repos to ensure that the dictionary is not # modified while another thread is attempting to iterate over it. The # lock only needs to be held during operations that change the number @@ -308,12 +331,26 @@ class DNFPayload(payload.PackagePayload): self.requirements.set_apply_callback(self._apply_requirements) + # OE specific + self.image = {} + self.tasks = {} + + def setup(self, storage, instClass): + log.info("%s %s" % (self.__class__.__name__, inspect.stack()[0][3])) + + self.image, self.tasks = instClass.read_buildstamp() + super().setup(storage, instClass) + def unsetup(self): super().unsetup() self._base = None self._configure() self._repoMD_list = [] + # OE specific + self.image = {} + self.tasks = {} + def _replace_vars(self, url): """Replace url variables with their values. @@ -413,21 +450,12 @@ class DNFPayload(payload.PackagePayload): super().addRepo(ksrepo) def _apply_selections(self): - if self.data.packages.nocore: - log.info("skipping core group due to %%packages --nocore; system may not be complete") - else: - try: - self._select_group('core', required=True) - log.info("selected group: core") - except payload.NoSuchGroup as e: - self._miss(e) - env = None - if self.data.packages.default and self.environments: - env = self.environments[0] - elif self.data.packages.environment: + if self.data.packages.environment: env = self.data.packages.environment + elif self.environments: + env = self.environments[0] excludedGroups = [group.name for group in self.data.packages.excludedGroupList] @@ -480,7 +508,7 @@ class DNFPayload(payload.PackagePayload): log.debug("selected package: %s, requirement for %s %s", req.id, req.reasons, ", ".join(ignore_msgs)) - for req in self.requirements.groups: + for req in self.requirements.groups or []: try: self._select_group(req.id, required=req.strong) log.debug("selected group: %s, requirement for %s", @@ -519,6 +547,33 @@ class DNFPayload(payload.PackagePayload): conf.proxy_username = None conf.proxy_password = None + def _oe_configure(self, dnf_conf): + # Refers oe-core's recipes of dnf and rpm + os.environ['RPM_NO_CHROOT_FOR_SCRIPTS'] = "1" + + # Refers oe-core's meta/lib/oe/package_manager.py + # def _prepare_pkg_transaction(self) in class RpmPM(PackageManager) + target_rootfs = util.getSysroot() + os.environ['D'] = target_rootfs + os.environ['OFFLINE_ROOT'] = target_rootfs + os.environ['IPKG_OFFLINE_ROOT'] = target_rootfs + os.environ['OPKG_OFFLINE_ROOT'] = target_rootfs + + # Refer oe-core's meta/lib/oe/package_manager.py + # def _configure_dnf(self) in class RpmPM(PackageManager) + # directly read its result from installer system + dnf_conf.substitutions['arch'] = open("/etc/dnf/vars/arch", "r").readline() + + def _oe_configure_post(self): + # Refers oe-core's meta/lib/oe/package_manager.py + # def _configure_rpm(self) and def _configure_dnf(self) + # in class RpmPM(PackageManager) + # directly copy its result from installer system + target_sysconf = "%s/etc" % util.getSysroot() + util.execWithRedirect("cp", ["-fr", "/etc/rpmrc", target_sysconf]) + util.execWithRedirect("cp", ["-rf", "/etc/dnf", target_sysconf]) + util.execWithRedirect("cp", ["-rf", "/etc/rpm", target_sysconf]) + def _configure(self): self._base = dnf.Base() conf = self._base.conf @@ -530,12 +585,17 @@ class DNFPayload(payload.PackagePayload): conf.installroot = util.getSysroot() conf.prepend_installroot('persistdir') + self._oe_configure(conf) + self._base.conf.substitutions.update_from_etc(conf.installroot) # NSS won't survive the forking we do to shield out chroot during # transaction, disable it in RPM: conf.tsflags.append('nocrypto') + conf.errorlevel = dnf.const.VERBOSE_LEVEL + conf.debuglevel = dnf.const.VERBOSE_LEVEL + if self.data.packages.multiLib: conf.multilib_policy = "all" @@ -588,9 +648,9 @@ class DNFPayload(payload.PackagePayload): # as excluded from the installable set return self._base.sack.add_excludes(pkgs) - def _install_package(self, pkg_name, required=False): + def _install_package(self, pkg_name, required=False, strict=True): try: - return self._base.install(pkg_name) + return self._base.install(pkg_name, strict=strict) except dnf.exceptions.MarkingError: raise payload.NoSuchPackage(pkg_name, required=required) @@ -626,24 +686,48 @@ class DNFPayload(payload.PackagePayload): return pkgdir def _select_group(self, group_id, default=True, optional=False, required=False): - grp = self._base.comps.group_by_pattern(group_id) - if grp is None: - raise payload.NoSuchGroup(group_id, required=required) - types = {'mandatory'} - if default: - types.add('default') - if optional: - types.add('optional') - try: - self._base.group_install(grp.id, types) - except dnf.exceptions.MarkingError as e: - # dnf-1.1.9 raises this error when a package is missing from a group - raise payload.NoSuchPackage(str(e), required=True) - except dnf.exceptions.CompsError as e: - # DNF raises this when it is already selected - log.debug(e) + log.info("_select_group %s" % group_id) + if not group_id: + return + + glob = self._complementary_glob(group_id) + + # get all available languages in repos + available_packs = self._base.sack.query().available() \ + .filter(name__glob = "*-%s"%glob) + allpkgnames = [p.name for p in available_packs] + log.info("allpkgnames %d" % len(allpkgnames)) + for pkgname in allpkgnames: + self._install_package(pkgname, strict=False) + + def _select_linguas(self, image_linguas): + lc_globs = [] + for ling in image_linguas.split(): + if ling not in lc_globs: + lc_globs.append(ling) + + baselang = ling.split('-')[0] + if ling != baselang and baselang not in lc_globs: + lc_globs.append(baselang) + + log.info("lc_globs: %s" % lc_globs) + for glob in lc_globs: + self._select_group("locale-%s" % glob) + def _select_environment(self, env_id, excluded): + (image_name, description) = self.environmentDescription(env_id) + log.info("image_name %s, des %s" % (image_name, description)) + image_id = image_name.split()[0] + (image_summary, image_description, package_install, package_install_attemptonly, image_linguas) = self.image[image_id] + log.info("package_install %s, attempt %s, linguas %s" % + (package_install, package_install_attemptonly, image_linguas)) + + for pkg in package_install.split(): + self._install_package(pkg) + + self._select_linguas(image_linguas) + # dnf.base.environment_install excludes on packages instead of groups, # which is unhelpful. Instead, use group_install for each group in # the environment so we can skip the ones that are excluded. @@ -693,20 +777,25 @@ class DNFPayload(payload.PackagePayload): def baseRepo(self): # is any locking needed here? repo_names = [constants.BASE_REPO_NAME] + self.DEFAULT_REPOS + log.info("repo_names %s" % repo_names) with self._repos_lock: for repo in self._base.repos.iter_enabled(): + log.info("repo.id %s" % repo.id) if repo.id in repo_names: return repo.id return None @property def environments(self): - return [env.id for env in self._base.comps.environments] + """ List of environment ids. """ + log.info("%s %s: %s" % (self.__class__.__name__, inspect.stack()[0][3], self.tasks.keys())) + return sorted(self.tasks.keys()) @property def groups(self): - groups = self._base.comps.groups_iter() - return [g.id for g in groups] + """ List of group ids. """ + log.info("%s %s" % (self.__class__.__name__, inspect.stack()[0][3])) + return sorted(self.tasks.keys()) @property def mirrorEnabled(self): @@ -776,7 +865,7 @@ class DNFPayload(payload.PackagePayload): return grp.visible def _groupHasInstallableMembers(self, grpid): - return True + return False def checkSoftwareSelection(self): log.info("checking software selection") @@ -831,50 +920,49 @@ class DNFPayload(payload.PackagePayload): super().enableRepo(repo_id) def environmentDescription(self, environmentid): - env = self._base.comps.environment_by_pattern(environmentid) - if env is None: - raise payload.NoSuchGroup(environmentid) - return (env.ui_name, env.ui_description) + log.info("%s %s, %s" % (self.__class__.__name__, inspect.stack()[0][3], environmentid)) + + if environmentid not in self.tasks: + raise NoSuchGroup(environmentid) + + if environmentid in self.tasks: + log.info("environmentDescription %s" % self.tasks) + (name, description, group) = self.tasks[environmentid] + + return (name, description) + return (environmentid, environmentid) def environmentId(self, environment): """Return environment id for the environment specified by id or name.""" - env = self._base.comps.environment_by_pattern(environment) - if env is None: - raise payload.NoSuchGroup(environment) - return env.id + log.info("%s %s, environment %s" % (self.__class__.__name__, inspect.stack()[0][3], environment)) + # TODO + return environment def environmentGroups(self, environmentid, optional=True): - env = self._base.comps.environment_by_pattern(environmentid) - if env is None: - raise payload.NoSuchGroup(environmentid) - group_ids = (id_.name for id_ in env.group_ids) - option_ids = (id_.name for id_ in env.option_ids) - if optional: - return list(itertools.chain(group_ids, option_ids)) - else: - return list(group_ids) + log.info("%s %s, %s" % (self.__class__.__name__, inspect.stack()[0][3], environmentid)) + + if environmentid in self.tasks: + log.info("environmentDescription %s" % self.tasks) + (name, description, groups) = self.tasks[environmentid] + + return groups.split() + return [environmentid] def environmentHasOption(self, environmentid, grpid): - env = self._base.comps.environment_by_pattern(environmentid) - if env is None: - raise payload.NoSuchGroup(environmentid) - return grpid in (id_.name for id_ in env.option_ids) + log.info("%s %s, %s, %s" % (self.__class__.__name__, inspect.stack()[0][3], environmentid, grpid)) + # TODO + return True def environmentOptionIsDefault(self, environmentid, grpid): - env = self._base.comps.environment_by_pattern(environmentid) - if env is None: - raise payload.NoSuchGroup(environmentid) - - # Look for a group in the optionlist that matches the group_id and has - # default set - return any(grp for grp in env.option_ids if grp.name == grpid and grp.default) + log.info("%s %s, %s, %s" % (self.__class__.__name__, inspect.stack()[0][3], environmentid, grpid)) + # TODO + return True def groupDescription(self, grpid): """Return name/description tuple for the group specified by id.""" - grp = self._base.comps.group_by_pattern(grpid) - if grp is None: - raise payload.NoSuchGroup(grpid) - return (grp.ui_name, grp.ui_description) + log.info("%s %s, %s" % (self.__class__.__name__, inspect.stack()[0][3], grpid)) + (name, description, group) = self.tasks[grpid] + return (name, description) def groupId(self, group_name): """Translate group name to group ID. @@ -897,6 +985,31 @@ class DNFPayload(payload.PackagePayload): self._base.read_comps() self._refreshEnvironmentAddons() + # Refer upstream oe-core meta/lib/oe/package_manager.py + def _save_rpmpostinst(self, pkg): + def _script_num_prefix(path): + files = os.listdir(path) + numbers = set() + numbers.add(99) + for f in files: + numbers.add(int(f.split("-")[0])) + return max(numbers) + 1 + + log.info("Saving postinstall script of %s" % (pkg)) + + target_rootfs = util.getSysroot() + args = ["-q", "--root=%s" % target_rootfs, "--queryformat", "%{postin}", pkg] + + output = util.execWithCapture('rpm', args, log_output=False) + log.info(output) + + target_path = "%s/etc/rpm-postinsts" % target_rootfs + util.execWithRedirect("mkdir", ["-p", target_path]) + num = _script_num_prefix(target_path) + saved_script_name = os.path.join(target_path, "%d-%s" % (num, pkg)) + open(saved_script_name, 'w').write(output) + os.chmod(saved_script_name, 0o755) + def install(self): progress_message(N_('Starting package installation process')) @@ -931,6 +1044,7 @@ class DNFPayload(payload.PackagePayload): _failure_limbo() log.info('Downloading packages finished.') + failed_scriptlets_pkgnames = collections.OrderedDict() pre_msg = (N_("Preparing transaction from installation source")) progress_message(pre_msg) @@ -963,6 +1077,9 @@ class DNFPayload(payload.PackagePayload): elif token == 'quit': msg = ("Payload error - DNF installation has ended up abruptly: %s" % msg) raise payload.PayloadError(msg) + elif token == 'warn': + if msg.startswith("Non-fatal POSTIN scriptlet failure in rpm package"): + failed_scriptlets_pkgnames[msg.split()[-1]] = True elif token == 'error': exc = payload.PayloadInstallError("DNF error: %s" % msg) if errors.errorHandler.cb(exc) == errors.ERROR_RAISE: @@ -972,6 +1089,10 @@ class DNFPayload(payload.PackagePayload): process.join() self._base.close() + + for pkg in failed_scriptlets_pkgnames.keys(): + self._save_rpmpostinst(pkg) + if os.path.exists(self._download_location): log.info("Cleaning up downloaded packages: %s", self._download_location) shutil.rmtree(self._download_location) @@ -982,6 +1103,39 @@ class DNFPayload(payload.PackagePayload): # we don't have to care about clearing the download location ourselves. log.warning("Can't delete nonexistent download location: %s", self._download_location) + def _list_installed(self): + output = util.execWithCapture( + 'dnf', + [ + 'repoquery', + '--installed', + '--installroot='+util.getSysroot(), + '--queryformat=Package:%{name}' + ], + log_output=False + ) + output = output.rstrip() + lines = output.splitlines() + installed = [] + for line in lines: + if not line.startswith("Package:"): + continue + installed.append(line.replace("Package:", "")) + return installed + + def _complementary_glob(self, group): + complementary_glob = {} + complementary_glob['dev-pkgs'] = 'dev' + complementary_glob['staticdev-pkgs'] = 'staticdev' + complementary_glob['doc-pkgs'] = 'doc' + complementary_glob['dbg-pkgs'] = 'dbg' + complementary_glob['ptest-pkgs'] = 'ptest' + + if group in complementary_glob: + return complementary_glob[group] + + return group + def getRepo(self, repo_id): """Return the yum repo object.""" return self._base.repos[repo_id] @@ -1004,16 +1158,7 @@ class DNFPayload(payload.PackagePayload): return True def languageGroups(self): - localization_proxy = LOCALIZATION.get_proxy() - locales = [localization_proxy.Language] + localization_proxy.LanguageSupport - match_fn = pyanaconda.localization.langcode_matches_locale - gids = set() - gl_tuples = ((g.id, g.lang_only) for g in self._base.comps.groups_iter()) - for (gid, lang) in gl_tuples: - for locale in locales: - if match_fn(lang, locale): - gids.add(gid) - return list(gids) + return [] def reset(self): super().reset() @@ -1025,7 +1170,6 @@ class DNFPayload(payload.PackagePayload): self._repoMD_list = [] def updateBaseRepo(self, fallback=True, checkmount=True): - log.info('configuring base repo') self.reset() url, mirrorlist, metalink = self._setupInstallDevice(self.storage, checkmount) @@ -1033,6 +1177,8 @@ class DNFPayload(payload.PackagePayload): sslverify = True if method.method == "url": sslverify = not (method.noverifyssl or flags.noverifyssl) + log.info('configuring base repo: url %s, mirrorlist %s, sslverify %s, method %s' + % (url, mirrorlist, sslverify, method.method)) # Read in all the repos from the installation environment, make a note of which # are enabled, and then disable them all. If the user gave us a method, we want @@ -1204,6 +1350,17 @@ class DNFPayload(payload.PackagePayload): super().postInstall() + self._oe_configure_post() + + # For the generation of anaconda-ks.cfg + for pkg in self._list_installed(): + if pkg not in self.data.packages.packageList: + self.data.packages.packageList.append(pkg) + + util.execWithRedirect("depmod", + ["-a"], + root=util.getSysroot()) + def writeStorageLate(self): pass @@ -1277,3 +1434,48 @@ class RepoMDMetaHash(object): log.debug("Can't download new repomd.xml from %s with proxy: %s. Error: %s", url, proxies, e) return repomd + + +# For testing +if __name__ == "__main__": + log.info("God bless us") + + from pykickstart.version import makeVersion + + # set up ksdata + ksdata = makeVersion() + + ksdata.method.method = "cdrom" + + from pyanaconda.installclass import factory + instClass = factory.get_best_install_class() + + from pyanaconda.storage.osinstall import InstallerStorage + storage = InstallerStorage(ksdata=ksdata) + storage.set_default_fstype("ext4") + + _payload = DNFPayload(ksdata) + + _payload.setup(storage, instClass) + + # Keep setting up package-based repositories + # Download package metadata + try: + _payload.updateBaseRepo(fallback=False, checkmount=True) + except payload.PayloadError as e: + log.error("PayloadError: %s", e) + _payload.unsetup() + exit(0) + + # Gather the group data + _payload.gatherRepoMetadata() + _payload.release() + _payload.addDriverRepos() + + log.info("baseRepo %s" % _payload.baseRepo) + + _payload.preInstall() + + _payload.install() + + _payload.postInstall() -- 2.7.4