path: root/recipes-installer/anaconda/files/0007-dnfpayload.py-customize-for-OE.patch

From 0a983d1368af8709c2ebe37dca34adde575cabdc Mon Sep 17 00:00:00 2001
From: Hongxu Jia <hongxu.jia@windriver.com>
Date: Mon, 23 Jul 2018 17:46:13 +0800
Subject: [PATCH 07/65] dnfpayload.py: customize for OE

Upstream-Status: Inappropriate [oe specific]

Signed-off-by: Hongxu Jia <hongxu.jia@windriver.com>
---
 pyanaconda/payload/dnfpayload.py | 360 ++++++++++++++++++++++++++++++---------
 1 file changed, 281 insertions(+), 79 deletions(-)

diff --git a/pyanaconda/payload/dnfpayload.py b/pyanaconda/payload/dnfpayload.py
index 664a8f6..afff294 100644
--- a/pyanaconda/payload/dnfpayload.py
+++ b/pyanaconda/payload/dnfpayload.py
@@ -18,6 +18,7 @@
 # Red Hat, Inc.
 #
 import os
+import os.path
 
 from blivet.size import Size
 from pykickstart.constants import GROUP_ALL, GROUP_DEFAULT, KS_MISSING_IGNORE
@@ -45,6 +46,11 @@ import time
 import threading
 from requests.exceptions import RequestException
 
+import inspect
+if __name__ == "__main__":
+    from pyanaconda import anaconda_logging
+    anaconda_logging.init()
+    anaconda_logging.logger.setupVirtio()
 
 from pyanaconda.anaconda_loggers import get_packaging_logger, get_dnf_logger
 log = get_packaging_logger()
@@ -56,6 +62,7 @@ import dnf.repo
 import dnf.callback
 import dnf.conf.parser
 import dnf.conf.substitutions
+import dnf.const
 import rpm
 import librepo
 
@@ -227,7 +234,17 @@ class PayloadRPMDisplay(dnf.callback.TransactionProgress):
         """Report an error that occurred during the transaction. Message is a
         string which describes the error.
         """
-        self._queue.put(('error', message))
+        token = "error"
+        if message.startswith("Non-fatal"):
+            token = "warn"
+        self._queue.put((token, message))
+        log.info("error: %s" % message)
+
+    def scriptout(self, msgs):
+        if msgs is None:
+            return
+
+        log.info("scriptout: %s" % msgs)
 
 
 class DownloadProgress(dnf.callback.DownloadProgress):
@@ -297,6 +314,12 @@ class DNFPayload(payload.PackagePayload):
         self._updates_enabled = True
         self._configure()
 
+        self.requirements.add_packages(['base-files', 'base-passwd', 'shadow'], reason="basic packages")
+        # Support grub-mkconfig
+        self.requirements.add_packages(['sed', 'coreutils'], reason="Support grub-mkconfig")
+        # Support create new user
+        self.requirements.add_packages(['shadow'], reason="Support create new user")
+
         # Protect access to _base.repos to ensure that the dictionary is not
         # modified while another thread is attempting to iterate over it. The
         # lock only needs to be held during operations that change the number
@@ -308,12 +331,26 @@ class DNFPayload(payload.PackagePayload):
 
         self.requirements.set_apply_callback(self._apply_requirements)
 
+        # OE specific
+        self.image = {}
+        self.tasks = {}
+
+    def setup(self, storage, instClass):
+        log.info("%s %s" % (self.__class__.__name__, inspect.stack()[0][3]))
+
+        self.image, self.tasks = instClass.read_buildstamp()
+        super().setup(storage, instClass)
+
     def unsetup(self):
         super().unsetup()
         self._base = None
         self._configure()
         self._repoMD_list = []
 
+        # OE specific
+        self.image = {}
+        self.tasks = {}
+
     def _replace_vars(self, url):
         """Replace url variables with their values.
 
@@ -413,21 +450,12 @@ class DNFPayload(payload.PackagePayload):
         super().addRepo(ksrepo)
 
     def _apply_selections(self):
-        if self.data.packages.nocore:
-            log.info("skipping core group due to %%packages --nocore; system may not be complete")
-        else:
-            try:
-                self._select_group('core', required=True)
-                log.info("selected group: core")
-            except payload.NoSuchGroup as e:
-                self._miss(e)
-
         env = None
 
-        if self.data.packages.default and self.environments:
-            env = self.environments[0]
-        elif self.data.packages.environment:
+        if self.data.packages.environment:
             env = self.data.packages.environment
+        elif self.environments:
+            env = self.environments[0]
 
         excludedGroups = [group.name for group in self.data.packages.excludedGroupList]
 
@@ -480,7 +508,7 @@ class DNFPayload(payload.PackagePayload):
             log.debug("selected package: %s, requirement for %s %s",
                        req.id, req.reasons, ", ".join(ignore_msgs))
 
-        for req in self.requirements.groups:
+        for req in self.requirements.groups or []:
             try:
                 self._select_group(req.id, required=req.strong)
                 log.debug("selected group: %s, requirement for %s",
@@ -519,6 +547,33 @@ class DNFPayload(payload.PackagePayload):
             conf.proxy_username = None
             conf.proxy_password = None
 
+    def _oe_configure(self, dnf_conf):
+        # Refers oe-core's recipes of dnf and rpm
+        os.environ['RPM_NO_CHROOT_FOR_SCRIPTS'] = "1"
+
+        # Refers oe-core's meta/lib/oe/package_manager.py
+        # def _prepare_pkg_transaction(self) in class RpmPM(PackageManager)
+        target_rootfs = util.getSysroot()
+        os.environ['D'] = target_rootfs
+        os.environ['OFFLINE_ROOT'] = target_rootfs
+        os.environ['IPKG_OFFLINE_ROOT'] = target_rootfs
+        os.environ['OPKG_OFFLINE_ROOT'] = target_rootfs
+
+        # Refer oe-core's meta/lib/oe/package_manager.py
+        # def _configure_dnf(self) in class RpmPM(PackageManager)
+        # directly read its result from installer system
+        dnf_conf.substitutions['arch'] = open("/etc/dnf/vars/arch", "r").readline()
+
+    def _oe_configure_post(self):
+        # Refers oe-core's meta/lib/oe/package_manager.py
+        # def _configure_rpm(self) and def _configure_dnf(self)
+        # in class RpmPM(PackageManager)
+        # directly copy its result from installer system
+        target_sysconf = "%s/etc" % util.getSysroot()
+        util.execWithRedirect("cp", ["-fr", "/etc/rpmrc", target_sysconf])
+        util.execWithRedirect("cp", ["-rf", "/etc/dnf", target_sysconf])
+        util.execWithRedirect("cp", ["-rf", "/etc/rpm", target_sysconf])
+
     def _configure(self):
         self._base = dnf.Base()
         conf = self._base.conf
@@ -530,12 +585,17 @@ class DNFPayload(payload.PackagePayload):
         conf.installroot = util.getSysroot()
         conf.prepend_installroot('persistdir')
 
+        self._oe_configure(conf)
+
         self._base.conf.substitutions.update_from_etc(conf.installroot)
 
         # NSS won't survive the forking we do to shield out chroot during
         # transaction, disable it in RPM:
         conf.tsflags.append('nocrypto')
 
+        conf.errorlevel = dnf.const.VERBOSE_LEVEL
+        conf.debuglevel = dnf.const.VERBOSE_LEVEL
+
         if self.data.packages.multiLib:
             conf.multilib_policy = "all"
 
@@ -588,9 +648,9 @@ class DNFPayload(payload.PackagePayload):
         # as excluded from the installable set
         return self._base.sack.add_excludes(pkgs)
 
-    def _install_package(self, pkg_name, required=False):
+    def _install_package(self, pkg_name, required=False, strict=True):
         try:
-            return self._base.install(pkg_name)
+            return self._base.install(pkg_name, strict=strict)
         except dnf.exceptions.MarkingError:
             raise payload.NoSuchPackage(pkg_name, required=required)
 
@@ -626,24 +686,48 @@ class DNFPayload(payload.PackagePayload):
         return pkgdir
 
     def _select_group(self, group_id, default=True, optional=False, required=False):
-        grp = self._base.comps.group_by_pattern(group_id)
-        if grp is None:
-            raise payload.NoSuchGroup(group_id, required=required)
-        types = {'mandatory'}
-        if default:
-            types.add('default')
-        if optional:
-            types.add('optional')
-        try:
-            self._base.group_install(grp.id, types)
-        except dnf.exceptions.MarkingError as e:
-            # dnf-1.1.9 raises this error when a package is missing from a group
-            raise payload.NoSuchPackage(str(e), required=True)
-        except dnf.exceptions.CompsError as e:
-            # DNF raises this when it is already selected
-            log.debug(e)
+        log.info("_select_group %s" % group_id)
+        if not group_id:
+            return
+
+        glob = self._complementary_glob(group_id)
+
+        # get all available languages in repos
+        available_packs = self._base.sack.query().available() \
+            .filter(name__glob = "*-%s"%glob)
+        allpkgnames = [p.name for p in available_packs]
+        log.info("allpkgnames %d" % len(allpkgnames))
+        for pkgname in allpkgnames:
+            self._install_package(pkgname, strict=False)
+
+    def _select_linguas(self, image_linguas):
+        lc_globs = []
+        for ling in image_linguas.split():
+            if ling not in lc_globs:
+                lc_globs.append(ling)
+
+            baselang = ling.split('-')[0]
+            if ling != baselang and baselang not in lc_globs:
+                lc_globs.append(baselang)
+
+        log.info("lc_globs: %s" % lc_globs)
+        for glob in lc_globs:
+            self._select_group("locale-%s" % glob)
+
 
     def _select_environment(self, env_id, excluded):
+        (image_name, description)  = self.environmentDescription(env_id)
+        log.info("image_name %s, des %s" % (image_name, description))
+        image_id = image_name.split()[0]
+        (image_summary, image_description, package_install, package_install_attemptonly, image_linguas) = self.image[image_id]
+        log.info("package_install %s, attempt %s, linguas %s" %
+                   (package_install, package_install_attemptonly, image_linguas))
+
+        for pkg in package_install.split():
+            self._install_package(pkg)
+
+        self._select_linguas(image_linguas)
+
         # dnf.base.environment_install excludes on packages instead of groups,
         # which is unhelpful. Instead, use group_install for each group in
         # the environment so we can skip the ones that are excluded.
@@ -693,20 +777,25 @@ class DNFPayload(payload.PackagePayload):
     def baseRepo(self):
         # is any locking needed here?
         repo_names = [constants.BASE_REPO_NAME] + self.DEFAULT_REPOS
+        log.info("repo_names %s" % repo_names)
         with self._repos_lock:
             for repo in self._base.repos.iter_enabled():
+                log.info("repo.id %s" % repo.id)
                 if repo.id in repo_names:
                     return repo.id
         return None
 
     @property
     def environments(self):
-        return [env.id for env in self._base.comps.environments]
+        """ List of environment ids. """
+        log.info("%s %s: %s" % (self.__class__.__name__, inspect.stack()[0][3], self.tasks.keys()))
+        return sorted(self.tasks.keys())
 
     @property
     def groups(self):
-        groups = self._base.comps.groups_iter()
-        return [g.id for g in groups]
+        """ List of group ids. """
+        log.info("%s %s" % (self.__class__.__name__, inspect.stack()[0][3]))
+        return sorted(self.tasks.keys())
 
     @property
     def mirrorEnabled(self):
@@ -776,7 +865,7 @@ class DNFPayload(payload.PackagePayload):
         return grp.visible
 
     def _groupHasInstallableMembers(self, grpid):
-        return True
+        return False
 
     def checkSoftwareSelection(self):
         log.info("checking software selection")
@@ -831,50 +920,49 @@ class DNFPayload(payload.PackagePayload):
         super().enableRepo(repo_id)
 
     def environmentDescription(self, environmentid):
-        env = self._base.comps.environment_by_pattern(environmentid)
-        if env is None:
-            raise payload.NoSuchGroup(environmentid)
-        return (env.ui_name, env.ui_description)
+        log.info("%s %s, %s" % (self.__class__.__name__, inspect.stack()[0][3], environmentid))
+
+        if environmentid not in self.tasks:
+                raise NoSuchGroup(environmentid)
+
+        if environmentid in self.tasks:
+            log.info("environmentDescription %s" % self.tasks)
+            (name, description, group) = self.tasks[environmentid]
+
+            return (name, description)
+        return (environmentid, environmentid)
 
     def environmentId(self, environment):
         """Return environment id for the environment specified by id or name."""
-        env = self._base.comps.environment_by_pattern(environment)
-        if env is None:
-            raise payload.NoSuchGroup(environment)
-        return env.id
+        log.info("%s %s, environment %s" % (self.__class__.__name__, inspect.stack()[0][3], environment))
+        # TODO
+        return environment
 
     def environmentGroups(self, environmentid, optional=True):
-        env = self._base.comps.environment_by_pattern(environmentid)
-        if env is None:
-            raise payload.NoSuchGroup(environmentid)
-        group_ids = (id_.name for id_ in env.group_ids)
-        option_ids = (id_.name for id_ in env.option_ids)
-        if optional:
-            return list(itertools.chain(group_ids, option_ids))
-        else:
-            return list(group_ids)
+        log.info("%s %s, %s" % (self.__class__.__name__, inspect.stack()[0][3], environmentid))
+
+        if environmentid in self.tasks:
+            log.info("environmentDescription %s" % self.tasks)
+            (name, description, groups) = self.tasks[environmentid]
+
+            return groups.split()
+        return [environmentid]
 
     def environmentHasOption(self, environmentid, grpid):
-        env = self._base.comps.environment_by_pattern(environmentid)
-        if env is None:
-            raise payload.NoSuchGroup(environmentid)
-        return grpid in (id_.name for id_ in env.option_ids)
+        log.info("%s %s, %s, %s" % (self.__class__.__name__, inspect.stack()[0][3], environmentid, grpid))
+        # TODO
+        return True
 
     def environmentOptionIsDefault(self, environmentid, grpid):
-        env = self._base.comps.environment_by_pattern(environmentid)
-        if env is None:
-            raise payload.NoSuchGroup(environmentid)
-
-        # Look for a group in the optionlist that matches the group_id and has
-        # default set
-        return any(grp for grp in env.option_ids if grp.name == grpid and grp.default)
+        log.info("%s %s, %s, %s" % (self.__class__.__name__, inspect.stack()[0][3], environmentid, grpid))
+        # TODO
+        return True
 
     def groupDescription(self, grpid):
         """Return name/description tuple for the group specified by id."""
-        grp = self._base.comps.group_by_pattern(grpid)
-        if grp is None:
-            raise payload.NoSuchGroup(grpid)
-        return (grp.ui_name, grp.ui_description)
+        log.info("%s %s, %s" % (self.__class__.__name__, inspect.stack()[0][3], grpid))
+        (name, description, group) = self.tasks[grpid]
+        return (name, description)
 
     def groupId(self, group_name):
         """Translate group name to group ID.
@@ -897,6 +985,31 @@ class DNFPayload(payload.PackagePayload):
         self._base.read_comps()
         self._refreshEnvironmentAddons()
 
+    # Refer upstream oe-core meta/lib/oe/package_manager.py
+    def _save_rpmpostinst(self, pkg):
+        def _script_num_prefix(path):
+            files = os.listdir(path)
+            numbers = set()
+            numbers.add(99)
+            for f in files:
+                numbers.add(int(f.split("-")[0]))
+            return max(numbers) + 1
+
+        log.info("Saving postinstall script of %s" % (pkg))
+
+        target_rootfs = util.getSysroot()
+        args = ["-q", "--root=%s" % target_rootfs, "--queryformat", "%{postin}", pkg]
+
+        output = util.execWithCapture('rpm', args, log_output=False)
+        log.info(output)
+
+        target_path = "%s/etc/rpm-postinsts" % target_rootfs
+        util.execWithRedirect("mkdir", ["-p", target_path])
+        num = _script_num_prefix(target_path)
+        saved_script_name = os.path.join(target_path, "%d-%s" % (num, pkg))
+        open(saved_script_name, 'w').write(output)
+        os.chmod(saved_script_name, 0o755)
+
     def install(self):
         progress_message(N_('Starting package installation process'))
 
@@ -931,6 +1044,7 @@ class DNFPayload(payload.PackagePayload):
                 _failure_limbo()
 
         log.info('Downloading packages finished.')
+        failed_scriptlets_pkgnames = collections.OrderedDict()
 
         pre_msg = (N_("Preparing transaction from installation source"))
         progress_message(pre_msg)
@@ -963,6 +1077,9 @@ class DNFPayload(payload.PackagePayload):
             elif token == 'quit':
                 msg = ("Payload error - DNF installation has ended up abruptly: %s" % msg)
                 raise payload.PayloadError(msg)
+            elif token == 'warn':
+                if msg.startswith("Non-fatal POSTIN scriptlet failure in rpm package"):
+                    failed_scriptlets_pkgnames[msg.split()[-1]] = True
             elif token == 'error':
                 exc = payload.PayloadInstallError("DNF error: %s" % msg)
                 if errors.errorHandler.cb(exc) == errors.ERROR_RAISE:
@@ -972,6 +1089,10 @@ class DNFPayload(payload.PackagePayload):
 
         process.join()
         self._base.close()
+
+        for pkg in failed_scriptlets_pkgnames.keys():
+            self._save_rpmpostinst(pkg)
+
         if os.path.exists(self._download_location):
             log.info("Cleaning up downloaded packages: %s", self._download_location)
             shutil.rmtree(self._download_location)
@@ -982,6 +1103,39 @@ class DNFPayload(payload.PackagePayload):
             # we don't have to care about clearing the download location ourselves.
             log.warning("Can't delete nonexistent download location: %s", self._download_location)
 
+    def _list_installed(self):
+        output = util.execWithCapture(
+                      'dnf',
+                      [
+                        'repoquery',
+                        '--installed',
+                        '--installroot='+util.getSysroot(),
+                        '--queryformat=Package:%{name}'
+                      ],
+                      log_output=False
+                 )
+        output = output.rstrip()
+        lines = output.splitlines()
+        installed = []
+        for line in lines:
+            if not line.startswith("Package:"):
+                continue
+            installed.append(line.replace("Package:", ""))
+        return installed
+
+    def _complementary_glob(self, group):
+        complementary_glob = {}
+        complementary_glob['dev-pkgs'] = 'dev'
+        complementary_glob['staticdev-pkgs'] = 'staticdev'
+        complementary_glob['doc-pkgs'] = 'doc'
+        complementary_glob['dbg-pkgs'] = 'dbg'
+        complementary_glob['ptest-pkgs'] = 'ptest'
+
+        if group in complementary_glob:
+            return complementary_glob[group]
+
+        return group
+
     def getRepo(self, repo_id):
         """Return the yum repo object."""
         return self._base.repos[repo_id]
@@ -1004,16 +1158,7 @@ class DNFPayload(payload.PackagePayload):
         return True
 
     def languageGroups(self):
-        localization_proxy = LOCALIZATION.get_proxy()
-        locales = [localization_proxy.Language] + localization_proxy.LanguageSupport
-        match_fn = pyanaconda.localization.langcode_matches_locale
-        gids = set()
-        gl_tuples = ((g.id, g.lang_only) for g in self._base.comps.groups_iter())
-        for (gid, lang) in gl_tuples:
-            for locale in locales:
-                if match_fn(lang, locale):
-                    gids.add(gid)
-        return list(gids)
+        return []
 
     def reset(self):
         super().reset()
@@ -1025,7 +1170,6 @@ class DNFPayload(payload.PackagePayload):
         self._repoMD_list = []
 
     def updateBaseRepo(self, fallback=True, checkmount=True):
-        log.info('configuring base repo')
         self.reset()
         url, mirrorlist, metalink = self._setupInstallDevice(self.storage,
                                                              checkmount)
@@ -1033,6 +1177,8 @@ class DNFPayload(payload.PackagePayload):
         sslverify = True
         if method.method == "url":
             sslverify = not (method.noverifyssl or flags.noverifyssl)
+        log.info('configuring base repo: url %s, mirrorlist %s, sslverify %s, method %s'
+                    % (url, mirrorlist, sslverify, method.method))
 
         # Read in all the repos from the installation environment, make a note of which
         # are enabled, and then disable them all.  If the user gave us a method, we want
@@ -1204,6 +1350,17 @@ class DNFPayload(payload.PackagePayload):
 
         super().postInstall()
 
+        self._oe_configure_post()
+
+        # For the generation of anaconda-ks.cfg
+        for pkg in self._list_installed():
+            if pkg not in self.data.packages.packageList:
+                self.data.packages.packageList.append(pkg)
+
+        util.execWithRedirect("depmod",
+                              ["-a"],
+                              root=util.getSysroot())
+
     def writeStorageLate(self):
         pass
 
@@ -1277,3 +1434,48 @@ class RepoMDMetaHash(object):
                 log.debug("Can't download new repomd.xml from %s with proxy: %s. Error: %s", url, proxies, e)
 
         return repomd
+
+
+# For testing
+if __name__ == "__main__":
+    log.info("God bless us")
+
+    from pykickstart.version import makeVersion
+
+    # set up ksdata
+    ksdata = makeVersion()
+
+    ksdata.method.method = "cdrom"
+
+    from pyanaconda.installclass import factory
+    instClass = factory.get_best_install_class()
+
+    from pyanaconda.storage.osinstall import InstallerStorage
+    storage = InstallerStorage(ksdata=ksdata)
+    storage.set_default_fstype("ext4")
+
+    _payload = DNFPayload(ksdata)
+
+    _payload.setup(storage, instClass)
+
+    # Keep setting up package-based repositories
+    # Download package metadata
+    try:
+        _payload.updateBaseRepo(fallback=False, checkmount=True)
+    except payload.PayloadError as e:
+        log.error("PayloadError: %s", e)
+        _payload.unsetup()
+        exit(0)
+
+    # Gather the group data
+    _payload.gatherRepoMetadata()
+    _payload.release()
+    _payload.addDriverRepos()
+
+    log.info("baseRepo %s" % _payload.baseRepo)
+
+    _payload.preInstall()
+
+    _payload.install()
+
+    _payload.postInstall()
-- 
2.7.4