path: root/recipes-installer/anaconda/files/0007-dnfpayload.py-customize-for-OE.patch

From f1e88262ef77bcd9d0ec50180ecd1a9c0f646049 Mon Sep 17 00:00:00 2001
From: Hongxu Jia <hongxu.jia@windriver.com>
Date: Tue, 25 Jun 2019 10:34:14 +0800
Subject: [PATCH] dnfpayload.py: customize for OE

Upstream-Status: Inappropriate [oe specific]

Signed-off-by: Hongxu Jia <hongxu.jia@windriver.com>
---
 pyanaconda/payload/dnfpayload.py | 397 +++++++++++++++++++++++++++++++++------
 1 file changed, 340 insertions(+), 57 deletions(-)

diff --git a/pyanaconda/payload/dnfpayload.py b/pyanaconda/payload/dnfpayload.py
index 1f2925d..53d40eb 100644
--- a/pyanaconda/payload/dnfpayload.py
+++ b/pyanaconda/payload/dnfpayload.py
@@ -17,6 +17,7 @@
 # Red Hat, Inc.
 #
 import os
+import os.path
 import configparser
 import collections
 import multiprocessing
@@ -55,6 +56,7 @@ import dnf.transaction
 import libdnf.conf
 import rpm
 from dnf.const import GROUP_PACKAGE_TYPES
+import dnf.const
 
 from blivet.size import Size
 from pykickstart.constants import GROUP_ALL, GROUP_DEFAULT, KS_MISSING_IGNORE
@@ -62,6 +64,11 @@ from pykickstart.constants import GROUP_ALL, GROUP_DEFAULT, KS_MISSING_IGNORE
 from pyanaconda.anaconda_loggers import get_packaging_logger, get_dnf_logger
 log = get_packaging_logger()
 
+import inspect
+if __name__ == "__main__":
+    from pyanaconda import anaconda_logging
+    anaconda_logging.init()
+    anaconda_logging.logger.setupVirtio(constants.VIRTIO_PORT)
 
 DNF_CACHE_DIR = '/tmp/dnf.cache'
 DNF_PLUGINCONF_DIR = '/tmp/dnf.pluginconf'
@@ -232,7 +239,17 @@ class PayloadRPMDisplay(dnf.callback.TransactionProgress):
         """Report an error that occurred during the transaction. Message is a
         string which describes the error.
         """
-        self._queue.put(('error', message))
+        token = "error"
+        if message.startswith("Error in POSTIN scriptlet in rpm package"):
+            token = "warn"
+        self._queue.put((token, message))
+        log.info("error: %s" % message)
+
+    def scriptout(self, msgs):
+        if msgs is None:
+            return
+
+        log.info("scriptout: %s" % msgs)
 
 
 class DownloadProgress(dnf.callback.DownloadProgress):
@@ -302,6 +319,12 @@ class DNFPayload(payload.PackagePayload):
         self._updates_enabled = True
         self._configure()
 
+        self.requirements.add_packages(['base-files', 'base-passwd', 'shadow'], reason="basic packages")
+        # Support grub-mkconfig
+        self.requirements.add_packages(['sed', 'coreutils'], reason="Support grub-mkconfig")
+        # Support create new user
+        self.requirements.add_packages(['shadow'], reason="Support create new user")
+
         # Protect access to _base.repos to ensure that the dictionary is not
         # modified while another thread is attempting to iterate over it. The
         # lock only needs to be held during operations that change the number
@@ -315,12 +338,28 @@ class DNFPayload(payload.PackagePayload):
         self._req_packages = set()
         self.requirements.set_apply_callback(self._apply_requirements)
 
+        # OE specific
+        self.image = {}
+        self.tasks = {}
+
+    def setup(self, storage):
+        log.info("%s %s" % (self.__class__.__name__, inspect.stack()[0][3]))
+
+        # OE specific
+        self.image, self.tasks = self._read_buildstamp()
+
+        super().setup(storage)
+
     def unsetup(self):
         super().unsetup()
         self._base = None
         self._configure()
         self._repoMD_list = []
 
+        # OE specific
+        self.image = {}
+        self.tasks = {}
+
     def _replace_vars(self, url):
         """Replace url variables with their values.
 
@@ -482,24 +521,16 @@ class DNFPayload(payload.PackagePayload):
             log.debug("excluding group %s", group.name)
             exclude_list.append("@{}".format(group.name))
 
-        # core groups
-        if self.data.packages.nocore:
-            log.info("skipping core group due to %%packages --nocore; system may not be complete")
-            exclude_list.append("@core")
-        else:
-            log.info("selected group: core")
-            include_list.append("@core")
-
         # environment
         env = None
-        if self.data.packages.default and self.environments:
-            env = self.environments[0]
-            log.info("selecting default environment: %s", env)
-        elif self.data.packages.environment:
+        if self.data.packages.environment:
             env = self.data.packages.environment
             log.info("selected environment: %s", env)
-        if env:
-            include_list.append("@{}".format(env))
+        elif self.environments:
+            env = self.environments[0]
+        for pkg_name in self._apply_yocto_environment(env):
+            log.info("selected yocto package: '%s'", pkg_name)
+            include_list.append(pkg_name)
 
         # groups from kickstart data
         for group in self.data.packages.groupList:
@@ -579,6 +610,62 @@ class DNFPayload(payload.PackagePayload):
         except Exception as e:  # pylint: disable=broad-except
             self._payload_setup_error(e)
 
+    def _apply_yocto_environment(self, env):
+        if not env:
+            return []
+
+        (image_name, description)  = self.environment_description(env)
+        log.info("image_name %s, des %s" % (image_name, description))
+        image_id = image_name.split()[0]
+        (image_summary, image_description, package_install, package_install_attemptonly, image_linguas) = self.image[image_id]
+        log.info("package_install %s, attempt %s, linguas %s" %
+                   (package_install, package_install_attemptonly, image_linguas))
+
+        # Language locale
+        self._select_yocto_linguas(image_linguas)
+
+        # -dbg/-doc/-dev
+        for group in self.tasks[env][2].split():
+            self._select_yocto_group(group)
+
+        # Installing packages
+        return package_install.split()
+
+    def _select_yocto_group(self, group_id):
+        log.info("_select_yocto_group %s" % group_id)
+        if not group_id:
+            return []
+
+        glob = self._yocto_complementary_glob(group_id)
+
+        # get all available languages in repos
+        available_packs = self._base.sack.query().available() \
+            .filter(name__glob = "*-%s"%glob)
+        allpkgnames = [p.name for p in available_packs]
+        log.info("allpkgnames %d" % len(allpkgnames))
+        for pkgname in allpkgnames:
+            self._try_install_package(pkgname)
+
+    def _select_yocto_linguas(self, image_linguas):
+        lc_globs = []
+        for ling in image_linguas.split():
+            if ling not in lc_globs:
+                lc_globs.append(ling)
+
+            baselang = ling.split('-')[0]
+            if ling != baselang and baselang not in lc_globs:
+                lc_globs.append(baselang)
+
+        log.info("lc_globs: %s" % lc_globs)
+        for glob in lc_globs:
+            self._select_yocto_group("locale-%s" % glob)
+
+    def _try_install_package(self, pkg_name):
+        try:
+            return self._base.install(pkg_name, strict=False)
+        except dnf.exceptions.MarkingError:
+            raise payload.NoSuchPackage(pkg_name, required=False)
+
     def _apply_requirements(self, requirements):
         self._req_groups = set()
         self._req_packages = set()
@@ -594,7 +681,7 @@ class DNFPayload(payload.PackagePayload):
             log.debug("selected package: %s, requirement for %s %s",
                       req.id, req.reasons, ", ".join(ignore_msgs))
 
-        for req in self.requirements.groups:
+        for req in self.requirements.groups or []:
             # NOTE: req.strong not handled yet
             log.debug("selected group: %s, requirement for %s",
                       req.id, req.reasons)
@@ -645,16 +732,47 @@ class DNFPayload(payload.PackagePayload):
         if os.path.exists("/etc/os-release"):
             config = SimpleConfigFile()
             config.read("/etc/os-release")
-            os_release_platform_id = config.get("PLATFORM_ID")
+            os_release_platform_id = config.get("VERSION_ID")
             # simpleconfig return "" for keys that are not found
             if os_release_platform_id:
                 platform_id = os_release_platform_id
             else:
-                log.error("PLATFORM_ID missing from /etc/os-release")
+                log.error("VERSION_ID missing from /etc/os-release")
         else:
             log.error("/etc/os-release is missing, platform id can't be obtained")
         return platform_id
 
+    def _oe_configure(self, dnf_conf):
+        # Refers oe-core's recipes of dnf and rpm
+        os.environ['RPM_NO_CHROOT_FOR_SCRIPTS'] = "1"
+
+        # Refers oe-core's meta/lib/oe/package_manager.py
+        # def _prepare_pkg_transaction(self) in class RpmPM(PackageManager)
+        target_rootfs = util.getSysroot()
+        os.environ['D'] = target_rootfs
+        os.environ['OFFLINE_ROOT'] = target_rootfs
+        os.environ['IPKG_OFFLINE_ROOT'] = target_rootfs
+        os.environ['OPKG_OFFLINE_ROOT'] = target_rootfs
+
+        # Refer oe-core's meta/lib/oe/package_manager.py
+        # def _configure_dnf(self) in class RpmPM(PackageManager)
+        # directly read its result from installer system
+        dnf_conf.substitutions['arch'] = open("/etc/dnf/vars/arch", "r").readline()
+
+        dnf_conf.errorlevel = dnf.const.VERBOSE_LEVEL
+        dnf_conf.debuglevel = dnf.const.VERBOSE_LEVEL
+
+
+    def _oe_configure_post(self):
+        # Refers oe-core's meta/lib/oe/package_manager.py
+        # def _configure_rpm(self) and def _configure_dnf(self)
+        # in class RpmPM(PackageManager)
+        # directly copy its result from installer system
+        target_sysconf = "%s/etc" % util.getSysroot()
+        util.execWithRedirect("cp", ["-fr", "/etc/rpmrc", target_sysconf])
+        util.execWithRedirect("cp", ["-rf", "/etc/dnf", target_sysconf])
+        util.execWithRedirect("cp", ["-rf", "/etc/rpm", target_sysconf])
+
     def _configure(self):
         self._base = dnf.Base()
         config = self._base.conf
@@ -674,6 +792,8 @@ class DNFPayload(payload.PackagePayload):
         config.installroot = util.getSysroot()
         config.prepend_installroot('persistdir')
 
+        self._oe_configure(config)
+
         self._base.conf.substitutions.update_from_etc(config.installroot)
 
         if self.data.packages.multiLib:
@@ -801,20 +921,23 @@ class DNFPayload(payload.PackagePayload):
     def base_repo(self):
         # is any locking needed here?
         repo_names = [constants.BASE_REPO_NAME] + constants.DEFAULT_REPOS
+        log.info("repo_names %s" % repo_names)
         with self._repos_lock:
             for repo in self._base.repos.iter_enabled():
+                log.info("repo.id %s" % repo.id)
                 if repo.id in repo_names:
                     return repo.id
         return None
 
     @property
     def environments(self):
-        return [env.id for env in self._base.comps.environments]
+        log.info("%s %s: %s" % (self.__class__.__name__, inspect.stack()[0][3], self.tasks.keys()))
+        return sorted(self.tasks.keys())
 
     @property
     def groups(self):
-        groups = self._base.comps.groups_iter()
-        return [g.id for g in groups]
+        log.info("%s %s: %s" % (self.__class__.__name__, inspect.stack()[0][3], self.tasks.keys()))
+        return []
 
     @property
     def repos(self):
@@ -936,42 +1059,39 @@ class DNFPayload(payload.PackagePayload):
         super().enable_repo(repo_id)
 
     def environment_description(self, environment_id):
-        env = self._base.comps.environment_by_pattern(environment_id)
-        if env is None:
-            raise NoSuchGroup(environment_id)
-        return (env.ui_name, env.ui_description)
+        log.info("%s %s, %s" % (self.__class__.__name__, inspect.stack()[0][3], environment_id))
+
+        if environment_id not in self.tasks:
+                raise NoSuchGroup(environment_id)
+
+        if environment_id in self.tasks:
+            log.info("environmentDescription %s" % self.tasks)
+            (name, description, group) = self.tasks[environment_id]
+
+            return (name, description)
+        return (environment_id, environment_id)
 
     def environment_id(self, environment):
         """Return environment id for the environment specified by id or name."""
-        # the enviroment must be string or else DNF >=3 throws an assert error
-        if not isinstance(environment, str):
-            log.warning("environment_id() called with non-string argument: %s", environment)
-        env = self._base.comps.environment_by_pattern(environment)
-        if env is None:
-            raise NoSuchGroup(environment)
-        return env.id
+        log.info("%s %s, environment %s" % (self.__class__.__name__, inspect.stack()[0][3], environment))
+        # TODO
+        return environment
 
     def environment_has_option(self, environment_id, grpid):
-        env = self._base.comps.environment_by_pattern(environment_id)
-        if env is None:
-            raise NoSuchGroup(environment_id)
-        return grpid in (id_.name for id_ in env.option_ids)
+        log.info("%s %s, %s, %s" % (self.__class__.__name__, inspect.stack()[0][3], environment_id, grpid))
+        # TODO
+        return True
 
     def environment_option_is_default(self, environment_id, grpid):
-        env = self._base.comps.environment_by_pattern(environment_id)
-        if env is None:
-            raise NoSuchGroup(environment_id)
-
-        # Look for a group in the optionlist that matches the group_id and has
-        # default set
-        return any(grp for grp in env.option_ids if grp.name == grpid and grp.default)
+        log.info("%s %s, %s, %s" % (self.__class__.__name__, inspect.stack()[0][3], environment_id, grpid))
+        # TODO
+        return True
 
     def group_description(self, grpid):
         """Return name/description tuple for the group specified by id."""
-        grp = self._base.comps.group_by_pattern(grpid)
-        if grp is None:
-            raise NoSuchGroup(grpid)
-        return (grp.ui_name, grp.ui_description)
+        log.info("%s %s, %s" % (self.__class__.__name__, inspect.stack()[0][3], grpid))
+        (name, description, group) = self.tasks[grpid]
+        return (name, description)
 
     def group_id(self, group_name):
         """Translate group name to group ID.
@@ -994,6 +1114,31 @@ class DNFPayload(payload.PackagePayload):
         self._base.read_comps()
         self._refresh_environment_addons()
 
+    # Refer upstream oe-core meta/lib/oe/package_manager.py
+    def _save_rpmpostinst(self, pkg):
+        def _script_num_prefix(path):
+            files = os.listdir(path)
+            numbers = set()
+            numbers.add(99)
+            for f in files:
+                numbers.add(int(f.split("-")[0]))
+            return max(numbers) + 1
+
+        log.info("Saving postinstall script of %s" % (pkg))
+
+        target_rootfs = util.getSysroot()
+        args = ["-q", "--root=%s" % target_rootfs, "--queryformat", "%{postin}", pkg]
+
+        output = util.execWithCapture('rpm', args, log_output=False)
+        log.info(output)
+
+        target_path = "%s/etc/rpm-postinsts" % target_rootfs
+        util.execWithRedirect("mkdir", ["-p", target_path])
+        num = _script_num_prefix(target_path)
+        saved_script_name = os.path.join(target_path, "%d-%s" % (num, pkg))
+        open(saved_script_name, 'w').write(output)
+        os.chmod(saved_script_name, 0o755)
+
     def install(self):
         progress_message(N_('Starting package installation process'))
 
@@ -1028,6 +1173,7 @@ class DNFPayload(payload.PackagePayload):
                 _failure_limbo()
 
         log.info('Downloading packages finished.')
+        failed_scriptlets_pkgnames = collections.OrderedDict()
 
         pre_msg = (N_("Preparing transaction from installation source"))
         progress_message(pre_msg)
@@ -1060,6 +1206,9 @@ class DNFPayload(payload.PackagePayload):
             elif token == 'quit':
                 msg = ("Payload error - DNF installation has ended up abruptly: %s" % msg)
                 raise PayloadError(msg)
+            elif token == 'warn':
+                if msg.startswith("Error in POSTIN scriptlet in rpm package"):
+                    failed_scriptlets_pkgnames[msg.split()[-1]] = True
             elif token == 'error':
                 exc = PayloadInstallError("DNF error: %s" % msg)
                 if errors.errorHandler.cb(exc) == errors.ERROR_RAISE:
@@ -1068,6 +1217,10 @@ class DNFPayload(payload.PackagePayload):
             (token, msg) = queue_instance.get()
 
         process.join()
+
+        for pkg in failed_scriptlets_pkgnames.keys():
+            self._save_rpmpostinst(pkg)
+
         # Don't close the mother base here, because we still need it.
         if os.path.exists(self._download_location):
             log.info("Cleaning up downloaded packages: %s", self._download_location)
@@ -1079,6 +1232,39 @@ class DNFPayload(payload.PackagePayload):
             # we don't have to care about clearing the download location ourselves.
             log.warning("Can't delete nonexistent download location: %s", self._download_location)
 
+    def _list_installed(self):
+        output = util.execWithCapture(
+                      'dnf',
+                      [
+                        'repoquery',
+                        '--installed',
+                        '--installroot='+util.getSysroot(),
+                        '--queryformat=Package:%{name}'
+                      ],
+                      log_output=False
+                 )
+        output = output.rstrip()
+        lines = output.splitlines()
+        installed = []
+        for line in lines:
+            if not line.startswith("Package:"):
+                continue
+            installed.append(line.replace("Package:", ""))
+        return installed
+
+    def _yocto_complementary_glob(self, group):
+        complementary_glob = {}
+        complementary_glob['dev-pkgs'] = 'dev'
+        complementary_glob['staticdev-pkgs'] = 'staticdev'
+        complementary_glob['doc-pkgs'] = 'doc'
+        complementary_glob['dbg-pkgs'] = 'dbg'
+        complementary_glob['ptest-pkgs'] = 'ptest'
+
+        if group in complementary_glob:
+            return complementary_glob[group]
+
+        return group
+
     def get_repo(self, repo_id):
         """Return the yum repo object."""
         return self._base.repos[repo_id]
@@ -1101,16 +1287,7 @@ class DNFPayload(payload.PackagePayload):
         return True
 
     def language_groups(self):
-        localization_proxy = LOCALIZATION.get_proxy()
-        locales = [localization_proxy.Language] + localization_proxy.LanguageSupport
-        match_fn = pyanaconda.localization.langcode_matches_locale
-        gids = set()
-        gl_tuples = ((g.id, g.lang_only) for g in self._base.comps.groups_iter())
-        for (gid, lang) in gl_tuples:
-            for locale in locales:
-                if match_fn(lang, locale):
-                    gids.add(gid)
-        return list(gids)
+        return []
 
     def reset(self):
         super().reset()
@@ -1134,6 +1311,8 @@ class DNFPayload(payload.PackagePayload):
         sslverify = True
         if method.method == "url":
             sslverify = not (method.noverifyssl or flags.noverifyssl)
+        log.info('configuring base repo: url %s, mirrorlist %s, sslverify %s, method %s'
+                    % (base_repo_url, mirrorlist, sslverify, method.method))
 
         # Read in all the repos from the installation environment, make a note of which
         # are enabled, and then disable them all.  If the user gave us a method, we want
@@ -1179,6 +1358,7 @@ class DNFPayload(payload.PackagePayload):
                     sslcacert=getattr(method, 'sslcacert', None),
                     sslclientcert=getattr(method, 'sslclientcert', None),
                     sslclientkey=getattr(method, 'sslclientkey', None))
+                log.debug("base_ksrepo enabled %s" % base_ksrepo.enabled)
                 self._add_repo(base_ksrepo)
                 self._fetch_md(base_ksrepo.name)
             except (MetadataError, PayloadError) as e:
@@ -1397,6 +1577,66 @@ class DNFPayload(payload.PackagePayload):
         self._base.close()
         super().post_install()
 
+        self._oe_configure_post()
+
+        # For the generation of anaconda-ks.cfg
+        for pkg in self._list_installed():
+            if pkg not in self.data.packages.packageList:
+                self.data.packages.packageList.append(pkg)
+
+        util.execWithRedirect("depmod",
+                              ["-a"],
+                              root=util.getSysroot())
+
+    def _read_buildstamp(self):
+        image = {}
+        tasks = {}
+
+        config = configparser.ConfigParser()
+        config.read(["/tmp/product/.buildstamp", "/.buildstamp", os.environ.get("PRODBUILDPATH", "")])
+
+        image_list = (config.get("Rootfs", "LIST") or "").split()
+        for image_name in image_list:
+            _id_ = image_name.split('-')[0]
+            image_summary = config.get(image_name, "SUMMARY")
+            image_description = config.get(image_name, "DESCRIPTION")
+            package_install = config.get(image_name, "PACKAGE_INSTALL")
+            package_install_attemptonly = config.get(image_name,
+                                                         "PACKAGE_INSTALL_ATTEMPTONLY")
+            image_linguas = config.get(image_name, "IMAGE_LINGUAS")
+            image[image_name] = (image_summary,
+                                 image_description,
+                                 package_install,
+                                 package_install_attemptonly,
+                                 image_linguas)
+
+            short_image = image_name.replace("%s-image-" % _id_, "")
+
+            taskid = short_image
+            name = image_name
+            description = "%s" % image_summary
+            group = ""
+            tasks[taskid] = (name, description, group)
+
+            taskid = "%s-dev" % short_image
+            name = "%s dev-pkgs staticdev-pkgs" % image_name
+            description = "%s with development files" % image_summary
+            group = "dev-pkgs staticdev-pkgs"
+            tasks[taskid] = (name, description, group)
+
+            taskid = "%s-dbg" % short_image
+            name = "%s dbg-pkgs" % image_name
+            description = "%s with debug symbols" % image_summary
+            group = "dbg-pkgs"
+            tasks[taskid] = (name, description, group)
+
+            taskid = "%s-dev-dbg" % short_image
+            name = "%s dev-pkgs staticdev-pkgs dbg-pkgs" % image_name
+            description = "%s with development files and debug symbols" % image_summary
+            group = "dev-pkgs staticdev-pkgs dbg-pkgs"
+            tasks[taskid] = (name, description, group)
+
+        return image, tasks
 
 class RepoMDMetaHash(object):
     """Class that holds hash of a repomd.xml file content from a repository.
@@ -1469,3 +1709,46 @@ class RepoMDMetaHash(object):
                           url, proxies, e)
 
         return repomd
+
+
+# For testing
+if __name__ == "__main__":
+    log.info("God bless us")
+
+    from pykickstart.version import makeVersion
+
+    # set up ksdata
+    from pyanaconda import kickstart
+    ksdata = kickstart.AnacondaKSHandler()
+
+    ksdata.method.method = "cdrom"
+
+    from pyanaconda.storage.osinstall import InstallerStorage
+    storage = InstallerStorage()
+    storage.set_default_fstype("ext4")
+
+    _payload = DNFPayload(ksdata)
+
+    _payload.setup(storage)
+
+    # Keep setting up package-based repositories
+    # Download package metadata
+    try:
+        _payload.update_base_repo(fallback=False, checkmount=True)
+    except payload.PayloadError as e:
+        log.error("PayloadError: %s", e)
+        _payload.unsetup()
+        exit(0)
+
+    # Gather the group data
+    _payload.gather_repo_metadata()
+    _payload.release()
+    _payload.add_driver_repos()
+
+    log.info("baseRepo %s" % _payload.base_repo)
+
+    _payload.pre_install()
+
+    _payload.install()
+
+    _payload.post_install()
-- 
2.7.4