diff options
Diffstat (limited to 'scripts/lib/wic/plugins/source')
| -rw-r--r-- | scripts/lib/wic/plugins/source/bootimg-efi.py | 271 | ||||
| -rw-r--r-- | scripts/lib/wic/plugins/source/bootimg-partition.py | 11 | ||||
| -rw-r--r-- | scripts/lib/wic/plugins/source/bootimg-pcbios.py | 12 | ||||
| -rw-r--r-- | scripts/lib/wic/plugins/source/empty.py | 89 | ||||
| -rw-r--r-- | scripts/lib/wic/plugins/source/isoimage-isohybrid.py | 28 | ||||
| -rw-r--r-- | scripts/lib/wic/plugins/source/rawcopy.py | 44 | ||||
| -rw-r--r-- | scripts/lib/wic/plugins/source/rootfs.py | 170 |
7 files changed, 552 insertions, 73 deletions
diff --git a/scripts/lib/wic/plugins/source/bootimg-efi.py b/scripts/lib/wic/plugins/source/bootimg-efi.py index 2cfdc10ecd..13a9cddf4e 100644 --- a/scripts/lib/wic/plugins/source/bootimg-efi.py +++ b/scripts/lib/wic/plugins/source/bootimg-efi.py @@ -12,7 +12,11 @@ import logging import os +import tempfile import shutil +import re + +from glob import glob from wic import WicError from wic.engine import get_custom_config @@ -31,6 +35,26 @@ class BootimgEFIPlugin(SourcePlugin): name = 'bootimg-efi' @classmethod + def _copy_additional_files(cls, hdddir, initrd, dtb): + bootimg_dir = get_bitbake_var("DEPLOY_DIR_IMAGE") + if not bootimg_dir: + raise WicError("Couldn't find DEPLOY_DIR_IMAGE, exiting") + + if initrd: + initrds = initrd.split(';') + for rd in initrds: + cp_cmd = "cp %s/%s %s" % (bootimg_dir, rd, hdddir) + exec_cmd(cp_cmd, True) + else: + logger.debug("Ignoring missing initrd") + + if dtb: + if ';' in dtb: + raise WicError("Only one DTB supported, exiting") + cp_cmd = "cp %s/%s %s" % (bootimg_dir, dtb, hdddir) + exec_cmd(cp_cmd, True) + + @classmethod def do_configure_grubefi(cls, hdddir, creator, cr_workdir, source_params): """ Create loader-specific (grub-efi) config @@ -49,18 +73,9 @@ class BootimgEFIPlugin(SourcePlugin): "get it from %s." % configfile) initrd = source_params.get('initrd') + dtb = source_params.get('dtb') - if initrd: - bootimg_dir = get_bitbake_var("DEPLOY_DIR_IMAGE") - if not bootimg_dir: - raise WicError("Couldn't find DEPLOY_DIR_IMAGE, exiting") - - initrds = initrd.split(';') - for rd in initrds: - cp_cmd = "cp %s/%s %s" % (bootimg_dir, rd, hdddir) - exec_cmd(cp_cmd, True) - else: - logger.debug("Ignoring missing initrd") + cls._copy_additional_files(hdddir, initrd, dtb) if not custom_cfg: # Create grub configuration using parameters from wks file @@ -94,6 +109,9 @@ class BootimgEFIPlugin(SourcePlugin): grubefi_conf += " /%s" % rd grubefi_conf += "\n" + if dtb: + grubefi_conf += "devicetree /%s\n" % dtb + grubefi_conf += "}\n" logger.debug("Writing grubefi config %s/hdd/boot/EFI/BOOT/grub.cfg", @@ -115,24 +133,18 @@ class BootimgEFIPlugin(SourcePlugin): bootloader = creator.ks.bootloader + unified_image = source_params.get('create-unified-kernel-image') == "true" + loader_conf = "" - loader_conf += "default boot\n" + if not unified_image: + loader_conf += "default boot\n" loader_conf += "timeout %d\n" % bootloader.timeout initrd = source_params.get('initrd') + dtb = source_params.get('dtb') - if initrd: - # obviously we need to have a common common deploy var - bootimg_dir = get_bitbake_var("DEPLOY_DIR_IMAGE") - if not bootimg_dir: - raise WicError("Couldn't find DEPLOY_DIR_IMAGE, exiting") - - initrds = initrd.split(';') - for rd in initrds: - cp_cmd = "cp %s/%s %s" % (bootimg_dir, rd, hdddir) - exec_cmd(cp_cmd, True) - else: - logger.debug("Ignoring missing initrd") + if not unified_image: + cls._copy_additional_files(hdddir, initrd, dtb) logger.debug("Writing systemd-boot config " "%s/hdd/boot/loader/loader.conf", cr_workdir) @@ -180,11 +192,15 @@ class BootimgEFIPlugin(SourcePlugin): for rd in initrds: boot_conf += "initrd /%s\n" % rd - logger.debug("Writing systemd-boot config " - "%s/hdd/boot/loader/entries/boot.conf", cr_workdir) - cfg = open("%s/hdd/boot/loader/entries/boot.conf" % cr_workdir, "w") - cfg.write(boot_conf) - cfg.close() + if dtb: + boot_conf += "devicetree /%s\n" % dtb + + if not unified_image: + logger.debug("Writing systemd-boot config " + "%s/hdd/boot/loader/entries/boot.conf", cr_workdir) + cfg = open("%s/hdd/boot/loader/entries/boot.conf" % cr_workdir, "w") + cfg.write(boot_conf) + cfg.close() @classmethod @@ -204,11 +220,64 @@ class BootimgEFIPlugin(SourcePlugin): cls.do_configure_grubefi(hdddir, creator, cr_workdir, source_params) elif source_params['loader'] == 'systemd-boot': cls.do_configure_systemdboot(hdddir, creator, cr_workdir, source_params) + elif source_params['loader'] == 'uefi-kernel': + pass else: raise WicError("unrecognized bootimg-efi loader: %s" % source_params['loader']) except KeyError: raise WicError("bootimg-efi requires a loader, none specified") + if get_bitbake_var("IMAGE_EFI_BOOT_FILES") is None: + logger.debug('No boot files defined in IMAGE_EFI_BOOT_FILES') + else: + boot_files = None + for (fmt, id) in (("_uuid-%s", part.uuid), ("_label-%s", part.label), (None, None)): + if fmt: + var = fmt % id + else: + var = "" + + boot_files = get_bitbake_var("IMAGE_EFI_BOOT_FILES" + var) + if boot_files: + break + + logger.debug('Boot files: %s', boot_files) + + # list of tuples (src_name, dst_name) + deploy_files = [] + for src_entry in re.findall(r'[\w;\-\./\*]+', boot_files): + if ';' in src_entry: + dst_entry = tuple(src_entry.split(';')) + if not dst_entry[0] or not dst_entry[1]: + raise WicError('Malformed boot file entry: %s' % src_entry) + else: + dst_entry = (src_entry, src_entry) + + logger.debug('Destination entry: %r', dst_entry) + deploy_files.append(dst_entry) + + cls.install_task = []; + for deploy_entry in deploy_files: + src, dst = deploy_entry + if '*' in src: + # by default install files under their basename + entry_name_fn = os.path.basename + if dst != src: + # unless a target name was given, then treat name + # as a directory and append a basename + entry_name_fn = lambda name: \ + os.path.join(dst, + os.path.basename(name)) + + srcs = glob(os.path.join(kernel_dir, src)) + + logger.debug('Globbed sources: %s', ', '.join(srcs)) + for entry in srcs: + src = os.path.relpath(entry, kernel_dir) + entry_dst_name = entry_name_fn(entry) + cls.install_task.append((src, entry_dst_name)) + else: + cls.install_task.append((src, dst)) @classmethod def do_prepare_partition(cls, part, source_params, creator, cr_workdir, @@ -234,10 +303,114 @@ class BootimgEFIPlugin(SourcePlugin): kernel = "%s-%s.bin" % \ (get_bitbake_var("KERNEL_IMAGETYPE"), get_bitbake_var("INITRAMFS_LINK_NAME")) - install_cmd = "install -m 0644 %s/%s %s/%s" % \ - (staging_kernel_dir, kernel, hdddir, kernel) - exec_cmd(install_cmd) + if source_params.get('create-unified-kernel-image') == "true": + initrd = source_params.get('initrd') + if not initrd: + raise WicError("initrd= must be specified when create-unified-kernel-image=true, exiting") + + deploy_dir = get_bitbake_var("DEPLOY_DIR_IMAGE") + efi_stub = glob("%s/%s" % (deploy_dir, "linux*.efi.stub")) + if len(efi_stub) == 0: + raise WicError("Unified Kernel Image EFI stub not found, exiting") + efi_stub = efi_stub[0] + with tempfile.TemporaryDirectory() as tmp_dir: + label = source_params.get('label') + label_conf = "root=%s" % creator.rootdev + if label: + label_conf = "LABEL=%s" % label + + bootloader = creator.ks.bootloader + cmdline = open("%s/cmdline" % tmp_dir, "w") + cmdline.write("%s %s" % (label_conf, bootloader.append)) + cmdline.close() + + initrds = initrd.split(';') + initrd = open("%s/initrd" % tmp_dir, "wb") + for f in initrds: + with open("%s/%s" % (deploy_dir, f), 'rb') as in_file: + shutil.copyfileobj(in_file, initrd) + initrd.close() + + # Searched by systemd-boot: + # https://systemd.io/BOOT_LOADER_SPECIFICATION/#type-2-efi-unified-kernel-images + install_cmd = "install -d %s/EFI/Linux" % hdddir + exec_cmd(install_cmd) + + staging_dir_host = get_bitbake_var("STAGING_DIR_HOST") + target_sys = get_bitbake_var("TARGET_SYS") + + objdump_cmd = "%s-objdump" % target_sys + objdump_cmd += " -p %s" % efi_stub + objdump_cmd += " | awk '{ if ($1 == \"SectionAlignment\"){print $2} }'" + + ret, align_str = exec_native_cmd(objdump_cmd, native_sysroot) + align = int(align_str, 16) + + objdump_cmd = "%s-objdump" % target_sys + objdump_cmd += " -h %s | tail -2" % efi_stub + ret, output = exec_native_cmd(objdump_cmd, native_sysroot) + + offset = int(output.split()[2], 16) + int(output.split()[3], 16) + + osrel_off = offset + align - offset % align + osrel_path = "%s/usr/lib/os-release" % staging_dir_host + osrel_sz = os.stat(osrel_path).st_size + + cmdline_off = osrel_off + osrel_sz + cmdline_off = cmdline_off + align - cmdline_off % align + cmdline_sz = os.stat(cmdline.name).st_size + + dtb_off = cmdline_off + cmdline_sz + dtb_off = dtb_off + align - dtb_off % align + + dtb = source_params.get('dtb') + if dtb: + if ';' in dtb: + raise WicError("Only one DTB supported, exiting") + dtb_path = "%s/%s" % (deploy_dir, dtb) + dtb_params = '--add-section .dtb=%s --change-section-vma .dtb=0x%x' % \ + (dtb_path, dtb_off) + linux_off = dtb_off + os.stat(dtb_path).st_size + linux_off = linux_off + align - linux_off % align + else: + dtb_params = '' + linux_off = dtb_off + + linux_path = "%s/%s" % (staging_kernel_dir, kernel) + linux_sz = os.stat(linux_path).st_size + + initrd_off = linux_off + linux_sz + initrd_off = initrd_off + align - initrd_off % align + + # https://www.freedesktop.org/software/systemd/man/systemd-stub.html + objcopy_cmd = "%s-objcopy" % target_sys + objcopy_cmd += " --enable-deterministic-archives" + objcopy_cmd += " --preserve-dates" + objcopy_cmd += " --add-section .osrel=%s" % osrel_path + objcopy_cmd += " --change-section-vma .osrel=0x%x" % osrel_off + objcopy_cmd += " --add-section .cmdline=%s" % cmdline.name + objcopy_cmd += " --change-section-vma .cmdline=0x%x" % cmdline_off + objcopy_cmd += dtb_params + objcopy_cmd += " --add-section .linux=%s" % linux_path + objcopy_cmd += " --change-section-vma .linux=0x%x" % linux_off + objcopy_cmd += " --add-section .initrd=%s" % initrd.name + objcopy_cmd += " --change-section-vma .initrd=0x%x" % initrd_off + objcopy_cmd += " %s %s/EFI/Linux/linux.efi" % (efi_stub, hdddir) + + exec_native_cmd(objcopy_cmd, native_sysroot) + else: + if source_params.get('install-kernel-into-boot-dir') != 'false': + install_cmd = "install -m 0644 %s/%s %s/%s" % \ + (staging_kernel_dir, kernel, hdddir, kernel) + exec_cmd(install_cmd) + + if get_bitbake_var("IMAGE_EFI_BOOT_FILES"): + for src_path, dst_path in cls.install_task: + install_cmd = "install -m 0644 -D %s %s" \ + % (os.path.join(kernel_dir, src_path), + os.path.join(hdddir, dst_path)) + exec_cmd(install_cmd) try: if source_params['loader'] == 'grub-efi': @@ -252,6 +425,28 @@ class BootimgEFIPlugin(SourcePlugin): for mod in [x for x in os.listdir(kernel_dir) if x.startswith("systemd-")]: cp_cmd = "cp %s/%s %s/EFI/BOOT/%s" % (kernel_dir, mod, hdddir, mod[8:]) exec_cmd(cp_cmd, True) + elif source_params['loader'] == 'uefi-kernel': + kernel = get_bitbake_var("KERNEL_IMAGETYPE") + if not kernel: + raise WicError("Empty KERNEL_IMAGETYPE %s\n" % target) + target = get_bitbake_var("TARGET_SYS") + if not target: + raise WicError("Unknown arch (TARGET_SYS) %s\n" % target) + + if re.match("x86_64", target): + kernel_efi_image = "bootx64.efi" + elif re.match('i.86', target): + kernel_efi_image = "bootia32.efi" + elif re.match('aarch64', target): + kernel_efi_image = "bootaa64.efi" + elif re.match('arm', target): + kernel_efi_image = "bootarm.efi" + else: + raise WicError("UEFI stub kernel is incompatible with target %s" % target) + + for mod in [x for x in os.listdir(kernel_dir) if x.startswith(kernel)]: + cp_cmd = "cp %s/%s %s/EFI/BOOT/%s" % (kernel_dir, mod, hdddir, kernel_efi_image) + exec_cmd(cp_cmd, True) else: raise WicError("unrecognized bootimg-efi loader: %s" % source_params['loader']) @@ -263,6 +458,11 @@ class BootimgEFIPlugin(SourcePlugin): cp_cmd = "cp %s %s/" % (startup, hdddir) exec_cmd(cp_cmd, True) + for paths in part.include_path or []: + for path in paths: + cp_cmd = "cp -r %s %s/" % (path, hdddir) + exec_cmd(cp_cmd, True) + du_cmd = "du -bks %s" % hdddir out = exec_cmd(du_cmd) blocks = int(out.split()[0]) @@ -277,6 +477,13 @@ class BootimgEFIPlugin(SourcePlugin): logger.debug("Added %d extra blocks to %s to get to %d total blocks", extra_blocks, part.mountpoint, blocks) + # required for compatibility with certain devices expecting file system + # block count to be equal to partition block count + if blocks < part.fixed_size: + blocks = part.fixed_size + logger.debug("Overriding %s to %d total blocks for compatibility", + part.mountpoint, blocks) + # dosfs image, created by mkdosfs bootimg = "%s/boot.img" % cr_workdir diff --git a/scripts/lib/wic/plugins/source/bootimg-partition.py b/scripts/lib/wic/plugins/source/bootimg-partition.py index 138986a71e..1071d1af3f 100644 --- a/scripts/lib/wic/plugins/source/bootimg-partition.py +++ b/scripts/lib/wic/plugins/source/bootimg-partition.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # # DESCRIPTION @@ -30,6 +32,7 @@ class BootimgPartitionPlugin(SourcePlugin): """ name = 'bootimg-partition' + image_boot_files_var_name = 'IMAGE_BOOT_FILES' @classmethod def do_configure_partition(cls, part, source_params, cr, cr_workdir, @@ -54,12 +57,12 @@ class BootimgPartitionPlugin(SourcePlugin): else: var = "" - boot_files = get_bitbake_var("IMAGE_BOOT_FILES" + var) + boot_files = get_bitbake_var(cls.image_boot_files_var_name + var) if boot_files is not None: break if boot_files is None: - raise WicError('No boot files defined, IMAGE_BOOT_FILES unset for entry #%d' % part.lineno) + raise WicError('No boot files defined, %s unset for entry #%d' % (cls.image_boot_files_var_name, part.lineno)) logger.debug('Boot files: %s', boot_files) @@ -110,7 +113,7 @@ class BootimgPartitionPlugin(SourcePlugin): # Use a custom configuration for extlinux.conf extlinux_conf = custom_cfg logger.debug("Using custom configuration file " - "%s for extlinux.cfg", configfile) + "%s for extlinux.conf", configfile) else: raise WicError("configfile is specified but failed to " "get it from %s." % configfile) @@ -141,7 +144,7 @@ class BootimgPartitionPlugin(SourcePlugin): break if not kernel_name: - raise WicError('No kernel file founded') + raise WicError('No kernel file found') # Compose the extlinux.conf extlinux_conf = "default Yocto\n" diff --git a/scripts/lib/wic/plugins/source/bootimg-pcbios.py b/scripts/lib/wic/plugins/source/bootimg-pcbios.py index f2639e7004..a207a83530 100644 --- a/scripts/lib/wic/plugins/source/bootimg-pcbios.py +++ b/scripts/lib/wic/plugins/source/bootimg-pcbios.py @@ -122,7 +122,7 @@ class BootimgPcbiosPlugin(SourcePlugin): syslinux_conf += "DEFAULT boot\n" syslinux_conf += "LABEL boot\n" - kernel = "/vmlinuz" + kernel = "/" + get_bitbake_var("KERNEL_IMAGETYPE") syslinux_conf += "KERNEL " + kernel + "\n" syslinux_conf += "APPEND label=boot root=%s %s\n" % \ @@ -155,8 +155,8 @@ class BootimgPcbiosPlugin(SourcePlugin): kernel = "%s-%s.bin" % \ (get_bitbake_var("KERNEL_IMAGETYPE"), get_bitbake_var("INITRAMFS_LINK_NAME")) - cmds = ("install -m 0644 %s/%s %s/vmlinuz" % - (staging_kernel_dir, kernel, hdddir), + cmds = ("install -m 0644 %s/%s %s/%s" % + (staging_kernel_dir, kernel, hdddir, get_bitbake_var("KERNEL_IMAGETYPE")), "install -m 444 %s/syslinux/ldlinux.sys %s/ldlinux.sys" % (bootimg_dir, hdddir), "install -m 0644 %s/syslinux/vesamenu.c32 %s/vesamenu.c32" % @@ -186,8 +186,10 @@ class BootimgPcbiosPlugin(SourcePlugin): # dosfs image, created by mkdosfs bootimg = "%s/boot%s.img" % (cr_workdir, part.lineno) - dosfs_cmd = "mkdosfs -n boot -i %s -S 512 -C %s %d" % \ - (part.fsuuid, bootimg, blocks) + label = part.label if part.label else "boot" + + dosfs_cmd = "mkdosfs -n %s -i %s -S 512 -C %s %d" % \ + (label, part.fsuuid, bootimg, blocks) exec_native_cmd(dosfs_cmd, native_sysroot) mcopy_cmd = "mcopy -i %s -s %s/* ::/" % (bootimg, hdddir) diff --git a/scripts/lib/wic/plugins/source/empty.py b/scripts/lib/wic/plugins/source/empty.py new file mode 100644 index 0000000000..4178912377 --- /dev/null +++ b/scripts/lib/wic/plugins/source/empty.py @@ -0,0 +1,89 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +# The empty wic plugin is used to create unformatted empty partitions for wic +# images. +# To use it you must pass "empty" as argument for the "--source" parameter in +# the wks file. For example: +# part foo --source empty --ondisk sda --size="1024" --align 1024 +# +# The plugin supports writing zeros to the start of the +# partition. This is useful to overwrite old content like +# filesystem signatures which may be re-recognized otherwise. +# This feature can be enabled with +# '--sourceparams="[fill|size=<N>[S|s|K|k|M|G]][,][bs=<N>[S|s|K|k|M|G]]"' +# Conflicting or missing options throw errors. + +import logging +import os + +from wic import WicError +from wic.ksparser import sizetype +from wic.pluginbase import SourcePlugin + +logger = logging.getLogger('wic') + +class EmptyPartitionPlugin(SourcePlugin): + """ + Populate unformatted empty partition. + + The following sourceparams are supported: + - fill + Fill the entire partition with zeros. Requires '--fixed-size' option + to be set. + - size=<N>[S|s|K|k|M|G] + Set the first N bytes of the partition to zero. Default unit is 'K'. + - bs=<N>[S|s|K|k|M|G] + Write at most N bytes at a time during source file creation. + Defaults to '1M'. Default unit is 'K'. + """ + + name = 'empty' + + @classmethod + def do_prepare_partition(cls, part, source_params, cr, cr_workdir, + oe_builddir, bootimg_dir, kernel_dir, + rootfs_dir, native_sysroot): + """ + Called to do the actual content population for a partition i.e. it + 'prepares' the partition to be incorporated into the image. + """ + get_byte_count = sizetype('K', True) + size = 0 + + if 'fill' in source_params and 'size' in source_params: + raise WicError("Conflicting source parameters 'fill' and 'size' specified, exiting.") + + # Set the size of the zeros to be written to the partition + if 'fill' in source_params: + if part.fixed_size == 0: + raise WicError("Source parameter 'fill' only works with the '--fixed-size' option, exiting.") + size = get_byte_count(part.fixed_size) + elif 'size' in source_params: + size = get_byte_count(source_params['size']) + + if size == 0: + # Nothing to do, create empty partition + return + + if 'bs' in source_params: + bs = get_byte_count(source_params['bs']) + else: + bs = get_byte_count('1M') + + # Create a binary file of the requested size filled with zeros + source_file = os.path.join(cr_workdir, 'empty-plugin-zeros%s.bin' % part.lineno) + if not os.path.exists(os.path.dirname(source_file)): + os.makedirs(os.path.dirname(source_file)) + + quotient, remainder = divmod(size, bs) + with open(source_file, 'wb') as file: + for _ in range(quotient): + file.write(bytearray(bs)) + file.write(bytearray(remainder)) + + part.size = (size + 1024 - 1) // 1024 # size in KB rounded up + part.source_file = source_file diff --git a/scripts/lib/wic/plugins/source/isoimage-isohybrid.py b/scripts/lib/wic/plugins/source/isoimage-isohybrid.py index 24299c1ece..607356ad13 100644 --- a/scripts/lib/wic/plugins/source/isoimage-isohybrid.py +++ b/scripts/lib/wic/plugins/source/isoimage-isohybrid.py @@ -1,4 +1,6 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # # DESCRIPTION @@ -216,6 +218,18 @@ class IsoImagePlugin(SourcePlugin): creator.name = source_params['image_name'].strip() logger.debug("The name of the image is: %s", creator.name) + @staticmethod + def _install_payload(source_params, iso_dir): + """ + Copies contents of payload directory (as specified in 'payload_dir' param) into iso_dir + """ + + if source_params.get('payload_dir'): + payload_dir = source_params['payload_dir'] + + logger.debug("Payload directory: %s", payload_dir) + shutil.copytree(payload_dir, iso_dir, symlinks=True, dirs_exist_ok=True) + @classmethod def do_prepare_partition(cls, part, source_params, creator, cr_workdir, oe_builddir, bootimg_dir, kernel_dir, @@ -228,6 +242,8 @@ class IsoImagePlugin(SourcePlugin): isodir = "%s/ISO" % cr_workdir + cls._install_payload(source_params, isodir) + if part.rootfs_dir is None: if not 'ROOTFS_DIR' in rootfs_dir: raise WicError("Couldn't find --rootfs-dir, exiting.") @@ -336,19 +352,23 @@ class IsoImagePlugin(SourcePlugin): (img_iso_dir, isodir) exec_cmd(install_cmd) else: + # Default to 100 blocks of extra space for file system overhead + esp_extra_blocks = int(source_params.get('esp_extra_blocks', '100')) + du_cmd = "du -bks %s/EFI" % isodir out = exec_cmd(du_cmd) blocks = int(out.split()[0]) - # Add some extra space for file system overhead - blocks += 100 + blocks += esp_extra_blocks logger.debug("Added 100 extra blocks to %s to get to %d " "total blocks", part.mountpoint, blocks) # dosfs image for EFI boot bootimg = "%s/efi.img" % isodir - dosfs_cmd = 'mkfs.vfat -n "EFIimg" -S 512 -C %s %d' \ - % (bootimg, blocks) + esp_label = source_params.get('esp_label', 'EFIimg') + + dosfs_cmd = 'mkfs.vfat -n \'%s\' -S 512 -C %s %d' \ + % (esp_label, bootimg, blocks) exec_native_cmd(dosfs_cmd, native_sysroot) mmd_cmd = "mmd -i %s ::/EFI" % bootimg diff --git a/scripts/lib/wic/plugins/source/rawcopy.py b/scripts/lib/wic/plugins/source/rawcopy.py index 82970ce51b..21903c2f23 100644 --- a/scripts/lib/wic/plugins/source/rawcopy.py +++ b/scripts/lib/wic/plugins/source/rawcopy.py @@ -1,9 +1,13 @@ # +# Copyright OpenEmbedded Contributors +# # SPDX-License-Identifier: GPL-2.0-only # import logging import os +import signal +import subprocess from wic import WicError from wic.pluginbase import SourcePlugin @@ -21,6 +25,10 @@ class RawCopyPlugin(SourcePlugin): @staticmethod def do_image_label(fstype, dst, label): + # don't create label when fstype is none + if fstype == 'none': + return + if fstype.startswith('ext'): cmd = 'tune2fs -L %s %s' % (label, dst) elif fstype in ('msdos', 'vfat'): @@ -29,15 +37,35 @@ class RawCopyPlugin(SourcePlugin): cmd = 'btrfs filesystem label %s %s' % (dst, label) elif fstype == 'swap': cmd = 'mkswap -L %s %s' % (label, dst) - elif fstype == 'squashfs': - raise WicError("It's not possible to update a squashfs " - "filesystem label '%s'" % (label)) + elif fstype in ('squashfs', 'erofs'): + raise WicError("It's not possible to update a %s " + "filesystem label '%s'" % (fstype, label)) else: raise WicError("Cannot update filesystem label: " "Unknown fstype: '%s'" % (fstype)) exec_cmd(cmd) + @staticmethod + def do_image_uncompression(src, dst, workdir): + def subprocess_setup(): + # Python installs a SIGPIPE handler by default. This is usually not what + # non-Python subprocesses expect. + # SIGPIPE errors are known issues with gzip/bash + signal.signal(signal.SIGPIPE, signal.SIG_DFL) + + extension = os.path.splitext(src)[1] + decompressor = { + ".bz2": "bzip2", + ".gz": "gzip", + ".xz": "xz", + ".zst": "zstd -f", + }.get(extension) + if not decompressor: + raise WicError("Not supported compressor filename extension: %s" % extension) + cmd = "%s -dc %s > %s" % (decompressor, src, dst) + subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True, cwd=workdir) + @classmethod def do_prepare_partition(cls, part, source_params, cr, cr_workdir, oe_builddir, bootimg_dir, kernel_dir, @@ -56,8 +84,14 @@ class RawCopyPlugin(SourcePlugin): if 'file' not in source_params: raise WicError("No file specified") - src = os.path.join(kernel_dir, source_params['file']) - dst = os.path.join(cr_workdir, "%s.%s" % (source_params['file'], part.lineno)) + if 'unpack' in source_params: + img = os.path.join(kernel_dir, source_params['file']) + src = os.path.join(cr_workdir, os.path.splitext(source_params['file'])[0]) + RawCopyPlugin.do_image_uncompression(img, src, cr_workdir) + else: + src = os.path.join(kernel_dir, source_params['file']) + + dst = os.path.join(cr_workdir, "%s.%s" % (os.path.basename(source_params['file']), part.lineno)) if not os.path.exists(os.path.dirname(dst)): os.makedirs(os.path.dirname(dst)) diff --git a/scripts/lib/wic/plugins/source/rootfs.py b/scripts/lib/wic/plugins/source/rootfs.py index e26e95b991..c990143c0d 100644 --- a/scripts/lib/wic/plugins/source/rootfs.py +++ b/scripts/lib/wic/plugins/source/rootfs.py @@ -17,10 +17,11 @@ import shutil import sys from oe.path import copyhardlinktree +from pathlib import Path from wic import WicError from wic.pluginbase import SourcePlugin -from wic.misc import get_bitbake_var +from wic.misc import get_bitbake_var, exec_native_cmd logger = logging.getLogger('wic') @@ -32,8 +33,24 @@ class RootfsPlugin(SourcePlugin): name = 'rootfs' @staticmethod + def __validate_path(cmd, rootfs_dir, path): + if os.path.isabs(path): + logger.error("%s: Must be relative: %s" % (cmd, path)) + sys.exit(1) + + # Disallow climbing outside of parent directory using '..', + # because doing so could be quite disastrous (we will delete the + # directory, or modify a directory outside OpenEmbedded). + full_path = os.path.realpath(os.path.join(rootfs_dir, path)) + if not full_path.startswith(os.path.realpath(rootfs_dir)): + logger.error("%s: Must point inside the rootfs: %s" % (cmd, path)) + sys.exit(1) + + return full_path + + @staticmethod def __get_rootfs_dir(rootfs_dir): - if os.path.isdir(rootfs_dir): + if rootfs_dir and os.path.isdir(rootfs_dir): return os.path.realpath(rootfs_dir) image_rootfs_dir = get_bitbake_var("IMAGE_ROOTFS", rootfs_dir) @@ -44,6 +61,15 @@ class RootfsPlugin(SourcePlugin): return os.path.realpath(image_rootfs_dir) + @staticmethod + def __get_pseudo(native_sysroot, rootfs, pseudo_dir): + pseudo = "export PSEUDO_PREFIX=%s/usr;" % native_sysroot + pseudo += "export PSEUDO_LOCALSTATEDIR=%s;" % pseudo_dir + pseudo += "export PSEUDO_PASSWD=%s;" % rootfs + pseudo += "export PSEUDO_NOSYMLINKEXP=1;" + pseudo += "%s " % get_bitbake_var("FAKEROOTCMD") + return pseudo + @classmethod def do_prepare_partition(cls, part, source_params, cr, cr_workdir, oe_builddir, bootimg_dir, kernel_dir, @@ -68,45 +94,143 @@ class RootfsPlugin(SourcePlugin): "it is not a valid path, exiting" % part.rootfs_dir) part.rootfs_dir = cls.__get_rootfs_dir(rootfs_dir) + part.has_fstab = os.path.exists(os.path.join(part.rootfs_dir, "etc/fstab")) + pseudo_dir = os.path.join(part.rootfs_dir, "../pseudo") + if not os.path.lexists(pseudo_dir): + pseudo_dir = os.path.join(cls.__get_rootfs_dir(None), '../pseudo') + + if not os.path.lexists(pseudo_dir): + logger.warn("%s folder does not exist. " + "Usernames and permissions will be invalid " % pseudo_dir) + pseudo_dir = None new_rootfs = None + new_pseudo = None # Handle excluded paths. - if part.exclude_path is not None: - # We need a new rootfs directory we can delete files from. Copy to - # workdir. + if part.exclude_path or part.include_path or part.change_directory or part.update_fstab_in_rootfs: + # We need a new rootfs directory we can safely modify without + # interfering with other tasks. Copy to workdir. new_rootfs = os.path.realpath(os.path.join(cr_workdir, "rootfs%d" % part.lineno)) if os.path.lexists(new_rootfs): shutil.rmtree(os.path.join(new_rootfs)) - copyhardlinktree(part.rootfs_dir, new_rootfs) + if part.change_directory: + cd = part.change_directory + if cd[-1] == '/': + cd = cd[:-1] + orig_dir = cls.__validate_path("--change-directory", part.rootfs_dir, cd) + else: + orig_dir = part.rootfs_dir + copyhardlinktree(orig_dir, new_rootfs) + + # Convert the pseudo directory to its new location + if (pseudo_dir): + new_pseudo = os.path.realpath( + os.path.join(cr_workdir, "pseudo%d" % part.lineno)) + if os.path.lexists(new_pseudo): + shutil.rmtree(new_pseudo) + os.mkdir(new_pseudo) + shutil.copy(os.path.join(pseudo_dir, "files.db"), + os.path.join(new_pseudo, "files.db")) + + pseudo_cmd = "%s -B -m %s -M %s" % (cls.__get_pseudo(native_sysroot, + new_rootfs, + new_pseudo), + orig_dir, new_rootfs) + exec_native_cmd(pseudo_cmd, native_sysroot) + + for in_path in part.include_path or []: + #parse arguments + include_path = in_path[0] + if len(in_path) > 2: + logger.error("'Invalid number of arguments for include-path") + sys.exit(1) + if len(in_path) == 2: + path = in_path[1] + else: + path = None + + # Pack files to be included into a tar file. + # We need to create a tar file, because that way we can keep the + # permissions from the files even when they belong to different + # pseudo enviroments. + # If we simply copy files using copyhardlinktree/copytree... the + # copied files will belong to the user running wic. + tar_file = os.path.realpath( + os.path.join(cr_workdir, "include-path%d.tar" % part.lineno)) + if os.path.isfile(include_path): + parent = os.path.dirname(os.path.realpath(include_path)) + tar_cmd = "tar c --owner=root --group=root -f %s -C %s %s" % ( + tar_file, parent, os.path.relpath(include_path, parent)) + exec_native_cmd(tar_cmd, native_sysroot) + else: + if include_path in krootfs_dir: + include_path = krootfs_dir[include_path] + include_path = cls.__get_rootfs_dir(include_path) + include_pseudo = os.path.join(include_path, "../pseudo") + if os.path.lexists(include_pseudo): + pseudo = cls.__get_pseudo(native_sysroot, include_path, + include_pseudo) + tar_cmd = "tar cf %s -C %s ." % (tar_file, include_path) + else: + pseudo = None + tar_cmd = "tar c --owner=root --group=root -f %s -C %s ." % ( + tar_file, include_path) + exec_native_cmd(tar_cmd, native_sysroot, pseudo) + + #create destination + if path: + destination = cls.__validate_path("--include-path", new_rootfs, path) + Path(destination).mkdir(parents=True, exist_ok=True) + else: + destination = new_rootfs + + #extract destination + untar_cmd = "tar xf %s -C %s" % (tar_file, destination) + if new_pseudo: + pseudo = cls.__get_pseudo(native_sysroot, new_rootfs, new_pseudo) + else: + pseudo = None + exec_native_cmd(untar_cmd, native_sysroot, pseudo) + os.remove(tar_file) - for orig_path in part.exclude_path: + for orig_path in part.exclude_path or []: path = orig_path - if os.path.isabs(path): - logger.error("Must be relative: --exclude-path=%s" % orig_path) - sys.exit(1) - full_path = os.path.realpath(os.path.join(new_rootfs, path)) + full_path = cls.__validate_path("--exclude-path", new_rootfs, path) - # Disallow climbing outside of parent directory using '..', - # because doing so could be quite disastrous (we will delete the - # directory). - if not full_path.startswith(new_rootfs): - logger.error("'%s' points to a path outside the rootfs" % orig_path) - sys.exit(1) + if not os.path.lexists(full_path): + continue + if new_pseudo: + pseudo = cls.__get_pseudo(native_sysroot, new_rootfs, new_pseudo) + else: + pseudo = None if path.endswith(os.sep): # Delete content only. for entry in os.listdir(full_path): full_entry = os.path.join(full_path, entry) - if os.path.isdir(full_entry) and not os.path.islink(full_entry): - shutil.rmtree(full_entry) - else: - os.remove(full_entry) + rm_cmd = "rm -rf %s" % (full_entry) + exec_native_cmd(rm_cmd, native_sysroot, pseudo) else: # Delete whole directory. - shutil.rmtree(full_path) + rm_cmd = "rm -rf %s" % (full_path) + exec_native_cmd(rm_cmd, native_sysroot, pseudo) + + # Update part.has_fstab here as fstab may have been added or + # removed by the above modifications. + part.has_fstab = os.path.exists(os.path.join(new_rootfs, "etc/fstab")) + if part.update_fstab_in_rootfs and part.has_fstab and not part.no_fstab_update: + fstab_path = os.path.join(new_rootfs, "etc/fstab") + # Assume that fstab should always be owned by root with fixed permissions + install_cmd = "install -m 0644 -p %s %s" % (part.updated_fstab_path, fstab_path) + if new_pseudo: + pseudo = cls.__get_pseudo(native_sysroot, new_rootfs, new_pseudo) + else: + pseudo = None + exec_native_cmd(install_cmd, native_sysroot, pseudo) part.prepare_rootfs(cr_workdir, oe_builddir, - new_rootfs or part.rootfs_dir, native_sysroot) + new_rootfs or part.rootfs_dir, native_sysroot, + pseudo_dir = new_pseudo or pseudo_dir) |