diff options
author | Felipe Sateler <fsateler@debian.org> | 2019-07-07 14:11:24 -0400 |
---|---|---|
committer | Felipe Sateler <fsateler@debian.org> | 2019-07-07 14:11:24 -0400 |
commit | fbb4bdb95b3c524757498ae81ff238d05dc63cb5 (patch) | |
tree | 50de90a08247ac1ffd617ec689f4909e08a04908 | |
parent | 2e218703225835565700bce3f1d06349b21e9d20 (diff) |
New upstream version 5
-rw-r--r-- | .gitignore | 6 | ||||
-rwxr-xr-x | ci/semaphore.sh | 15 | ||||
-rwxr-xr-x | mkosi | 362 | ||||
-rw-r--r-- | mkosi.cache/.gitignore | 2 | ||||
-rw-r--r-- | mkosi.files/mkosi.ubuntu-btrfs | 13 | ||||
-rw-r--r-- | mkosi.output/.gitignore | 2 | ||||
-rwxr-xr-x | setup.py | 2 |
7 files changed, 238 insertions, 164 deletions
@@ -7,13 +7,7 @@ /__pycache__ /build /dist -/image -/image.raw -/image.raw.xz -/image.roothash -/image.tar.xz /mkosi.build -/mkosi.cache /mkosi.egg-info /mkosi.extra /mkosi.nspawn diff --git a/ci/semaphore.sh b/ci/semaphore.sh index 759b908..e5d6460 100755 --- a/ci/semaphore.sh +++ b/ci/semaphore.sh @@ -6,6 +6,17 @@ sudo add-apt-repository --yes ppa:jonathonf/python-3.6 sudo apt --yes update sudo apt --yes install python3.6 debootstrap systemd-container squashfs-tools -sudo python3.6 ./mkosi --default ./mkosi.files/mkosi.ubuntu +testimg() +{ + img="$1" + sudo python3.6 ./mkosi --default ./mkosi.files/mkosi."$img" + test -f mkosi.output/"$img".raw + rm mkosi.output/"$img".raw +} -test -f ubuntu.raw +# Only test ubuntu images for now, as semaphore is based on Ubuntu +for i in ./mkosi.files/mkosi.ubuntu* +do + imgname="$(basename "$i" | cut -d. -f 2-)" + testimg "$imgname" +done @@ -50,7 +50,7 @@ from typing import ( cast, ) -__version__ = '4' +__version__ = '5' if sys.version_info < (3, 6): sys.exit("Sorry, we need at least Python 3.6.") @@ -174,6 +174,7 @@ GPT_BIOS = uuid.UUID("2168614864496e6f744e656564454649") # NOQA: GPT_SWAP = uuid.UUID("0657fd6da4ab43c484e50933c84b4f4f") # NOQA: E221 GPT_HOME = uuid.UUID("933ac7e12eb44f13b8440e14e2aef915") # NOQA: E221 GPT_SRV = uuid.UUID("3b8f842520e04f3b907f1a25a76f98e8") # NOQA: E221 +GPT_XBOOTLDR = uuid.UUID("bc13c2ff59e64262a352b275fd6f7172") # NOQA: E221 GPT_ROOT_X86_VERITY = uuid.UUID("d13c5d3bb5d1422ab29f9454fdc89d76") # NOQA: E221 GPT_ROOT_X86_64_VERITY = uuid.UUID("2c7357edebd246d9aec123d437ec2bf5") # NOQA: E221 GPT_ROOT_ARM_VERITY = uuid.UUID("7386cdf2203c47a9a498f2ecce45a2d6") # NOQA: E221 @@ -240,15 +241,15 @@ def unshare(flags: int) -> None: raise OSError(e, os.strerror(e)) -def format_bytes(bytes: int) -> str: - if bytes >= 1024*1024*1024: - return f'{bytes/1024**3 :0.1f}G' - if bytes >= 1024*1024: - return f'{bytes/1024**2 :0.1f}M' - if bytes >= 1024: - return f'{bytes/1024 :0.1f}K' +def format_bytes(num_bytes: int) -> str: + if num_bytes >= 1024*1024*1024: + return f'{num_bytes/1024**3 :0.1f}G' + if num_bytes >= 1024*1024: + return f'{num_bytes/1024**2 :0.1f}M' + if num_bytes >= 1024: + return f'{num_bytes/1024 :0.1f}K' - return f'{bytes}B' + return f'{num_bytes}B' def roundup512(x: int) -> int: @@ -292,13 +293,13 @@ _IOC_WRITE = 1 # NOQA: E221 _IOC_READ = 2 # NOQA: E221 -def _IOC(dir: int, type: int, nr: int, argtype: str) -> int: +def _IOC(dir_rw: int, type_drv: int, nr: int, argtype: str) -> int: size = {'int': 4, 'size_t': 8}[argtype] - return dir << _IOC_DIRSHIFT | type << _IOC_TYPESHIFT | nr << _IOC_NRSHIFT | size << _IOC_SIZESHIFT + return dir_rw << _IOC_DIRSHIFT | type_drv << _IOC_TYPESHIFT | nr << _IOC_NRSHIFT | size << _IOC_SIZESHIFT -def _IOW(type: int, nr: int, size: str) -> int: - return _IOC(_IOC_WRITE, type, nr, size) +def _IOW(type_drv: int, nr: int, size: str) -> int: + return _IOC(_IOC_WRITE, type_drv, nr, size) FICLONE = _IOW(0x94, 9, 'int') @@ -473,6 +474,8 @@ def image_size(args: CommandLineArguments) -> int: size += args.esp_size if "bios" in args.boot_protocols: size += BIOS_PARTITION_SIZE + if args.xbootldr_size is not None: + size += args.xbootldr_size if args.swap_size is not None: size += args.swap_size if args.verity_size is not None: @@ -507,6 +510,13 @@ def determine_partition_table(args: CommandLineArguments) -> Tuple[str, bool]: run_sfdisk = True + if args.xbootldr_size is not None: + table += f'size={args.xbootldr_size // 512}, type={GPT_XBOOTLDR}, name="Boot Loader Partition"\n' + args.xbootldr_partno = pn + pn += 1 + else: + args.xbootldr_partno = None + if args.swap_size is not None: table += f'size={args.swap_size // 512}, type={GPT_SWAP}, name="Swap Partition"\n' args.swap_partno = pn @@ -575,14 +585,14 @@ def create_image(args: CommandLineArguments, workspace: str, for_cache: bool) -> def reuse_cache_image(args: CommandLineArguments, workspace: str, - run_build_script: bool, + do_run_build_script: bool, for_cache: bool) -> Tuple[Optional[BinaryIO], bool]: if not args.incremental: return None, False if not args.output_format.is_disk_rw(): return None, False - fname = args.cache_pre_dev if run_build_script else args.cache_pre_inst + fname = args.cache_pre_dev if do_run_build_script else args.cache_pre_inst if for_cache: if fname and os.path.exists(fname): # Cache already generated, skip generation, note that manually removing the exising cache images is @@ -618,7 +628,7 @@ def reuse_cache_image(args: CommandLineArguments, disable_cow(f.name) copy_file_object(source, f) - table, run_sfdisk = determine_partition_table(args) + _, run_sfdisk = determine_partition_table(args) args.ran_sfdisk = run_sfdisk return f, True @@ -679,6 +689,18 @@ def prepare_esp(args: CommandLineArguments, loopdev: Optional[str], cached: bool run(["mkfs.fat", "-nEFI", "-F32", partition(loopdev, args.esp_partno)], check=True) +def prepare_xbootldr(args: CommandLineArguments, loopdev: Optional[str], cached: bool) -> None: + if loopdev is None: + return + if cached: + return + if args.xbootldr_partno is None: + return + + with complete_step('Formatting XBOOTLDR partition'): + run(["mkfs.fat", "-nXBOOTLDR", "-F32", partition(loopdev, args.xbootldr_partno)], check=True) + + def mkfs_ext4(label: str, mount: str, dev: str) -> None: run(["mkfs.ext4", "-L", label, "-M", mount, dev], check=True) @@ -732,7 +754,7 @@ def luks_close(dev: Optional[str], text: str) -> None: def luks_format_root(args: CommandLineArguments, loopdev: str, - run_build_script: bool, + do_run_build_script: bool, cached: bool, inserting_squashfs: bool = False) -> None: if args.encrypt != "all": @@ -741,7 +763,7 @@ def luks_format_root(args: CommandLineArguments, return if args.output_format == OutputFormat.gpt_squashfs and not inserting_squashfs: return - if run_build_script: + if do_run_build_script: return if cached: return @@ -750,12 +772,12 @@ def luks_format_root(args: CommandLineArguments, luks_format(partition(loopdev, args.root_partno), args.passphrase) -def luks_format_home(args: CommandLineArguments, loopdev: str, run_build_script: bool, cached: bool) -> None: +def luks_format_home(args: CommandLineArguments, loopdev: str, do_run_build_script: bool, cached: bool) -> None: if args.encrypt is None: return if args.home_partno is None: return - if run_build_script: + if do_run_build_script: return if cached: return @@ -764,12 +786,12 @@ def luks_format_home(args: CommandLineArguments, loopdev: str, run_build_script: luks_format(partition(loopdev, args.home_partno), args.passphrase) -def luks_format_srv(args: CommandLineArguments, loopdev: str, run_build_script: bool, cached: bool) -> None: +def luks_format_srv(args: CommandLineArguments, loopdev: str, do_run_build_script: bool, cached: bool) -> None: if args.encrypt is None: return if args.srv_partno is None: return - if run_build_script: + if do_run_build_script: return if cached: return @@ -780,7 +802,7 @@ def luks_format_srv(args: CommandLineArguments, loopdev: str, run_build_script: def luks_setup_root(args: CommandLineArguments, loopdev: str, - run_build_script: bool, + do_run_build_script: bool, inserting_squashfs: bool = False) -> Optional[str]: if args.encrypt != "all": return None @@ -788,31 +810,31 @@ def luks_setup_root(args: CommandLineArguments, return None if args.output_format == OutputFormat.gpt_squashfs and not inserting_squashfs: return None - if run_build_script: + if do_run_build_script: return None with complete_step("Opening LUKS root partition"): return luks_open(partition(loopdev, args.root_partno), args.passphrase) -def luks_setup_home(args: CommandLineArguments, loopdev: str, run_build_script: bool) -> Optional[str]: +def luks_setup_home(args: CommandLineArguments, loopdev: str, do_run_build_script: bool) -> Optional[str]: if args.encrypt is None: return None if args.home_partno is None: return None - if run_build_script: + if do_run_build_script: return None with complete_step("Opening LUKS home partition"): return luks_open(partition(loopdev, args.home_partno), args.passphrase) -def luks_setup_srv(args: CommandLineArguments, loopdev: str, run_build_script: bool) -> Optional[str]: +def luks_setup_srv(args: CommandLineArguments, loopdev: str, do_run_build_script: bool) -> Optional[str]: if args.encrypt is None: return None if args.srv_partno is None: return None - if run_build_script: + if do_run_build_script: return None with complete_step("Opening LUKS server data partition"): @@ -822,21 +844,21 @@ def luks_setup_srv(args: CommandLineArguments, loopdev: str, run_build_script: b @contextlib.contextmanager def luks_setup_all(args: CommandLineArguments, loopdev: Optional[str], - run_build_script: bool) -> Generator[Tuple[Optional[str], - Optional[str], - Optional[str]], - None, None]: + do_run_build_script: bool) -> Generator[Tuple[Optional[str], + Optional[str], + Optional[str]], + None, None]: if not args.output_format.is_disk(): yield (None, None, None) return assert loopdev is not None try: - root = luks_setup_root(args, loopdev, run_build_script) + root = luks_setup_root(args, loopdev, do_run_build_script) try: - home = luks_setup_home(args, loopdev, run_build_script) + home = luks_setup_home(args, loopdev, do_run_build_script) try: - srv = luks_setup_srv(args, loopdev, run_build_script) + srv = luks_setup_srv(args, loopdev, do_run_build_script) yield (optional_partition(loopdev, args.root_partno) if root is None else root, optional_partition(loopdev, args.home_partno) if home is None else home, @@ -917,16 +939,15 @@ def mount_image(args: CommandLineArguments, home_dev: Optional[str], srv_dev: Optional[str], root_read_only: bool = False) -> Generator[None, None, None]: - if loopdev is None: - yield None - return - assert root_dev is not None - with complete_step('Mounting image'): root = os.path.join(workspace, "root") - if args.output_format != OutputFormat.gpt_squashfs: + if root_dev is not None: mount_loop(args, root_dev, root, root_read_only) + else: + # always have a root of the tree as a mount point so we can + # recursively unmount anything that ends up mounted there + mount_bind(root, root) if home_dev is not None: mount_loop(args, home_dev, os.path.join(root, "home")) @@ -937,6 +958,9 @@ def mount_image(args: CommandLineArguments, if args.esp_partno is not None: mount_loop(args, partition(loopdev, args.esp_partno), os.path.join(root, "efi")) + if args.xbootldr_partno is not None: + mount_loop(args, partition(loopdev, args.xbootldr_partno), os.path.join(root, "boot")) + # Make sure /tmp and /run are not part of the image mount_tmpfs(os.path.join(root, "run")) mount_tmpfs(os.path.join(root, "tmp")) @@ -1017,7 +1041,7 @@ def umount(where: str) -> None: @completestep('Setting up basic OS tree') -def prepare_tree(args: CommandLineArguments, workspace: str, run_build_script: bool, cached: bool) -> None: +def prepare_tree(args: CommandLineArguments, workspace: str, do_run_build_script: bool, cached: bool) -> None: if args.output_format == OutputFormat.subvolume: btrfs_subvol_create(os.path.join(workspace, "root")) else: @@ -1045,20 +1069,33 @@ def prepare_tree(args: CommandLineArguments, workspace: str, run_build_script: b f.write(args.machine_id) f.write("\n") - os.mkdir(os.path.join(workspace, "root", "boot"), 0o700) + if args.xbootldr_partno is not None: + # Create directories for kernels and entries if this is enabled + os.mkdir(os.path.join(workspace, "root", "boot/EFI"), 0o700) + os.mkdir(os.path.join(workspace, "root", "boot/EFI/Linux"), 0o700) + os.mkdir(os.path.join(workspace, "root", "boot/loader"), 0o700) + os.mkdir(os.path.join(workspace, "root", "boot/loader/entries"), 0o700) + os.mkdir(os.path.join(workspace, "root", "boot", args.machine_id), 0o700) + else: + # If this is not enabled, let's create an empty directory on /boot + os.mkdir(os.path.join(workspace, "root", "boot"), 0o700) if args.esp_partno is not None: os.mkdir(os.path.join(workspace, "root", "efi/EFI"), 0o700) os.mkdir(os.path.join(workspace, "root", "efi/EFI/BOOT"), 0o700) - os.mkdir(os.path.join(workspace, "root", "efi/EFI/Linux"), 0o700) os.mkdir(os.path.join(workspace, "root", "efi/EFI/systemd"), 0o700) os.mkdir(os.path.join(workspace, "root", "efi/loader"), 0o700) - os.mkdir(os.path.join(workspace, "root", "efi/loader/entries"), 0o700) - os.mkdir(os.path.join(workspace, "root", "efi", args.machine_id), 0o700) - os.symlink("../efi", os.path.join(workspace, "root", "boot/efi")) - os.symlink("efi/loader", os.path.join(workspace, "root", "boot/loader")) - os.symlink("efi/" + args.machine_id, os.path.join(workspace, "root", "boot", args.machine_id)) + if args.xbootldr_partno is None: + # Create directories for kernels and entries, unless the XBOOTLDR partition is turned on + os.mkdir(os.path.join(workspace, "root", "efi/EFI/Linux"), 0o700) + os.mkdir(os.path.join(workspace, "root", "efi/loader/entries"), 0o700) + os.mkdir(os.path.join(workspace, "root", "efi", args.machine_id), 0o700) + + # Create some compatibility symlinks in /boot in case that is not set up otherwise + os.symlink("../efi", os.path.join(workspace, "root", "boot/efi")) + os.symlink("../efi/loader", os.path.join(workspace, "root", "boot/loader")) + os.symlink("../efi/" + args.machine_id, os.path.join(workspace, "root", "boot", args.machine_id)) os.mkdir(os.path.join(workspace, "root", "etc/kernel"), 0o755) @@ -1066,7 +1103,7 @@ def prepare_tree(args: CommandLineArguments, workspace: str, run_build_script: b cmdline.write(args.kernel_command_line) cmdline.write("\n") - if run_build_script: + if do_run_build_script: os.mkdir(os.path.join(workspace, "root", "root"), 0o750) os.mkdir(os.path.join(workspace, "root", "root/dest"), 0o755) @@ -1305,7 +1342,7 @@ def invoke_dnf(args: CommandLineArguments, @completestep('Installing Clear Linux') -def install_clear(args: CommandLineArguments, workspace: str, run_build_script: bool) -> None: +def install_clear(args: CommandLineArguments, workspace: str, do_run_build_script: bool) -> None: if args.release == "latest": release = "clear" else: @@ -1314,7 +1351,7 @@ def install_clear(args: CommandLineArguments, workspace: str, run_build_script: root = os.path.join(workspace, "root") packages = ['os-core'] + args.packages - if run_build_script: + if do_run_build_script: packages.extend(args.build_packages) if args.bootable: packages += ['kernel-native'] @@ -1346,7 +1383,7 @@ ensure that you have openssl program in your system. # Clear Linux doesn't have a /etc/shadow at install time, it gets # created when the root first login. To set the password via # mkosi, create one. - if not run_build_script and args.password is not None: + if not do_run_build_script and args.password is not None: shadow_file = os.path.join(root, "etc/shadow") with open(shadow_file, "w") as f: f.write('root::::::::') @@ -1357,7 +1394,7 @@ ensure that you have openssl program in your system. @completestep('Installing Fedora') -def install_fedora(args: CommandLineArguments, workspace: str, run_build_script: bool) -> None: +def install_fedora(args: CommandLineArguments, workspace: str, do_run_build_script: bool) -> None: if args.release == 'rawhide': last = sorted(FEDORA_KEYS_MAP)[-1] warn(f'Assuming rawhide is version {last} — ' + @@ -1412,7 +1449,7 @@ gpgkey={gpg_key} packages += args.packages or [] if args.bootable: packages += ['kernel-core', 'systemd-udev', 'binutils'] - if run_build_script: + if do_run_build_script: packages += args.build_packages or [] invoke_dnf(args, workspace, args.repositories or ["fedora", "updates"], @@ -1426,7 +1463,7 @@ gpgkey={gpg_key} @completestep('Installing Mageia') -def install_mageia(args: CommandLineArguments, workspace: str, run_build_script: bool) -> None: +def install_mageia(args: CommandLineArguments, workspace: str, do_run_build_script: bool) -> None: masked = disable_kernel_install(args, workspace) # Mageia does not (yet) have RPM GPG key on the web @@ -1516,7 +1553,7 @@ def invoke_dnf_or_yum(args: CommandLineArguments, @completestep('Installing CentOS') -def install_centos(args: CommandLineArguments, workspace: str, run_build_script: bool) -> None: +def install_centos(args: CommandLineArguments, workspace: str, do_run_build_script: bool) -> None: masked = disable_kernel_install(args, workspace) gpg_key = f"/etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-{args.release}" @@ -1567,7 +1604,7 @@ def debootstrap_knows_arg(arg: str) -> bool: def install_debian_or_ubuntu(args: CommandLineArguments, workspace: str, *, - run_build_script: bool, + do_run_build_script: bool, mirror: str) -> None: repos = args.repositories if args.repositories else ["main"] # Ubuntu needs the 'universe' repo to install 'dracut' @@ -1591,7 +1628,7 @@ def install_debian_or_ubuntu(args: CommandLineArguments, mirror] if args.bootable and args.output_format == OutputFormat.gpt_btrfs: - cmdline[4] += ",btrfs-tools" + cmdline[4] += ",btrfs-progs" run(cmdline, check=True) @@ -1604,7 +1641,7 @@ def install_debian_or_ubuntu(args: CommandLineArguments, # can deal better with any conflicts extra_packages.extend(args.packages) - if run_build_script: + if do_run_build_script: extra_packages.extend(args.build_packages) # Work around debian bug #835628 @@ -1671,17 +1708,17 @@ def install_debian_or_ubuntu(args: CommandLineArguments, @completestep('Installing Debian') -def install_debian(args: CommandLineArguments, workspace: str, run_build_script: bool) -> None: - install_debian_or_ubuntu(args, workspace, run_build_script=run_build_script, mirror=args.mirror) +def install_debian(args: CommandLineArguments, workspace: str, do_run_build_script: bool) -> None: + install_debian_or_ubuntu(args, workspace, do_run_build_script=do_run_build_script, mirror=args.mirror) @completestep('Installing Ubuntu') -def install_ubuntu(args: CommandLineArguments, workspace: str, run_build_script: bool) -> None: - install_debian_or_ubuntu(args, workspace, run_build_script=run_build_script, mirror=args.mirror) +def install_ubuntu(args: CommandLineArguments, workspace: str, do_run_build_script: bool) -> None: + install_debian_or_ubuntu(args, workspace, do_run_build_script=do_run_build_script, mirror=args.mirror) @completestep('Installing Arch Linux') -def install_arch(args: CommandLineArguments, workspace: str, run_build_script: bool) -> None: +def install_arch(args: CommandLineArguments, workspace: str, do_run_build_script: bool) -> None: if args.release is not None: sys.stderr.write("Distribution release specification is not supported for Arch Linux, ignoring.\n") @@ -1709,7 +1746,7 @@ Architecture = auto UseSyslog Color CheckSpace -SigLevel = Required DatabaseOptional +SigLevel = Required DatabaseOptional TrustAll [core] {server} @@ -1818,7 +1855,7 @@ SigLevel = Required DatabaseOptional if args.bootable: packages |= kernel_packages - if run_build_script: + if do_run_build_script: packages.update(args.build_packages) # Remove already installed packages c = run_pacman(['-Qq'], stdout=PIPE, universal_newlines=True) @@ -1828,6 +1865,7 @@ SigLevel = Required DatabaseOptional # Kill the gpg-agent used by pacman and pacman-key run(['gpg-connect-agent', '--homedir', os.path.join(root, 'etc/pacman.d/gnupg'), 'KILLAGENT', '/bye']) + run(['gpg-connect-agent', '--homedir', os.path.join(root, 'etc/pacman.d/gnupg'), '--dirmngr', 'KILLDIRMNGR', '/bye']) if "networkmanager" in args.packages: enable_networkmanager(workspace) @@ -1842,12 +1880,9 @@ SigLevel = Required DatabaseOptional with open(os.path.join(workspace, 'root', 'etc/locale.conf'), 'w') as f: f.write('LANG=en_US.UTF-8\n') - # At this point, no process should be left running, kill then - run(["fuser", "-c", root, "--kill"]) - @completestep('Installing openSUSE') -def install_opensuse(args: CommandLineArguments, workspace: str, run_build_script: bool) -> None: +def install_opensuse(args: CommandLineArguments, workspace: str, do_run_build_script: bool) -> None: root = os.path.join(workspace, "root") release = args.release.strip('"') @@ -1902,7 +1937,7 @@ def install_opensuse(args: CommandLineArguments, workspace: str, run_build_scrip extra_packages.extend(args.packages) - if run_build_script: + if do_run_build_script: extra_packages.extend(args.build_packages) if extra_packages: @@ -1933,7 +1968,7 @@ def install_opensuse(args: CommandLineArguments, workspace: str, run_build_scrip def install_distribution(args: CommandLineArguments, workspace: str, *, - run_build_script: bool, + do_run_build_script: bool, cached: bool) -> None: if cached: return @@ -1949,10 +1984,10 @@ def install_distribution(args: CommandLineArguments, Distribution.clear: install_clear, } - install[args.distribution](args, workspace, run_build_script) + install[args.distribution](args, workspace, do_run_build_script) -def reset_machine_id(args: CommandLineArguments, workspace: str, run_build_script: bool, for_cache: bool) -> None: +def reset_machine_id(args: CommandLineArguments, workspace: str, do_run_build_script: bool, for_cache: bool) -> None: """Make /etc/machine-id an empty file. This way, on the next boot is either initialized and committed (if /etc is @@ -1960,7 +1995,7 @@ def reset_machine_id(args: CommandLineArguments, workspace: str, run_build_scrip each boot (if the image is read-only). """ - if run_build_script: + if do_run_build_script: return if for_cache: return @@ -1992,10 +2027,10 @@ def reset_random_seed(args: CommandLineArguments, workspace: str) -> None: pass -def set_root_password(args: CommandLineArguments, workspace: str, run_build_script: bool, for_cache: bool) -> None: +def set_root_password(args: CommandLineArguments, workspace: str, do_run_build_script: bool, for_cache: bool) -> None: "Set the root account password, or just delete it so it's easy to log in" - if run_build_script: + if do_run_build_script: return if for_cache: return @@ -2018,13 +2053,13 @@ def set_root_password(args: CommandLineArguments, workspace: str, run_build_scri patch_file(os.path.join(workspace, 'root', 'etc/shadow'), jj) -def run_postinst_script(args: CommandLineArguments, workspace: str, run_build_script: bool, for_cache: bool) -> None: +def run_postinst_script(args: CommandLineArguments, workspace: str, do_run_build_script: bool, for_cache: bool) -> None: if args.postinst_script is None: return if for_cache: return - verb = "build" if run_build_script else "final" + verb = "build" if do_run_build_script else "final" with complete_step('Running postinstall script'): @@ -2096,9 +2131,9 @@ def install_grub(args: CommandLineArguments, workspace: str, loopdev: str, grub: loopdev, nspawn_params=nspawn_params) run_workspace_command( - args, workspace, f"{grub}-mkconfig", - f"--output=/boot/{grub}/grub.cfg", - nspawn_params=nspawn_params) + args, workspace, f"{grub}-mkconfig", + f"--output=/boot/{grub}/grub.cfg", + nspawn_params=nspawn_params) def install_boot_loader_fedora(args: CommandLineArguments, workspace: str, loopdev: str) -> None: @@ -2169,10 +2204,10 @@ def install_boot_loader(args: CommandLineArguments, workspace: str, loopdev: Opt with complete_step("Installing boot loader"): if args.esp_partno: shutil.copyfile(os.path.join(workspace, "root", "usr/lib/systemd/boot/efi/systemd-bootx64.efi"), - os.path.join(workspace, "root", "boot/efi/EFI/systemd/systemd-bootx64.efi")) + os.path.join(workspace, "root", "efi/EFI/systemd/systemd-bootx64.efi")) shutil.copyfile(os.path.join(workspace, "root", "usr/lib/systemd/boot/efi/systemd-bootx64.efi"), - os.path.join(workspace, "root", "boot/efi/EFI/BOOT/bootx64.efi")) + os.path.join(workspace, "root", "efi/EFI/BOOT/bootx64.efi")) if args.distribution == Distribution.fedora: install_boot_loader_fedora(args, workspace, loopdev) @@ -2262,8 +2297,8 @@ def copy_git_files(src: str, dest: str, *, source_file_transfer: SourceFileTrans copy_file(src_path, dest_path) -def install_build_src(args: CommandLineArguments, workspace: str, run_build_script: bool, for_cache: bool) -> None: - if not run_build_script: +def install_build_src(args: CommandLineArguments, workspace: str, do_run_build_script: bool, for_cache: bool) -> None: + if not do_run_build_script: return if for_cache: return @@ -2295,8 +2330,8 @@ def install_build_src(args: CommandLineArguments, workspace: str, run_build_scri shutil.copytree(args.build_sources, target, symlinks=True, ignore=ignore) -def install_build_dest(args: CommandLineArguments, workspace: str, run_build_script: bool, for_cache: bool) -> None: - if run_build_script: +def install_build_dest(args: CommandLineArguments, workspace: str, do_run_build_script: bool, for_cache: bool) -> None: + if do_run_build_script: return if for_cache: return @@ -2323,9 +2358,9 @@ def make_read_only(args: CommandLineArguments, workspace: str, for_cache: bool) def make_tar(args: CommandLineArguments, workspace: str, - run_build_script: bool, + do_run_build_script: bool, for_cache: bool) -> Optional[BinaryIO]: - if run_build_script: + if do_run_build_script: return None if args.output_format != OutputFormat.tar: return None @@ -2382,7 +2417,7 @@ def read_partition_table(loopdev: str) -> Tuple[List[str], int]: table.append(stripped) - name, rest = stripped.split(":", 1) + _, rest = stripped.split(":", 1) fields = rest.split(",") start = None @@ -2412,7 +2447,7 @@ def insert_partition(args: CommandLineArguments, blob: BinaryIO, name: str, type_uuid: uuid.UUID, - uuid: Optional[uuid.UUID] = None) -> int: + uuid_opt: Optional[uuid.UUID] = None) -> int: if args.ran_sfdisk: old_table, last_partition_sector = read_partition_table(loopdev) else: @@ -2436,8 +2471,8 @@ def insert_partition(args: CommandLineArguments, for t in old_table: table += t + "\n" - if uuid is not None: - table += "uuid=" + str(uuid) + ", " + if uuid_opt is not None: + table += "uuid=" + str(uuid_opt) + ", " n_sectores = (blob_size + luks_extra) // 512 table += f'size={n_sectores}, type={type_uuid}, attrs=GUID:60, name="{name}"\n' @@ -2487,9 +2522,9 @@ def insert_squashfs(args: CommandLineArguments, def make_verity(args: CommandLineArguments, workspace: str, dev: Optional[str], - run_build_script: bool, + do_run_build_script: bool, for_cache: bool) -> Tuple[Optional[BinaryIO], Optional[str]]: - if run_build_script or not args.verity: + if do_run_build_script or not args.verity: return None, None if for_cache: return None, None @@ -2551,7 +2586,7 @@ def patch_root_uuid(args: CommandLineArguments, def install_unified_kernel(args: CommandLineArguments, workspace: str, - run_build_script: bool, + do_run_build_script: bool, for_cache: bool, root_hash: Optional[str]) -> None: # Iterates through all kernel versions included in the image and @@ -2575,7 +2610,7 @@ def install_unified_kernel(args: CommandLineArguments, # not be relevant for building, and dracut is simply very slow, # hence let's avoid it invoking it needlessly, given that we never # actually invoke the boot loader on the development image. - if run_build_script: + if do_run_build_script: return if args.distribution not in (Distribution.fedora, Distribution.mageia): @@ -2591,7 +2626,10 @@ def install_unified_kernel(args: CommandLineArguments, if not kver.is_dir(): continue - boot_binary = "/efi/EFI/Linux/linux-" + kver.name + # Place kernel in XBOOTLDR partition if it is turned on, otherwise in the ESP + prefix = "/efi" if args.xbootldr_size is None else "/boot" + + boot_binary = prefix + "/EFI/Linux/linux-" + kver.name if root_hash is not None: boot_binary += "-" + root_hash boot_binary += ".efi" @@ -2619,8 +2657,8 @@ def install_unified_kernel(args: CommandLineArguments, run_workspace_command(args, workspace, *dracut) -def secure_boot_sign(args: CommandLineArguments, workspace: str, run_build_script: bool, for_cache: bool) -> None: - if run_build_script: +def secure_boot_sign(args: CommandLineArguments, workspace: str, do_run_build_script: bool, for_cache: bool) -> None: + if do_run_build_script: return if not args.bootable: return @@ -2629,7 +2667,7 @@ def secure_boot_sign(args: CommandLineArguments, workspace: str, run_build_scrip if for_cache: return - for path, dirnames, filenames in os.walk(os.path.join(workspace, "root", "efi")): + for path, _, filenames in os.walk(os.path.join(workspace, "root", "efi")): for i in filenames: if not i.endswith(".efi") and not i.endswith(".EFI"): continue @@ -2674,7 +2712,7 @@ def qcow2_output(args: CommandLineArguments, raw: Optional[BinaryIO]) -> Optiona with complete_step('Converting image file to qcow2'): f: BinaryIO = cast(BinaryIO, tempfile.NamedTemporaryFile(prefix=".mkosi-", dir=os.path.dirname(args.output))) - run(["qemu-img", "convert", "-fraw", "-Oqcow2", raw.name, f.name], check=True) + run(["qemu-img", "convert", "-onocow=on", "-fraw", "-Oqcow2", raw.name, f.name], check=True) return f @@ -2873,7 +2911,7 @@ def link_output_bmap(args: CommandLineArguments, bmap: Optional[str]) -> None: def dir_size(path: str) -> int: - sum = 0 + dir_sum = 0 for entry in os.scandir(path): if entry.is_symlink(): # We can ignore symlinks because they either point into our tree, @@ -2881,10 +2919,10 @@ def dir_size(path: str) -> int: # or outside, in which case we don't need to. continue elif entry.is_file(): - sum += entry.stat().st_blocks * 512 + dir_sum += entry.stat().st_blocks * 512 elif entry.is_dir(): - sum += dir_size(entry.path) - return sum + dir_sum += dir_size(entry.path) + return dir_sum def print_output_size(args: CommandLineArguments) -> None: @@ -3035,6 +3073,8 @@ def parse_args() -> CommandLineArguments: help='Set size of root partition (only gpt_ext4, gpt_xfs, gpt_btrfs)', metavar='BYTES') group.add_argument("--esp-size", help='Set size of EFI system partition (only gpt_ext4, gpt_xfs, gpt_btrfs, gpt_squashfs)', metavar='BYTES') # NOQA: E501 + group.add_argument("--xbootldr-size", + help='Set size of the XBOOTLDR partition (only gpt_ext4, gpt_xfs, gpt_btrfs, gpt_squashfs)', metavar='BYTES') # NOQA: E501 group.add_argument("--swap-size", help='Set size of swap partition (only gpt_ext4, gpt_xfs, gpt_btrfs, gpt_squashfs)', metavar='BYTES') # NOQA: E501 group.add_argument("--home-size", @@ -3084,23 +3124,23 @@ def parse_args() -> CommandLineArguments: return args -def parse_bytes(bytes: Optional[str]) -> Optional[int]: - if bytes is None: - return bytes +def parse_bytes(num_bytes: Optional[str]) -> Optional[int]: + if num_bytes is None: + return num_bytes - if bytes.endswith('G'): + if num_bytes.endswith('G'): factor = 1024**3 - elif bytes.endswith('M'): + elif num_bytes.endswith('M'): factor = 1024**2 - elif bytes.endswith('K'): + elif num_bytes.endswith('K'): factor = 1024 else: factor = 1 if factor > 1: - bytes = bytes[:-1] + num_bytes = num_bytes[:-1] - result = int(bytes) * factor + result = int(num_bytes) * factor if result <= 0: raise ValueError("Size out of range") @@ -3119,14 +3159,14 @@ def detect_distribution() -> Tuple[Optional[Distribution], Optional[str]]: except IOError: return None, None - id = None + dist_id = None version_id = None version_codename = None extracted_codename = None for ln in f: if ln.startswith("ID="): - id = ln[3:].strip() + dist_id = ln[3:].strip() if ln.startswith("VERSION_ID="): version_id = ln[11:].strip() if ln.startswith("VERSION_CODENAME="): @@ -3140,12 +3180,12 @@ def detect_distribution() -> Tuple[Optional[Distribution], Optional[str]]: if len(codename_list) == 1: extracted_codename = codename_list[0] - if id == "clear-linux-os": - id = "clear" + if dist_id == "clear-linux-os": + dist_id = "clear" d: Optional[Distribution] = None - if id is not None: - d = Distribution.__members__.get(id, None) + if dist_id is not None: + d = Distribution.__members__.get(dist_id, None) if d == Distribution.debian and (version_codename or extracted_codename): # debootstrap needs release codenames, not version numbers @@ -3398,6 +3438,9 @@ def process_setting(args: CommandLineArguments, section: str, key: Optional[str] elif key == "ESPSize": if args.esp_size is None: args.esp_size = value + elif key == "BootLoaderSize": + if args.xbootldr_size is None: + args.xbootldr_size = value elif key == "SwapSize": if args.swap_size is None: args.swap_size = value @@ -3527,12 +3570,12 @@ def args_find_path(args: CommandLineArguments, name: str, path: str, *, - type: Callable[[str], Any] = lambda x: x) -> None: + type_call: Callable[[str], Any] = lambda x: x) -> None: if getattr(args, name) is not None: return if os.path.exists(path): path = os.path.abspath(path) - path = type(path) + path = type_call(path) setattr(args, name, path) @@ -3643,7 +3686,7 @@ def load_args(args) -> CommandLineArguments: args_find_path(args, 'postinst_script', "mkosi.postinst") args_find_path(args, 'finalize_script', "mkosi.finalize") args_find_path(args, 'output_dir', "mkosi.output/") - args_find_path(args, 'mksquashfs_tool', "mkosi.mksquashfs-tool", type=lambda x: [x]) + args_find_path(args, 'mksquashfs_tool', "mkosi.mksquashfs-tool", type_call=lambda x: [x]) find_extra(args) find_skeleton(args) @@ -3822,6 +3865,7 @@ def load_args(args) -> CommandLineArguments: args.home_size = parse_bytes(args.home_size) args.srv_size = parse_bytes(args.srv_size) args.esp_size = parse_bytes(args.esp_size) + args.xbootldr_size = parse_bytes(args.xbootldr_size) args.swap_size = parse_bytes(args.swap_size) if args.output_format in (OutputFormat.gpt_ext4, OutputFormat.gpt_btrfs) and args.root_size is None: @@ -4000,6 +4044,7 @@ def print_summary(args: CommandLineArguments) -> None: sys.stderr.write(" ESP: " + format_bytes_or_disabled(args.esp_size) + "\n") if "bios" in args.boot_protocols: sys.stderr.write(" BIOS: " + format_bytes_or_disabled(BIOS_PARTITION_SIZE) + "\n") + sys.stderr.write(" XBOOTLDR Partition: " + format_bytes_or_disabled(args.xbootldr_size) + "\n") sys.stderr.write(" /home Partition: " + format_bytes_or_disabled(args.home_size) + "\n") sys.stderr.write(" /srv Partition: " + format_bytes_or_disabled(args.srv_size) + "\n") @@ -4015,7 +4060,7 @@ def print_summary(args: CommandLineArguments) -> None: def reuse_cache_tree(args: CommandLineArguments, workspace: str, - run_build_script: bool, + do_run_build_script: bool, for_cache: bool, cached: bool) -> bool: """If there's a cached version of this tree around, use it and @@ -4032,7 +4077,7 @@ def reuse_cache_tree(args: CommandLineArguments, if args.output_format.is_disk_rw(): return False - fname = args.cache_pre_dev if run_build_script else args.cache_pre_inst + fname = args.cache_pre_dev if do_run_build_script else args.cache_pre_inst if fname is None: return False @@ -4064,17 +4109,17 @@ def make_build_dir(args: CommandLineArguments) -> None: def build_image(args: CommandLineArguments, workspace: tempfile.TemporaryDirectory, *, - run_build_script: bool, + do_run_build_script: bool, for_cache: bool = False, cleanup: bool = False) -> Tuple[Optional[BinaryIO], Optional[BinaryIO], Optional[str]]: # If there's no build script set, there's no point in executing # the build script iteration. Let's quit early. - if args.build_script is None and run_build_script: + if args.build_script is None and do_run_build_script: return None, None, None make_build_dir(args) - raw, cached = reuse_cache_image(args, workspace.name, run_build_script, for_cache) + raw, cached = reuse_cache_image(args, workspace.name, do_run_build_script, for_cache) if for_cache and cached: # Found existing cache image, exiting build_image return None, None, None @@ -4086,44 +4131,45 @@ def build_image(args: CommandLineArguments, prepare_swap(args, loopdev, cached) prepare_esp(args, loopdev, cached) + prepare_xbootldr(args, loopdev, cached) if loopdev is not None: - luks_format_root(args, loopdev, run_build_script, cached) - luks_format_home(args, loopdev, run_build_script, cached) - luks_format_srv(args, loopdev, run_build_script, cached) + luks_format_root(args, loopdev, do_run_build_script, cached) + luks_format_home(args, loopdev, do_run_build_script, cached) + luks_format_srv(args, loopdev, do_run_build_script, cached) - with luks_setup_all(args, loopdev, run_build_script) as (encrypted_root, encrypted_home, encrypted_srv): + with luks_setup_all(args, loopdev, do_run_build_script) as (encrypted_root, encrypted_home, encrypted_srv): prepare_root(args, encrypted_root, cached) prepare_home(args, encrypted_home, cached) prepare_srv(args, encrypted_srv, cached) with mount_image(args, workspace.name, loopdev, encrypted_root, encrypted_home, encrypted_srv): - prepare_tree(args, workspace.name, run_build_script, cached) + prepare_tree(args, workspace.name, do_run_build_script, cached) with mount_cache(args, workspace.name): - cached = reuse_cache_tree(args, workspace.name, run_build_script, for_cache, cached) + cached = reuse_cache_tree(args, workspace.name, do_run_build_script, for_cache, cached) install_skeleton_trees(args, workspace.name, for_cache) install_distribution(args, workspace.name, - run_build_script=run_build_script, cached=cached) + do_run_build_script=do_run_build_script, cached=cached) install_etc_hostname(args, workspace.name) install_boot_loader(args, workspace.name, loopdev, cached) install_extra_trees(args, workspace.name, for_cache) - install_build_src(args, workspace.name, run_build_script, for_cache) - install_build_dest(args, workspace.name, run_build_script, for_cache) - set_root_password(args, workspace.name, run_build_script, for_cache) - run_postinst_script(args, workspace.name, run_build_script, for_cache) + install_build_src(args, workspace.name, do_run_build_script, for_cache) + install_build_dest(args, workspace.name, do_run_build_script, for_cache) + set_root_password(args, workspace.name, do_run_build_script, for_cache) + run_postinst_script(args, workspace.name, do_run_build_script, for_cache) if cleanup: clean_package_manager_metadata(workspace.name) - reset_machine_id(args, workspace.name, run_build_script, for_cache) + reset_machine_id(args, workspace.name, do_run_build_script, for_cache) reset_random_seed(args, workspace.name) make_read_only(args, workspace.name, for_cache) squashfs = make_squashfs(args, workspace.name, for_cache) insert_squashfs(args, workspace.name, raw, loopdev, squashfs, for_cache) - verity, root_hash = make_verity(args, workspace.name, encrypted_root, run_build_script, for_cache) + verity, root_hash = make_verity(args, workspace.name, encrypted_root, do_run_build_script, for_cache) patch_root_uuid(args, loopdev, root_hash, for_cache) insert_verity(args, workspace.name, raw, loopdev, verity, root_hash, for_cache) @@ -4132,10 +4178,10 @@ def build_image(args: CommandLineArguments, # image anymore. with mount_image(args, workspace.name, loopdev, encrypted_root, encrypted_home, encrypted_srv, root_read_only=True): - install_unified_kernel(args, workspace.name, run_build_script, for_cache, root_hash) - secure_boot_sign(args, workspace.name, run_build_script, for_cache) + install_unified_kernel(args, workspace.name, do_run_build_script, for_cache, root_hash) + secure_boot_sign(args, workspace.name, do_run_build_script, for_cache) - tar = make_tar(args, workspace.name, run_build_script, for_cache) + tar = make_tar(args, workspace.name, do_run_build_script, for_cache) return raw or squashfs, tar, root_hash @@ -4211,11 +4257,11 @@ def remove_artifacts(args: CommandLineArguments, workspace: str, raw: Optional[BinaryIO], tar: Optional[BinaryIO], - run_build_script: bool, + do_run_build_script: bool, for_cache: bool = False) -> None: if for_cache: what = "cache build" - elif run_build_script: + elif do_run_build_script: what = "development build" else: return @@ -4244,42 +4290,47 @@ def build_stuff(args: CommandLineArguments) -> None: setup_package_cache(args) workspace = setup_workspace(args) + # Make sure tmpfiles' aging doesn't interfere with our workspace + # while we are working on it. + dir_fd = os.open(workspace.name, os.O_RDONLY|os.O_DIRECTORY|os.O_CLOEXEC) + fcntl.flock(dir_fd, fcntl.LOCK_EX) + # If caching is requested, then make sure we have cache images around we can make use of if need_cache_images(args): # There is no point generating a pre-dev cache image if no build script is provided if args.build_script: # Generate the cache version of the build image, and store it as "cache-pre-dev" - raw, tar, root_hash = build_image(args, workspace, run_build_script=True, for_cache=True) + raw, tar, root_hash = build_image(args, workspace, do_run_build_script=True, for_cache=True) save_cache(args, workspace.name, raw.name if raw is not None else None, args.cache_pre_dev) - remove_artifacts(args, workspace.name, raw, tar, run_build_script=True) + remove_artifacts(args, workspace.name, raw, tar, do_run_build_script=True) # Generate the cache version of the build image, and store it as "cache-pre-inst" - raw, tar, root_hash = build_image(args, workspace, run_build_script=False, for_cache=True) + raw, tar, root_hash = build_image(args, workspace, do_run_build_script=False, for_cache=True) if raw: save_cache(args, workspace.name, raw.name, args.cache_pre_inst) - remove_artifacts(args, workspace.name, raw, tar, run_build_script=False) + remove_artifacts(args, workspace.name, raw, tar, do_run_build_script=False) run_finalize_script(args, workspace.name, verb='build') if args.build_script: # Run the image builder for the first (develpoment) stage in preparation for the build script - raw, tar, root_hash = build_image(args, workspace, run_build_script=True) + raw, tar, root_hash = build_image(args, workspace, do_run_build_script=True) run_build_script(args, workspace.name, raw) - remove_artifacts(args, workspace.name, raw, tar, run_build_script=True) + remove_artifacts(args, workspace.name, raw, tar, do_run_build_script=True) run_finalize_script(args, workspace.name, verb='final') # Run the image builder for the second (final) stage - raw, tar, root_hash = build_image(args, workspace, run_build_script=False, cleanup=True) + raw, tar, root_hash = build_image(args, workspace, do_run_build_script=False, cleanup=True) raw = qcow2_output(args, raw) raw = xz_output(args, raw) @@ -4308,6 +4359,7 @@ def build_stuff(args: CommandLineArguments) -> None: if root_hash is not None: print_step(f'Root hash is {root_hash}.') + os.close(dir_fd) def check_root() -> None: if os.getuid() != 0: diff --git a/mkosi.cache/.gitignore b/mkosi.cache/.gitignore new file mode 100644 index 0000000..120f485 --- /dev/null +++ b/mkosi.cache/.gitignore @@ -0,0 +1,2 @@ +* +!/.gitignore diff --git a/mkosi.files/mkosi.ubuntu-btrfs b/mkosi.files/mkosi.ubuntu-btrfs new file mode 100644 index 0000000..45d45fe --- /dev/null +++ b/mkosi.files/mkosi.ubuntu-btrfs @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: LGPL-2.1+ + +[Distribution] +Distribution=ubuntu +Release=xenial + +[Output] +Format=raw_btrfs +Output=ubuntu-btrfs.raw + +[Packages] +Packages= + tzdata diff --git a/mkosi.output/.gitignore b/mkosi.output/.gitignore new file mode 100644 index 0000000..120f485 --- /dev/null +++ b/mkosi.output/.gitignore @@ -0,0 +1,2 @@ +* +!/.gitignore @@ -11,7 +11,7 @@ if sys.version_info < (3, 6): setup( name="mkosi", - version="4", + version="5", description="Create legacy-free OS images", url="https://github.com/systemd/mkosi", maintainer="mkosi contributors", |