summaryrefslogtreecommitdiff
path: root/mkosi
diff options
context:
space:
mode:
authorFelipe Sateler <fsateler@debian.org>2016-10-19 12:18:38 -0300
committerFelipe Sateler <fsateler@debian.org>2016-10-19 12:18:38 -0300
commitdccb5cc431c22df7cc2deb0454eb02f632f0703b (patch)
tree5ebf86b8ea79a7b460132494ad38d28e5632dc59 /mkosi
parentbdec22d6ecbcba710098c3dfdea2957fc310f992 (diff)
New upstream version 0~20161012
Diffstat (limited to 'mkosi')
-rwxr-xr-xmkosi349
1 files changed, 216 insertions, 133 deletions
diff --git a/mkosi b/mkosi
index 1bbd8d8..ca06423 100755
--- a/mkosi
+++ b/mkosi
@@ -2,7 +2,9 @@
import argparse
import configparser
+import contextlib
import ctypes, ctypes.util
+import crypt
import hashlib
import os
import platform
@@ -47,8 +49,10 @@ GPT_SRV = uuid.UUID("3b8f842520e04f3b907f1a25a76f98e8")
if platform.machine() == "x86_64":
GPT_ROOT_NATIVE = GPT_ROOT_X86_64
+elif platform.machine() == "aarch64":
+ GPT_ROOT_NATIVE = GPT_ROOT_ARM_64
else:
- sys.stderr.write("Don't known the native architecture.")
+ sys.stderr.write("Don't known the %s architecture.\n" % platform.machine())
sys.exit(1)
CLONE_NEWNS = 0x00020000
@@ -162,28 +166,24 @@ def create_image(args, workspace):
return f
+@contextlib.contextmanager
def attach_image_loopback(args, raw):
if raw is None:
+ yield None
return
print_step("Attaching image file...")
+ c = subprocess.run(["losetup", "--find", "--show", "--partscan", raw.name],
+ stdout=subprocess.PIPE, check=True)
+ loopdev = c.stdout.decode("utf-8").strip()
+ print_step("Attached image file as " + loopdev + ".")
- c = subprocess.run(["losetup", "--find", "--show", "--partscan", raw.name], stdout=subprocess.PIPE, check=True)
- n = c.stdout.decode("utf-8").strip()
-
- print_step("Attached image file as " + n + ".")
-
- return n
-
-def detach_image_loopback(args, loopdev):
- if loopdev is None:
- return
-
- print_step("Detaching image file...");
-
- subprocess.run(["losetup", "--detach", loopdev], check=True)
-
- print_step("Detaching image file completed.");
+ try:
+ yield loopdev
+ finally:
+ print_step("Detaching image file...");
+ subprocess.run(["losetup", "--detach", loopdev], check=True)
+ print_step("Detaching image file completed.");
def partition(loopdev, partno):
return loopdev + "p" + str(partno)
@@ -269,29 +269,48 @@ def mount_bind(what, where):
os.makedirs(where, 0o755, True)
subprocess.run(["mount", "--bind", what, where], check=True)
+@contextlib.contextmanager
def mount_image(args, workspace, loopdev):
if loopdev is None:
+ yield None
return
print_step("Mounting image...");
- mount_loop(args, loopdev, args.root_partno, os.path.join(workspace, "root"))
+ root = os.path.join(workspace, "root")
+ mount_loop(args, loopdev, args.root_partno, root)
if args.home_partno is not None:
- mount_loop(args, loopdev, args.home_partno, os.path.join(workspace, "root", "home"))
+ mount_loop(args, loopdev, args.home_partno, os.path.join(root, "home"))
if args.srv_partno is not None:
- mount_loop(args, loopdev, args.srv_partno, os.path.join(workspace, "root", "srv"))
+ mount_loop(args, loopdev, args.srv_partno, os.path.join(root, "srv"))
if args.esp_partno is not None:
- mount_loop(args, loopdev, args.esp_partno, os.path.join(workspace, "root", "boot/efi"))
+ mount_loop(args, loopdev, args.esp_partno, os.path.join(root, "boot/efi"))
if args.distribution == Distribution.fedora:
- mount_bind("/proc", os.path.join(workspace, "root", "proc"))
- mount_bind("/dev", os.path.join(workspace, "root", "dev"))
- mount_bind("/sys", os.path.join(workspace, "root", "sys"))
+ mount_bind("/proc", os.path.join(root, "proc"))
+ mount_bind("/dev", os.path.join(root, "dev"))
+ mount_bind("/sys", os.path.join(root, "sys"))
print_step("Mounting image completed.");
+ try:
+ yield
+ finally:
+ print_step("Unmounting image...");
+
+ umount(os.path.join(root, "home"))
+ umount(os.path.join(root, "srv"))
+ umount(os.path.join(root, "boot/efi"))
+ umount(os.path.join(root, "proc"))
+ umount(os.path.join(root, "sys"))
+ umount(os.path.join(root, "dev"))
+ umount(os.path.join(root, "var/cache/dnf"))
+ umount(os.path.join(root, "var/cache/apt/archives"))
+ umount(os.path.join(root))
+
+ print_step("Unmounting image completed.");
def mount_cache(args, workspace):
if not args.distribution in (Distribution.fedora, Distribution.debian, Distribution.ubuntu):
@@ -310,24 +329,6 @@ def umount(where):
# Ignore failures and error messages
subprocess.run(["umount", "-n", where], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-def umount_image(args, workspace, loopdev):
- if loopdev is None:
- return
-
- print_step("Unmounting image...");
-
- umount(os.path.join(workspace, "root", "home"))
- umount(os.path.join(workspace, "root", "srv"))
- umount(os.path.join(workspace, "root", "boot/efi"))
- umount(os.path.join(workspace, "root", "proc"))
- umount(os.path.join(workspace, "root", "sys"))
- umount(os.path.join(workspace, "root", "dev"))
- umount(os.path.join(workspace, "root", "var/cache/dnf"))
- umount(os.path.join(workspace, "root", "var/cache/apt/archives"))
- umount(os.path.join(workspace, "root"))
-
- print_step("Unmounting image completed.");
-
def prepare_tree(args, workspace):
print_step("Setting up basic OS tree...");
@@ -397,11 +398,25 @@ def enable_networkd(workspace):
lambda line: " ".join(["resolve" if w == "dns" else w for w in line.split(" ")]) if line.startswith("hosts:") else line)
with open(os.path.join(workspace, "root", "etc/systemd/network/all-ethernet.network"), "w") as f:
- f.write("[Match]\n")
- f.write("Type=ether\n")
- f.write("\n")
- f.write("[Network]\n")
- f.write("DHCP=yes\n")
+ f.write("""\
+[Match]
+Type=ether
+
+[Network]
+DHCP=yes
+""")
+
+def run_workspace_command(workspace, *cmd, network=False):
+ cmdline = ["systemd-nspawn",
+ '--quiet',
+ "--directory", os.path.join(workspace, "root"),
+ "--as-pid2",
+ "--register=no"]
+ if not network:
+ cmdline += ["--private-network"]
+
+ cmdline += ['--', *cmd]
+ subprocess.run(cmdline, check=True)
def install_fedora(args, workspace, run_build_script):
print_step("Installing Fedora...")
@@ -412,27 +427,42 @@ def install_fedora(args, workspace, run_build_script):
else:
gpg_key = "https://getfedora.org/static/81B46521.txt"
- with open(os.path.join(workspace, "dnf.conf"), "w") as f:
- f.write("[main]\n")
- f.write("gpgcheck=1\n")
- f.write("\n")
- f.write("[fedora]\n")
- f.write("name=Fedora %s - base\n" % args.release)
- f.write("baseurl=%s/releases/%s/Everything/x86_64/os/\n" % (args.mirror, args.release))
- f.write("gpgkey=%s\n" % gpg_key)
- f.write("\n")
- f.write("[updates]\n")
- f.write("name=Fedora %s - updates\n" % args.release)
- f.write("baseurl=%s/updates/%s/x86_64/\n" % (args.mirror, args.release))
- f.write("gpgkey=%s\n" % gpg_key)
+ if args.mirror:
+ release_url = "baseurl={.mirror}/releases/{.release}/Everything/x86_64/os/".format(args)
+ updates_url = "baseurl={.mirror}/updates/{.release}/x86_64/".format(args)
+ else:
+ release_url = ("metalink=https://mirrors.fedoraproject.org/metalink?" +
+ "repo=fedora-{.release}&arch=x86_64".format(args))
+ updates_url = ("metalink=https://mirrors.fedoraproject.org/metalink?" +
+ "repo=updates-released-f{.release}&arch=x86_64".format(args))
+ with open(os.path.join(workspace, "dnf.conf"), "w") as f:
+ f.write("""\
+[main]
+gpgcheck=1
+
+[fedora]
+name=Fedora {args.release} - base
+{release_url}
+gpgkey={gpg_key}
+
+[updates]
+name=Fedora {args.release} - updates
+{updates_url}
+gpgkey={gpg_key}
+""".format(args=args,
+ gpg_key=gpg_key,
+ release_url=release_url,
+ updates_url=updates_url))
+
+ root = os.path.join(workspace, "root")
cmdline = ["dnf",
"-y",
"--config=" + os.path.join(workspace, "dnf.conf"),
"--best",
"--allowerasing",
"--releasever=" + args.release,
- "--installroot=" + workspace + "/root",
+ "--installroot=" + root,
"--disablerepo=*",
"--enablerepo=fedora",
"--enablerepo=updates",
@@ -457,6 +487,7 @@ def install_fedora(args, workspace, run_build_script):
if args.bootable:
cmdline.extend(["kernel", "systemd-udev"])
+ os.makedirs(os.path.join(root, 'efi'), exist_ok=True)
subprocess.run(cmdline, check=True)
@@ -466,43 +497,53 @@ def install_debian_or_ubuntu(args, workspace, run_build_script, mirror):
cmdline = ["debootstrap",
"--verbose",
"--variant=minbase",
- "--include=systemd,dbus,libpam-systemd",
+ "--include=systemd-sysv",
"--exclude=sysv-rc,initscripts,startpar,lsb-base,insserv",
args.release,
workspace + "/root",
mirror]
-
- if args.packages is not None:
- cmdline[3] += "," + ",".join(args.packages)
-
- if run_build_script and args.build_packages is not None:
- cmdline[3] += "," + ",".join(args.build_packages)
-
if args.bootable and args.output_format == OutputFormat.raw_btrfs:
cmdline[3] += ",btrfs-tools"
subprocess.run(cmdline, check=True)
+
+ # Debootstrap is not smart enough to deal correctly with alternative dependencies
+ # Installing libpam-systemd via debootstrap results in systemd-shim being installed
+ # Therefore, prefer to install via apt from inside the container
+ extra_packages = [ 'dbus', 'libpam-systemd']
+
+ # Also install extra packages via the secondary APT run, because it is smarter and
+ # can deal better with any conflicts
+ if args.packages is not None:
+ extra_packages += args.packages
+
+ if run_build_script and args.build_packages is not None:
+ extra_packages += args.build_packages
+
# Work around debian bug #835628
os.makedirs(os.path.join(workspace, "root/etc/dracut.conf.d"), exist_ok=True)
with open(os.path.join(workspace, "root/etc/dracut.conf.d/99-generic.conf"), "w") as f:
f.write("hostonly=no")
- # We cannot install this directly in the debootstrap phase.
- # linux-image prefers initramfs-tools over dracut, and debootstrap is not smart enough
- # to realize the solution when installing linux-image-amd64 + dracut is to not install
- # initramfs-tools...
if args.bootable:
- subprocess.run(["systemd-nspawn",
- "--directory", os.path.join(workspace, "root"),
- "--as-pid2",
- "--register=no",
- "/usr/bin/apt-get", "--assume-yes", "--no-install-recommends", "install",
- "linux-image-amd64",
- "dracut",
- "systemd-sysv",
- ],
- check=True)
+ extra_packages += ["linux-image-amd64", "dracut"]
+
+ if extra_packages:
+ # Debian policy is to start daemons by default.
+ # The policy-rc.d script can be used choose which ones to start
+ # Let's install one that denies all daemon startups
+ # See https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt
+ # Note: despite writing in /usr/sbin, this file is not shipped by the OS
+ # and instead should be managed by the admin.
+ policyrcd = os.path.join(workspace, "root/usr/sbin/policy-rc.d")
+ with open(policyrcd, "w") as f:
+ f.write("#!/bin/sh\n")
+ f.write("exit 101")
+ os.chmod(policyrcd, 0o755)
+ cmdline = ["/usr/bin/apt-get", "--assume-yes", "--no-install-recommends", "install"] + extra_packages
+ run_workspace_command(workspace, network=True, *cmdline)
+ os.unlink(policyrcd)
def install_debian(args, workspace, run_build_script):
print_step("Installing Debian...")
@@ -524,25 +565,38 @@ def install_arch(args, workspace, run_build_script):
print_step("Installing ArchLinux...")
+ keyring = "archlinux"
+
+ if platform.machine() == "aarch64":
+ keyring += "arm"
+
subprocess.run(["pacman-key", "--nocolor", "--init"], check=True)
- subprocess.run(["pacman-key", "--nocolor", "--populate", "archlinux"], check=True)
+ subprocess.run(["pacman-key", "--nocolor", "--populate", keyring], check=True)
+
+
+ if platform.machine() == "aarch64":
+ server = "Server = {}/$arch/$repo".format(args.mirror)
+ else:
+ server = "Server = {}/$repo/os/$arch".format(args.mirror)
with open(os.path.join(workspace, "pacman.conf"), "w") as f:
- f.write("[options]\n")
- f.write("HookDir = /no_hook/\n")
- f.write("HoldPkg = pacman glibc\n")
- f.write("Architecture = auto\n")
- f.write("CheckSpace\n")
- f.write("SigLevel = Required DatabaseOptional\n")
- f.write("\n")
- f.write("[core]\n")
- f.write("Server = %s/$repo/os/$arch\n" % args.mirror)
- f.write("\n")
- f.write("[extra]\n")
- f.write("Server = %s/$repo/os/$arch\n" % args.mirror)
- f.write("\n")
- f.write("[community]\n")
- f.write("Server = %s/$repo/os/$arch\n" % args.mirror)
+ f.write("""\
+[options]
+HookDir = /no_hook/
+HoldPkg = pacman glibc
+Architecture = auto
+CheckSpace
+SigLevel = Required DatabaseOptional
+
+[core]
+{server}
+
+[extra]
+{server}
+
+[community]
+{server}
+""".format(args=args, server=server))
subprocess.run(["pacman", "--color", "never", "--config", os.path.join(workspace, "pacman.conf"), "-Sy"], check=True)
c = subprocess.run(["pacman", "--color", "never", "--config", os.path.join(workspace, "pacman.conf"), "-Sg", "base"], stdout=subprocess.PIPE, universal_newlines=True, check=True)
@@ -567,7 +621,8 @@ def install_arch(args, workspace, run_build_script):
elif args.output_format == OutputFormat.raw_btrfs:
packages.add("btrfs-progs")
else:
- packages.remove("linux")
+ if "linux" in packages:
+ packages.remove("linux")
if args.packages is not None:
packages |= set(args.packages)
@@ -598,31 +653,34 @@ def install_distribution(args, workspace, run_build_script):
install[args.distribution](args, workspace, run_build_script)
+def set_root_password(args, workspace):
+ "Set the root account password, or just delete it so it's easy to log in"
+ if args.password == '':
+ print_step("Deleting root password...")
+ jj = lambda line: (':'.join(['root', ''] + line.split(':')[2:])
+ if line.startswith('root:') else line)
+ patch_file(os.path.join(workspace, 'root', 'etc/passwd'), jj)
+ elif args.password:
+ print_step("Setting root password...")
+ password = crypt.crypt(args.password, crypt.mksalt(crypt.METHOD_SHA512))
+ jj = lambda line: (':'.join(['root', password] + line.split(':')[2:])
+ if line.startswith('root:') else line)
+ patch_file(os.path.join(workspace, 'root', 'etc/shadow'), jj)
+
def install_boot_loader_arch(args, workspace):
patch_file(os.path.join(workspace, "root", "etc/mkinitcpio.conf"),
lambda line: "HOOKS=\"systemd modconf block filesystems fsck\"\n" if line.startswith("HOOKS=") else line)
kernel_version = next(filter(lambda x: x[0].isdigit(), os.listdir(os.path.join(workspace, "root", "lib/modules"))))
- subprocess.run(["systemd-nspawn",
- "--directory", os.path.join(workspace, "root"),
- "--as-pid2",
- "--private-network",
- "--register=no",
- "/usr/bin/kernel-install", "add", kernel_version, "/boot/vmlinuz-linux"],
- check=True)
+ run_workspace_command(workspace,
+ "/usr/bin/kernel-install", "add", kernel_version, "/boot/vmlinuz-linux")
def install_boot_loader_debian(args, workspace):
kernel_version = next(filter(lambda x: x[0].isdigit(), os.listdir(os.path.join(workspace, "root", "lib/modules"))))
- subprocess.run(["systemd-nspawn",
- "--directory", os.path.join(workspace, "root"),
- "--as-pid2",
- "--private-network",
- "--register=no",
- "/usr/bin/kernel-install", "add", kernel_version, "/boot/vmlinuz-" + kernel_version],
- check=True)
-
+ run_workspace_command(workspace,
+ "/usr/bin/kernel-install", "add", kernel_version, "/boot/vmlinuz-" + kernel_version)
def install_boot_loader(args, workspace):
if not args.bootable:
@@ -674,6 +732,22 @@ def install_extra_trees(args, workspace):
print_step("Copying in extra file trees completed.")
+def git_files_ignore():
+ "Creates a function to be used as a ignore callable argument for copytree"
+ c = subprocess.run(['git', 'ls-files', '-z', '--others', '--cached',
+ '--exclude-standard', '--exclude', '/.mkosi-*'],
+ stdout=subprocess.PIPE,
+ universal_newlines=False,
+ check=True)
+ files = {x.decode("utf-8") for x in c.stdout.split(b'\0')}
+ del c
+
+ def ignore(src, names):
+ return [name for name in names
+ if (os.path.relpath(os.path.join(src, name)) not in files
+ and not os.path.isdir(os.path.join(src, name)))]
+ return ignore
+
def install_build_src(args, workspace, run_build_script):
if not run_build_script:
return
@@ -686,7 +760,16 @@ def install_build_src(args, workspace, run_build_script):
shutil.copy(args.build_script, os.path.join(workspace, "root", "root", os.path.basename(args.build_script)))
if args.build_sources is not None:
- shutil.copytree(args.build_sources, os.path.join(workspace, "root", "root/src"), symlinks=True, ignore=shutil.ignore_patterns('.mkosi-*'))
+ target = os.path.join(workspace, "root", "root/src")
+ use_git = args.use_git_files
+ if use_git is None:
+ use_git = os.path.exists('.git')
+
+ if use_git:
+ ignore = git_files_ignore()
+ else:
+ ignore = shutil.ignore_patterns('.mkosi-*', '.git')
+ shutil.copytree(args.build_sources, target, symlinks=True, ignore=ignore)
print_step("Copying in build script and sources completed.")
@@ -937,7 +1020,8 @@ def parse_args():
group.add_argument('-t', "--format", dest='output_format', choices=OutputFormat.__members__, help='Output Format')
group.add_argument('-o', "--output", help='Output image path', metavar='PATH')
group.add_argument('-f', "--force", action='store_true', help='Remove existing image file before operation')
- group.add_argument('-b', "--bootable", action='store_true', help='Make image bootable on EFI (only raw_gpt, raw_btrfs)')
+ group.add_argument('-b', "--bootable", type=parse_boolean, nargs='?', const=True,
+ help='Make image bootable on EFI (only raw_gpt, raw_btrfs)')
group.add_argument("--read-only", action='store_true', help='Make root volume read-only (only raw_btrfs, subvolume)')
group.add_argument("--compress", action='store_true', help='Enable compression in file system (only raw_btrfs, subvolume)')
group.add_argument("--xz", action='store_true', help='Compress resulting image with xz (only raw_gpt, raw_btrfs, implied on tar)')
@@ -947,9 +1031,11 @@ def parse_args():
group.add_argument("--with-docs", action='store_true', help='Install documentation (only fedora)')
group.add_argument("--cache", dest='cache_path', help='Package cache path (only fedora, debian, ubuntu)', metavar='PATH')
group.add_argument("--extra-tree", action='append', dest='extra_trees', help='Copy an extra tree on top of image', metavar='PATH')
- group.add_argument("--build-script", dest='build_script', help='Build script to run inside image', metavar='PATH')
- group.add_argument("--build-sources", dest='build_sources', help='Path for sources to build', metavar='PATH')
+ group.add_argument("--build-script", help='Build script to run inside image', metavar='PATH')
+ group.add_argument("--build-sources", help='Path for sources to build', metavar='PATH')
group.add_argument("--build-package", action=PackageAction, dest='build_packages', help='Additional packages needed for build script', metavar='PACKAGE')
+ group.add_argument('--use-git-files', type=parse_boolean,
+ help='Ignore any files that git itself ignores (default: guess)')
group.add_argument("--settings", dest='nspawn_settings', help='Add in .spawn settings file', metavar='PATH')
group = parser.add_argument_group("Partitions")
@@ -963,6 +1049,7 @@ def parse_args():
group.add_argument("--checksum", action='store_true', help='Write SHA256SUM file')
group.add_argument("--sign", action='store_true', help='Write and sign SHA256SUM file')
group.add_argument("--key", help='GPG key to use for signing')
+ group.add_argument("--password", help='Set the root password')
group = parser.add_argument_group("Additional Configuration")
group.add_argument('-C', "--directory", help='Change to specified directory before doing anything', metavar='PATH')
@@ -1061,7 +1148,7 @@ def parse_boolean(s):
if s in {"0", "false", "no"}:
return False
- return KeyError("Unknown setting")
+ raise ValueError("invalid literal for bool(): {!r}".format(s))
def process_setting(args, section, key, value):
if section == "Distribution":
@@ -1279,13 +1366,15 @@ def load_args():
if args.mirror is None:
if args.distribution == Distribution.fedora:
- args.mirror = "https://mirrors.kernel.org/fedora"
+ args.mirror = None
elif args.distribution == Distribution.debian:
args.mirror = "http://httpredir.debian.org/debian"
elif args.distribution == Distribution.ubuntu:
args.mirror = "http://archive.ubuntu.com/ubuntu"
elif args.distribution == Distribution.arch:
args.mirror = "https://mirrors.kernel.org/archlinux"
+ if platform.machine() == "aarch64":
+ args.mirror = "http://mirror.archlinuxarm.org"
if args.bootable:
if args.distribution not in (Distribution.fedora, Distribution.arch, Distribution.debian):
@@ -1391,7 +1480,8 @@ def print_summary(args):
sys.stderr.write("DISTRIBUTION:\n")
sys.stderr.write(" Distribution: " + args.distribution.name + "\n")
sys.stderr.write(" Release: " + none_to_na(args.release) + "\n")
- sys.stderr.write(" Mirror: " + args.mirror + "\n")
+ if args.mirror is not None:
+ sys.stderr.write(" Mirror: " + args.mirror + "\n")
sys.stderr.write("\nOUTPUT:\n")
sys.stderr.write(" Output Format: " + args.output_format.name + "\n")
sys.stderr.write(" Output: " + args.output + "\n")
@@ -1443,18 +1533,14 @@ def build_image(args, workspace, run_build_script):
tar = None
raw = create_image(args, workspace.name)
- loopdev = attach_image_loopback(args, raw)
-
- try:
+ with attach_image_loopback(args, raw) as loopdev:
prepare_swap(args, loopdev)
prepare_esp(args, loopdev)
prepare_root(args, loopdev)
prepare_home(args, loopdev)
prepare_srv(args, loopdev)
- mount_image(args, workspace.name, loopdev)
-
- try:
+ with mount_image(args, workspace.name, loopdev):
prepare_tree(args, workspace.name)
mount_cache(args, workspace.name)
install_distribution(args, workspace.name, run_build_script)
@@ -1464,13 +1550,9 @@ def build_image(args, workspace, run_build_script):
install_build_dest(args, workspace.name, run_build_script)
if not run_build_script:
+ set_root_password(args, workspace.name)
make_read_only(args, workspace.name)
tar = make_tar(args, workspace.name)
- finally:
- umount_image(args, workspace.name, loopdev)
-
- finally:
- detach_image_loopback(args, loopdev)
return raw, tar
@@ -1484,6 +1566,7 @@ def run_build_script(args, workspace, raw):
os.mkdir(dest, 0o755)
cmdline = ["systemd-nspawn",
+ '--quiet',
"--directory=" + os.path.join(workspace, "root") if raw is None else "--image=" + raw.name,
"--as-pid2",
"--private-network",