Merge tag 'upstream/0_20161004'

Upstream version 0~20161004

# gpg: Signature made Wed 05 Oct 2016 19:52:43 CLST
# gpg:                using RSA key 0xA3BABAE2408DD6CF
# gpg: Good signature from "Felipe Sateler <fsateler@debian.org>" [ultimate]
# gpg:                 aka "Felipe Sateler <fsateler@uc.cl>" [ultimate]
# gpg:                 aka "Felipe Sateler <fsateler@gmail.com>" [ultimate]
# Primary key fingerprint: 218E E036 2033 C87B 6C13  5FA4 A3BA BAE2 408D D6CF

1 files changed, 104 insertions, 91 deletions

diff --git a/mkosi b/mkosi
index 67fb0a5..1bbd8d8 100755
--- a/mkosi
+++ b/mkosi
@@ -1,7 +1,18 @@
 #!/usr/bin/python3
 
+import argparse
+import configparser
+import ctypes, ctypes.util
+import hashlib
+import os
+import platform
+import shutil
+import subprocess
+import sys
+import tempfile
+import time
+import uuid
 from enum import Enum
-import argparse, sys, os, subprocess, uuid, platform, tempfile, shutil, time, hashlib, configparser, ctypes, ctypes.util
 
 # TODO
 # - squashfs root
@@ -64,11 +75,10 @@ def print_step(text):
 
 def setup_workspace(args):
     print_step("Setting up temporary workspace.")
-
     if args.output_format in (OutputFormat.directory, OutputFormat.subvolume):
-        d = tempfile.TemporaryDirectory(dir = os.path.dirname(args.output), prefix='.mkosi-')
+        d = tempfile.TemporaryDirectory(dir=os.path.dirname(args.output), prefix='.mkosi-')
     else:
-        d = tempfile.TemporaryDirectory(dir = '/var/tmp', prefix='mkosi-')
+        d = tempfile.TemporaryDirectory(dir='/var/tmp', prefix='mkosi-')
 
     print_step("Temporary workspace in " + d.name + " is now set up.")
     return d
@@ -87,13 +97,13 @@ def btrfs_subvol_make_ro(path, b=True):
 def image_size(args):
     size = args.root_size
 
-    if not args.home_size is None:
+    if args.home_size is not None:
         size += args.home_size
-    if not args.srv_size is None:
+    if args.srv_size is not None:
         size += args.srv_size
     if args.bootable:
         size += args.esp_size
-    if not args.swap_size is None:
+    if args.swap_size is not None:
         size += args.swap_size
 
     return size
@@ -118,7 +128,7 @@ def create_image(args, workspace):
     else:
         args.esp_partno = None
 
-    if not args.swap_size is None:
+    if args.swap_size is not None:
         table += 'size={}, type={}, name="Swap Partition"\n'.format(str(int(args.swap_size / 512)), GPT_SWAP)
         args.swap_partno = pn
         pn += 1
@@ -129,12 +139,12 @@ def create_image(args, workspace):
     args.srv_partno = None
 
     if args.output_format != OutputFormat.raw_btrfs:
-        if not args.home_size is None:
+        if args.home_size is not None:
             table += 'size={}, type={}, name="Home Partition"\n'.format(str(int(args.home_size / 512)), GPT_HOME)
             args.home_partno = pn
             pn += 1
 
-        if not args.srv_size is None:
+        if args.srv_size is not None:
             table += 'size={}, type={}, name="Server Data Partition"\n'.format(str(int(args.srv_size / 512)), GPT_SRV)
             args.srv_partno = pn
             pn += 1
@@ -204,7 +214,7 @@ def prepare_esp(args, loopdev):
     print_step("Formatting ESP partition completed.");
 
 def mkfs_ext4(label, mount, loopdev, partno):
-    subprocess.run(["mkfs.ext4", "-L" + label, "-M" + mount, partition(loopdev, partno)], check=True)
+    subprocess.run(["mkfs.ext4", "-L", label, "-M", mount, partition(loopdev, partno)], check=True)
 
 def prepare_root(args, loopdev):
     if loopdev is None:
@@ -267,13 +277,13 @@ def mount_image(args, workspace, loopdev):
 
     mount_loop(args, loopdev, args.root_partno, os.path.join(workspace, "root"))
 
-    if not args.home_partno is None:
+    if args.home_partno is not None:
         mount_loop(args, loopdev, args.home_partno, os.path.join(workspace, "root", "home"))
 
-    if not args.srv_partno is None:
+    if args.srv_partno is not None:
         mount_loop(args, loopdev, args.srv_partno, os.path.join(workspace, "root", "srv"))
 
-    if not args.esp_partno is None:
+    if args.esp_partno is not None:
         mount_loop(args, loopdev, args.esp_partno, os.path.join(workspace, "root", "boot/efi"))
 
     if args.distribution == Distribution.fedora:
@@ -356,7 +366,9 @@ def prepare_tree(args, workspace):
 
         os.mkdir(os.path.join(workspace, "root", "etc/kernel"), 0o755)
 
-        open(os.path.join(workspace, "root", "etc/kernel/cmdline"), "w").write("rhgb quiet selinux=0 audit=0 rw\n")
+        with open(os.path.join(workspace, "root", "etc/kernel/cmdline"), "w") as cmdline:
+            cmdline.write(args.kernel_commandline)
+            cmdline.write("\n")
 
     print_step("Setting up basic OS tree completed.");
 
@@ -437,10 +449,10 @@ def install_fedora(args, workspace, run_build_script):
                "fedora-release",
                "passwd"])
 
-    if not args.packages is None:
+    if args.packages is not None:
         cmdline.extend(args.packages)
 
-    if run_build_script and not args.build_packages is None:
+    if run_build_script and args.build_packages is not None:
         cmdline.extend(args.build_packages)
 
     if args.bootable:
@@ -454,16 +466,16 @@ def install_debian_or_ubuntu(args, workspace, run_build_script, mirror):
     cmdline = ["debootstrap",
                "--verbose",
                "--variant=minbase",
-               "--include=systemd",
+               "--include=systemd,dbus,libpam-systemd",
                "--exclude=sysv-rc,initscripts,startpar,lsb-base,insserv",
                args.release,
                workspace + "/root",
                mirror]
 
-    if not args.packages is None:
+    if args.packages is not None:
         cmdline[3] += "," + ",".join(args.packages)
 
-    if run_build_script and not args.build_packages is None:
+    if run_build_script and args.build_packages is not None:
         cmdline[3] += "," + ",".join(args.build_packages)
 
     if args.bootable and args.output_format == OutputFormat.raw_btrfs:
@@ -472,7 +484,7 @@ def install_debian_or_ubuntu(args, workspace, run_build_script, mirror):
     subprocess.run(cmdline, check=True)
 
     # Work around debian bug #835628
-    os.makedirs(os.path.join(workspace, "root/etc/dracut.conf.d"))
+    os.makedirs(os.path.join(workspace, "root/etc/dracut.conf.d"), exist_ok=True)
     with open(os.path.join(workspace, "root/etc/dracut.conf.d/99-generic.conf"), "w") as f:
         f.write("hostonly=no")
 
@@ -482,15 +494,15 @@ def install_debian_or_ubuntu(args, workspace, run_build_script, mirror):
     # initramfs-tools...
     if args.bootable:
         subprocess.run(["systemd-nspawn",
-                    "--directory=" + os.path.join(workspace, "root"),
-                    "--as-pid2",
-                    "--register=no",
-                    "/usr/bin/apt-get", "--assume-yes", "--no-install-recommends", "install",
-                    "linux-image-amd64",
-                    "dracut",
-                    "systemd-sysv",
-                    ],
-                   check=True)
+                        "--directory", os.path.join(workspace, "root"),
+                        "--as-pid2",
+                        "--register=no",
+                        "/usr/bin/apt-get", "--assume-yes", "--no-install-recommends", "install",
+                        "linux-image-amd64",
+                        "dracut",
+                        "systemd-sysv",
+                       ],
+                       check=True)
 
 def install_debian(args, workspace, run_build_script):
     print_step("Installing Debian...")
@@ -507,13 +519,13 @@ def install_ubuntu(args, workspace, run_build_script):
     print_step("Installing Ubuntu completed.")
 
 def install_arch(args, workspace, run_build_script):
-    if not args.release is None:
+    if args.release is not None:
         sys.stderr.write("Distribution release specification is not supported for ArchLinux, ignoring.")
 
     print_step("Installing ArchLinux...")
 
-    subprocess.run(["pacman-key", "--init"], check=True)
-    subprocess.run(["pacman-key", "--populate", "archlinux"], check=True)
+    subprocess.run(["pacman-key", "--nocolor", "--init"], check=True)
+    subprocess.run(["pacman-key", "--nocolor", "--populate", "archlinux"], check=True)
 
     with open(os.path.join(workspace, "pacman.conf"), "w") as f:
         f.write("[options]\n")
@@ -532,8 +544,8 @@ def install_arch(args, workspace, run_build_script):
         f.write("[community]\n")
         f.write("Server = %s/$repo/os/$arch\n" % args.mirror)
 
-    subprocess.run(["pacman", "--config", os.path.join(workspace, "pacman.conf"), "-Sy"], check=True)
-    c = subprocess.run(["pacman", "--config", os.path.join(workspace, "pacman.conf"), "-Sg", "base"], stdout=subprocess.PIPE, universal_newlines=True, check=True)
+    subprocess.run(["pacman", "--color", "never", "--config", os.path.join(workspace, "pacman.conf"), "-Sy"], check=True)
+    c = subprocess.run(["pacman", "--color", "never", "--config", os.path.join(workspace, "pacman.conf"), "-Sg", "base"], stdout=subprocess.PIPE, universal_newlines=True, check=True)
     packages = set(c.stdout.split())
     packages.remove("base")
 
@@ -593,7 +605,7 @@ def install_boot_loader_arch(args, workspace):
     kernel_version = next(filter(lambda x: x[0].isdigit(), os.listdir(os.path.join(workspace, "root", "lib/modules"))))
 
     subprocess.run(["systemd-nspawn",
-                    "--directory=" + os.path.join(workspace, "root"),
+                    "--directory", os.path.join(workspace, "root"),
                     "--as-pid2",
                     "--private-network",
                     "--register=no",
@@ -604,7 +616,7 @@ def install_boot_loader_debian(args, workspace):
     kernel_version = next(filter(lambda x: x[0].isdigit(), os.listdir(os.path.join(workspace, "root", "lib/modules"))))
 
     subprocess.run(["systemd-nspawn",
-                    "--directory=" + os.path.join(workspace, "root"),
+                    "--directory", os.path.join(workspace, "root"),
                     "--as-pid2",
                     "--private-network",
                     "--register=no",
@@ -637,11 +649,9 @@ def enumerate_and_copy(source, dest, suffix = ""):
         dest_path = dest + suffix + "/" + entry.name
 
         if entry.is_dir():
-            try:
-                os.mkdir(dest_path, entry.stat(follow_symlinks=False).st_mode & 0o7777)
-            except FileExistsError:
-                pass
-
+            os.makedirs(dest_path,
+                        mode=entry.stat(follow_symlinks=False).st_mode & 0o7777,
+                        exist_ok=True)
             enumerate_and_copy(source, dest, suffix + "/" + entry.name)
         else:
             try:
@@ -675,7 +685,7 @@ def install_build_src(args, workspace, run_build_script):
 
     shutil.copy(args.build_script, os.path.join(workspace, "root", "root", os.path.basename(args.build_script)))
 
-    if not args.build_sources is None:
+    if args.build_sources is not None:
         shutil.copytree(args.build_sources, os.path.join(workspace, "root", "root/src"), symlinks=True, ignore=shutil.ignore_patterns('.mkosi-*'))
 
     print_step("Copying in build script and sources completed.")
@@ -744,7 +754,6 @@ def copy_nspawn_settings(args):
     f = tempfile.NamedTemporaryFile(mode = "w+b", dir = os.path.dirname(args.output_nspawn_settings), prefix=".mkosi-")
 
     with open(args.nspawn_settings, "rb") as c:
-
         bs = 65536
         buf = c.read(bs)
         while len(buf) > 0:
@@ -775,13 +784,13 @@ def calculate_sha256sum(args, raw, tar, nspawn_settings):
 
     print_step("Calculating SHA256SUM...")
 
-    f = tempfile.NamedTemporaryFile(mode = "w+", dir = os.path.dirname(args.output_checksum), prefix=".mkosi-", encoding = "utf-8")
+    f = tempfile.NamedTemporaryFile(mode="w+", dir=os.path.dirname(args.output_checksum), prefix=".mkosi-", encoding="utf-8")
 
-    if not raw is None:
+    if raw is not None:
         hash_file(f, raw, os.path.basename(args.output))
-    if not tar is None:
+    if tar is not None:
         hash_file(f, tar, os.path.basename(args.output))
-    if not nspawn_settings is None:
+    if nspawn_settings is not None:
         hash_file(f, nspawn_settings, os.path.basename(args.output_nspawn_settings))
 
     print_step("Calculating SHA256SUM complete.")
@@ -796,11 +805,11 @@ def calculate_signature(args, checksum):
 
     print_step("Signing SHA256SUM...")
 
-    f = tempfile.NamedTemporaryFile(mode = "wb", prefix=".mkosi-", dir = os.path.dirname(args.output_signature))
+    f = tempfile.NamedTemporaryFile(mode="wb", prefix=".mkosi-", dir=os.path.dirname(args.output_signature))
 
     cmdline = ["gpg", "--detach-sign"]
 
-    if not args.key is None:
+    if args.key is not None:
         cmdline.extend(["--default-key", args.key])
 
     checksum.seek(0)
@@ -859,18 +868,23 @@ def link_output_signature(args, signature):
 
 def format_bytes(bytes):
     if bytes >= 1024*1024*1024:
-        return "{:0.1f}G".format(float(bytes) / 1024/1024/1024)
+        return "{:0.1f}G".format(bytes / 1024**3)
     if bytes >= 1024*1024:
-        return "{:0.1f}M".format(float(bytes) / 1024/1024)
+        return "{:0.1f}M".format(bytes / 1024**2)
     if bytes >= 1024:
-        return "{:0.1f}K".format(float(bytes) / 1024)
+        return "{:0.1f}K".format(bytes / 1024)
 
     return "{}B".format(bytes)
 
 def dir_size(path):
     sum = 0
     for entry in os.scandir(path):
-        if entry.is_file():
+        if entry.is_symlink():
+            # We can ignore symlinks because they either point into our tree,
+            # in which case we'll include the size of target directory anyway,
+            # or outside, in which case we don't need to.
+            continue
+        elif entry.is_file():
             sum += entry.stat().st_blocks * 512
         elif entry.is_dir():
             sum += dir_size(entry.path)
@@ -890,7 +904,7 @@ def setup_cache(args):
     print_step("Setting up package cache...")
 
     if args.cache_path is None:
-        d = tempfile.TemporaryDirectory(dir = os.path.dirname(args.output), prefix=".mkosi-")
+        d = tempfile.TemporaryDirectory(dir=os.path.dirname(args.output), prefix=".mkosi-")
         args.cache_path = d.name
     else:
         os.makedirs(args.cache_path, 0o700, True)
@@ -953,6 +967,7 @@ def parse_args():
     group = parser.add_argument_group("Additional Configuration")
     group.add_argument('-C', "--directory", help='Change to specified directory before doing anything', metavar='PATH')
     group.add_argument("--default", dest='default_path', help='Read configuration data from file', metavar='PATH')
+    group.add_argument("--kernel-commandline", help='Set the kernel command line (only bootable images)')
 
     args = parser.parse_args()
 
@@ -967,9 +982,9 @@ def parse_bytes(bytes):
         return bytes
 
     if bytes.endswith('G'):
-        factor = 1024*1024*1024
+        factor = 1024**3
     elif bytes.endswith('M'):
-        factor = 1024*1024
+        factor = 1024**2
     elif bytes.endswith('K'):
         factor = 1024
     else:
@@ -1005,13 +1020,7 @@ def detect_distribution():
         if ln.startswith("VERSION_ID="):
             version_id = ln[11:].strip()
 
-    d = None
-    if not id is None:
-        try:
-            d = Distribution[id]
-        except KeyError:
-            pass
-
+    d = Distribution.__members__.get(id, None)
     return d, version_id
 
 def unlink_try_hard(path):
@@ -1042,14 +1051,14 @@ def unlink_output(args):
     if args.sign:
         unlink_try_hard(args.output_signature)
 
-    if not args.nspawn_settings is None:
+    if args.nspawn_settings is not None:
         unlink_try_hard(args.output_nspawn_settings)
 
 def parse_boolean(s):
-    if s in ("1", "true", "yes"):
+    if s in {"1", "true", "yes"}:
         return True
 
-    if s in ("0", "false", "no"):
+    if s in {"0", "false", "no"}:
         return False
 
     return KeyError("Unknown setting")
@@ -1110,10 +1119,10 @@ def process_setting(args, section, key, value):
             else:
                 args.extra_trees.extend(value.split())
         elif key == "BuildScript":
-            if not args.build_script is None:
+            if args.build_script is not None:
                 args.build_script = value
         elif key == "BuildSources":
-            if not args.build_sources is None:
+            if args.build_sources is not None:
                 args.build_sources = value
         elif key == "BuildPackages":
             if args.build_packages is None:
@@ -1121,7 +1130,7 @@ def process_setting(args, section, key, value):
             else:
                 args.build_packages.extend(value.split())
         elif key == "NSpawnSettings":
-            if not args.nspawn_settings is None:
+            if args.nspawn_settings is not None:
                 args.nspawn_settings = value
         elif key is None:
             return True
@@ -1187,7 +1196,7 @@ def load_defaults(args):
                 sys.stderr.write("Unknown key in section [{}] in {}, ignoring: {}=\n".format(section, fname, key))
 
 def find_nspawn_settings(args):
-    if not args.nspawn_settings is None:
+    if args.nspawn_settings is not None:
         return
 
     if os.path.exists("mkosi.nspawn"):
@@ -1201,14 +1210,14 @@ def find_extra(args):
             args.extra_trees.append("mkosi.extra")
 
 def find_build_script(args):
-    if not args.build_script is None:
+    if args.build_script is not None:
         return
 
     if os.path.exists("mkosi.build"):
         args.build_script = "mkosi.build"
 
 def find_build_sources(args):
-    if not args.build_sources is None:
+    if args.build_sources is not None:
         return
 
     args.build_sources = os.getcwd()
@@ -1230,7 +1239,7 @@ def build_nspawn_settings_path(path):
 def load_args():
     args = parse_args()
 
-    if not args.directory is None:
+    if args.directory is not None:
         os.chdir(args.directory)
 
     load_defaults(args)
@@ -1244,7 +1253,7 @@ def load_args():
     else:
         args.output_format = OutputFormat[args.output_format]
 
-    if not args.distribution is None:
+    if args.distribution is not None:
         args.distribution = Distribution[args.distribution]
 
     if args.distribution is None or args.release is None:
@@ -1312,17 +1321,17 @@ def load_args():
     if args.sign:
         args.output_signature = os.path.join(os.path.dirname(args.output), "SHA256SUM.gpg")
 
-    if not args.nspawn_settings is None:
+    if args.nspawn_settings is not None:
         args.nspawn_settings = os.path.abspath(args.nspawn_settings)
         args.output_nspawn_settings = build_nspawn_settings_path(args.output)
 
-    if not args.build_script is None:
+    if args.build_script is not None:
         args.build_script = os.path.abspath(args.build_script)
 
-    if not args.build_sources is None:
+    if args.build_sources is not None:
         args.build_sources = os.path.abspath(args.build_sources)
 
-    if not args.extra_trees is None:
+    if args.extra_trees is not None:
         for i in range(len(args.extra_trees)):
             args.extra_trees[i] = os.path.abspath(args.extra_trees[i])
 
@@ -1338,13 +1347,16 @@ def load_args():
     if args.bootable and args.esp_size is None:
         args.esp_size = 256*1024*1024
 
+    if args.bootable and args.kernel_commandline is None:
+        args.kernel_commandline = "rhgb quiet selinux=0 audit=0 rw"
+
     return args
 
 def check_output(args):
     for f in (args.output,
               args.output_checksum if args.checksum else None,
               args.output_signature if args.sign else None,
-              args.output_nspawn_settings if not args.nspawn_settings is None else None):
+              args.output_nspawn_settings if args.nspawn_settings is not None else None):
 
         if f is None:
             continue
@@ -1476,11 +1488,11 @@ def run_build_script(args, workspace, raw):
                "--as-pid2",
                "--private-network",
                "--register=no",
-               "--bind=" + dest + ":/root/dest",
+               "--bind", dest + ":/root/dest",
                "--setenv=WITH_DOCS=" + ("1" if args.with_docs else "0"),
                "--setenv=DESTDIR=/root/dest"]
 
-    if not args.build_sources is None:
+    if args.build_sources is not None:
         cmdline.append("--setenv=SRCDIR=/root/src")
         cmdline.append("--chdir=/root/src")
     else:
@@ -1498,17 +1510,17 @@ def build_stuff(args):
     workspace = setup_workspace(args)
 
     # Run the image builder twice, once for running the build script and once for the final build
-    (raw, tar) = build_image(args, workspace, run_build_script=True)
+    raw, tar = build_image(args, workspace, run_build_script=True)
 
     run_build_script(args, workspace.name, raw)
 
-    if not raw is None:
+    if raw is not None:
         del raw
 
-    if not tar is None:
+    if tar is not None:
         del tar
 
-    (raw, tar) = build_image(args, workspace, run_build_script=False)
+    raw, tar = build_image(args, workspace, run_build_script=False)
 
     raw = xz_output(args, raw)
     settings = copy_nspawn_settings(args)
@@ -1517,17 +1529,18 @@ def build_stuff(args):
 
     link_output(args,
                 workspace.name,
-                None if raw is None else raw.name,
-                None if tar is None else tar.name)
+                raw.name if raw is not None else None,
+                tar.name if tar is not None else None)
 
     link_output_checksum(args,
-                         None if checksum is None else checksum.name)
+                         checksum.name if checksum is not None else None)
 
     link_output_signature(args,
-                          None if signature is None else signature.name)
+                          signature.name if signature is not None else None)
 
     link_output_nspawn_settings(args,
-                                None if settings is None else settings.name)
+                                settings.name if settings is not None else None)
+
 
 def main():
     args = load_args()