core: introduce system state enum

The system state knows the states starting →
running/degraded/maintenance → stopping, where:

starting = system startup
running = normal operation
degraded = at least one unit is currently in failed state
maintenance = rescue/emergency mode is active or queued
stopping = system shutdown

4 files changed, 124 insertions, 5 deletions

diff --git a/src/core/dbus-manager.c b/src/core/dbus-manager.c
index 30f28b6e2..135d31465 100644
--- a/src/core/dbus-manager.c
+++ b/src/core/dbus-manager.c
@@ -239,6 +239,24 @@ static int property_get_n_names(
         return sd_bus_message_append(reply, "u", (uint32_t) hashmap_size(m->units));
 }
 
+static int property_get_n_failed_units(
+                sd_bus *bus,
+                const char *path,
+                const char *interface,
+                const char *property,
+                sd_bus_message *reply,
+                void *userdata,
+                sd_bus_error *error) {
+
+        Manager *m = userdata;
+
+        assert(bus);
+        assert(reply);
+        assert(m);
+
+        return sd_bus_message_append(reply, "u", (uint32_t) set_size(m->failed_units));
+}
+
 static int property_get_n_jobs(
                 sd_bus *bus,
                 const char *path,
@@ -281,6 +299,24 @@ static int property_get_progress(
         return sd_bus_message_append(reply, "d", d);
 }
 
+static int property_get_system_state(
+                sd_bus *bus,
+                const char *path,
+                const char *interface,
+                const char *property,
+                sd_bus_message *reply,
+                void *userdata,
+                sd_bus_error *error) {
+
+        Manager *m = userdata;
+
+        assert(bus);
+        assert(reply);
+        assert(m);
+
+        return sd_bus_message_append(reply, "s", manager_state_to_string(manager_state(m)));
+}
+
 static int property_set_runtime_watchdog(
                 sd_bus *bus,
                 const char *path,
@@ -1598,6 +1634,7 @@ const sd_bus_vtable bus_manager_vtable[] = {
         SD_BUS_WRITABLE_PROPERTY("LogLevel", "s", property_get_log_level, property_set_log_level, 0, 0),
         SD_BUS_WRITABLE_PROPERTY("LogTarget", "s", property_get_log_target, property_set_log_target, 0, 0),
         SD_BUS_PROPERTY("NNames", "u", property_get_n_names, 0, 0),
+        SD_BUS_PROPERTY("NFailedUnits", "u", property_get_n_failed_units, 0, 0),
         SD_BUS_PROPERTY("NJobs", "u", property_get_n_jobs, 0, 0),
         SD_BUS_PROPERTY("NInstalledJobs", "u", bus_property_get_unsigned, offsetof(Manager, n_installed_jobs), 0),
         SD_BUS_PROPERTY("NFailedJobs", "u", bus_property_get_unsigned, offsetof(Manager, n_failed_jobs), 0),
@@ -1611,6 +1648,7 @@ const sd_bus_vtable bus_manager_vtable[] = {
         SD_BUS_WRITABLE_PROPERTY("RuntimeWatchdogUSec", "t", bus_property_get_usec, property_set_runtime_watchdog, offsetof(Manager, runtime_watchdog), SD_BUS_VTABLE_PROPERTY_CONST),
         SD_BUS_WRITABLE_PROPERTY("ShutdownWatchdogUSec", "t", bus_property_get_usec, bus_property_set_usec, offsetof(Manager, shutdown_watchdog), SD_BUS_VTABLE_PROPERTY_CONST),
         SD_BUS_PROPERTY("ControlGroup", "s", NULL, offsetof(Manager, cgroup_root), 0),
+        SD_BUS_PROPERTY("SystemState", "s", property_get_system_state, 0, 0),
 
         SD_BUS_METHOD("GetUnit", "s", "o", method_get_unit, SD_BUS_VTABLE_UNPRIVILEGED),
         SD_BUS_METHOD("GetUnitByPID", "u", "o", method_get_unit_by_pid, SD_BUS_VTABLE_UNPRIVILEGED),
diff --git a/src/core/manager.c b/src/core/manager.c
index 78f4f3dfd..f618bed42 100644
--- a/src/core/manager.c
+++ b/src/core/manager.c
@@ -140,6 +140,8 @@ static void draw_cylon(char buffer[], size_t buflen, unsigned width, unsigned po
 }
 
 void manager_flip_auto_status(Manager *m, bool enable) {
+        assert(m);
+
         if (enable) {
                 if (m->show_status == SHOW_STATUS_AUTO)
                         manager_set_show_status(m, SHOW_STATUS_TEMPORARY);
@@ -451,6 +453,10 @@ int manager_new(SystemdRunningAs running_as, Manager **_m) {
         if (r < 0)
                 goto fail;
 
+        r = set_ensure_allocated(&m->failed_units, trivial_hash_func, trivial_compare_func);
+        if (r < 0)
+                goto fail;
+
         r = sd_event_default(&m->event);
         if (r < 0)
                 goto fail;
@@ -778,6 +784,8 @@ void manager_free(Manager *m) {
         hashmap_free(m->watch_pids2);
         hashmap_free(m->watch_bus);
 
+        set_free(m->failed_units);
+
         sd_event_source_unref(m->signal_event_source);
         sd_event_source_unref(m->notify_event_source);
         sd_event_source_unref(m->time_change_event_source);
@@ -1799,7 +1807,7 @@ int manager_loop(Manager *m) {
         RATELIMIT_DEFINE(rl, 1*USEC_PER_SEC, 50000);
 
         assert(m);
-        m->exit_code = MANAGER_RUNNING;
+        m->exit_code = MANAGER_OK;
 
         /* Release the path cache */
         set_free_free(m->unit_path_cache);
@@ -1813,7 +1821,7 @@ int manager_loop(Manager *m) {
         if (r < 0)
                 return r;
 
-        while (m->exit_code == MANAGER_RUNNING) {
+        while (m->exit_code == MANAGER_OK) {
                 usec_t wait_usec;
 
                 if (m->runtime_watchdog > 0 && m->running_as == SYSTEMD_SYSTEM)
@@ -2463,10 +2471,12 @@ void manager_check_finished(Manager *m) {
                 m->jobs_in_progress_event_source = sd_event_source_unref(m->jobs_in_progress_event_source);
 
         if (hashmap_size(m->jobs) > 0) {
+
                 if (m->jobs_in_progress_event_source) {
-                        uint64_t next = now(CLOCK_MONOTONIC) + JOBS_IN_PROGRESS_WAIT_USEC;
-                        sd_event_source_set_time(m->jobs_in_progress_event_source, next);
+                        sd_event_source_set_time(m->jobs_in_progress_event_source,
+                                                 now(CLOCK_MONOTONIC) + JOBS_IN_PROGRESS_WAIT_USEC);
                 }
+
                 return;
         }
 
@@ -2858,8 +2868,51 @@ Set *manager_get_units_requiring_mounts_for(Manager *m, const char *path) {
 }
 
 const char *manager_get_runtime_prefix(Manager *m) {
+        assert(m);
 
         return m->running_as == SYSTEMD_SYSTEM ?
                "/run" :
                getenv("XDG_RUNTIME_DIR");
 }
+
+ManagerState manager_state(Manager *m) {
+        Unit *u;
+
+        assert(m);
+
+        /* Did we ever finish booting? If not then we are still starting up */
+        if (!dual_timestamp_is_set(&m->finish_timestamp))
+                return MANAGER_STARTING;
+
+        /* Is the special shutdown target queued? If so, we are in shutdown state */
+        u = manager_get_unit(m, SPECIAL_SHUTDOWN_TARGET);
+        if (u && u->job && IN_SET(u->job->type, JOB_START, JOB_RESTART, JOB_TRY_RESTART, JOB_RELOAD_OR_START))
+                return MANAGER_STOPPING;
+
+        /* Are the rescue or emergency targets active or queued? If so we are in maintenance state */
+        u = manager_get_unit(m, SPECIAL_RESCUE_TARGET);
+        if (u && (UNIT_IS_ACTIVE_OR_ACTIVATING(unit_active_state(u)) ||
+                  (u->job && IN_SET(u->job->type, JOB_START, JOB_RESTART, JOB_TRY_RESTART, JOB_RELOAD_OR_START))))
+                return MANAGER_MAINTENANCE;
+
+        u = manager_get_unit(m, SPECIAL_EMERGENCY_TARGET);
+        if (u && (UNIT_IS_ACTIVE_OR_ACTIVATING(unit_active_state(u)) ||
+                  (u->job && IN_SET(u->job->type, JOB_START, JOB_RESTART, JOB_TRY_RESTART, JOB_RELOAD_OR_START))))
+                return MANAGER_MAINTENANCE;
+
+        /* Are there any failed units? If so, we are in degraded mode */
+        if (set_size(m->failed_units) > 0)
+                return MANAGER_DEGRADED;
+
+        return MANAGER_RUNNING;
+}
+
+static const char *const manager_state_table[_MANAGER_STATE_MAX] = {
+        [MANAGER_STARTING] = "starting",
+        [MANAGER_RUNNING] = "running",
+        [MANAGER_DEGRADED] = "degraded",
+        [MANAGER_MAINTENANCE] = "maintenance",
+        [MANAGER_STOPPING] = "stopping",
+};
+
+DEFINE_STRING_TABLE_LOOKUP(manager_state, ManagerState);
diff --git a/src/core/manager.h b/src/core/manager.h
index e014abd04..38f1c89c9 100644
--- a/src/core/manager.h
+++ b/src/core/manager.h
@@ -35,8 +35,18 @@
 
 typedef struct Manager Manager;
 
-typedef enum ManagerExitCode {
+typedef enum ManagerState {
+        MANAGER_STARTING,
         MANAGER_RUNNING,
+        MANAGER_DEGRADED,
+        MANAGER_MAINTENANCE,
+        MANAGER_STOPPING,
+        _MANAGER_STATE_MAX,
+        _MANAGER_STATE_INVALID = -1
+} ManagerState;
+
+typedef enum ManagerExitCode {
+        MANAGER_OK,
         MANAGER_EXIT,
         MANAGER_RELOAD,
         MANAGER_REEXECUTE,
@@ -106,6 +116,9 @@ struct Manager {
         Hashmap *watch_pids1;  /* pid => Unit object n:1 */
         Hashmap *watch_pids2;  /* pid => Unit object n:1 */
 
+        /* A set which contains all currently failed units */
+        Set *failed_units;
+
         sd_event_source *run_queue_event_source;
 
         char *notify_socket;
@@ -321,3 +334,8 @@ void manager_flip_auto_status(Manager *m, bool enable);
 Set *manager_get_units_requiring_mounts_for(Manager *m, const char *path);
 
 const char *manager_get_runtime_prefix(Manager *m);
+
+ManagerState manager_state(Manager *m);
+
+const char *manager_state_to_string(ManagerState m) _const_;
+ManagerState manager_state_from_string(const char *s) _pure_;
diff --git a/src/core/unit.c b/src/core/unit.c
index aa723cb8c..5c9884354 100644
--- a/src/core/unit.c
+++ b/src/core/unit.c
@@ -471,6 +471,8 @@ void unit_free(Unit *u) {
                 free(u->cgroup_path);
         }
 
+        set_remove(u->manager->failed_units, u);
+
         free(u->description);
         strv_free(u->documentation);
         free(u->fragment_path);
@@ -1507,6 +1509,7 @@ void unit_notify(Unit *u, UnitActiveState os, UnitActiveState ns, bool reload_su
 
         m = u->manager;
 
+        /* Update timestamps for state changes */
         if (m->n_reloading <= 0) {
                 dual_timestamp ts;
 
@@ -1523,6 +1526,13 @@ void unit_notify(Unit *u, UnitActiveState os, UnitActiveState ns, bool reload_su
                         u->active_exit_timestamp = ts;
         }
 
+        /* Keep track of failed of units */
+        if (ns == UNIT_FAILED && os != UNIT_FAILED)
+                set_put(u->manager->failed_units, u);
+        else if (os == UNIT_FAILED && ns != UNIT_FAILED)
+                set_remove(u->manager->failed_units, u);
+
+        /* Make sure the cgroup is always removed when we become inactive */
         if (UNIT_IS_INACTIVE_OR_FAILED(ns))
                 unit_destroy_cgroup(u);