importd: automatically grow /var/lib/machines/ loopback filesystem during downloads

If /var/lib/machines is mounted as btrfs loopback file system in
/var/lib/machines.raw with this change we automatically grow the file
system as it fills up. After each 10M we write to it during imports, we
check the free disk space, and if the fill level grows beyond 66% we
increase the size of the file system to 3x the fill level (thus lowering
it to 33%).

5 files changed, 31 insertions, 0 deletions

diff --git a/src/import/import-dkr.c b/src/import/import-dkr.c
index fb72f6cee..2d4e9b398 100644
--- a/src/import/import-dkr.c
+++ b/src/import/import-dkr.c
@@ -28,6 +28,7 @@
 #include "btrfs-util.h"
 #include "utf8.h"
 #include "mkdir.h"
+#include "path-util.h"
 #include "import-util.h"
 #include "curl-util.h"
 #include "aufs-util.h"
@@ -72,6 +73,7 @@ struct DkrImport {
 
         char *local;
         bool force_local;
+        bool grow_machine_directory;
 
         char *temp_path;
         char *final_path;
@@ -156,6 +158,8 @@ int dkr_import_new(
         if (!i->image_root)
                 return -ENOMEM;
 
+        i->grow_machine_directory = path_startswith(i->image_root, "/var/lib/machines");
+
         i->index_url = strdup(index_url);
         if (!i->index_url)
                 return -ENOMEM;
@@ -561,6 +565,7 @@ static int dkr_import_pull_layer(DkrImport *i) {
         i->layer_job->on_finished = dkr_import_job_on_finished;
         i->layer_job->on_open_disk = dkr_import_job_on_open_disk;
         i->layer_job->on_progress = dkr_import_job_on_progress;
+        i->layer_job->grow_machine_directory = i->grow_machine_directory;
 
         r = import_job_begin(i->layer_job);
         if (r < 0)
diff --git a/src/import/import-job.c b/src/import/import-job.c
index 809486500..980b639b5 100644
--- a/src/import/import-job.c
+++ b/src/import/import-job.c
@@ -22,8 +22,12 @@
 #include <sys/xattr.h>
 
 #include "strv.h"
+#include "machine-pool.h"
 #include "import-job.h"
 
+/* Grow the /var/lib/machines directory after each 10MiB written */
+#define IMPORT_GROW_INTERVAL_BYTES (UINT64_C(10) * UINT64_C(1024) * UINT64_C(1024))
+
 ImportJob* import_job_unref(ImportJob *j) {
         if (!j)
                 return NULL;
@@ -197,6 +201,11 @@ static int import_job_write_uncompressed(ImportJob *j, void *p, size_t sz) {
 
         if (j->disk_fd >= 0) {
 
+                if (j->grow_machine_directory && j->written_since_last_grow >= IMPORT_GROW_INTERVAL_BYTES) {
+                        j->written_since_last_grow = 0;
+                        grow_machine_directory();
+                }
+
                 if (j->allow_sparse)
                         n = sparse_write(j->disk_fd, p, sz, 64);
                 else
@@ -219,6 +228,7 @@ static int import_job_write_uncompressed(ImportJob *j, void *p, size_t sz) {
         }
 
         j->written_uncompressed += sz;
+        j->written_since_last_grow += sz;
 
         return 0;
 }
@@ -667,6 +677,9 @@ int import_job_begin(ImportJob *j) {
         if (j->state != IMPORT_JOB_INIT)
                 return -EBUSY;
 
+        if (j->grow_machine_directory)
+                grow_machine_directory();
+
         r = curl_glue_make(&j->curl, j->url, j);
         if (r < 0)
                 return r;
diff --git a/src/import/import-job.h b/src/import/import-job.h
index dcf89cb28..2c01d723d 100644
--- a/src/import/import-job.h
+++ b/src/import/import-job.h
@@ -107,6 +107,9 @@ struct ImportJob {
         gcry_md_hd_t checksum_context;
 
         char *checksum;
+
+        bool grow_machine_directory;
+        uint64_t written_since_last_grow;
 };
 
 int import_job_new(ImportJob **job, const char *url, CurlGlue *glue, void *userdata);
diff --git a/src/import/import-raw.c b/src/import/import-raw.c
index 8d99f1085..89c064cb3 100644
--- a/src/import/import-raw.c
+++ b/src/import/import-raw.c
@@ -31,6 +31,7 @@
 #include "util.h"
 #include "macro.h"
 #include "mkdir.h"
+#include "path-util.h"
 #include "import-util.h"
 #include "curl-util.h"
 #include "qcow2-util.h"
@@ -61,6 +62,7 @@ struct RawImport {
 
         char *local;
         bool force_local;
+        bool grow_machine_directory;
 
         char *temp_path;
         char *final_path;
@@ -115,6 +117,8 @@ int raw_import_new(
         if (!i->image_root)
                 return -ENOMEM;
 
+        i->grow_machine_directory = path_startswith(i->image_root, "/var/lib/machines");
+
         if (event)
                 i->event = sd_event_ref(event);
         else {
@@ -480,6 +484,7 @@ int raw_import_pull(RawImport *i, const char *url, const char *local, bool force
         i->raw_job->on_open_disk = raw_import_job_on_open_disk;
         i->raw_job->on_progress = raw_import_job_on_progress;
         i->raw_job->calc_checksum = verify != IMPORT_VERIFY_NO;
+        i->raw_job->grow_machine_directory = i->grow_machine_directory;
 
         r = import_find_old_etags(url, i->image_root, DT_REG, ".raw-", ".raw", &i->raw_job->old_etags);
         if (r < 0)
diff --git a/src/import/import-tar.c b/src/import/import-tar.c
index 493252a13..472e33624 100644
--- a/src/import/import-tar.c
+++ b/src/import/import-tar.c
@@ -30,6 +30,7 @@
 #include "util.h"
 #include "macro.h"
 #include "mkdir.h"
+#include "path-util.h"
 #include "import-util.h"
 #include "curl-util.h"
 #include "import-job.h"
@@ -58,6 +59,7 @@ struct TarImport {
 
         char *local;
         bool force_local;
+        bool grow_machine_directory;
 
         pid_t tar_pid;
 
@@ -121,6 +123,8 @@ int tar_import_new(
         if (!i->image_root)
                 return -ENOMEM;
 
+        i->grow_machine_directory = path_startswith(i->image_root, "/var/lib/machines");
+
         if (event)
                 i->event = sd_event_ref(event);
         else {
@@ -376,6 +380,7 @@ int tar_import_pull(TarImport *i, const char *url, const char *local, bool force
         i->tar_job->on_open_disk = tar_import_job_on_open_disk;
         i->tar_job->on_progress = tar_import_job_on_progress;
         i->tar_job->calc_checksum = verify != IMPORT_VERIFY_NO;
+        i->tar_job->grow_machine_directory = i->grow_machine_directory;
 
         r = import_find_old_etags(url, i->image_root, DT_DIR, ".tar-", NULL, &i->tar_job->old_etags);
         if (r < 0)