journald: when we detect the journal file we are about to write to has been deleted, rotate

https://bugzilla.redhat.com/show_bug.cgi?id=1171719

4 files changed, 57 insertions, 18 deletions

diff --git a/src/journal/journal-file.c b/src/journal/journal-file.c
index 1736ff57a..304ce03bd 100644
--- a/src/journal/journal-file.c
+++ b/src/journal/journal-file.c
@@ -67,6 +67,9 @@
 /* How much to increase the journal file size at once each time we allocate something new. */
 #define FILE_SIZE_INCREASE (8ULL*1024ULL*1024ULL)              /* 8MB */
 
+/* Reread fstat() of the file for detecting deletions at least this often */
+#define LAST_STAT_REFRESH_USEC (5*USEC_PER_SEC)
+
 /* The mmap context to use for the header we pick as one above the last defined typed */
 #define CONTEXT_HEADER _OBJECT_TYPE_MAX
 
@@ -319,6 +322,22 @@ static int journal_file_verify_header(JournalFile *f) {
         return 0;
 }
 
+static int journal_file_fstat(JournalFile *f) {
+        assert(f);
+        assert(f->fd >= 0);
+
+        if (fstat(f->fd, &f->last_stat) < 0)
+                return -errno;
+
+        f->last_stat_usec = now(CLOCK_MONOTONIC);
+
+        /* Refuse appending to files that are already deleted */
+        if (f->last_stat.st_nlink <= 0)
+                return -EIDRM;
+
+        return 0;
+}
+
 static int journal_file_allocate(JournalFile *f, uint64_t offset, uint64_t size) {
         uint64_t old_size, new_size;
         int r;
@@ -340,8 +359,21 @@ static int journal_file_allocate(JournalFile *f, uint64_t offset, uint64_t size)
         if (new_size < le64toh(f->header->header_size))
                 new_size = le64toh(f->header->header_size);
 
-        if (new_size <= old_size)
-                return 0;
+        if (new_size <= old_size) {
+
+                /* We already pre-allocated enough space, but before
+                 * we write to it, let's check with fstat() if the
+                 * file got deleted, in order make sure we don't throw
+                 * away the data immediately. Don't check fstat() for
+                 * all writes though, but only once ever 10s. */
+
+                if (f->last_stat_usec + LAST_STAT_REFRESH_USEC > now(CLOCK_MONOTONIC))
+                        return 0;
+
+                return journal_file_fstat(f);
+        }
+
+        /* Allocate more space. */
 
         if (f->metrics.max_size > 0 && new_size > f->metrics.max_size)
                 return -E2BIG;
@@ -376,12 +408,9 @@ static int journal_file_allocate(JournalFile *f, uint64_t offset, uint64_t size)
         if (r != 0)
                 return -r;
 
-        if (fstat(f->fd, &f->last_stat) < 0)
-                return -errno;
-
         f->header->arena_size = htole64(new_size - le64toh(f->header->header_size));
 
-        return 0;
+        return journal_file_fstat(f);
 }
 
 static unsigned type_to_context(ObjectType type) {
@@ -392,6 +421,8 @@ static unsigned type_to_context(ObjectType type) {
 }
 
 static int journal_file_move_to(JournalFile *f, ObjectType type, bool keep_always, uint64_t offset, uint64_t size, void **ret) {
+        int r;
+
         assert(f);
         assert(ret);
 
@@ -403,8 +434,11 @@ static int journal_file_move_to(JournalFile *f, ObjectType type, bool keep_alway
                 /* Hmm, out of range? Let's refresh the fstat() data
                  * first, before we trust that check. */
 
-                if (fstat(f->fd, &f->last_stat) < 0 ||
-                    offset + size > (uint64_t) f->last_stat.st_size)
+                r = journal_file_fstat(f);
+                if (r < 0)
+                        return r;
+
+                if (offset + size > (uint64_t) f->last_stat.st_size)
                         return -EADDRNOTAVAIL;
         }
 
@@ -2548,10 +2582,9 @@ int journal_file_open(
                 goto fail;
         }
 
-        if (fstat(f->fd, &f->last_stat) < 0) {
-                r = -errno;
+        r = journal_file_fstat(f);
+        if (r < 0)
                 goto fail;
-        }
 
         if (f->last_stat.st_size == 0 && f->writable) {
                 /* Let's attach the creation time to the journal file,
@@ -2580,10 +2613,9 @@ int journal_file_open(
                 if (r < 0)
                         goto fail;
 
-                if (fstat(f->fd, &f->last_stat) < 0) {
-                        r = -errno;
+                r = journal_file_fstat(f);
+                if (r < 0)
                         goto fail;
-                }
 
                 newly_created = true;
         }
@@ -2700,8 +2732,11 @@ int journal_file_rotate(JournalFile **f, bool compress, bool seal) {
         if (r < 0)
                 return -ENOMEM;
 
+        /* Try to rename the file to the archived version. If the file
+         * already was deleted, we'll get ENOENT, let's ignore that
+         * case. */
         r = rename(old_file->path, p);
-        if (r < 0)
+        if (r < 0 && errno != ENOENT)
                 return -errno;
 
         old_file->header->state = STATE_ARCHIVED;
diff --git a/src/journal/journal-file.h b/src/journal/journal-file.h
index 19fd7257f..b3a0679b9 100644
--- a/src/journal/journal-file.h
+++ b/src/journal/journal-file.h
@@ -82,6 +82,7 @@ typedef struct JournalFile {
 
         char *path;
         struct stat last_stat;
+        usec_t last_stat_usec;
 
         Header *header;
         HashItem *data_hash_table;
diff --git a/src/journal/journald-server.c b/src/journal/journald-server.c
index 019c3a649..c28aba8fd 100644
--- a/src/journal/journald-server.c
+++ b/src/journal/journald-server.c
@@ -318,6 +318,7 @@ static int do_rotate(
                         log_error_errno(r, "Failed to create new %s journal: %m", name);
         else
                 server_fix_perms(s, *f, uid);
+
         return r;
 }
 
@@ -466,7 +467,8 @@ static bool shall_try_append_again(JournalFile *f, int r) {
            -EPROTONOSUPPORT  Unsupported feature
            -EBADMSG          Corrupted
            -ENODATA          Truncated
-           -ESHUTDOWN        Already archived */
+           -ESHUTDOWN        Already archived
+           -EIDRM            Journal file has been deleted */
 
         if (r == -E2BIG || r == -EFBIG || r == -EDQUOT || r == -ENOSPC)
                 log_debug("%s: Allocation limit reached, rotating.", f->path);
@@ -480,6 +482,8 @@ static bool shall_try_append_again(JournalFile *f, int r) {
                 log_warning("%s: Journal file corrupted, rotating.", f->path);
         else if (r == -EIO)
                 log_warning("%s: IO error, rotating.", f->path);
+        else if (r == -EIDRM)
+                log_warning("%s: Journal file has been deleted, rotating.", f->path);
         else
                 return false;
 
diff --git a/src/journal/test-journal-flush.c b/src/journal/test-journal-flush.c
index 0ca24e0cb..40ede4a92 100644
--- a/src/journal/test-journal-flush.c
+++ b/src/journal/test-journal-flush.c
@@ -39,8 +39,6 @@ int main(int argc, char *argv[]) {
         r = journal_file_open(fn, O_CREAT|O_RDWR, 0644, false, false, NULL, NULL, NULL, &new_journal);
         assert_se(r >= 0);
 
-        unlink(fn);
-
         r = sd_journal_open(&j, 0);
         assert_se(r >= 0);
 
@@ -68,6 +66,7 @@ int main(int argc, char *argv[]) {
 
         journal_file_close(new_journal);
 
+        unlink(fn);
         assert_se(rmdir(dn) == 0);
 
         return 0;