create placeholders for messages over MaxSize

this is vastly more useful than just omitting the messages with no
indication at all.

NEWS

@@ -15,6 +15,8 @@ The IMAP user query can be scripted now.
 
 Added built-in support for macOS Keychain.
 
+Messages excluded by MaxSize will now result in placeholders.
+
 The use of Master/Slave terminology has been deprecated.
 
 [1.3.0]

TODO

@@ -61,9 +61,8 @@ messages.
 
 use MULTIAPPEND and FETCH with multiple messages.
 
-create dummies describing MIME structure of messages bigger than MaxSize.
-flagging the dummy would fetch the real message. possibly remove --renew.
-note that all interaction needs to happen on the near side probably.
+dummy messages resulting from MaxSize should contain a dump of the original
+message's MIME structure and its (reasonably sized) text parts.
 
 don't SELECT boxes unless really needed; in particular not for appending,
 and in write-only mode not before changes are made.

src/driver.h

@@ -80,7 +80,6 @@ typedef struct message {
 #define OPEN_OLD        (1<<0)  // Paired messages *in* this store.
 #define OPEN_NEW        (1<<1)  // Messages (possibly) not yet propagated *from* this store.
 #define OPEN_FLAGS      (1<<2)  // Note that fetch_msg() gets the flags regardless.
-#define OPEN_OLD_SIZE   (1<<3)
 #define OPEN_NEW_SIZE   (1<<4)
 #define OPEN_EXPUNGE    (1<<5)
 #define OPEN_SETFLAGS   (1<<6)
@@ -217,8 +216,10 @@ struct driver {
 	void (*load_box)( store_t *ctx, uint minuid, uint maxuid, uint finduid, uint pairuid, uint newuid, uint_array_t excs,
 	                  void (*cb)( int sts, message_t *msgs, int total_msgs, int recent_msgs, void *aux ), void *aux );
 
-	/* Fetch the contents and flags of the given message from the current mailbox. */
-	void (*fetch_msg)( store_t *ctx, message_t *msg, msg_data_t *data,
+	/* Fetch the contents and flags of the given message from the current mailbox.
+	 * If minimal is non-zero, fetch only a placeholder for the requested message -
+	 * ideally, this is precisely the header, but it may be more. */
+	void (*fetch_msg)( store_t *ctx, message_t *msg, msg_data_t *data, int minimal,
 	                   void (*cb)( int sts, void *aux ), void *aux );
 
 	/* Store the given message to either the current mailbox or the trash folder.

src/drv_imap.c

@@ -1098,7 +1098,7 @@ parse_fetch_rsp( imap_store_t *ctx, list_t *list, char *s ATTR_UNUSED )
 				error( "IMAP error: unable to parse RFC822.SIZE\n" );
 				goto ffail;
 			}
-		} else if (!strcmp( "BODY[]", name )) {
+		} else if (!strcmp( "BODY[]", name ) || !strcmp( "BODY[HEADER]", name )) {
 			if (!is_atom( tmp )) {
 				error( "IMAP error: unable to parse BODY[]\n" );
 				goto ffail;
@@ -2714,9 +2714,8 @@ imap_load_box( store_t *gctx, uint minuid, uint maxuid, uint finduid, uint pairu
 			ranges[0].last = maxuid;
 			ranges[0].flags = 0;
 			uint nranges = 1;
-			if (ctx->opts & (OPEN_OLD_SIZE | OPEN_NEW_SIZE))
-				imap_set_range( ranges, &nranges, shifted_bit( ctx->opts, OPEN_OLD_SIZE, WantSize),
-				                                  shifted_bit( ctx->opts, OPEN_NEW_SIZE, WantSize), newuid );
+			if (ctx->opts & OPEN_NEW_SIZE)
+				imap_set_range( ranges, &nranges, 0, WantSize, newuid );
 			if (ctx->opts & OPEN_FIND)
 				imap_set_range( ranges, &nranges, 0, WantTuids, finduid - 1 );
 			if (ctx->opts & OPEN_OLD_IDS)
@@ -2811,7 +2810,7 @@ imap_submit_load_p3( imap_store_t *ctx, imap_load_box_state_t *sts )
 static void imap_fetch_msg_p2( imap_store_t *, imap_cmd_t *, int );
 
 static void
-imap_fetch_msg( store_t *ctx, message_t *msg, msg_data_t *data,
+imap_fetch_msg( store_t *ctx, message_t *msg, msg_data_t *data, int minimal,
                 void (*cb)( int sts, void *aux ), void *aux )
 {
 	imap_cmd_fetch_msg_t *cmd;
@@ -2821,9 +2820,10 @@ imap_fetch_msg( store_t *ctx, message_t *msg, msg_data_t *data,
 	cmd->msg_data = data;
 	data->data = NULL;
 	imap_exec( (imap_store_t *)ctx, &cmd->gen.gen, imap_fetch_msg_p2,
-	           "UID FETCH %u (%s%sBODY.PEEK[])", msg->uid,
+	           "UID FETCH %u (%s%sBODY.PEEK[%s])", msg->uid,
 	           !(msg->status & M_FLAGS) ? "FLAGS " : "",
-	           (data->date== -1) ? "INTERNALDATE " : "" );
+	           (data->date== -1) ? "INTERNALDATE " : "",
+	           minimal ? "HEADER" : "" );
 }
 
 static void

src/drv_maildir.c

@@ -1139,7 +1139,7 @@ maildir_scan( maildir_store_t *ctx, msg_t_array_alloc_t *msglist )
 				free( entry->base );
 				entry->base = nfstrndup( buf + bl + 4, (size_t)fnl );
 			}
-			int want_size = (uid > ctx->newuid) ? (ctx->opts & OPEN_NEW_SIZE) : (ctx->opts & OPEN_OLD_SIZE);
+			int want_size = ((ctx->opts & OPEN_NEW_SIZE) && uid > ctx->newuid);
 			int want_tuid = ((ctx->opts & OPEN_FIND) && uid >= ctx->finduid);
 			int want_msgid = ((ctx->opts & OPEN_OLD_IDS) && uid <= ctx->pairuid);
 			if (!want_size && !want_tuid && !want_msgid)
@@ -1533,7 +1533,7 @@ maildir_again( maildir_store_t *ctx, maildir_message_t *msg, const char *err, ..
 }
 
 static void
-maildir_fetch_msg( store_t *gctx, message_t *gmsg, msg_data_t *data,
+maildir_fetch_msg( store_t *gctx, message_t *gmsg, msg_data_t *data, int minimal ATTR_UNUSED,
                    void (*cb)( int sts, void *aux ), void *aux )
 {
 	maildir_store_t *ctx = (maildir_store_t *)gctx;

src/mbsync.1

@@ -163,8 +163,16 @@ directory.
 .
 .TP
 \fBMaxSize\fR \fIsize\fR[\fBk\fR|\fBm\fR][\fBb\fR]
-Messages larger than that will not be propagated into this Store.
-This is useful for weeding out messages with large attachments.
+Messages larger than \fIsize\fR will have only a small placeholder message
+propagated into this Store. To propagate the full message, it must be
+flagged in either Store; that can be done retroactively, in which case
+the \fBReNew\fR operation needs to be executed instead of \fBNew\fR.
+This is useful for avoiding downloading messages with large attachments
+unless they are actually needed.
+Caveat: Setting a size limit on a Store you never read directly (which is
+typically the case for servers) is not recommended, as you may never
+notice that affected messages were not propagated to it.
+.br
 \fBK\fR and \fBM\fR can be appended to the size to specify KiBytes resp.
 MeBytes instead of bytes. \fBB\fR is accepted but superfluous.
 If \fIsize\fR is 0, the maximum message size is \fBunlimited\fR.
@@ -574,9 +582,8 @@ Select the synchronization operation(s) to perform:
 .br
 \fBNew\fR - propagate newly appeared messages.
 .br
-\fBReNew\fR - previously refused messages are re-evaluated for propagation.
-Useful after flagging affected messages in the source Store or enlarging
-MaxSize in the destination Store.
+\fBReNew\fR - upgrade placeholders to full messages. Useful only with
+a configured \fBMaxSize\fR.
 .br
 \fBDelete\fR - propagate message deletions. This applies only to messages that
 are actually gone, i.e., were expunged. The affected messages in the remote

src/run-tests.pl

@@ -161,29 +161,75 @@ my @x10 = (
     ],
 );
 
-my @O11 = ("MaxSize 1k\n", "MaxSize 1k\n", "");
+my @O11 = ("MaxSize 1k\n", "MaxSize 1k\n", "Expunge Near");
 #show("10", "11", "11");
 my @X11 = (
- [ 2,
-   A, 1, "", B, 2, "*" ],
- [ 2,
-   C, 1, "*", A, 2, "" ],
- [ 2, 0, 2,
-   0, 1, "^", 1, 2, "", 2, 0, "^" ],
+ [ 3,
+   A, 1, "", B, 2, "*", C, 3, "?" ],
+ [ 3,
+   C, 1, "*", A, 2, "", B, 3, "?" ],
+ [ 3, 0, 3,
+   3, 1, "<", 1, 2, "", 2, 3, ">" ],
 );
 test("max size", \@x10, \@X11, @O11);
 
-my @O22 = ("", "MaxSize 1k\n", "");
-#show("11", "22", "22");
-my @X22 = (
+my @x22 = (
  [ 3,
-   A, 1, "", B, 2, "*", C, 3, "*" ],
+   A, 1, "", B, 2, "*", C, 3, "?" ],
+ [ 3,
+   C, 1, "F*", A, 2, "", B, 3, "F?" ],
+ [ 3, 0, 3,
+   3, 1, "<", 1, 2, "", 2, 3, ">" ],
+);
+
+#show("22", "22", "11");
+my @X22 = (
+ [ 4,
+   A, 1, "", B, 2, "*", C, 3, "T?", C, 4, "F*" ],
+ [ 4,
+   C, 1, "F*", A, 2, "", B, 4, "*" ],
+ [ 4, 0, 4,
+   4, 1, "F", 3, 0, "T", 1, 2, "", 2, 4, "" ],
+);
+test("max size + flagging", \@x22, \@X22, @O11);
+
+my @x23 = (
  [ 2,
-   C, 1, "*", A, 2, "" ],
- [ 3, 0, 2,
-   3, 1, "", 1, 2, "", 2, 0, "^" ],
+   A, 1, "", B, 2, "F*" ],
+ [ 1,
+   C, 1, "F*" ],
+ [ 0, 0, 0,
+    ],
+);
+
+my @X23 = (
+ [ 3,
+   A, 1, "", B, 2, "F*", C, 3, "F*" ],
+ [ 3,
+   C, 1, "F*", A, 2, "", B, 3, "F*" ],
+ [ 3, 0, 3,
+   3, 1, "F", 1, 2, "", 2, 3, "F" ]
+);
+test("max size + initial flagging", \@x23, \@X23, @O11);
+
+my @x24 = (
+ [ 3,
+   A, 1, "", B, 2, "*", C, 3, "F*" ],
+ [ 1,
+   A, 1, "" ],
+ [ 3, 0, 1,
+   1, 1, "", 2, 0, "^", 3, 0, "^" ],
+);
+
+my @X24 = (
+ [ 3,
+   A, 1, "", B, 2, "*", C, 3, "F*" ],
+ [ 3,
+   A, 1, "", B, 2, "?", C, 3, "F*" ],
+ [ 3, 0, 3,
+   1, 1, "", 2, 2, ">", 3, 3, "F" ],
 );
-test("near side max size", \@X11, \@X22, @O22);
+test("max size (pre-1.4 legacy)", \@x24, \@X24, @O11);
 
 # expiration tests
 
@@ -329,7 +375,7 @@ sub readbox($)
 	for my $d ("cur", "new") {
 		opendir(DIR, $bn."/".$d) or next;
 		for my $f (grep(!/^\.\.?$/, readdir(DIR))) {
-			my ($uid, $flg, $num);
+			my ($uid, $flg, $ph, $num);
 			if ($f =~ /^\d+\.\d+_\d+\.[-[:alnum:]]+,U=(\d+):2,(.*)$/) {
 				($uid, $flg) = ($1, $2);
 			} else {
@@ -339,7 +385,7 @@ sub readbox($)
 			open(FILE, "<", $bn."/".$d."/".$f) or die "Cannot read message '$f' in '$bn'.\n";
 			my $sz = 0;
 			while (<FILE>) {
-				/^Subject: (\d+)$/ && ($num = $1);
+				/^Subject: (\[placeholder\] )?(\d+)$/ && ($ph = defined($1), $num = $2);
 				$sz += length($_);
 			}
 			close FILE;
@@ -347,7 +393,7 @@ sub readbox($)
 				print STDERR "message '$f' in '$bn' has no identifier.\n";
 				exit 1;
 			}
-			@{ $ms{$uid} } = ($num, $flg.($sz>1000?"*":""));
+			@{ $ms{$uid} } = ($num, $flg.($sz>1000?"*":"").($ph?"?":""));
 		}
 	}
 	return ($mu, %ms);
@@ -455,9 +501,10 @@ sub mkbox($$@)
 	while (@ms) {
 		my ($num, $uid, $flg) = (shift @ms, shift @ms, shift @ms);
 		my $big = $flg =~ s/\*//;
+		my $ph = $flg =~ s/\?//;
 		open(FILE, ">", $bn."/".($flg =~ /S/ ? "cur" : "new")."/0.1_".$num.".local,U=".$uid.":2,".$flg) or
 			die "Cannot create message ".mn($num)." in mailbox $bn.\n";
-		print FILE "From: foo\nTo: bar\nDate: Thu, 1 Jan 1970 00:00:00 +0000\nSubject: $num\n\n".(("A"x50)."\n")x($big*30);
+		print FILE "From: foo\nTo: bar\nDate: Thu, 1 Jan 1970 00:00:00 +0000\nSubject: ".($ph?"[placeholder] ":"").$num."\n\n".(("A"x50)."\n")x($big*30);
 		close FILE;
 	}
 }

src/sync.c

@@ -136,20 +136,23 @@ make_flags( uchar flags, char *buf )
 #define S_EXPIRE       (1<<0)  // the entry is being expired (near side message removal scheduled)
 #define S_EXPIRED      (1<<1)  // the entry is expired (near side message removal confirmed)
 #define S_PENDING      (1<<2)  // the entry is new and awaits propagation (possibly a retry)
-#define S_SKIPPED      (1<<3)  // the entry was not propagated (message is too big)
+#define S_DUMMY(fn)    (1<<(3+(fn)))  // f/n message is only a placeholder
+#define S_SKIPPED      (1<<5)  // pre-1.4 legacy: the entry was not propagated (message is too big)
 #define S_DEAD         (1<<7)  // ephemeral: the entry was killed and should be ignored
 
 // Ephemeral working set.
 #define W_NEXPIRE      (1<<0)  // temporary: new expiration state
 #define W_DELETE       (1<<1)  // ephemeral: flags propagation is a deletion
 #define W_DEL(fn)      (1<<(2+(fn)))  // ephemeral: f/n message would be subject to expunge
+#define W_UPGRADE      (1<<4)  // ephemeral: upgrading placeholder, do not apply MaxSize
+#define W_PURGE        (1<<5)  // ephemeral: placeholder is being nuked
 
 typedef struct sync_rec {
 	struct sync_rec *next;
 	/* string_list_t *keywords; */
 	uint uid[2];
 	message_t *msg[2];
-	uchar status, wstate, flags, aflags[2], dflags[2];
+	uchar status, wstate, flags, pflags, aflags[2], dflags[2];
 	char tuid[TUIDL];
 } sync_rec_t;
 
@@ -274,6 +277,7 @@ assign_uid( sync_vars_t *svars, sync_rec_t *srec, int t, uint uid )
 	if (uid == svars->maxuid[t] + 1)
 		svars->maxuid[t] = uid;
 	srec->status &= ~S_PENDING;
+	srec->wstate &= ~W_UPGRADE;
 	srec->tuid[0] = 0;
 }
 
@@ -353,6 +357,7 @@ typedef struct copy_vars {
 	sync_rec_t *srec; /* also ->tuid */
 	message_t *msg;
 	msg_data_t data;
+	int minimal;
 } copy_vars_t;
 
 static void msg_fetched( int sts, void *aux );
@@ -365,7 +370,7 @@ copy_msg( copy_vars_t *vars )
 	t ^= 1;
 	vars->data.flags = vars->msg->flags;
 	vars->data.date = svars->chan->use_internal_date ? -1 : 0;
-	svars->drv[t]->fetch_msg( svars->ctx[t], vars->msg, &vars->data, msg_fetched, vars );
+	svars->drv[t]->fetch_msg( svars->ctx[t], vars->msg, &vars->data, vars->minimal, msg_fetched, vars );
 }
 
 static void msg_stored( int sts, uint uid, void *aux );
@@ -405,8 +410,10 @@ copy_msg_convert( int in_cr, int out_cr, copy_vars_t *vars )
 {
 	char *in_buf = vars->data.data;
 	uint in_len = vars->data.len;
-	uint idx = 0, sbreak = 0, ebreak = 0;
+	uint idx = 0, sbreak = 0, ebreak = 0, break2 = 0;
 	uint lines = 0, hdr_crs = 0, bdy_crs = 0, app_cr = 0, extra = 0;
+	uint add_subj = 0;
+
 	if (vars->srec) {
 	  nloop: ;
 		uint start = idx;
@@ -416,14 +423,29 @@ copy_msg_convert( int in_cr, int out_cr, copy_vars_t *vars )
 			if (c == '\r') {
 				line_crs++;
 			} else if (c == '\n') {
-				if (starts_with_upper( in_buf + start, (int)(in_len - start), "X-TUID: ", 8 )) {
+				if (!ebreak && starts_with_upper( in_buf + start, (int)(in_len - start), "X-TUID: ", 8 )) {
 					extra = (sbreak = start) - (ebreak = idx);
-					goto oke;
+					if (!vars->minimal)
+						goto oke;
+				} else {
+					if (!break2 && vars->minimal && !strncasecmp( in_buf + start, "Subject:", 8 )) {
+						break2 = start + 8;
+						if (in_buf[break2] == ' ')
+							break2++;
+					}
+					lines++;
+					hdr_crs += line_crs;
 				}
-				lines++;
-				hdr_crs += line_crs;
 				if (idx - line_crs - 1 == start) {
-					sbreak = ebreak = start;
+					if (!ebreak)
+						sbreak = ebreak = start;
+					if (vars->minimal) {
+						in_len = idx;
+						if (!break2) {
+							break2 = start;
+							add_subj = 1;
+						}
+					}
 					goto oke;
 				}
 				goto nloop;
@@ -449,10 +471,36 @@ copy_msg_convert( int in_cr, int out_cr, copy_vars_t *vars )
 			extra += lines;
 	}
 
+	uint dummy_msg_len = 0;
+	char dummy_msg_buf[180];
+	static const char dummy_pfx[] = "[placeholder] ";
+	static const char dummy_subj[] = "Subject: [placeholder] (No Subject)";
+	static const char dummy_msg[] =
+		"Having a size of %s, this message is over the MaxSize limit.%s"
+		"Flag it and sync again (Sync mode ReNew) to fetch its real contents.%s";
+
+	if (vars->minimal) {
+		char sz[32];
+
+		if (vars->msg->size < 1024000)
+			sprintf( sz, "%dKiB", (int)(vars->msg->size >> 10) );
+		else
+			sprintf( sz, "%.1fMiB", vars->msg->size / 1048576. );
+		const char *nl = app_cr ? "\r\n" : "\n";
+		dummy_msg_len = (uint)sprintf( dummy_msg_buf, dummy_msg, sz, nl, nl );
+		extra += dummy_msg_len;
+		extra += add_subj ? strlen(dummy_subj) + app_cr + 1 : strlen(dummy_pfx);
+	}
+
 	vars->data.len = in_len + extra;
 	char *out_buf = vars->data.data = nfmalloc( vars->data.len );
 	idx = 0;
 	if (vars->srec) {
+		if (break2 && break2 < sbreak) {
+			copy_msg_bytes( &out_buf, in_buf, &idx, break2, in_cr, out_cr );
+			memcpy( out_buf, dummy_pfx, strlen(dummy_pfx) );
+			out_buf += strlen(dummy_pfx);
+		}
 		copy_msg_bytes( &out_buf, in_buf, &idx, sbreak, in_cr, out_cr );
 
 		memcpy( out_buf, "X-TUID: ", 8 );
@@ -463,9 +511,26 @@ copy_msg_convert( int in_cr, int out_cr, copy_vars_t *vars )
 			*out_buf++ = '\r';
 		*out_buf++ = '\n';
 		idx = ebreak;
+
+		if (break2 >= sbreak) {
+			copy_msg_bytes( &out_buf, in_buf, &idx, break2, in_cr, out_cr );
+			if (!add_subj) {
+				memcpy( out_buf, dummy_pfx, strlen(dummy_pfx) );
+				out_buf += strlen(dummy_pfx);
+			} else {
+				memcpy( out_buf, dummy_subj, strlen(dummy_subj) );
+				out_buf += strlen(dummy_subj);
+				if (app_cr)
+					*out_buf++ = '\r';
+				*out_buf++ = '\n';
+			}
+		}
 	}
 	copy_msg_bytes( &out_buf, in_buf, &idx, in_len, in_cr, out_cr );
 
+	if (vars->minimal)
+		memcpy( out_buf, dummy_msg_buf, dummy_msg_len );
+
 	free( in_buf );
 	return 1;
 }
@@ -648,6 +713,33 @@ clean_strdup( const char *s )
 }
 
 
+static sync_rec_t *
+upgrade_srec( sync_vars_t *svars, sync_rec_t *srec )
+{
+	// Create an entry and append it to the current one.
+	sync_rec_t *nsrec = nfcalloc( sizeof(*nsrec) );
+	nsrec->next = srec->next;
+	srec->next = nsrec;
+	if (svars->srecadd == &srec->next)
+		svars->srecadd = &nsrec->next;
+	// Move the placeholder to the new entry.
+	int t = (srec->status & S_DUMMY(F)) ? F : N;
+	nsrec->uid[t] = srec->uid[t];
+	srec->uid[t] = 0;
+	if (srec->msg[t]) {  // NULL during journal replay; is assigned later.
+		nsrec->msg[t] = srec->msg[t];
+		nsrec->msg[t]->srec = nsrec;
+		srec->msg[t] = NULL;
+	}
+	// Mark the original entry for upgrade.
+	srec->status = (srec->status & ~(S_DUMMY(F)|S_DUMMY(N))) | S_PENDING;
+	srec->wstate |= W_UPGRADE;
+	// Mark the placeholder for nuking.
+	nsrec->wstate = W_PURGE;
+	nsrec->aflags[t] = F_DELETED;
+	return nsrec;
+}
+
 static int
 prepare_state( sync_vars_t *svars )
 {
@@ -741,7 +833,8 @@ save_state( sync_vars_t *svars )
 		if (srec->status & S_DEAD)
 			continue;
 		make_flags( srec->flags, fbuf );
-		Fprintf( svars->nfp, "%u %u %s%s\n", srec->uid[F], srec->uid[N],
+		Fprintf( svars->nfp, "%u %u %s%s%s\n", srec->uid[F], srec->uid[N],
+		         (srec->status & S_DUMMY(F)) ? "<" : (srec->status & S_DUMMY(N)) ? ">" : "",
 		         (srec->status & S_SKIPPED) ? "^" : (srec->status & S_EXPIRED) ? "~" : "", fbuf );
 	}
 
@@ -836,7 +929,14 @@ load_state( sync_vars_t *svars )
 			srec->uid[F] = t1;
 			srec->uid[N] = t2;
 			s = fbuf;
-			if (*s == '^') {
+			if (*s == '<') {
+				s++;
+				srec->status = S_DUMMY(F);
+			} else if (*s == '>') {
+				s++;
+				srec->status = S_DUMMY(N);
+			}
+			if (*s == '^') {  // Pre-1.4 legacy
 				s++;
 				srec->status = S_SKIPPED;
 			} else if (*s == '~' || *s == 'X' /* Pre-1.3 legacy */) {
@@ -850,8 +950,9 @@ load_state( sync_vars_t *svars )
 				srec->status = S_SKIPPED;
 			}
 			srec->flags = parse_flags( s );
-			debug( "  entry (%u,%u,%u,%s)\n", srec->uid[F], srec->uid[N], srec->flags,
-			       (srec->status & S_SKIPPED) ? "SKIP" : (srec->status & S_EXPIRED) ? "XPIRE" : "" );
+			debug( "  entry (%u,%u,%u,%s%s)\n", srec->uid[F], srec->uid[N], srec->flags,
+			       (srec->status & S_SKIPPED) ? "SKIP" : (srec->status & S_EXPIRED) ? "XPIRE" : "",
+			       (srec->status & S_DUMMY(F)) ? ",F-DUMMY" : (srec->status & S_DUMMY(N)) ? ",N-DUMMY" : "" );
 			*svars->srecadd = srec;
 			svars->srecadd = &srec->next;
 			svars->nsrecs++;
@@ -919,14 +1020,16 @@ load_state( sync_vars_t *svars )
 				}
 				buf[ll] = 0;
 				int tn;
-				uint t1, t2, t3;
+				uint t1, t2, t3, t4;
 				if ((c = buf[0]) == '#' ?
 				      (tn = 0, (sscanf( buf + 2, "%u %u %n", &t1, &t2, &tn ) < 2) || !tn || (ll - (uint)tn != TUIDL + 2)) :
 				      c == '!' ?
 				        (sscanf( buf + 2, "%u", &t1 ) != 1) :
-				        c == 'N' || c == 'F' || c == 'T' || c == '+' || c == '&' || c == '-' || c == '=' || c == '|' ?
+				        c == 'N' || c == 'F' || c == 'T' || c == '+' || c == '&' || c == '-' || c == '=' || c == '_' || c == '|' ?
 				          (sscanf( buf + 2, "%u %u", &t1, &t2 ) != 2) :
-				          (sscanf( buf + 2, "%u %u %u", &t1, &t2, &t3 ) != 3))
+				          c != '^' ?
+				            (sscanf( buf + 2, "%u %u %u", &t1, &t2, &t3 ) != 3) :
+				            (sscanf( buf + 2, "%u %u %u %u", &t1, &t2, &t3, &t4 ) != 4))
 				{
 					error( "Error: malformed journal entry at %s:%d\n", svars->jname, line );
 					goto jbail;
@@ -992,11 +1095,24 @@ load_state( sync_vars_t *svars )
 					case '*':
 						debug( "flags now %u\n", t3 );
 						srec->flags = (uchar)t3;
+						srec->aflags[F] = srec->aflags[N] = 0;
+						srec->wstate &= ~W_PURGE;
 						break;
 					case '~':
 						debug( "status now %#x\n", t3 );
 						srec->status = (uchar)t3;
 						break;
+					case '_':
+						debug( "has placeholder now\n" );
+						srec->status = S_PENDING;  // Pre-1.4 legacy only
+						srec->status |= !srec->uid[F] ? S_DUMMY(F) : S_DUMMY(N);
+						break;
+					case '^':
+						debug( "is being upgraded, flags %u, srec flags %u\n", t3, t4 );
+						srec->pflags = (uchar)t3;
+						srec->flags = (uchar)t4;
+						srec = upgrade_srec( svars, srec );
+						break;
 					default:
 						error( "Error: unrecognized journal entry at %s:%d\n", svars->jname, line );
 						goto jbail;
@@ -1275,18 +1391,17 @@ box_opened2( sync_vars_t *svars, int t )
 		}
 		if (chan->ops[t] & (OP_NEW|OP_RENEW)) {
 			opts[t] |= OPEN_APPEND;
-			if (chan->ops[t] & OP_RENEW)
-				opts[1-t] |= OPEN_OLD;
-			if (chan->ops[t] & OP_NEW)
+			if (chan->ops[t] & OP_NEW) {
 				opts[1-t] |= OPEN_NEW;
-			if (chan->ops[t] & OP_EXPUNGE)  // Don't propagate doomed msgs
-				opts[1-t] |= OPEN_FLAGS;
-			if (chan->stores[t]->max_size != UINT_MAX) {
-				if (chan->ops[t] & OP_RENEW)
-					opts[1-t] |= OPEN_FLAGS|OPEN_OLD_SIZE;
-				if (chan->ops[t] & OP_NEW)
+				if (chan->stores[t]->max_size != UINT_MAX)
 					opts[1-t] |= OPEN_FLAGS|OPEN_NEW_SIZE;
 			}
+			if (chan->ops[t] & OP_RENEW) {
+				opts[t] |= OPEN_OLD|OPEN_FLAGS|OPEN_SETFLAGS;
+				opts[1-t] |= OPEN_OLD|OPEN_FLAGS;
+			}
+			if (chan->ops[t] & OP_EXPUNGE)  // Don't propagate doomed msgs
+				opts[1-t] |= OPEN_FLAGS;
 		}
 		if (chan->ops[t] & OP_EXPUNGE) {
 			opts[t] |= OPEN_EXPUNGE;
@@ -1312,6 +1427,15 @@ box_opened2( sync_vars_t *svars, int t )
 				else
 					warn( "Warning: sync record (%u,%u) has stray TUID. Ignoring.\n", srec->uid[F], srec->uid[N] );
 			}
+			if (srec->wstate & W_PURGE) {
+				t = srec->uid[F] ? F : N;
+				opts[t] |= OPEN_SETFLAGS;
+			}
+			if (srec->wstate & W_UPGRADE) {
+				t = !srec->uid[F] ? F : N;
+				opts[t] |= OPEN_APPEND;
+				opts[1-t] |= OPEN_OLD;
+			}
 		}
 	svars->opts[F] = svars->drv[F]->prepare_load_box( ctx[F], opts[F] );
 	svars->opts[N] = svars->drv[N]->prepare_load_box( ctx[N], opts[N] );
@@ -1583,6 +1707,12 @@ box_loaded( int sts, message_t *msgs, int total_msgs, int recent_msgs, void *aux
 							debug( "  near side expiring\n" );
 							sflags &= ~F_DELETED;
 						}
+						if (srec->status & S_DUMMY(1-t)) {
+							// For placeholders, don't propagate:
+							// - Seen, because the real contents were obviously not seen yet
+							// - Flagged, because it's just a request to upgrade
+							sflags &= ~(F_SEEN|F_FLAGGED);
+						}
 						srec->aflags[t] = sflags & ~srec->flags;
 						srec->dflags[t] = ~sflags & srec->flags;
 						if ((DFlags & DEBUG_SYNC) && (srec->aflags[t] || srec->dflags[t])) {
@@ -1594,6 +1724,41 @@ box_loaded( int sts, message_t *msgs, int total_msgs, int recent_msgs, void *aux
 					}
 				}
 			}
+
+			sync_rec_t *nsrec = srec;
+			if (((srec->status & S_DUMMY(F)) && (svars->chan->ops[F] & OP_RENEW)) ||
+			     ((srec->status & S_DUMMY(N)) && (svars->chan->ops[N] & OP_RENEW))) {
+				// Flagging the message on either side causes an upgrade of the dummy.
+				// We ignore flag resets, because that corner case is not worth it.
+				ushort muflags = srec->msg[F] ? srec->msg[F]->flags : 0;
+				ushort suflags = srec->msg[N] ? srec->msg[N]->flags : 0;
+				if ((muflags | suflags) & F_FLAGGED) {
+					t = (srec->status & S_DUMMY(F)) ? F : N;
+					// We calculate the flags for the replicated message already now,
+					// because after an interruption the dummy may be already gone.
+					srec->pflags = ((srec->msg[t]->flags & ~(F_SEEN|F_FLAGGED)) | srec->aflags[t]) & ~srec->dflags[t];
+					// Consequently, the srec's flags are committed right away as well.
+					srec->flags = (srec->flags | srec->aflags[t]) & ~srec->dflags[t];
+					JLOG( "^ %u %u %u %u", (srec->uid[F], srec->uid[N], srec->pflags, srec->flags), "upgrading placeholder" );
+					nsrec = upgrade_srec( svars, srec );
+				}
+			}
+			// This is separated, because the upgrade can come from the journal.
+			if (srec->wstate & W_UPGRADE) {
+				t = !srec->uid[F] ? F : N;
+				tmsg = srec->msg[1-t];
+				if ((svars->chan->ops[t] & OP_EXPUNGE) && (srec->pflags & F_DELETED)) {
+					JLOG( "- %u %u", (srec->uid[F], srec->uid[N]), "killing upgrade - would be expunged anyway" );
+					tmsg->srec = NULL;
+					srec->status = S_DEAD;
+				} else {
+					// Pretend that the source message has the adjusted flags of the dummy.
+					tmsg->flags = srec->pflags;
+					tmsg->status = M_FLAGS;
+					any_new[t] = 1;
+				}
+			}
+			srec = nsrec;
 		}
 	}
 
@@ -1603,12 +1768,20 @@ box_loaded( int sts, message_t *msgs, int total_msgs, int recent_msgs, void *aux
 			srec = tmsg->srec;
 			if (srec) {
 				if (srec->status & S_SKIPPED) {
-					// The message was skipped due to being too big.
+					// Pre-1.4 legacy only: The message was skipped due to being too big.
 					// We must have already seen the UID, but we might have been interrupted.
 					if (svars->maxuid[1-t] < tmsg->uid)
 						svars->maxuid[1-t] = tmsg->uid;
 					if (!(svars->chan->ops[t] & OP_RENEW))
 						continue;
+					srec->status = S_PENDING;
+					// The message size was not queried, so this won't be dummified below.
+					if (!(tmsg->flags & F_FLAGGED)) {
+						srec->status |= S_DUMMY(t);
+						JLOG( "_ %u %u", (srec->uid[F], srec->uid[N]), "placeholder only - was previously skipped" );
+					} else {
+						JLOG( "~ %u %u %u", (srec->uid[F], srec->uid[N], srec->status), "was previously skipped" );
+					}
 				} else {
 					if (!(svars->chan->ops[t] & OP_NEW))
 						continue;
@@ -1623,8 +1796,8 @@ box_loaded( int sts, message_t *msgs, int total_msgs, int recent_msgs, void *aux
 					if (!(srec->status & S_PENDING))
 						continue;  // Nothing to do - the message is paired or expired
 					// Propagation was scheduled, but we got interrupted
+					debug( "unpropagated old message %u\n", tmsg->uid );
 				}
-				debug( "unpropagated old message %u\n", tmsg->uid );
 
 				if ((svars->chan->ops[t] & OP_EXPUNGE) && (tmsg->flags & F_DELETED)) {
 					JLOG( "- %u %u", (srec->uid[F], srec->uid[N]), "killing - would be expunged anyway" );
@@ -1660,20 +1833,12 @@ box_loaded( int sts, message_t *msgs, int total_msgs, int recent_msgs, void *aux
 				tmsg->srec = srec;
 				JLOG( "+ %u %u", (srec->uid[F], srec->uid[N]), "fresh" );
 			}
-			if ((tmsg->flags & F_FLAGGED) || tmsg->size <= svars->chan->stores[t]->max_size) {
-				if (srec->status != S_PENDING) {
-					srec->status = S_PENDING;
-					JLOG( "~ %u %u %u", (srec->uid[F], srec->uid[N], srec->status), "was too big" );
-				}
-				any_new[t] = 1;
-			} else {
-				if (srec->status == S_SKIPPED) {
-					debug( "-> still too big\n" );
-				} else {
-					srec->status = S_SKIPPED;
-					JLOG( "~ %u %u %u", (srec->uid[F], srec->uid[N], srec->status), "skipping - too big" );
-				}
+			if (!(tmsg->flags & F_FLAGGED) && tmsg->size > svars->chan->stores[t]->max_size &&
+			    !(srec->wstate & W_UPGRADE) && !(srec->status & (S_DUMMY(F)|S_DUMMY(N)))) {
+				srec->status |= S_DUMMY(t);
+				JLOG( "_ %u %u", (srec->uid[F], srec->uid[N]), "placeholder only - too big" );
 			}
+			any_new[t] = 1;
 		}
 	}
 
@@ -1790,14 +1955,22 @@ box_loaded( int sts, message_t *msgs, int total_msgs, int recent_msgs, void *aux
 
 	debug( "synchronizing flags\n" );
 	for (srec = svars->srecs; srec; srec = srec->next) {
-		if ((srec->status & S_DEAD) || !srec->uid[F] || !srec->uid[N])
+		if (srec->status & S_DEAD)
 			continue;
 		for (t = 0; t < 2; t++) {
+			if (!srec->uid[t])
+				continue;
 			aflags = srec->aflags[t];
 			dflags = srec->dflags[t];
-			if (srec->wstate & W_DELETE) {
+			if (srec->wstate & (W_DELETE|W_PURGE)) {
 				if (!aflags) {
-					/* This deletion propagation goes the other way round. */
+					// This deletion propagation goes the other way round, or
+					// this deletion of a dummy happens on the other side.
+					continue;
+				}
+				if (!srec->msg[t] && (svars->opts[t] & OPEN_OLD)) {
+					// The message disappeared. This can happen, because the wstate may
+					// come from the journal, and things could have happened meanwhile.
 					continue;
 				}
 			} else {
@@ -1874,7 +2047,7 @@ msg_copied( int sts, uint uid, copy_vars_t *vars )
 	sync_rec_t *srec = vars->srec;
 	switch (sts) {
 	case SYNC_OK:
-		if (vars->msg->flags != srec->flags) {
+		if (!(srec->wstate & W_UPGRADE) && vars->msg->flags != srec->flags) {
 			srec->flags = vars->msg->flags;
 			JLOG( "* %u %u %u", (srec->uid[F], srec->uid[N], srec->flags), "%sed with flags", str_hl[t] );
 		}
@@ -1937,6 +2110,7 @@ msgs_copied( sync_vars_t *svars, int t )
 				cv->aux = AUX;
 				cv->srec = srec;
 				cv->msg = tmsg;
+				cv->minimal = (srec->status & S_DUMMY(t));
 				copy_msg( cv );
 				svars->state[t] &= ~ST_SENDING_NEW;
 				if (check_cancel( svars ))
@@ -2085,6 +2259,7 @@ msgs_flags_set( sync_vars_t *svars, int t )
 							cv->aux = INV_AUX;
 							cv->srec = NULL;
 							cv->msg = tmsg;
+							cv->minimal = 0;
 							copy_msg( cv );
 							if (check_cancel( svars ))
 								goto out;