add ExpungeSolo option

REFMAIL: CAOgBZNonT0s0b_yPs2vx81Ru3cQp5M93xpZ3syWBW-2CNoX_ow@mail.gmail.com

NEWS

@@ -20,6 +20,8 @@ A proper summary is now printed prior to exiting.
 
 Added new sync operation 'Old'.
 
+Added support for mirroring deletions more accurately.
+
 [1.4.0]
 
 The 'isync' compatibility wrapper was removed.

src/config.c

@@ -174,6 +174,7 @@ static const struct {
 	const char *name;
 } boxOps[] = {
 	{ OP_EXPUNGE, "Expunge" },
+	{ OP_EXPUNGE_SOLO, "ExpungeSolo" },
 	{ OP_CREATE, "Create" },
 	{ OP_REMOVE, "Remove" },
 };

src/driver.h

@@ -54,6 +54,7 @@ flag_str_t ATTR_OPTIMIZE /* force RVO */ fmt_lone_flags( uchar flags );
 BIT_ENUM(
 	M_RECENT,   // unsyncable flag; maildir_*() depend on this being bit 0
 	M_DEAD,     // expunged
+	M_EXPUNGE,  // for driver_t->close_box()
 	M_FLAGS,    // flags are valid
 	// The following are only for IMAP FETCH response parsing
 	M_DATE,

src/drv_imap.c

@@ -3122,7 +3122,7 @@ imap_close_box( store_t *gctx,
 
 		for (msg = ctx->msgs.head; ; ) {
 			for (bl = 0; msg && bl < 960; msg = msg->next) {
-				if ((msg->status & M_DEAD) || !(msg->flags & F_DELETED))
+				if ((msg->status & M_DEAD) || !(msg->status & M_EXPUNGE))
 					continue;
 				if (bl)
 					buf[bl++] = ',';
@@ -3136,7 +3136,7 @@ imap_close_box( store_t *gctx,
 					} else {
 						if (nmsg->seq > 1)
 							break;
-						if (!(nmsg->flags & F_DELETED))
+						if (!(nmsg->flags & M_EXPUNGE))
 							break;
 					}
 				}

src/drv_maildir.c

@@ -1806,7 +1806,7 @@ maildir_close_box( store_t *gctx,
 		retry = 0;
 		basel = nfsnprintf( buf, sizeof(buf), "%s/", ctx->path );
 		for (msg = ctx->msgs; msg; msg = msg->next) {
-			if (!(msg->status & M_DEAD) && (msg->flags & F_DELETED)) {
+			if (!(msg->status & M_DEAD) && (msg->status & M_EXPUNGE)) {
 				nfsnprintf( buf + basel, _POSIX_PATH_MAX - basel, "%s/%s", subdirs[msg->status & M_RECENT], msg->base );
 				if (unlink( buf )) {
 					if (errno == ENOENT)

src/main.c

@@ -43,6 +43,7 @@ PACKAGE " " VERSION " - mailbox synchronizer\n"
 "  -C, --create		propagate creations of mailboxes\n"
 "  -R, --remove		propagate deletions of mailboxes\n"
 "  -X, --expunge		expunge deleted messages\n"
+"  -x, --expunge-solo	expunge deleted messages that are not paired\n"
 "  -c, --config CONFIG	read an alternate config file (default: ~/." EXE "rc)\n"
 "  -D, --debug		debugging modes (see manual)\n"
 "  -V, --verbose		display what is happening\n"
@@ -52,7 +53,8 @@ PACKAGE " " VERSION " - mailbox synchronizer\n"
 "\nIf neither --pull nor --push are specified, both are active.\n"
 "If neither --new, --gone, --flags, nor --upgrade are specified, all are\n"
 "active. Direction and operation can be concatenated like --pull-new, etc.\n"
-"--create, --remove, and --expunge can be suffixed with -far/-near.\n"
+"--create, --remove, --expunge, and --expunge-solo can be suffixed with"
+"-far/-near.\n"
 "See the man page for details.\n"
 "\nSupported mailbox formats are: IMAP4rev1, Maildir\n"
 "\nCompile time options:\n"
@@ -235,15 +237,21 @@ main( int argc, char **argv )
 						mvars->ops[N] |= op, ms_warn = 1;
 					else
 						goto badopt;
-					mvars->ops[F] |= op & (XOP_HAVE_CREATE | XOP_HAVE_REMOVE | XOP_HAVE_EXPUNGE);
+					mvars->ops[F] |= op & (XOP_HAVE_CREATE | XOP_HAVE_REMOVE | XOP_HAVE_EXPUNGE | XOP_HAVE_EXPUNGE_SOLO);
 				} else if (starts_with( opt, -1, "remove", 6 )) {
 					opt += 6;
 					op = OP_REMOVE|XOP_HAVE_REMOVE;
 					goto lcop;
+				} else if (starts_with( opt, -1, "expunge-solo", 12 )) {
+					opt += 12;
+					op = OP_EXPUNGE_SOLO | XOP_HAVE_EXPUNGE_SOLO;
+					goto lcop;
 				} else if (starts_with( opt, -1, "expunge", 7 )) {
 					opt += 7;
 					op = OP_EXPUNGE|XOP_HAVE_EXPUNGE;
 					goto lcop;
+				} else if (!strcmp( opt, "no-expunge-solo" )) {
+					mvars->ops[F] |= XOP_EXPUNGE_SOLO_NOOP | XOP_HAVE_EXPUNGE_SOLO;
 				} else if (!strcmp( opt, "no-expunge" )) {
 					mvars->ops[F] |= XOP_EXPUNGE_NOOP | XOP_HAVE_EXPUNGE;
 				} else if (!strcmp( opt, "no-create" )) {
@@ -340,11 +348,14 @@ main( int argc, char **argv )
 				ochar++;
 			else
 				cops |= op;
-			mvars->ops[F] |= op & (XOP_HAVE_CREATE | XOP_HAVE_REMOVE | XOP_HAVE_EXPUNGE);
+			mvars->ops[F] |= op & (XOP_HAVE_CREATE | XOP_HAVE_REMOVE | XOP_HAVE_EXPUNGE | XOP_HAVE_EXPUNGE_SOLO);
 			break;
 		case 'R':
 			op = OP_REMOVE|XOP_HAVE_REMOVE;
 			goto cop;
+		case 'x':
+			op = OP_EXPUNGE_SOLO | XOP_HAVE_EXPUNGE_SOLO;
+			goto cop;
 		case 'X':
 			op = OP_EXPUNGE|XOP_HAVE_EXPUNGE;
 			goto cop;

src/main_sync.c

@@ -186,13 +186,20 @@ add_channel( chan_ent_t ***chanapp, channel_conf_t *chan, int ops[] )
 	merge_actions( chan, ops, XOP_HAVE_CREATE, OP_CREATE, 0 );
 	merge_actions( chan, ops, XOP_HAVE_REMOVE, OP_REMOVE, 0 );
 	merge_actions( chan, ops, XOP_HAVE_EXPUNGE, OP_EXPUNGE, 0 );
+	merge_actions( chan, ops, XOP_HAVE_EXPUNGE_SOLO, OP_EXPUNGE_SOLO, 0 );
 	debug( "channel ops (%s):\n  far: %s\n  near: %s\n",
 	       chan->name, fmt_ops( ops[F] ).str, fmt_ops( ops[N] ).str );
 
 	for (int t = 0; t < 2; t++) {
+		if (!(~ops[t] & (OP_EXPUNGE | OP_EXPUNGE_SOLO))) {
+			error( "Specified both Expunge and ExpungeSolo for %s of Channel '%s'.\n",
+			       str_fn[t], chan->stores[t]->name );
+			free( ce );
+			return NULL;
+		}
 		if (chan->ops[t] & OP_MASK_TYPE)
 			ops_any[t] = 1;
-		if ((chan->ops[t] & OP_EXPUNGE) &&
+		if ((chan->ops[t] & (OP_EXPUNGE | OP_EXPUNGE_SOLO)) &&
 		    (chan->stores[t]->trash ||
 		     (chan->stores[t^1]->trash && chan->stores[t^1]->trash_remote_new)))
 			trash_any[t] = 1;
@@ -253,6 +260,8 @@ add_named_channel( chan_ent_t ***chanapp, char *channame, int ops[] )
 	}
 
 	chan_ent_t *ce = add_channel( chanapp, chan, ops );
+	if (!ce)
+		return NULL;
 	ce->boxes = boxes;
 	ce->boxlist = boxlist;
 	return ce;
@@ -297,7 +306,8 @@ sync_chans( core_vars_t *cvars, char **argv )
 
 	if (cvars->all) {
 		for (channel_conf_t *chan = channels; chan; chan = chan->next) {
-			add_channel( &chanapp, chan, cvars->ops );
+			if (!add_channel( &chanapp, chan, cvars->ops ))
+				cvars->ret = 1;
 			if (!chan->patterns)
 				boxes_total++;
 		}

src/mbsync.1

@@ -659,10 +659,24 @@ Note that for safety, non-empty mailboxes are never deleted.
 \fBExpunge\fR {\fBNone\fR|\fBFar\fR|\fBNear\fR|\fBBoth\fR}
 Permanently remove all messages [on the far/near side] which are marked
 for deletion.
+Mutually exclusive with \fBExpungeSolo\fR for the same side.
 See \fBRECOMMENDATIONS\fR below.
 (Global default: \fBNone\fR)
 .
 .TP
+\fBExpungeSolo\fR {\fBNone\fR|\fBFar\fR|\fBNear\fR|\fBBoth\fR}
+Permanently remove all messages [on the far/near side] which are both
+marked for deletion and have no corresponding message in the opposite
+Store.
+Together with \fBSync Gone\fR, this allows actual mirroring of
+expunges. Note, however, that this makes sense only if nothing else
+expunges the other messages which are marked for deletion.
+Also note that this does not work for IMAP Stores which do not support
+the UIDPLUS extension.
+Mutually exclusive with \fBExpunge\fR for the same side.
+(Global default: \fBNone\fR)
+.
+.TP
 \fBCopyArrivalDate\fR {\fByes\fR|\fBno\fR}
 Selects whether their arrival time should be propagated together with
 the messages.
@@ -673,7 +687,7 @@ date\fR) is actually the arrival time, but it is usually close enough.
 (Global default: \fBno\fR)
 .
 .P
-\fBSync\fR, \fBCreate\fR, \fBRemove\fR, \fBExpunge\fR,
+\fBSync\fR, \fBCreate\fR, \fBRemove\fR, \fBExpunge\fR, \fBExpungeSolo\fR,
 \fBMaxMessages\fR, \fBExpireUnread\fR, and \fBCopyArrivalDate\fR
 can be used before any section for a global effect.
 The global settings are overridden by Channel-specific options,

src/run-tests.pl

@@ -1816,4 +1816,104 @@ my @X13 = (
 );
 test("trash new remotely", \@x10, \@X13, \@O13);
 
+# Test "mirroring" expunges.
+
+my @xa0 = (
+  M, 0, M,
+  # pair
+  A, "*", "*", "*",
+  # expire
+  B, "*", "*", "*S",
+  # expire with del
+  C, "*T", "*", "*S",
+  # pair flag del
+  D, "*T", "*", "*",
+  E, "*", "*", "*T",
+  # pair flag undel
+  F, "*", "*T", "*T",
+  G, "*T", "*T", "*",
+  # pair gone
+  H, "_", "*", "*",
+  I, "*", "*", "_",
+  # upgrade
+  J, "**", "*>", "*F?",
+  K, "*F?", "*<", "**",
+  # doomed upgrade
+  L, "*T*", "*>", "*F?",
+  M, "*F?", "*<", "*T*",
+  # doomed new
+  N, "", "", "*T",
+  O, "*T", "", "",
+);
+
+my @Oa1 = ("", "", "ExpungeSolo Both\nMaxMessages 1\nExpireUnread false\n");
+my @Xa1 = (
+  N, B, O,
+  B, "+S", "/", "/",
+  C, "+S", "+ST", "+T",  # This is weird, but it's not worth handling.
+  D, "", "+T", "+T",
+  E, "+T", "+T", "",
+  F, "", "-T", "-T",
+  G, "-T", "-T", "",
+  H, "", "/", "/",
+  I, "/", "/", "",
+  J, "", ">->", "^*",
+  J, "", "", "&1/",
+  K, "^*", "<-<", "",
+  K, "&1/", "", "",
+  L, "", ">->+T", "^*T",
+  L, "", "", "&1/",
+  M, "^*T", "<-<+T", "",
+  M, "&1/", "", "",
+  N, "*T", "*T", "",
+  O, "", "*T", "*T",
+);
+test("expunge solo both", \@xa0, \@Xa1, \@Oa1);
+
+my @Oa2 = ("", "", "ExpungeSolo Near\nMaxMessages 1\nExpireUnread false\n");
+my @Xa2 = (
+  N, B, O,
+  B, "+S", "/", "/",
+  C, "+S", "+ST", "+T",  # As above.
+  D, "", "+T", "+T",
+  E, "+T", "+T", "",
+  F, "", "-T", "-T",
+  G, "-T", "-T", "",
+  H, "", "/", "/",
+  I, "+T", ">", "",
+  J, "", ">->", "^*",
+  J, "", "", "&1/",
+  K, "^*", "<-<", "",
+  K, "&1+T", "^", "|",
+  L, "", ">->+T", "^*T",
+  L, "", "", "&1/",
+  M, "^*T", "<-<+T", "",
+  M, "&1+T", "^", "|",
+  N, "*T", "*T", "",
+  O, "", "*T", "*T",
+);
+test("expunge solo near", \@xa0, \@Xa2, \@Oa2);
+
+my @Oa3 = ("", "", "Expunge Far\nExpungeSolo Near\nMaxMessages 1\nExpireUnread false\n");
+my @Xa3 = (
+  K, B, J,
+  B, "+S", "/", "/",
+  C, "/", "/", "/",
+  D, "/", "/", "/",
+  E, "/", "/", "/",
+  F, "", "-T", "-T",
+  G, "-T", "-T", "",
+  H, "", "/", "/",
+  I, "/", "/", "",
+  J, "", ">->", "^*",
+  J, "", "", "&1/",
+  K, "^*", "<-<", "",
+  K, "&1/", "", "",
+  L, "/", "/", "/",
+  M, "/", "/", "/",
+  N, "", "", "/",
+  O, "/", "", "",
+);
+test("expunge far & solo near", \@xa0, \@Xa3, \@Oa3);
+
 print "OK.\n";

src/sync.c

@@ -788,9 +788,12 @@ box_opened2( sync_vars_t *svars, int t )
 			if ((chan->ops[t] | chan->ops[t^1]) & OP_EXPUNGE)  // Don't propagate doomed msgs
 				opts[t^1] |= OPEN_FLAGS;
 		}
-		if (chan->ops[t] & OP_EXPUNGE) {
+		if (chan->ops[t] & (OP_EXPUNGE | OP_EXPUNGE_SOLO)) {
 			opts[t] |= OPEN_EXPUNGE;
-			if (chan->stores[t]->trash) {
+			if (chan->ops[t] & OP_EXPUNGE_SOLO) {
+				opts[t] |= OPEN_OLD | OPEN_NEW | OPEN_FLAGS | OPEN_UID_EXPUNGE;
+				opts[t^1] |= OPEN_OLD;
+			} else if (chan->stores[t]->trash) {
 				if (!chan->stores[t]->trash_only_new)
 					opts[t] |= OPEN_OLD;
 				opts[t] |= OPEN_NEW | OPEN_FLAGS | OPEN_UID_EXPUNGE;
@@ -816,6 +819,11 @@ box_opened2( sync_vars_t *svars, int t )
 	for (t = 0; t < 2; t++) {
 		svars->opts[t] = svars->drv[t]->prepare_load_box( ctx[t], opts[t] );
 		if (opts[t] & ~svars->opts[t] & OPEN_UID_EXPUNGE) {
+			if (chan->ops[t] & OP_EXPUNGE_SOLO) {
+				error( "Error: Store %s does not support ExpungeSolo.\n",
+				       svars->chan->stores[t]->name );
+				goto bail;
+			}
 			if (!ctx[t]->racy_trash) {
 				ctx[t]->racy_trash = 1;
 				notice( "Notice: Trashing in Store %s is prone to race conditions.\n",
@@ -1490,7 +1498,8 @@ box_loaded( int sts, message_t *msgs, int total_msgs, int recent_msgs, void *aux
 						dflags |= F_DELETED;
 				}
 			}
-			if ((svars->chan->ops[t] & OP_EXPUNGE) && (((srec->msg[t] ? srec->msg[t]->flags : 0) | aflags) & ~dflags & F_DELETED) &&
+			if ((svars->chan->ops[t] & OP_EXPUNGE) &&
+			    (((srec->msg[t] ? srec->msg[t]->flags : 0) | aflags) & ~dflags & F_DELETED) &&
 			    (!svars->ctx[t]->conf->trash || svars->ctx[t]->conf->trash_only_new))
 			{
 				/* If the message is going to be expunged, don't propagate anything but the deletion. */
@@ -1748,8 +1757,54 @@ msgs_flags_set( sync_vars_t *svars, int t )
 	if (check_cancel( svars ))
 		goto out;
 
-	if (!(svars->chan->ops[t] & OP_EXPUNGE))
+	int only_solo;
+	if (svars->chan->ops[t] & OP_EXPUNGE_SOLO)
+		only_solo = 1;
+	else if (svars->chan->ops[t] & OP_EXPUNGE)
+		only_solo = 0;
+	else
 		goto skip;
+	int expunge_other = (svars->chan->ops[t^1] & OP_EXPUNGE);
+	// Driver-wise, this makes sense only if (svars->opts[t] & OPEN_UID_EXPUNGE),
+	// but the trashing loop uses the result as well.
+	debug( "preparing expunge of %s on %s, %sexpunging %s\n",
+	       only_solo ? "solo" : "all", str_fn[t], expunge_other ? "" : "NOT ", str_fn[t^1] );
+	for (tmsg = svars->msgs[t]; tmsg; tmsg = tmsg->next) {
+		if (tmsg->status & M_DEAD)
+			continue;
+		if (!(tmsg->flags & F_DELETED)) {
+			//debug( "  message %u is not deleted\n", tmsg->uid );  // Too noisy
+			continue;
+		}
+		debugn( "  message %u ", tmsg->uid );
+		if (only_solo) {
+			if ((srec = tmsg->srec)) {
+				if (!srec->uid[t^1]) {
+					debugn( "(solo) " );
+				} else if (srec->status & S_GONE(t^1)) {
+					debugn( "(orphaned) " );
+				} else if (expunge_other && (srec->status & S_DEL(t^1))) {
+					debugn( "(orphaning) " );
+				} else if (t == N && (srec->status & (S_EXPIRE | S_EXPIRED))) {
+					// Expiration overrides mirroring, as otherwise the combination
+					// makes no sense at all.
+					debugn( "(expire) " );
+				} else {
+					debug( "is not solo\n" );
+					continue;
+				}
+				if (srec->status & S_PENDING) {
+					debug( "is being paired\n" );
+					continue;
+				}
+			} else {
+				debugn( "(isolated) " );
+			}
+		}
+		debug( "- expunging\n" );
+		tmsg->status |= M_EXPUNGE;
+	}
+
 	int remote, only_new;
 	if (svars->ctx[t]->conf->trash) {
 		only_new = svars->ctx[t]->conf->trash_only_new;
@@ -1765,8 +1820,8 @@ msgs_flags_set( sync_vars_t *svars, int t )
 	for (tmsg = svars->msgs[t]; tmsg; tmsg = tmsg->next) {
 		if (tmsg->status & M_DEAD)
 			continue;
-		if (!(tmsg->flags & F_DELETED)) {
+		if (!(tmsg->status & M_EXPUNGE)) {
-			//debug( "  message %u is not deleted\n", tmsg->uid );  // Too noisy
+			//debug( "  message %u is not being expunged\n", tmsg->uid );  // Too noisy
 			continue;
 		}
 		debugn( "  message %u ", tmsg->uid );
@@ -1881,7 +1936,7 @@ sync_close( sync_vars_t *svars, int t )
 		return;
 	svars->state[t] |= ST_CLOSING;
 
-	if ((svars->chan->ops[t] & OP_EXPUNGE) && !(DFlags & FAKEEXPUNGE)
+	if ((svars->chan->ops[t] & (OP_EXPUNGE | OP_EXPUNGE_SOLO)) && !(DFlags & FAKEEXPUNGE)
 	    /*&& !(svars->state[t] & ST_TRASH_BAD)*/) {
 		debug( "expunging %s\n", str_fn[t] );
 		svars->drv[t]->close_box( svars->ctx[t], box_closed, AUX );

src/sync.h

@@ -21,6 +21,7 @@ BIT_ENUM(
 	OP_GONE,
 	OP_FLAGS,
 	OP_EXPUNGE,
+	OP_EXPUNGE_SOLO,
 	OP_CREATE,
 	OP_REMOVE,
 
@@ -29,12 +30,14 @@ BIT_ENUM(
 	XOP_HAVE_TYPE,  // Aka mode; have at least one of dir and type (see below)
 	// The following must all have the same bit shift from the corresponding OP_* flags.
 	XOP_HAVE_EXPUNGE,
+	XOP_HAVE_EXPUNGE_SOLO,
 	XOP_HAVE_CREATE,
 	XOP_HAVE_REMOVE,
 	// ... until here.
 	XOP_TYPE_NOOP,
 	// ... and here again from scratch.
 	XOP_EXPUNGE_NOOP,
+	XOP_EXPUNGE_SOLO_NOOP,
 	XOP_CREATE_NOOP,
 	XOP_REMOVE_NOOP,
 )