From 224a783e837554f406a238c5e7cff7ef004c40dd Mon Sep 17 00:00:00 2001
From: Oswald Buddenhagen <ossi@users.sf.net>
Date: Sun, 18 Dec 2016 21:10:07 +0100
Subject: [PATCH] *** implement fallback path in UIDVALIDITY recovery

mailboxes with mostly messages with no Message-Id (such as Drafts and
Templates) cannot recover the usual way.

this patch implements a statistical method. as such, it has a (very)
slight chance to yield a false positive which would subsequently clobber
actual messages (specifically, the ones which were not sufficient to
throw off the algorithm). it would be possible to be even stricter, but
then any actual modification of the box would make the algorithm fail.
and even in this case, it would be still possible that even though the
UIDs match (and thus nothing gets deleted), the actual messages are in
reality different (though this is ridiculously unlikely).
the added tolerance doesn't help with stores which are synced seldomly,
as too much is likely to happen in the mean time.

*** actually make it stricter?
---
 src/sync.c | 28 +++++++++++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

diff --git a/src/sync.c b/src/sync.c
index eb48dc5..3984c6e 100644
--- a/src/sync.c
+++ b/src/sync.c
@@ -1434,11 +1434,12 @@ box_loaded( int sts, void *aux )
 
 	for (t = 0; t < 2; t++) {
 		if (svars->uidval[t] >= 0 && svars->uidval[t] != svars->ctx[t]->uidvalidity) {
-			unsigned need = 0, got = 0;
+			unsigned total = 0, need = 0, got = 0;
 			debug( "trying to re-approve uid validity of %s\n", str_ms[t] );
 			for (srec = svars->srecs; srec; srec = srec->next) {
 				if (srec->status & S_DEAD)
 					continue;
+				total++;
 				if (!srec->msg[t])
 					continue;  // Message disappeared.
 				need++;  // Present paired messages require re-validation.
@@ -1461,12 +1462,37 @@ box_loaded( int sts, void *aux )
 				// A proper fallback would be fetching more headers (which potentially need
 				// normalization) or the message body (which should be truncated for sanity)
 				// and comparing.
+				// But for simplicity, we employ a heuristic instead. We limit it to mailboxes
+				// with at most 10 messages as a means to limit (hypothetical) damage (though
+				// with more messages, the likelihood of a false positive is even lower).
+				// The algorithm checks whether enough (80%) of the messages are still in place,
+				// not too many (20%) new messages appeared, and that no new messages appeared
+				// in the already seen UID range. A genuine UID re-enumeration would quickly
+				// invalidate these assertions. The algorithm is most reliable when the UIDs
+				// are non-adjacent (which increases with mailbox use) due to looking random.
+				if (total <= 10 && need * 5 >= total * 4) {
+					unsigned neu = 0;
+					debug( "doing purely UID-based recovery\n" );
+					for (tmsg = svars->ctx[t]->msgs; tmsg; tmsg = tmsg->next) {
+						if (!tmsg->srec) {
+							if (tmsg->uid <= svars->maxuid[t]) {
+								error( "Error: channel %s, %s %s: UIDVALIDITY genuinely changed (at UID %d).\n",
+								       svars->chan->name, str_ms[t], svars->orig_name[t], tmsg->uid );
+								goto uvchg;
+							}
+							neu++;
+						}
+					}
+					if (neu * 5 <= total)
+						goto uvchgok;
+				}
 				error( "Error: channel %s, %s %s: Unable to recover from UIDVALIDITY change\n"
 				       "(got %d, expected %d).\n",
 				       svars->chan->name, str_ms[t], svars->orig_name[t],
 				       svars->ctx[t]->uidvalidity, svars->uidval[t] );
 				goto uvchg;
 			}
+		  uvchgok:
 			notice( "Notice: channel %s, %s %s: Recovered from change of UIDVALIDITY.\n",
 			        svars->chan->name, str_ms[t], svars->orig_name[t] );
 			svars->uidval[t] = -1;