nettika/geary
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'wip/713530-background-sync'. Fixes Bug 713530.

Browse source
This commit is contained in:
Michael James Gratton 2017-12-02 12:18:57 +11:00
commit b1d3d494bc
7 changed files with 444 additions and 336 deletions
Download patch file Download diff file Expand all files Collapse all files

109
src/engine/api/geary-folder.vala
View file

@ -5,25 +5,55 @@
*/
/**
* Folder represents the basic unit of organization for email.
* A Folder represents the basic unit of organization for email.
*
* Each {@link Account} offers a hierarcichal listing of Folders. Folders must be opened (with
* {@link open_async} before using most of its methods and should be closed with
* {@link close_async} when completed, even if a method has failed with an IOError.
* Folders may contain a set of messages, either stored purely locally
* (for example, in the case of an ''outbox'' for mail queued for
* sending), or as a representation of those found in a mailbox on a
* remote mail server, such as those provided by an IMAP server. For
* folders that represent a remote mailbox, messages are cached
* locally, and the set of cached messages may be a subset of those
* available in the mailbox, depending on an account's settings. Note
* that some folders may not be able to contain any messages, and may
* exist purely to construct a hierarchy.
*
* Folder offers various open states indicating when its "local" (disk or database) connection and
* "remote" (network) connections are ready. Generally the local connection opens first and the
* remote connection takes time to establish. When in this state, Folder's methods still operate,
* but will only return locally stored information.
* The set of locally stored messages is called the folder's
* ''vector'', and contains generally the most recent message in the
* mailbox at the upper end, back through to some older message at the
* start or lower end of the vector. Thus the ordering of the vector
* is the ''natural'' ordering, based on the order in which messages
* were appended to the folder, not when messages were sent or some
* other criteria. For remote-backed folders, the engine will maintain
* the vector in accordance with the value of {@link
* AccountInformation.prefetch-period-days}, however the start of the
* vector will be extended back past that over time and in response to
* certain operations that cause the vector to be ''expanded'' ---
* that is for additional messages to be loaded from the remote
* server, extending the vector. The upper end of the vector is
* similarly extended as new messages are appended to the folder by
* another on the server or in response to user operations such as
* moving a message.
*
* Folder only offers a small selection of guaranteed functionality (in particular, the ability
* to list its {@link Email}). Additional functionality for Folders is indicated by the presence
* of {@link FolderSupport} interfaces, include {@link FolderSupport.Remove},
* {@link FolderSupport.Copy}, and so forth.
* Each {@link Account} offers a hierarchical listing of Folders.
* Folders must be opened (with {@link open_async} before using most
* of its methods and should be closed with {@link close_async} when
* completed, even if a method has failed with an IOError.
*
* Folders offer various open states indicating when its "local" (disk
* or database) connection and "remote" (network) connections are
* ready. Generally the local connection opens first and the remote
* connection takes time to establish. When in this state, Folder's
* methods still operate, but will only return locally stored
* information.
*
* This class only offers a small selection of guaranteed
* functionality (in particular, the ability to list its {@link
* Email}). Additional functionality for Folders is indicated by the
* presence of {@link FolderSupport} interfaces, include {@link
* FolderSupport.Remove}, {@link FolderSupport.Copy}, and so forth.
*
* @see Geary.SpecialFolderType
*/
public abstract class Geary.Folder : BaseObject {
public enum OpenState {
CLOSED,
@ -475,26 +505,49 @@ public abstract class Geary.Folder : BaseObject {
public abstract async void find_boundaries_async(Gee.Collection<Geary.EmailIdentifier> ids,
out Geary.EmailIdentifier? low, out Geary.EmailIdentifier? high,
Cancellable? cancellable = null) throws Error;
/**
* List emails from the {@link Folder} starting at a particular location within the vector
* and moving either direction along the mail stack.
* List a number of contiguous emails in the folder's vector.
*
* If the {@link EmailIdentifier} is null, it indicates the end of the vector. Which end
* depends on the {@link ListFlags.OLDEST_TO_NEWEST} flag. Without, the default is to traverse
* from newest to oldest, with null being the newest email. If set, the direction is reversed
* and null indicates the oldest email.
* Emails in the folder are listed starting at a particular
* location within the vector and moving either direction along
* it. For remote-backed folders, the remote server is contacted
* if any messages stored locally do not meet the requirements
* given by `required_fields`, or if `count` extends back past the
* low end of the vector.
*
* If not null, the EmailIdentifier ''must'' have originated from this Folder.
* If the {@link EmailIdentifier} is null, it indicates the end of
* the vector, not the end of the remote. Which end depends on
* the {@link ListFlags.OLDEST_TO_NEWEST} flag. If not set, the
* default is to traverse from newest to oldest, with null being
* the newest email in the vector. If set, the direction is
* reversed and null indicates the oldest email in the vector, not
* the oldest in the mailbox.
*
* To fetch all available messages in one call, use a count of int.MAX.
* If not null, the EmailIdentifier ''must'' have originated from
* this Folder.
*
* Use {@link ListFlags.INCLUDING_ID} to include the {@link Email} for the particular identifier
* in the results. Otherwise, the specified email will not be included. A null
* EmailIdentifier implies that the top most email is included in the result (i.e.
* To fetch all available messages in one call, use a count of
* `int.MAX`. If the {@link ListFlags.OLDEST_TO_NEWEST} flag is
* set then the listing will contain all messages in the vector,
* and no expansion will be performed. It may still access the
* remote however in case of any of the messages not meeting the
* given `required_fields`. If {@link ListFlags.OLDEST_TO_NEWEST}
* is not set, the call will cause the vector to be fully expanded
* and the listing will return all messages in the remote
* mailbox. Note that specifying `int.MAX` in either case may be a
* expensive operation (in terms of both computation and memory)
* if the number of messages in the folder or mailbox is large,
* hence should be avoided if possible.
*
* Use {@link ListFlags.INCLUDING_ID} to include the {@link Email}
* for the particular identifier in the results. Otherwise, the
* specified email will not be included. A null EmailIdentifier
* implies that the top most email is included in the result (i.e.
* ListFlags.INCLUDING_ID is not required);
*
* If the remote connection fails, this call will return locally-available Email without error.
* If the remote connection fails, this call will return
* locally-available Email without error.
*
* There's no guarantee of the returned messages' order.
*
@ -503,8 +556,10 @@ public abstract class Geary.Folder : BaseObject {
public abstract async Gee.List<Geary.Email>? list_email_by_id_async(Geary.EmailIdentifier? initial_id,
int count, Geary.Email.Field required_fields, ListFlags flags, Cancellable? cancellable = null)
throws Error;
/**
* List a set of non-contiguous emails in the folder's vector.
*
* Similar in contract to {@link list_email_by_id_async}, but uses a list of
* {@link Geary.EmailIdentifier}s rather than a range.
*

329
src/engine/imap-db/imap-db-folder.vala
View file

@ -66,21 +66,21 @@ private class Geary.ImapDB.Folder : BaseObject, Geary.ReferenceSemantics {
return result;
}
}
private class LocationIdentifier {
public int64 message_id;
public Imap.UID uid;
public ImapDB.EmailIdentifier email_id;
public bool marked_removed;
public LocationIdentifier(int64 message_id, Imap.UID uid, bool marked_removed) {
this.message_id = message_id;
this.uid = uid;
email_id = new ImapDB.EmailIdentifier(message_id, uid);
this.email_id = new ImapDB.EmailIdentifier(message_id, uid);
this.marked_removed = marked_removed;
}
}
protected int manual_ref_count { get; protected set; }
private ImapDB.Database db;
@ -1194,52 +1194,45 @@ private class Geary.ImapDB.Folder : BaseObject, Geary.ReferenceSemantics {
stmt.reset(Db.ResetScope.SAVE_BINDINGS);
}
}
// Returns message_id if duplicate found, associated set to true if message is already associated
// with this folder. Only call this on emails that came from the IMAP Folder.
private LocationIdentifier? do_search_for_duplicates(Db.Connection cx, Geary.Email email,
out bool associated, Cancellable? cancellable) throws Error {
associated = false;
ImapDB.EmailIdentifier email_id = (ImapDB.EmailIdentifier) email.id;
// This should only ever get invoked for messages that came from the
// IMAP layer, which don't have a message id, but should have a UID.
assert(email_id.message_id == Db.INVALID_ROWID);
LocationIdentifier? location = null;
// See if it already exists; first by UID (which is only guaranteed to
// be unique in a folder, not account-wide)
if (email_id.uid != null)
location = do_get_location_for_uid(cx, email_id.uid, ListFlags.INCLUDE_MARKED_FOR_REMOVE,
cancellable);
if (location != null) {
associated = true;
return location;
}
/**
* Returns the id of any existing message matching the given.
*
* Searches for an existing message that matches `email`, based on
* its message attributes. Currently, since ImapDB only requests
* the IMAP internal date and RFC822 message size, these are the
* only attributes used.
*
* The unique, internal message ID of the first matching message
* is returned, else `-1` if no matching message was found.
*
* This should only be called on messages obtained via the IMAP
* stack.
*/
private int64 do_search_for_duplicates(Db.Connection cx,
Geary.Email email,
ImapDB.EmailIdentifier email_id,
Cancellable? cancellable)
throws Error {
int64 id = -1;
// if fields not present, then no duplicate can reliably be found
if (!email.fields.is_all_set(REQUIRED_FIELDS)) {
debug("Unable to detect duplicates for %s (%s available)", email.id.to_string(),
email.fields.to_list_string());
return null;
return id;
}
// what's more, actually need all those fields to be available, not merely attempted,
// to err on the side of safety
Imap.EmailProperties? imap_properties = (Imap.EmailProperties) email.properties;
string? internaldate = (imap_properties != null && imap_properties.internaldate != null)
? imap_properties.internaldate.serialize() : null;
int64 rfc822_size = (imap_properties != null) ? imap_properties.rfc822_size.value : -1;
if (String.is_empty(internaldate) || rfc822_size < 0) {
debug("Unable to detect duplicates for %s (%s available but invalid)", email.id.to_string(),
email.fields.to_list_string());
return null;
return id;
}
// look for duplicate in IMAP message properties
@ -1255,49 +1248,32 @@ private class Geary.ImapDB.Folder : BaseObject, Geary.ReferenceSemantics {
stmt.bind_string(2, email.message_id.to_string());
Db.Result results = stmt.exec(cancellable);
// no duplicates found
if (results.finished)
return null;
int64 message_id = results.rowid_at(0);
if (results.next(cancellable)) {
debug("Warning: multiple messages with the same internaldate (%s) and size (%s) in %s",
internaldate, rfc822_size.to_string(), to_string());
if (!results.finished) {
id = results.int64_at(0);
}
Db.Statement search_stmt = cx.prepare(
"SELECT ordering, remove_marker FROM MessageLocationTable WHERE message_id=? AND folder_id=?");
search_stmt.bind_rowid(0, message_id);
search_stmt.bind_rowid(1, folder_id);
Db.Result search_results = search_stmt.exec(cancellable);
if (!search_results.finished) {
associated = true;
location = new LocationIdentifier(message_id, new Imap.UID(search_results.int64_at(0)),
search_results.bool_at(1));
} else {
assert(email_id.uid != null);
location = new LocationIdentifier(message_id, email_id.uid, false);
}
return location;
return id;
}
// Note: does NOT check if message is already associated with thie folder
private void do_associate_with_folder(Db.Connection cx, int64 message_id, Imap.UID uid,
Cancellable? cancellable) throws Error {
assert(message_id != Db.INVALID_ROWID);
// insert email at supplied position
/**
* Adds a message to the folder.
*
* Note: does NOT check if message is already associated with thie
* folder.
*/
private void do_associate_with_folder(Db.Connection cx,
int64 message_id,
Imap.UID uid,
Cancellable? cancellable)
throws Error {
Db.Statement stmt = cx.prepare(
"INSERT INTO MessageLocationTable (message_id, folder_id, ordering) VALUES (?, ?, ?)");
stmt.bind_rowid(0, message_id);
stmt.bind_rowid(1, folder_id);
stmt.bind_rowid(1, this.folder_id);
stmt.bind_int64(2, uid.value);
stmt.exec(cancellable);
}
private void do_remove_association_with_folder(Db.Connection cx, LocationIdentifier location,
Cancellable? cancellable) throws Error {
Db.Statement stmt = cx.prepare(
@ -1307,112 +1283,143 @@ private class Geary.ImapDB.Folder : BaseObject, Geary.ReferenceSemantics {
stmt.exec(cancellable);
}
/**
* Adds a single message to the folder, creating or merging it.
*
* This creates the message and appends it to the folder if the
* message does not already exist, else appends and merges if the
* message exists but not in the given position in this folder,
* else it exists in the given position, so simply merges it.
*
* Returns `true` if created, else was merged and returns `false`.
*/
private bool do_create_or_merge_email(Db.Connection cx, Geary.Email email,
out Geary.Email.Field pre_fields, out Geary.Email.Field post_fields,
out Gee.Collection<Contact> updated_contacts, ref int unread_count_change,
Cancellable? cancellable) throws Error {
// see if message already present in current folder, if not, search for duplicate throughout
// mailbox
bool associated;
LocationIdentifier? location = do_search_for_duplicates(cx, email, out associated, cancellable);
// if found, merge, and associate if necessary
// This should only ever get invoked for messages that came
// from the IMAP layer, which should not have a message id,
// but should have a UID.
ImapDB.EmailIdentifier? email_id = email.id as ImapDB.EmailIdentifier;
if (email_id == null ||
email_id.message_id != Db.INVALID_ROWID ||
email_id.uid == null) {
throw new EngineError.INCOMPLETE_MESSAGE(
"IMAP message with UID required"
);
}
int64 message_id = -1;
bool is_associated = false;
// First, look for the same message at the same location
LocationIdentifier? location = do_get_location_for_uid(
cx,
email_id.uid,
ListFlags.INCLUDE_MARKED_FOR_REMOVE,
cancellable
);
if (location != null) {
if (!associated)
do_associate_with_folder(cx, location.message_id, location.uid, cancellable);
// If the email came from the Imap layer, we need to fill in the id.
ImapDB.EmailIdentifier email_id = (ImapDB.EmailIdentifier) email.id;
if (email_id.message_id == Db.INVALID_ROWID)
email_id.promote_with_message_id(location.message_id);
// special-case updating flags, which happens often and should only write to the DB
// if necessary
// Already at the specified location, so no need to create
// or associate with this folder — just merge it
message_id = location.message_id;
is_associated = true;
} else {
// Not already at the specified location, so look for the
// same message in other locations or other folders
message_id = do_search_for_duplicates(
cx, email, email_id, cancellable
);
if (message_id >= 0) {
location = new LocationIdentifier(
message_id, email_id.uid, false
);
}
}
bool was_created = false;
if (location != null) {
// Found the same or a duplicate message, so merge it. We
// special-case flag-only updates, which happens often and
// will only write to the DB if necessary.
if (email.fields != Geary.Email.Field.FLAGS) {
do_merge_email(cx, location, email, out pre_fields, out post_fields,
out updated_contacts, ref unread_count_change, cancellable);
// Already associated with folder and flags were known.
if (associated && pre_fields.is_all_set(Geary.Email.Field.FLAGS))
if (is_associated && pre_fields.is_all_set(Geary.Email.Field.FLAGS))
unread_count_change = 0;
} else {
do_merge_email_flags(cx, location, email, out pre_fields, out post_fields,
out updated_contacts, ref unread_count_change, cancellable);
}
// return false to indicate a merge
return false;
} else {
// Message was not found, so create a new message for it
was_created = true;
MessageRow row = new MessageRow.from_email(email);
pre_fields = Geary.Email.Field.NONE;
post_fields = email.fields;
Db.Statement stmt = cx.prepare(
"INSERT INTO MessageTable "
+ "(fields, date_field, date_time_t, from_field, sender, reply_to, to_field, cc, bcc, "
+ "message_id, in_reply_to, reference_ids, subject, header, body, preview, flags, "
+ "internaldate, internaldate_time_t, rfc822_size) "
+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
stmt.bind_int(0, row.fields);
stmt.bind_string(1, row.date);
stmt.bind_int64(2, row.date_time_t);
stmt.bind_string(3, row.from);
stmt.bind_string(4, row.sender);
stmt.bind_string(5, row.reply_to);
stmt.bind_string(6, row.to);
stmt.bind_string(7, row.cc);
stmt.bind_string(8, row.bcc);
stmt.bind_string(9, row.message_id);
stmt.bind_string(10, row.in_reply_to);
stmt.bind_string(11, row.references);
stmt.bind_string(12, row.subject);
stmt.bind_string_buffer(13, row.header);
stmt.bind_string_buffer(14, row.body);
stmt.bind_string(15, row.preview);
stmt.bind_string(16, row.email_flags);
stmt.bind_string(17, row.internaldate);
stmt.bind_int64(18, row.internaldate_time_t);
stmt.bind_int64(19, row.rfc822_size);
message_id = stmt.exec_insert(cancellable);
// write out attachments, if any
// TODO: Because this involves saving files, it potentially means holding up access to the
// database while they're being written; may want to do this outside of transaction.
if (email.fields.fulfills(Attachment.REQUIRED_FIELDS))
do_save_attachments(cx, message_id, email.get_message().get_attachments(), cancellable);
do_add_email_to_search_table(cx, message_id, email, cancellable);
MessageAddresses message_addresses =
new MessageAddresses.from_email(account_owner_email, email);
foreach (Contact contact in message_addresses.contacts)
do_update_contact(cx, contact, cancellable);
updated_contacts = message_addresses.contacts;
// Update unread count if our new email is unread.
if (email.email_flags != null && email.email_flags.is_unread())
unread_count_change++;
}
// not found, so create and associate with this folder
MessageRow row = new MessageRow.from_email(email);
ImapDB.EmailIdentifier email_id = (ImapDB.EmailIdentifier) email.id;
// the create case *requires* a UID be present (originating from Imap.Folder)
Imap.UID? uid = email_id.uid;
assert(uid != null);
pre_fields = Geary.Email.Field.NONE;
post_fields = email.fields;
Db.Statement stmt = cx.prepare(
"INSERT INTO MessageTable "
+ "(fields, date_field, date_time_t, from_field, sender, reply_to, to_field, cc, bcc, "
+ "message_id, in_reply_to, reference_ids, subject, header, body, preview, flags, "
+ "internaldate, internaldate_time_t, rfc822_size) "
+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
stmt.bind_int(0, row.fields);
stmt.bind_string(1, row.date);
stmt.bind_int64(2, row.date_time_t);
stmt.bind_string(3, row.from);
stmt.bind_string(4, row.sender);
stmt.bind_string(5, row.reply_to);
stmt.bind_string(6, row.to);
stmt.bind_string(7, row.cc);
stmt.bind_string(8, row.bcc);
stmt.bind_string(9, row.message_id);
stmt.bind_string(10, row.in_reply_to);
stmt.bind_string(11, row.references);
stmt.bind_string(12, row.subject);
stmt.bind_string_buffer(13, row.header);
stmt.bind_string_buffer(14, row.body);
stmt.bind_string(15, row.preview);
stmt.bind_string(16, row.email_flags);
stmt.bind_string(17, row.internaldate);
stmt.bind_int64(18, row.internaldate_time_t);
stmt.bind_int64(19, row.rfc822_size);
int64 message_id = stmt.exec_insert(cancellable);
// Make sure the id is filled in even if it came from the Imap layer.
if (email_id.message_id == Db.INVALID_ROWID)
email_id.promote_with_message_id(message_id);
do_associate_with_folder(cx, message_id, uid, cancellable);
// write out attachments, if any
// TODO: Because this involves saving files, it potentially means holding up access to the
// database while they're being written; may want to do this outside of transaction.
if (email.fields.fulfills(Attachment.REQUIRED_FIELDS))
do_save_attachments(cx, message_id, email.get_message().get_attachments(), cancellable);
do_add_email_to_search_table(cx, message_id, email, cancellable);
MessageAddresses message_addresses =
new MessageAddresses.from_email(account_owner_email, email);
foreach (Contact contact in message_addresses.contacts)
do_update_contact(cx, contact, cancellable);
updated_contacts = message_addresses.contacts;
// Update unread count if our new email is unread.
if (email.email_flags != null && email.email_flags.is_unread())
unread_count_change++;
return true;
// Finally, update the email's message id and add it to the
// folder, if needed
email_id.promote_with_message_id(message_id);
if (!is_associated) {
do_associate_with_folder(cx, message_id, email_id.uid, cancellable);
}
return was_created;
}
internal static void do_add_email_to_search_table(Db.Connection cx, int64 message_id,
Geary.Email email, Cancellable? cancellable) throws Error {
string? body = null;

140
src/engine/imap-engine/imap-engine-account-synchronizer.vala
View file

@ -269,69 +269,69 @@ private class Geary.ImapEngine.AccountSynchronizer : Geary.BaseObject {
Cancellable cancellable)
throws Error {
Logging.debug(
Logging.Flag.PERIODIC, "Background sync'ing %s", folder.to_string()
Logging.Flag.PERIODIC,
"Background sync'ing %s to %s",
folder.to_string(),
epoch.to_string()
);
// get oldest local email and its time, as well as number of messages in local store
// If we aren't checking the folder because it became
// available, then it has changed and we need to check it.
// Otherwise compare the oldest mail in the local store and
// see if it is before the epoch; if so, no need to
// synchronize simply because this Folder is available; wait
// for its contents to change instead.
//
// Note we can't compare the local and remote folder counts
// here, since the folder may not have opened yet to determine
// what the actual remote count is, which is particularly
// problematic when an existing folder is seen for the first
// time, e.g. when the account was just added.
DateTime? oldest_local = null;
Geary.EmailIdentifier? oldest_local_id = null;
int local_count = 0;
Gee.List<Geary.Email>? list = yield folder.local_folder.list_email_by_id_async(
null,
1,
Email.Field.PROPERTIES,
ImapDB.Folder.ListFlags.NONE | ImapDB.Folder.ListFlags.OLDEST_TO_NEWEST,
cancellable
);
if (list != null && list.size > 0) {
oldest_local = list[0].properties.date_received;
oldest_local_id = list[0].id;
}
local_count = yield folder.local_folder.get_email_count_async(
ImapDB.Folder.ListFlags.NONE,
cancellable
);
bool do_sync = true;
if (availability_check) {
// Compare the oldest mail in the local store and see if it is before the epoch; if so, no
// need to synchronize simply because this Folder is available; wait for its contents to
// change instead
if (oldest_local != null) {
if (oldest_local.compare(epoch) < 0) {
// Oldest local email before epoch, don't sync from network
do_sync = false;
} else if (folder.properties.email_total == local_count) {
// Local earliest email is after epoch, but there's nothing before it
do_sync = false;
} else {
Logging.debug(
Logging.Flag.PERIODIC,
"Oldest local email in %s not old enough (%s vs. %s), email_total=%d vs. local_count=%d, synchronizing...",
folder.to_string(),
oldest_local.to_string(),
epoch.to_string(),
folder.properties.email_total,
local_count
);
}
} else if (folder.properties.email_total == 0) {
// no local messages, no remote messages -- this is as good as having everything up
// to the epoch
do_sync = false;
} else {
Logging.debug(
Logging.Flag.PERIODIC,
"No oldest message found for %s, synchronizing...",
folder.to_string()
);
}
} else {
if (!availability_check) {
// Folder already available, so it must have changed
Logging.debug(
Logging.Flag.PERIODIC,
"Folder %s changed, synchronizing...",
folder.to_string()
);
} else {
// get oldest local email and its time, as well as number
// of messages in local store
Gee.List<Geary.Email>? list =yield folder.local_folder.list_email_by_id_async(
null,
1,
Email.Field.PROPERTIES,
ImapDB.Folder.ListFlags.NONE | ImapDB.Folder.ListFlags.OLDEST_TO_NEWEST,
cancellable
);
if (list != null && list.size > 0) {
oldest_local = list[0].properties.date_received;
oldest_local_id = list[0].id;
}
if (oldest_local == null) {
// No oldest message found, so we haven't seen the folder
// before or it has no messages. Either way we need to
// open it to check, so sync it.
Logging.debug(
Logging.Flag.PERIODIC,
"No oldest message found for %s, synchronizing...",
folder.to_string()
);
} else if (oldest_local.compare(epoch) < 0) {
// Oldest local email before epoch, don't sync from network
do_sync = false;
Logging.debug(
Logging.Flag.PERIODIC,
"Oldest local message is older than the epoch for %s",
folder.to_string()
);
}
}
if (do_sync) {
@ -352,6 +352,10 @@ private class Geary.ImapEngine.AccountSynchronizer : Geary.BaseObject {
}
}
}
Logging.debug(
Logging.Flag.PERIODIC, "Background sync of %s completed",
folder.to_string()
);
}
private async void sync_folder_async(MinimalFolder folder,
@ -375,11 +379,14 @@ private class Geary.ImapEngine.AccountSynchronizer : Geary.BaseObject {
// no need to keep searching once this happens
int local_count = yield folder.local_folder.get_email_count_async(ImapDB.Folder.ListFlags.NONE,
cancellable);
if (local_count >= folder.properties.email_total) {
int remote_count = folder.properties.email_total;
if (local_count >= remote_count) {
Logging.debug(
Logging.Flag.PERIODIC,
"Total vector normalization for %s: %d/%d emails", folder.to_string(), local_count,
folder.properties.email_total
"Final vector normalization for %s: %d/%d emails",
folder.to_string(),
local_count,
remote_count
);
break;
}
@ -394,11 +401,24 @@ private class Geary.ImapEngine.AccountSynchronizer : Geary.BaseObject {
max_epoch.to_string(),
folder.to_string(),
local_count,
folder.properties.email_total
remote_count
);
yield folder.list_email_by_id_async(null, 1, Geary.Email.Field.NONE,
Geary.Folder.ListFlags.OLDEST_TO_NEWEST, cancellable);
// Per the contract for list_email_by_id_async, we
// need to specify int.MAX count and ensure that
// ListFlags.OLDEST_TO_NEWEST is *not* specified
// to get all messages listed.
//
// XXX This is expensive, but should only usually
// happen once per folder - at the end of a full
// sync.
yield folder.list_email_by_id_async(
null,
int.MAX,
Geary.Email.Field.NONE,
Geary.Folder.ListFlags.NONE,
cancellable
);
} else {
// don't go past proscribed epoch
if (current_epoch.compare(epoch) < 0)
@ -410,7 +430,7 @@ private class Geary.ImapEngine.AccountSynchronizer : Geary.BaseObject {
folder.to_string(),
current_epoch.to_string(),
local_count,
folder.properties.email_total
remote_count
);
Geary.EmailIdentifier? earliest_span_id = yield folder.find_earliest_email_async(current_epoch,
oldest_local_id, cancellable);

8
src/engine/imap-engine/imap-engine-email-prefetcher.vala
View file

@ -36,8 +36,8 @@ private class Geary.ImapEngine.EmailPrefetcher : Object {
folder.opened.connect(on_opened);
folder.closed.connect(on_closed);
folder.email_appended.connect(on_local_expansion);
folder.email_inserted.connect(on_local_expansion);
folder.email_locally_appended.connect(on_local_expansion);
folder.email_locally_inserted.connect(on_local_expansion);
}
~EmailPrefetcher() {
@ -46,8 +46,8 @@ private class Geary.ImapEngine.EmailPrefetcher : Object {
folder.opened.disconnect(on_opened);
folder.closed.disconnect(on_closed);
folder.email_appended.disconnect(on_local_expansion);
folder.email_inserted.disconnect(on_local_expansion);
folder.email_locally_appended.disconnect(on_local_expansion);
folder.email_locally_inserted.disconnect(on_local_expansion);
}
private void on_opened(Geary.Folder.OpenState open_state) {

147
src/engine/imap-engine/replay-ops/imap-engine-abstract-list-email.vala
View file

@ -4,6 +4,9 @@
* (version 2.1 or later). See the COPYING file in this distribution.
*/
/**
* A base class for building replay operations that list messages.
*/
private abstract class Geary.ImapEngine.AbstractListEmail : Geary.ImapEngine.SendReplayOperation {
private static int total_fetches_avoided = 0;
@ -187,7 +190,10 @@ private abstract class Geary.ImapEngine.AbstractListEmail : Geary.ImapEngine.Sen
return ReplayOperation.Status.COMPLETED;
}
/**
* Determines if the owning folder's vector is fully expanded.
*/
protected async Trillian is_fully_expanded_async() throws Error {
int remote_count;
owner.get_remote_counts(out remote_count, null);
@ -204,16 +210,29 @@ private abstract class Geary.ImapEngine.AbstractListEmail : Geary.ImapEngine.Sen
return Trillian.from_boolean(local_count_with_marked >= remote_count);
}
// Adds everything in the expansion to the unfulfilled set with ImapDB's field requirements ...
// UIDs are returned if anything else needs to be added to them
/**
* Expands the owning folder's vector.
*
* Lists on the remote messages needed to fulfill ImapDB's
* requirements from `initial_uid` (inclusive) forward to the
* start of the vector if the OLDEST_TO_NEWEST flag is set, else
* from `initial_uid` (inclusive) back at most by `count` number
* of messages. If `initial_uid` is null, the start or end of the
* remote is used, respectively.
*
* The returned UIDs are those added to the vector, which can then
* be examined and added to the messages to be fulfilled if
* needed.
*/
protected async Gee.Set<Imap.UID>? expand_vector_async(Imap.UID? initial_uid, int count) throws Error {
debug("%s: expanding vector...", owner.to_string());
// watch out for situations where the entire folder is represented locally (i.e. no
// expansion necessary)
int remote_count = owner.get_remote_counts(null, null);
if (remote_count < 0)
if (remote_count <= 0)
return null;
// include marked for removed in the count in case this is being called while a removal
// is in process, in which case don't want to expand vector this moment because the
// vector is in flux
@ -223,74 +242,62 @@ private abstract class Geary.ImapEngine.AbstractListEmail : Geary.ImapEngine.Sen
// watch out for attempts to expand vector when it's expanded as far as it will go
if (local_count >= remote_count)
return null;
// determine low and high position for expansion ... default in most code paths for high
// is the SequenceNumber just below the lowest known message, unless no local messages
// are present
Imap.SequenceNumber? low_pos = null;
Imap.SequenceNumber? high_pos = null;
if (local_count > 0)
high_pos = new Imap.SequenceNumber(Numeric.int_floor(remote_count - local_count, 1));
if (flags.is_oldest_to_newest()) {
if (initial_uid == null) {
// if oldest to newest and initial-id is null, then start at the bottom
low_pos = new Imap.SequenceNumber(Imap.SequenceNumber.MIN);
} else {
Gee.Map<Imap.UID, Imap.SequenceNumber>? map = yield owner.remote_folder.uid_to_position_async(
new Imap.MessageSet.uid(initial_uid), cancellable);
if (map == null || map.size == 0 || !map.has_key(initial_uid)) {
debug("%s: Unable to expand vector for initial_uid=%s: unable to convert to position",
to_string(), initial_uid.to_string());
return null;
}
low_pos = map.get(initial_uid);
}
} else {
// newest to oldest
//
// if initial_id is null or no local earliest UID, then vector expansion is simple:
// merely count backwards from the top of the locally available vector
if (initial_uid == null || local_count == 0) {
low_pos = new Imap.SequenceNumber(Numeric.int_floor((remote_count - local_count) - count, 1));
// don't set high_pos, leave null to use symbolic "highest" in MessageSet
high_pos = null;
} else {
// not so simple; need to determine the *remote* position of the earliest local
// UID and count backward from that; if no UIDs present, then it's as if no initial_id
// is specified.
//
// low position: count backwards; note that it's possible this will overshoot and
// pull in more email than technically required, but without a round-trip to the
// server to determine the position number of a particular UID, this makes sense
assert(high_pos != null);
low_pos = new Imap.SequenceNumber(
Numeric.int64_floor((high_pos.value - count) + 1, 1));
// Determine low and high position for expansion. The vector
// start position is based on the assumption that the vector
// end is the same as the remote end.
int64 vector_start = (remote_count - local_count + 1);
int64 low_pos = -1;
int64 high_pos = -1;
int64 initial_pos = -1;
if (initial_uid != null) {
Gee.Map<Imap.UID, Imap.SequenceNumber>? map =
yield owner.remote_folder.uid_to_position_async(
new Imap.MessageSet.uid(initial_uid), cancellable
);
Imap.SequenceNumber? pos = map.get(initial_uid);
if (pos != null) {
initial_pos = pos.value;
}
}
// low_pos must be defined by this point
assert(low_pos != null);
if (high_pos != null && low_pos.value > high_pos.value) {
debug("%s: Aborting vector expansion, low_pos=%s > high_pos=%s", owner.to_string(),
low_pos.to_string(), high_pos.to_string());
// Determine low and high position for expansion
if (flags.is_oldest_to_newest()) {
low_pos = Imap.SequenceNumber.MIN;
if (initial_pos > Imap.SequenceNumber.MIN) {
low_pos = initial_pos;
}
high_pos = vector_start - 1;
} else {
// Newest to oldest.
if (initial_pos <= Imap.SequenceNumber.MIN) {
high_pos = remote_count;
low_pos = Numeric.int64_floor(
high_pos - count + 1, Imap.SequenceNumber.MIN
);
} else {
high_pos = Numeric.int64_floor(
initial_pos, vector_start - 1
);
low_pos = Numeric.int64_floor(
initial_pos - (count - 1), Imap.SequenceNumber.MIN
);
}
}
if (low_pos > high_pos) {
debug("%s: Aborting vector expansion, low_pos=%s > high_pos=%s",
owner.to_string(), low_pos.to_string(), high_pos.to_string());
return null;
}
Imap.MessageSet msg_set;
int64 actual_count = -1;
if (high_pos != null) {
msg_set = new Imap.MessageSet.range_by_first_last(low_pos, high_pos);
actual_count = (high_pos.value - low_pos.value) + 1;
} else {
msg_set = new Imap.MessageSet.range_to_highest(low_pos);
}
Imap.MessageSet msg_set = new Imap.MessageSet.range_by_first_last(
new Imap.SequenceNumber(low_pos),
new Imap.SequenceNumber(high_pos)
);
int64 actual_count = (high_pos - low_pos) + 1;
debug("%s: Performing vector expansion using %s for initial_uid=%s count=%d actual_count=%s local_count=%d remote_count=%d oldest_to_newest=%s",
owner.to_string(), msg_set.to_string(),
(initial_uid != null) ? initial_uid.to_string() : "(null)", count, actual_count.to_string(),

9
src/engine/imap-engine/replay-ops/imap-engine-list-email-by-id.vala
View file

@ -57,7 +57,14 @@ private class Geary.ImapEngine.ListEmailByID : Geary.ImapEngine.AbstractListEmai
add_unfulfilled_fields(uid, required_fields.clear(email.fields));
}
}
if (this.flags.is_including_id() && this.initial_uid == null) {
throw new EngineError.NOT_FOUND(
"Initial id not found in local set: %s",
this.initial_id.to_string()
);
}
// report fulfilled items
fulfilled_count = fulfilled.size;
if (fulfilled_count > 0)

38
src/engine/imap/api/imap-folder.vala
View file

@ -590,27 +590,39 @@ private class Geary.Imap.Folder : BaseObject {
return (email_list.size > 0) ? email_list : null;
}
public async Gee.Map<UID, SequenceNumber>? uid_to_position_async(MessageSet msg_set,
Cancellable? cancellable) throws Error {
/**
* Returns the sequence numbers for a set of UIDs.
*
* The `msg_set` parameter must be a set containing UIDs. An error
* is thrown if the sequence numbers cannot be determined.
*/
public async Gee.Map<UID, SequenceNumber> uid_to_position_async(MessageSet msg_set,
Cancellable? cancellable)
throws Error {
check_open();
// MessageSet better be UID addressing
assert(msg_set.is_uid);
if (!msg_set.is_uid) {
throw new ImapError.NOT_SUPPORTED("Message set must contain UIDs");
}
Gee.List<Command> cmds = new Gee.ArrayList<Command>();
cmds.add(new FetchCommand.data_type(msg_set, FetchDataSpecifier.UID));
Gee.HashMap<SequenceNumber, FetchedData>? fetched;
yield exec_commands_async(cmds, out fetched, null, cancellable);
if (fetched == null || fetched.size == 0)
return null;
Gee.Map<UID, SequenceNumber> map = new Gee.HashMap<UID, SequenceNumber>();
foreach (SequenceNumber seq_num in fetched.keys)
map.set((UID) fetched.get(seq_num).data_map.get(FetchDataSpecifier.UID), seq_num);
if (fetched == null || fetched.is_empty) {
throw new ImapError.INVALID("Server returned no sequence numbers");
}
Gee.Map<UID,SequenceNumber> map = new Gee.HashMap<UID,SequenceNumber>();
foreach (SequenceNumber seq_num in fetched.keys) {
map.set(
(UID) fetched.get(seq_num).data_map.get(FetchDataSpecifier.UID),
seq_num
);
}
return map;
}