nettika/geary
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Further optimizations with network and database access.

Browse source 
These changes are to speed up GenericImapFolder.prepare_opened_folder(), which is
the initial synchronization stage to ensure the local database is normalized against
the remote folder.  The two big improvements are parallelization of database writes
(creates, removes) and skipping writes when unnecessary (if the message flags on
the server haven't changed, don't re-write them to the database).
This commit is contained in:
Jim Nelson 2011-11-17 15:21:16 -08:00
parent dcd5d2b175
commit 9982df56e2
9 changed files with 237 additions and 82 deletions
Download patch file Download diff file Expand all files Collapse all files

52
src/engine/api/geary-batch-operations.vala Executable file
View file

@ -0,0 +1,52 @@
/* Copyright 2011 Yorba Foundation
*
* This software is licensed under the GNU Lesser General Public License
* (version 2.1 or later). See the COPYING file in this distribution.
*/
/**
* CreateEmailOperation is a common Geary.NonblockingBatchOperation that can be used with
* Geary.NonblockingBatch.
*
* Note that this operation always returns null, as Geary.Folder.create_email_async() has no returned
* value.
*/
public class Geary.CreateEmailOperation : Geary.NonblockingBatchOperation {
public Geary.Folder folder { get; private set; }
public Geary.Email email { get; private set; }
public CreateEmailOperation(Geary.Folder folder, Geary.Email email) {
this.folder = folder;
this.email = email;
}
public override async Object? execute_async(Cancellable? cancellable) throws Error {
yield folder.create_email_async(email, cancellable);
return null;
}
}
/**
* RemoveEmailOperation is a common NonblockingBatchOperation that can be used with
* NonblockingBatch.
*
* Note that this operation always returns null, as Geary.Folder.remove_email_async() has no returned
* value.
*/
public class Geary.RemoveEmailOperation : Geary.NonblockingBatchOperation {
public Geary.Folder folder { get; private set; }
public Geary.EmailIdentifier email_id { get; private set; }
public RemoveEmailOperation(Geary.Folder folder, Geary.EmailIdentifier email_id) {
this.folder = folder;
this.email_id = email_id;
}
public override async Object? execute_async(Cancellable? cancellable) throws Error {
yield folder.remove_email_async(email_id, cancellable);
return null;
}
}

76
src/engine/common/common-message-data.vala
View file

@ -18,49 +18,117 @@ public abstract class Geary.Common.MessageData {
public abstract string to_string();
}
public abstract class Geary.Common.StringMessageData : Geary.Common.MessageData {
public abstract class Geary.Common.StringMessageData : Geary.Common.MessageData, Hashable, Equalable {
public string value { get; private set; }
private uint hash = uint.MAX;
public StringMessageData(string value) {
this.value = value;
}
/**
* Default definition of equals is case-sensitive comparison.
*/
public virtual bool equals(Equalable e) {
StringMessageData? other = e as StringMessageData;
if (other == null)
return false;
if (this == other)
return true;
if (to_hash() != other.to_hash())
return false;
return (value == other.value);
}
public virtual uint to_hash() {
return (hash != uint.MAX) ? hash : (hash = str_hash(value));
}
public override string to_string() {
return value;
}
}
public abstract class Geary.Common.IntMessageData : Geary.Common.MessageData {
public abstract class Geary.Common.IntMessageData : Geary.Common.MessageData, Hashable, Equalable {
public int value { get; private set; }
public IntMessageData(int value) {
this.value = value;
}
public virtual bool equals(Equalable e) {
IntMessageData? other = e as IntMessageData;
if (other == null)
return false;
if (this == other)
return true;
return (value == other.value);
}
public virtual uint to_hash() {
return int_hash(value);
}
public override string to_string() {
return value.to_string();
}
}
public abstract class Geary.Common.LongMessageData : Geary.Common.MessageData {
public abstract class Geary.Common.LongMessageData : Geary.Common.MessageData, Hashable, Equalable {
public long value { get; private set; }
public LongMessageData(long value) {
this.value = value;
}
public virtual bool equals(Equalable e) {
LongMessageData? other = e as LongMessageData;
if (other == null)
return false;
if (this == other)
return true;
return (value == other.value);
}
public virtual uint to_hash() {
return int64_hash((int64) value);
}
public override string to_string() {
return value.to_string();
}
}
public abstract class Geary.Common.Int64MessageData : Geary.Common.MessageData {
public abstract class Geary.Common.Int64MessageData : Geary.Common.MessageData, Hashable, Equalable {
public int64 value { get; private set; }
public Int64MessageData(int64 value) {
this.value = value;
}
public virtual bool equals(Equalable e) {
Int64MessageData? other = e as Int64MessageData;
if (other == null)
return false;
if (this == other)
return true;
return (value == other.value);
}
public virtual uint to_hash() {
return int64_hash(value);
}
public override string to_string() {
return value.to_string();
}

22
src/engine/imap/api/imap-email-properties.vala
View file

@ -4,7 +4,7 @@
* (version 2.1 or later). See the COPYING file in this distribution.
*/
public class Geary.Imap.EmailProperties : Geary.EmailProperties {
public class Geary.Imap.EmailProperties : Geary.EmailProperties, Equalable {
public bool answered { get; private set; }
public bool deleted { get; private set; }
public bool draft { get; private set; }
@ -31,5 +31,25 @@ public class Geary.Imap.EmailProperties : Geary.EmailProperties {
public override bool is_unread() {
return !flags.contains(MessageFlag.SEEN);
}
public bool equals(Equalable e) {
Imap.EmailProperties? other = e as Imap.EmailProperties;
if (other == null)
return false;
if (this == other)
return true;
// for simplicity and robustness, internaldate and rfc822_size must be present in both
// to be considered equal
if (internaldate == null || other.internaldate == null)
return false;
if (rfc822_size == null || other.rfc822_size == null)
return false;
return flags.equals(other.flags) && internaldate.equals(other.internaldate)
&& rfc822_size.equals(other.rfc822_size);
}
}

21
src/engine/imap/message/imap-message-data.vala
View file

@ -41,7 +41,7 @@ public class Geary.Imap.MessageNumber : Geary.Common.IntMessageData, Geary.Imap.
}
}
public abstract class Geary.Imap.Flags : Geary.Common.MessageData, Geary.Imap.MessageData {
public abstract class Geary.Imap.Flags : Geary.Common.MessageData, Geary.Imap.MessageData, Equalable {
public int size { get { return list.size; } }
private Gee.Set<Flag> list;
@ -67,6 +67,25 @@ public abstract class Geary.Imap.Flags : Geary.Common.MessageData, Geary.Imap.Me
return to_string();
}
public bool equals(Equalable e) {
Imap.Flags? other = e as Imap.Flags;
if (other == null)
return false;
if (this == other)
return true;
if (other.size != size)
return false;
foreach (Flag flag in list) {
if (!other.contains(flag))
return false;
}
return true;
}
public override string to_string() {
StringBuilder builder = new StringBuilder();
foreach (Flag flag in list) {

103
src/engine/impl/geary-generic-imap-folder.vala
View file

@ -12,6 +12,8 @@ private class Geary.GenericImapFolder : Geary.EngineFolder {
// Check if the remote folder's ordering has changed since last opened
protected override async bool prepare_opened_folder(Geary.Folder local_folder,
Geary.Folder remote_folder, Cancellable? cancellable) throws Error {
debug("prepare_opened_folder %s", to_string());
Geary.Imap.FolderProperties? local_properties =
(Geary.Imap.FolderProperties?) local_folder.get_properties();
Geary.Imap.FolderProperties? remote_properties =
@ -57,7 +59,13 @@ private class Geary.GenericImapFolder : Geary.EngineFolder {
Geary.Imap.Folder imap_remote_folder = (Geary.Imap.Folder) remote_folder;
Geary.Sqlite.Folder imap_local_folder = (Geary.Sqlite.Folder) local_folder;
// if same, no problem-o
// from here on the only operations being performed on the folder are creating or updating
// existing emails or removing them, both operations being performed using EmailIdentifiers
// rather than positional addressing ... this means the order of operation is not important
// and can be batched up rather than performed serially
NonblockingBatch batch = new NonblockingBatch();
// if same, no problem-o, move on
if (local_properties.uid_next.value != remote_properties.uid_next.value) {
debug("UID next changed for %s: %lld -> %lld", to_string(), local_properties.uid_next.value,
remote_properties.uid_next.value);
@ -65,8 +73,6 @@ private class Geary.GenericImapFolder : Geary.EngineFolder {
// fetch everything from the last seen UID (+1) to the current next UID that's not
// already in the local store (since the uidnext field isn't reported by NOOP or IDLE,
// it's possible these were fetched the last time the folder was selected)
//
// TODO: Could break this fetch up in chunks if it helps
int64 uid_start_value = local_properties.uid_next.value;
for (;;) {
Geary.EmailIdentifier start_id = new Imap.EmailIdentifier(new Imap.UID(uid_start_value));
@ -82,6 +88,8 @@ private class Geary.GenericImapFolder : Geary.EngineFolder {
break;
}
// store all the new emails' UIDs and properties (primarily flags) in the local store,
// to normalize the database against the remote folder
if (uid_start_value < remote_properties.uid_next.value) {
Geary.Imap.EmailIdentifier uid_start = new Geary.Imap.EmailIdentifier(
new Geary.Imap.UID(uid_start_value));
@ -91,15 +99,8 @@ private class Geary.GenericImapFolder : Geary.EngineFolder {
cancellable);
if (newest != null && newest.size > 0) {
debug("saving %d newest emails starting at %s in %s", newest.size, uid_start.to_string(),
to_string());
foreach (Geary.Email email in newest) {
try {
yield local_folder.create_email_async(email, cancellable);
} catch (Error newest_err) {
debug("Unable to save new email in %s: %s", to_string(), newest_err.message);
}
}
foreach (Geary.Email email in newest)
batch.add(new CreateEmailOperation(local_folder, email));
}
}
}
@ -141,7 +142,8 @@ private class Geary.GenericImapFolder : Geary.EngineFolder {
return true;
}
// Get the remote emails in the range
// Get the remote emails in the range to either add any not known, remove deleted messages,
// and update the flags of the remainder
Gee.List<Geary.Email>? old_remote = yield imap_remote_folder.list_email_by_id_async(
earliest_id, full_id_count, Geary.Email.Field.PROPERTIES, Geary.Folder.ListFlags.NONE,
cancellable);
@ -149,6 +151,7 @@ private class Geary.GenericImapFolder : Geary.EngineFolder {
int remote_ctr = 0;
int local_ctr = 0;
Gee.ArrayList<Geary.EmailIdentifier> removed_ids = new Gee.ArrayList<Geary.EmailIdentifier>();
for (;;) {
if (local_ctr >= local_length || remote_ctr >= remote_length)
break;
@ -159,66 +162,64 @@ private class Geary.GenericImapFolder : Geary.EngineFolder {
((Geary.Imap.EmailIdentifier) old_local[local_ctr].id).uid;
if (remote_uid.value == local_uid.value) {
// same, update flags and move on
try {
yield local_folder.create_email_async(old_remote[remote_ctr], cancellable);
} catch (Error update_err) {
debug("Unable to update old email in %s: %s", to_string(), update_err.message);
}
// same, update flags (if changed) and move on
Geary.Imap.EmailProperties local_email_properties =
(Geary.Imap.EmailProperties) old_local[local_ctr].properties;
Geary.Imap.EmailProperties remote_email_properties =
(Geary.Imap.EmailProperties) old_remote[remote_ctr].properties;
if (!local_email_properties.equals(remote_email_properties))
batch.add(new CreateEmailOperation(local_folder, old_remote[remote_ctr]));
remote_ctr++;
local_ctr++;
} else if (remote_uid.value < local_uid.value) {
// one we'd not seen before is present, add and move to next remote
try {
yield local_folder.create_email_async(old_remote[remote_ctr], cancellable);
} catch (Error add_err) {
debug("Unable to add new email to %s: %s", to_string(), add_err.message);
}
batch.add(new CreateEmailOperation(local_folder, old_remote[remote_ctr]));
remote_ctr++;
} else {
assert(remote_uid.value > local_uid.value);
// local's email on the server has been removed, remove locally
try {
yield local_folder.remove_email_async(old_local[local_ctr].id, cancellable);
} catch (Error remove_err) {
debug("Unable to remove discarded email from %s: %s", to_string(),
remove_err.message);
}
notify_message_removed(old_local[local_ctr].id);
batch.add(new RemoveEmailOperation(local_folder, old_local[local_ctr].id));
removed_ids.add(old_local[local_ctr].id);
local_ctr++;
}
}
// add newly-discovered emails to local store
// add newly-discovered emails to local store ... only report these as appended; earlier
// CreateEmailOperations were updates of emails existing previously or additions of emails
// that were on the server earlier but not stored locally (i.e. this value represents emails
// added to the top of the stack)
int appended = 0;
for (; remote_ctr < remote_length; remote_ctr++) {
try {
yield local_folder.create_email_async(old_remote[remote_ctr], cancellable);
appended++;
} catch (Error append_err) {
debug("Unable to append new email to %s: %s", to_string(), append_err.message);
}
batch.add(new CreateEmailOperation(local_folder, old_remote[remote_ctr]));
appended++;
}
if (appended > 0)
notify_messages_appended(appended);
// remove anything left over ... use local count rather than remote as we're still in a stage
// where only the local messages are available
for (; local_ctr < local_length; local_ctr++) {
try {
yield local_folder.remove_email_async(old_local[local_ctr].id, cancellable);
} catch (Error discard_err) {
debug("Unable to discard email from %s: %s", to_string(), discard_err.message);
}
notify_message_removed(old_local[local_ctr].id);
}
for (; local_ctr < local_length; local_ctr++)
batch.add(new RemoveEmailOperation(local_folder, old_local[local_ctr].id));
// execute them all at once
yield batch.execute_all_async(cancellable);
// throw the first exception, if one occurred
batch.throw_first_exception();
// notify emails that have been removed (see note above about why not all Creates are
// signalled)
foreach (Geary.EmailIdentifier removed_id in removed_ids)
notify_message_removed(removed_id);
// notify additions
if (appended > 0)
notify_messages_appended(appended);
debug("completed prepare_opened_folder %s", to_string());
return true;
}

2
src/engine/nonblocking/nonblocking-batch.vala
View file

@ -84,7 +84,7 @@ public class Geary.NonblockingBatch : Object {
/**
* Returns the number of NonblockingBatchOperations added.
*/
public int count {
public int size {
get { return contexts.size; }
}

40
src/engine/rfc822/rfc822-message-data.vala
View file

@ -13,31 +13,10 @@
public interface Geary.RFC822.MessageData : Geary.Common.MessageData {
}
public class Geary.RFC822.MessageID : Geary.Common.StringMessageData, Geary.RFC822.MessageData,
Geary.Equalable, Geary.Hashable {
private uint hash = 0;
public class Geary.RFC822.MessageID : Geary.Common.StringMessageData, Geary.RFC822.MessageData {
public MessageID(string value) {
base (value);
}
public bool equals(Equalable e) {
MessageID? message_id = e as MessageID;
if (message_id == null)
return false;
if (this == message_id)
return true;
if (to_hash() != message_id.to_hash())
return false;
return value == message_id.value;
}
public uint to_hash() {
return (hash != 0) ? hash : (hash = str_hash(value));
}
}
/**
@ -62,7 +41,7 @@ public class Geary.RFC822.MessageIDList : Geary.Common.StringMessageData, Geary.
}
}
public class Geary.RFC822.Date : Geary.RFC822.MessageData, Geary.Common.MessageData {
public class Geary.RFC822.Date : Geary.RFC822.MessageData, Geary.Common.MessageData, Equalable, Hashable {
public string original { get; private set; }
public DateTime value { get; private set; }
public time_t as_time_t { get; private set; }
@ -76,6 +55,21 @@ public class Geary.RFC822.Date : Geary.RFC822.MessageData, Geary.Common.MessageD
original = iso8601;
}
public virtual bool equals(Equalable e) {
RFC822.Date? other = e as RFC822.Date;
if (other == null)
return false;
if (this == other)
return true;
return value.equal(other.value);
}
public virtual uint to_hash() {
return value.hash();
}
public override string to_string() {
return original;
}

2
src/engine/sqlite/imap/sqlite-imap-message-properties-table.vala
View file

@ -45,7 +45,7 @@ public class Geary.Sqlite.ImapMessagePropertiesTable : Geary.Sqlite.Table {
cancellable);
SQLHeavy.Query query = locked.prepare(
"SELECT id, flags internaldate, rfc822_size FROM ImapMessagePropertiesTable "
"SELECT id, flags, internaldate, rfc822_size FROM ImapMessagePropertiesTable "
+ "WHERE message_id = ?");
query.bind_int64(0, message_id);

1
src/wscript
View file

@ -17,6 +17,7 @@ def build(bld):
bld.engine_src = [
'../engine/api/geary-account.vala',
'../engine/api/geary-batch-operations.vala',
'../engine/api/geary-composed-email.vala',
'../engine/api/geary-conversation.vala',
'../engine/api/geary-conversations.vala',