diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index 437df0f6..c52b2ae6 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -6,3 +6,4 @@ install(FILES version-003.sql DESTINATION ${SQL_DEST}) install(FILES version-004.sql DESTINATION ${SQL_DEST}) install(FILES version-005.sql DESTINATION ${SQL_DEST}) install(FILES version-006.sql DESTINATION ${SQL_DEST}) +install(FILES version-007.sql DESTINATION ${SQL_DEST}) diff --git a/sql/version-007.sql b/sql/version-007.sql new file mode 100644 index 00000000..0768dd05 --- /dev/null +++ b/sql/version-007.sql @@ -0,0 +1,9 @@ +-- +-- Gmail has a serious bug: its STATUS command returns a message count that includes chat messages, +-- but the SELECT/EXAMINE result codes do not. That means its difficult to confirm changes to a +-- mailbox without SELECTing it each pass. This schema modification allows for Geary to store both +-- the SELECT/EXAMINE count and STATUS count in the database for comparison. +-- + +ALTER TABLE FolderTable ADD COLUMN last_seen_status_total; + diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 181fb150..03f408f1 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -33,6 +33,7 @@ engine/api/geary-engine-error.vala engine/api/geary-engine.vala engine/api/geary-folder.vala engine/api/geary-folder-path.vala +engine/api/geary-folder-properties.vala engine/api/geary-folder-supports-archive.vala engine/api/geary-folder-supports-copy.vala engine/api/geary-folder-supports-create.vala @@ -113,8 +114,11 @@ engine/imap-db/imap-db-message-row.vala engine/imap-db/outbox/smtp-outbox-email-identifier.vala engine/imap-db/outbox/smtp-outbox-email-properties.vala engine/imap-db/outbox/smtp-outbox-folder.vala +engine/imap-db/outbox/smtp-outbox-folder-properties.vala engine/imap-db/outbox/smtp-outbox-folder-root.vala +engine/imap-engine/imap-engine.vala +engine/imap-engine/imap-engine-account-synchronizer.vala engine/imap-engine/imap-engine-batch-operations.vala engine/imap-engine/imap-engine-email-flag-watcher.vala engine/imap-engine/imap-engine-email-prefetcher.vala @@ -152,6 +156,7 @@ engine/nonblocking/nonblocking-mutex.vala engine/nonblocking/nonblocking-reporting-semaphore.vala engine/nonblocking/nonblocking-variants.vala +engine/rfc822/rfc822.vala engine/rfc822/rfc822-error.vala engine/rfc822/rfc822-gmime-filter-flowed.vala engine/rfc822/rfc822-mailbox-addresses.vala diff --git a/src/engine/abstract/geary-abstract-account.vala b/src/engine/abstract/geary-abstract-account.vala index 97881f01..72252cf3 100644 --- a/src/engine/abstract/geary-abstract-account.vala +++ b/src/engine/abstract/geary-abstract-account.vala @@ -24,6 +24,10 @@ public abstract class Geary.AbstractAccount : Object, Geary.Account { folders_added_removed(added, removed); } + protected virtual void notify_folders_contents_altered(Gee.Collection altered) { + folders_contents_altered(altered); + } + protected virtual void notify_opened() { opened(); } diff --git a/src/engine/abstract/geary-abstract-folder.vala b/src/engine/abstract/geary-abstract-folder.vala index df3973ee..1d7d3ea2 100644 --- a/src/engine/abstract/geary-abstract-folder.vala +++ b/src/engine/abstract/geary-abstract-folder.vala @@ -52,7 +52,7 @@ public abstract class Geary.AbstractFolder : Object, Geary.Folder { public abstract Geary.FolderPath get_path(); - public abstract Geary.Trillian has_children(); + public abstract Geary.FolderProperties get_properties(); public abstract Geary.SpecialFolderType get_special_folder_type(); @@ -70,9 +70,9 @@ public abstract class Geary.AbstractFolder : Object, Geary.Folder { public abstract async void open_async(bool readonly, Cancellable? cancellable = null) throws Error; - public abstract async void close_async(Cancellable? cancellable = null) throws Error; + public abstract async void wait_for_open_async(Cancellable? cancellable = null) throws Error; - public abstract async int get_email_count_async(Cancellable? cancellable = null) throws Error; + public abstract async void close_async(Cancellable? cancellable = null) throws Error; public abstract async Gee.List? list_email_async(int low, int count, Geary.Email.Field required_fields, Folder.ListFlags flags, Cancellable? cancellable = null) diff --git a/src/engine/api/geary-account.vala b/src/engine/api/geary-account.vala index e33a333f..0e4df813 100644 --- a/src/engine/api/geary-account.vala +++ b/src/engine/api/geary-account.vala @@ -38,6 +38,11 @@ public interface Geary.Account : Object { public signal void folders_added_removed(Gee.Collection? added, Gee.Collection? removed); + /** + * Fired when a Folder's contents is detected having changed. + */ + public signal void folders_contents_altered(Gee.Collection altered); + /** * Signal notification method for subclasses to use. */ @@ -70,6 +75,11 @@ public interface Geary.Account : Object { protected abstract void notify_folders_added_removed(Gee.Collection? added, Gee.Collection? removed); + /** + * Signal notification method for subclasses to use. + */ + protected abstract void notify_folders_contents_altered(Gee.Collection altered); + /** * */ diff --git a/src/engine/api/geary-conversation-monitor.vala b/src/engine/api/geary-conversation-monitor.vala index 938be060..2900a46a 100644 --- a/src/engine/api/geary-conversation-monitor.vala +++ b/src/engine/api/geary-conversation-monitor.vala @@ -364,17 +364,19 @@ public class Geary.ConversationMonitor : Object { folder.opened.connect(on_folder_opened); folder.closed.connect(on_folder_closed); - bool reseed_now = true; - if (folder.get_open_state() == Geary.Folder.OpenState.CLOSED) { - try { - yield folder.open_async(readonly, cancellable); - } catch (Error err) { - is_monitoring = false; - - throw err; - } + bool reseed_now = (folder.get_open_state() != Geary.Folder.OpenState.CLOSED); + try { + yield folder.open_async(readonly, cancellable); + } catch (Error err) { + is_monitoring = false; - reseed_now = false; + folder.email_appended.disconnect(on_folder_email_appended); + folder.email_removed.disconnect(on_folder_email_removed); + folder.email_flags_changed.disconnect(on_folder_email_flags_changed); + folder.opened.disconnect(on_folder_opened); + folder.closed.disconnect(on_folder_closed); + + throw err; } notify_monitoring_started(); diff --git a/src/engine/api/geary-email-properties.vala b/src/engine/api/geary-email-properties.vala index acd85f12..8b000614 100644 --- a/src/engine/api/geary-email-properties.vala +++ b/src/engine/api/geary-email-properties.vala @@ -9,13 +9,26 @@ * held here and retrieved via Email.Field.PROPERTIES, but as they're mutable, they were broken out * for efficiency reasons. * - * Currently EmailProperties offers nothing to clients of the Geary engine. In the future it may - * be expanded to supply details like when the message was added to the local store, checksums, - * and so forth. + * EmailProperties may be expanded in the future to supply details like when the message was added + * to the local store, checksums, and so forth. */ public abstract class Geary.EmailProperties : Object { - public EmailProperties() { + /** + * date_received may be the date/time received on the server or in the local store, depending + * on whether the information is available on the server. For example, with IMAP, this is + * the INTERNALDATE supplied by the server. + */ + public DateTime date_received { get; protected set; } + + /** + * Total size of the email (header and body) in bytes. + */ + public long total_bytes { get; protected set; } + + public EmailProperties(DateTime date_received, long total_bytes) { + this.date_received = date_received; + this.total_bytes = total_bytes; } public abstract string to_string(); diff --git a/src/engine/api/geary-email.vala b/src/engine/api/geary-email.vala index 4c01b9b8..a8ecf945 100644 --- a/src/engine/api/geary-email.vala +++ b/src/engine/api/geary-email.vala @@ -30,7 +30,8 @@ public class Geary.Email : Object { FLAGS = 1 << 9, ENVELOPE = DATE | ORIGINATORS | RECEIVERS | REFERENCES | SUBJECT, - ALL = 0xFFFFFFFF; + ALL = DATE | ORIGINATORS | RECEIVERS | REFERENCES | SUBJECT | HEADER | BODY + | PROPERTIES | PREVIEW | FLAGS; public static Field[] all() { return { @@ -382,5 +383,59 @@ public class Geary.Email : Object { public static int compare_id_descending(void* a, void *b) { return compare_id_ascending(b, a); } + + /** + * CompareFunc to sort Email by EmailProperties.total_bytes. If not available, emails are + * compared by EmailIdentifier. + */ + public static int compare_size_ascending(void *a, void *b) { + Geary.EmailProperties? aprop = (Geary.EmailProperties) ((Geary.Email *) a)->properties; + Geary.EmailProperties? bprop = (Geary.EmailProperties) ((Geary.Email *) b)->properties; + + if (aprop == null || bprop == null) + return compare_id_ascending(a, b); + + long asize = aprop.total_bytes; + long bsize = bprop.total_bytes; + + if (asize < bsize) + return -1; + else if (asize > bsize) + return 1; + else + return compare_id_ascending(a, b); + } + + /** + * CompareFunc to sort Email by EmailProperties.total_bytes. If not available, emails are + * compared by EmailIdentifier. + */ + public static int compare_size_descending(void *a, void *b) { + return compare_size_ascending(b, a); + } + + /** + * CompareFunc to sort Email by EmailProperties.date_received. If not available, emails are + * compared by EmailIdentifier. + */ + public static int compare_date_received_ascending(void *a, void *b) { + Geary.Email aemail = (Geary.Email) a; + Geary.Email bemail = (Geary.Email) b; + + if (aemail.properties == null || bemail.properties == null) + return compare_id_ascending(a, b); + + int cmp = aemail.properties.date_received.compare(bemail.properties.date_received); + + return (cmp != 0) ? cmp : compare_id_ascending(a, b); + } + + /** + * CompareFunc to sort Email by EmailProperties.date_received. If not available, emails are + * compared by EmailIdentifier. + */ + public static int compare_date_received_descending(void *a, void *b) { + return compare_date_received_ascending(b, a); + } } diff --git a/src/engine/api/geary-engine.vala b/src/engine/api/geary-engine.vala index 8c0ade4b..3f68f412 100644 --- a/src/engine/api/geary-engine.vala +++ b/src/engine/api/geary-engine.vala @@ -22,17 +22,15 @@ public class Geary.Engine { private static Engine? _instance = null; public static Engine instance { get { - if (_instance == null) - _instance = new Engine(); - - return _instance; + return (_instance != null) ? _instance : (_instance = new Engine()); } } public File? user_data_dir { get; private set; default = null; } public File? resource_dir { get; private set; default = null; } public Geary.CredentialsMediator? authentication_mediator { get; private set; default = null; } - + + private bool is_initialized = false; private bool is_open = false; private Gee.HashMap? accounts = null; private Gee.HashMap? account_instances = null; @@ -72,15 +70,29 @@ public class Geary.Engine { public signal void account_removed(AccountInformation account); private Engine() { - // Initialize GMime - GMime.init(0); } - + private void check_opened() throws EngineError { if (!is_open) throw new EngineError.OPEN_REQUIRED("Geary.Engine instance not open"); } - + + // This can't be called from within the ctor, as initialization code may want to access the + // Engine instance to make their own calls and, in particular, subscribe to signals. + // + // TODO: It would make sense to have a terminate_library() call, but it technically should not + // be called until the application is exiting, not merely if the Engine is closed, as termination + // means shutting down resources for good + private void initialize_library() { + if (is_initialized) + return; + + is_initialized = true; + + RFC822.init(); + ImapEngine.init(); + } + /** * Initializes the engine, and makes all existing accounts available. The * given authentication mediator will be used to retrieve all passwords @@ -89,9 +101,13 @@ public class Geary.Engine { public async void open_async(File user_data_dir, File resource_dir, Geary.CredentialsMediator? authentication_mediator, Cancellable? cancellable = null) throws Error { + // initialize *before* opening the Engine ... all initialize code should assume the Engine + // is closed + initialize_library(); + if (is_open) throw new EngineError.ALREADY_OPEN("Geary.Engine instance already open"); - + this.user_data_dir = user_data_dir; this.resource_dir = resource_dir; this.authentication_mediator = authentication_mediator; diff --git a/src/engine/api/geary-folder-properties.vala b/src/engine/api/geary-folder-properties.vala new file mode 100644 index 00000000..4a02f524 --- /dev/null +++ b/src/engine/api/geary-folder-properties.vala @@ -0,0 +1,44 @@ +/* Copyright 2013 Yorba Foundation + * + * This software is licensed under the GNU Lesser General Public License + * (version 2.1 or later). See the COPYING file in this distribution. + */ + +public abstract class Geary.FolderProperties : Object { + /** + * The total count of email in the Folder. + */ + public int email_total { get; protected set; } + + /** + * The total count of unread email in the Folder. + */ + public int email_unread { get; protected set; } + + /** + * Returns a Trillian indicating if this Folder has children. has_children == Trillian.TRUE + * implies supports_children == Trilian.TRUE. + */ + public Trillian has_children { get; protected set; } + + /** + * Returns a Trillian indicating if this Folder can parent new children Folders. This does + * *not* mean creating a sub-folder is guaranteed to succeed. + */ + public Trillian supports_children { get; protected set; } + + /** + * Returns a Trillian indicating if Folder.open_async() *can* succeed remotely. + */ + public Trillian is_openable { get; protected set; } + + protected FolderProperties(int email_total, int email_unread, Trillian has_children, + Trillian supports_children, Trillian is_openable) { + this.email_total = email_total; + this.email_unread = email_unread; + this.has_children = has_children; + this.supports_children = supports_children; + this.is_openable = is_openable; + } +} + diff --git a/src/engine/api/geary-folder.vala b/src/engine/api/geary-folder.vala index 7d20b518..ccb81659 100644 --- a/src/engine/api/geary-folder.vala +++ b/src/engine/api/geary-folder.vala @@ -189,7 +189,18 @@ public interface Geary.Folder : Object { public abstract Geary.FolderPath get_path(); - public abstract Geary.Trillian has_children(); + /** + * Returns a FolderProperties that represents, if fully open, accurate values for this Folder, + * and if not, values that represent the last time the Folder was opened or examined by the + * Engine. + * + * The returned object is not guaranteed to be long-lived. If the Folder's state changes, it's + * possible a new FolderProperties will be set in its place. Instead of monitoring the fields + * of the FolderProperties for changes, use Account.folders_contents_changed() to be notified + * of changes and use the (potentially new) FolderProperties returned by this method at that + * point. + */ + public abstract Geary.FolderProperties get_properties(); /** * Returns the special folder type of the folder. @@ -223,16 +234,17 @@ public interface Geary.Folder : Object { * may not reflect the full state of the Folder, however, and returned emails may subsequently * have their state changed (such as their position). Making a call that requires * accessing the remote store before OpenState.BOTH has been signalled will result in that - * call blocking until the remote is open or an error state has occurred. See list_email_async() - * for special notes on its operation. + * call blocking until the remote is open or an error state has occurred. It's also possible for + * the command to return early without waiting, depending on prior information of the folder. + * See list_email_async() for special notes on its operation. Also see wait_for_open_async(). * * If there's an error while opening, "open-failed" will be fired. (See that signal for more * information on how many times it may fire, and when.) To prevent the Folder from going into * a halfway state, it will immediately schedule a close_async() to cleanup, and those * associated signals will be fired as well. * - * If the Folder has been opened previously, EngineError.ALREADY_OPEN is thrown. There are no - * other side-effects. + * If the Folder has been opened previously, an internal open count is incremented and the + * method returns. There are no other side-effects. * * A Folder may be reopened after it has been closed. This allows for Folder objects to be * emitted by the Account object cheaply, but the client should only have a few open at a time, @@ -240,32 +252,30 @@ public interface Geary.Folder : Object { */ public abstract async void open_async(bool readonly, Cancellable? cancellable = null) throws Error; + /** + * Wait for the Folder to become fully open or fails to open due to error. If not opened + * due to error, throws EngineError.ALREADY_CLOSED. + * + * NOTE: The current implementation requirements are only that should be work after an + * open_async() call has completed (i.e. an open is in progress). Calling this method + * otherwise will throw an EngineError.OPEN_REQUIRED. + */ + public abstract async void wait_for_open_async(Cancellable? cancellable = null) throws Error; + /** * The Folder should be closed when operations on it are concluded. Depending on the * implementation this might entail closing a network connection or reverting it to another * state, or closing file handles or database connections. * + * If the Folder is open, an internal open count is decremented. If it remains above zero, the + * method returns with no other side-effects. If it decrements to zero, the Folder is closed, + * tearing down network connections, closing files, and so forth. See "closed" for signals + * indicating the closing states. + * * If the Folder is already closed, the method silently returns. */ public abstract async void close_async(Cancellable? cancellable = null) throws Error; - /** - * Returns the number of messages in the Folder. They can be addressed by their position, - * from 1 to n. - * - * Note that this only returns the number of messages available to the backing medium. In the - * case of the local store, this might differ from the number on the network server. Folders - * created by Engine are aggregating objects and will return the true count. However, this - * might require a round-trip to the server. - * - * Also note that local folders may be sparsely populated. get_email_count_async() returns the - * total number of recorded emails, but it's possible none of them have more than placeholder - * information. - * - * The Folder must be opened prior to attempting this operation. - */ - public abstract async int get_email_count_async(Cancellable? cancellable = null) throws Error; - /** * Returns a list of messages that fulfill the required_fields flags starting at the low * position and moving up to (low + count). If count is -1, the returned list starts at low diff --git a/src/engine/db/db-connection.vala b/src/engine/db/db-connection.vala index 05f3b415..9dec4f29 100644 --- a/src/engine/db/db-connection.vala +++ b/src/engine/db/db-connection.vala @@ -71,6 +71,8 @@ public class Geary.Db.Connection : Geary.Db.Context { check_cancelled("Connection.ctor", cancellable); + // TODO: open_v2() can return a database connection even when an error is returned; the + // database must still be closed in this case throw_on_error("Connection.ctor", Sqlite.Database.open_v2(database.db_file.get_path(), out db, sqlite_flags, null)); } diff --git a/src/engine/imap-db/imap-db-account.vala b/src/engine/imap-db/imap-db-account.vala index bef1b9be..86b8b994 100644 --- a/src/engine/imap-db/imap-db-account.vala +++ b/src/engine/imap-db/imap-db-account.vala @@ -115,16 +115,17 @@ private class Geary.ImapDB.Account : Object { // create the folder object Db.Statement stmt = cx.prepare( - "INSERT INTO FolderTable (name, parent_id, last_seen_total, uid_validity, uid_next, attributes) " - + "VALUES (?, ?, ?, ?, ?, ?)"); + "INSERT INTO FolderTable (name, parent_id, last_seen_total, last_seen_status_total, " + + "uid_validity, uid_next, attributes) VALUES (?, ?, ?, ?, ?, ?)"); stmt.bind_string(0, path.basename); stmt.bind_rowid(1, parent_id); - stmt.bind_int(2, properties.messages); - stmt.bind_int64(3, (properties.uid_validity != null) ? properties.uid_validity.value + stmt.bind_int(2, Numeric.int_floor(properties.select_examine_messages, 0)); + stmt.bind_int(3, Numeric.int_floor(properties.status_messages, 0)); + stmt.bind_int64(4, (properties.uid_validity != null) ? properties.uid_validity.value : Imap.UIDValidity.INVALID); - stmt.bind_int64(4, (properties.uid_next != null) ? properties.uid_next.value + stmt.bind_int64(5, (properties.uid_next != null) ? properties.uid_next.value : Imap.UID.INVALID); - stmt.bind_string(5, properties.attrs.serialize()); + stmt.bind_string(6, properties.attrs.serialize()); stmt.exec(cancellable); @@ -136,11 +137,7 @@ private class Geary.ImapDB.Account : Object { throws Error { check_open(); - Geary.Imap.FolderProperties? properties = (Geary.Imap.FolderProperties?) imap_folder.get_properties(); - - // properties *must* be available - assert(properties != null); - + Geary.Imap.FolderProperties properties = imap_folder.get_properties(); Geary.FolderPath path = imap_folder.get_path(); yield db.exec_transaction_async(Db.TransactionType.RW, (cx) => { @@ -154,31 +151,39 @@ private class Geary.ImapDB.Account : Object { Db.Statement stmt; if (parent_id != Db.INVALID_ROWID) { stmt = cx.prepare( - "UPDATE FolderTable SET last_seen_total=?, uid_validity=?, uid_next=?, attributes=? " + "UPDATE FolderTable SET uid_validity=?, uid_next=?, attributes=? " + "WHERE parent_id=? AND name=?"); - stmt.bind_int(0, properties.messages); - stmt.bind_int64(1, (properties.uid_validity != null) ? properties.uid_validity.value + stmt.bind_int64(0, (properties.uid_validity != null) ? properties.uid_validity.value : Imap.UIDValidity.INVALID); - stmt.bind_int64(2, (properties.uid_next != null) ? properties.uid_next.value + stmt.bind_int64(1, (properties.uid_next != null) ? properties.uid_next.value : Imap.UID.INVALID); - stmt.bind_string(3, properties.attrs.serialize()); - stmt.bind_rowid(4, parent_id); - stmt.bind_string(5, path.basename); + stmt.bind_string(2, properties.attrs.serialize()); + stmt.bind_rowid(3, parent_id); + stmt.bind_string(4, path.basename); } else { stmt = cx.prepare( - "UPDATE FolderTable SET last_seen_total=?, uid_validity=?, uid_next=?, attributes=? " + "UPDATE FolderTable SET uid_validity=?, uid_next=?, attributes=? " + "WHERE parent_id IS NULL AND name=?"); - stmt.bind_int(0, properties.messages); - stmt.bind_int64(1, (properties.uid_validity != null) ? properties.uid_validity.value + stmt.bind_int64(0, (properties.uid_validity != null) ? properties.uid_validity.value : Imap.UIDValidity.INVALID); - stmt.bind_int64(2, (properties.uid_next != null) ? properties.uid_next.value + stmt.bind_int64(1, (properties.uid_next != null) ? properties.uid_next.value : Imap.UID.INVALID); - stmt.bind_string(3, properties.attrs.serialize()); - stmt.bind_string(4, path.basename); + stmt.bind_string(2, properties.attrs.serialize()); + stmt.bind_string(3, path.basename); } stmt.exec(); + if (properties.select_examine_messages >= 0) { + do_update_last_seen_total(cx, parent_id, path.basename, properties.select_examine_messages, + cancellable); + } + + if (properties.status_messages >= 0) { + do_update_last_seen_status_total(cx, parent_id, path.basename, properties.status_messages, + cancellable); + } + return Db.TransactionOutcome.COMMIT; }, cancellable); @@ -247,12 +252,12 @@ private class Geary.ImapDB.Account : Object { Db.Statement stmt; if (parent_id != Db.INVALID_ROWID) { stmt = cx.prepare( - "SELECT id, name, last_seen_total, uid_validity, uid_next, attributes " + "SELECT id, name, last_seen_total, last_seen_status_total, uid_validity, uid_next, attributes " + "FROM FolderTable WHERE parent_id=?"); stmt.bind_rowid(0, parent_id); } else { stmt = cx.prepare( - "SELECT id, name, last_seen_total, uid_validity, uid_next, attributes " + "SELECT id, name, last_seen_total, last_seen_status_total, uid_validity, uid_next, attributes " + "FROM FolderTable WHERE parent_id IS NULL"); } @@ -268,6 +273,7 @@ private class Geary.ImapDB.Account : Object { new Imap.UIDValidity(result.int64_for("uid_validity")), new Imap.UID(result.int64_for("uid_next")), Geary.Imap.MailboxAttributes.deserialize(result.string_for("attributes"))); + properties.set_status_message_count(result.int_for("last_seen_status_total")); id_map.set(path, result.rowid_for("id")); prop_map.set(path, properties); @@ -288,7 +294,7 @@ private class Geary.ImapDB.Account : Object { Gee.Collection folders = new Gee.ArrayList(); foreach (Geary.FolderPath path in id_map.keys) { Geary.ImapDB.Folder? folder = get_local_folder(path); - if (folder == null) + if (folder == null && id_map.has_key(path) && prop_map.has_key(path)) folder = create_local_folder(path, id_map.get(path), prop_map.get(path)); folders.add(folder); @@ -342,7 +348,8 @@ private class Geary.ImapDB.Account : Object { return Db.TransactionOutcome.DONE; Db.Statement stmt = cx.prepare( - "SELECT last_seen_total, uid_validity, uid_next, attributes FROM FolderTable WHERE id=?"); + "SELECT last_seen_total, last_seen_status_total, uid_validity, uid_next, attributes " + + "FROM FolderTable WHERE id=?"); stmt.bind_rowid(0, folder_id); Db.Result results = stmt.exec(cancellable); @@ -351,12 +358,13 @@ private class Geary.ImapDB.Account : Object { new Imap.UIDValidity(results.int64_for("uid_validity")), new Imap.UID(results.int64_for("uid_next")), Geary.Imap.MailboxAttributes.deserialize(results.string_for("attributes"))); + properties.set_status_message_count(results.int_for("last_seen_status_total")); } return Db.TransactionOutcome.DONE; }, cancellable); - if (folder_id == Db.INVALID_ROWID) + if (folder_id == Db.INVALID_ROWID || properties == null) throw new EngineError.NOT_FOUND("%s not found in local database", path.to_string()); return create_local_folder(path, folder_id, properties); @@ -369,13 +377,12 @@ private class Geary.ImapDB.Account : Object { } private Geary.ImapDB.Folder create_local_folder(Geary.FolderPath path, int64 folder_id, - Imap.FolderProperties? properties) throws Error { + Imap.FolderProperties properties) throws Error { // return current if already created ImapDB.Folder? folder = get_local_folder(path); if (folder != null) { - // update properties if available - if (properties != null) - folder.set_properties(properties); + // update properties + folder.set_properties(properties); return folder; } @@ -522,5 +529,34 @@ private class Geary.ImapDB.Account : Object { return do_fetch_folder_id(cx, path.get_parent(), create, out parent_id, cancellable); } + + private void do_update_last_seen_total(Db.Connection cx, int64 parent_id, string name, int total, + Cancellable? cancellable) throws Error { + do_update_total(cx, parent_id, name, "last_seen_total", total, cancellable); + } + + private void do_update_last_seen_status_total(Db.Connection cx, int64 parent_id, string name, + int total, Cancellable? cancellable) throws Error { + do_update_total(cx, parent_id, name, "last_seen_status_total", total, cancellable); + } + + private void do_update_total(Db.Connection cx, int64 parent_id, string name, string colname, + int total, Cancellable? cancellable) throws Error { + Db.Statement stmt; + if (parent_id != Db.INVALID_ROWID) { + stmt = cx.prepare( + "UPDATE FolderTable SET %s=? WHERE parent_id=? AND name=?".printf(colname)); + stmt.bind_int(0, Numeric.int_floor(total, 0)); + stmt.bind_rowid(1, parent_id); + stmt.bind_string(2, name); + } else { + stmt = cx.prepare( + "UPDATE FolderTable SET %s=? WHERE parent_id IS NULL AND name=?".printf(colname)); + stmt.bind_int(0, Numeric.int_floor(total, 0)); + stmt.bind_string(1, name); + } + + stmt.exec(cancellable); + } } diff --git a/src/engine/imap-db/imap-db-folder.vala b/src/engine/imap-db/imap-db-folder.vala index 8fdd0d60..405bf17c 100644 --- a/src/engine/imap-db/imap-db-folder.vala +++ b/src/engine/imap-db/imap-db-folder.vala @@ -58,12 +58,12 @@ private class Geary.ImapDB.Folder : Object, Geary.ReferenceSemantics { private ContactStore contact_store; private string account_owner_email; private int64 folder_id; - private Geary.Imap.FolderProperties? properties; + private Geary.Imap.FolderProperties properties; private Gee.HashSet marked_removed = new Gee.HashSet( Hashable.hash_func, Equalable.equal_func); - + internal Folder(ImapDB.Database db, Geary.FolderPath path, ContactStore contact_store, - string account_owner_email, int64 folder_id, Geary.Imap.FolderProperties? properties) { + string account_owner_email, int64 folder_id, Geary.Imap.FolderProperties properties) { assert(folder_id != Db.INVALID_ROWID); this.db = db; @@ -83,12 +83,11 @@ private class Geary.ImapDB.Folder : Object, Geary.ReferenceSemantics { return path; } - public Geary.Imap.FolderProperties? get_properties() { - // TODO: TBD: alteration/updated signals for folders + public Geary.Imap.FolderProperties get_properties() { return properties; } - internal void set_properties(Geary.Imap.FolderProperties? properties) { + internal void set_properties(Geary.Imap.FolderProperties properties) { this.properties = properties; } @@ -170,9 +169,34 @@ private class Geary.ImapDB.Folder : Object, Geary.ReferenceSemantics { // Updates both the FolderProperties and the value in the local store. Must be called while // open. - public async void update_remote_message_count(int count, Cancellable? cancellable) throws Error { + public async void update_remote_status_message_count(int count, Cancellable? cancellable) throws Error { check_open(); + if (count < 0) + return; + + yield db.exec_transaction_async(Db.TransactionType.RW, (cx) => { + Db.Statement stmt = cx.prepare( + "UPDATE FolderTable SET last_seen_status_total=? WHERE id=?"); + stmt.bind_int(0, Numeric.int_floor(count, 0)); + stmt.bind_rowid(1, folder_id); + + stmt.exec(cancellable); + + return Db.TransactionOutcome.COMMIT; + }, cancellable); + + properties.set_status_message_count(count); + } + + // Updates both the FolderProperties and the value in the local store. Must be called while + // open. + public async void update_remote_selected_message_count(int count, Cancellable? cancellable) throws Error { + check_open(); + + if (count < 0) + return; + yield db.exec_transaction_async(Db.TransactionType.RW, (cx) => { Db.Statement stmt = cx.prepare( "UPDATE FolderTable SET last_seen_total=? WHERE id=?"); @@ -184,8 +208,7 @@ private class Geary.ImapDB.Folder : Object, Geary.ReferenceSemantics { return Db.TransactionOutcome.COMMIT; }, cancellable); - if (properties != null) - properties.messages = count; + properties.set_select_examine_message_count(count); } public async int get_id_position_async(Geary.EmailIdentifier id, ListFlags flags, @@ -231,9 +254,24 @@ private class Geary.ImapDB.Folder : Object, Geary.ReferenceSemantics { return results; } + // NOTE: This can be used to check local messages without opening the folder, useful since + // opening a Geary.Folder implies remote connection ... this skips check_open() (and, by + // implication, means the ImapDB.Folder can be in an odd state), so USE CAREFULLY. + public async Gee.List? local_list_email_async(int low, int count, + Geary.Email.Field required_fields, ListFlags flags, Cancellable? cancellable) throws Error { + return yield internal_list_email_async(low, count, required_fields, flags, true, cancellable); + } + public async Gee.List? list_email_async(int low, int count, Geary.Email.Field required_fields, ListFlags flags, Cancellable? cancellable) throws Error { - check_open(); + return yield internal_list_email_async(low, count, required_fields, flags, false, cancellable); + } + + private async Gee.List? internal_list_email_async(int low, int count, + Geary.Email.Field required_fields, ListFlags flags, bool skip_open_check, + Cancellable? cancellable) throws Error { + if (!skip_open_check) + check_open(); // TODO: A more efficient way to do this would be to pull in all the columns at once in // a single SELECT operation ... this might be less efficient than current practice if @@ -704,8 +742,11 @@ private class Geary.ImapDB.Folder : Object, Geary.ReferenceSemantics { ? imap_properties.internaldate.original : null; long rfc822_size = (imap_properties != null) ? imap_properties.rfc822_size.value : -1; - if (String.is_empty(internaldate) || rfc822_size < 0) + if (String.is_empty(internaldate) || rfc822_size < 0) { + debug("Unable to detect duplicates for %s (%s available but invalid)", email.id.to_string(), + email.fields.to_list_string()); return Db.INVALID_ROWID; + } // look for duplicate in IMAP message properties Db.Statement stmt = cx.prepare( diff --git a/src/engine/imap-db/imap-db-message-row.vala b/src/engine/imap-db/imap-db-message-row.vala index b8862d28..4f9f9cd0 100644 --- a/src/engine/imap-db/imap-db-message-row.vala +++ b/src/engine/imap-db/imap-db-message-row.vala @@ -136,8 +136,11 @@ public class Geary.ImapDB.MessageRow { if (fields.is_all_set(Geary.Email.Field.FLAGS)) email.set_flags(get_generic_email_flags()); - if (fields.is_all_set(Geary.Email.Field.PROPERTIES)) - email.set_email_properties(get_imap_email_properties()); + if (fields.is_all_set(Geary.Email.Field.PROPERTIES)) { + Imap.EmailProperties? properties = get_imap_email_properties(); + if (properties != null) + email.set_email_properties(properties); + } return email; } diff --git a/src/engine/imap-db/outbox/smtp-outbox-email-properties.vala b/src/engine/imap-db/outbox/smtp-outbox-email-properties.vala index 76ff7d61..0cdf238a 100644 --- a/src/engine/imap-db/outbox/smtp-outbox-email-properties.vala +++ b/src/engine/imap-db/outbox/smtp-outbox-email-properties.vala @@ -5,7 +5,8 @@ */ private class Geary.SmtpOutboxEmailProperties : Geary.EmailProperties { - public SmtpOutboxEmailProperties() { + public SmtpOutboxEmailProperties(DateTime date_received, long total_bytes) { + base(date_received, total_bytes); } public override string to_string() { diff --git a/src/engine/imap-db/outbox/smtp-outbox-folder-properties.vala b/src/engine/imap-db/outbox/smtp-outbox-folder-properties.vala new file mode 100644 index 00000000..5e665b2f --- /dev/null +++ b/src/engine/imap-db/outbox/smtp-outbox-folder-properties.vala @@ -0,0 +1,16 @@ +/* Copyright 2013 Yorba Foundation + * + * This software is licensed under the GNU Lesser General Public License + * (version 2.1 or later). See the COPYING file in this distribution. + */ + +private class Geary.SmtpOutboxFolderProperties : Geary.FolderProperties { + public SmtpOutboxFolderProperties(int total, int unread) { + base (total, unread, Trillian.FALSE, Trillian.FALSE, Trillian.TRUE); + } + + public void set_total(int total) { + this.email_total = total; + } +} + diff --git a/src/engine/imap-db/outbox/smtp-outbox-folder.vala b/src/engine/imap-db/outbox/smtp-outbox-folder.vala index a8f497a9..70362037 100644 --- a/src/engine/imap-db/outbox/smtp-outbox-folder.vala +++ b/src/engine/imap-db/outbox/smtp-outbox-folder.vala @@ -43,8 +43,9 @@ private class Geary.SmtpOutboxFolder : Geary.AbstractFolder, Geary.FolderSupport private ImapDB.Database db; private weak Account _account; private Geary.Smtp.ClientSession smtp; - private bool opened = false; + private int open_count = 0; private NonblockingMailbox outbox_queue = new NonblockingMailbox(); + private SmtpOutboxFolderProperties properties = new SmtpOutboxFolderProperties(0, 0); public override Account account { get { return _account; } } @@ -89,6 +90,9 @@ private class Geary.SmtpOutboxFolder : Geary.AbstractFolder, Geary.FolderSupport }, null); if (list.size > 0) { + // set properties now (can't do yield in ctor) + properties.set_total(list.size); + debug("Priming outbox postman with %d stored messages", list.size); foreach (OutboxRow row in list) outbox_queue.send(row); @@ -128,11 +132,7 @@ private class Geary.SmtpOutboxFolder : Geary.AbstractFolder, Geary.FolderSupport } catch (Error send_err) { debug("Outbox postman send error, retrying: %s", send_err.message); - try { - outbox_queue.send(row); - } catch (Error send_err) { - debug("Outbox postman: Unable to re-enqueue message, dropping on floor: %s", send_err.message); - } + outbox_queue.send(row); if (send_err is SmtpError.AUTHENTICATION_FAILED) { bool report = true; @@ -167,7 +167,15 @@ private class Geary.SmtpOutboxFolder : Geary.AbstractFolder, Geary.FolderSupport } catch (Error rm_err) { debug("Outbox postman: Unable to remove row from database: %s", rm_err.message); } - + + // update properties + try { + properties.set_total(yield get_email_count_async(null)); + } catch (Error err) { + debug("Outbox postman: Unable to fetch updated email count for properties: %s", + err.message); + } + // If we got this far the send was successful, so reset the send retry interval. send_retry_seconds = MIN_SEND_RETRY_INTERVAL_SEC; } @@ -182,8 +190,8 @@ private class Geary.SmtpOutboxFolder : Geary.AbstractFolder, Geary.FolderSupport return path; } - public override Geary.Trillian has_children() { - return Geary.Trillian.FALSE; + public override Geary.FolderProperties get_properties() { + return properties; } public override Geary.SpecialFolderType get_special_folder_type() { @@ -191,36 +199,36 @@ private class Geary.SmtpOutboxFolder : Geary.AbstractFolder, Geary.FolderSupport } public override Geary.Folder.OpenState get_open_state() { - return opened ? Geary.Folder.OpenState.LOCAL : Geary.Folder.OpenState.CLOSED; + return open_count > 0 ? Geary.Folder.OpenState.LOCAL : Geary.Folder.OpenState.CLOSED; } private void check_open() throws EngineError { - if (!opened) + if (open_count == 0) throw new EngineError.OPEN_REQUIRED("%s not open", to_string()); } + public override async void wait_for_open_async(Cancellable? cancellable = null) throws Error { + if (open_count == 0) + throw new EngineError.OPEN_REQUIRED("Outbox not open"); + } + public override async void open_async(bool readonly, Cancellable? cancellable = null) throws Error { - if (opened) - throw new EngineError.ALREADY_OPEN("Folder %s already open", to_string()); + if (open_count++ > 0) + return; - opened = true; - notify_opened(Geary.Folder.OpenState.LOCAL, yield get_email_count_async(cancellable)); + notify_opened(Geary.Folder.OpenState.LOCAL, properties.email_total); } public override async void close_async(Cancellable? cancellable = null) throws Error { - if (!opened) + if (open_count == 0 || --open_count > 0) return; - opened = false; - notify_closed(Geary.Folder.CloseReason.LOCAL_CLOSE); notify_closed(Geary.Folder.CloseReason.FOLDER_CLOSED); } - public override async int get_email_count_async(Cancellable? cancellable = null) throws Error { - check_open(); - + private async int get_email_count_async(Cancellable? cancellable) throws Error { int count = 0; yield db.exec_transaction_async(Db.TransactionType.RO, (cx) => { count = do_get_email_count(cx, cancellable); @@ -267,11 +275,14 @@ private class Geary.SmtpOutboxFolder : Geary.AbstractFolder, Geary.FolderSupport // should have thrown an error if this failed assert(row != null); + // update properties + properties.set_total(yield get_email_count_async(cancellable)); + // immediately add to outbox queue for delivery outbox_queue.send(row); // notify only if opened - if (opened) { + if (open_count > 0) { Gee.List list = new Gee.ArrayList(); list.add(row.outbox_id); @@ -479,7 +490,7 @@ private class Geary.SmtpOutboxFolder : Geary.AbstractFolder, Geary.FolderSupport return false; // notify only if opened - if (opened) { + if (open_count > 0) { notify_email_removed(removed); notify_email_count_changed(final_count, CountChangeReason.REMOVED); } @@ -500,7 +511,8 @@ private class Geary.SmtpOutboxFolder : Geary.AbstractFolder, Geary.FolderSupport RFC822.Message message = new RFC822.Message.from_string(row.message); Geary.Email email = message.get_email(row.position, row.outbox_id); - email.set_email_properties(new SmtpOutboxEmailProperties()); + // TODO: Determine message's total size (header + body) to store in Properties. + email.set_email_properties(new SmtpOutboxEmailProperties(new DateTime.now_local(), -1)); email.set_flags(new Geary.EmailFlags()); return email; diff --git a/src/engine/imap-engine/imap-engine-account-synchronizer.vala b/src/engine/imap-engine/imap-engine-account-synchronizer.vala new file mode 100644 index 00000000..9138cdb3 --- /dev/null +++ b/src/engine/imap-engine/imap-engine-account-synchronizer.vala @@ -0,0 +1,324 @@ +/* Copyright 2013 Yorba Foundation + * + * This software is licensed under the GNU Lesser General Public License + * (version 2.1 or later). See the COPYING file in this distribution. + */ + +private class Geary.ImapEngine.AccountSynchronizer { + private const int SYNC_DEPTH_DAYS = 15; + private const int FETCH_DATE_RECEIVED_CHUNK_COUNT = 25; + + public GenericAccount account { get; private set; } + + private NonblockingMailbox? bg_queue = null; + private Gee.HashSet made_available = new Gee.HashSet(); + private Cancellable? bg_cancellable = null; + private NonblockingSemaphore stopped = new NonblockingSemaphore(); + private NonblockingSemaphore prefetcher_semaphore = new NonblockingSemaphore(); + + public AccountSynchronizer(GenericAccount account) { + this.account = account; + + account.opened.connect(on_account_opened); + account.closed.connect(on_account_closed); + account.folders_available_unavailable.connect(on_folders_available_unavailable); + account.folders_contents_altered.connect(on_folders_contents_altered); + } + + ~AccountSynchronizer() { + account.opened.disconnect(on_account_opened); + account.closed.disconnect(on_account_closed); + account.folders_available_unavailable.disconnect(on_folders_available_unavailable); + account.folders_contents_altered.disconnect(on_folders_contents_altered); + } + + public async void stop_async() { + bg_cancellable.cancel(); + + try { + yield stopped.wait_async(); + } catch (Error err) { + debug("Error waiting for AccountSynchronizer background task for %s to complete: %s", + account.to_string(), err.message); + } + } + + private void on_account_opened() { + if (stopped.is_passed()) + return; + + bg_queue = new NonblockingMailbox(bg_queue_comparator); + bg_queue.allow_duplicates = false; + bg_queue.requeue_duplicate = false; + bg_cancellable = new Cancellable(); + + // immediately start processing folders as they are announced as available + process_queue_async.begin(); + } + + private void on_account_closed() { + bg_cancellable.cancel(); + bg_queue.clear(); + } + + private void on_folders_available_unavailable(Gee.Collection? available, + Gee.Collection? unavailable) { + if (stopped.is_passed()) + return; + + if (available != null) + send_all(available, true); + + if (unavailable != null) + revoke_all(unavailable); + } + + private void on_folders_contents_altered(Gee.Collection altered) { + send_all(altered, false); + } + + private void send_all(Gee.Collection folders, bool reason_available) { + foreach (Folder folder in folders) { + GenericFolder? generic_folder = folder as GenericFolder; + if (generic_folder != null) + bg_queue.send(generic_folder); + + // If adding because now available, make sure it's flagged as such, since there's an + // additional check for available folders ... if not, remove from the map so it's + // not treated as such, in case both of these come in back-to-back + if (reason_available) + made_available.add(generic_folder); + else + made_available.remove(generic_folder); + } + } + + private void revoke_all(Gee.Collection folders) { + foreach (Folder folder in folders) { + GenericFolder? generic_folder = folder as GenericFolder; + if (generic_folder != null) { + bg_queue.revoke(generic_folder); + made_available.remove(generic_folder); + } + } + } + + // This is used to ensure that certain special folders get prioritized over others, so folders + // important to the user (i.e. Inbox) and folders handy for pulling all mail (i.e. All Mail) go + // first while less-used folders (Trash, Spam) are fetched last + private static int bg_queue_comparator(GenericFolder a, GenericFolder b) { + if (a == b) + return 0; + + int cmp = score_folder(a) - score_folder(b); + if (cmp != 0) + return cmp; + + // sort by path to stabilize the sort + return a.get_path().compare(b.get_path()); + } + + // Lower the score, the higher the importance. + private static int score_folder(Folder a) { + switch (a.get_special_folder_type()) { + case SpecialFolderType.INBOX: + return -60; + + case SpecialFolderType.ALL_MAIL: + return -50; + + case SpecialFolderType.SENT: + return -40; + + case SpecialFolderType.FLAGGED: + return -30; + + case SpecialFolderType.IMPORTANT: + return -20; + + case SpecialFolderType.DRAFTS: + return -10; + + case SpecialFolderType.SPAM: + return 10; + + case SpecialFolderType.TRASH: + return 20; + + default: + return 0; + } + } + + private async void process_queue_async() { + for (;;) { + GenericFolder folder; + try { + folder = yield bg_queue.recv_async(bg_cancellable); + } catch (Error err) { + if (!(err is IOError.CANCELLED)) + debug("Failed to receive next folder for background sync: %s", err.message); + + break; + } + + // generate the current epoch for synchronization (could cache this value, obviously, but + // doesn't seem like this biggest win in this class) + DateTime epoch = new DateTime.now_local(); + epoch = epoch.add_days(0 - SYNC_DEPTH_DAYS); + + if (!yield process_folder_async(folder, made_available.remove(folder), epoch)) + break; + } + + // clear queue of any remaining folders so references aren't held + bg_queue.clear(); + + // same with made_available table + made_available.clear(); + + // flag as stopped for any waiting tasks + stopped.blind_notify(); + } + + // Returns false if IOError.CANCELLED received + private async bool process_folder_async(GenericFolder folder, bool availability_check, DateTime epoch) { + if (availability_check) { + // Fetch the oldest mail in the local store and see if it is before the epoch; if so, no + // need to synchronize simply because this Folder is available; wait for its contents to + // change instead + Gee.List? oldest_local = null; + try { + oldest_local = yield folder.local_folder.local_list_email_async(1, 1, + Email.Field.PROPERTIES, ImapDB.Folder.ListFlags.NONE, bg_cancellable); + } catch (Error err) { + debug("Unable to fetch oldest local email for %s: %s", folder.to_string(), err.message); + } + + if (oldest_local != null && oldest_local.size > 0) { + if (oldest_local[0].properties.date_received.compare(epoch) < 0) { + debug("Oldest local email in %s before epoch, don't sync from network", folder.to_string()); + + return true; + } else { + debug("Oldest local email in %s not old enough (%s), synchronizing...", folder.to_string(), + oldest_local[0].properties.date_received.to_string()); + } + } else if (folder.get_properties().email_total == 0) { + // no local messages, no remote messages -- this is as good as having everything up + // to the epoch + debug("No messages in local or remote folder %s, don't sync from network", + folder.to_string()); + + return true; + } else { + debug("No oldest message found for %s, synchronizing...", folder.to_string()); + } + } + + try { + yield folder.open_async(true, bg_cancellable); + yield folder.wait_for_open_async(bg_cancellable); + } catch (Error err) { + // don't need to close folder; if either calls throws an error, the folder is not open + if (err is IOError.CANCELLED) + return false; + + debug("Unable to open %s: %s", folder.to_string(), err.message); + + return true; + } + + // set up monitoring the Folder's prefetcher so an exception doesn't leave dangling + // signal subscriptions + prefetcher_semaphore = new NonblockingSemaphore(); + folder.email_prefetcher.halting.connect(on_email_prefetcher_completed); + folder.closed.connect(on_email_prefetcher_completed); + + try { + yield sync_folder_async(folder, epoch); + } catch (Error err) { + if (err is IOError.CANCELLED) + return false; + + debug("Error background syncing folder %s: %s", folder.to_string(), err.message); + + // fallthrough and close + } finally { + folder.email_prefetcher.halting.disconnect(on_email_prefetcher_completed); + folder.closed.disconnect(on_email_prefetcher_completed); + } + + try { + // don't pass Cancellable; really need this to complete in all cases + yield folder.close_async(); + } catch (Error err) { + debug("Error closing %s: %s", folder.to_string(), err.message); + } + + return true; + } + + private async void sync_folder_async(GenericFolder folder, DateTime epoch) throws Error { + debug("Background sync'ing %s", folder.to_string()); + + // TODO: This could be done in a single IMAP SEARCH command, as INTERNALDATE may be searched + // upon (returning all messages that fit the criteria). For now, simply iterating backward + // in the folder until the oldest is found, then pulling the email down in chunks + int low = -1; + int count = FETCH_DATE_RECEIVED_CHUNK_COUNT; + for (;;) { + Gee.List? list = yield folder.list_email_async(low, count, Geary.Email.Field.PROPERTIES, + Folder.ListFlags.NONE, bg_cancellable); + if (list == null || list.size == 0) + break; + + // sort these by their received date so they're walked in order + Gee.TreeSet sorted_list = new Gee.TreeSet(Email.compare_date_received_descending); + sorted_list.add_all(list); + + // look for any that are older than epoch and bail out if found + bool found = false; + int lowest = int.MAX; + foreach (Email email in sorted_list) { + if (email.properties.date_received.compare(epoch) < 0) { + debug("Found epoch for %s at %s (%s)", folder.to_string(), email.id.to_string(), + email.properties.date_received.to_string()); + + found = true; + + break; + } + + // find lowest position for next round of fetching + if (email.position < lowest) + lowest = email.position; + } + + if (found || low == 1) + break; + + low = Numeric.int_floor(lowest - FETCH_DATE_RECEIVED_CHUNK_COUNT, 1); + count = (lowest - low).clamp(1, FETCH_DATE_RECEIVED_CHUNK_COUNT); + } + + if (folder.email_prefetcher.has_work()) { + // expanding an already opened folder doesn't guarantee the prefetcher will start + debug("Waiting for email prefetcher to complete %s...", folder.to_string()); + try { + yield prefetcher_semaphore.wait_async(bg_cancellable); + } catch (Error err) { + debug("Error waiting for email prefetcher to complete %s: %s", folder.to_string(), + err.message); + } + } + + debug("Done background sync'ing %s", folder.to_string()); + } + + private void on_email_prefetcher_completed() { + debug("on_email_prefetcher_completed"); + prefetcher_semaphore.blind_notify(); + } +} + diff --git a/src/engine/imap-engine/imap-engine-email-prefetcher.vala b/src/engine/imap-engine/imap-engine-email-prefetcher.vala index e2ca7dfc..5ebfe724 100644 --- a/src/engine/imap-engine/imap-engine-email-prefetcher.vala +++ b/src/engine/imap-engine/imap-engine-email-prefetcher.vala @@ -12,7 +12,9 @@ * The EmailPrefetcher does not maintain a reference to the folder. */ public class Geary.ImapEngine.EmailPrefetcher : Object { - public const int PREFETCH_DELAY_SEC = 5; + public const int PREFETCH_DELAY_SEC = 1; + + private const Geary.Email.Field PREFETCH_FIELDS = Geary.Email.Field.ALL; private unowned Geary.Folder folder; private int start_delay_sec; @@ -22,6 +24,8 @@ public class Geary.ImapEngine.EmailPrefetcher : Object { private uint schedule_id = 0; private Cancellable cancellable = new Cancellable(); + public signal void halting(); + public EmailPrefetcher(Geary.Folder folder, int start_delay_sec = PREFETCH_DELAY_SEC) { assert(start_delay_sec > 0); @@ -42,12 +46,16 @@ public class Geary.ImapEngine.EmailPrefetcher : Object { folder.email_locally_appended.disconnect(on_locally_appended); } + public bool has_work() { + return prefetch_ids.size > 0; + } + private void on_opened(Geary.Folder.OpenState open_state) { if (open_state != Geary.Folder.OpenState.BOTH) return; cancellable = new Cancellable(); - schedule_prefetch_all(); + schedule_prefetch_all_local(); } private void on_closed(Geary.Folder.CloseReason close_reason) { @@ -65,9 +73,9 @@ public class Geary.ImapEngine.EmailPrefetcher : Object { schedule_prefetch(ids); } - private void schedule_prefetch_all() { + private void schedule_prefetch_all_local() { // Async method will schedule prefetch once ids are known - do_prefetch_all.begin(); + do_prefetch_all_local.begin(); } private void schedule_prefetch(Gee.Collection ids) { @@ -76,7 +84,7 @@ public class Geary.ImapEngine.EmailPrefetcher : Object { if (schedule_id != 0) Source.remove(schedule_id); - schedule_id = Timeout.add_seconds(start_delay_sec, on_start_prefetch, Priority.LOW); + schedule_id = Timeout.add_seconds(start_delay_sec, on_start_prefetch); } private bool on_start_prefetch() { @@ -87,7 +95,7 @@ public class Geary.ImapEngine.EmailPrefetcher : Object { return false; } - private async void do_prefetch_all() { + private async void do_prefetch_all_local() { Gee.List? list = null; try { // by listing NONE, retrieving only the EmailIdentifier for the range (which here is all) @@ -119,6 +127,10 @@ public class Geary.ImapEngine.EmailPrefetcher : Object { debug("Error while prefetching emails for %s: %s", folder.to_string(), err.message); } + // only signal "halting" if it looks like nothing more is waiting for another round + if (prefetch_ids.size == 0) + halting(); + if (token != NonblockingMutex.INVALID_TOKEN) { try { mutex.release(ref token); @@ -129,8 +141,6 @@ public class Geary.ImapEngine.EmailPrefetcher : Object { } private async void do_prefetch_batch() throws Error { - debug("do_prefetch_batch %s %d", folder.to_string(), prefetch_ids.size); - // snarf up all requested EmailIdentifiers for this round Gee.HashSet ids = prefetch_ids; prefetch_ids = new Gee.HashSet(Hashable.hash_func, Equalable.equal_func); @@ -148,8 +158,11 @@ public class Geary.ImapEngine.EmailPrefetcher : Object { return; } + debug("do_prefetch_batch %s %d", folder.to_string(), ids.size); + // Sort email by size - Gee.TreeSet sorted_email = new Gee.TreeSet(email_size_ascending_comparator); + Gee.TreeSet sorted_email = new Gee.TreeSet( + Email.compare_date_received_descending); foreach (Geary.EmailIdentifier id in local_fields.keys) { sorted_email.add(yield folder.fetch_email_async(id, Geary.Email.Field.PROPERTIES, Geary.Folder.ListFlags.LOCAL_ONLY, cancellable)); @@ -159,64 +172,36 @@ public class Geary.ImapEngine.EmailPrefetcher : Object { // constituting it) and PREVIEW from HEADER and BODY if available. When it can do that // won't need to prefetch ENVELOPE or PREVIEW; prefetching HEADER and BODY will be enough. + // Another big TODO: The engine needs to be able to chunk BODY requests so a large email + // doesn't monopolize the pipe and prevent other requests from going through + + int skipped = 0; foreach (Geary.Email email in sorted_email) { - Geary.EmailIdentifier id = email.id; - Geary.Email.Field has_fields = local_fields.get(id); - - if (!yield prefetch_field_async(Geary.Email.Field.ENVELOPE, has_fields, id, "envelope")) - break; - - if (!yield prefetch_field_async(Geary.Email.Field.HEADER, has_fields, id, "headers")) - break; - - if (!yield prefetch_field_async(Geary.Email.Field.BODY, has_fields, id, "body")) - break; - - if (!yield prefetch_field_async(Geary.Email.Field.PREVIEW, has_fields, id, "preview")) - break; + if (local_fields.get(email.id).fulfills(PREFETCH_FIELDS)) { + skipped++; + + continue; + } if (cancellable.is_cancelled()) break; + + try { + yield folder.fetch_email_async(email.id, PREFETCH_FIELDS, Folder.ListFlags.NONE, + cancellable); + } catch (Error err) { + if (!(err is IOError.CANCELLED)) { + debug("Error prefetching %s for %s: %s", folder.to_string(), email.id.to_string(), + err.message); + } else { + // only exit if cancelled; fetch_email_async() can error out on lots of things, + // including mail that's been deleted, and that shouldn't stop the prefetcher + break; + } + } } - debug("finished do_prefetch_batch %s %d", folder.to_string(), ids.size); - } - - private async bool prefetch_field_async(Geary.Email.Field field, Geary.Email.Field has_fields, - Geary.EmailIdentifier id, string name) { - if (has_fields.fulfills(field)) - return true; - - if (cancellable.is_cancelled()) - return false; - - try { - yield folder.fetch_email_async(id, field, Geary.Folder.ListFlags.NONE, cancellable); - } catch (Error err) { - if (!(err is IOError.CANCELLED)) - debug("Error prefetching %s for %s: %s", name, id.to_string(), err.message); - } - - return true; - } - - private static int email_size_ascending_comparator(void *a, void *b) { - long asize = 0; - Geary.Imap.EmailProperties? aprop = (Geary.Imap.EmailProperties) ((Geary.Email *) a)->properties; - if (aprop != null && aprop.rfc822_size != null) - asize = aprop.rfc822_size.value; - - long bsize = 0; - Geary.Imap.EmailProperties? bprop = (Geary.Imap.EmailProperties) ((Geary.Email *) b)->properties; - if (bprop != null && bprop.rfc822_size != null) - bsize = bprop.rfc822_size.value; - - if (asize < bsize) - return -1; - else if (asize > bsize) - return 1; - else - return 0; + debug("finished do_prefetch_batch %s %d skipped=%d", folder.to_string(), ids.size, skipped); } } diff --git a/src/engine/imap-engine/imap-engine-generic-account.vala b/src/engine/imap-engine/imap-engine-generic-account.vala index 95a58083..5bf362d0 100644 --- a/src/engine/imap-engine/imap-engine-generic-account.vala +++ b/src/engine/imap-engine/imap-engine-generic-account.vala @@ -5,6 +5,8 @@ */ private abstract class Geary.ImapEngine.GenericAccount : Geary.AbstractAccount { + private const int REFRESH_FOLDER_LIST_SEC = 1 * 60; + private static Geary.FolderPath? inbox_path = null; private static Geary.FolderPath? outbox_path = null; @@ -17,6 +19,9 @@ private abstract class Geary.ImapEngine.GenericAccount : Geary.AbstractAccount { FolderPath, GenericFolder>(Hashable.hash_func, Equalable.equal_func); private Gee.HashMap local_only = new Gee.HashMap( Hashable.hash_func, Equalable.equal_func); + private uint refresh_folder_timeout_id = 0; + private bool in_refresh_enumerate = false; + private Cancellable refresh_cancellable = new Cancellable(); public GenericAccount(string name, Geary.AccountInformation information, Imap.Account remote, ImapDB.Account local) { @@ -53,7 +58,7 @@ private abstract class Geary.ImapEngine.GenericAccount : Geary.AbstractAccount { if (available != null) { foreach(Folder f in available) { - if (f.has_children().is_possible()) + if (f.get_properties().has_children.is_possible()) enumerate_folders_async.begin(f.get_path(), null, on_enumerate_folders_async_complete); } } @@ -96,7 +101,10 @@ private abstract class Geary.ImapEngine.GenericAccount : Geary.AbstractAccount { notify_opened(); notify_folders_available_unavailable(local_only.values, null); - yield enumerate_folders_async(null, cancellable); + + // schedule an immediate sweep of the folders; once this is finished, folders will be + // regularly enumerated + reschedule_folder_refresh(true); } public override async void close_async(Cancellable? cancellable = null) throws Error { @@ -172,6 +180,7 @@ private abstract class Geary.ImapEngine.GenericAccount : Geary.AbstractAccount { if (built_folders.size > 0) notify_folders_available_unavailable(built_folders, null); + return return_folders; } @@ -193,11 +202,51 @@ private abstract class Geary.ImapEngine.GenericAccount : Geary.AbstractAccount { public override Gee.Collection list_folders() throws Error { check_open(); + return existing_folders.values; } - private async Gee.Collection enumerate_folders_async(Geary.FolderPath? parent, - Cancellable? cancellable = null) throws Error { + private void reschedule_folder_refresh(bool immediate) { + if (in_refresh_enumerate) + return; + + cancel_folder_refresh(); + + refresh_folder_timeout_id = immediate + ? Idle.add(on_refresh_folders) + : Timeout.add_seconds(REFRESH_FOLDER_LIST_SEC, on_refresh_folders); + } + + private void cancel_folder_refresh() { + if (refresh_folder_timeout_id != 0) { + Source.remove(refresh_folder_timeout_id); + refresh_folder_timeout_id = 0; + } + } + + private bool on_refresh_folders() { + in_refresh_enumerate = true; + enumerate_folders_async.begin(null, refresh_cancellable, on_refresh_completed); + + refresh_folder_timeout_id = 0; + + return false; + } + + private void on_refresh_completed(Object? source, AsyncResult result) { + try { + enumerate_folders_async.end(result); + } catch (Error err) { + if (!(err is IOError.CANCELLED)) + debug("Refresh of account %s folders did not complete: %s", to_string(), err.message); + } + + in_refresh_enumerate = false; + reschedule_folder_refresh(false); + } + + private async void enumerate_folders_async(Geary.FolderPath? parent,Cancellable? cancellable = null) + throws Error { check_open(); Gee.Collection? local_list = null; @@ -219,8 +268,6 @@ private abstract class Geary.ImapEngine.GenericAccount : Geary.AbstractAccount { engine_list.add_all(local_only.values); background_update_folders.begin(parent, engine_list, cancellable); - - return engine_list; } public override Geary.ContactStore get_contact_store() { @@ -284,7 +331,25 @@ private abstract class Geary.ImapEngine.GenericAccount : Geary.AbstractAccount { } // update all remote folders properties in the local store and active in the system + Gee.HashSet altered_paths = new Gee.HashSet( + Hashable.hash_func, Equalable.equal_func); foreach (Imap.Folder remote_folder in remote_folders) { + // only worry about alterations if the remote is openable + if (remote_folder.get_properties().is_openable.is_possible()) { + ImapDB.Folder? local_folder = null; + try { + local_folder = yield local.fetch_folder_async(remote_folder.get_path(), cancellable); + } catch (Error err) { + debug("Unable to fetch local folder for remote %s: %s", remote_folder.get_path().to_string(), + err.message); + } + + if (local_folder != null) { + if (remote_folder.get_properties().have_contents_changed(local_folder.get_properties()).is_possible()) + altered_paths.add(remote_folder.get_path()); + } + } + try { yield local.update_folder_async(remote_folder, cancellable); } catch (Error update_error) { @@ -379,6 +444,32 @@ private abstract class Geary.ImapEngine.GenericAccount : Geary.AbstractAccount { if (engine_added != null) notify_folders_added_removed(engine_added, null); + + // report all altered folders + if (altered_paths.size > 0) { + Gee.ArrayList altered = new Gee.ArrayList(); + foreach (Geary.FolderPath path in altered_paths) { + if (existing_folders.has_key(path)) + altered.add(existing_folders.get(path)); + else + debug("Unable to report %s altered: no local representation", path.to_string()); + } + + if (altered.size > 0) + notify_folders_contents_altered(altered); + } + + // enumerate childen of each remote folder + foreach (Imap.Folder remote_folder in remote_folders) { + if (remote_folder.get_properties().has_children.is_possible()) { + try { + yield enumerate_folders_async(remote_folder.get_path(), cancellable); + } catch (Error err) { + debug("Unable to enumerate children of %s: %s", remote_folder.get_path().to_string(), + err.message); + } + } + } } public override async void send_email_async(Geary.ComposedEmail composed, diff --git a/src/engine/imap-engine/imap-engine-generic-folder.vala b/src/engine/imap-engine/imap-engine-generic-folder.vala index b180fa0c..ee4e2051 100644 --- a/src/engine/imap-engine/imap-engine-generic-folder.vala +++ b/src/engine/imap-engine/imap-engine-generic-folder.vala @@ -11,18 +11,19 @@ private class Geary.ImapEngine.GenericFolder : Geary.AbstractFolder, Geary.Folde private const Geary.Email.Field NORMALIZATION_FIELDS = Geary.Email.Field.PROPERTIES | Geary.Email.Field.FLAGS | ImapDB.Folder.REQUIRED_FOR_DUPLICATE_DETECTION; - private weak GenericAccount _account; public override Account account { get { return _account; } } internal ImapDB.Folder local_folder { get; protected set; } internal Imap.Folder? remote_folder { get; protected set; default = null; } + internal EmailPrefetcher email_prefetcher { get; private set; } + + private weak GenericAccount _account; private Imap.Account remote; private ImapDB.Account local; private EmailFlagWatcher email_flag_watcher; - private EmailPrefetcher email_prefetcher; private SpecialFolderType special_folder_type; - private bool opened = false; - private NonblockingReportingSemaphore remote_semaphore; + private int open_count = 0; + private NonblockingReportingSemaphore? remote_semaphore = null; private ReplayQueue? replay_queue = null; private NonblockingMutex normalize_email_positions_mutex = new NonblockingMutex(); private int remote_count = -1; @@ -42,7 +43,7 @@ private class Geary.ImapEngine.GenericFolder : Geary.AbstractFolder, Geary.Folde } ~EngineFolder() { - if (opened) + if (open_count > 0) warning("Folder %s destroyed without closing", to_string()); } @@ -50,6 +51,16 @@ private class Geary.ImapEngine.GenericFolder : Geary.AbstractFolder, Geary.Folde return local_folder.get_path(); } + public override Geary.FolderProperties get_properties() { + // Get properties in order of authoritativeness: + // - From open remote folder + // - Fetch from local store + if (remote_folder != null && get_open_state() == OpenState.BOTH) + return remote_folder.get_properties(); + + return local_folder.get_properties(); + } + public override Geary.SpecialFolderType get_special_folder_type() { return special_folder_type; } @@ -64,33 +75,8 @@ private class Geary.ImapEngine.GenericFolder : Geary.AbstractFolder, Geary.Folde notify_special_folder_type_changed(old_type, new_type); } - private Imap.FolderProperties? get_folder_properties() { - Imap.FolderProperties? properties = null; - - // Get properties in order of authoritativeness: - // - Ask open remote folder - // - Query account object if it's seen them in its traversals - // - Fetch from local store - if (remote_folder != null) - properties = remote_folder.get_properties(); - - if (properties == null) - properties = _account.get_properties_for_folder(local_folder.get_path()); - - if (properties == null) - properties = local_folder.get_properties(); - - return properties; - } - - public override Geary.Trillian has_children() { - Imap.FolderProperties? properties = get_folder_properties(); - - return (properties != null) ? properties.has_children : Trillian.UNKNOWN; - } - public override Geary.Folder.OpenState get_open_state() { - if (!opened) + if (open_count == 0) return Geary.Folder.OpenState.CLOSED; if (local_folder.opened) @@ -109,7 +95,9 @@ private class Geary.ImapEngine.GenericFolder : Geary.AbstractFolder, Geary.Folde // last_seen_remote_count (which may be -1). internal int get_remote_counts(out int remote_count, out int last_seen_remote_count) { remote_count = this.remote_count; - last_seen_remote_count = (local_folder.get_properties() != null) ? local_folder.get_properties().messages : -1; + last_seen_remote_count = local_folder.get_properties().select_examine_messages; + if (last_seen_remote_count < 0) + last_seen_remote_count = local_folder.get_properties().status_messages; return (remote_count >= 0) ? remote_count : last_seen_remote_count; } @@ -117,21 +105,8 @@ private class Geary.ImapEngine.GenericFolder : Geary.AbstractFolder, Geary.Folde private async bool normalize_folders(Geary.Imap.Folder remote_folder, Cancellable? cancellable) throws Error { debug("normalize_folders %s", to_string()); - Geary.Imap.FolderProperties? local_properties = local_folder.get_properties(); - Geary.Imap.FolderProperties? remote_properties = remote_folder.get_properties(); - - // both sets of properties must be available - if (local_properties == null) { - debug("Unable to verify UID validity for %s: missing local properties", get_path().to_string()); - - return false; - } - - if (remote_properties == null) { - debug("Unable to verify UID validity for %s: missing remote properties", get_path().to_string()); - - return false; - } + Geary.Imap.FolderProperties local_properties = local_folder.get_properties(); + Geary.Imap.FolderProperties remote_properties = remote_folder.get_properties(); // and both must have their next UID's (it's possible they don't if it's a non-selectable // folder) @@ -403,11 +378,17 @@ private class Geary.ImapEngine.GenericFolder : Geary.AbstractFolder, Geary.Folde return true; } - public override async void open_async(bool readonly, Cancellable? cancellable = null) throws Error { - if (opened) - throw new EngineError.ALREADY_OPEN("Folder %s already open", to_string()); + public override async void wait_for_open_async(Cancellable? cancellable = null) throws Error { + if (open_count == 0 || remote_semaphore == null) + throw new EngineError.OPEN_REQUIRED("wait_for_open_async() can only be called after open_async()"); - opened = true; + if (!yield remote_semaphore.wait_for_result_async(cancellable)) + throw new EngineError.ALREADY_CLOSED("%s failed to open", to_string()); + } + + public override async void open_async(bool readonly, Cancellable? cancellable = null) throws Error { + if (open_count++ > 0) + return; remote_semaphore = new Geary.NonblockingReportingSemaphore(false); @@ -528,13 +509,9 @@ private class Geary.ImapEngine.GenericFolder : Geary.AbstractFolder, Geary.Folde private async void close_internal_async(Folder.CloseReason local_reason, Folder.CloseReason remote_reason, Cancellable? cancellable) { - if (!opened) + if (open_count == 0 || --open_count > 0) return; - // set this now to avoid multiple close_async(), particularly nested inside one of the signals - // fired here - opened = false; - // Notify all callers waiting for the remote folder that it's not coming available Imap.Folder? closing_remote_folder = remote_folder; try { @@ -667,7 +644,7 @@ private class Geary.ImapEngine.GenericFolder : Geary.AbstractFolder, Geary.Folde bool changed = (remote_count != new_remote_count); remote_count = new_remote_count; try { - yield local_folder.update_remote_message_count(remote_count, null); + yield local_folder.update_remote_selected_message_count(remote_count, null); } catch (Error update_err) { debug("Unable to save appended remote count for %s: %s", to_string(), update_err.message); } @@ -748,7 +725,7 @@ private class Geary.ImapEngine.GenericFolder : Geary.AbstractFolder, Geary.Folde bool changed = (remote_count != new_remote_count); remote_count = new_remote_count; try { - yield local_folder.update_remote_message_count(remote_count, null); + yield local_folder.update_remote_selected_message_count(remote_count, null); } catch (Error update_err) { debug("Unable to save removed remote count for %s: %s", to_string(), update_err.message); } @@ -787,19 +764,6 @@ private class Geary.ImapEngine.GenericFolder : Geary.AbstractFolder, Geary.Folde }); } - public override async int get_email_count_async(Cancellable? cancellable = null) throws Error { - check_open("get_email_count_async"); - - // if connected or connecting, use stashed remote count (which is always kept current once - // remote folder is opened) - if (opened) { - if (yield remote_semaphore.wait_for_result_async(cancellable)) - return remote_count; - } - - return yield local_folder.get_email_count_async(ImapDB.Folder.ListFlags.NONE, cancellable); - } - // // list_email variants // @@ -998,7 +962,7 @@ private class Geary.ImapEngine.GenericFolder : Geary.AbstractFolder, Geary.Folde } private void check_open(string method) throws EngineError { - if (!opened) + if (open_count == 0) throw new EngineError.OPEN_REQUIRED("%s failed: folder %s is not open", method, to_string()); } @@ -1098,9 +1062,6 @@ private class Geary.ImapEngine.GenericFolder : Geary.AbstractFolder, Geary.Folde int prefetch_count = local_low - high; - debug("expanding normalized range to %d (%d needed) for %s (local_low=%d) from remote", - high, prefetch_count, to_string(), local_low); - // Normalize the local folder by fetching EmailIdentifiers for all missing email as well // as fields for duplicate detection Gee.List? list = yield remote_folder.list_email_async( diff --git a/src/engine/imap-engine/imap-engine-replay-queue.vala b/src/engine/imap-engine/imap-engine-replay-queue.vala index d7206af8..17bd53c3 100644 --- a/src/engine/imap-engine/imap-engine-replay-queue.vala +++ b/src/engine/imap-engine/imap-engine-replay-queue.vala @@ -139,14 +139,7 @@ private class Geary.ImapEngine.ReplayQueue { // in order), it's *vital* that even REMOTE_ONLY operations go through the local queue, // only being scheduled on the remote queue *after* local operations ahead of it have // completed; thus, no need for get_scope() to be called here. - try { - local_queue.send(op); - } catch (Error err) { - debug("Replay operation %s not scheduled on local queue %s: %s", op.to_string(), - to_string(), err.message); - - return false; - } + local_queue.send(op); scheduled(op); @@ -273,12 +266,7 @@ private class Geary.ImapEngine.ReplayQueue { } if (remote_enqueue) { - try { - remote_queue.send(op); - } catch (Error send_err) { - error("ReplayOperation %s not scheduled on remote queue %s: %s", op.to_string(), - to_string(), send_err.message); - } + remote_queue.send(op); } else { // all code paths to this point should have notified ready if not enqueuing for // next stage diff --git a/src/engine/imap-engine/imap-engine.vala b/src/engine/imap-engine/imap-engine.vala new file mode 100644 index 00000000..bf8e80c5 --- /dev/null +++ b/src/engine/imap-engine/imap-engine.vala @@ -0,0 +1,62 @@ +/* Copyright 2013 Yorba Foundation + * + * This software is licensed under the GNU Lesser General Public License + * (version 2.1 or later). See the COPYING file in this distribution. + */ + +namespace Geary.ImapEngine { + +private int init_count = 0; +private Gee.HashMap? account_synchronizers = null; + +internal void init() { + if (init_count++ != 0) + return; + + account_synchronizers = new Gee.HashMap(); + + // create a FullAccountSync object for each Account as it comes and goes + Engine.instance.account_available.connect(on_account_available); + Engine.instance.account_unavailable.connect(on_account_unavailable); +} + +private GenericAccount? get_imap_account(AccountInformation account_info) { + try { + return Engine.instance.get_account_instance(account_info) as GenericAccount; + } catch (Error err) { + debug("Unable to get account instance %s: %s", account_info.email, err.message); + } + + return null; +} + +private void on_account_available(AccountInformation account_info) { + GenericAccount? imap_account = get_imap_account(account_info); + if (imap_account == null) + return; + + assert(!account_synchronizers.has_key(imap_account)); + account_synchronizers.set(imap_account, new AccountSynchronizer(imap_account)); +} + +private void on_account_unavailable(AccountInformation account_info) { + GenericAccount? imap_account = get_imap_account(account_info); + if (imap_account == null) + return; + + AccountSynchronizer? account_synchronizer = account_synchronizers.get(imap_account); + assert(account_synchronizer != null); + + account_synchronizer.stop_async.begin(on_synchronizer_stopped); +} + +private void on_synchronizer_stopped(Object? source, AsyncResult result) { + AccountSynchronizer account_synchronizer = (AccountSynchronizer) source; + account_synchronizer.stop_async.end(result); + + bool removed = account_synchronizers.unset(account_synchronizer.account); + assert(removed); +} + +} + diff --git a/src/engine/imap-engine/replay-ops/imap-engine-list-email.vala b/src/engine/imap-engine/replay-ops/imap-engine-list-email.vala index 0c0ca6e7..81f64274 100644 --- a/src/engine/imap-engine/replay-ops/imap-engine-list-email.vala +++ b/src/engine/imap-engine/replay-ops/imap-engine-list-email.vala @@ -101,10 +101,31 @@ private class Geary.ImapEngine.ListEmail : Geary.ImapEngine.SendReplayOperation Folder.normalize_span_specifiers(ref low, ref count, usable_remote_count); - // calculate local availability - int local_low = GenericFolder.remote_position_to_local_position(low, local_count, usable_remote_count); - int local_available_low = local_low.clamp(1, local_count); - int local_available_count = (local_low > 0) ? (count - local_low) + 1 : (count + local_low) - 1; + // + // Convert the requested low/count values into values that correspond do the number of + // emails in the database and their lowest position value relative to the remote's list. + // + + // convert remote low position to the low position relative to the database's contents ... + // this can return zero or a negative value if the position is not present in the local store + int local_low = GenericFolder.remote_position_to_local_position(low, local_count, + usable_remote_count); + + // from local_low and the user's request count, calculate the low position in the database + // for what is available (again, can be zero if nothing is available in the database) + int local_available_low = 0; + if (local_low >= 1) + local_available_low = local_low; + else if ((local_low + count) >= 1) + local_available_low = 1; + + // finally, determine how much in the requested span is available locally, if any + int local_available_count; + if (local_low >= 1) + local_available_count = (local_count - local_low) + 1; + else + local_available_count = count + local_low; + local_available_count = local_available_count.clamp(0, count); Logging.debug(Logging.Flag.REPLAY, "ListEmail.replay_local_async %s: low=%d count=%d local_low=%d local_count=%d " @@ -114,7 +135,7 @@ private class Geary.ImapEngine.ListEmail : Geary.ImapEngine.SendReplayOperation local_available_count, remote_count, last_seen_remote_count, usable_remote_count, local_only.to_string(), remote_only.to_string()); - if (!remote_only && local_available_count > 0) { + if (!remote_only && local_available_count > 0 && local_available_low > 0) { try { local_list = yield engine.local_folder.list_email_async(local_available_low, local_available_count, required_fields, ImapDB.Folder.ListFlags.PARTIAL_OK, @@ -145,6 +166,7 @@ private class Geary.ImapEngine.ListEmail : Geary.ImapEngine.SendReplayOperation } else { // strip fulfilled fields so only remaining are fetched from server Geary.Email.Field remaining = required_fields.clear(email.fields); + assert(remaining != Geary.Email.Field.NONE); unfulfilled.set(remaining, email.id); } } @@ -258,7 +280,7 @@ private class Geary.ImapEngine.ListEmail : Geary.ImapEngine.SendReplayOperation } } - Logging.debug(Logging.Flag.REPLAY, "ListEmail.replay_remote %s: Scheduling %d FETCH operations", + Logging.debug(Logging.Flag.REPLAY, "ListEmail.replay_remote %s: Scheduling %d partial list operations", engine.to_string(), batch.size); yield batch.execute_all_async(cancellable); diff --git a/src/engine/imap/api/imap-account.vala b/src/engine/imap/api/imap-account.vala index 56b23504..6a4f3625 100644 --- a/src/engine/imap/api/imap-account.vala +++ b/src/engine/imap/api/imap-account.vala @@ -86,7 +86,7 @@ private class Geary.Imap.Account : Object { if (!mbox.attrs.contains(MailboxAttribute.NO_SELECT)) batch.add(new StatusOperation(session_mgr, mbox, path)); else - folders.add(new Geary.Imap.Folder(session_mgr, path, null, mbox)); + folders.add(new Geary.Imap.Folder.unselectable(session_mgr, path, mbox)); } yield batch.execute_all_async(cancellable); @@ -95,7 +95,7 @@ private class Geary.Imap.Account : Object { StatusOperation op = (StatusOperation) batch.get_operation(id); try { folders.add(new Geary.Imap.Folder(session_mgr, op.path, - (StatusResults?) batch.get_result(id), op.mbox)); + (StatusResults) batch.get_result(id), op.mbox)); } catch (Error status_err) { message("Unable to fetch status for %s: %s", op.path.to_string(), status_err.message); } @@ -125,15 +125,11 @@ private class Geary.Imap.Account : Object { if (mbox == null) throw_not_found(path); - StatusResults? status = null; - if (!mbox.attrs.contains(MailboxAttribute.NO_SELECT)) { - try { - status = yield session_mgr.status_async(processed.get_fullpath(), - StatusDataType.all(), cancellable); - } catch (Error status_err) { - debug("Unable to get status for %s: %s", processed.to_string(), status_err.message); - } - } + if (mbox.attrs.contains(MailboxAttribute.NO_SELECT)) + return new Geary.Imap.Folder.unselectable(session_mgr, processed, mbox); + + StatusResults status = yield session_mgr.status_async(processed.get_fullpath(), + StatusDataType.all(), cancellable); return new Geary.Imap.Folder(session_mgr, processed, status, mbox); } catch (ImapError err) { diff --git a/src/engine/imap/api/imap-email-properties.vala b/src/engine/imap/api/imap-email-properties.vala index 0dc828f3..b0659e04 100644 --- a/src/engine/imap/api/imap-email-properties.vala +++ b/src/engine/imap/api/imap-email-properties.vala @@ -9,6 +9,8 @@ public class Geary.Imap.EmailProperties : Geary.EmailProperties, Equalable { public RFC822.Size? rfc822_size { get; private set; } public EmailProperties(InternalDate? internaldate, RFC822.Size? rfc822_size) { + base (internaldate.value, rfc822_size.value); + this.internaldate = internaldate; this.rfc822_size = rfc822_size; } diff --git a/src/engine/imap/api/imap-folder-properties.vala b/src/engine/imap/api/imap-folder-properties.vala index ed643da2..c46c7d42 100644 --- a/src/engine/imap/api/imap-folder-properties.vala +++ b/src/engine/imap/api/imap-folder-properties.vala @@ -4,21 +4,30 @@ * (version 2.1 or later). See the COPYING file in this distribution. */ -public class Geary.Imap.FolderProperties { - // messages can be updated a variety of ways, so it's available as a public set - public int messages { get; set; } - public int recent { get; private set; } +public class Geary.Imap.FolderProperties : Geary.FolderProperties { + /** + * -1 if the Folder was not opened via SELECT or EXAMINE. + */ + public int select_examine_messages { get; private set; } + /** + * -1 if the FolderProperties were not obtained via a STATUS command + */ + public int status_messages { get; private set; } public int unseen { get; private set; } + public int recent { get; private set; } public UIDValidity? uid_validity { get; private set; } public UID? uid_next { get; private set; } public MailboxAttributes attrs { get; private set; } - public Trillian supports_children { get; private set; } - public Trillian has_children { get; private set; } - public Trillian is_openable { get; private set; } + // Note that unseen from SELECT/EXAMINE is the *position* of the first unseen message, + // not the total unseen count, so it should not be passed in here, but rather the unseen + // count from a STATUS command public FolderProperties(int messages, int recent, int unseen, UIDValidity? uid_validity, UID? uid_next, MailboxAttributes attrs) { - this.messages = messages; + base (messages, unseen, Trillian.UNKNOWN, Trillian.UNKNOWN, Trillian.UNKNOWN); + + select_examine_messages = messages; + status_messages = -1; this.recent = recent; this.unseen = unseen; this.uid_validity = uid_validity; @@ -29,7 +38,10 @@ public class Geary.Imap.FolderProperties { } public FolderProperties.status(StatusResults status, MailboxAttributes attrs) { - messages = status.messages; + base (status.messages, status.unseen, Trillian.UNKNOWN, Trillian.UNKNOWN, Trillian.UNKNOWN); + + select_examine_messages = -1; + status_messages = status.messages; recent = status.recent; unseen = status.unseen; uid_validity = status.uid_validity; @@ -39,8 +51,39 @@ public class Geary.Imap.FolderProperties { init_flags(); } + /** + * Use with FolderProperties of the *same folder* seen at different times (i.e. after SELECTing + * versus data stored locally). Only compares fields that suggest the contents of the folder + * have changed. + * + * Note that this is *not* concerned with message flags changing. + */ + public Trillian have_contents_changed(Geary.Imap.FolderProperties other) { + // UIDNEXT changes indicate messages have been added, but not if they've been removed + if (uid_next != null && other.uid_next != null && !uid_next.equals(other.uid_next)) + return Trillian.TRUE; + + // Gmail includes Chat messages in STATUS results but not in SELECT/EXAMINE + // results, so message count comparison has to be from the same origin ... use SELECT/EXAMINE + // first, as it's more authoritative in many ways + // + // TODO: If this continues to work, it might be worthwhile to change the result of this + // method to boolean + if (select_examine_messages >= 0 && other.select_examine_messages >= 0 + && select_examine_messages != other.select_examine_messages) { + return Trillian.TRUE; + } + + if (status_messages >= 0 && other.status_messages >= 0 && status_messages != other.status_messages) { + return Trillian.TRUE; + } + + return Trillian.FALSE; + } + private void init_flags() { supports_children = Trillian.from_boolean(!attrs.contains(MailboxAttribute.NO_INFERIORS)); + // \HasNoChildren & \HasChildren are optional attributes (could check for CHILDREN extension, // but unnecessary here) if (attrs.contains(MailboxAttribute.HAS_NO_CHILDREN)) @@ -49,7 +92,29 @@ public class Geary.Imap.FolderProperties { has_children = Trillian.TRUE; else has_children = Trillian.UNKNOWN; + is_openable = Trillian.from_boolean(!attrs.contains(MailboxAttribute.NO_SELECT)); } + + public void set_status_message_count(int messages) { + if (messages < 0) + return; + + status_messages = messages; + + // select/examine more authoritative than status + if (select_examine_messages < 0) + email_total = messages; + } + + public void set_select_examine_message_count(int messages) { + if (messages < 0) + return; + + select_examine_messages = messages; + + // select/examine more authoritative than status + email_total = messages; + } } diff --git a/src/engine/imap/api/imap-folder.vala b/src/engine/imap/api/imap-folder.vala index 4aacfb82..f6610cf3 100644 --- a/src/engine/imap/api/imap-folder.vala +++ b/src/engine/imap/api/imap-folder.vala @@ -20,7 +20,7 @@ private class Geary.Imap.Folder : Object { public signal void disconnected(Geary.Folder.CloseReason reason); - internal Folder(ClientSessionManager session_mgr, Geary.FolderPath path, StatusResults? status, + internal Folder(ClientSessionManager session_mgr, Geary.FolderPath path, StatusResults status, MailboxInformation info) { this.session_mgr = session_mgr; this.info = info; @@ -28,9 +28,18 @@ private class Geary.Imap.Folder : Object { readonly = Trillian.UNKNOWN; - properties = (status != null) - ? new Imap.FolderProperties.status(status, info.attrs) - : new Imap.FolderProperties(0, 0, 0, null, null, info.attrs); + properties = new Imap.FolderProperties.status(status, info.attrs); + } + + internal Folder.unselectable(ClientSessionManager session_mgr, Geary.FolderPath path, + MailboxInformation info) { + this.session_mgr = session_mgr; + this.info = info; + this.path = path; + + readonly = Trillian.UNKNOWN; + + properties = new Imap.FolderProperties(0, 0, 0, null, null, info.attrs); } public Geary.FolderPath get_path() { @@ -57,8 +66,10 @@ private class Geary.Imap.Folder : Object { mailbox.expunged.connect(on_expunged); mailbox.disconnected.connect(on_disconnected); - properties = new Imap.FolderProperties(mailbox.exists, mailbox.recent, mailbox.unseen, + int old_status_messages = properties.status_messages; + properties = new Imap.FolderProperties(mailbox.exists, mailbox.recent, properties.unseen, mailbox.uid_validity, mailbox.uid_next, properties.attrs); + properties.set_status_message_count(old_status_messages); } public async void close_async(Cancellable? cancellable = null) throws Error { diff --git a/src/engine/imap/decoders/imap-select-examine-results.vala b/src/engine/imap/decoders/imap-select-examine-results.vala index 3e0c2904..5ae562e6 100644 --- a/src/engine/imap/decoders/imap-select-examine-results.vala +++ b/src/engine/imap/decoders/imap-select-examine-results.vala @@ -13,23 +13,20 @@ public class Geary.Imap.SelectExamineResults : Geary.Imap.CommandResults { * -1 if not specified. */ public int recent { get; private set; } - /** - * -1 if not specified. - */ - public int unseen { get; private set; } + public MessageNumber? unseen_position { get; private set; } public UIDValidity? uid_validity { get; private set; } public UID? uid_next { get; private set; } public Flags? flags { get; private set; } public Flags? permanentflags { get; private set; } public bool readonly { get; private set; } - private SelectExamineResults(StatusResponse status_response, int exists, int recent, int unseen, + private SelectExamineResults(StatusResponse status_response, int exists, int recent, MessageNumber? unseen_position, UIDValidity? uid_validity, UID? uid_next, Flags? flags, Flags? permanentflags, bool readonly) { base (status_response); this.exists = exists; this.recent = recent; - this.unseen = unseen; + this.unseen_position = unseen_position; this.uid_validity = uid_validity; this.uid_next = uid_next; this.flags = flags; @@ -42,7 +39,7 @@ public class Geary.Imap.SelectExamineResults : Geary.Imap.CommandResults { int exists = -1; int recent = -1; - int unseen = -1; + MessageNumber? unseen_position = null; UIDValidity? uid_validity = null; UID? uid_next = null; MessageFlags? flags = null; @@ -73,7 +70,8 @@ public class Geary.Imap.SelectExamineResults : Geary.Imap.CommandResults { // the ResponseCode is what we're interested in switch (ok_response.response_code.get_code_type()) { case ResponseCodeType.UNSEEN: - unseen = ok_response.response_code.get_as_string(1).as_int(0, int.MAX); + unseen_position = new MessageNumber( + ok_response.response_code.get_as_string(1).as_int(1, int.MAX)); break; case ResponseCodeType.UIDVALIDITY: @@ -130,7 +128,7 @@ public class Geary.Imap.SelectExamineResults : Geary.Imap.CommandResults { if (flags == null || exists < 0 || recent < 0) throw new ImapError.PARSE_ERROR("Incomplete SELECT/EXAMINE Response: \"%s\"", response.to_string()); - return new SelectExamineResults(response.status_response, exists, recent, unseen, + return new SelectExamineResults(response.status_response, exists, recent, unseen_position, uid_validity, uid_next, flags, permanentflags, readonly); } } diff --git a/src/engine/imap/transport/imap-client-session-manager.vala b/src/engine/imap/transport/imap-client-session-manager.vala index a6487213..f69485cb 100644 --- a/src/engine/imap/transport/imap-client-session-manager.vala +++ b/src/engine/imap/transport/imap-client-session-manager.vala @@ -5,7 +5,7 @@ */ public class Geary.Imap.ClientSessionManager { - public const int DEFAULT_MIN_POOL_SIZE = 2; + public const int DEFAULT_MIN_POOL_SIZE = 4; private AccountInformation account_information; private int min_pool_size; diff --git a/src/engine/imap/transport/imap-mailbox.vala b/src/engine/imap/transport/imap-mailbox.vala index b347fd5b..4d9a14b8 100644 --- a/src/engine/imap/transport/imap-mailbox.vala +++ b/src/engine/imap/transport/imap-mailbox.vala @@ -22,7 +22,6 @@ public class Geary.Imap.Mailbox : Geary.SmartReference { public string name { get { return context.name; } } public int exists { get { return context.exists; } } public int recent { get { return context.recent; } } - public int unseen { get { return context.unseen; } } public bool is_readonly { get { return context.is_readonly; } } public UIDValidity? uid_validity { get { return context.uid_validity; } } public UID? uid_next { get { return context.uid_next; } } @@ -641,7 +640,6 @@ private class Geary.Imap.SelectedContext : Object, Geary.ReferenceSemantics { public string name { get; protected set; } public int exists { get; protected set; } public int recent { get; protected set; } - public int unseen { get; protected set; } public bool is_readonly { get; protected set; } public UIDValidity? uid_validity { get; protected set; } public UID? uid_next { get; protected set; } @@ -669,7 +667,6 @@ private class Geary.Imap.SelectedContext : Object, Geary.ReferenceSemantics { is_readonly = results.readonly; exists = results.exists; recent = results.recent; - unseen = results.unseen; uid_validity = results.uid_validity; uid_next = results.uid_next; diff --git a/src/engine/nonblocking/nonblocking-abstract-semaphore.vala b/src/engine/nonblocking/nonblocking-abstract-semaphore.vala index 5dd12a80..1fc7cde2 100644 --- a/src/engine/nonblocking/nonblocking-abstract-semaphore.vala +++ b/src/engine/nonblocking/nonblocking-abstract-semaphore.vala @@ -146,6 +146,10 @@ public abstract class Geary.NonblockingAbstractSemaphore { notify_at_reset(); } + public bool is_passed() { + return passed; + } + public bool is_cancelled() { return (cancellable != null) ? cancellable.is_cancelled() : false; } diff --git a/src/engine/nonblocking/nonblocking-mailbox.vala b/src/engine/nonblocking/nonblocking-mailbox.vala index b2002ef1..489155bd 100644 --- a/src/engine/nonblocking/nonblocking-mailbox.vala +++ b/src/engine/nonblocking/nonblocking-mailbox.vala @@ -6,30 +6,58 @@ public class Geary.NonblockingMailbox : Object { public int size { get { return queue.size; } } + public bool allow_duplicates { get; set; default = true; } + public bool requeue_duplicate { get; set; default = false; } - private Gee.List queue; + private Gee.Queue queue; private NonblockingSpinlock spinlock = new NonblockingSpinlock(); - public NonblockingMailbox() { - queue = new Gee.LinkedList(); + public NonblockingMailbox(CompareFunc? comparator = null) { + // can't use ternary here, Vala bug + if (comparator == null) + queue = new Gee.LinkedList(); + else + queue = new Gee.PriorityQueue(comparator); } - public void send(G msg) throws Error { - queue.add(msg); - spinlock.notify(); + public bool send(G msg) { + if (!allow_duplicates && queue.contains(msg)) { + if (requeue_duplicate) + queue.remove(msg); + else + return false; + } + + if (!queue.offer(msg)) + return false; + + spinlock.blind_notify(); + + return true; } /** * Returns true if the message was revoked. */ - public bool revoke(G msg) throws Error { + public bool revoke(G msg) { return queue.remove(msg); } + /** + * Returns number of removed items. + */ + public int clear() { + int count = queue.size; + if (count != 0) + queue.clear(); + + return count; + } + public async G recv_async(Cancellable? cancellable = null) throws Error { for (;;) { if (queue.size > 0) - return queue.remove_at(0); + return queue.poll(); yield spinlock.wait_async(cancellable); } @@ -42,7 +70,7 @@ public class Geary.NonblockingMailbox : Object { * This returns a read-only list in queue-order. Altering will not affect the queue. Use * revoke() to remove enqueued operations. */ - public Gee.List get_all() { + public Gee.Collection get_all() { return queue.read_only_view; } } diff --git a/src/engine/rfc822/rfc822.vala b/src/engine/rfc822/rfc822.vala new file mode 100644 index 00000000..90a354b3 --- /dev/null +++ b/src/engine/rfc822/rfc822.vala @@ -0,0 +1,18 @@ +/* Copyright 2013 Yorba Foundation + * + * This software is licensed under the GNU Lesser General Public License + * (version 2.1 or later). See the COPYING file in this distribution. + */ + +namespace Geary.RFC822 { + +private int init_count = 0; + +internal void init() { + if (init_count++ != 0) + return; + + GMime.init(0); +} + +} diff --git a/src/engine/util/util-interfaces.vala b/src/engine/util/util-interfaces.vala index 1d3bac8a..a8b2975f 100644 --- a/src/engine/util/util-interfaces.vala +++ b/src/engine/util/util-interfaces.vala @@ -8,12 +8,21 @@ public interface Geary.Comparable { public abstract int compare(Comparable other); /** - * A CompareFunc for any object that implements Comparable. + * A CompareFunc for any object that implements Comparable + * (ascending order). */ public static int compare_func(void *a, void *b) { return ((Comparable *) a)->compare((Comparable *) b); } + /** + * A reverse CompareFunc for any object that implements + * Comparable (descending order). + */ + public static int reverse_compare_func(void *a, void *b) { + return ((Comparable *) b)->compare((Comparable *) a); + } + /** * A CompareFunc for DateTime. */