diff --git a/po/POTFILES.in b/po/POTFILES.in index 47439258..08cfa043 100644 --- a/po/POTFILES.in +++ b/po/POTFILES.in @@ -172,12 +172,14 @@ src/engine/db/db-versioned-database.vala src/engine/db/db.vala src/engine/imap/imap.vala src/engine/imap/imap-error.vala -src/engine/imap/api/imap-account.vala +src/engine/imap/api/imap-account-session.vala src/engine/imap/api/imap-email-flags.vala src/engine/imap/api/imap-email-properties.vala +src/engine/imap/api/imap-folder.vala src/engine/imap/api/imap-folder-properties.vala src/engine/imap/api/imap-folder-root.vala -src/engine/imap/api/imap-folder.vala +src/engine/imap/api/imap-folder-session.vala +src/engine/imap/api/imap-session-object.vala src/engine/imap/command/imap-append-command.vala src/engine/imap/command/imap-capability-command.vala src/engine/imap/command/imap-close-command.vala diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ab2d27ca..52ca9696 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -86,12 +86,14 @@ engine/db/db-versioned-database.vala engine/imap/imap.vala engine/imap/imap-error.vala -engine/imap/api/imap-account.vala +engine/imap/api/imap-account-session.vala engine/imap/api/imap-email-flags.vala engine/imap/api/imap-email-properties.vala -engine/imap/api/imap-folder-properties.vala engine/imap/api/imap-folder.vala +engine/imap/api/imap-folder-properties.vala engine/imap/api/imap-folder-root.vala +engine/imap/api/imap-folder-session.vala +engine/imap/api/imap-session-object.vala engine/imap/command/imap-append-command.vala engine/imap/command/imap-capability-command.vala engine/imap/command/imap-close-command.vala diff --git a/src/engine/api/geary-abstract-local-folder.vala b/src/engine/api/geary-abstract-local-folder.vala index 7d72d0c5..c7e35658 100644 --- a/src/engine/api/geary-abstract-local-folder.vala +++ b/src/engine/api/geary-abstract-local-folder.vala @@ -32,12 +32,12 @@ public abstract class Geary.AbstractLocalFolder : Geary.Folder { protected bool is_open() { return open_count > 0; } - - public override async void wait_for_open_async(Cancellable? cancellable = null) throws Error { + + public override async void wait_for_remote_async(Cancellable? cancellable = null) throws Error { if (open_count == 0) throw new EngineError.OPEN_REQUIRED("%s not open", to_string()); } - + public override async bool open_async(Geary.Folder.OpenFlags open_flags, Cancellable? cancellable = null) throws Error { if (open_count++ > 0) diff --git a/src/engine/api/geary-engine.vala b/src/engine/api/geary-engine.vala index 937a208a..b1f4c028 100644 --- a/src/engine/api/geary-engine.vala +++ b/src/engine/api/geary-engine.vala @@ -398,28 +398,38 @@ public class Geary.Engine : BaseObject { return account_instances.get(account_information.id); ImapDB.Account local_account = new ImapDB.Account(account_information); - Imap.Account remote_account = new Imap.Account(account_information); - Geary.Account account; switch (account_information.service_provider) { case ServiceProvider.GMAIL: - account = new ImapEngine.GmailAccount("Gmail:%s".printf(account_information.id), - account_information, remote_account, local_account); + account = new ImapEngine.GmailAccount( + "Gmail:%s".printf(account_information.id), + account_information, + local_account + ); break; case ServiceProvider.YAHOO: - account = new ImapEngine.YahooAccount("Yahoo:%s".printf(account_information.id), - account_information, remote_account, local_account); + account = new ImapEngine.YahooAccount( + "Yahoo:%s".printf(account_information.id), + account_information, + local_account + ); break; case ServiceProvider.OUTLOOK: - account = new ImapEngine.OutlookAccount("Outlook:%s".printf(account_information.id), - account_information, remote_account, local_account); + account = new ImapEngine.OutlookAccount( + "Outlook:%s".printf(account_information.id), + account_information, + local_account + ); break; case ServiceProvider.OTHER: - account = new ImapEngine.OtherAccount("Other:%s".printf(account_information.id), - account_information, remote_account, local_account); + account = new ImapEngine.OtherAccount( + "Other:%s".printf(account_information.id), + account_information, + local_account + ); break; default: diff --git a/src/engine/api/geary-folder.vala b/src/engine/api/geary-folder.vala index d07a4004..e9396953 100644 --- a/src/engine/api/geary-folder.vala +++ b/src/engine/api/geary-folder.vala @@ -63,13 +63,12 @@ public abstract class Geary.Folder : BaseObject { LOCAL, BOTH } - + public enum OpenFailed { - LOCAL_FAILED, - REMOTE_FAILED, - CANCELLED + LOCAL_ERROR, + REMOTE_ERROR, } - + /** * Provides the reason why the folder is closing or closed when the {@link closed} signal * is fired. @@ -195,51 +194,68 @@ public abstract class Geary.Folder : BaseObject { public abstract Geary.SpecialFolderType special_folder_type { get; } public abstract Geary.ProgressMonitor opening_monitor { get; } - + /** - * Fired when the folder is successfully opened by a caller. + * Fired when the folder moves through stages of being opened. * - * It will only fire once until the Folder is closed, with the {@link OpenState} indicating what - * has been opened and the count indicating the number of messages in the folder. In the case - * of {@link OpenState.BOTH} or {@link OpenState.REMOTE}, it refers to the authoritative number. - * For {@link OpenState.LOCAL}, it refers to the number of messages in the local store. + * It will fire at least once if the folder successfully opens, + * with the {@link OpenState} indicating what has been opened and + * the count indicating the number of messages in the folder. it + * may fire additional times as remote sessions are established + * and re-established after being lost. * - * {@link OpenState.REMOTE} will only be passed if there's no local store, indicating that it's - * not a synchronized folder but rather one entirely backed by a network server. Geary - * currently has no such folder implemented like this. + * If //state// is {@link OpenState.LOCAL}, the local store for + * the folder has opened and the count reflects the number of + * messages in the local store. * - * This signal will never fire with {@link OpenState.CLOSED} as a parameter. + * If //state// is {@link OpenState.BOTH}, it indicates both the + * local store and a remote session has been established, and the + * count reflects the number of messages on the remote. + * + * If //state// is {@link OpenState.REMOTE}, it indicates a folder + * is not synchronized locally but rather one entirely backed by a + * network server. + * + * In the case of {@link OpenState.BOTH} or {@link + * OpenState.REMOTE}, it refers to the authoritative count. + * + * This signal will never fire with {@link OpenState.CLOSED} as a + * parameter. * * @see get_open_state */ public signal void opened(OpenState state, int count); - + /** * Fired when {@link open_async} fails for one or more reasons. * - * See open_async and {@link opened} for more information on how opening a Folder works, i particular - * how open_async may return immediately although the remote has not completely opened. - * This signal may be called in the context of, or after completion of, open_async. It will - * ''not'' be called after {@link close_async} has completed, however. + * See open_async and {@link opened} for more information on how + * opening a Folder works, in particular how open_async may return + * immediately although the remote has not completely opened. + * This signal may be called in the context of, or after + * completion of, open_async. It will ''not'' be called after + * {@link close_async} has completed, however. * - * Note that this signal may be fired ''and'' open_async throw an Error. + * Note that this signal may be fired ''and'' open_async throw an + * Error. * - * This signal may be fired more than once before the Folder is closed. It will only fire once - * for each type of failure, however. + * This signal may be fired more than once before the Folder is + * closed, especially in the case of a remote session */ public signal void open_failed(OpenFailed failure, Error? err); - + /** - * Fired when the Folder is closed, either by the caller or due to errors in the local - * or remote store(s). + * Fired when the Folder is closed, either by the caller or due to + * errors in the local or remote store(s). * - * It will fire three times: to report how the local store closed - * (gracefully or due to error), how the remote closed (similarly) and finally with - * {@link CloseReason.FOLDER_CLOSED}. The first two may come in either order; the third is - * always the last. + * It will fire a number of times: to report how the local store + * closed (gracefully or due to error), how the remote closed + * (similarly) and finally with {@link CloseReason.FOLDER_CLOSED}. + * The first two may come in either order; the third is always the + * last. */ public signal void closed(CloseReason reason); - + /** * Fired when email has been appended to the list of messages in the folder. * @@ -384,7 +400,7 @@ public abstract class Geary.Folder : BaseObject { protected virtual void notify_email_locally_complete(Gee.Collection ids) { email_locally_complete(ids); } - + /** * In its default implementation, this will also call {@link notify_display_name_changed} since * that's often the case; if not, subclasses should override. @@ -392,13 +408,12 @@ public abstract class Geary.Folder : BaseObject { protected virtual void notify_special_folder_type_changed(Geary.SpecialFolderType old_type, Geary.SpecialFolderType new_type) { special_folder_type_changed(old_type, new_type); - + // in default implementation, this may also mean the display name changed; subclasses may // override this behavior, but no way to detect this, so notify - if (special_folder_type != Geary.SpecialFolderType.NONE) - notify_display_name_changed(); + notify_display_name_changed(); } - + protected virtual void notify_display_name_changed() { display_name_changed(); } @@ -418,8 +433,10 @@ public abstract class Geary.Folder : BaseObject { * Returns the state of the Folder's connections to the local and remote stores. */ public abstract OpenState get_open_state(); - + /** + * Marks the folder's operations as being required for use. + * * The Folder must be opened before most operations may be performed on it. Depending on the * implementation this might entail opening a network connection or setting the connection to * a particular state, opening a file or database, and so on. @@ -444,7 +461,7 @@ public abstract class Geary.Folder : BaseObject { * accessing the remote store before OpenState.BOTH has been signalled will result in that * call blocking until the remote is open or an error state has occurred. It's also possible for * the command to return early without waiting, depending on prior information of the folder. - * See list_email_async() for special notes on its operation. Also see wait_for_open_async(). + * See list_email_async() for special notes on its operation. Also see wait_for_remote_async(). * * If there's an error while opening, "open-failed" will be fired. (See that signal for more * information on how many times it may fire, and when.) To prevent the Folder from going into @@ -462,18 +479,20 @@ public abstract class Geary.Folder : BaseObject { * Returns false if already opened. */ public abstract async bool open_async(OpenFlags open_flags, Cancellable? cancellable = null) throws Error; - + /** - * Wait for the Folder to become fully open or fails to open due to error. If not opened - * due to error, throws EngineError.ALREADY_CLOSED. + * Blocks waiting for the folder to establish a remote session. * - * NOTE: The current implementation requirements are only that should be work after an - * open_async() call has completed (i.e. an open is in progress). Calling this method - * otherwise will throw an EngineError.OPEN_REQUIRED. + * @throws EngineError.OPEN_REQUIRED if the folder has not already + * been opened. + * @throws EngineError.ALREADY_CLOSED if not opened due to error. */ - public abstract async void wait_for_open_async(Cancellable? cancellable = null) throws Error; - + public abstract async void wait_for_remote_async(Cancellable? cancellable = null) + throws Error; + /** + * Marks one use of the folder's operations as being completed. + * * The Folder should be closed when operations on it are concluded. Depending on the * implementation this might entail closing a network connection or reverting it to another * state, or closing file handles or database connections. @@ -487,13 +506,15 @@ public abstract class Geary.Folder : BaseObject { * {@link wait_for_close_async} to block until the folder is completely closed. Otherwise, * returns false. Note that this semantic is slightly different than the result code for * {@link open_async}. + * + * @see open_async */ public abstract async bool close_async(Cancellable? cancellable = null) throws Error; - + /** * Wait for the {@link Folder} to fully close. * - * Unlike {@link wait_for_open_async}, this will ''always'' block until a {@link Folder} is + * Unlike {@link wait_for_remote_async}, this will ''always'' block until a {@link Folder} is * closed, even if it's not open. */ public abstract async void wait_for_close_async(Cancellable? cancellable = null) throws Error; diff --git a/src/engine/imap-db/imap-db-account.vala b/src/engine/imap-db/imap-db-account.vala index 37c22505..a5faad19 100644 --- a/src/engine/imap-db/imap-db-account.vala +++ b/src/engine/imap-db/imap-db-account.vala @@ -65,7 +65,6 @@ private class Geary.ImapDB.Account : BaseObject { // Only available when the Account is opened public SmtpOutboxFolder? outbox { get; private set; default = null; } - public Geary.SearchFolder? search_folder { get; private set; default = null; } public ImapEngine.ContactStore contact_store { get; private set; } public IntervalProgressMonitor search_index_monitor { get; private set; default = new IntervalProgressMonitor(ProgressType.SEARCH_INDEX, 0, 0); } @@ -340,11 +339,8 @@ private class Geary.ImapDB.Account : BaseObject { // ImapDB.Account holds the Outbox, which is tied to the database it maintains outbox = new SmtpOutboxFolder(db, account, sending_monitor); outbox.email_sent.connect(on_outbox_email_sent); - - // Search folder - search_folder = ((ImapEngine.GenericAccount) account).new_search_folder(); } - + public async void close_async(Cancellable? cancellable) throws Error { if (db == null) return; @@ -361,9 +357,8 @@ private class Geary.ImapDB.Account : BaseObject { outbox.email_sent.disconnect(on_outbox_email_sent); outbox = null; - search_folder = null; } - + private void on_outbox_email_sent(Geary.RFC822.Message rfc822) { email_sent(rfc822); } @@ -519,12 +514,14 @@ private class Geary.ImapDB.Account : BaseObject { Geary.FolderPath path = (parent != null) ? parent.get_child(basename) : new Imap.FolderRoot(basename); - - Geary.Imap.FolderProperties properties = new Geary.Imap.FolderProperties( - result.int_for("last_seen_total"), result.int_for("unread_count"), 0, + + Geary.Imap.FolderProperties properties = new Geary.Imap.FolderProperties.from_imapdb( + Geary.Imap.MailboxAttributes.deserialize(result.string_for("attributes")), + result.int_for("last_seen_total"), + result.int_for("unread_count"), new Imap.UIDValidity(result.int64_for("uid_validity")), - new Imap.UID(result.int64_for("uid_next")), - Geary.Imap.MailboxAttributes.deserialize(result.string_for("attributes"))); + new Imap.UID(result.int64_for("uid_next")) + ); // due to legacy code, can't set last_seen_total to -1 to indicate that the folder // hasn't been SELECT/EXAMINE'd yet, so the STATUS count should be used as the // authoritative when the other is zero ... this is important when first creating a @@ -608,17 +605,21 @@ private class Geary.ImapDB.Account : BaseObject { Db.Result results = stmt.exec(cancellable); if (!results.finished) { - properties = new Imap.FolderProperties(results.int_for("last_seen_total"), - results.int_for("unread_count"), 0, + properties = new Imap.FolderProperties.from_imapdb( + Geary.Imap.MailboxAttributes.deserialize(results.string_for("attributes")), + results.int_for("last_seen_total"), + results.int_for("unread_count"), new Imap.UIDValidity(results.int64_for("uid_validity")), - new Imap.UID(results.int64_for("uid_next")), - Geary.Imap.MailboxAttributes.deserialize(results.string_for("attributes"))); + new Imap.UID(results.int64_for("uid_next")) + ); // due to legacy code, can't set last_seen_total to -1 to indicate that the folder // hasn't been SELECT/EXAMINE'd yet, so the STATUS count should be used as the // authoritative when the other is zero ... this is important when first creating a // folder, as the STATUS is the count that is known first - properties.set_status_message_count(results.int_for("last_seen_status_total"), - (properties.select_examine_messages == 0)); + properties.set_status_message_count( + results.int_for("last_seen_status_total"), + (properties.select_examine_messages == 0) + ); } return Db.TransactionOutcome.DONE; diff --git a/src/engine/imap-db/imap-db-folder.vala b/src/engine/imap-db/imap-db-folder.vala index 56651f62..8c236e10 100644 --- a/src/engine/imap-db/imap-db-folder.vala +++ b/src/engine/imap-db/imap-db-folder.vala @@ -271,48 +271,6 @@ private class Geary.ImapDB.Folder : BaseObject, Geary.ReferenceSemantics { properties.set_select_examine_message_count(count); } - public async Imap.StatusData fetch_status_data(ListFlags flags, Cancellable? cancellable) throws Error { - Imap.StatusData? status_data = null; - yield db.exec_transaction_async(Db.TransactionType.RO, (cx) => { - Db.Statement stmt = cx.prepare(""" - SELECT uid_next, uid_validity, unread_count - FROM FolderTable - WHERE id = ? - """); - stmt.bind_rowid(0, folder_id); - - Db.Result result = stmt.exec(cancellable); - if (result.finished) - return Db.TransactionOutcome.DONE; - - int messages = do_get_email_count(cx, flags, cancellable); - Imap.UID? uid_next = !result.is_null_for("uid_next") - ? new Imap.UID(result.int64_for("uid_next")) - : null; - Imap.UIDValidity? uid_validity = !result.is_null_for("uid_validity") - ? new Imap.UIDValidity(result.int64_for("uid_validity")) - : null; - - // Note that recent is not stored - status_data = new Imap.StatusData( - // XXX using to_string here very sketchy - new Imap.MailboxSpecifier(this.path.to_string()), - messages, - 0, - uid_next, - uid_validity, - result.int_for("unread_count") - ); - - return Db.TransactionOutcome.DONE; - }, cancellable); - - if (status_data == null) - throw new EngineError.NOT_FOUND("%s STATUS not found in database", path.to_string()); - - return status_data; - } - // Returns a Map with the created or merged email as the key and the result of the operation // (true if created, false if merged) as the value. Note that every email // object passed in's EmailIdentifier will be fully filled out by this diff --git a/src/engine/imap-engine/gmail/imap-engine-gmail-account.vala b/src/engine/imap-engine/gmail/imap-engine-gmail-account.vala index 971b2cb9..ea43d616 100644 --- a/src/engine/imap-engine/gmail/imap-engine-gmail-account.vala +++ b/src/engine/imap-engine/gmail/imap-engine-gmail-account.vala @@ -21,7 +21,7 @@ private class Geary.ImapEngine.GmailAccount : Geary.ImapEngine.GenericAccount { Geary.Endpoint.Flags.SSL, Imap.ClientConnection.RECOMMENDED_TIMEOUT_SEC); } - + public static Geary.Endpoint generate_smtp_endpoint() { return new Geary.Endpoint( "smtp.gmail.com", @@ -30,9 +30,10 @@ private class Geary.ImapEngine.GmailAccount : Geary.ImapEngine.GenericAccount { Smtp.ClientConnection.DEFAULT_TIMEOUT_SEC); } - public GmailAccount(string name, Geary.AccountInformation account_information, - Imap.Account remote, ImapDB.Account local) { - base (name, account_information, remote, local); + public GmailAccount(string name, + Geary.AccountInformation account_information, + ImapDB.Account local) { + base(name, account_information, local); } protected override Geary.SpecialFolderType[] get_supported_special_folders() { diff --git a/src/engine/imap-engine/gmail/imap-engine-gmail-folder.vala b/src/engine/imap-engine/gmail/imap-engine-gmail-folder.vala index ccaa5820..fb2d9259 100644 --- a/src/engine/imap-engine/gmail/imap-engine-gmail-folder.vala +++ b/src/engine/imap-engine/gmail/imap-engine-gmail-folder.vala @@ -65,27 +65,21 @@ private class Geary.ImapEngine.GmailFolder : MinimalFolder, FolderSupport.Archiv return; } - - // For speed reasons, use a detached Imap.Folder object to delete moved emails; this is a + + // For speed reasons, use a standalone Imap.Folder object to delete moved emails; this is a // separate connection and is not synchronized with the database, but also avoids a full // folder normalization, which can be a heavyweight operation - Imap.Folder imap_trash = yield ((GenericAccount) folder.account).fetch_detached_folder_async( - trash.path, cancellable); - - yield imap_trash.open_async(cancellable); + GenericAccount account = (GenericAccount) folder.account; + Imap.FolderSession imap_trash = yield account.open_folder_session( + trash.path, cancellable + ); try { yield imap_trash.remove_email_async(Imap.MessageSet.uid_sparse(uids), cancellable); } finally { - try { - // don't use cancellable, need to close this connection no matter what - yield imap_trash.close_async(null); - } catch (Error err) { - // ignored - } + account.release_folder_session(imap_trash); } - + debug("%s: Successfully true-removed %d/%d emails", folder.to_string(), uids.size, email_ids.size); } } - diff --git a/src/engine/imap-engine/imap-engine-account-synchronizer.vala b/src/engine/imap-engine/imap-engine-account-synchronizer.vala index a14fdedb..c7ed9951 100644 --- a/src/engine/imap-engine/imap-engine-account-synchronizer.vala +++ b/src/engine/imap-engine/imap-engine-account-synchronizer.vala @@ -106,7 +106,7 @@ private class Geary.ImapEngine.RefreshFolderSync : FolderOperation { try { yield this.folder.open_async(Folder.OpenFlags.FAST_OPEN, cancellable); opened = true; - yield this.folder.wait_for_open_async(cancellable); + yield this.folder.wait_for_remote_async(cancellable); yield sync_folder(cancellable); } finally { if (opened) { diff --git a/src/engine/imap-engine/imap-engine-generic-account.vala b/src/engine/imap-engine/imap-engine-generic-account.vala index 03816c80..aefed4bf 100644 --- a/src/engine/imap-engine/imap-engine-generic-account.vala +++ b/src/engine/imap-engine/imap-engine-generic-account.vala @@ -8,6 +8,10 @@ private abstract class Geary.ImapEngine.GenericAccount : Geary.Account { + + /** Default IMAP session pool size. */ + private const int IMAP_MIN_POOL_SIZE = 2; + // This is high since it's an expensive operation, and we'll go // looking changes caused by local operations as they happen, so // we don't need to double check. @@ -24,27 +28,49 @@ private abstract class Geary.ImapEngine.GenericAccount : Geary.Account { private static Geary.FolderPath? outbox_path = null; private static Geary.FolderPath? search_path = null; - internal Imap.Account remote { get; private set; } + /** This account's IMAP session pool. */ + public Imap.ClientSessionManager session_pool { get; private set; } + internal ImapDB.Account local { get; private set; } private bool open = false; + private Cancellable? open_cancellable = null; + + private Geary.SearchFolder? search_folder { get; private set; default = null; } + + private Nonblocking.Mutex remote_open_lock = new Nonblocking.Mutex(); + private Nonblocking.Semaphore? remote_ready_lock = null; + private Imap.AccountSession? remote_session { get; private set; default = null; } + private Gee.HashMap folder_map = new Gee.HashMap< FolderPath, MinimalFolder>(); private Gee.HashMap local_only = new Gee.HashMap(); + private AccountProcessor? processor; private AccountSynchronizer sync; private TimeoutManager refresh_folder_timer; + private uint authentication_failures = 0; + + private Gee.Map> special_search_names = new Gee.HashMap>(); - public GenericAccount(string name, Geary.AccountInformation information, - Imap.Account remote, ImapDB.Account local) { - base (name, information); + public GenericAccount(string name, + Geary.AccountInformation information, + ImapDB.Account local) { + base(name, information); - this.remote = remote; - this.remote.report_problem.connect(notify_report_problem); + this.session_pool = new Imap.ClientSessionManager( + this.information.id, + this.information.get_imap_endpoint(), + this.information.imap_credentials + ); + this.session_pool.min_pool_size = IMAP_MIN_POOL_SIZE; + this.session_pool.ready.connect(on_pool_session_ready); + this.session_pool.connection_failed.connect(on_pool_connection_failed); + this.session_pool.login_failed.connect(on_pool_login_failed); this.local = local; this.local.contacts_loaded.connect(() => { contacts_loaded(); }); @@ -74,89 +100,40 @@ private abstract class Geary.ImapEngine.GenericAccount : Geary.Account { compile_special_search_names(); } - /** - * Queues an operation for execution by this account. - * - * The operation will added to the account's {@link - * AccountProcessor} and executed asynchronously by that when it - * reaches the front. - */ - public void queue_operation(AccountOperation op) - throws EngineError { - check_open(); - debug("%s: Enqueuing operation: %s", this.to_string(), op.to_string()); - this.processor.enqueue(op); - } - - protected override void notify_folders_available_unavailable(Gee.List? available, - Gee.List? unavailable) { - base.notify_folders_available_unavailable(available, unavailable); - if (available != null) { - foreach (Geary.Folder folder in available) { - folder.email_appended.connect(notify_email_appended); - folder.email_inserted.connect(notify_email_inserted); - folder.email_removed.connect(notify_email_removed); - folder.email_locally_complete.connect(notify_email_locally_complete); - folder.email_flags_changed.connect(notify_email_flags_changed); - } - } - if (unavailable != null) { - foreach (Geary.Folder folder in unavailable) { - folder.email_appended.disconnect(notify_email_appended); - folder.email_inserted.disconnect(notify_email_inserted); - folder.email_removed.disconnect(notify_email_removed); - folder.email_locally_complete.disconnect(notify_email_locally_complete); - folder.email_flags_changed.disconnect(notify_email_flags_changed); - } - } - } - - protected override void notify_email_appended(Geary.Folder folder, Gee.Collection ids) { - base.notify_email_appended(folder, ids); - schedule_unseen_update(folder); - } - - protected override void notify_email_inserted(Geary.Folder folder, Gee.Collection ids) { - base.notify_email_inserted(folder, ids); - schedule_unseen_update(folder); - } - - protected override void notify_email_removed(Geary.Folder folder, Gee.Collection ids) { - base.notify_email_removed(folder, ids); - schedule_unseen_update(folder); - } - - protected override void notify_email_flags_changed(Geary.Folder folder, - Gee.Map flag_map) { - base.notify_email_flags_changed(folder, flag_map); - schedule_unseen_update(folder); - } - - private void check_open() throws EngineError { - if (!open) - throw new EngineError.OPEN_REQUIRED("Account %s not opened", to_string()); - } - + /** {@inheritDoc} */ public override async void open_async(Cancellable? cancellable = null) throws Error { if (open) throw new EngineError.ALREADY_OPEN("Account %s already opened", to_string()); - + opening_monitor.notify_start(); - - Error? throw_err = null; try { yield internal_open_async(cancellable); - } catch (Error err) { - throw_err = err; + } finally { + opening_monitor.notify_finish(); } - - opening_monitor.notify_finish(); - - if (throw_err != null) - throw throw_err; } - + private async void internal_open_async(Cancellable? cancellable) throws Error { + this.open_cancellable = new Cancellable(); + this.remote_ready_lock = new Nonblocking.Semaphore(this.open_cancellable); + + // Reset this so we start trying to authenticate again + this.authentication_failures = 0; + + // To prevent spurious connection failures, we make sure we have the + // IMAP password before attempting a connection. This might have to be + // reworked when we allow passwordless logins. + if (!this.information.imap_credentials.is_complete()) + yield this.information.get_passwords_async(ServiceFlag.IMAP); + + this.session_pool.credentials_updated( + this.information.imap_credentials + ); + + // This will cause the session manager to open at least one + // connection if we are online + yield this.session_pool.open_async(cancellable); + this.processor = new AccountProcessor(this.to_string()); this.processor.operation_error.connect(on_operation_error); @@ -174,36 +151,16 @@ private abstract class Geary.ImapEngine.GenericAccount : Geary.Account { else throw err; } - - // outbox is now available + + // Local folders + local.outbox.report_problem.connect(notify_report_problem); local_only.set(outbox_path, local.outbox); - - // Search folder. - local_only.set(search_path, local.search_folder); - - // To prevent spurious connection failures, we make sure we have the - // IMAP password before attempting a connection. This might have to be - // reworked when we allow passwordless logins. - if (!information.imap_credentials.is_complete()) - yield information.get_passwords_async(ServiceFlag.IMAP); - // need to back out local.open_async() if remote fails - try { - yield remote.open_async(cancellable); - } catch (Error err) { - // back out - try { - yield local.close_async(cancellable); - } catch (Error close_err) { - // ignored - } - - throw err; - } + this.search_folder = new_search_folder(); + local_only.set(search_path, this.search_folder); this.open = true; - notify_opened(); notify_folders_available_unavailable(sort_by_path(local_only.values), null); @@ -211,9 +168,9 @@ private abstract class Geary.ImapEngine.GenericAccount : Geary.Account { new LoadFolders(this, this.local, get_supported_special_folders()) ); - this.remote.ready.connect(on_remote_ready); - if (this.remote.is_ready) { - this.update_remote_folders(); + // If the pool is already ready, let's go get a session! + if (this.session_pool.is_ready) { + this.open_remote_session.begin(cancellable); } } @@ -221,16 +178,16 @@ private abstract class Geary.ImapEngine.GenericAccount : Geary.Account { if (!open) return; - this.remote.prepare_to_close(); - this.remote.ready.disconnect(on_remote_ready); + // Stop trying to re-use IMAP server connections + this.session_pool.discard_returned_sessions = true; // Halt internal tasks early so they stop using local and // remote connections. + this.refresh_folder_timer.reset(); + this.open_cancellable.cancel(); this.processor.stop(); - this.refresh_folder_timer.reset(); - - // Notify folders and ensure they are closed + // Close folders and ensure they do in fact close Gee.List locals = sort_by_path(this.local_only.values); Gee.List remotes = sort_by_path(this.folder_map.values); @@ -250,38 +207,36 @@ private abstract class Geary.ImapEngine.GenericAccount : Geary.Account { yield folder.wait_for_close_async(); } - this.local.outbox.report_problem.disconnect(notify_report_problem); + // Close remote infrastructure - // Close accounts - Error? local_err = null; + yield close_remote_session(cancellable); + this.remote_ready_lock = null; + try { + yield this.session_pool.close_async(cancellable); + } catch (Error err) { + debug("%s: Error closing IMAP session pool: %s", + to_string(), + this.session_pool.to_string() + ); + } + + // Close local infrastructure + + this.search_folder = null; + this.local.outbox.report_problem.disconnect(notify_report_problem); try { yield local.close_async(cancellable); - } catch (Error lclose_err) { - local_err = lclose_err; + } finally { + this.open = false; + notify_closed(); } - - Error? remote_err = null; - try { - yield remote.close_async(cancellable); - } catch (Error rclose_err) { - remote_err = rclose_err; - } - - this.open = false; - - notify_closed(); - - if (local_err != null) - throw local_err; - - if (remote_err != null) - throw remote_err; } - + + /** {@inheritDoc} */ public override bool is_open() { return open; } - + public override async void rebuild_async(Cancellable? cancellable = null) throws Error { if (open) throw new EngineError.ALREADY_OPEN("Account cannot be open during rebuild"); @@ -308,7 +263,7 @@ private abstract class Geary.ImapEngine.GenericAccount : Geary.Account { } /** - * This starts the outbox postman running. + * Starts the outbox postman running. */ public override async void start_outgoing_client() throws Error { @@ -317,18 +272,115 @@ private abstract class Geary.ImapEngine.GenericAccount : Geary.Account { } /** - * This closes then reopens the IMAP account. + * Closes then reopens the IMAP account if it is not ready. */ public override async void start_incoming_client() throws Error { check_open(); + if (!this.session_pool.is_ready) { + try { + yield this.session_pool.close_async(this.open_cancellable); + } catch (Error err) { + debug("Ignoring error closing IMAP session pool for restart: %s", + err.message); + } + + yield this.session_pool.open_async(this.open_cancellable); + } + } + + /** + * Queues an operation for execution by this account. + * + * The operation will added to the account's {@link + * AccountProcessor} and executed asynchronously by that when it + * reaches the front. + */ + public void queue_operation(AccountOperation op) + throws EngineError { + check_open(); + debug("%s: Enqueuing operation: %s", this.to_string(), op.to_string()); + this.processor.enqueue(op); + } + + /** + * Returns a valid IMAP account session when one is available. + * + * Implementations may use this to acquire an IMAP session for + * performing account-related work. The call will wait until a + * connection is established then return the session. + * + * The session returned is guaranteed to be open upon return, + * however may close afterwards due to this account closing, or + * the network connection going away. + * + * The account must have been opened before calling this method. + */ + public async Imap.AccountSession claim_account_session(Cancellable? cancellable = null) + throws Error { + check_open(); + debug("%s: Acquiring account session", this.to_string()); + yield this.remote_ready_lock.wait_async(cancellable); + return this.remote_session; + } + + /** + * Establishes a new IMAP folder session. + * + * A new IMAP client session will be retrieved from the pool, + * connecting if needed, and used for a new folder session. This + * call will wait until the pool is ready to provide sessions. The + * session must be returned via {@link release_folder_session} + * after use. + */ + public async Imap.FolderSession open_folder_session(Geary.FolderPath path, + Cancellable cancellable) + throws Error { + check_open(); + debug("%s: Opening account session", this.to_string()); + Imap.ClientSession? client = null; + Imap.Folder? folder = null; try { - yield this.remote.close_async(); + // Do the claim_account_session first ensure the pool is + // ready. + Imap.AccountSession account = yield claim_account_session(); + folder = yield account.fetch_folder_async(path, cancellable); + client = yield this.session_pool.claim_authorized_session_async( + cancellable + ); } catch (Error err) { - debug("Ignoring error closing IMAP account for restart: %s", err.message); + if (client != null) { + yield this.session_pool.release_session_async(client); + } + throw err; } - yield this.remote.open_async(); + return yield new Imap.FolderSession( + this.information.id, client, folder, cancellable + ); + } + + /** + * Returns an IMAP folder session to the pool for cleanup and re-use. + */ + public void release_folder_session(Imap.FolderSession session) { + debug("%s: Releasing folder session", this.to_string()); + Imap.ClientSession? old_session = session.close(); + if (old_session != null) { + this.session_pool.release_session_async.begin( + old_session, + (obj, res) => { + try { + this.session_pool.release_session_async.end(res); + } catch (Error err) { + debug("%s: Error releasing %s session: %s", + to_string(), + session.folder.path.to_string(), + err.message); + } + } + ); + } } public override Gee.Collection list_matching_folders(Geary.FolderPath? parent) @@ -358,16 +410,15 @@ private abstract class Geary.ImapEngine.GenericAccount : Geary.Account { return local.contact_store; } + /** {@inheritDoc} */ public override async bool folder_exists_async(Geary.FolderPath path, - Cancellable? cancellable = null) throws Error { + Cancellable? cancellable = null) + throws Error { check_open(); - - if (yield local.folder_exists_async(path, cancellable)) - return true; - - return yield remote.folder_exists_async(path, cancellable); + return this.local_only.has_key(path) || this.folder_map.has_key(path); } + /** {@inheritDoc} */ public override async Geary.Folder fetch_folder_async(Geary.FolderPath path, Cancellable? cancellable = null) throws Error { @@ -384,39 +435,6 @@ private abstract class Geary.ImapEngine.GenericAccount : Geary.Account { return folder; } - /** - * Returns an Imap.Folder that is not connected (is detached) to a - * MinimalFolder or any other ImapEngine container. - * - * This is useful for one-shot operations that need to bypass the - * heavyweight synchronization routines inside MinimalFolder. - * This also means that operations performed on this Folder will - * not be reflected in the local database unless there's a - * separate connection to the server that is notified or detects - * these changes. - * - * The returned Folder must be opened prior to use and closed once - * completed. - * - * ''Leaving a Folder open will cause a connection leak.'' - * - * It is not recommended this object be held open long-term, or - * that its status or notifications be directly written to the - * database unless you know exactly what you're doing. - * ''Caveat implementor.'' - */ - public async Imap.Folder fetch_detached_folder_async(Geary.FolderPath path, Cancellable? cancellable) - throws Error { - check_open(); - - if (local_only.has_key(path)) { - throw new EngineError.NOT_FOUND("%s: path %s points to local-only folder, not IMAP", - to_string(), path.to_string()); - } - - return yield remote.fetch_folder_async(path, cancellable); - } - public override async Geary.Folder get_required_special_folder_async(Geary.SpecialFolderType special, Cancellable? cancellable) throws Error { if (!(special in get_supported_special_folders())) { @@ -500,12 +518,6 @@ private abstract class Geary.ImapEngine.GenericAccount : Geary.Account { return yield local.get_containing_folders_async(ids, cancellable); } - // Subclasses with specific SearchFolder implementations should override - // this to return the correct subclass. - internal virtual SearchFolder new_search_folder() { - return new ImapDB.SearchFolder(this); - } - /** * Constructs a set of folders and adds them to the account. * @@ -565,6 +577,37 @@ private abstract class Geary.ImapEngine.GenericAccount : Geary.Account { } } + /** + * Marks a folder as a specific special folder type. + */ + internal void promote_folders(Gee.Map specials) { + Gee.Set changed = new Gee.HashSet(); + foreach (Geary.SpecialFolderType special in specials.keys) { + MinimalFolder? minimal = specials.get(special) as MinimalFolder; + if (minimal.special_folder_type != special) { + minimal.set_special_folder_type(special); + changed.add(minimal); + + MinimalFolder? existing = null; + try { + existing = get_special_folder(special) as MinimalFolder; + } catch (Error err) { + debug("%s: Error getting special folder: %s", + to_string(), err.message); + } + + if (existing != null && existing != minimal) { + existing.set_special_folder_type(SpecialFolderType.NONE); + changed.add(existing); + } + } + } + + if (!changed.is_empty) { + folders_special_type(changed); + } + } + /** * Removes a set of folders from the account. * @@ -604,6 +647,7 @@ private abstract class Geary.ImapEngine.GenericAccount : Geary.Account { if (folder != null) return folder; + Imap.AccountSession account = yield claim_account_session(); MinimalFolder? minimal_folder = null; Geary.FolderPath? path = information.get_special_folder_path(special); if (path != null) { @@ -611,8 +655,8 @@ private abstract class Geary.ImapEngine.GenericAccount : Geary.Account { } else { // This is the first time we're turning a non-special folder into a special one. // After we do this, we'll record which one we picked in the account info. - - Geary.FolderPath root = yield remote.get_default_personal_namespace(cancellable); + Geary.FolderPath root = + yield account.get_default_personal_namespace(cancellable); Gee.List search_names = special_search_names.get(special); foreach (string search_name in search_names) { Geary.FolderPath search_path = root.get_child(search_name); @@ -639,21 +683,165 @@ private abstract class Geary.ImapEngine.GenericAccount : Geary.Account { } else { debug("Creating %s to use as special folder %s", path.to_string(), special.to_string()); // TODO: ignore error due to already existing. - yield remote.create_folder_async(path, special, cancellable); + yield account.create_folder_async(path, special, cancellable); minimal_folder = (MinimalFolder) yield fetch_folder_async(path, cancellable); } - minimal_folder.set_special_folder_type(special); + Gee.Map specials = + new Gee.HashMap(); + specials.set(special, minimal_folder); + promote_folders(specials); + return minimal_folder; } - // Subclasses should implement this to return their flavor of a MinimalFolder with the - // appropriate interfaces attached. The returned folder should have its SpecialFolderType - // set using either the properties from the local folder or its path. - // - // This won't be called to build the Outbox or search folder, but for all others (including Inbox) it will. + /** + * Constructs a concrete folder implementation. + * + * Subclasses should implement this to return their flavor of a + * MinimalFolder with the appropriate interfaces attached. The + * returned folder should have its SpecialFolderType set using + * either the properties from the local folder or its path. + * + * This won't be called to build the Outbox or search folder, but + * for all others (including Inbox) it will. + */ protected abstract MinimalFolder new_folder(ImapDB.Folder local_folder); + /** + * Constructs a concrete search folder implementation. + * + * Subclasses with specific SearchFolder implementations should + * override this to return the correct subclass. + */ + protected virtual SearchFolder new_search_folder() { + return new ImapDB.SearchFolder(this); + } + + /** {@inheritDoc} */ + protected override void notify_folders_available_unavailable(Gee.List? available, + Gee.List? unavailable) { + base.notify_folders_available_unavailable(available, unavailable); + if (available != null) { + foreach (Geary.Folder folder in available) { + folder.email_appended.connect(notify_email_appended); + folder.email_inserted.connect(notify_email_inserted); + folder.email_removed.connect(notify_email_removed); + folder.email_locally_complete.connect(notify_email_locally_complete); + folder.email_flags_changed.connect(notify_email_flags_changed); + } + } + if (unavailable != null) { + foreach (Geary.Folder folder in unavailable) { + folder.email_appended.disconnect(notify_email_appended); + folder.email_inserted.disconnect(notify_email_inserted); + folder.email_removed.disconnect(notify_email_removed); + folder.email_locally_complete.disconnect(notify_email_locally_complete); + folder.email_flags_changed.disconnect(notify_email_flags_changed); + } + } + } + + /** {@inheritDoc} */ + protected override void notify_email_appended(Geary.Folder folder, Gee.Collection ids) { + base.notify_email_appended(folder, ids); + schedule_unseen_update(folder); + } + + /** {@inheritDoc} */ + protected override void notify_email_inserted(Geary.Folder folder, Gee.Collection ids) { + base.notify_email_inserted(folder, ids); + schedule_unseen_update(folder); + } + + /** {@inheritDoc} */ + protected override void notify_email_removed(Geary.Folder folder, Gee.Collection ids) { + base.notify_email_removed(folder, ids); + schedule_unseen_update(folder); + } + + /** {@inheritDoc} */ + protected override void notify_email_flags_changed(Geary.Folder folder, + Gee.Map flag_map) { + base.notify_email_flags_changed(folder, flag_map); + schedule_unseen_update(folder); + } + + /** Fires a {@link report_problem} signal for an IMAP service. */ + protected void notify_imap_problem(Geary.ProblemType type, Error? err) { + notify_service_problem(type, Service.IMAP, err); + } + + /** + * Establishes a new account session with the IMAP server. + */ + private async void open_remote_session(Cancellable cancellable) { + try { + int token = yield this.remote_open_lock.claim_async(cancellable); + if (this.remote_session != null) { + return; + } + + try { + check_open(); + debug("%s: Opening remote session", to_string()); + Imap.ClientSession client = + yield this.session_pool.claim_authorized_session_async( + cancellable + ); + + this.remote_session = new Imap.AccountSession( + this.information.id, client + ); + this.remote_session.disconnected.connect(on_remote_disconnect); + + this.remote_ready_lock.notify(); + } catch (Error err) { + notify_imap_problem(ProblemType.CONNECTION_ERROR, err); + } + + this.remote_open_lock.release(ref token); + + // Now we have a valid remote session again, update our idea + // of what the remote folders are in case they have changed + update_remote_folders(); + } catch (Error err) { + // Oh well + } + } + + /** + * Drops the current account session, if any. + */ + private async void close_remote_session(Cancellable cancellable) { + try { + int token = yield this.remote_open_lock.claim_async(cancellable); + if (this.remote_session == null) { + return; + } + + try { + this.remote_ready_lock.reset(); + + Imap.ClientSession? old_session = this.remote_session.close(); + + this.remote_session.disconnected.connect(on_remote_disconnect); + this.remote_session = null; + + if (old_session != null) { + yield this.session_pool.release_session_async(old_session); + } + } catch (Error err) { + debug("%s: Error closing remote session: %s", + this.to_string(), err.message); + } + + this.remote_open_lock.release(ref token); + } catch (Error err) { + // Oh well + } + } + /** * Hooks up and queues an {@link UpdateRemoteFolders} operation. */ @@ -662,8 +850,6 @@ private abstract class Geary.ImapEngine.GenericAccount : Geary.Account { UpdateRemoteFolders op = new UpdateRemoteFolders( this, - this.remote, - this.local, this.local_only.keys, get_supported_special_folders() ); @@ -791,8 +977,9 @@ private abstract class Geary.ImapEngine.GenericAccount : Geary.Account { return loc_names; } - private void on_remote_ready() { - this.update_remote_folders(); + private void check_open() throws EngineError { + if (!open) + throw new EngineError.OPEN_REQUIRED("Account %s not opened", to_string()); } private void on_operation_error(AccountOperation op, Error error) { @@ -808,6 +995,78 @@ private abstract class Geary.ImapEngine.GenericAccount : Geary.Account { } } + private void on_pool_session_ready() { + // Now have a valid session, so credentials must be good + this.authentication_failures = 0; + this.open_remote_session.begin(this.open_cancellable); + } + + private void on_pool_connection_failed(Error error) { + if (error is ImapError.UNAUTHENTICATED) { + // This is effectively a login failure + on_pool_login_failed(null); + } else { + notify_imap_problem(ProblemType.CONNECTION_ERROR, error); + } + } + + private void on_pool_login_failed(Geary.Imap.StatusResponse? response) { + this.authentication_failures++; + if (this.authentication_failures >= Geary.Account.AUTH_ATTEMPTS_MAX) { + // We have tried auth too many times, so bail out + notify_imap_problem(ProblemType.LOGIN_FAILED, null); + } else { + // login can fail due to an invalid password hence we + // should re-ask it, but it can also fail due to server + // inaccessibility, for instance "[UNAVAILABLE] / Maximum + // number of connections from user+IP exceeded". In that + // case, resetting password seems unneeded. + bool reask_password = false; + Error? login_error = null; + try { + reask_password = ( + response == null || + response.response_code == null || + response.response_code.get_response_code_type().value != Geary.Imap.ResponseCodeType.UNAVAILABLE + ); + } catch (ImapError err) { + login_error = err; + debug("Unable to parse ResponseCode %s: %s", response.response_code.to_string(), + err.message); + } + + if (!reask_password) { + // Either the server was unavailable, or we were unable to + // parse the login response. Either way, indicate a + // non-login error. + notify_imap_problem(ProblemType.SERVER_ERROR, login_error); + } else { + // Now, we should ask the user for their password + this.information.fetch_passwords_async.begin( + ServiceFlag.IMAP, true, + (obj, ret) => { + try { + if (this.information.fetch_passwords_async.end(ret)) { + // Have a new password, so try that + this.session_pool.credentials_updated( + this.information.imap_credentials + ); + } else { + // User cancelled, so indicate a login problem + notify_imap_problem(ProblemType.LOGIN_FAILED, null); + } + } catch (Error err) { + notify_imap_problem(ProblemType.GENERIC_ERROR, err); + } + }); + } + } + } + + private void on_remote_disconnect(Imap.ClientSession.DisconnectReason reason) { + this.close_remote_session.begin(this.open_cancellable); + } + } @@ -832,24 +1091,14 @@ internal class Geary.ImapEngine.LoadFolders : AccountOperation { public override async void execute(Cancellable cancellable) throws Error { GenericAccount generic = (GenericAccount) this.account; Gee.List folders = new Gee.LinkedList(); - yield enumerate_local_folders_async(folders, null, cancellable); - debug("%s: found %u folders", to_string(), folders.size); - generic.add_folders(folders, true); + yield enumerate_local_folders_async(folders, null, cancellable); + generic.add_folders(folders, true); if (!folders.is_empty) { // If we have some folders to load, then this isn't the // first run, and hence the special folders should already // exist - foreach (Geary.SpecialFolderType special in this.specials) { - try { - yield generic.ensure_special_folder_async(special, cancellable); - } catch (Error e) { - warning( - "Unable to ensure special folder %s: %s", - special.to_string(), e.message - ); - } - } + yield check_special_folders(cancellable); } } @@ -876,6 +1125,28 @@ internal class Geary.ImapEngine.LoadFolders : AccountOperation { } } } + + private async void check_special_folders(Cancellable cancellable) + throws Error { + GenericAccount generic = (GenericAccount) this.account; + Gee.Map specials = + new Gee.HashMap(); + foreach (Geary.SpecialFolderType special in this.specials) { + Geary.FolderPath? path = generic.information.get_special_folder_path(special); + if (path != null) { + try { + Geary.Folder target = yield generic.fetch_folder_async(path, cancellable); + specials.set(special, target); + } catch (Error err) { + debug("%s: Previously used special folder %s does not exist: %s", + generic.information.id, special.to_string(), err.message); + } + } + } + + generic.promote_folders(specials); + } + } @@ -886,40 +1157,41 @@ internal class Geary.ImapEngine.UpdateRemoteFolders : AccountOperation { private weak GenericAccount generic_account; - private weak Imap.Account remote; - private weak ImapDB.Account local; private Gee.Collection local_folders; private Geary.SpecialFolderType[] specials; internal UpdateRemoteFolders(GenericAccount account, - Imap.Account remote, - ImapDB.Account local, Gee.Collection local_folders, Geary.SpecialFolderType[] specials) { base(account); this.generic_account = account; - this.remote = remote; - this.local = local; this.local_folders = local_folders; this.specials = specials; } public override async void execute(Cancellable cancellable) throws Error { + Imap.AccountSession remote = + yield ((GenericAccount) this.account).claim_account_session(cancellable); + Gee.Map existing_folders = Geary.traverse(this.account.list_folders()) .to_hash_map(f => f.path); Gee.Map remote_folders = new Gee.HashMap(); + bool is_suspect = yield enumerate_remote_folders_async( - remote_folders, null, cancellable + remote, remote_folders, null, cancellable ); // pair the local and remote folders and make sure everything is up-to-date - yield update_folders_async(existing_folders, remote_folders, is_suspect, cancellable); + yield update_folders_async( + remote, existing_folders, remote_folders, is_suspect, cancellable + ); } - private async bool enumerate_remote_folders_async(Gee.Map folders, + private async bool enumerate_remote_folders_async(Imap.AccountSession remote, + Gee.Map folders, Geary.FolderPath? parent, Cancellable? cancellable) throws Error { @@ -927,7 +1199,7 @@ internal class Geary.ImapEngine.UpdateRemoteFolders : AccountOperation { Gee.List? children = null; try { - children = yield this.remote.fetch_child_folders_async(parent, cancellable); + children = yield remote.fetch_child_folders_async(parent, cancellable); } catch (Error err) { // ignore everything but I/O and IMAP errors (cancellation is an IOError) if (err is IOError || err is ImapError) @@ -942,7 +1214,8 @@ internal class Geary.ImapEngine.UpdateRemoteFolders : AccountOperation { FolderPath path = child.path; folders.set(path, child); if (child.properties.has_children.is_possible() && - yield enumerate_remote_folders_async(folders, path, cancellable)) { + yield enumerate_remote_folders_async( + remote, folders, path, cancellable)) { results_suspect = true; } } @@ -951,9 +1224,13 @@ internal class Geary.ImapEngine.UpdateRemoteFolders : AccountOperation { return results_suspect; } - private async void update_folders_async(Gee.Map existing_folders, - Gee.Map remote_folders, bool remote_folders_suspect, Cancellable? cancellable) { - // update all remote folders properties in the local store and active in the system + private async void update_folders_async(Imap.AccountSession remote, + Gee.Map existing_folders, + Gee.Map remote_folders, + bool remote_folders_suspect, + Cancellable? cancellable) { + // update all remote folders properties in the local store and + // active in the system Gee.HashSet altered_paths = new Gee.HashSet(); foreach (Imap.Folder remote_folder in remote_folders.values) { MinimalFolder? minimal_folder = existing_folders.get(remote_folder.path) @@ -979,7 +1256,7 @@ internal class Geary.ImapEngine.UpdateRemoteFolders : AccountOperation { ); } catch (Error update_error) { debug("Unable to update local folder %s with remote properties: %s", - remote_folder.to_string(), update_error.message); + remote_folder.path.to_string(), update_error.message); } // set the engine folder's special type @@ -1003,6 +1280,7 @@ internal class Geary.ImapEngine.UpdateRemoteFolders : AccountOperation { .to_array_list(); // For folders to add, clone them and their properties locally + ImapDB.Account local = ((GenericAccount) this.account).local; foreach (Geary.Imap.Folder remote_folder in to_add) { try { yield local.clone_folder_async(remote_folder, cancellable); @@ -1016,7 +1294,7 @@ internal class Geary.ImapEngine.UpdateRemoteFolders : AccountOperation { Gee.ArrayList to_build = new Gee.ArrayList(); foreach (Geary.Imap.Folder remote_folder in to_add) { try { - to_build.add(yield this.local.fetch_folder_async(remote_folder.path, cancellable)); + to_build.add(yield local.fetch_folder_async(remote_folder.path, cancellable)); } catch (Error convert_err) { // This isn't fatal, but irksome ... in the future, when local folders are // removed, it's possible for one to disappear between cloning it and fetching @@ -1037,14 +1315,14 @@ internal class Geary.ImapEngine.UpdateRemoteFolders : AccountOperation { foreach (Geary.Folder folder in removed) { try { debug("Locally deleting removed folder %s", folder.to_string()); - yield this.local.delete_folder_async(folder, cancellable); + yield local.delete_folder_async(folder, cancellable); } catch (Error e) { debug("Unable to locally delete removed folder %s: %s", folder.to_string(), e.message); } } // Let the remote know as well - this.remote.folders_removed( + remote.folders_removed( Geary.traverse(removed) .map(f => f.path).to_array_list() ); @@ -1091,13 +1369,15 @@ internal class Geary.ImapEngine.RefreshFolderUnseen : FolderOperation { } public override async void execute(Cancellable cancellable) throws Error { + Imap.AccountSession remote = + yield ((GenericAccount) this.account).claim_account_session(cancellable); + if (this.folder.get_open_state() == Geary.Folder.OpenState.CLOSED) { - Imap.Folder remote_folder = - yield ((GenericAccount) this.account).remote.fetch_folder_cached_async( - folder.path, - true, - cancellable - ); + Imap.Folder remote_folder = yield remote.fetch_folder_cached_async( + folder.path, + true, + cancellable + ); // Although this is called when the folder is closed, we // can safely use local_folder since we are only using its diff --git a/src/engine/imap-engine/imap-engine-minimal-folder.vala b/src/engine/imap-engine/imap-engine-minimal-folder.vala index e85f7ef3..f5a75d7b 100644 --- a/src/engine/imap-engine/imap-engine-minimal-folder.vala +++ b/src/engine/imap-engine/imap-engine-minimal-folder.vala @@ -55,12 +55,13 @@ private class Geary.ImapEngine.MinimalFolder : Geary.Folder, Geary.FolderSupport private ProgressMonitor _opening_monitor = new Geary.ReentrantProgressMonitor(Geary.ProgressType.ACTIVITY); public override Geary.ProgressMonitor opening_monitor { get { return _opening_monitor; } } - + internal ImapDB.Folder local_folder { get; protected set; } - internal Imap.Folder? remote_folder { get; protected set; default = null; } - internal EmailPrefetcher email_prefetcher { get; private set; } + internal Imap.FolderSession? remote_folder { get; protected set; default = null; } internal int remote_count { get; private set; default = -1; } + internal ReplayQueue replay_queue { get; private set; } + internal EmailPrefetcher email_prefetcher { get; private set; } private weak GenericAccount _account; private Geary.AggregatedFolderProperties _properties = @@ -68,9 +69,9 @@ private class Geary.ImapEngine.MinimalFolder : Geary.Folder, Geary.FolderSupport private Folder.OpenFlags open_flags = OpenFlags.NONE; private int open_count = 0; - private bool remote_opened = false; + private TimeoutManager remote_open_timer; - private Nonblocking.ReportingSemaphore remote_semaphore = + private Nonblocking.ReportingSemaphore remote_wait_semaphore = new Nonblocking.ReportingSemaphore(false); private Nonblocking.Semaphore closed_semaphore = new Nonblocking.Semaphore(); private Nonblocking.Mutex open_mutex = new Nonblocking.Mutex(); @@ -110,9 +111,6 @@ private class Geary.ImapEngine.MinimalFolder : Geary.Folder, Geary.FolderSupport ImapDB.Folder local_folder, SpecialFolderType special_folder_type) { this._account = account; - this.remote_open_timer = new TimeoutManager.seconds( - FORCE_OPEN_REMOTE_TIMEOUT_SEC, () => { start_open_remote(); } - ); this.local_folder = local_folder; this.local_folder.email_complete.connect(on_email_complete); @@ -121,6 +119,10 @@ private class Geary.ImapEngine.MinimalFolder : Geary.Folder, Geary.FolderSupport this.replay_queue = new ReplayQueue(this); this.email_prefetcher = new EmailPrefetcher(this); + this.remote_open_timer = new TimeoutManager.seconds( + FORCE_OPEN_REMOTE_TIMEOUT_SEC, () => { this.open_remote_session.begin(); } + ); + this.update_flags_timer = new TimeoutManager.seconds( FLAG_UPDATE_TIMEOUT_SEC, () => { on_update_flags.begin(); } ); @@ -175,19 +177,21 @@ private class Geary.ImapEngine.MinimalFolder : Geary.Folder, Geary.FolderSupport if (old_type != new_type) notify_special_folder_type_changed(old_type, new_type); } - + public override Geary.Folder.OpenState get_open_state() { - if (open_count == 0) + if (this.open_count == 0) return Geary.Folder.OpenState.CLOSED; - - return (remote_folder != null) ? Geary.Folder.OpenState.BOTH : Geary.Folder.OpenState.LOCAL; + + return (this.remote_folder != null) + ? Geary.Folder.OpenState.BOTH + : Geary.Folder.OpenState.LOCAL; } - + // Returns the synchronized remote count (-1 if not opened) and the last seen remote count (stored // locally, -1 if not available) // // Return value is the remote_count, unless the remote is unopened, in which case it's the - // last_seen_remote_count (which may be -1). + // last_seen_remote_count (which may also be -1). // // remote_count, last_seen_remote_count, and returned value do not reflect any notion of // messages marked for removal @@ -196,10 +200,100 @@ private class Geary.ImapEngine.MinimalFolder : Geary.Folder, Geary.FolderSupport last_seen_remote_count = local_folder.get_properties().select_examine_messages; if (last_seen_remote_count < 0) last_seen_remote_count = local_folder.get_properties().status_messages; - + return (remote_count >= 0) ? remote_count : last_seen_remote_count; } - + + /** {@inheritDoc} */ + public override async bool open_async(Geary.Folder.OpenFlags open_flags, Cancellable? cancellable = null) + throws Error { + if (open_count++ > 0) { + // even if opened or opening, or if forcing a re-open, respect the NO_DELAY flag + if (open_flags.is_all_set(OpenFlags.NO_DELAY)) { + // add NO_DELAY flag if it forces an open + if (this.remote_folder == null) + this.open_flags |= OpenFlags.NO_DELAY; + + this.open_remote_session.begin(); + } + return false; + } + + // first open gets to name the flags, but see note above + this.open_flags = open_flags; + + // reset to force waiting in wait_for_remote_async() + this.remote_wait_semaphore.reset(); + + // reset to force waiting in wait_for_close_async() + this.closed_semaphore.reset(); + + // reset unseen count refresh since it will be updated when + // the remote opens + this.refresh_unseen_timer.reset(); + + this.open_cancellable = new Cancellable(); + + // Notify the email prefetcher + this.email_prefetcher.open(); + + // notify about the local open + int local_count = 0; + get_remote_counts(null, out local_count); + notify_opened(Geary.Folder.OpenState.LOCAL, local_count); + + // Unless NO_DELAY is set, do NOT open the remote side here; wait for the ReplayQueue to + // require a remote connection or wait_for_remote_async() to be called ... this allows for + // fast local-only operations to occur, local-only either because (a) the folder has all + // the information required (for a list or fetch operation), or (b) the operation was de + // facto local-only. In particular, EmailStore will open and close lots of folders, + // causing a lot of connection setup and teardown + // + // However, want to eventually open, otherwise if there's no user interaction (i.e. a + // second account Inbox they don't manipulate), no remote connection will ever be made, + // meaning that folder normalization never happens and unsolicited notifications never + // arrive + this._account.session_pool.ready.connect(on_remote_ready); + if (open_flags.is_all_set(OpenFlags.NO_DELAY)) { + this.open_remote_session.begin(); + } else { + this.remote_open_timer.start(); + } + return true; + } + + /** {@inheritDoc} */ + public override async void wait_for_remote_async(Cancellable? cancellable = null) throws Error { + check_open("wait_for_remote_async"); + + // if remote has not yet been opened, do it now ... + if (this.remote_folder == null) { + this.open_remote_session.begin(); + } + + if (!yield this.remote_wait_semaphore.wait_for_result_async(cancellable)) + throw new EngineError.ALREADY_CLOSED("%s failed to open", to_string()); + } + + /** {@inheritDoc} */ + public override async bool close_async(Cancellable? cancellable = null) throws Error { + // Check open_count but only decrement inside of replay queue + if (open_count <= 0) + return false; + + UserClose user_close = new UserClose(this, cancellable); + this.replay_queue.schedule(user_close); + + yield user_close.wait_for_ready_async(cancellable); + return user_close.closing; + } + + /** {@inheritDoc} */ + public override async void wait_for_close_async(Cancellable? cancellable = null) + throws Error { + yield this.closed_semaphore.wait_async(cancellable); + } + // used by normalize_folders() during the normalization process; should not be used elsewhere private async void detach_all_emails_async(Cancellable? cancellable) throws Error { Gee.List? all = yield local_folder.list_email_by_id_async(null, -1, @@ -214,32 +308,35 @@ private class Geary.ImapEngine.MinimalFolder : Geary.Folder, Geary.FolderSupport notify_email_count_changed(0, Folder.CountChangeReason.REMOVED); } } - - private async bool normalize_folders(Geary.Imap.Folder remote_folder, Cancellable? cancellable) + + private async void normalize_folders(Geary.Imap.FolderSession remote_folder, + Cancellable? cancellable) throws Error { debug("%s: Begin normalizing remote and local folders", to_string()); - - Geary.Imap.FolderProperties local_properties = local_folder.get_properties(); - Geary.Imap.FolderProperties remote_properties = remote_folder.properties; - + + Geary.Imap.FolderProperties local_properties = this.local_folder.get_properties(); + Geary.Imap.FolderProperties remote_properties = remote_folder.folder.properties; + // and both must have their next UID's (it's possible they don't if it's a non-selectable // folder) if (local_properties.uid_next == null || local_properties.uid_validity == null) { - debug("%s: Unable to verify UIDs: missing local UIDNEXT (%s) and/or UIDVALIDITY (%s)", - to_string(), (local_properties.uid_next == null).to_string(), - (local_properties.uid_validity == null).to_string()); - - return false; + throw new ImapError.NOT_SUPPORTED( + "%s: Unable to verify UIDs: missing local UIDNEXT (%s) and/or UIDVALIDITY (%s)", + to_string(), + (local_properties.uid_next == null).to_string(), + (local_properties.uid_validity == null).to_string() + ); } - + if (remote_properties.uid_next == null || remote_properties.uid_validity == null) { - debug("%s: Unable to verify UIDs: missing remote UIDNEXT (%s) and/or UIDVALIDITY (%s)", - to_string(), (remote_properties.uid_next == null).to_string(), - (remote_properties.uid_validity == null).to_string()); - - return false; + throw new ImapError.NOT_SUPPORTED( + "%s: Unable to verify UIDs: missing remote UIDNEXT (%s) and/or UIDVALIDITY (%s)", + to_string(), + (remote_properties.uid_next == null).to_string(), + (remote_properties.uid_validity == null).to_string() + ); } - + // If UIDVALIDITY changes, all email in the folder must be removed as the UIDs are now // invalid ... we merely detach the emails (leaving their contents behind) so duplicate // detection can fix them up. But once all UIDs are removed, it's much like the next @@ -250,12 +347,10 @@ private class Geary.ImapEngine.MinimalFolder : Geary.Folder, Geary.FolderSupport debug("%s: UID validity changed, detaching all email: %s -> %s", to_string(), local_properties.uid_validity.value.to_string(), remote_properties.uid_validity.value.to_string()); - yield detach_all_emails_async(cancellable); - - return true; + return; } - + // fetch email from earliest email to last to (a) remove any deletions and (b) update // any flags that may have changed ImapDB.EmailIdentifier? local_earliest_id = yield local_folder.get_earliest_id_async(cancellable); @@ -264,14 +359,13 @@ private class Geary.ImapEngine.MinimalFolder : Geary.Folder, Geary.FolderSupport // verify still open; this is required throughout after each yield, as a close_async() can // come in ay any time since this does not run in the context of open_async() check_open("normalize_folders (local earliest/latest UID)"); - + // if no earliest UID, that means no messages in local store, so nothing to update if (local_earliest_id == null || local_latest_id == null) { debug("%s: local store empty, nothing to normalize", to_string()); - - return true; + return; } - + assert(local_earliest_id.has_uid()); assert(local_latest_id.has_uid()); @@ -306,12 +400,10 @@ private class Geary.ImapEngine.MinimalFolder : Geary.Folder, Geary.FolderSupport debug("%s: Local UID(s) higher than remote UIDNEXT, detaching all email: %s/%s remote=%s", to_string(), local_earliest_id.uid.to_string(), local_latest_id.uid.to_string(), last_uid.to_string()); - yield detach_all_emails_async(cancellable); - - return true; + return; } - + // if UIDNEXT has changed, that indicates messages have been appended (and possibly removed) int64 uidnext_diff = remote_properties.uid_next.value - local_properties.uid_next.value; @@ -324,10 +416,9 @@ private class Geary.ImapEngine.MinimalFolder : Geary.Folder, Geary.FolderSupport // nothing has been added or removed if (!is_dirty && uidnext_diff == 0 && local_message_count == remote_message_count) { debug("%s: No messages added/removed since last opened, normalization completed", to_string()); - - return true; + return; } - + // if the difference in UIDNEXT values equals the difference in message count, then only // an append could have happened, so only pull in the new messages ... note that this is not foolproof, // as UIDs are not guaranteed to increase by 1; however, this is a standard implementation practice, @@ -544,303 +635,36 @@ private class Geary.ImapEngine.MinimalFolder : Geary.Folder, Geary.FolderSupport count_change_reason, remote_message_count); notify_email_count_changed(remote_message_count, count_change_reason); } - + debug("%s: Completed normalize_folder", to_string()); - - return true; - } - - public override async void wait_for_open_async(Cancellable? cancellable = null) throws Error { - if (open_count == 0) - throw new EngineError.OPEN_REQUIRED("wait_for_open_async() can only be called after open_async()"); - - // if remote has not yet been opened, do it now ... this bool can go true only once after - // an open_async, it's reset at close time - if (!remote_opened) { - // Someone wants this open right now, so cancel the timer and just do it already - this.remote_open_timer.reset(); - start_open_remote(); - } - - if (!yield remote_semaphore.wait_for_result_async(cancellable)) - throw new EngineError.ALREADY_CLOSED("%s failed to open", to_string()); } - public override async bool open_async(Geary.Folder.OpenFlags open_flags, Cancellable? cancellable = null) - throws Error { - if (open_count++ > 0) { - // even if opened or opening, or if forcing a re-open, respect the NO_DELAY flag - if (open_flags.is_all_set(OpenFlags.NO_DELAY)) { - // add NO_DELAY flag if it forces an open - if (!remote_opened) - this.open_flags |= OpenFlags.NO_DELAY; + /** + * Unhooks the IMAP folder session and returns it to the account. + */ + internal async void close_remote_session(Folder.CloseReason remote_reason) { + // Block anyone calling wait_for_remote_async(), as the session + // will no longer available. + this.remote_wait_semaphore.reset(); - start_open_remote(); - } - return false; - } + Imap.FolderSession session = this.remote_folder; + this.remote_folder = null; + this.remote_count = -1; - // first open gets to name the flags, but see note above - this.open_flags = open_flags; + if (session != null) { + session.appended.disconnect(on_remote_appended); + session.updated.disconnect(on_remote_updated); + session.removed.disconnect(on_remote_removed); + session.disconnected.disconnect(on_remote_disconnected); + this._account.release_folder_session(session); - // reset to force waiting in wait_for_open_async() - this.remote_semaphore.reset(); - - // reset to force waiting in wait_for_close_async() - this.closed_semaphore.reset(); - - // reset unseen count refresh since it will be updated when - // the remote opens - this.refresh_unseen_timer.reset(); - - this.open_cancellable = new Cancellable(); - - // Notify the email prefetcher - this.email_prefetcher.open(); - - // Unless NO_DELAY is set, do NOT open the remote side here; wait for the ReplayQueue to - // require a remote connection or wait_for_open_async() to be called ... this allows for - // fast local-only operations to occur, local-only either because (a) the folder has all - // the information required (for a list or fetch operation), or (b) the operation was de - // facto local-only. In particular, EmailStore will open and close lots of folders, - // causing a lot of connection setup and teardown - // - // However, want to eventually open, otherwise if there's no user interaction (i.e. a - // second account Inbox they don't manipulate), no remote connection will ever be made, - // meaning that folder normalization never happens and unsolicited notifications never - // arrive - this._account.remote.ready.connect(on_remote_ready); - if (open_flags.is_all_set(OpenFlags.NO_DELAY)) { - start_open_remote(); - } else { - this.remote_open_timer.start(); - } - return true; - } - - private void start_open_remote() { - if (!this.remote_opened && this._account.remote.is_ready) { - this.remote_opened = true; - this.remote_open_timer.reset(); - this.open_remote_async.begin(null); + notify_closed(remote_reason); } } - // Open the remote connection using a Mutex to prevent concurrency. - // - // start_open_remote() *should* prevent more than one open from occurring at the same time, - // but it's still wise to use a nonblocking primitive to prevent it if that does occur to at - // least keep Folder state cogent. - private async void open_remote_async(Cancellable? cancellable) { - debug("%s: Opening remote folder", to_string()); - int token; - try { - token = yield open_mutex.claim_async(cancellable); - } catch (Error err) { - return; - } - - yield open_remote_locked_async(cancellable); - - try { - open_mutex.release(ref token); - } catch (Error err) { - } - } - - // Should only be called when open_mutex is locked, i.e. use open_remote_async() - private async void open_remote_locked_async(Cancellable? cancellable) { - // watch for folder closing before this call got a chance to execute - if (open_count == 0) - return; - - // to ensure this isn't running when open_remote_async() is called again (due to a connection - // reestablishment), stop this monitoring from running *before* launching close_internal_async - // ... in essence, guard against reentrancy, which is possible - opening_monitor.notify_start(); - - // following blocks of code are fairly tricky because if the remote open fails need to - // carefully back out and possibly retry - Imap.Folder? opening_folder = null; - FolderPath path = local_folder.get_path(); - try { - // Fetch the local status first anyway, since if it - // doesn't exist we haven't seen the folder before anyway - Imap.StatusData? local_status = yield local_folder.fetch_status_data( - ImapDB.Folder.ListFlags.NONE, - cancellable - ); - - debug("Fetching information for remote folder %s", to_string()); - try { - opening_folder = yield this._account.remote.fetch_folder_cached_async( - path, false, cancellable - ); - } catch (EngineError.NOT_FOUND err) { - if (err is EngineError.NOT_FOUND) { - throw err; - } - - // Use local STATUS data cache to be able to present - // something to the user at least. XXX get the attrs - // from somewhere for Bug 714775 - opening_folder = this._account.remote.new_selectable_folder( - path, - local_status, - new Imap.MailboxAttributes(new Gee.ArrayList()) - ); - } - - debug("Opening remote folder %s", opening_folder.to_string()); - yield opening_folder.open_async(cancellable); - - // allow subclasses to examine the opened folder and resolve any vital - // inconsistencies - if (yield normalize_folders(opening_folder, cancellable)) { - // update flags, properties, etc. - yield local_folder.update_folder_select_examine( - opening_folder.properties, cancellable - ); - - // signals - opening_folder.appended.connect(on_remote_appended); - opening_folder.updated.connect(on_remote_updated); - opening_folder.removed.connect(on_remote_removed); - opening_folder.disconnected.connect(on_remote_disconnected); - - // state - remote_count = opening_folder.properties.email_total; - - // all set; bless the remote folder as opened (don't do this until completely - // open, as other functions rely on this to determine folder-open state) - remote_folder = opening_folder; - } else { - debug("Unable to prepare remote folder %s: normalize_folders() failed", to_string()); - notify_open_failed(Geary.Folder.OpenFailed.REMOTE_FAILED, null); - - // be sure to close opening_folder, close_internal_async won't do it - try { - yield opening_folder.close_async(null); - } catch (Error err) { - debug("%s: Error closing remote folder %s: %s", to_string(), opening_folder.to_string(), - err.message); - - // fall through - } - - // stop before starting the close - opening_monitor.notify_finish(); - - // normalize_folders() returning false indicates a soft error, but hard in the sense - // that opening cannot proceed, even with a connection retry - open_count = 0; - - // schedule immediate close - close_internal_async.begin(CloseReason.LOCAL_CLOSE, CloseReason.REMOTE_CLOSE, false, - cancellable); - - return; - } - } catch (Error open_err) { - bool hard_failure; - bool is_cancellation = false; - if (open_err is ImapError || open_err is EngineError) { - // "hard" error in the sense of network conditions make connection impossible - // at the moment, "soft" error in the sense that some logical error prevented - // connect (like bad credentials) - hard_failure = is_hard_failure(open_err); - } else if (open_err is IOError.CANCELLED) { - // user cancelled open, treat like soft error - hard_failure = false; - is_cancellation = true; - } else { - // a different IOError, a hard failure - hard_failure = true; - } - - Folder.CloseReason remote_reason; - if (hard_failure) { - // hard failure, retry - debug("Hard failure opening or preparing remote folder %s, retrying: %s", to_string(), - open_err.message); - - remote_reason = CloseReason.REMOTE_ERROR; - } else { - // soft failure, treat as failure to open - debug("Soft failure opening or preparing remote folder %s, closing: %s", to_string(), - open_err.message); - notify_open_failed( - is_cancellation ? Folder.OpenFailed.CANCELLED : Folder.OpenFailed.REMOTE_FAILED, - open_err); - - remote_reason = CloseReason.REMOTE_CLOSE; - - // clear open_count to ensure that close_internal_async() doesn't attempt to - // reestablish the connection - open_count = 0; - } - - // be sure to close opening_folder if it was fetched or opened - try { - if (opening_folder != null) - yield opening_folder.close_async(null); - } catch (Error err) { - debug("%s: Error closing remote folder %s: %s", to_string(), opening_folder.to_string(), - err.message); - } - - // stop before starting the close - opening_monitor.notify_finish(); - - // schedule immediate close (and possible connection reestablishment) - close_internal_async.begin(CloseReason.LOCAL_CLOSE, remote_reason, false, null); - - return; - } - - opening_monitor.notify_finish(); - - // at this point, remote_folder should be set; there's no notion of a local-only open (yet) - assert(remote_folder != null); - - // notify any threads of execution waiting for the remote folder to open that the result - // of that operation is ready - try { - remote_semaphore.notify_result(true, null); - } catch (Error notify_err) { - // This should only happen if cancelled, which can't happen without a Cancellable - warning("%s: Unable to fire semaphore notifying remote folder ready/not ready: %s", - to_string(), notify_err.message); - } - - _properties.add(remote_folder.properties); - - // notify any subscribers with similar information - notify_opened(Geary.Folder.OpenState.BOTH, remote_count); - - // Update flags once the folder has opened. We will receive - // notifications of changes as long as it remains open, so - // only need to do this once - this.update_flags_timer.start(); - } - - public override async bool close_async(Cancellable? cancellable = null) throws Error { - // Check open_count but only decrement inside of replay queue - if (open_count <= 0) - return false; - - UserClose user_close = new UserClose(this, cancellable); - replay_queue.schedule(user_close); - - yield user_close.wait_for_ready_async(cancellable); - - return user_close.closing; - } - - public override async void wait_for_close_async(Cancellable? cancellable = null) throws Error { - yield closed_semaphore.wait_async(cancellable); - } - + /** + * Starts closing the folder, called from {@link UserClose}. + */ internal async bool user_close_async(Cancellable? cancellable) { // decrement open_count and, if zero, continue closing Folder if (open_count == 0 || --open_count > 0) @@ -849,196 +673,249 @@ private class Geary.ImapEngine.MinimalFolder : Geary.Folder, Geary.FolderSupport // Close the prefetcher early so it stops using the remote ASAP this.email_prefetcher.close(); - if (remote_folder != null) - _properties.remove(remote_folder.properties); + if (this.remote_folder != null) + _properties.remove(this.remote_folder.folder.properties); + + // block anyone from wait_for_remote_async(), as this is no longer open + this.remote_wait_semaphore.reset(); - // block anyone from wait_until_open_async(), as this is no longer open - remote_semaphore.reset(); - // don't yield here, close_internal_async() needs to be called outside of the replay queue // the open_count protects against this path scheduling it more than once - close_internal_async.begin(CloseReason.LOCAL_CLOSE, CloseReason.REMOTE_CLOSE, true, cancellable); - + this.close_internal_async.begin( + CloseReason.LOCAL_CLOSE, + CloseReason.REMOTE_CLOSE, + true, + cancellable + ); + return true; } - - // Close the remote connection and, if open_count is zero, the Folder itself. A Mutex is used - // to prevent concurrency. - // - // This is best called using a ReplayDisconnect operation, which ensures an orderly disconnect - // by going through the ReplayQueue. There are certain situations in open_remote_async() where - // this is not possible (because the queue hasn't been started). - // - // NOTE: This bypasses open_count and forces the Folder closed - internal async void close_internal_async(Folder.CloseReason local_reason, Folder.CloseReason remote_reason, - bool flush_pending, Cancellable? cancellable) { - this.remote_open_timer.reset(); - int token; + /** + * Forces closes the folder. + * + * NOTE: This bypasses open_count and forces the Folder closed. + */ + internal async void close_internal_async(Folder.CloseReason local_reason, + Folder.CloseReason remote_reason, + bool flush_pending, + Cancellable? cancellable) { try { - token = yield close_mutex.claim_async(cancellable); - } catch (Error err) { - return; - } - - yield close_internal_locked_async(local_reason, remote_reason, flush_pending, cancellable); - - try { - close_mutex.release(ref token); + int token = yield this.close_mutex.claim_async(cancellable); + yield close_internal_locked_async( + local_reason, remote_reason, flush_pending, cancellable + ); + this.close_mutex.release(ref token); } catch (Error err) { + // oh well } } - + // Should only be called when close_mutex is locked, i.e. use close_internal_async() private async void close_internal_locked_async(Folder.CloseReason local_reason, - Folder.CloseReason remote_reason, bool flush_pending, Cancellable? cancellable) { + Folder.CloseReason remote_reason, + bool flush_pending, + Cancellable? cancellable) { + // Ensure we don't attempt to start opening a remote while + // closing + this._account.session_pool.ready.disconnect(on_remote_ready); + this.remote_open_timer.reset(); + // only flushing pending ReplayOperations if this is a "clean" close, not forced due to // error and if specified by caller (could be a non-error close on the server, i.e. "BYE", // but the connection is dropping, so don't flush pending) - flush_pending = flush_pending && !remote_reason.is_error(); - - // If closing due to error, notify all operations waiting for the remote that it's not - // coming available ... this wakes up any ReplayOperation blocking on wait_for_open_async(), - // necessary in order to finish ReplayQueue.close_async (i.e. to prevent deadlock); this - // is necessary because it's possible for this method to be called before the remote_folder - // has even had a chance to open. - // - // Note that we don't want to do this for a clean close, because we want to flush out - // pending operations first - Imap.Folder? closing_remote_folder = null; - if (!flush_pending) - closing_remote_folder = clear_remote_folder(); - - // That said, only flush, close, and destroy the ReplayQueue if fully closing and not - // allowing for a connection reestablishment - if (open_count <= 0) { - // if closing and flushing the queue, give Revokables a chance to schedule their + flush_pending = ( + flush_pending && + !local_reason.is_error() && + !remote_reason.is_error() + ); + + if (flush_pending) { + // We are flushing the queue, so gather operations from + // Revokables to give them a chance to schedule their // commit operations before going down - if (flush_pending) { - Gee.List final_ops = new Gee.ArrayList(); - notify_closing(final_ops); - - foreach (ReplayOperation op in final_ops) - replay_queue.schedule(op); - } - - // Close the replay queues; if a "clean" close, flush pending operations so everything - // gets a chance to run; if forced close, drop everything outstanding - try { - // swap out the ReplayQueue while closing so, if re-opened, future commands can - // be queued on the new queue - ReplayQueue closing_replay_queue = replay_queue; - replay_queue = new ReplayQueue(this); - - debug("Closing replay queue for %s (flush_pending=%s): %s", to_string(), - flush_pending.to_string(), closing_replay_queue.to_string()); - yield closing_replay_queue.close_async(flush_pending); - debug("Closed replay queue for %s: %s", to_string(), closing_replay_queue.to_string()); - } catch (Error replay_queue_err) { - debug("Error closing %s replay queue: %s", to_string(), replay_queue_err.message); - } - } - - // if a "clean" close, now go ahead and close the folder - if (flush_pending) - closing_remote_folder = clear_remote_folder(); - - // now treat remote as closed, i.e. a call to open_async() will reinitiate opening and not fall - // through (unless open_count is > 0) ... do this before close_remote_folder_async() since - // it's *possible* for it to loop back to open_async() before returning - remote_opened = false; - - if (closing_remote_folder != null) { - // to avoid keeping the caller waiting while the remote end closes (i.e. drops the - // connection or performs an IMAP CLOSE operation), close it in the background + Gee.List final_ops = new Gee.ArrayList(); + notify_closing(final_ops); + foreach (ReplayOperation op in final_ops) + replay_queue.schedule(op); + } else { + // Not flushing the queue, so notify all operations + // waiting for the remote that it's not coming available + // ... this wakes up any ReplayOperation blocking on + // wait_for_remote_async(), necessary in order to finish + // ReplayQueue.close_async (i.e. to prevent deadlock); + // this is necessary because it's possible for this method + // to be called before a session has even had a chance to + // open. // - // TODO: Problem with this is that we cannot effectively signal or report a close error, - // because by the time this operation completes the folder is considered closed. That - // may not be important to most callers, however. - // - // It also means the reference to the Folder must be maintained until completely - // closed. Also not a problem, as GenericAccount does that internally. However, this - // might be an issue if GenericAccount removes this folder due to a user command or - // detection on the server, so this background op keeps a reference to the Folder - close_remote_folder_async.begin(this, closing_remote_folder); + // We don't want to do this for a clean close yet, because + // some pending operations may still need to use the + // session. + notify_remote_waiters(false); } - // Only mark the folder as closed if there are no more - // users of this instance - if (open_count == 0) { - // forced closed one way or another, so reset state - open_flags = OpenFlags.NONE; + // swap out the ReplayQueue while closing so, if re-opened, + // future commands can be queued on the new queue + ReplayQueue closing_replay_queue = this.replay_queue; + this.replay_queue = new ReplayQueue(this); - // use remote_reason even if remote_folder was null; it - // could be that the error occurred while opening and - // remote_folder was yet unassigned ... also, need to call - // this every time, even if remote was not fully opened, - // as some callers rely on order of signals - notify_closed(remote_reason); - - // see above note for why this must be called every time - notify_closed(local_reason); - - notify_closed(CloseReason.FOLDER_CLOSED); - - // If not closing in the background, notify waiting callers here - if (closing_remote_folder == null) - closed_semaphore.blind_notify(); - - debug("Folder %s closed", to_string()); - } - } - - // Returns the remote_folder, if it was set - private Imap.Folder? clear_remote_folder() { - // Cancel any internal pending operations before unhooking - this.open_cancellable.cancel(); - this.open_cancellable = null; - - if (remote_folder != null) { - // disconnect signals before ripping out reference - remote_folder.appended.disconnect(on_remote_appended); - remote_folder.updated.disconnect(on_remote_updated); - remote_folder.removed.disconnect(on_remote_removed); - remote_folder.disconnected.disconnect(on_remote_disconnected); - } - - Imap.Folder? old_remote_folder = remote_folder; - remote_folder = null; - remote_count = -1; - - // only signal waiters in wait_for_open_async() that the open failed if there is no cx - // reestablishment to occur - if (open_count <= 0) { - try { - remote_semaphore.notify_result(false, null); - } catch (Error err) { - debug("Error attempting to notify that remote folder %s is now closed: %s", to_string(), - err.message); - } - } - - return old_remote_folder; - } - - // See note in close_async() for why this method is static and uses an owned ref - private static async void close_remote_folder_async(owned MinimalFolder folder, - owned Imap.Folder? remote_folder) { - // force the remote closed; if due to a remote disconnect and plan on reopening, *still* - // need to do this ... don't set remote_folder to null, as that will make some code paths - // think the folder is closing or closed when in fact it will be re-opening in a moment + // Close the replay queues; if a "clean" close, flush pending operations so everything + // gets a chance to run; if forced close, drop everything outstanding try { - if (remote_folder != null) - yield remote_folder.close_async(null); - } catch (Error err) { - debug("Unable to close remote %s: %s", remote_folder.to_string(), err.message); - // fallthrough + debug("Closing replay queue for %s (flush_pending=%s): %s", to_string(), + flush_pending.to_string(), closing_replay_queue.to_string()); + yield closing_replay_queue.close_async(flush_pending); + debug("Closed replay queue for %s: %s", to_string(), closing_replay_queue.to_string()); + } catch (Error replay_queue_err) { + debug("Error closing %s replay queue: %s", to_string(), replay_queue_err.message); } - // need to do it here if not done in close_internal_locked_async() - if (folder.open_count <= 0 && remote_folder != null) { - folder.closed_semaphore.blind_notify(); + // If flushing, now notify waiters that the queue has bee flushed + if (flush_pending) { + notify_remote_waiters(false); } + + // forced closed one way or another, so reset state + this.open_count = 0; + this.open_flags = OpenFlags.NONE; + + // Actually close the remote folder + yield close_remote_session(remote_reason); + + // need to call these every time, even if remote was not fully + // opened, as some callers rely on order of signals + notify_closed(local_reason); + notify_closed(CloseReason.FOLDER_CLOSED); + + // Notify waiting tasks + this.closed_semaphore.blind_notify(); + + debug("Folder %s closed", to_string()); + } + + /** + * Establishes a new IMAP session, normalising local and remote folders. + */ + private async void open_remote_session() { + try { + int token = yield this.open_mutex.claim_async(this.open_cancellable); + + // Ensure we are open already and guard against someone + // else having called this just before we did. + if (this.open_count > 0 && + this._account.session_pool.is_ready && + this.remote_folder == null) { + + this.opening_monitor.notify_start(); + yield open_remote_session_locked(this.open_cancellable); + this.opening_monitor.notify_finish(); + } + + this.open_mutex.release(ref token); + } catch (Error err) { + // Lock error + } + } + + // Should only be called when open_mutex is locked, i.e. use open_remote_session() + private async void open_remote_session_locked(Cancellable? cancellable) { + debug("%s: Opening remote session", to_string()); + + // Don't try to re-open again + this.remote_open_timer.reset(); + + // Phase 1: Acquire a new session + + Imap.FolderSession? session = null; + try { + session = yield this._account.open_folder_session(this.path, cancellable); + } catch (Error err) { + // Notify that there was a connection error, but don't + // force the folder closed, since it might come good again + // if the user fixes an auth problem or the network comes + // back or whatever. + notify_open_failed(Folder.OpenFailed.REMOTE_ERROR, err); + return; + } + + // Phase 2: Update local state based on the remote session + + // Signals need to be hooked up before normalisation so that + // notifications of state changes are not lost when that is + // running. + session.appended.connect(on_remote_appended); + session.updated.connect(on_remote_updated); + session.removed.connect(on_remote_removed); + session.disconnected.connect(on_remote_disconnected); + + try { + yield normalize_folders(session, cancellable); + } catch (Error err) { + // Normalisation failed, which is also a serious problem + // so treat as in the error case above, after resolving if + // the issue was local or remote. + this._account.release_folder_session(session); + if (err is IOError.CANCELLED) { + notify_open_failed(OpenFailed.LOCAL_ERROR, err); + } else { + Folder.CloseReason local_reason = CloseReason.LOCAL_ERROR; + Folder.CloseReason remote_reason = CloseReason.REMOTE_CLOSE; + if (!is_remote_error(err)) { + notify_open_failed(OpenFailed.LOCAL_ERROR, err); + } else { + notify_open_failed(OpenFailed.REMOTE_ERROR, err); + local_reason = CloseReason.LOCAL_CLOSE; + remote_reason = CloseReason.REMOTE_ERROR; + } + + this.close_internal_async.begin( + local_reason, + remote_reason, + false, + null // Don't pass cancellable, close must complete + ); + } + return; + } + + try { + yield local_folder.update_folder_select_examine( + session.folder.properties, cancellable + ); + this.remote_count = session.folder.properties.email_total; + } catch (Error err) { + // Database failed, so we have a pretty serious problem + // and should not try to use the folder further, unless + // the open was simply cancelled. So clean up, and force + // the folder closed if needed. + this._account.release_folder_session(session); + notify_open_failed(Folder.OpenFailed.LOCAL_ERROR, err); + if (!(err is IOError.CANCELLED)) { + this.close_internal_async.begin( + CloseReason.LOCAL_ERROR, + CloseReason.REMOTE_CLOSE, + false, + null // Don't pass cancellable, close must complete + ); + } + return; + } + + // Phase 3: Move in place and notify waiters + + this.remote_folder = session; + + // notify any subscribers with similar information + notify_opened(Geary.Folder.OpenState.BOTH, this.remote_count); + + // notify any threads of execution waiting for the remote + // folder to open that the result of that operation is ready + notify_remote_waiters(true); + + // Update flags once the folder has opened. We will receive + // notifications of changes as long as the session remains + // open, so only need to do this once + this.update_flags_timer.start(); } public override async void find_boundaries_async(Gee.Collection ids, @@ -1134,14 +1011,9 @@ private class Geary.ImapEngine.MinimalFolder : Geary.Folder, Geary.FolderSupport private void on_remote_disconnected(Imap.ClientSession.DisconnectReason reason) { debug("on_remote_disconnected: reason=%s", reason.to_string()); - - // reset remote_semaphore to indicate that callers must again wait for the remote to open... - // do this now to avoid race conditions w/ wait_for_open_async() - remote_semaphore.reset(); - replay_queue.schedule(new ReplayDisconnect(this, reason, false, null)); } - + // // list email variants // @@ -1342,8 +1214,9 @@ private class Geary.ImapEngine.MinimalFolder : Geary.Folder, Geary.FolderSupport } public override string to_string() { - return "%s (open_count=%d remote_opened=%s)".printf(base.to_string(), open_count, - remote_opened.to_string()); + return "%s (open_count=%d remote_opened=%s)".printf( + base.to_string(), open_count, (remote_folder != null).to_string() + ); } /** @@ -1459,6 +1332,14 @@ private class Geary.ImapEngine.MinimalFolder : Geary.Folder, Geary.FolderSupport marked_email_removed(removed); } + private inline void notify_remote_waiters(bool successful) { + try { + this.remote_wait_semaphore.notify_result(successful, null); + } catch (Error err) { + // Can't happen because semaphore has no cancellable + } + } + /** * Checks for changes to {@link EmailFlags} after a folder opens. */ @@ -1467,7 +1348,8 @@ private class Geary.ImapEngine.MinimalFolder : Geary.Folder, Geary.FolderSupport // we support IMAP CONDSTORE (Bug 713117). int chunk_size = FLAG_UPDATE_START_CHUNK; Geary.EmailIdentifier? lowest = null; - while (!this.open_cancellable.is_cancelled() && this._account.remote.is_ready) { + for (;;) { + yield wait_for_remote_async(this.open_cancellable); Gee.List? list_local = yield list_email_by_id_async( lowest, chunk_size, Geary.Email.Field.FLAGS, @@ -1531,9 +1413,7 @@ private class Geary.ImapEngine.MinimalFolder : Geary.Folder, Geary.FolderSupport } private void on_remote_ready() { - if (this.open_count > 0) { - start_open_remote(); - } + this.open_remote_session.begin(); } } diff --git a/src/engine/imap-engine/imap-engine-replay-queue.vala b/src/engine/imap-engine/imap-engine-replay-queue.vala index d092f75a..eb888518 100644 --- a/src/engine/imap-engine/imap-engine-replay-queue.vala +++ b/src/engine/imap-engine/imap-engine-replay-queue.vala @@ -491,7 +491,7 @@ private class Geary.ImapEngine.ReplayQueue : Geary.BaseObject { // wait until the remote folder is opened (or throws an exception, in which case closed) try { if (!is_close_op && folder_opened && state == State.OPEN) - yield owner.wait_for_open_async(); + yield owner.wait_for_remote_async(); } catch (Error remote_err) { debug("Folder %s closed or failed to open, remote replay queue closing: %s", to_string(), remote_err.message); diff --git a/src/engine/imap-engine/imap-engine-revokable-committed-move.vala b/src/engine/imap-engine/imap-engine-revokable-committed-move.vala index 0f3f24aa..daed4417 100644 --- a/src/engine/imap-engine/imap-engine-revokable-committed-move.vala +++ b/src/engine/imap-engine/imap-engine-revokable-committed-move.vala @@ -22,42 +22,34 @@ private class Geary.ImapEngine.RevokableCommittedMove : Revokable { this.destination = destination; this.destination_uids = destination_uids; } - + protected override async void internal_revoke_async(Cancellable? cancellable) throws Error { - Imap.Folder? detached_destination = null; + Imap.FolderSession? session = null; try { // use a detached folder to quickly open, issue command, and leave, without full // normalization that MinimalFolder requires - detached_destination = yield account.fetch_detached_folder_async(destination, cancellable); - - yield detached_destination.open_async(cancellable); - + session = yield this.account.open_folder_session(destination, cancellable); foreach (Imap.MessageSet msg_set in Imap.MessageSet.uid_sparse(destination_uids)) { // don't use Cancellable to try to make operations atomic - yield detached_destination.copy_email_async(msg_set, source, null); - yield detached_destination.remove_email_async(msg_set.to_list(), null); - + yield session.copy_email_async(msg_set, source, null); + yield session.remove_email_async(msg_set.to_list(), null); + if (cancellable != null && cancellable.is_cancelled()) throw new IOError.CANCELLED("Revoke cancelled"); } - + notify_revoked(); Geary.Folder target = yield this.account.fetch_folder_async(this.destination); this.account.update_folder(target); } finally { - if (detached_destination != null) { - try { - yield detached_destination.close_async(cancellable); - } catch (Error err) { - // ignored - } + if (session != null) { + this.account.release_folder_session(session); } - set_invalid(); } } - + protected override async void internal_commit_async(Cancellable? cancellable) throws Error { // pretty simple: already committed, so done notify_committed(null); diff --git a/src/engine/imap-engine/imap-engine.vala b/src/engine/imap-engine/imap-engine.vala index 967c9994..5bad3088 100644 --- a/src/engine/imap-engine/imap-engine.vala +++ b/src/engine/imap-engine/imap-engine.vala @@ -25,4 +25,21 @@ private static bool is_hard_failure(Error err) { || err is EngineError.SERVER_UNAVAILABLE; } +/** + * Determines if this IOError related to a remote host or not. + */ +private static bool is_remote_error(GLib.Error err) { + return err is ImapError + || err is IOError.CONNECTION_CLOSED + || err is IOError.CONNECTION_REFUSED + || err is IOError.HOST_UNREACHABLE + || err is IOError.MESSAGE_TOO_LARGE + || err is IOError.NETWORK_UNREACHABLE + || err is IOError.NOT_CONNECTED + || err is IOError.PROXY_AUTH_FAILED + || err is IOError.PROXY_FAILED + || err is IOError.PROXY_NEED_AUTH + || err is IOError.PROXY_NOT_ALLOWED; +} + } diff --git a/src/engine/imap-engine/other/imap-engine-other-account.vala b/src/engine/imap-engine/other/imap-engine-other-account.vala index 4ee8097b..ba0f98ea 100644 --- a/src/engine/imap-engine/other/imap-engine-other-account.vala +++ b/src/engine/imap-engine/other/imap-engine-other-account.vala @@ -5,9 +5,11 @@ */ private class Geary.ImapEngine.OtherAccount : Geary.ImapEngine.GenericAccount { - public OtherAccount(string name, AccountInformation account_information, - Imap.Account remote, ImapDB.Account local) { - base (name, account_information, remote, local); + + public OtherAccount(string name, + AccountInformation account_information, + ImapDB.Account local) { + base (name, account_information, local); } protected override MinimalFolder new_folder(ImapDB.Folder local_folder) { @@ -20,4 +22,5 @@ private class Geary.ImapEngine.OtherAccount : Geary.ImapEngine.GenericAccount { return new OtherFolder(this, local_folder, type); } + } diff --git a/src/engine/imap-engine/outlook/imap-engine-outlook-account.vala b/src/engine/imap-engine/outlook/imap-engine-outlook-account.vala index d9a7f979..f8e6f9f4 100644 --- a/src/engine/imap-engine/outlook/imap-engine-outlook-account.vala +++ b/src/engine/imap-engine/outlook/imap-engine-outlook-account.vala @@ -5,6 +5,7 @@ */ private class Geary.ImapEngine.OutlookAccount : Geary.ImapEngine.GenericAccount { + public static Geary.Endpoint generate_imap_endpoint() { Geary.Endpoint endpoint = new Geary.Endpoint( "imap-mail.outlook.com", @@ -30,9 +31,10 @@ private class Geary.ImapEngine.OutlookAccount : Geary.ImapEngine.GenericAccount Smtp.ClientConnection.DEFAULT_TIMEOUT_SEC); } - public OutlookAccount(string name, AccountInformation account_information, Imap.Account remote, - ImapDB.Account local) { - base (name, account_information, remote, local); + public OutlookAccount(string name, + AccountInformation account_information, + ImapDB.Account local) { + base(name, account_information, local); } protected override MinimalFolder new_folder(ImapDB.Folder local_folder) { diff --git a/src/engine/imap-engine/replay-ops/imap-engine-replay-disconnect.vala b/src/engine/imap-engine/replay-ops/imap-engine-replay-disconnect.vala index 238759ea..7aee3c08 100644 --- a/src/engine/imap-engine/replay-ops/imap-engine-replay-disconnect.vala +++ b/src/engine/imap-engine/replay-ops/imap-engine-replay-disconnect.vala @@ -31,25 +31,15 @@ private class Geary.ImapEngine.ReplayDisconnect : Geary.ImapEngine.ReplayOperati public override async ReplayOperation.Status replay_local_async() throws Error { debug("%s ReplayDisconnect reason=%s", owner.to_string(), reason.to_string()); - + Geary.Folder.CloseReason remote_reason = reason.is_error() - ? Geary.Folder.CloseReason.REMOTE_ERROR : Geary.Folder.CloseReason.REMOTE_CLOSE; - - // because close_internal_async() may schedule a ReplayOperation before its first yield, - // that means a ReplayOperation is scheduling a ReplayOperation, which isn't something - // we want to encourage, so use the Idle queue to schedule close_internal_async - Idle.add(() => { - // ReplayDisconnect is only used when remote disconnects, so never flush pending, the - // connection is down or going down - owner.close_internal_async.begin(Geary.Folder.CloseReason.LOCAL_CLOSE, remote_reason, - flush_pending, cancellable); - - return false; - }); - + ? Geary.Folder.CloseReason.REMOTE_ERROR + : Geary.Folder.CloseReason.REMOTE_CLOSE; + + this.owner.close_remote_session.begin(remote_reason); return ReplayOperation.Status.COMPLETED; } - + public override async void backout_local_async() throws Error { } @@ -62,4 +52,3 @@ private class Geary.ImapEngine.ReplayDisconnect : Geary.ImapEngine.ReplayOperati return "reason=%s".printf(reason.to_string()); } } - diff --git a/src/engine/imap-engine/yahoo/imap-engine-yahoo-account.vala b/src/engine/imap-engine/yahoo/imap-engine-yahoo-account.vala index 49333a92..e2524e2f 100644 --- a/src/engine/imap-engine/yahoo/imap-engine-yahoo-account.vala +++ b/src/engine/imap-engine/yahoo/imap-engine-yahoo-account.vala @@ -5,6 +5,7 @@ */ private class Geary.ImapEngine.YahooAccount : Geary.ImapEngine.GenericAccount { + public static Geary.Endpoint generate_imap_endpoint() { return new Geary.Endpoint( "imap.mail.yahoo.com", @@ -12,7 +13,7 @@ private class Geary.ImapEngine.YahooAccount : Geary.ImapEngine.GenericAccount { Geary.Endpoint.Flags.SSL, Imap.ClientConnection.RECOMMENDED_TIMEOUT_SEC); } - + public static Geary.Endpoint generate_smtp_endpoint() { return new Geary.Endpoint( "smtp.mail.yahoo.com", @@ -20,16 +21,17 @@ private class Geary.ImapEngine.YahooAccount : Geary.ImapEngine.GenericAccount { Geary.Endpoint.Flags.SSL, Smtp.ClientConnection.DEFAULT_TIMEOUT_SEC); } - + private static Gee.HashMap? special_map = null; - public YahooAccount(string name, AccountInformation account_information, - Imap.Account remote, ImapDB.Account local) { - base (name, account_information, remote, local); + public YahooAccount(string name, + AccountInformation account_information, + ImapDB.Account local) { + base(name, account_information, local); if (special_map == null) { special_map = new Gee.HashMap(); - + special_map.set(Imap.MailboxSpecifier.inbox.to_folder_path(null, null), Geary.SpecialFolderType.INBOX); special_map.set(new Imap.FolderRoot("Sent"), Geary.SpecialFolderType.SENT); special_map.set(new Imap.FolderRoot("Draft"), Geary.SpecialFolderType.DRAFTS); diff --git a/src/engine/imap/api/imap-account.vala b/src/engine/imap/api/imap-account-session.vala similarity index 53% rename from src/engine/imap/api/imap-account.vala rename to src/engine/imap/api/imap-account-session.vala index 0710064e..1601c783 100644 --- a/src/engine/imap/api/imap-account.vala +++ b/src/engine/imap/api/imap-account-session.vala @@ -14,12 +14,6 @@ * (in particular, {@link Geary.ImapEngine.GenericAccount}) and makes * them into simple async calls. * - * This class maintains an {@link ClientSessionManager} instance to - * maintain a pool of connections to an account's IMAP endpoint. On - * opening, it will open the pool, then claim an IMAP session and - * maintain it in a non-selected state for executing - * non-mailbox-specific operations. - * * Geary.Imap.Account manages the {@link Imap.Folder} objects it * returns, but only in the sense that it will not create new * instances repeatedly. Otherwise, it does not refresh or update the @@ -27,100 +21,24 @@ * Imap.StatusData} periodically). That's the responsibility of the * higher layers of the stack. */ -private class Geary.Imap.Account : BaseObject { +internal class Geary.Imap.AccountSession : Geary.Imap.SessionObject { - /** Default IMAP session pool size. */ - private const int IMAP_MIN_POOL_SIZE = 2; + private Gee.HashMap folders = + new Gee.HashMap(); - /** Determines if the IMAP account has been opened. */ - public bool is_open { get; private set; default = false; } - - /** - * Determines if the IMAP account has a working connection. - * - * See {@link ClientSessionManager.is_open} for more details. - */ - public bool is_ready { get { return this.session_mgr.is_ready; } } - - private string name; - private AccountInformation account; - private ClientSessionManager session_mgr; - private uint authentication_failures = 0; - private ClientSession? account_session = null; - private Nonblocking.Mutex account_session_mutex = new Nonblocking.Mutex(); private Nonblocking.Mutex cmd_mutex = new Nonblocking.Mutex(); - private Gee.HashMap folders = new Gee.HashMap(); private Gee.List? list_collector = null; private Gee.List? status_collector = null; private Gee.List? server_data_collector = null; - /** - * Fired after opening when the account has a working connection. - * - * This may be fired multiple times, see @{link - * ClientSessionManager.ready} for details. - */ - public signal void ready(); + internal AccountSession(string account_id, + ClientSession session) { + base("%s:account".printf(account_id), session); - /** Fired if a user-notifiable problem occurs. */ - public signal void report_problem(ProblemReport report); - - - public Account(Geary.AccountInformation account) { - this.name = account.id + ":imap"; - this.account = account; - this.session_mgr = new ClientSessionManager(account); - this.session_mgr.min_pool_size = IMAP_MIN_POOL_SIZE; - this.session_mgr.ready.connect(on_session_ready); - this.session_mgr.connection_failed.connect(on_connection_failed); - this.session_mgr.login_failed.connect(on_login_failed); - } - - /** - * Prepares the account for use. - * - * Opening the account will kick off at establishing least one - * connection to the IMAP server, if accessible. - */ - public async void open_async(Cancellable? cancellable = null) throws Error { - if (is_open) - throw new EngineError.ALREADY_OPEN("Imap.Account already open"); - - // Reset this so we start trying to authenticate again - this.authentication_failures = 0; - - // This will cause the session manager to open at least one - // connection. We can't attempt to claim one straight away - // since we might not be online. - yield session_mgr.open_async(cancellable); - - is_open = true; - } - - /** - * Notifies the account that the engine is preparing to exit. - */ - public void prepare_to_close() { - this.session_mgr.discard_returned_sessions = true; - } - - /** - * Closes the account, releasing its IMAP session and session pool. - */ - public async void close_async(Cancellable? cancellable = null) throws Error { - if (!is_open) - return; - - yield drop_session_async(cancellable); - - try { - yield session_mgr.close_async(cancellable); - } catch (Error err) { - // ignored - } - - is_open = false; + session.list.connect(on_list_data); + session.status.connect(on_status_data); + session.server_data_received.connect(on_server_data_received); } /** @@ -128,7 +46,7 @@ private class Geary.Imap.Account : BaseObject { */ public async FolderPath get_default_personal_namespace(Cancellable? cancellable) throws Error { - ClientSession session = yield claim_session_async(cancellable); + ClientSession session = claim_session(); if (session.personal_namespaces.is_empty) { throw new ImapError.INVALID("No personal namespace found"); } @@ -145,7 +63,7 @@ private class Geary.Imap.Account : BaseObject { public async bool folder_exists_async(FolderPath path, Cancellable? cancellable) throws Error { - ClientSession session = yield claim_session_async(cancellable); + ClientSession session = claim_session(); Gee.List mailboxes = yield send_list_async(session, path, false, cancellable); bool exists = mailboxes.is_empty; if (!exists) { @@ -171,7 +89,7 @@ private class Geary.Imap.Account : BaseObject { Geary.SpecialFolderType? type, Cancellable? cancellable) throws Error { - ClientSession session = yield claim_session_async(cancellable); + ClientSession session = claim_session(); MailboxSpecifier mailbox = session.get_mailbox_for_path(path); bool can_create_special = session.capabilities.has_capability(Capabilities.CREATE_SPECIAL_USE); CreateCommand cmd = (type != null && can_create_special) @@ -199,14 +117,16 @@ private class Geary.Imap.Account : BaseObject { */ public async Imap.Folder fetch_folder_async(FolderPath path, Cancellable? cancellable) throws Error { - ClientSession session = yield claim_session_async(cancellable); + ClientSession session = claim_session(); - Gee.List? mailboxes = yield send_list_async(session, path, false, cancellable); + Gee.List? mailboxes = yield send_list_async( + session, path, false, cancellable + ); if (mailboxes.is_empty) throw_not_found(path); - Imap.Folder? folder = null; MailboxInformation mailbox_info = mailboxes.get(0); + Imap.FolderProperties? props = null; if (!mailbox_info.attrs.is_no_select) { StatusData status = yield send_status_async( session, @@ -214,12 +134,16 @@ private class Geary.Imap.Account : BaseObject { StatusDataType.all(), cancellable ); - folder = new_selectable_folder(path, status, mailbox_info.attrs); + props = new Imap.FolderProperties.selectable( + mailbox_info.attrs, + status, + session.capabilities + ); } else { - folder = new_unselectable_folder(path, mailbox_info.attrs); + props = new Imap.FolderProperties.not_selectable(mailbox_info.attrs); } - return folder; + return new Imap.Folder(path, props); } /** @@ -230,33 +154,22 @@ private class Geary.Imap.Account : BaseObject { * server and cached for future use. */ public async Imap.Folder fetch_folder_cached_async(FolderPath path, - bool refresh_counts, + bool refresh_status, Cancellable? cancellable) - throws Error { - check_open(); - + throws Error { + ClientSession session = claim_session(); Imap.Folder? folder = this.folders.get(path); if (folder == null) { folder = yield fetch_folder_async(path, cancellable); this.folders.set(path, folder); - } else { - if (refresh_counts && !folder.properties.attrs.is_no_select) { - try { - ClientSession session = yield claim_session_async(cancellable); - StatusData data = yield send_status_async( - session, - session.get_mailbox_for_path(path), - { StatusDataType.UNSEEN, StatusDataType.MESSAGES }, - cancellable - ); - folder.properties.set_status_unseen(data.unseen); - folder.properties.set_status_message_count(data.messages, false); - } catch (ImapError e) { - this.folders.unset(path); - // XXX notify someone - throw_not_found(path); - } - } + } else if (refresh_status && !folder.properties.attrs.is_no_select) { + StatusData status = yield send_status_async( + session, + session.get_mailbox_for_path(path), + { StatusDataType.UNSEEN, StatusDataType.MESSAGES }, + cancellable + ); + folder.properties.update_status(status); } return folder; } @@ -272,7 +185,7 @@ private class Geary.Imap.Account : BaseObject { */ public async Gee.List fetch_child_folders_async(FolderPath? parent, Cancellable? cancellable) throws Error { - ClientSession session = yield claim_session_async(cancellable); + ClientSession session = claim_session(); Gee.List children = new Gee.ArrayList(); Gee.List mailboxes = yield send_list_async(session, parent, true, cancellable); if (mailboxes.size == 0) { @@ -300,7 +213,10 @@ private class Geary.Imap.Account : BaseObject { FolderPath path = session.get_path_for_mailbox(mailbox_info.mailbox); Folder? child = this.folders.get(path); if (child == null) { - child = new_unselectable_folder(path, mailbox_info.attrs); + child = new Imap.Folder( + path, + new Imap.FolderProperties.not_selectable(mailbox_info.attrs) + ); this.folders.set(path, child); } children.add(child); @@ -347,10 +263,18 @@ private class Geary.Imap.Account : BaseObject { FolderPath child_path = session.get_path_for_mailbox(mailbox_info.mailbox); Imap.Folder? child = this.folders.get(child_path); + if (child != null) { child.properties.update_status(status); } else { - child = new_selectable_folder(child_path, status, mailbox_info.attrs); + child = new Imap.Folder( + child_path, + new Imap.FolderProperties.selectable( + mailbox_info.attrs, + status, + session.capabilities + ) + ); this.folders.set(child_path, child); } @@ -364,12 +288,6 @@ private class Geary.Imap.Account : BaseObject { return children; } - internal Imap.Folder new_selectable_folder(FolderPath path, StatusData status, MailboxAttributes attrs) { - return new Imap.Folder( - path, new Imap.FolderProperties.status(status, attrs), this.session_mgr - ); - } - internal void folders_removed(Gee.Collection paths) { foreach (FolderPath path in paths) { if (folders.has_key(path)) @@ -377,88 +295,15 @@ private class Geary.Imap.Account : BaseObject { } } - // Claiming session in open_async() would delay opening, which make take too long ... rather, - // this is used by the various calls to put off claiming a session until needed (which - // possibly is long enough for ClientSessionManager to get a few ready). - private async ClientSession claim_session_async(Cancellable? cancellable) - throws Error { - check_open(); - // check if available session is in good state - if (account_session != null - && account_session.get_protocol_state(null) != ClientSession.ProtocolState.AUTHORIZED) { - yield drop_session_async(cancellable); + /** {@inheritDoc} */ + public override ClientSession? close() { + ClientSession old_session = base.close(); + if (old_session != null) { + old_session.list.disconnect(on_list_data); + old_session.status.disconnect(on_status_data); + old_session.server_data_received.disconnect(on_server_data_received); } - - int token = yield account_session_mutex.claim_async(cancellable); - - Error? err = null; - if (account_session == null) { - try { - account_session = yield session_mgr.claim_authorized_session_async(cancellable); - - account_session.list.connect(on_list_data); - account_session.status.connect(on_status_data); - account_session.server_data_received.connect(on_server_data_received); - account_session.disconnected.connect(on_disconnected); - } catch (Error claim_err) { - err = claim_err; - } - } - - account_session_mutex.release(ref token); - - if (err != null) { - if (account_session != null) - yield drop_session_async(null); - - throw err; - } - - return account_session; - } - - private async void drop_session_async(Cancellable? cancellable) { - debug("[%s] Dropping account session...", to_string()); - - int token; - try { - token = yield account_session_mutex.claim_async(cancellable); - } catch (Error err) { - debug("Unable to claim Imap.Account session mutex: %s", err.message); - - return; - } - - string desc = account_session != null ? account_session.to_string() : "(none)"; - - if (account_session != null) { - // disconnect signals before releasing (in particular, "disconnected" will in turn - // reenter this method, so avoid that) - account_session.list.disconnect(on_list_data); - account_session.status.disconnect(on_status_data); - account_session.server_data_received.disconnect(on_server_data_received); - account_session.disconnected.disconnect(on_disconnected); - - debug("[%s] Releasing account session %s", to_string(), desc); - - try { - yield session_mgr.release_session_async(account_session, cancellable); - } catch (Error err) { - // ignored - } - - debug("[%s] Released account session %s", to_string(), desc); - - account_session = null; - } - - try { - account_session_mutex.release(ref token); - } catch (Error err) { - // ignored - } - - debug("[%s] Dropped account session (%s)", to_string(), desc); + return old_session; } // Performs a LIST against the server, returning the results @@ -566,61 +411,49 @@ private class Geary.Imap.Account : BaseObject { return Geary.Collection.get_first(responses.values); } - - private async Gee.Map send_multiple_async( - ClientSession session, - Gee.Collection cmds, - Gee.List? list_results, - Gee.List? status_results, - Cancellable? cancellable) + + private async Gee.Map + send_multiple_async(ClientSession session, + Gee.Collection cmds, + Gee.List? list_results, + Gee.List? status_results, + Cancellable? cancellable) throws Error { - int token = yield cmd_mutex.claim_async(cancellable); - - // set up collectors - list_collector = list_results; - status_collector = status_results; - Gee.Map? responses = null; - Error? err = null; + int token = yield this.cmd_mutex.claim_async(cancellable); + + // set up collectors + this.list_collector = list_results; + this.status_collector = status_results; + + Error? cmd_err = null; try { - responses = yield session.send_multiple_commands_async(cmds, cancellable); - } catch (Error send_err) { - err = send_err; + responses = yield session.send_multiple_commands_async( + cmds, cancellable + ); + } catch (Error err) { + cmd_err = err; } - - // disconnect collectors - list_collector = null; - status_collector = null; - - cmd_mutex.release(ref token); - - if (err != null) - throw err; - - assert(responses != null); - + + // tear down collectors + this.list_collector = null; + this.status_collector = null; + + this.cmd_mutex.release(ref token); + + if (cmd_err != null) { + throw cmd_err; + } + return responses; } - private void check_open() throws Error { - if (!is_open) - throw new EngineError.OPEN_REQUIRED("Imap.Account not open"); - } - - private inline Imap.Folder new_unselectable_folder(FolderPath path, MailboxAttributes attrs) { - return new Imap.Folder( - path, new Imap.FolderProperties(0, 0, 0, null, null, attrs), this.session_mgr - ); - } - - private void notify_report_problem(ProblemType problem, Error? err) { - report_problem(new ServiceProblemReport(problem, this.account, Service.IMAP, err)); - } - [NoReturn] private void throw_not_found(Geary.FolderPath? path) throws EngineError { - throw new EngineError.NOT_FOUND("Folder %s not found on %s", - (path != null) ? path.to_string() : "root", session_mgr.to_string()); + throw new EngineError.NOT_FOUND( + "Folder not found: %s", + (path != null) ? path.to_string() : "[root]" + ); } private void on_list_data(MailboxInformation mailbox_info) { @@ -638,80 +471,4 @@ private class Geary.Imap.Account : BaseObject { server_data_collector.add(server_data); } - private void on_disconnected() { - drop_session_async.begin(null); - } - - private void on_session_ready() { - // Now have a valid session, so credentials must be good - this.authentication_failures = 0; - ready(); - } - - private void on_connection_failed(Error error) { - // There was an error connecting to the IMAP host - this.authentication_failures = 0; - if (error is ImapError.UNAUTHENTICATED) { - // This is effectively a login failure - on_login_failed(null); - } else { - notify_report_problem(ProblemType.CONNECTION_ERROR, error); - } - } - - private void on_login_failed(Geary.Imap.StatusResponse? response) { - this.authentication_failures++; - if (this.authentication_failures >= Geary.Account.AUTH_ATTEMPTS_MAX) { - // We have tried auth too many times, so bail out - notify_report_problem(ProblemType.LOGIN_FAILED, null); - } else { - // login can fail due to an invalid password hence we - // should re-ask it but it can also fail due to server - // inaccessibility, for instance "[UNAVAILABLE] / Maximum - // number of connections from user+IP exceeded". In that - // case, resetting password seems unneeded. - bool reask_password = false; - Error? login_error = null; - try { - reask_password = ( - response == null || - response.response_code == null || - response.response_code.get_response_code_type().value != Geary.Imap.ResponseCodeType.UNAVAILABLE - ); - } catch (ImapError err) { - login_error = err; - debug("Unable to parse ResponseCode %s: %s", response.response_code.to_string(), - err.message); - } - - if (!reask_password) { - // Either the server was unavailable, or we were unable to - // parse the login response. Either way, indicate a - // non-login error. - notify_report_problem(ProblemType.SERVER_ERROR, login_error); - } else { - // Now, we should ask the user for their password - this.account.fetch_passwords_async.begin( - ServiceFlag.IMAP, true, - (obj, ret) => { - try { - if (this.account.fetch_passwords_async.end(ret)) { - // Have a new password, so try that - this.session_mgr.credentials_updated(); - } else { - // User cancelled, so indicate a login problem - notify_report_problem(ProblemType.LOGIN_FAILED, null); - } - } catch (Error err) { - notify_report_problem(ProblemType.GENERIC_ERROR, err); - } - }); - } - } - } - - public string to_string() { - return name; - } - } diff --git a/src/engine/imap/api/imap-folder-properties.vala b/src/engine/imap/api/imap-folder-properties.vala index d8c07cf6..4dc4b275 100644 --- a/src/engine/imap/api/imap-folder-properties.vala +++ b/src/engine/imap/api/imap-folder-properties.vala @@ -56,45 +56,93 @@ public class Geary.Imap.FolderProperties : Geary.FolderProperties { public UIDValidity? uid_validity { get; internal set; } public UID? uid_next { get; internal set; } public MailboxAttributes attrs { get; internal set; } - + + /** - * Note that unseen from SELECT/EXAMINE is the *position* of the first unseen message, - * not the total unseen count, so it's not be passed in here, but rather only from the unseen - * count from a STATUS command + * Constructs properties for an IMAP folder that can be selected. */ - public FolderProperties(int messages, int email_unread, int recent, UIDValidity? uid_validity, - UID? uid_next, MailboxAttributes attrs) { - // give the base class a zero email_unread, as the notion of "unknown" doesn't exist in - // its contract - base (messages, email_unread, Trillian.UNKNOWN, Trillian.UNKNOWN, Trillian.UNKNOWN, false, - false, false); - - select_examine_messages = messages; - status_messages = -1; - this.recent = recent; + public FolderProperties.selectable(MailboxAttributes attrs, + StatusData status, + Capabilities capabilities) { + this( + attrs, + status.messages, + status.unseen, + capabilities.supports_uidplus() + ); + + this.select_examine_messages = -1; + this.status_messages = status.messages; + this.recent = status.recent; + this.unseen = status.unseen; + this.uid_validity = status.uid_validity; + this.uid_next = status.uid_next; + } + + /** + * Constructs properties for an IMAP folder that can not be selected. + */ + public FolderProperties.not_selectable(MailboxAttributes attrs) { + this(attrs, 0, 0, false); + + this.select_examine_messages = 0; + this.status_messages = -1; + this.recent = 0; + this.unseen = -1; + this.uid_validity = null; + this.uid_next = null; + } + + /** + * Reconstitutes properties for an IMAP folder from the database + */ + internal FolderProperties.from_imapdb(MailboxAttributes attrs, + int email_total, + int email_unread, + UIDValidity? uid_validity, + UID? uid_next) { + this(attrs, email_total, email_unread, false); + + this.select_examine_messages = email_total; + this.status_messages = -1; + this.recent = 0; this.unseen = -1; this.uid_validity = uid_validity; this.uid_next = uid_next; - this.attrs = attrs; - - init_flags(); } - - public FolderProperties.status(StatusData status, MailboxAttributes attrs) { - base (status.messages, status.unseen, Trillian.UNKNOWN, Trillian.UNKNOWN, Trillian.UNKNOWN, - false, false, false); - - select_examine_messages = -1; - status_messages = status.messages; - recent = status.recent; - unseen = status.unseen; - uid_validity = status.uid_validity; - uid_next = status.uid_next; + + protected FolderProperties(MailboxAttributes attrs, + int email_total, + int email_unread, + bool supports_uid) { + Trillian has_children = Trillian.UNKNOWN; + if (attrs.contains(MailboxAttribute.HAS_NO_CHILDREN)) + has_children = Trillian.FALSE; + else if (attrs.contains(MailboxAttribute.HAS_CHILDREN)) + has_children = Trillian.TRUE; + + Trillian supports_children = Trillian.UNKNOWN; + // has_children implies supports_children + if (has_children != Trillian.UNKNOWN) { + supports_children = has_children; + } else { + // !supports_children implies !has_children + supports_children = Trillian.from_boolean(!attrs.contains(MailboxAttribute.NO_INFERIORS)); + if (supports_children.is_impossible()) + has_children = Trillian.FALSE; + } + + Trillian is_openable = Trillian.from_boolean(!attrs.is_no_select); + + base(email_total, email_unread, + has_children, supports_children, is_openable, + false, // not local + false, // not virtual + !supports_uid); + this.attrs = attrs; - - init_flags(); } - + /** * Use with {@link FolderProperties} of the *same folder* seen at different times (i.e. after * SELECTing versus data stored locally). Only compares fields that suggest the contents of @@ -145,30 +193,7 @@ public class Geary.Imap.FolderProperties : Geary.FolderProperties { return false; } - - private void init_flags() { - // \HasNoChildren & \HasChildren are optional attributes (could check for CHILDREN extension, - // but unnecessary here) - if (attrs.contains(MailboxAttribute.HAS_NO_CHILDREN)) - has_children = Trillian.FALSE; - else if (attrs.contains(MailboxAttribute.HAS_CHILDREN)) - has_children = Trillian.TRUE; - else - has_children = Trillian.UNKNOWN; - - // has_children implies supports_children - if (has_children != Trillian.UNKNOWN) { - supports_children = has_children; - } else { - // !supports_children implies !has_children - supports_children = Trillian.from_boolean(!attrs.contains(MailboxAttribute.NO_INFERIORS)); - if (supports_children.is_impossible()) - has_children = Trillian.FALSE; - } - - is_openable = Trillian.from_boolean(!attrs.is_no_select); - } - + /** * Update an existing {@link FolderProperties} with fresh {@link StatusData}. * diff --git a/src/engine/imap/api/imap-folder-session.vala b/src/engine/imap/api/imap-folder-session.vala new file mode 100644 index 00000000..3093ec35 --- /dev/null +++ b/src/engine/imap/api/imap-folder-session.vala @@ -0,0 +1,1069 @@ +/* + * Copyright 2016 Software Freedom Conservancy Inc. + * Copyright 2018 Michael Gratton . + * + * This software is licensed under the GNU Lesser General Public License + * (version 2.1 or later). See the COPYING file in this distribution. + */ + +// this is used internally to indicate a recoverable failure +private errordomain Geary.Imap.FolderError { + RETRY +} + +/** + * An interface between the high-level engine API and an IMAP mailbox. + * + * Because of the complexities of the IMAP protocol, class takes + * common operations that a Geary.Folder implementation would need + * (in particular, {@link Geary.ImapEngine.MinimalFolder}) and makes + * them into simple async calls. + * + * When constructed, this class will issue an IMAP SELECT command for + * the mailbox represented by this folder, placing the session in the + * Selected state. + */ +private class Geary.Imap.FolderSession : Geary.Imap.SessionObject { + + private const Geary.Email.Field BASIC_FETCH_FIELDS = Email.Field.ENVELOPE | Email.Field.DATE + | Email.Field.ORIGINATORS | Email.Field.RECEIVERS | Email.Field.REFERENCES + | Email.Field.SUBJECT | Email.Field.HEADER; + + + /** The folder this session operates on. */ + public Imap.Folder folder { get; private set; } + + /** Determines if this folder immutable. */ + public Trillian readonly { get; private set; default = Trillian.UNKNOWN; } + + /** Determines if this folder accepts custom IMAP flags. */ + public Trillian accepts_user_flags { get; private set; default = Trillian.UNKNOWN; } + + /** Determines if this folder accepts custom IMAP flags. */ + public MessageFlags? permanent_flags { get; private set; default = null; } + + /** + * Set to true when it's detected that the server doesn't allow a + * space between "header.fields" and the list of email headers to + * be requested via FETCH; see: + * [[https://bugzilla.gnome.org/show_bug.cgi?id=714902|Bug * 714902]] + */ + public bool imap_header_fields_hack { get; private set; default = false; } + + private Nonblocking.Mutex cmd_mutex = new Nonblocking.Mutex(); + private Gee.HashMap? fetch_accumulator = null; + private Gee.Set? search_accumulator = null; + + /** + * A (potentially unsolicited) response from the server. + * + * See [[http://tools.ietf.org/html/rfc3501#section-7.3.1]] + */ + public signal void exists(int total); + + /** + * A (potentially unsolicited) response from the server. + * + * See [[http://tools.ietf.org/html/rfc3501#section-7.3.2]] + */ + public signal void recent(int total); + + /** + * A (potentially unsolicited) response from the server. + * + * See [[http://tools.ietf.org/html/rfc3501#section-7.4.1]] + */ + public signal void expunge(SequenceNumber position); + + /** + * Fabricated from the IMAP signals and state obtained at open_async(). + */ + public signal void appended(int total); + + /** + * Fabricated from the IMAP signals and state obtained at open_async(). + */ + public signal void updated(SequenceNumber pos, FetchedData data); + + /** + * Fabricated from the IMAP signals and state obtained at open_async(). + */ + public signal void removed(SequenceNumber pos, int total); + + + public async FolderSession(string account_id, + ClientSession session, + Imap.Folder folder, + Cancellable cancellable) + throws Error { + base("%s:%s".printf(account_id, folder.path.to_string()), session); + this.folder = folder; + + if (folder.properties.attrs.is_no_select) { + throw new ImapError.NOT_SUPPORTED( + "Folder cannot be selected: %s", + folder.path.to_string() + ); + } + + // Update based on our current session + folder.properties.set_from_session_capabilities(session.capabilities); + + // connect to interesting signals *before* selecting + session.exists.connect(on_exists); + session.expunge.connect(on_expunge); + session.fetch.connect(on_fetch); + session.recent.connect(on_recent); + session.search.connect(on_search); + session.status_response_received.connect(on_status_response); + + MailboxSpecifier mailbox = session.get_mailbox_for_path(folder.path); + StatusResponse? response = yield session.select_async( + mailbox, cancellable + ); + + switch (response.status) { + case Status.OK: + // all good + break; + + case Status.BAD: + case Status.NO: + throw new ImapError.NOT_SUPPORTED( + "Server disallowed SELECT %s: %s", + this.folder.path.to_string(), + response.to_string() + ); + + default: + throw new ImapError.SERVER_ERROR( + "Unable to SELECT %s: %s", + this.folder.path.to_string(), + response.to_string() + ); + } + + // if at end of SELECT command accepts_user_flags is still + // UNKKNOWN, treat as TRUE because, according to IMAP spec, if + // PERMANENTFLAGS are not returned, then assume OK + if (this.accepts_user_flags == Trillian.UNKNOWN) + this.accepts_user_flags = Trillian.TRUE; + } + + /** + * {@inheritDoc} + */ + public override ClientSession? close() { + ClientSession? old_session = base.close(); + if (old_session != null) { + old_session.exists.disconnect(on_exists); + old_session.expunge.disconnect(on_expunge); + old_session.fetch.disconnect(on_fetch); + old_session.recent.disconnect(on_recent); + old_session.search.disconnect(on_search); + old_session.status_response_received.disconnect(on_status_response); + } + return old_session; + } + + private void on_exists(int total) { + debug("%s EXISTS %d", to_string(), total); + + int old_total = this.folder.properties.select_examine_messages; + this.folder.properties.set_select_examine_message_count(total); + + exists(total); + if (old_total < total) + appended(total); + } + + private void on_expunge(SequenceNumber pos) { + debug("%s EXPUNGE %s", to_string(), pos.to_string()); + + this.folder.properties.set_select_examine_message_count( + this.folder.properties.select_examine_messages - 1 + ); + + expunge(pos); + removed(pos, this.folder.properties.select_examine_messages); + } + + private void on_fetch(FetchedData data) { + // add if not found, merge if already received data for this email + if (this.fetch_accumulator != null) { + FetchedData? existing = this.fetch_accumulator.get(data.seq_num); + this.fetch_accumulator.set( + data.seq_num, (existing != null) ? data.combine(existing) : data + ); + } else { + debug("%s: FETCH (unsolicited): %s:", + to_string(), + data.to_string()); + updated(data.seq_num, data); + } + } + + private void on_recent(int total) { + debug("%s RECENT %d", to_string(), total); + this.folder.properties.recent = total; + recent(total); + } + + private void on_search(int64[] seq_or_uid) { + // All SEARCH from this class are UID SEARCH, so can reliably convert and add to + // accumulator + if (this.search_accumulator != null) { + foreach (int64 uid in seq_or_uid) { + try { + this.search_accumulator.add(new UID.checked(uid)); + } catch (ImapError imaperr) { + debug("%s Unable to process SEARCH UID result: %s", to_string(), imaperr.message); + } + } + } else { + debug("%s Not handling unsolicited SEARCH response", to_string()); + } + } + + private void on_status_response(StatusResponse status_response) { + // only interested in ResponseCodes here + ResponseCode? response_code = status_response.response_code; + if (response_code == null) + return; + + try { + // Have to take a copy of the string property before evaluation due to this bug: + // https://bugzilla.gnome.org/show_bug.cgi?id=703818 + string value = response_code.get_response_code_type().value; + switch (value) { + case ResponseCodeType.READONLY: + this.readonly = Trillian.TRUE; + break; + + case ResponseCodeType.READWRITE: + this.readonly = Trillian.FALSE; + break; + + case ResponseCodeType.UIDNEXT: + this.folder.properties.uid_next = response_code.get_uid_next(); + break; + + case ResponseCodeType.UIDVALIDITY: + this.folder.properties.uid_validity = response_code.get_uid_validity(); + break; + + case ResponseCodeType.UNSEEN: + // do NOT update properties.unseen, as the UNSEEN response code (here) means + // the sequence number of the first unseen message, not the total count of + // unseen messages + break; + + case ResponseCodeType.PERMANENT_FLAGS: + this.permanent_flags = response_code.get_permanent_flags(); + this.accepts_user_flags = Trillian.from_boolean( + this.permanent_flags.contains(MessageFlag.ALLOWS_NEW) + ); + break; + + default: + // ignored + break; + } + } catch (ImapError ierr) { + debug("Unable to parse ResponseCode %s: %s", response_code.to_string(), + ierr.message); + } + } + + // All commands must executed inside the cmd_mutex; returns FETCH or STORE results + // + // FETCH commands can generate a FolderError.RETRY. State will be updated to accomodate retry, + // but all Commands must be regenerated to ensure new state is reflected in requests. + private async Gee.Map? exec_commands_async(Gee.Collection cmds, + Gee.HashMap? fetch_results, + Gee.Set? search_results, + Cancellable? cancellable) + throws Error { + ClientSession session = claim_session(); + Gee.Map? responses = null; + int token = yield this.cmd_mutex.claim_async(cancellable); + + this.fetch_accumulator = fetch_results; + this.search_accumulator = search_results; + + Error? cmd_err = null; + try { + responses = yield session.send_multiple_commands_async( + cmds, cancellable + ); + } catch (Error err) { + cmd_err = err; + } + + this.fetch_accumulator = null; + this.search_accumulator = null; + + this.cmd_mutex.release(ref token); + + if (cmd_err != null) { + throw cmd_err; + } + + foreach (Command cmd in responses.keys) { + throw_on_failed_status(responses.get(cmd), cmd); + } + + return responses; + } + + // HACK: See https://bugzilla.gnome.org/show_bug.cgi?id=714902 + // + // Detect when a server has returned a BAD response to FETCH BODY[HEADER.FIELDS (HEADER-LIST)] + // due to space between HEADER.FIELDS and (HEADER-LIST) + private bool retry_bad_header_fields_response(Command cmd, StatusResponse response) { + if (response.status != Status.BAD) + return false; + + FetchCommand? fetch = cmd as FetchCommand; + if (fetch == null) + return false; + + foreach (FetchBodyDataSpecifier body_specifier in fetch.for_body_data_specifiers) { + switch (body_specifier.section_part) { + case FetchBodyDataSpecifier.SectionPart.HEADER_FIELDS: + case FetchBodyDataSpecifier.SectionPart.HEADER_FIELDS_NOT: + // use value stored in specifier, not this folder's setting, as it's possible + // the folder's setting was enabled after sending command but before response + // returned + if (body_specifier.request_header_fields_space) + return true; + break; + } + } + + return false; + } + + private void throw_on_failed_status(StatusResponse response, Command cmd) throws Error { + assert(response.is_completion); + + switch (response.status) { + case Status.OK: + return; + + case Status.NO: + throw new ImapError.SERVER_ERROR("Request %s failed on %s: %s", cmd.to_string(), + to_string(), response.to_string()); + + case Status.BAD: { + // if a FetchBodyDataSpecifier is used to request for a header field BAD is returned, + // could be a specific formatting mistake some servers make of not allowing a space + // between the "header.fields" and list of email header names, i.e. + // + // "body[header.fields (references)]" + // + // If so, then enable a hack to work around this and retry the FETCH + if (retry_bad_header_fields_response(cmd, response)) { + imap_header_fields_hack = true; + + throw new FolderError.RETRY("BAD response to header.fields FETCH BODY, retry with hack"); + } + + throw new ImapError.INVALID("Bad request %s on %s: %s", cmd.to_string(), + to_string(), response.to_string()); + } + + default: + throw new ImapError.NOT_SUPPORTED("Unknown response status to %s on %s: %s", + cmd.to_string(), to_string(), response.to_string()); + } + } + + // Utility method for listing UIDs on the remote within the supplied range + public async Gee.Set? list_uids_async(MessageSet msg_set, Cancellable? cancellable) + throws Error { + // Although FETCH could be used, SEARCH is more efficient in returning pure UID results, + // which is all we're interested in here + SearchCriteria criteria = new SearchCriteria(SearchCriterion.message_set(msg_set)); + SearchCommand cmd = new SearchCommand.uid(criteria); + + Gee.Set search_results = new Gee.HashSet(); + yield exec_commands_async( + Geary.iterate(cmd).to_array_list(), + null, + search_results, + cancellable + ); + + return (search_results.size > 0) ? search_results : null; + } + + private Gee.Collection assemble_list_commands(Imap.MessageSet msg_set, + Geary.Email.Field fields, out FetchBodyDataSpecifier? header_specifier, + out FetchBodyDataSpecifier? body_specifier, out FetchBodyDataSpecifier? preview_specifier, + out FetchBodyDataSpecifier? preview_charset_specifier) { + // getting all the fields can require multiple FETCH commands (some servers don't handle + // well putting every required data item into single command), so aggregate FetchCommands + Gee.Collection cmds = new Gee.ArrayList(); + + // if not a UID FETCH, request UIDs for all messages so their EmailIdentifier can be + // created without going back to the database (assuming the messages have already been + // pulled down, not a guarantee); if request is for NONE, that guarantees that the + // EmailIdentifier will be set, and so fetch UIDs (which looks funny but works when + // listing a range for contents: UID FETCH x:y UID) + if (!msg_set.is_uid || fields == Geary.Email.Field.NONE) + cmds.add(new FetchCommand.data_type(msg_set, FetchDataSpecifier.UID)); + + // convert bulk of the "basic" fields into a one or two FETCH commands (some servers have + // exhibited bugs or return NO when too many FETCH data types are combined on a single + // command) + if (fields.requires_any(BASIC_FETCH_FIELDS)) { + Gee.List data_types = new Gee.ArrayList(); + fields_to_fetch_data_types(fields, data_types, out header_specifier); + + // Add all simple data types as one FETCH command + if (data_types.size > 0) + cmds.add(new FetchCommand(msg_set, data_types, null)); + + // Add all body data types as separate FETCH command + if (header_specifier != null) + cmds.add(new FetchCommand.body_data_type(msg_set, header_specifier)); + } else { + header_specifier = null; + } + + // RFC822 BODY is a separate command + if (fields.require(Email.Field.BODY)) { + body_specifier = new FetchBodyDataSpecifier.peek(FetchBodyDataSpecifier.SectionPart.TEXT, + null, -1, -1, null); + + cmds.add(new FetchCommand.body_data_type(msg_set, body_specifier)); + } else { + body_specifier = null; + } + + // PREVIEW obtains the content type and a truncated version of + // the first part of the message, which often leads to poor + // results. It can also be also be synthesised from the + // email's RFC822 message in fetched_data_to_email, if the + // fields needed for reconstructing the RFC822 message are + // present. If so, rely on that and don't also request any + // additional data for the preview here. + if (fields.require(Email.Field.PREVIEW) && + !fields.require(Email.REQUIRED_FOR_MESSAGE)) { + // Get the preview text (the initial MAX_PREVIEW_BYTES of + // the first MIME section + + preview_specifier = new FetchBodyDataSpecifier.peek(FetchBodyDataSpecifier.SectionPart.NONE, + { 1 }, 0, Geary.Email.MAX_PREVIEW_BYTES, null); + cmds.add(new FetchCommand.body_data_type(msg_set, preview_specifier)); + + // Also get the character set to properly decode it + preview_charset_specifier = new FetchBodyDataSpecifier.peek( + FetchBodyDataSpecifier.SectionPart.MIME, { 1 }, -1, -1, null); + cmds.add(new FetchCommand.body_data_type(msg_set, preview_charset_specifier)); + } else { + preview_specifier = null; + preview_charset_specifier = null; + } + + // PROPERTIES and FLAGS are a separate command + if (fields.requires_any(Email.Field.PROPERTIES | Email.Field.FLAGS)) { + Gee.List data_types = new Gee.ArrayList(); + + if (fields.require(Geary.Email.Field.PROPERTIES)) { + data_types.add(FetchDataSpecifier.INTERNALDATE); + data_types.add(FetchDataSpecifier.RFC822_SIZE); + } + + if (fields.require(Geary.Email.Field.FLAGS)) + data_types.add(FetchDataSpecifier.FLAGS); + + cmds.add(new FetchCommand(msg_set, data_types, null)); + } + + return cmds; + } + + // Returns a no-message-id ImapDB.EmailIdentifier with the UID stored in it. + public async Gee.List? list_email_async(MessageSet msg_set, + Geary.Email.Field fields, + Cancellable? cancellable) + throws Error { + Gee.HashMap fetched = + new Gee.HashMap(); + FetchBodyDataSpecifier? header_specifier = null; + FetchBodyDataSpecifier? body_specifier = null; + FetchBodyDataSpecifier? preview_specifier = null; + FetchBodyDataSpecifier? preview_charset_specifier = null; + for (;;) { + Gee.Collection cmds = assemble_list_commands(msg_set, fields, + out header_specifier, out body_specifier, out preview_specifier, + out preview_charset_specifier); + if (cmds.size == 0) { + throw new ImapError.INVALID("No FETCH commands generate for list request %s %s", + msg_set.to_string(), fields.to_list_string()); + } + + // Commands prepped, do the fetch and accumulate all the responses + try { + yield exec_commands_async(cmds, fetched, null, cancellable); + } catch (Error err) { + if (err is FolderError.RETRY) { + debug("Retryable server failure detected for %s: %s", to_string(), err.message); + + continue; + } + + throw err; + } + + break; + } + + if (fetched.size == 0) + return null; + + // Convert fetched data into Geary.Email objects + // because this could be for a lot of email, do in a background thread + Gee.List email_list = new Gee.ArrayList(); + yield Nonblocking.Concurrent.global.schedule_async(() => { + foreach (SequenceNumber seq_num in fetched.keys) { + FetchedData fetched_data = fetched.get(seq_num); + + // the UID should either have been fetched (if using positional addressing) or should + // have come back with the response (if using UID addressing) + UID? uid = fetched_data.data_map.get(FetchDataSpecifier.UID) as UID; + if (uid == null) { + message("Unable to list message #%s on %s: No UID returned from server", + seq_num.to_string(), to_string()); + + continue; + } + + try { + Geary.Email email = fetched_data_to_email(to_string(), uid, fetched_data, fields, + header_specifier, body_specifier, preview_specifier, preview_charset_specifier); + if (!email.fields.fulfills(fields)) { + message("%s: %s missing=%s fetched=%s", to_string(), email.id.to_string(), + fields.clear(email.fields).to_list_string(), fetched_data.to_string()); + + continue; + } + + email_list.add(email); + } catch (Error err) { + debug("%s: Unable to convert email for %s %s: %s", to_string(), uid.to_string(), + fetched_data.to_string(), err.message); + } + } + }, cancellable); + + return (email_list.size > 0) ? email_list : null; + } + + /** + * Returns the sequence numbers for a set of UIDs. + * + * The `msg_set` parameter must be a set containing UIDs. An error + * is thrown if the sequence numbers cannot be determined. + */ + public async Gee.Map uid_to_position_async(MessageSet msg_set, + Cancellable? cancellable) + throws Error { + if (!msg_set.is_uid) { + throw new ImapError.NOT_SUPPORTED("Message set must contain UIDs"); + } + + Gee.List cmds = new Gee.ArrayList(); + cmds.add(new FetchCommand.data_type(msg_set, FetchDataSpecifier.UID)); + + Gee.HashMap fetched = + new Gee.HashMap(); + yield exec_commands_async(cmds, fetched, null, cancellable); + + if (fetched.is_empty) { + throw new ImapError.INVALID("Server returned no sequence numbers"); + } + + Gee.Map map = new Gee.HashMap(); + foreach (SequenceNumber seq_num in fetched.keys) { + map.set( + (UID) fetched.get(seq_num).data_map.get(FetchDataSpecifier.UID), + seq_num + ); + } + return map; + } + + public async void remove_email_async(Gee.List msg_sets, Cancellable? cancellable) + throws Error { + ClientSession session = claim_session(); + Gee.List flags = new Gee.ArrayList(); + flags.add(MessageFlag.DELETED); + + Gee.List cmds = new Gee.ArrayList(); + + // Build STORE command for all MessageSets, see if all are UIDs so we can use UID EXPUNGE + bool all_uid = true; + foreach (MessageSet msg_set in msg_sets) { + if (!msg_set.is_uid) + all_uid = false; + + cmds.add(new StoreCommand(msg_set, flags, StoreCommand.Option.ADD_FLAGS)); + } + + // TODO: Only use old-school EXPUNGE when closing folder (or rely on CLOSE to do that work + // for us). See: + // http://redmine.yorba.org/issues/7532 + // + // However, current client implementation doesn't properly close INBOX when application + // shuts down, which means deleted messages return at application start. See: + // http://redmine.yorba.org/issues/6865 + if (all_uid && session.capabilities.supports_uidplus()) { + foreach (MessageSet msg_set in msg_sets) + cmds.add(new ExpungeCommand.uid(msg_set)); + } else { + cmds.add(new ExpungeCommand()); + } + + yield exec_commands_async(cmds, null, null, cancellable); + } + + public async void mark_email_async(Gee.List msg_sets, Geary.EmailFlags? flags_to_add, + Geary.EmailFlags? flags_to_remove, Cancellable? cancellable) throws Error { + Gee.List msg_flags_add = new Gee.ArrayList(); + Gee.List msg_flags_remove = new Gee.ArrayList(); + MessageFlag.from_email_flags(flags_to_add, flags_to_remove, out msg_flags_add, + out msg_flags_remove); + + if (msg_flags_add.size == 0 && msg_flags_remove.size == 0) + return; + + Gee.Collection cmds = new Gee.ArrayList(); + foreach (MessageSet msg_set in msg_sets) { + if (msg_flags_add.size > 0) + cmds.add(new StoreCommand(msg_set, msg_flags_add, StoreCommand.Option.ADD_FLAGS)); + + if (msg_flags_remove.size > 0) + cmds.add(new StoreCommand(msg_set, msg_flags_remove, StoreCommand.Option.REMOVE_FLAGS)); + } + + yield exec_commands_async(cmds, null, null, cancellable); + } + + // Returns a mapping of the source UID to the destination UID. If the MessageSet is not for + // UIDs, then null is returned. If the server doesn't support COPYUID, null is returned. + public async Gee.Map? copy_email_async(MessageSet msg_set, FolderPath destination, + Cancellable? cancellable) throws Error { + ClientSession session = claim_session(); + + MailboxSpecifier mailbox = session.get_mailbox_for_path(destination); + CopyCommand cmd = new CopyCommand(msg_set, mailbox); + + Gee.Map? responses = yield exec_commands_async( + Geary.iterate(cmd).to_array_list(), null, null, cancellable); + + if (!responses.has_key(cmd)) + return null; + + StatusResponse response = responses.get(cmd); + if (response.response_code != null && msg_set.is_uid) { + Gee.List? src_uids = null; + Gee.List? dst_uids = null; + try { + response.response_code.get_copyuid(null, out src_uids, out dst_uids); + } catch (ImapError ierr) { + debug("Unable to retrieve COPYUID UIDs: %s", ierr.message); + } + + if (!Collection.is_empty(src_uids) && !Collection.is_empty(dst_uids)) { + Gee.Map copyuids = new Gee.HashMap(); + int ctr = 0; + for (;;) { + UID? src_uid = (ctr < src_uids.size) ? src_uids[ctr] : null; + UID? dst_uid = (ctr < dst_uids.size) ? dst_uids[ctr] : null; + + if (src_uid != null && dst_uid != null) + copyuids.set(src_uid, dst_uid); + else + break; + + ctr++; + } + + if (copyuids.size > 0) + return copyuids; + } + } + + return null; + } + + public async Gee.SortedSet? search_async(SearchCriteria criteria, Cancellable? cancellable) + throws Error { + // always perform a UID SEARCH + Gee.Collection cmds = new Gee.ArrayList(); + cmds.add(new SearchCommand.uid(criteria)); + + Gee.Set search_results = new Gee.HashSet(); + yield exec_commands_async(cmds, null, search_results, cancellable); + + Gee.SortedSet tree = null; + if (search_results.size > 0) { + tree = new Gee.TreeSet(); + tree.add_all(search_results); + } + return tree; + } + + // NOTE: If fields are added or removed from this method, BASIC_FETCH_FIELDS *must* be updated + // as well + private void fields_to_fetch_data_types(Geary.Email.Field fields, + Gee.List data_types_list, out FetchBodyDataSpecifier? header_specifier) { + // pack all the needed headers into a single FetchBodyDataType + string[] field_names = new string[0]; + + // The assumption here is that because ENVELOPE is such a common fetch command, the + // server will have optimizations for it, whereas if we called for each header in the + // envelope separately, the server has to chunk harder parsing the RFC822 header ... have + // to add References because IMAP ENVELOPE doesn't return them for some reason (but does + // return Message-ID and In-Reply-To) + if (fields.is_all_set(Geary.Email.Field.ENVELOPE)) { + data_types_list.add(FetchDataSpecifier.ENVELOPE); + field_names += "References"; + + // remove those flags and process any remaining + fields = fields.clear(Geary.Email.Field.ENVELOPE); + } + + foreach (Geary.Email.Field field in Geary.Email.Field.all()) { + switch (fields & field) { + case Geary.Email.Field.DATE: + field_names += "Date"; + break; + + case Geary.Email.Field.ORIGINATORS: + field_names += "From"; + field_names += "Sender"; + field_names += "Reply-To"; + break; + + case Geary.Email.Field.RECEIVERS: + field_names += "To"; + field_names += "Cc"; + field_names += "Bcc"; + break; + + case Geary.Email.Field.REFERENCES: + field_names += "References"; + field_names += "Message-ID"; + field_names += "In-Reply-To"; + break; + + case Geary.Email.Field.SUBJECT: + field_names += "Subject"; + break; + + case Geary.Email.Field.HEADER: + // TODO: If the entire header is being pulled, then no need to pull down partial + // headers; simply get them all and decode what is needed directly + data_types_list.add(FetchDataSpecifier.RFC822_HEADER); + break; + + case Geary.Email.Field.NONE: + case Geary.Email.Field.BODY: + case Geary.Email.Field.PROPERTIES: + case Geary.Email.Field.FLAGS: + case Geary.Email.Field.PREVIEW: + // not set or fetched separately + break; + + default: + assert_not_reached(); + } + } + + // convert field names into single FetchBodyDataType object + if (field_names.length > 0) { + header_specifier = new FetchBodyDataSpecifier.peek( + FetchBodyDataSpecifier.SectionPart.HEADER_FIELDS, null, -1, -1, field_names); + if (imap_header_fields_hack) + header_specifier.omit_request_header_fields_space(); + } else { + header_specifier = null; + } + } + + private static Geary.Email fetched_data_to_email(string folder_name, UID uid, + FetchedData fetched_data, Geary.Email.Field required_fields, + FetchBodyDataSpecifier? header_specifier, FetchBodyDataSpecifier? body_specifier, + FetchBodyDataSpecifier? preview_specifier, FetchBodyDataSpecifier? preview_charset_specifier) throws Error { + // note the use of INVALID_ROWID, as the rowid for this email (if one is present in the + // database) is unknown at this time; this means ImapDB *must* create a new EmailIdentifier + // for this email after create/merge is completed + Geary.Email email = new Geary.Email(new ImapDB.EmailIdentifier.no_message_id(uid)); + + // accumulate these to submit Imap.EmailProperties all at once + InternalDate? internaldate = null; + RFC822.Size? rfc822_size = null; + + // accumulate these to submit References all at once + RFC822.MessageID? message_id = null; + RFC822.MessageIDList? in_reply_to = null; + RFC822.MessageIDList? references = null; + + // loop through all available FetchDataTypes and gather converted data + foreach (FetchDataSpecifier data_type in fetched_data.data_map.keys) { + MessageData? data = fetched_data.data_map.get(data_type); + if (data == null) + continue; + + switch (data_type) { + case FetchDataSpecifier.ENVELOPE: + Envelope envelope = (Envelope) data; + + email.set_send_date(envelope.sent); + email.set_message_subject(envelope.subject); + email.set_originators( + envelope.from, + envelope.sender.equal_to(envelope.from) || envelope.sender.size == 0 ? null : envelope.sender[0], + envelope.reply_to.equal_to(envelope.from) ? null : envelope.reply_to + ); + email.set_receivers(envelope.to, envelope.cc, envelope.bcc); + + // store these to add to References all at once + message_id = envelope.message_id; + in_reply_to = envelope.in_reply_to; + break; + + case FetchDataSpecifier.RFC822_HEADER: + email.set_message_header((RFC822.Header) data); + break; + + case FetchDataSpecifier.RFC822_TEXT: + email.set_message_body((RFC822.Text) data); + break; + + case FetchDataSpecifier.RFC822_SIZE: + rfc822_size = (RFC822.Size) data; + break; + + case FetchDataSpecifier.FLAGS: + email.set_flags(new Imap.EmailFlags((MessageFlags) data)); + break; + + case FetchDataSpecifier.INTERNALDATE: + internaldate = (InternalDate) data; + break; + + default: + // everything else dropped on the floor (not applicable to Geary.Email) + break; + } + } + + // Only set PROPERTIES if all have been found + if (internaldate != null && rfc822_size != null) + email.set_email_properties(new Geary.Imap.EmailProperties(internaldate, rfc822_size)); + + // if the header was requested, convert its fields now + bool has_header_specifier = fetched_data.body_data_map.has_key(header_specifier); + if (header_specifier != null && !has_header_specifier) { + message("[%s] No header specifier \"%s\" found:", folder_name, + header_specifier.to_string()); + foreach (FetchBodyDataSpecifier specifier in fetched_data.body_data_map.keys) + message("[%s] has %s", folder_name, specifier.to_string()); + } else if (header_specifier != null && has_header_specifier) { + RFC822.Header headers = new RFC822.Header( + fetched_data.body_data_map.get(header_specifier)); + + // DATE + if (required_but_not_set(Geary.Email.Field.DATE, required_fields, email)) { + string? value = headers.get_header("Date"); + if (!String.is_empty(value)) + email.set_send_date(new RFC822.Date(value)); + else + email.set_send_date(null); + } + + // ORIGINATORS + if (required_but_not_set(Geary.Email.Field.ORIGINATORS, required_fields, email)) { + RFC822.MailboxAddresses? from = null; + string? value = headers.get_header("From"); + if (!String.is_empty(value)) + from = new RFC822.MailboxAddresses.from_rfc822_string(value); + + RFC822.MailboxAddress? sender = null; + value = headers.get_header("Sender"); + if (!String.is_empty(value)) + sender = new RFC822.MailboxAddress.from_rfc822_string(value); + + RFC822.MailboxAddresses? reply_to = null; + value = headers.get_header("Reply-To"); + if (!String.is_empty(value)) + reply_to = new RFC822.MailboxAddresses.from_rfc822_string(value); + + email.set_originators(from, sender, reply_to); + } + + // RECEIVERS + if (required_but_not_set(Geary.Email.Field.RECEIVERS, required_fields, email)) { + RFC822.MailboxAddresses? to = null; + string? value = headers.get_header("To"); + if (!String.is_empty(value)) + to = new RFC822.MailboxAddresses.from_rfc822_string(value); + + RFC822.MailboxAddresses? cc = null; + value = headers.get_header("Cc"); + if (!String.is_empty(value)) + cc = new RFC822.MailboxAddresses.from_rfc822_string(value); + + RFC822.MailboxAddresses? bcc = null; + value = headers.get_header("Bcc"); + if (!String.is_empty(value)) + bcc = new RFC822.MailboxAddresses.from_rfc822_string(value); + + email.set_receivers(to, cc, bcc); + } + + // REFERENCES + // (Note that it's possible the request used an IMAP ENVELOPE, in which case only the + // References header will be present if REFERENCES were required, which is why + // REFERENCES is set at the bottom of the method, when all information has been gathered + if (message_id == null) { + string? value = headers.get_header("Message-ID"); + if (!String.is_empty(value)) + message_id = new RFC822.MessageID(value); + } + + if (in_reply_to == null) { + string? value = headers.get_header("In-Reply-To"); + if (!String.is_empty(value)) + in_reply_to = new RFC822.MessageIDList.from_rfc822_string(value); + } + + if (references == null) { + string? value = headers.get_header("References"); + if (!String.is_empty(value)) + references = new RFC822.MessageIDList.from_rfc822_string(value); + } + + // SUBJECT + // Unlike DATE, allow for empty subjects + if (required_but_not_set(Geary.Email.Field.SUBJECT, required_fields, email)) { + string? value = headers.get_header("Subject"); + if (value != null) + email.set_message_subject(new RFC822.Subject.decode(value)); + else + email.set_message_subject(null); + } + } + + // It's possible for all these fields to be null even though they were requested from + // the server, so use requested fields for determination + if (required_but_not_set(Geary.Email.Field.REFERENCES, required_fields, email)) + email.set_full_references(message_id, in_reply_to, references); + + // if preview was requested, get it now ... both identifiers + // must be supplied if one is + if (preview_specifier != null || preview_charset_specifier != null) { + assert(preview_specifier != null && preview_charset_specifier != null); + + if (fetched_data.body_data_map.has_key(preview_specifier) + && fetched_data.body_data_map.has_key(preview_charset_specifier)) { + email.set_message_preview(new RFC822.PreviewText.with_header( + fetched_data.body_data_map.get(preview_specifier), + fetched_data.body_data_map.get(preview_charset_specifier))); + } else { + message("[%s] No preview specifiers \"%s\" and \"%s\" found", folder_name, + preview_specifier.to_string(), preview_charset_specifier.to_string()); + foreach (FetchBodyDataSpecifier specifier in fetched_data.body_data_map.keys) + message("[%s] has %s", folder_name, specifier.to_string()); + } + } + + // If body was requested, get it now. We also set the preview + // here from the body if possible since for HTML messages at + // least there's a lot of boilerplate HTML to wade through to + // get some actual preview text, which usually requires more + // than Geary.Email.MAX_PREVIEW_BYTES will allow for + if (body_specifier != null) { + if (fetched_data.body_data_map.has_key(body_specifier)) { + email.set_message_body(new Geary.RFC822.Text( + fetched_data.body_data_map.get(body_specifier))); + + // Try to set the preview + Geary.RFC822.Message? message = null; + try { + message = email.get_message(); + } catch (Error e) { + // Not enough fields to construct the message + } + if (message != null) { + string preview = message.get_preview(); + if (preview.length > Geary.Email.MAX_PREVIEW_BYTES) { + preview = Geary.String.safe_byte_substring( + preview, Geary.Email.MAX_PREVIEW_BYTES + ); + } + email.set_message_preview( + new RFC822.PreviewText.from_string(preview) + ); + } + } else { + message("[%s] No body specifier \"%s\" found", folder_name, + body_specifier.to_string()); + foreach (FetchBodyDataSpecifier specifier in fetched_data.body_data_map.keys) + message("[%s] has %s", folder_name, specifier.to_string()); + } + } + + return email; + } + + // Returns a no-message-id ImapDB.EmailIdentifier with the UID stored in it. + // This method does not take a cancellable; there is currently no way to tell if an email was + // created or not if exec_commands_async() is cancelled during the append. For atomicity's sake, + // callers need to remove the returned email ID if a cancel occurred. + public async Geary.EmailIdentifier? create_email_async(RFC822.Message message, Geary.EmailFlags? flags, + DateTime? date_received) throws Error { + ClientSession session = claim_session(); + + MessageFlags? msg_flags = null; + if (flags != null) { + Imap.EmailFlags imap_flags = Imap.EmailFlags.from_api_email_flags(flags); + msg_flags = imap_flags.message_flags; + } else { + msg_flags = new MessageFlags(Geary.iterate(MessageFlag.SEEN).to_array_list()); + } + + InternalDate? internaldate = null; + if (date_received != null) + internaldate = new InternalDate.from_date_time(date_received); + + MailboxSpecifier mailbox = session.get_mailbox_for_path(this.folder.path); + AppendCommand cmd = new AppendCommand( + mailbox, msg_flags, internaldate, message.get_network_buffer(false) + ); + + Gee.Map responses = yield exec_commands_async( + Geary.iterate(cmd).to_array_list(), null, null, null); + + // Grab the response and parse out the UID, if available. + StatusResponse response = responses.get(cmd); + if (response.status == Status.OK && response.response_code != null && + response.response_code.get_response_code_type().is_value("appenduid")) { + UID new_id = new UID.checked(response.response_code.get_as_string(2).as_int64()); + + return new ImapDB.EmailIdentifier.no_message_id(new_id); + } + + // We didn't get a UID back from the server. + return null; + } + + private static bool required_but_not_set(Geary.Email.Field check, Geary.Email.Field users_fields, Geary.Email email) { + return users_fields.require(check) ? !email.fields.is_all_set(check) : false; + } +} diff --git a/src/engine/imap/api/imap-folder.vala b/src/engine/imap/api/imap-folder.vala index dc55f74c..2144bab8 100644 --- a/src/engine/imap/api/imap-folder.vala +++ b/src/engine/imap/api/imap-folder.vala @@ -1,1116 +1,34 @@ -/* Copyright 2016 Software Freedom Conservancy Inc. +/* + * Copyright 2016 Software Freedom Conservancy Inc. + * Copyright 2018 Michael Gratton . * * This software is licensed under the GNU Lesser General Public License * (version 2.1 or later). See the COPYING file in this distribution. */ -// this is used internally to indicate a recoverable failure -private errordomain Geary.Imap.FolderError { - RETRY -} - /** - * An interface between the high-level engine API and a single IMAP mailbox. + * Represents a mailbox on an IMAP server. * - * When opening, this class will claim a {@link ClientSession} and - * issue an IMAP SELECT command for the mailbox represented by this - * folder. On closing, the session is released, causing it to be - * returned to the pool where an IMAP CLOSE will be issued. + * Everything we can glean from an IMAP LIST for a specific folder is + * encapsulated here. Any information requires the folder to be + * selected, and hence there is no other information about + * non-selectable folders that can be obtained. + * + * Note the mailbox name is not represented since that may differ + * based on the client session being used to connect to the server. */ -private class Geary.Imap.Folder : BaseObject { +internal class Geary.Imap.Folder : Geary.BaseObject { - private const Geary.Email.Field BASIC_FETCH_FIELDS = Email.Field.ENVELOPE | Email.Field.DATE - | Email.Field.ORIGINATORS | Email.Field.RECEIVERS | Email.Field.REFERENCES - | Email.Field.SUBJECT | Email.Field.HEADER; - - public bool is_open { get; private set; default = false; } + /** The full path to this folder. */ public FolderPath path { get; private set; } - public Imap.FolderProperties properties { get; private set; } - public MessageFlags? permanent_flags { get; private set; default = null; } - public Trillian readonly { get; private set; default = Trillian.UNKNOWN; } - public Trillian accepts_user_flags { get; private set; default = Trillian.UNKNOWN; } - /** - * Set to true when it's detected that the server doesn't allow a - * space between "header.fields" and the list of email headers to - * be requested via FETCH; see: - * [[https://bugzilla.gnome.org/show_bug.cgi?id=714902|Bug * 714902]] - */ - public bool imap_header_fields_hack { get; private set; default = false; } - - private ClientSessionManager session_mgr; - private ClientSession? session = null; - private Nonblocking.Mutex cmd_mutex = new Nonblocking.Mutex(); - private Gee.HashMap? fetch_accumulator = null; - private Gee.Set? search_accumulator = null; - - /** - * A (potentially unsolicited) response from the server. - * - * See [[http://tools.ietf.org/html/rfc3501#section-7.3.1]] - */ - public signal void exists(int total); - - /** - * A (potentially unsolicited) response from the server. - * - * See [[http://tools.ietf.org/html/rfc3501#section-7.3.2]] - */ - public signal void recent(int total); - - /** - * A (potentially unsolicited) response from the server. - * - * See [[http://tools.ietf.org/html/rfc3501#section-7.4.1]] - */ - public signal void expunge(SequenceNumber position); - - /** - * Fabricated from the IMAP signals and state obtained at open_async(). - */ - public signal void appended(int total); - - /** - * Fabricated from the IMAP signals and state obtained at open_async(). - */ - public signal void updated(SequenceNumber pos, FetchedData data); - - /** - * Fabricated from the IMAP signals and state obtained at open_async(). - */ - public signal void removed(SequenceNumber pos, int total); - - /** - * Note that close_async() still needs to be called after this signal is fired. - */ - public signal void disconnected(ClientSession.DisconnectReason reason); + /** IMAP properties reported by the server. */ + public Imap.FolderProperties properties { get; private set; } - internal Folder(FolderPath path, Imap.FolderProperties properties, ClientSessionManager session_mgr) { + internal Folder(FolderPath path, Imap.FolderProperties properties) { this.path = path; this.properties = properties; - this.session_mgr = session_mgr; } - public async void open_async(Cancellable? cancellable) throws Error { - if (is_open) - throw new EngineError.ALREADY_OPEN("%s already open", to_string()); - - session = yield session_mgr.claim_authorized_session_async(cancellable); - - // connect to interesting signals *before* selecting - session.exists.connect(on_exists); - session.expunge.connect(on_expunge); - session.fetch.connect(on_fetch); - session.recent.connect(on_recent); - session.search.connect(on_search); - session.status_response_received.connect(on_status_response); - session.disconnected.connect(on_disconnected); - - properties.set_from_session_capabilities(session.capabilities); - - MailboxSpecifier mailbox = this.session.get_mailbox_for_path(this.path); - StatusResponse? response = null; - Error? select_err = null; - try { - response = yield this.session.select_async(mailbox, cancellable); - } catch (Error err) { - select_err = err; - } - - // if select_err is null, then response can not be null - if (select_err != null || response.status != Status.OK) { - // don't use user-supplied cancellable; it may be cancelled, and even if not, do not want - // to cancel this operation - yield release_session_async(null); - - if (select_err != null) - throw select_err; - - switch (response.status) { - case Status.BAD: - case Status.NO: - throw new ImapError.NOT_SUPPORTED("Server disallowed SELECT %s: %s", path.to_string(), - response.to_string()); - - default: - throw new ImapError.SERVER_ERROR("Unable to SELECT %s: %s", path.to_string(), - response.to_string()); - } - } - - // if at end of SELECT command accepts_user_flags is still UNKKNOWN, treat as TRUE because, - // according to IMAP spec, if PERMANENTFLAGS are not returned, then assume OK - if (accepts_user_flags == Trillian.UNKNOWN) - accepts_user_flags = Trillian.TRUE; - - is_open = true; - } - - public async void close_async(Cancellable? cancellable) throws Error { - if (!is_open) - return; - - yield release_session_async(cancellable); - - this.fetch_accumulator = null; - this.search_accumulator = null; - - this.readonly = Trillian.UNKNOWN; - this.accepts_user_flags = Trillian.UNKNOWN; - - this.is_open = false; - } - - private async void release_session_async(Cancellable? cancellable) { - if (this.session == null) - return; - - this.session.exists.disconnect(on_exists); - this.session.expunge.disconnect(on_expunge); - this.session.fetch.disconnect(on_fetch); - this.session.recent.disconnect(on_recent); - this.session.search.disconnect(on_search); - this.session.status_response_received.disconnect(on_status_response); - this.session.disconnected.disconnect(on_disconnected); - - ClientSession release_session = this.session; - this.session = null; - try { - yield session_mgr.release_session_async(release_session, cancellable); - } catch (Error err) { - debug("Unable to release session %s: %s", release_session.to_string(), err.message); - } - } - - private void on_exists(int total) { - debug("%s EXISTS %d", to_string(), total); - - int old_total = properties.select_examine_messages; - properties.set_select_examine_message_count(total); - - // don't fire signals until opened - if (!is_open) - return; - - exists(total); - if (old_total < total) - appended(total); - } - - private void on_expunge(SequenceNumber pos) { - debug("%s EXPUNGE %s", to_string(), pos.to_string()); - - properties.set_select_examine_message_count(properties.select_examine_messages - 1); - - // don't fire signals until opened - if (!is_open) - return; - - expunge(pos); - removed(pos, properties.select_examine_messages); - } - - private void on_fetch(FetchedData data) { - // add if not found, merge if already received data for this email - if (this.fetch_accumulator != null) { - FetchedData? existing = this.fetch_accumulator.get(data.seq_num); - this.fetch_accumulator.set( - data.seq_num, (existing != null) ? data.combine(existing) : data - ); - } else { - debug("%s: FETCH (unsolicited): %s:", - to_string(), - data.to_string()); - updated(data.seq_num, data); - } - } - - private void on_recent(int total) { - debug("%s RECENT %d", to_string(), total); - - properties.recent = total; - - // don't fire signal until opened - if (is_open) - recent(total); - } - - private void on_search(int64[] seq_or_uid) { - // All SEARCH from this class are UID SEARCH, so can reliably convert and add to - // accumulator - if (this.search_accumulator != null) { - foreach (int64 uid in seq_or_uid) { - try { - this.search_accumulator.add(new UID.checked(uid)); - } catch (ImapError imaperr) { - debug("%s Unable to process SEARCH UID result: %s", to_string(), imaperr.message); - } - } - } else { - debug("%s Not handling unsolicited SEARCH response", to_string()); - } - } - - private void on_status_response(StatusResponse status_response) { - // only interested in ResponseCodes here - ResponseCode? response_code = status_response.response_code; - if (response_code == null) - return; - - try { - // Have to take a copy of the string property before evaluation due to this bug: - // https://bugzilla.gnome.org/show_bug.cgi?id=703818 - string value = response_code.get_response_code_type().value; - switch (value) { - case ResponseCodeType.READONLY: - readonly = Trillian.TRUE; - break; - - case ResponseCodeType.READWRITE: - readonly = Trillian.FALSE; - break; - - case ResponseCodeType.UIDNEXT: - properties.uid_next = response_code.get_uid_next(); - break; - - case ResponseCodeType.UIDVALIDITY: - properties.uid_validity = response_code.get_uid_validity(); - break; - - case ResponseCodeType.UNSEEN: - // do NOT update properties.unseen, as the UNSEEN response code (here) means - // the sequence number of the first unseen message, not the total count of - // unseen messages - break; - - case ResponseCodeType.PERMANENT_FLAGS: - permanent_flags = response_code.get_permanent_flags(); - accepts_user_flags = Trillian.from_boolean( - permanent_flags.contains(MessageFlag.ALLOWS_NEW)); - break; - - default: - // ignored - break; - } - } catch (ImapError ierr) { - debug("Unable to parse ResponseCode %s: %s", response_code.to_string(), - ierr.message); - } - } - - private void on_disconnected(ClientSession.DisconnectReason reason) { - debug("%s DISCONNECTED %s", to_string(), reason.to_string()); - - disconnected(reason); - } - - private void check_open() throws Error { - if (!is_open || session == null) - throw new EngineError.OPEN_REQUIRED("Imap.Folder %s not open", to_string()); - } - - // All commands must executed inside the cmd_mutex; returns FETCH or STORE results - // - // FETCH commands can generate a FolderError.RETRY. State will be updated to accomodate retry, - // but all Commands must be regenerated to ensure new state is reflected in requests. - private async Gee.Map? exec_commands_async(Gee.Collection cmds, - Gee.HashMap? fetch_results, - Gee.Set? search_results, - Cancellable? cancellable) - throws Error { - Gee.Map? responses = null; - int token = yield cmd_mutex.claim_async(cancellable); - Error? thrown = null; - try { - check_open(); - - this.fetch_accumulator = fetch_results; - this.search_accumulator = search_results; - responses = yield session.send_multiple_commands_async(cmds, cancellable); - } catch (Error err) { - thrown = err; - } - - this.fetch_accumulator = null; - this.search_accumulator = null; - - cmd_mutex.release(ref token); - - if (thrown != null) { - throw thrown; - } - - foreach (Command cmd in responses.keys) { - throw_on_failed_status(responses.get(cmd), cmd); - } - - return responses; - } - - // HACK: See https://bugzilla.gnome.org/show_bug.cgi?id=714902 - // - // Detect when a server has returned a BAD response to FETCH BODY[HEADER.FIELDS (HEADER-LIST)] - // due to space between HEADER.FIELDS and (HEADER-LIST) - private bool retry_bad_header_fields_response(Command cmd, StatusResponse response) { - if (response.status != Status.BAD) - return false; - - FetchCommand? fetch = cmd as FetchCommand; - if (fetch == null) - return false; - - foreach (FetchBodyDataSpecifier body_specifier in fetch.for_body_data_specifiers) { - switch (body_specifier.section_part) { - case FetchBodyDataSpecifier.SectionPart.HEADER_FIELDS: - case FetchBodyDataSpecifier.SectionPart.HEADER_FIELDS_NOT: - // use value stored in specifier, not this folder's setting, as it's possible - // the folder's setting was enabled after sending command but before response - // returned - if (body_specifier.request_header_fields_space) - return true; - break; - } - } - - return false; - } - - private void throw_on_failed_status(StatusResponse response, Command cmd) throws Error { - assert(response.is_completion); - - switch (response.status) { - case Status.OK: - return; - - case Status.NO: - throw new ImapError.SERVER_ERROR("Request %s failed on %s: %s", cmd.to_string(), - to_string(), response.to_string()); - - case Status.BAD: { - // if a FetchBodyDataSpecifier is used to request for a header field BAD is returned, - // could be a specific formatting mistake some servers make of not allowing a space - // between the "header.fields" and list of email header names, i.e. - // - // "body[header.fields (references)]" - // - // If so, then enable a hack to work around this and retry the FETCH - if (retry_bad_header_fields_response(cmd, response)) { - imap_header_fields_hack = true; - - throw new FolderError.RETRY("BAD response to header.fields FETCH BODY, retry with hack"); - } - - throw new ImapError.INVALID("Bad request %s on %s: %s", cmd.to_string(), - to_string(), response.to_string()); - } - - default: - throw new ImapError.NOT_SUPPORTED("Unknown response status to %s on %s: %s", - cmd.to_string(), to_string(), response.to_string()); - } - } - - // Utility method for listing UIDs on the remote within the supplied range - public async Gee.Set? list_uids_async(MessageSet msg_set, Cancellable? cancellable) - throws Error { - check_open(); - - // Although FETCH could be used, SEARCH is more efficient in returning pure UID results, - // which is all we're interested in here - SearchCriteria criteria = new SearchCriteria(SearchCriterion.message_set(msg_set)); - SearchCommand cmd = new SearchCommand.uid(criteria); - - Gee.Set search_results = new Gee.HashSet(); - yield exec_commands_async( - Geary.iterate(cmd).to_array_list(), - null, - search_results, - cancellable - ); - - return (search_results.size > 0) ? search_results : null; - } - - private Gee.Collection assemble_list_commands(Imap.MessageSet msg_set, - Geary.Email.Field fields, out FetchBodyDataSpecifier? header_specifier, - out FetchBodyDataSpecifier? body_specifier, out FetchBodyDataSpecifier? preview_specifier, - out FetchBodyDataSpecifier? preview_charset_specifier) { - // getting all the fields can require multiple FETCH commands (some servers don't handle - // well putting every required data item into single command), so aggregate FetchCommands - Gee.Collection cmds = new Gee.ArrayList(); - - // if not a UID FETCH, request UIDs for all messages so their EmailIdentifier can be - // created without going back to the database (assuming the messages have already been - // pulled down, not a guarantee); if request is for NONE, that guarantees that the - // EmailIdentifier will be set, and so fetch UIDs (which looks funny but works when - // listing a range for contents: UID FETCH x:y UID) - if (!msg_set.is_uid || fields == Geary.Email.Field.NONE) - cmds.add(new FetchCommand.data_type(msg_set, FetchDataSpecifier.UID)); - - // convert bulk of the "basic" fields into a one or two FETCH commands (some servers have - // exhibited bugs or return NO when too many FETCH data types are combined on a single - // command) - if (fields.requires_any(BASIC_FETCH_FIELDS)) { - Gee.List data_types = new Gee.ArrayList(); - fields_to_fetch_data_types(fields, data_types, out header_specifier); - - // Add all simple data types as one FETCH command - if (data_types.size > 0) - cmds.add(new FetchCommand(msg_set, data_types, null)); - - // Add all body data types as separate FETCH command - if (header_specifier != null) - cmds.add(new FetchCommand.body_data_type(msg_set, header_specifier)); - } else { - header_specifier = null; - } - - // RFC822 BODY is a separate command - if (fields.require(Email.Field.BODY)) { - body_specifier = new FetchBodyDataSpecifier.peek(FetchBodyDataSpecifier.SectionPart.TEXT, - null, -1, -1, null); - - cmds.add(new FetchCommand.body_data_type(msg_set, body_specifier)); - } else { - body_specifier = null; - } - - // PREVIEW obtains the content type and a truncated version of - // the first part of the message, which often leads to poor - // results. It can also be also be synthesised from the - // email's RFC822 message in fetched_data_to_email, if the - // fields needed for reconstructing the RFC822 message are - // present. If so, rely on that and don't also request any - // additional data for the preview here. - if (fields.require(Email.Field.PREVIEW) && - !fields.require(Email.REQUIRED_FOR_MESSAGE)) { - // Get the preview text (the initial MAX_PREVIEW_BYTES of - // the first MIME section - - preview_specifier = new FetchBodyDataSpecifier.peek(FetchBodyDataSpecifier.SectionPart.NONE, - { 1 }, 0, Geary.Email.MAX_PREVIEW_BYTES, null); - cmds.add(new FetchCommand.body_data_type(msg_set, preview_specifier)); - - // Also get the character set to properly decode it - preview_charset_specifier = new FetchBodyDataSpecifier.peek( - FetchBodyDataSpecifier.SectionPart.MIME, { 1 }, -1, -1, null); - cmds.add(new FetchCommand.body_data_type(msg_set, preview_charset_specifier)); - } else { - preview_specifier = null; - preview_charset_specifier = null; - } - - // PROPERTIES and FLAGS are a separate command - if (fields.requires_any(Email.Field.PROPERTIES | Email.Field.FLAGS)) { - Gee.List data_types = new Gee.ArrayList(); - - if (fields.require(Geary.Email.Field.PROPERTIES)) { - data_types.add(FetchDataSpecifier.INTERNALDATE); - data_types.add(FetchDataSpecifier.RFC822_SIZE); - } - - if (fields.require(Geary.Email.Field.FLAGS)) - data_types.add(FetchDataSpecifier.FLAGS); - - cmds.add(new FetchCommand(msg_set, data_types, null)); - } - - return cmds; - } - - // Returns a no-message-id ImapDB.EmailIdentifier with the UID stored in it. - public async Gee.List? list_email_async(MessageSet msg_set, Geary.Email.Field fields, - Cancellable? cancellable) throws Error { - check_open(); - Gee.HashMap fetched = - new Gee.HashMap(); - FetchBodyDataSpecifier? header_specifier = null; - FetchBodyDataSpecifier? body_specifier = null; - FetchBodyDataSpecifier? preview_specifier = null; - FetchBodyDataSpecifier? preview_charset_specifier = null; - for (;;) { - Gee.Collection cmds = assemble_list_commands(msg_set, fields, - out header_specifier, out body_specifier, out preview_specifier, - out preview_charset_specifier); - if (cmds.size == 0) { - throw new ImapError.INVALID("No FETCH commands generate for list request %s %s", - msg_set.to_string(), fields.to_list_string()); - } - - // Commands prepped, do the fetch and accumulate all the responses - try { - yield exec_commands_async(cmds, fetched, null, cancellable); - } catch (Error err) { - if (err is FolderError.RETRY) { - debug("Retryable server failure detected for %s: %s", to_string(), err.message); - - continue; - } - - throw err; - } - - break; - } - - if (fetched.size == 0) - return null; - - // Convert fetched data into Geary.Email objects - // because this could be for a lot of email, do in a background thread - Gee.List email_list = new Gee.ArrayList(); - yield Nonblocking.Concurrent.global.schedule_async(() => { - foreach (SequenceNumber seq_num in fetched.keys) { - FetchedData fetched_data = fetched.get(seq_num); - - // the UID should either have been fetched (if using positional addressing) or should - // have come back with the response (if using UID addressing) - UID? uid = fetched_data.data_map.get(FetchDataSpecifier.UID) as UID; - if (uid == null) { - message("Unable to list message #%s on %s: No UID returned from server", - seq_num.to_string(), to_string()); - - continue; - } - - try { - Geary.Email email = fetched_data_to_email(to_string(), uid, fetched_data, fields, - header_specifier, body_specifier, preview_specifier, preview_charset_specifier); - if (!email.fields.fulfills(fields)) { - message("%s: %s missing=%s fetched=%s", to_string(), email.id.to_string(), - fields.clear(email.fields).to_list_string(), fetched_data.to_string()); - - continue; - } - - email_list.add(email); - } catch (Error err) { - debug("%s: Unable to convert email for %s %s: %s", to_string(), uid.to_string(), - fetched_data.to_string(), err.message); - } - } - }, cancellable); - - return (email_list.size > 0) ? email_list : null; - } - - /** - * Returns the sequence numbers for a set of UIDs. - * - * The `msg_set` parameter must be a set containing UIDs. An error - * is thrown if the sequence numbers cannot be determined. - */ - public async Gee.Map uid_to_position_async(MessageSet msg_set, - Cancellable? cancellable) - throws Error { - check_open(); - - if (!msg_set.is_uid) { - throw new ImapError.NOT_SUPPORTED("Message set must contain UIDs"); - } - - Gee.List cmds = new Gee.ArrayList(); - cmds.add(new FetchCommand.data_type(msg_set, FetchDataSpecifier.UID)); - - Gee.HashMap fetched = - new Gee.HashMap(); - yield exec_commands_async(cmds, fetched, null, cancellable); - - if (fetched.is_empty) { - throw new ImapError.INVALID("Server returned no sequence numbers"); - } - - Gee.Map map = new Gee.HashMap(); - foreach (SequenceNumber seq_num in fetched.keys) { - map.set( - (UID) fetched.get(seq_num).data_map.get(FetchDataSpecifier.UID), - seq_num - ); - } - return map; - } - - public async void remove_email_async(Gee.List msg_sets, Cancellable? cancellable) - throws Error { - check_open(); - - Gee.List flags = new Gee.ArrayList(); - flags.add(MessageFlag.DELETED); - - Gee.List cmds = new Gee.ArrayList(); - - // Build STORE command for all MessageSets, see if all are UIDs so we can use UID EXPUNGE - bool all_uid = true; - foreach (MessageSet msg_set in msg_sets) { - if (!msg_set.is_uid) - all_uid = false; - - cmds.add(new StoreCommand(msg_set, flags, StoreCommand.Option.ADD_FLAGS)); - } - - // TODO: Only use old-school EXPUNGE when closing folder (or rely on CLOSE to do that work - // for us). See: - // http://redmine.yorba.org/issues/7532 - // - // However, current client implementation doesn't properly close INBOX when application - // shuts down, which means deleted messages return at application start. See: - // http://redmine.yorba.org/issues/6865 - if (all_uid && session.capabilities.supports_uidplus()) { - foreach (MessageSet msg_set in msg_sets) - cmds.add(new ExpungeCommand.uid(msg_set)); - } else { - cmds.add(new ExpungeCommand()); - } - - yield exec_commands_async(cmds, null, null, cancellable); - } - - public async void mark_email_async(Gee.List msg_sets, Geary.EmailFlags? flags_to_add, - Geary.EmailFlags? flags_to_remove, Cancellable? cancellable) throws Error { - check_open(); - - Gee.List msg_flags_add = new Gee.ArrayList(); - Gee.List msg_flags_remove = new Gee.ArrayList(); - MessageFlag.from_email_flags(flags_to_add, flags_to_remove, out msg_flags_add, - out msg_flags_remove); - - if (msg_flags_add.size == 0 && msg_flags_remove.size == 0) - return; - - Gee.Collection cmds = new Gee.ArrayList(); - foreach (MessageSet msg_set in msg_sets) { - if (msg_flags_add.size > 0) - cmds.add(new StoreCommand(msg_set, msg_flags_add, StoreCommand.Option.ADD_FLAGS)); - - if (msg_flags_remove.size > 0) - cmds.add(new StoreCommand(msg_set, msg_flags_remove, StoreCommand.Option.REMOVE_FLAGS)); - } - - yield exec_commands_async(cmds, null, null, cancellable); - } - - // Returns a mapping of the source UID to the destination UID. If the MessageSet is not for - // UIDs, then null is returned. If the server doesn't support COPYUID, null is returned. - public async Gee.Map? copy_email_async(MessageSet msg_set, FolderPath destination, - Cancellable? cancellable) throws Error { - check_open(); - - MailboxSpecifier mailbox = this.session.get_mailbox_for_path(destination); - CopyCommand cmd = new CopyCommand(msg_set, mailbox); - - Gee.Map? responses = yield exec_commands_async( - Geary.iterate(cmd).to_array_list(), null, null, cancellable); - - if (!responses.has_key(cmd)) - return null; - - StatusResponse response = responses.get(cmd); - if (response.response_code != null && msg_set.is_uid) { - Gee.List? src_uids = null; - Gee.List? dst_uids = null; - try { - response.response_code.get_copyuid(null, out src_uids, out dst_uids); - } catch (ImapError ierr) { - debug("Unable to retrieve COPYUID UIDs: %s", ierr.message); - } - - if (!Collection.is_empty(src_uids) && !Collection.is_empty(dst_uids)) { - Gee.Map copyuids = new Gee.HashMap(); - int ctr = 0; - for (;;) { - UID? src_uid = (ctr < src_uids.size) ? src_uids[ctr] : null; - UID? dst_uid = (ctr < dst_uids.size) ? dst_uids[ctr] : null; - - if (src_uid != null && dst_uid != null) - copyuids.set(src_uid, dst_uid); - else - break; - - ctr++; - } - - if (copyuids.size > 0) - return copyuids; - } - } - - return null; - } - - public async Gee.SortedSet? search_async(SearchCriteria criteria, Cancellable? cancellable) - throws Error { - check_open(); - - // always perform a UID SEARCH - Gee.Collection cmds = new Gee.ArrayList(); - cmds.add(new SearchCommand.uid(criteria)); - - Gee.Set search_results = new Gee.HashSet(); - yield exec_commands_async(cmds, null, search_results, cancellable); - - Gee.SortedSet tree = null; - if (search_results.size > 0) { - tree = new Gee.TreeSet(); - tree.add_all(search_results); - } - return tree; - } - - // NOTE: If fields are added or removed from this method, BASIC_FETCH_FIELDS *must* be updated - // as well - private void fields_to_fetch_data_types(Geary.Email.Field fields, - Gee.List data_types_list, out FetchBodyDataSpecifier? header_specifier) { - // pack all the needed headers into a single FetchBodyDataType - string[] field_names = new string[0]; - - // The assumption here is that because ENVELOPE is such a common fetch command, the - // server will have optimizations for it, whereas if we called for each header in the - // envelope separately, the server has to chunk harder parsing the RFC822 header ... have - // to add References because IMAP ENVELOPE doesn't return them for some reason (but does - // return Message-ID and In-Reply-To) - if (fields.is_all_set(Geary.Email.Field.ENVELOPE)) { - data_types_list.add(FetchDataSpecifier.ENVELOPE); - field_names += "References"; - - // remove those flags and process any remaining - fields = fields.clear(Geary.Email.Field.ENVELOPE); - } - - foreach (Geary.Email.Field field in Geary.Email.Field.all()) { - switch (fields & field) { - case Geary.Email.Field.DATE: - field_names += "Date"; - break; - - case Geary.Email.Field.ORIGINATORS: - field_names += "From"; - field_names += "Sender"; - field_names += "Reply-To"; - break; - - case Geary.Email.Field.RECEIVERS: - field_names += "To"; - field_names += "Cc"; - field_names += "Bcc"; - break; - - case Geary.Email.Field.REFERENCES: - field_names += "References"; - field_names += "Message-ID"; - field_names += "In-Reply-To"; - break; - - case Geary.Email.Field.SUBJECT: - field_names += "Subject"; - break; - - case Geary.Email.Field.HEADER: - // TODO: If the entire header is being pulled, then no need to pull down partial - // headers; simply get them all and decode what is needed directly - data_types_list.add(FetchDataSpecifier.RFC822_HEADER); - break; - - case Geary.Email.Field.NONE: - case Geary.Email.Field.BODY: - case Geary.Email.Field.PROPERTIES: - case Geary.Email.Field.FLAGS: - case Geary.Email.Field.PREVIEW: - // not set or fetched separately - break; - - default: - assert_not_reached(); - } - } - - // convert field names into single FetchBodyDataType object - if (field_names.length > 0) { - header_specifier = new FetchBodyDataSpecifier.peek( - FetchBodyDataSpecifier.SectionPart.HEADER_FIELDS, null, -1, -1, field_names); - if (imap_header_fields_hack) - header_specifier.omit_request_header_fields_space(); - } else { - header_specifier = null; - } - } - - private static Geary.Email fetched_data_to_email(string folder_name, UID uid, - FetchedData fetched_data, Geary.Email.Field required_fields, - FetchBodyDataSpecifier? header_specifier, FetchBodyDataSpecifier? body_specifier, - FetchBodyDataSpecifier? preview_specifier, FetchBodyDataSpecifier? preview_charset_specifier) throws Error { - // note the use of INVALID_ROWID, as the rowid for this email (if one is present in the - // database) is unknown at this time; this means ImapDB *must* create a new EmailIdentifier - // for this email after create/merge is completed - Geary.Email email = new Geary.Email(new ImapDB.EmailIdentifier.no_message_id(uid)); - - // accumulate these to submit Imap.EmailProperties all at once - InternalDate? internaldate = null; - RFC822.Size? rfc822_size = null; - - // accumulate these to submit References all at once - RFC822.MessageID? message_id = null; - RFC822.MessageIDList? in_reply_to = null; - RFC822.MessageIDList? references = null; - - // loop through all available FetchDataTypes and gather converted data - foreach (FetchDataSpecifier data_type in fetched_data.data_map.keys) { - MessageData? data = fetched_data.data_map.get(data_type); - if (data == null) - continue; - - switch (data_type) { - case FetchDataSpecifier.ENVELOPE: - Envelope envelope = (Envelope) data; - - email.set_send_date(envelope.sent); - email.set_message_subject(envelope.subject); - email.set_originators( - envelope.from, - envelope.sender.equal_to(envelope.from) || envelope.sender.size == 0 ? null : envelope.sender[0], - envelope.reply_to.equal_to(envelope.from) ? null : envelope.reply_to - ); - email.set_receivers(envelope.to, envelope.cc, envelope.bcc); - - // store these to add to References all at once - message_id = envelope.message_id; - in_reply_to = envelope.in_reply_to; - break; - - case FetchDataSpecifier.RFC822_HEADER: - email.set_message_header((RFC822.Header) data); - break; - - case FetchDataSpecifier.RFC822_TEXT: - email.set_message_body((RFC822.Text) data); - break; - - case FetchDataSpecifier.RFC822_SIZE: - rfc822_size = (RFC822.Size) data; - break; - - case FetchDataSpecifier.FLAGS: - email.set_flags(new Imap.EmailFlags((MessageFlags) data)); - break; - - case FetchDataSpecifier.INTERNALDATE: - internaldate = (InternalDate) data; - break; - - default: - // everything else dropped on the floor (not applicable to Geary.Email) - break; - } - } - - // Only set PROPERTIES if all have been found - if (internaldate != null && rfc822_size != null) - email.set_email_properties(new Geary.Imap.EmailProperties(internaldate, rfc822_size)); - - // if the header was requested, convert its fields now - bool has_header_specifier = fetched_data.body_data_map.has_key(header_specifier); - if (header_specifier != null && !has_header_specifier) { - message("[%s] No header specifier \"%s\" found:", folder_name, - header_specifier.to_string()); - foreach (FetchBodyDataSpecifier specifier in fetched_data.body_data_map.keys) - message("[%s] has %s", folder_name, specifier.to_string()); - } else if (header_specifier != null && has_header_specifier) { - RFC822.Header headers = new RFC822.Header( - fetched_data.body_data_map.get(header_specifier)); - - // DATE - if (required_but_not_set(Geary.Email.Field.DATE, required_fields, email)) { - string? value = headers.get_header("Date"); - if (!String.is_empty(value)) - email.set_send_date(new RFC822.Date(value)); - else - email.set_send_date(null); - } - - // ORIGINATORS - if (required_but_not_set(Geary.Email.Field.ORIGINATORS, required_fields, email)) { - RFC822.MailboxAddresses? from = null; - string? value = headers.get_header("From"); - if (!String.is_empty(value)) - from = new RFC822.MailboxAddresses.from_rfc822_string(value); - - RFC822.MailboxAddress? sender = null; - value = headers.get_header("Sender"); - if (!String.is_empty(value)) - sender = new RFC822.MailboxAddress.from_rfc822_string(value); - - RFC822.MailboxAddresses? reply_to = null; - value = headers.get_header("Reply-To"); - if (!String.is_empty(value)) - reply_to = new RFC822.MailboxAddresses.from_rfc822_string(value); - - email.set_originators(from, sender, reply_to); - } - - // RECEIVERS - if (required_but_not_set(Geary.Email.Field.RECEIVERS, required_fields, email)) { - RFC822.MailboxAddresses? to = null; - string? value = headers.get_header("To"); - if (!String.is_empty(value)) - to = new RFC822.MailboxAddresses.from_rfc822_string(value); - - RFC822.MailboxAddresses? cc = null; - value = headers.get_header("Cc"); - if (!String.is_empty(value)) - cc = new RFC822.MailboxAddresses.from_rfc822_string(value); - - RFC822.MailboxAddresses? bcc = null; - value = headers.get_header("Bcc"); - if (!String.is_empty(value)) - bcc = new RFC822.MailboxAddresses.from_rfc822_string(value); - - email.set_receivers(to, cc, bcc); - } - - // REFERENCES - // (Note that it's possible the request used an IMAP ENVELOPE, in which case only the - // References header will be present if REFERENCES were required, which is why - // REFERENCES is set at the bottom of the method, when all information has been gathered - if (message_id == null) { - string? value = headers.get_header("Message-ID"); - if (!String.is_empty(value)) - message_id = new RFC822.MessageID(value); - } - - if (in_reply_to == null) { - string? value = headers.get_header("In-Reply-To"); - if (!String.is_empty(value)) - in_reply_to = new RFC822.MessageIDList.from_rfc822_string(value); - } - - if (references == null) { - string? value = headers.get_header("References"); - if (!String.is_empty(value)) - references = new RFC822.MessageIDList.from_rfc822_string(value); - } - - // SUBJECT - // Unlike DATE, allow for empty subjects - if (required_but_not_set(Geary.Email.Field.SUBJECT, required_fields, email)) { - string? value = headers.get_header("Subject"); - if (value != null) - email.set_message_subject(new RFC822.Subject.decode(value)); - else - email.set_message_subject(null); - } - } - - // It's possible for all these fields to be null even though they were requested from - // the server, so use requested fields for determination - if (required_but_not_set(Geary.Email.Field.REFERENCES, required_fields, email)) - email.set_full_references(message_id, in_reply_to, references); - - // if preview was requested, get it now ... both identifiers - // must be supplied if one is - if (preview_specifier != null || preview_charset_specifier != null) { - assert(preview_specifier != null && preview_charset_specifier != null); - - if (fetched_data.body_data_map.has_key(preview_specifier) - && fetched_data.body_data_map.has_key(preview_charset_specifier)) { - email.set_message_preview(new RFC822.PreviewText.with_header( - fetched_data.body_data_map.get(preview_specifier), - fetched_data.body_data_map.get(preview_charset_specifier))); - } else { - message("[%s] No preview specifiers \"%s\" and \"%s\" found", folder_name, - preview_specifier.to_string(), preview_charset_specifier.to_string()); - foreach (FetchBodyDataSpecifier specifier in fetched_data.body_data_map.keys) - message("[%s] has %s", folder_name, specifier.to_string()); - } - } - - // If body was requested, get it now. We also set the preview - // here from the body if possible since for HTML messages at - // least there's a lot of boilerplate HTML to wade through to - // get some actual preview text, which usually requires more - // than Geary.Email.MAX_PREVIEW_BYTES will allow for - if (body_specifier != null) { - if (fetched_data.body_data_map.has_key(body_specifier)) { - email.set_message_body(new Geary.RFC822.Text( - fetched_data.body_data_map.get(body_specifier))); - - // Try to set the preview - Geary.RFC822.Message? message = null; - try { - message = email.get_message(); - } catch (Error e) { - // Not enough fields to construct the message - } - if (message != null) { - string preview = message.get_preview(); - if (preview.length > Geary.Email.MAX_PREVIEW_BYTES) { - preview = Geary.String.safe_byte_substring( - preview, Geary.Email.MAX_PREVIEW_BYTES - ); - } - email.set_message_preview( - new RFC822.PreviewText.from_string(preview) - ); - } - } else { - message("[%s] No body specifier \"%s\" found", folder_name, - body_specifier.to_string()); - foreach (FetchBodyDataSpecifier specifier in fetched_data.body_data_map.keys) - message("[%s] has %s", folder_name, specifier.to_string()); - } - } - - return email; - } - - // Returns a no-message-id ImapDB.EmailIdentifier with the UID stored in it. - // This method does not take a cancellable; there is currently no way to tell if an email was - // created or not if exec_commands_async() is cancelled during the append. For atomicity's sake, - // callers need to remove the returned email ID if a cancel occurred. - public async Geary.EmailIdentifier? create_email_async(RFC822.Message message, Geary.EmailFlags? flags, - DateTime? date_received) throws Error { - check_open(); - - MessageFlags? msg_flags = null; - if (flags != null) { - Imap.EmailFlags imap_flags = Imap.EmailFlags.from_api_email_flags(flags); - msg_flags = imap_flags.message_flags; - } else { - msg_flags = new MessageFlags(Geary.iterate(MessageFlag.SEEN).to_array_list()); - } - - InternalDate? internaldate = null; - if (date_received != null) - internaldate = new InternalDate.from_date_time(date_received); - - MailboxSpecifier mailbox = this.session.get_mailbox_for_path(this.path); - AppendCommand cmd = new AppendCommand( - mailbox, msg_flags, internaldate, message.get_network_buffer(false) - ); - - Gee.Map responses = yield exec_commands_async( - Geary.iterate(cmd).to_array_list(), null, null, null); - - // Grab the response and parse out the UID, if available. - StatusResponse response = responses.get(cmd); - if (response.status == Status.OK && response.response_code != null && - response.response_code.get_response_code_type().is_value("appenduid")) { - UID new_id = new UID.checked(response.response_code.get_as_string(2).as_int64()); - - return new ImapDB.EmailIdentifier.no_message_id(new_id); - } - - // We didn't get a UID back from the server. - return null; - } - - private static bool required_but_not_set(Geary.Email.Field check, Geary.Email.Field users_fields, Geary.Email email) { - return users_fields.require(check) ? !email.fields.is_all_set(check) : false; - } - - public string to_string() { - return path.to_string(); - } } - diff --git a/src/engine/imap/api/imap-session-object.vala b/src/engine/imap/api/imap-session-object.vala new file mode 100644 index 00000000..8391fde1 --- /dev/null +++ b/src/engine/imap/api/imap-session-object.vala @@ -0,0 +1,102 @@ +/* + * Copyright 2018 Michael Gratton . + * + * This software is licensed under the GNU Lesser General Public License + * (version 2.1 or later). See the COPYING file in this distribution. + */ + +/** + * Base class for IMAP client session objects. + * + * Since a client session can come and go as the server and network + * changes, IMAP client objects need to be sensitive to the state of + * the connection. This abstract class manages access to an IMAP + * client session for objects that use connections to an IMAP server, + * ensuring it is no longer available if the client session is + * disconnected. + * + * This class is ''not'' thread safe. + */ +public abstract class Geary.Imap.SessionObject : BaseObject { + + + /** Determines if this object has a valid session or not. */ + public bool is_valid { get { return this.session != null; } } + + private string id; + private ClientSession? session; + + + /** Fired if the object's connection to the server is lost. */ + public signal void disconnected(ClientSession.DisconnectReason reason); + + + /** + * Constructs a new IMAP object with the given session. + */ + protected SessionObject(string id, ClientSession session) { + this.id = id; + this.session = session; + this.session.disconnected.connect(on_disconnected); + } + + ~SessionObject() { + if (close() != null) { + debug("%s: destroyed without releasing its session".printf(this.id)); + } + } + + /** + * Drops this object's association with its client session. + * + * Calling this method unhooks the object from its session, and + * makes it unavailable for further use. This does //not// + * disconnect the client session from its server. + * + * @return the old IMAP client session, for returning to the pool, + * etc, if any. + */ + public virtual ClientSession? close() { + ClientSession? old_session = this.session; + this.session = null; + + if (old_session != null) { + old_session.disconnected.disconnect(on_disconnected); + } + + return old_session; + } + + /** + * Returns a string representation of this object for debugging. + */ + public virtual string to_string() { + return "%s:%s".printf( + this.id, + this.session != null ? this.session.to_string() : "(session dropped)" + ); + } + + /** + * Obtains IMAP session the server for use by this object. + * + * @throws ImapError.NOT_CONNECTED if the session with the server + * server has been dropped via {@link close}, or because + * the connection was lost. + */ + protected ClientSession claim_session() + throws ImapError { + if (this.session == null) { + throw new ImapError.NOT_CONNECTED("IMAP object has no session"); + } + return this.session; + } + + private void on_disconnected(ClientSession.DisconnectReason reason) { + debug("%s: DISCONNECTED %s", to_string(), reason.to_string()); + + close(); + disconnected(reason); + } + +} diff --git a/src/engine/imap/transport/imap-client-session-manager.vala b/src/engine/imap/transport/imap-client-session-manager.vala index 7c36ab49..14ecec55 100644 --- a/src/engine/imap/transport/imap-client-session-manager.vala +++ b/src/engine/imap/transport/imap-client-session-manager.vala @@ -78,8 +78,9 @@ public class Geary.Imap.ClientSessionManager : BaseObject { */ public bool discard_returned_sessions = false; - private AccountInformation account_information; + private string id; private Endpoint endpoint; + private Credentials credentials; private Nonblocking.Mutex sessions_mutex = new Nonblocking.Mutex(); private Gee.Set all_sessions = @@ -110,16 +111,17 @@ public class Geary.Imap.ClientSessionManager : BaseObject { public signal void login_failed(StatusResponse? response); - public ClientSessionManager(AccountInformation account_information) { - this.account_information = account_information; + public ClientSessionManager(string id, + Endpoint imap_endpoint, + Credentials imap_credentials) { + this.id = "%s:%s".printf(id, imap_endpoint.to_string()); - // NOTE: This works because AccountInformation guarantees the IMAP endpoint not to change - // for the lifetime of the AccountInformation object; if this ever changes, will need to - // refactor for that - this.endpoint = account_information.get_imap_endpoint(); + this.endpoint = imap_endpoint; this.endpoint.notify[Endpoint.PROP_TRUST_UNTRUSTED_HOST].connect(on_imap_trust_untrusted_host); this.endpoint.untrusted_host.connect(on_imap_untrusted_host); + this.credentials = imap_credentials; + this.pool_start = new TimeoutManager.seconds( POOL_START_TIMEOUT_SEC, () => { this.check_pool.begin(); } @@ -133,7 +135,7 @@ public class Geary.Imap.ClientSessionManager : BaseObject { ~ClientSessionManager() { if (is_open) - warning("[%s] Destroying opened ClientSessionManager", to_string()); + warning("[%s] Destroying opened ClientSessionManager", this.id); this.endpoint.untrusted_host.disconnect(on_imap_untrusted_host); this.endpoint.notify[Endpoint.PROP_TRUST_UNTRUSTED_HOST].disconnect(on_imap_trust_untrusted_host); @@ -176,7 +178,7 @@ public class Geary.Imap.ClientSessionManager : BaseObject { // for now int attempts = 0; while (this.all_sessions.size > 0) { - debug("[%s] Waiting for client sessions to disconnect...", to_string()); + debug("[%s] Waiting for client sessions to disconnect...", this.id); Timeout.add(250, close_async.callback); yield; @@ -192,8 +194,9 @@ public class Geary.Imap.ClientSessionManager : BaseObject { * This will reset the manager's authentication state and if open, * attempt to open a connection to the server. */ - public void credentials_updated() { + public void credentials_updated(Credentials new_creds) { this.authentication_failed = false; + this.credentials = new_creds; if (this.is_open) { this.check_pool.begin(); } @@ -220,7 +223,7 @@ public class Geary.Imap.ClientSessionManager : BaseObject { throws Error { check_open(); debug("[%s] Claiming session from %d of %d free", - to_string(), this.free_queue.size, this.all_sessions.size); + this.id, this.free_queue.size, this.all_sessions.size); if (this.authentication_failed) throw new ImapError.UNAUTHENTICATED("Invalid ClientSessionManager credentials"); @@ -253,13 +256,13 @@ public class Geary.Imap.ClientSessionManager : BaseObject { return claimed; } - public async void release_session_async(ClientSession session, Cancellable? cancellable) + public async void release_session_async(ClientSession session) throws Error { // Don't check_open(), it's valid for this to be called when // is_open is false, that happens during mop-up debug("[%s] Returning session with %d of %d free", - to_string(), this.free_queue.size, this.all_sessions.size); + this.id, this.free_queue.size, this.all_sessions.size); if (!this.is_open || this.discard_returned_sessions) { yield force_disconnect(session); @@ -271,17 +274,12 @@ public class Geary.Imap.ClientSessionManager : BaseObject { // adding it back to the pool if (proto == ClientSession.ProtocolState.SELECTED || proto == ClientSession.ProtocolState.SELECTING) { - debug("[%s] Closing %s for released session %s", - to_string(), - mailbox != null ? mailbox.to_string() : "(unknown)", - session.to_string()); - // always close mailbox to return to authorized state try { - yield session.close_mailbox_async(cancellable); + yield session.close_mailbox_async(pool_cancellable); } catch (ImapError imap_error) { debug("[%s] Error attempting to close released session %s: %s", - to_string(), session.to_string(), imap_error.message); + this.id, session.to_string(), imap_error.message); free = false; } @@ -294,13 +292,19 @@ public class Geary.Imap.ClientSessionManager : BaseObject { } if (free) { - debug("[%s] Unreserving session %s", - to_string(), session.to_string()); + debug("[%s] Unreserving session %s", this.id, session.to_string()); this.free_queue.send(session); } } } + /** + * Returns a string representation of this object for debugging. + */ + public string to_string() { + return this.id; + } + private void check_open() throws Error { if (!is_open) throw new EngineError.OPEN_REQUIRED("ClientSessionManager is not open"); @@ -308,7 +312,7 @@ public class Geary.Imap.ClientSessionManager : BaseObject { private async void check_pool() { debug("[%s] Checking session pool with %d of %d free", - to_string(), this.free_queue.size, this.all_sessions.size); + this.id, this.free_queue.size, this.all_sessions.size); while (this.is_open && !this.authentication_failed && @@ -326,8 +330,7 @@ public class Geary.Imap.ClientSessionManager : BaseObject { this.free_queue.send(free); } catch (Error err) { debug("[%s] Error adding free session pool: %s", - to_string(), - err.message); + this.id, err.message); break; } @@ -361,7 +364,7 @@ public class Geary.Imap.ClientSessionManager : BaseObject { yield remove_session_async(target); } catch (Error err) { debug("[%s] Error removing unconnected session: %s", - to_string(), err.message); + this.id, err.message); } break; @@ -374,6 +377,7 @@ public class Geary.Imap.ClientSessionManager : BaseObject { } private async ClientSession create_new_authorized_session(Cancellable? cancellable) throws Error { + debug("[%s] Opening new session", this.id); ClientSession new_session = new ClientSession(endpoint); // Listen for auth failures early so the client is notified if @@ -390,7 +394,7 @@ public class Geary.Imap.ClientSessionManager : BaseObject { } try { - yield new_session.initiate_session_async(account_information.imap_credentials, cancellable); + yield new_session.initiate_session_async(this.credentials, cancellable); } catch (Error err) { debug("[%s] Initiate session failure: %s", new_session.to_string(), err.message); @@ -417,6 +421,7 @@ public class Geary.Imap.ClientSessionManager : BaseObject { // We now have a good connection, so signal us as ready if not // already done so. if (!this.is_ready) { + debug("[%s] Became ready", this.id); this.is_ready = true; ready(); } @@ -428,7 +433,7 @@ public class Geary.Imap.ClientSessionManager : BaseObject { private async void force_disconnect_all() throws Error { debug("[%s] Dropping and disconnecting %d sessions", - to_string(), this.all_sessions.size); + this.id, this.all_sessions.size); // Take a copy and work off that while scheduling disconnects, // since as they disconnect they'll remove themselves from the @@ -447,12 +452,12 @@ public class Geary.Imap.ClientSessionManager : BaseObject { } private async void force_disconnect(ClientSession session) { - debug("[%s] Dropping session %s", to_string(), session.to_string()); + debug("[%s] Dropping session %s", this.id, session.to_string()); try { yield remove_session_async(session); } catch (Error err) { - debug("[%s] Error removing session: %s", to_string(), err.message); + debug("[%s] Error removing session: %s", this.id, err.message); } // Don't wait for this to finish because we don't want to @@ -485,8 +490,7 @@ public class Geary.Imap.ClientSessionManager : BaseObject { this.remove_session_async.end(res); } catch (Error err) { debug("[%s] Error removing disconnected session: %s", - to_string(), - err.message); + this.id, err.message); } } ); @@ -536,13 +540,4 @@ public class Geary.Imap.ClientSessionManager : BaseObject { connection_failed(error); } - /** - * Use only for debugging and logging. - */ - public string to_string() { - return "%s:%s".printf( - this.account_information.id, - endpoint.to_string() - ); - } } diff --git a/src/engine/meson.build b/src/engine/meson.build index 5ce7094c..37fead0e 100644 --- a/src/engine/meson.build +++ b/src/engine/meson.build @@ -83,12 +83,14 @@ geary_engine_vala_sources = files( 'imap/imap.vala', 'imap/imap-error.vala', - 'imap/api/imap-account.vala', + 'imap/api/imap-account-session.vala', 'imap/api/imap-email-flags.vala', 'imap/api/imap-email-properties.vala', - 'imap/api/imap-folder-properties.vala', 'imap/api/imap-folder.vala', + 'imap/api/imap-folder-properties.vala', 'imap/api/imap-folder-root.vala', + 'imap/api/imap-folder-session.vala', + 'imap/api/imap-session-object.vala', 'imap/command/imap-append-command.vala', 'imap/command/imap-capability-command.vala', 'imap/command/imap-close-command.vala', diff --git a/test/engine/api/geary-folder-test.vala b/test/engine/api/geary-folder-test.vala index 718e4915..e208e424 100644 --- a/test/engine/api/geary-folder-test.vala +++ b/test/engine/api/geary-folder-test.vala @@ -56,7 +56,7 @@ public class Geary.MockFolder : Folder { throw new EngineError.UNSUPPORTED("Mock method"); } - public override async void wait_for_open_async(Cancellable? cancellable = null) + public override async void wait_for_remote_async(Cancellable? cancellable = null) throws Error { throw new EngineError.UNSUPPORTED("Mock method"); }