diff --git a/sql/Create.sql b/sql/Create.sql index 33b2bfad..dbb8ac57 100644 --- a/sql/Create.sql +++ b/sql/Create.sql @@ -39,7 +39,9 @@ CREATE TABLE MessageTable ( header TEXT, - body TEXT + body TEXT, + + preview TEXT ); CREATE INDEX MessageTableMessageIDIndex ON MessageTable(message_id); diff --git a/src/client/ui/main-window.vala b/src/client/ui/main-window.vala index b3358dd0..eefc159f 100644 --- a/src/client/ui/main-window.vala +++ b/src/client/ui/main-window.vala @@ -8,6 +8,58 @@ public class MainWindow : Gtk.Window { private const int MESSAGE_LIST_WIDTH = 250; private const int FETCH_EMAIL_CHUNK_COUNT = 50; + private class FetchPreviewOperation : Geary.NonblockingBatchOperation { + public MainWindow owner; + public Geary.Folder folder; + public Geary.EmailIdentifier email_id; + public int index; + + public FetchPreviewOperation(MainWindow owner, Geary.Folder folder, + Geary.EmailIdentifier email_id, int index) { + this.owner = owner; + this.folder = folder; + this.email_id = email_id; + this.index = index; + } + + public override async Object? execute(Cancellable? cancellable) throws Error { + Geary.Email? preview = yield folder.fetch_email_async(email_id, + MessageListStore.WITH_PREVIEW_FIELDS, cancellable); + + owner.message_list_store.set_preview_at_index(index, preview); + + return null; + } + } + + private class ListFoldersOperation : Geary.NonblockingBatchOperation { + public Geary.Account account; + public Geary.FolderPath path; + + public ListFoldersOperation(Geary.Account account, Geary.FolderPath path) { + this.account = account; + this.path = path; + } + + public override async Object? execute(Cancellable? cancellable) throws Error { + return yield account.list_folders_async(path, cancellable); + } + } + + private class FetchSpecialFolderOperation : Geary.NonblockingBatchOperation { + public Geary.Account account; + public Geary.SpecialFolder special_folder; + + public FetchSpecialFolderOperation(Geary.Account account, Geary.SpecialFolder special_folder) { + this.account = account; + this.special_folder = special_folder; + } + + public override async Object? execute(Cancellable? cancellable) throws Error { + return yield account.fetch_folder_async(special_folder.path); + } + } + private MainToolbar main_toolbar; private MessageListStore message_list_store = new MessageListStore(); private MessageListView message_list_view; @@ -60,9 +112,24 @@ public class MainWindow : Gtk.Window { // add all the special folders, which are assumed to always exist Geary.SpecialFolderMap? special_folders = account.get_special_folder_map(); if (special_folders != null) { - foreach (Geary.SpecialFolder special_folder in special_folders.get_all()) { - Geary.Folder folder = yield account.fetch_folder_async(special_folder.path); - folder_list_store.add_special_folder(special_folder, folder); + Geary.NonblockingBatch batch = new Geary.NonblockingBatch(); + foreach (Geary.SpecialFolder special_folder in special_folders.get_all()) + batch.add(new FetchSpecialFolderOperation(account, special_folder)); + + debug("Listing special folders"); + yield batch.execute_all(); + debug("Completed list of special folders"); + + foreach (int id in batch.get_ids()) { + FetchSpecialFolderOperation op = (FetchSpecialFolderOperation) batch.get_operation(id); + try { + Geary.Folder folder = (Geary.Folder) batch.get_result(id); + folder_list_store.add_special_folder(op.special_folder, folder); + } catch (Error inner_error) { + message("Unable to fetch special folder %s: %s", op.special_folder.path.to_string(), + inner_error.message); + } + } // If inbox is specified, select that @@ -78,7 +145,7 @@ public class MainWindow : Gtk.Window { else debug("no folders"); } catch (Error err) { - warning("%s", err.message); + message("%s", err.message); } } @@ -268,18 +335,21 @@ public class MainWindow : Gtk.Window { } private async void do_fetch_previews(Cancellable? cancellable) throws Error { + Geary.NonblockingBatch batch = new Geary.NonblockingBatch(); + int count = message_list_store.get_count(); for (int ctr = 0; ctr < count; ctr++) { Geary.Email? email = message_list_store.get_newest_message_at_index(ctr); - if (email == null) - continue; - - Geary.Email? body = yield current_folder.fetch_email_async(email.id, - Geary.Email.Field.HEADER | Geary.Email.Field.BODY | Geary.Email.Field.ENVELOPE | - Geary.Email.Field.PROPERTIES, cancellable); - message_list_store.set_preview_at_index(ctr, body); + if (email != null) + batch.add(new FetchPreviewOperation(this, current_folder, email.id, ctr)); } + debug("Fetching %d previews", count); + yield batch.execute_all(cancellable); + debug("Completed fetching %d previews", count); + + batch.throw_first_exception(); + // with all the previews fetched, now go back and do a full list (if required) if (second_list_pass_required) { second_list_pass_required = false; @@ -358,14 +428,28 @@ public class MainWindow : Gtk.Window { } private async void search_folders_for_children(Gee.Collection folders) { + Geary.NonblockingBatch batch = new Geary.NonblockingBatch(); + foreach (Geary.Folder folder in folders) + batch.add(new ListFoldersOperation(account, folder.get_path())); + + debug("Listing folder children"); + try { + yield batch.execute_all(); + } catch (Error err) { + debug("Unable to execute batch: %s", err.message); + + return; + } + debug("Completed listing folder children"); + Gee.ArrayList accumulator = new Gee.ArrayList(); - foreach (Geary.Folder folder in folders) { + foreach (int id in batch.get_ids()) { + ListFoldersOperation op = (ListFoldersOperation) batch.get_operation(id); try { - Gee.Collection children = yield account.list_folders_async( - folder.get_path(), null); + Gee.Collection children = (Gee.Collection) batch.get_result(id); accumulator.add_all(children); - } catch (Error err) { - debug("Unable to list children of %s: %s", folder.to_string(), err.message); + } catch (Error err2) { + debug("Unable to list children of %s: %s", op.path.to_string(), err2.message); } } diff --git a/src/client/ui/message-list-cell-renderer.vala b/src/client/ui/message-list-cell-renderer.vala index cd503891..7702d215 100644 --- a/src/client/ui/message-list-cell-renderer.vala +++ b/src/client/ui/message-list-cell-renderer.vala @@ -42,21 +42,14 @@ public class FormattedMessageData : Object { public FormattedMessageData.from_email(Geary.Email email, int num_emails) { assert(email.fields.fulfills(MessageListStore.REQUIRED_FIELDS)); - StringBuilder builder = new StringBuilder(); - if (email.fields.fulfills(Geary.Email.Field.BODY)) { - try { - Geary.Memory.AbstractBuffer buffer = email.get_message(). - get_first_mime_part_of_content_type("text/plain"); - builder.append(buffer.to_utf8()); - } catch (Error e) { - debug("Error displaying message body: %s".printf(e.message)); - } - } + string preview = ""; + if (email.fields.fulfills(Geary.Email.Field.PREVIEW) && email.preview != null) + preview = email.preview.buffer.to_utf8(); string from = (email.from != null && email.from.size > 0) ? email.from[0].get_short_address() : ""; this(email.properties.is_unread(), Date.pretty_print(email.date.value), - from, email.subject.value, Geary.String.reduce_whitespace(builder.str), num_emails); + from, email.subject.value, Geary.String.reduce_whitespace(preview), num_emails); this.email = email; } diff --git a/src/client/ui/message-list-store.vala b/src/client/ui/message-list-store.vala index 6557e22d..8ed0d939 100644 --- a/src/client/ui/message-list-store.vala +++ b/src/client/ui/message-list-store.vala @@ -5,10 +5,12 @@ */ public class MessageListStore : Gtk.TreeStore { - public const Geary.Email.Field REQUIRED_FIELDS = Geary.Email.Field.ENVELOPE | Geary.Email.Field.PROPERTIES; + public const Geary.Email.Field WITH_PREVIEW_FIELDS = + Geary.Email.Field.ENVELOPE | Geary.Email.Field.PROPERTIES | Geary.Email.Field.PREVIEW; + public enum Column { MESSAGE_DATA, MESSAGE_OBJECT, diff --git a/src/console/main.vala b/src/console/main.vala index a543543e..bc563db3 100644 --- a/src/console/main.vala +++ b/src/console/main.vala @@ -94,6 +94,7 @@ class ImapConsole : Gtk.Window { "gmail", "keepalive", "status", + "preview", "close" }; @@ -196,6 +197,10 @@ class ImapConsole : Gtk.Window { folder_status(cmd, args); break; + case "preview": + preview(cmd, args); + break; + default: status("Unknown command \"%s\"".printf(cmd)); break; @@ -407,7 +412,7 @@ class ImapConsole : Gtk.Window { status("Fetching fields %s".printf(args[0])); Geary.Imap.FetchBodyDataType fields = new Geary.Imap.FetchBodyDataType( - Geary.Imap.FetchBodyDataType.SectionPart.HEADER_FIELDS, args[1:args.length]); + Geary.Imap.FetchBodyDataType.SectionPart.HEADER_FIELDS, null, -1, -1, args[1:args.length]); Gee.List list = new Gee.ArrayList(); list.add(fields); @@ -463,6 +468,31 @@ class ImapConsole : Gtk.Window { } } + private void preview(string cmd, string[] args) throws Error { + check_min_connected(cmd, args, 1, ""); + + status("Preview %s".printf(args[0])); + + Geary.Imap.FetchBodyDataType preview_data_type = new Geary.Imap.FetchBodyDataType.peek( + Geary.Imap.FetchBodyDataType.SectionPart.NONE, { 1 }, 0, Geary.Email.MAX_PREVIEW_BYTES, + null); + + Gee.ArrayList list = new Gee.ArrayList(); + list.add(preview_data_type); + + cx.send_async.begin(new Geary.Imap.FetchCommand( + new Geary.Imap.MessageSet.custom(args[0]), null, list), null, on_preview_completed); + } + + private void on_preview_completed(Object? source, AsyncResult result) { + try { + cx.send_async.end(result); + status("Preview fetched"); + } catch (Error err) { + exception(err); + } + } + private void quit(string cmd, string[] args) throws Error { Gtk.main_quit(); } diff --git a/src/engine/api/geary-email.vala b/src/engine/api/geary-email.vala index 0a568047..fe0f7d90 100644 --- a/src/engine/api/geary-email.vala +++ b/src/engine/api/geary-email.vala @@ -5,6 +5,10 @@ */ public class Geary.Email : Object { + // This value is not persisted, but it does represent the expected max size of the preview + // when returned. + public const int MAX_PREVIEW_BYTES = 128; + // THESE VALUES ARE PERSISTED. Change them only if you know what you're doing. public enum Field { NONE = 0, @@ -16,6 +20,8 @@ public class Geary.Email : Object { HEADER = 1 << 5, BODY = 1 << 6, PROPERTIES = 1 << 7, + PREVIEW = 1 << 8, + ENVELOPE = DATE | ORIGINATORS | RECEIVERS | REFERENCES | SUBJECT, ALL = 0xFFFFFFFF; @@ -28,7 +34,8 @@ public class Geary.Email : Object { SUBJECT, HEADER, BODY, - PROPERTIES + PROPERTIES, + PREVIEW }; } @@ -108,6 +115,9 @@ public class Geary.Email : Object { // PROPERTIES public Geary.EmailProperties? properties { get; private set; default = null; } + // PREVIEW + public RFC822.Text? preview { get; private set; default = null; } + public Geary.Email.Field fields { get; private set; default = Field.NONE; } private Geary.RFC822.Message? message = null; @@ -188,6 +198,12 @@ public class Geary.Email : Object { fields |= Field.PROPERTIES; } + public void set_message_preview(Geary.RFC822.Text preview) { + this.preview = preview; + + fields |= Field.PREVIEW; + } + /** * This method requires Geary.Email.Field.HEADER and Geary.Email.Field.BODY be present. * If not, EngineError.INCOMPLETE_MESSAGE is thrown. diff --git a/src/engine/api/geary-engine-error.vala b/src/engine/api/geary-engine-error.vala index fd5ea968..6a2c610a 100644 --- a/src/engine/api/geary-engine-error.vala +++ b/src/engine/api/geary-engine-error.vala @@ -11,6 +11,7 @@ public errordomain Geary.EngineError { NOT_FOUND, READONLY, BAD_PARAMETERS, + BAD_RESPONSE, INCOMPLETE_MESSAGE, SERVER_UNAVAILABLE, CLOSED diff --git a/src/engine/imap/api/imap-account.vala b/src/engine/imap/api/imap-account.vala index 16969d10..f8fc5e08 100644 --- a/src/engine/imap/api/imap-account.vala +++ b/src/engine/imap/api/imap-account.vala @@ -10,6 +10,23 @@ private class Geary.Imap.Account : Geary.AbstractAccount, Geary.RemoteAccount { public const string INBOX_NAME = "INBOX"; public const string ASSUMED_SEPARATOR = "/"; + private class StatusOperation : Geary.NonblockingBatchOperation { + public ClientSessionManager session_mgr; + public MailboxInformation mbox; + public Geary.FolderPath path; + + public StatusOperation(ClientSessionManager session_mgr, MailboxInformation mbox, + Geary.FolderPath path) { + this.session_mgr = session_mgr; + this.mbox = mbox; + this.path = path; + } + + public override async Object? execute(Cancellable? cancellable) throws Error { + return yield session_mgr.status_async(path.get_fullpath(), StatusDataType.all(), cancellable); + } + } + private Geary.Credentials cred; private ClientSessionManager session_mgr; private Geary.Smtp.ClientSession smtp; @@ -47,6 +64,8 @@ private class Geary.Imap.Account : Geary.AbstractAccount, Geary.RemoteAccount { } Gee.Collection folders = new Gee.ArrayList(); + + Geary.NonblockingBatch batch = new Geary.NonblockingBatch(); foreach (MailboxInformation mbox in mboxes) { Geary.FolderPath path = process_path(processed, mbox.name, mbox.delim); @@ -55,17 +74,22 @@ private class Geary.Imap.Account : Geary.AbstractAccount, Geary.RemoteAccount { if (processed == null) delims.set(path.get_root().basename, mbox.delim); - StatusResults? status = null; - if (!mbox.attrs.contains(MailboxAttribute.NO_SELECT)) { - try { - status = yield session_mgr.status_async(path.get_fullpath(), - StatusDataType.all(), cancellable); - } catch (Error status_err) { - message("Unable to fetch status for %s: %s", path.to_string(), status_err.message); - } + if (!mbox.attrs.contains(MailboxAttribute.NO_SELECT)) + batch.add(new StatusOperation(session_mgr, mbox, path)); + else + folders.add(new Geary.Imap.Folder(session_mgr, path, null, mbox)); + } + + yield batch.execute_all(cancellable); + + foreach (int id in batch.get_ids()) { + StatusOperation op = (StatusOperation) batch.get_operation(id); + try { + folders.add(new Geary.Imap.Folder(session_mgr, op.path, + (StatusResults?) batch.get_result(id), op.mbox)); + } catch (Error status_err) { + message("Unable to fetch status for %s: %s", op.path.to_string(), status_err.message); } - - folders.add(new Geary.Imap.Folder(session_mgr, path, status, mbox)); } return folders; diff --git a/src/engine/imap/api/imap-folder.vala b/src/engine/imap/api/imap-folder.vala index 73651af6..6068910f 100644 --- a/src/engine/imap/api/imap-folder.vala +++ b/src/engine/imap/api/imap-folder.vala @@ -120,7 +120,7 @@ private class Geary.Imap.Folder : Geary.AbstractFolder, Geary.RemoteFolder { normalize_span_specifiers(ref low, ref count, mailbox.exists); - return yield mailbox.list_set_async(this, new MessageSet.range(low, count), fields, cancellable); + return yield mailbox.list_set_async(new MessageSet.range(low, count), fields, cancellable); } public override async Gee.List? list_email_sparse_async(int[] by_position, @@ -129,7 +129,7 @@ private class Geary.Imap.Folder : Geary.AbstractFolder, Geary.RemoteFolder { if (mailbox == null) throw new EngineError.OPEN_REQUIRED("%s not opened", to_string()); - return yield mailbox.list_set_async(this, new MessageSet.sparse(by_position), fields, cancellable); + return yield mailbox.list_set_async(new MessageSet.sparse(by_position), fields, cancellable); } public override async Gee.List? list_email_by_id_async(Geary.EmailIdentifier email_id, @@ -160,7 +160,7 @@ private class Geary.Imap.Folder : Geary.AbstractFolder, Geary.RemoteFolder { msg_set = new MessageSet.uid(uid); } - return yield mailbox.list_set_async(this, msg_set, fields, cancellable); + return yield mailbox.list_set_async(msg_set, fields, cancellable); } public override async Geary.Email fetch_email_async(Geary.EmailIdentifier id, @@ -168,9 +168,20 @@ private class Geary.Imap.Folder : Geary.AbstractFolder, Geary.RemoteFolder { if (mailbox == null) throw new EngineError.OPEN_REQUIRED("%s not opened", to_string()); - // TODO: If position out of range, throw EngineError.NOT_FOUND + Gee.List? list = yield mailbox.list_set_async( + new MessageSet.uid(((Imap.EmailIdentifier) id).uid), fields, cancellable); - return yield mailbox.fetch_async(this, ((Imap.EmailIdentifier) id).uid, fields, cancellable); + if (list == null || list.size == 0) { + throw new EngineError.NOT_FOUND("Unable to fetch email %s from %s", id.to_string(), + to_string()); + } + + if (list.size != 1) { + throw new EngineError.BAD_RESPONSE("Too many responses (%d) from %s when fetching %s", + list.size, to_string(), id.to_string()); + } + + return list[0]; } public override async void remove_email_async(Geary.EmailIdentifier email_id, Cancellable? cancellable = null) diff --git a/src/engine/imap/command/imap-command-response.vala b/src/engine/imap/command/imap-command-response.vala index def993f3..5aaf32d4 100644 --- a/src/engine/imap/command/imap-command-response.vala +++ b/src/engine/imap/command/imap-command-response.vala @@ -4,7 +4,7 @@ * (version 2.1 or later). See the COPYING file in this distribution. */ -public class Geary.Imap.CommandResponse { +public class Geary.Imap.CommandResponse : Object { public Gee.List server_data { get; private set; } public StatusResponse? status_response { get; private set; } diff --git a/src/engine/imap/decoders/imap-command-results.vala b/src/engine/imap/decoders/imap-command-results.vala index eab56314..e7d21e26 100644 --- a/src/engine/imap/decoders/imap-command-results.vala +++ b/src/engine/imap/decoders/imap-command-results.vala @@ -4,7 +4,7 @@ * (version 2.1 or later). See the COPYING file in this distribution. */ -public abstract class Geary.Imap.CommandResults { +public abstract class Geary.Imap.CommandResults : Object { public StatusResponse status_response { get; private set; } public CommandResults(StatusResponse status_response) { diff --git a/src/engine/imap/decoders/imap-fetch-results.vala b/src/engine/imap/decoders/imap-fetch-results.vala index 614fe009..12b9f26c 100644 --- a/src/engine/imap/decoders/imap-fetch-results.vala +++ b/src/engine/imap/decoders/imap-fetch-results.vala @@ -85,11 +85,11 @@ public class Geary.Imap.FetchResults : Geary.Imap.CommandResults { return map.keys; } - private void set_data(FetchDataType data_item, MessageData primitive) { + private new void set_data(FetchDataType data_item, MessageData primitive) { map.set(data_item, primitive); } - public MessageData? get_data(FetchDataType data_item) { + public new MessageData? get_data(FetchDataType data_item) { return map.get(data_item); } diff --git a/src/engine/imap/message/imap-fetch-body-data-type.vala b/src/engine/imap/message/imap-fetch-body-data-type.vala index 0dcb1c8a..a1bb1a0e 100644 --- a/src/engine/imap/message/imap-fetch-body-data-type.vala +++ b/src/engine/imap/message/imap-fetch-body-data-type.vala @@ -44,25 +44,47 @@ public class Geary.Imap.FetchBodyDataType { } private SectionPart section_part; + private int[]? part_number; + private int partial_start; + private int partial_count; private string[]? field_names; private bool is_peek; /** + * See RFC-3501 6.4.5 for some light beach reading on how the FETCH body data specifier is formed. + * + * A fully-qualified specifier looks something like this: + * + * BODY[part_number.section_part] + * + * or, when headers are specified: + * + * BODY[part_number.section_part (header_fields)] + * + * Note that Gmail apparently doesn't like BODY[1.TEXT] and instead must be specified with + * BODY[1]. + * + * Set part_number to null to ignore. Set partial_start less than zero to ignore. + * partial_count must be greater than zero if partial_start is greater than zero. + * * field_names are required for SectionPart.HEADER_FIELDS and SectionPart.HEADER_FIELDS_NOT * and must be null for all other SectionParts. */ - public FetchBodyDataType(SectionPart section_part, string[]? field_names) { - init(section_part, field_names, false); + public FetchBodyDataType(SectionPart section_part, int[]? part_number, int partial_start, + int partial_count, string[]? field_names) { + init(section_part, part_number, partial_start, partial_count, field_names, false); } /** * Like FetchBodyDataType, but the /seen flag will not be set when used on a message. */ - public FetchBodyDataType.peek(SectionPart section_part, string[]? field_names) { - init(section_part, field_names, true); + public FetchBodyDataType.peek(SectionPart section_part, int[]? part_number, int partial_start, + int partial_count, string[]? field_names) { + init(section_part, part_number, partial_start, partial_count, field_names, true); } - private void init(SectionPart section_part, string[]? field_names, bool is_peek) { + private void init(SectionPart section_part, int[]? part_number, int partial_start, int partial_count, + string[]? field_names, bool is_peek) { switch (section_part) { case SectionPart.HEADER_FIELDS: case SectionPart.HEADER_FIELDS_NOT: @@ -74,7 +96,13 @@ public class Geary.Imap.FetchBodyDataType { break; } + if (partial_start >= 0) + assert(partial_count > 0); + this.section_part = section_part; + this.part_number = part_number; + this.partial_start = partial_start; + this.partial_count = partial_count; this.field_names = field_names; this.is_peek = is_peek; } @@ -89,6 +117,25 @@ public class Geary.Imap.FetchBodyDataType { return new UnquotedStringParameter(serialize()); } + private string serialize_part_number() { + if (part_number == null || part_number.length == 0) + return ""; + + StringBuilder builder = new StringBuilder(); + foreach (int part in part_number) { + if (builder.len > 0) + builder.append_c('.'); + + builder.append_printf("%d", part); + } + + // if there's a SectionPart that follows, append a period as a separator + if (section_part != SectionPart.NONE) + builder.append_c('.'); + + return builder.str; + } + private string serialize_field_names() { if (field_names == null || field_names.length == 0) return ""; @@ -105,6 +152,10 @@ public class Geary.Imap.FetchBodyDataType { return builder.str; } + private string serialize_partial() { + return (partial_start < 0) ? "" : "<%d.%d>".printf(partial_start, partial_count); + } + public static bool is_fetch_body(StringParameter items) { string strd = items.value.down(); @@ -112,8 +163,11 @@ public class Geary.Imap.FetchBodyDataType { } public string to_string() { - return (!is_peek ? "body[%s%s]" : "body.peek[%s%s]").printf(section_part.serialize(), - serialize_field_names()); + return (!is_peek ? "body[%s%s%s]%s" : "body.peek[%s%s%s]%s").printf( + serialize_part_number(), + section_part.serialize(), + serialize_field_names(), + serialize_partial()); } } diff --git a/src/engine/imap/message/imap-fetch-data-type.vala b/src/engine/imap/message/imap-fetch-data-type.vala index d610fcee..d06c0274 100644 --- a/src/engine/imap/message/imap-fetch-data-type.vala +++ b/src/engine/imap/message/imap-fetch-data-type.vala @@ -4,7 +4,6 @@ * (version 2.1 or later). See the COPYING file in this distribution. */ -// TODO: Support body[section] and body.peek[section] forms public enum Geary.Imap.FetchDataType { UID, FLAGS, diff --git a/src/engine/imap/message/imap-parameter.vala b/src/engine/imap/message/imap-parameter.vala index bf4aba5a..9016fa6b 100644 --- a/src/engine/imap/message/imap-parameter.vala +++ b/src/engine/imap/message/imap-parameter.vala @@ -73,6 +73,11 @@ public class Geary.Imap.StringParameter : Geary.Imap.Parameter { return long.parse(value).clamp(clamp_min, clamp_max); } + // TODO: This does not check that the value is a properly-formed int64. + public int64 as_int64(int64 clamp_min = int64.MIN, int64 clamp_max = int64.MAX) throws ImapError { + return int64.parse(value).clamp(clamp_min, clamp_max); + } + public override string to_string() { return value; } diff --git a/src/engine/imap/transport/imap-client-session-manager.vala b/src/engine/imap/transport/imap-client-session-manager.vala index 1b4bd65e..f45a6157 100644 --- a/src/engine/imap/transport/imap-client-session-manager.vala +++ b/src/engine/imap/transport/imap-client-session-manager.vala @@ -5,8 +5,8 @@ */ public class Geary.Imap.ClientSessionManager { - public const int MIN_POOL_SIZE = 2; - public const int SELECTED_KEEPALIVE_SEC = 5; + private const int MIN_POOL_SIZE = 2; + private const int SELECTED_KEEPALIVE_SEC = 60; private Endpoint endpoint; private Credentials credentials; diff --git a/src/engine/imap/transport/imap-mailbox.vala b/src/engine/imap/transport/imap-mailbox.vala index 9e2dd31c..a2213f00 100644 --- a/src/engine/imap/transport/imap-mailbox.vala +++ b/src/engine/imap/transport/imap-mailbox.vala @@ -5,6 +5,20 @@ */ public class Geary.Imap.Mailbox : Geary.SmartReference { + private class MailboxOperation : NonblockingBatchOperation { + public SelectedContext context; + public Command cmd; + + public MailboxOperation(SelectedContext context, Command cmd) { + this.context = context; + this.cmd = cmd; + } + + public override async Object? execute(Cancellable? cancellable) throws Error { + return yield context.session.send_command_async(cmd, cancellable); + } + } + public string name { get { return context.name; } } public int exists { get { return context.exists; } } public int recent { get { return context.recent; } } @@ -49,70 +63,92 @@ public class Geary.Imap.Mailbox : Geary.SmartReference { context.recent_altered.disconnect(on_recent_altered); } - public async Gee.List? list_set_async(Geary.Folder folder, MessageSet msg_set, - Geary.Email.Field fields, Cancellable? cancellable = null) throws Error { + public async Gee.List? list_set_async(MessageSet msg_set, Geary.Email.Field fields, + Cancellable? cancellable = null) throws Error { if (context.is_closed()) throw new ImapError.NOT_SELECTED("Mailbox %s closed", name); if (fields == Geary.Email.Field.NONE) throw new EngineError.BAD_PARAMETERS("No email fields specified"); + NonblockingBatch batch = new NonblockingBatch(); + + Gee.List msgs = new Gee.ArrayList(); + Gee.HashMap map = new Gee.HashMap(); + Gee.List data_type_list = new Gee.ArrayList(); Gee.List body_data_type_list = new Gee.ArrayList(); - fields_to_fetch_data_types(fields, data_type_list, body_data_type_list, false); + fields_to_fetch_data_types(fields, data_type_list, body_data_type_list); + + // if nothing else, should always fetch the UID, which is gotten via data_type_list + // (necessary to create the EmailIdentifier, also provides mappings of position -> UID) + assert(data_type_list.size > 0); FetchCommand fetch_cmd = new FetchCommand.from_collection(msg_set, data_type_list, body_data_type_list); - CommandResponse resp = yield context.session.send_command_async(fetch_cmd, cancellable); - if (resp.status_response.status != Status.OK) { - throw new ImapError.SERVER_ERROR("Server error for %s: %s", fetch_cmd.to_string(), - resp.to_string()); + int plain_id = batch.add(new MailboxOperation(context, fetch_cmd)); + + int preview_id = NonblockingBatch.INVALID_ID; + if (fields.require(Geary.Email.Field.PREVIEW)) { + FetchBodyDataType fetch_preview = new FetchBodyDataType.peek(FetchBodyDataType.SectionPart.NONE, + { 1 }, 0, Geary.Email.MAX_PREVIEW_BYTES, null); + Gee.List list = new Gee.ArrayList(); + list.add(fetch_preview); + + FetchCommand preview_cmd = new FetchCommand(msg_set, null, list); + + preview_id = batch.add(new MailboxOperation(context, preview_cmd)); } - Gee.List msgs = new Gee.ArrayList(); + yield batch.execute_all(cancellable); - FetchResults[] results = FetchResults.decode(resp); - foreach (FetchResults res in results) { - UID? uid = res.get_data(FetchDataType.UID) as UID; + // process "plain" FETCH results ... these are fetched every time for, if nothing else, + // the UID which provides a position -> UID mapping that is kept in the map. + + MailboxOperation plain_op = (MailboxOperation) batch.get_operation(plain_id); + CommandResponse plain_resp = (CommandResponse) batch.get_result(plain_id); + + if (plain_resp.status_response.status != Status.OK) { + throw new ImapError.SERVER_ERROR("Server error for %s: %s", plain_op.cmd.to_string(), + plain_resp.to_string()); + } + + FetchResults[] plain_results = FetchResults.decode(plain_resp); + foreach (FetchResults plain_res in plain_results) { + UID? uid = plain_res.get_data(FetchDataType.UID) as UID; // see fields_to_fetch_data_types() for why this is guaranteed assert(uid != null); - Geary.Email email = new Geary.Email(res.msg_num, new Geary.Imap.EmailIdentifier(uid)); - fetch_results_to_email(res, fields, email); + Geary.Email email = new Geary.Email(plain_res.msg_num, new Geary.Imap.EmailIdentifier(uid)); + fetch_results_to_email(plain_res, fields, email); msgs.add(email); + map.set(plain_res.msg_num, email); + assert(map.get(plain_res.msg_num) != null); } - return (msgs != null && msgs.size > 0) ? msgs : null; - } - - public async Geary.Email fetch_async(Geary.Folder folder, Geary.Imap.UID uid, Geary.Email.Field fields, - Cancellable? cancellable = null) throws Error { - if (context.is_closed()) - throw new ImapError.NOT_SELECTED("Mailbox %s closed", name); + // process preview FETCH results - Gee.List data_type_list = new Gee.ArrayList(); - Gee.List body_data_type_list = new Gee.ArrayList(); - fields_to_fetch_data_types(fields, data_type_list, body_data_type_list, true); - - FetchCommand fetch_cmd = new FetchCommand.from_collection(new MessageSet.uid(uid), - data_type_list, body_data_type_list); - - CommandResponse resp = yield context.session.send_command_async(fetch_cmd, cancellable); - if (resp.status_response.status != Status.OK) { - throw new ImapError.SERVER_ERROR("Server error for %s: %s", fetch_cmd.to_string(), - resp.to_string()); + if (preview_id != NonblockingBatch.INVALID_ID) { + MailboxOperation preview_op = (MailboxOperation) batch.get_operation(preview_id); + CommandResponse preview_resp = (CommandResponse) batch.get_result(preview_id); + + if (preview_resp.status_response.status != Status.OK) { + throw new ImapError.SERVER_ERROR("Server error for %s: %s", preview_op.cmd.to_string(), + preview_resp.to_string()); + } + + FetchResults[] preview_results = FetchResults.decode(preview_resp); + foreach (FetchResults preview_res in preview_results) { + Geary.Email? preview_email = map.get(preview_res.msg_num); + assert(preview_email != null); + + preview_email.set_message_preview(new RFC822.Text(preview_res.get_body_data()[0])); + } } - FetchResults[] results = FetchResults.decode(resp); - if (results.length != 1) - throw new ImapError.SERVER_ERROR("Too many responses from server: %d", results.length); - - Geary.Email email = new Geary.Email(results[0].msg_num, new Geary.Imap.EmailIdentifier(uid)); - fetch_results_to_email(results[0], fields, email); - - return email; + return (msgs.size > 0) ? msgs : null; } private void on_closed() { @@ -139,14 +175,10 @@ public class Geary.Imap.Mailbox : Geary.SmartReference { flags_altered(flags); } - // store FetchDataTypes in a set because the same data type may be requested multiple times - // by different fields (i.e. ENVELOPE) private void fields_to_fetch_data_types(Geary.Email.Field fields, Gee.List data_types_list, - Gee.List body_data_types_list, bool is_specific_uid) { - // always fetch UID because it's needed for EmailIdentifier (unless single message is being - // fetched by UID, in which case, obviously not necessary) - if (!is_specific_uid) - data_types_list.add(FetchDataType.UID); + Gee.List body_data_types_list) { + // always fetch UID because it's needed for EmailIdentifier + data_types_list.add(FetchDataType.UID); // pack all the needed headers into a single FetchBodyDataType string[] field_names = new string[0]; @@ -207,7 +239,8 @@ public class Geary.Imap.Mailbox : Geary.SmartReference { break; case Geary.Email.Field.NONE: - // not set + case Geary.Email.Field.PREVIEW: + // not set (or, for previews, fetched separately) break; default: @@ -217,7 +250,7 @@ public class Geary.Imap.Mailbox : Geary.SmartReference { if (field_names.length > 0) { body_data_types_list.add(new FetchBodyDataType( - FetchBodyDataType.SectionPart.HEADER_FIELDS, field_names)); + FetchBodyDataType.SectionPart.HEADER_FIELDS, null, -1, -1, field_names)); } } diff --git a/src/engine/impl/geary-engine-folder.vala b/src/engine/impl/geary-engine-folder.vala index 94c80438..0a43e621 100644 --- a/src/engine/impl/geary-engine-folder.vala +++ b/src/engine/impl/geary-engine-folder.vala @@ -5,7 +5,7 @@ */ private class Geary.EngineFolder : Geary.AbstractFolder { - private const int REMOTE_FETCH_CHUNK_COUNT = 10; + private const int REMOTE_FETCH_CHUNK_COUNT = 5; private class ReplayAppend : ReplayOperation { public EngineFolder owner; @@ -52,6 +52,22 @@ private class Geary.EngineFolder : Geary.AbstractFolder { } } + private class CommitOperation : NonblockingBatchOperation { + public Folder folder; + public Geary.Email email; + + public CommitOperation(Folder folder, Geary.Email email) { + this.folder = folder; + this.email = email; + } + + public override async Object? execute(Cancellable? cancellable) throws Error { + yield folder.create_email_async(email, cancellable); + + return null; + } + } + private RemoteAccount remote; private LocalAccount local; private LocalFolder local_folder; @@ -762,9 +778,14 @@ private class Geary.EngineFolder : Geary.AbstractFolder { break; // if any were fetched, store locally - // TODO: Bulk writing + NonblockingBatch batch = new NonblockingBatch(); + foreach (Geary.Email email in remote_list) - yield local_folder.create_email_async(email, cancellable); + batch.add(new CommitOperation(local_folder, email)); + + yield batch.execute_all(cancellable); + + batch.throw_first_exception(); if (cb != null) cb(remote_list, null); diff --git a/src/engine/nonblocking/nonblocking-batch.vala b/src/engine/nonblocking/nonblocking-batch.vala new file mode 100755 index 00000000..16611b08 --- /dev/null +++ b/src/engine/nonblocking/nonblocking-batch.vala @@ -0,0 +1,244 @@ +/* Copyright 2011 Yorba Foundation + * + * This software is licensed under the GNU Lesser General Public License + * (version 2.1 or later). See the COPYING file in this distribution. + */ + +/** + * A NonblockingBatchOperation is an abstract base class used by NonblockingBatch. It represents + * a single task of asynchronous work. NonblockingBatch will execute it one time only. + */ +public abstract class Geary.NonblockingBatchOperation : Object { + public abstract async Object? execute(Cancellable? cancellable) throws Error; +} + +/** + * NonblockingBatch allows for multiple asynchronous tasks to be executed in parallel and for their + * results to be examined after all have completed. It's designed specifically with Vala's async + * keyword in mind. + * + * Although the yield keyword allows for async tasks to execute, it only allows them to performed + * in serial. In a loop, for example, the next task in the loop won't execute until the current + * one has completed. The thread of execution won't block waiting for it, but this can be + * suboptiminal and certain cases. + * + * NonblockingBatch allows for multiple async tasks to be gathered (via the add() method) into a + * single batch. Each task must subclass from NonblockingBatchOperation. It's expected that the + * subclass will maintain state particular to the operation, although NonblockingBatch does gather + * two types of a results the task may generate: a result object (which descends from Object) or + * a thrown exception. Other results should be stored by the subclass. + * + * To use, create a NonblockingBatch and populate it via the add() method. When all + * NonblockingBatchOperations have been added, call execute_all(). NonblockingBatch will fire off + * all at once and only complete execute_all() when all of them have finished. As mentioned + * earlier, it's also gather their returned objects and thrown exceptions while they run. See + * get_result() and throw_first_exception() for more information. + * + * The caller will want to call *either* get_result() or throw_first_exception() to ensure that + * errors are propagated. It's not necessary to call both. + * + * After execute_all() has completed, the results may be examined. The NonblockingBatch object + * can *not* be reused. + * + * Currently NonblockingBatch will fire off all operations at once and let them complete. It does + * not attempt to stop the others if one throws exception. Also, there's no algorithm to submit the + * operations in smaller chunks (to avoid flooding the thread's MainLoop). These may be added in + * the future. + */ +public class Geary.NonblockingBatch : Object { + public const int INVALID_ID = -1; + + private const int START_ID = 1; + + private class BatchContext { + public weak NonblockingBatch owner; + public int id; + public NonblockingBatchOperation op; + public bool completed = false; + public Object? returned = null; + public Error? threw = null; + + public BatchContext(NonblockingBatch owner, int id, NonblockingBatchOperation op) { + this.owner = owner; + this.id = id; + this.op = op; + } + + public void execute(Cancellable? cancellable) { + op.execute.begin(cancellable, on_op_completed); + } + + private void on_op_completed(Object? source, AsyncResult result) { + completed = true; + + try { + returned = op.execute.end(result); + } catch (Error err) { + threw = err; + } + + owner.on_context_completed(this); + } + } + + /** + * Returns the number of NonblockingBatchOperations added. + */ + public int count { + get { return contexts.size; } + } + + private Gee.HashMap contexts = new Gee.HashMap(); + private NonblockingSemaphore sem = new NonblockingSemaphore(); + private int next_result_id = START_ID; + private bool locked = false; + private int completed_ops = 0; + private Error? first_exception = null; + + public signal void added(NonblockingBatchOperation op, int id); + + public signal void started(int count); + + public signal void operation_completed(NonblockingBatchOperation op, Object? returned, + Error? threw); + + public signal void completed(int count, Error? first_error); + + public NonblockingBatch() { + } + + /** + * Adds a NonblockingBatchOperation to the batch. INVALID_ID is returned if the batch is + * executing or has already executed. Otherwise, returns an ID that can be used to fetch + * results of this particular NonblockingBatchOperation after execute_all() completes. + * + * The returned ID is only good for this NonblockingBatch. Since each instance uses the + * same algorithm, different instances will likely return the same ID, so they must be + * associated with the NonblockingBatch they originated from. + */ + public int add(NonblockingBatchOperation op) { + if (locked) { + warning("NonblockingBatch already executed or executing"); + + return INVALID_ID; + } + + int id = next_result_id++; + contexts.set(id, new BatchContext(this, id, op)); + + added(op, id); + + return id; + } + + /** + * Executes all the NonblockingBatchOperations added to the batch. The supplied Cancellable + * will be passed to each operation. + * + * If the batch is executing or already executed, IOError.PENDING will be thrown. If the + * Cancellable is already cancelled, IOError.CANCELLED is thrown. Other errors may be thrown + * as well; see NonblockingAbstractSemaphore.wait_async(). + * + * If there are no operations added to the batch, the method quietly exits. + */ + public async void execute_all(Cancellable? cancellable = null) throws Error { + if (locked) + throw new IOError.PENDING("NonblockingBatch already executed or executing"); + + locked = true; + + // if empty, quietly exit + if (contexts.size == 0) + return; + + // if already cancelled, not-so-quietly exit + if (cancellable != null && cancellable.is_cancelled()) + throw new IOError.CANCELLED("NonblockingBatch cancelled before executing"); + + started(contexts.size); + + // although they should technically be able to execute in any order, fire them off in the + // order they were submitted; this may hide bugs, but it also makes other bugs reproducible + int count = 0; + for (int id = START_ID; id < next_result_id; id++) { + BatchContext? context = contexts.get(id); + assert(context != null); + + context.execute(cancellable); + count++; + } + + assert(count == contexts.size); + + yield sem.wait_async(cancellable); + } + + /** + * Returns a Set of IDs for all added NonblockingBatchOperations. + */ + public Gee.Set get_ids() { + return contexts.keys; + } + + /** + * Returns the NonblockingBatchOperation for the supplied ID. Returns null if the ID is invalid + * or unknown. + */ + public NonblockingBatchOperation? get_operation(int id) { + BatchContext? context = contexts.get(id); + + return (context != null) ? context.op : null; + } + + /** + * Returns the resulting Object from the operation for the supplied ID. If the ID is invalid + * or unknown, or the operation returned null, null is returned. + * + * If the operation threw an exception, it will be thrown here. If all the operations' results + * are examined with this method, there is no need to call throw_first_exception(). + * + * If the operation has not completed, IOError.BUSY will be thrown. It *is* legal to query + * the result of a completed operation while others are executing. + */ + public Object? get_result(int id) throws Error { + BatchContext? context = contexts.get(id); + if (context == null) + return null; + + if (!context.completed) + throw new IOError.BUSY("NonblockingBatchOperation %d not completed", id); + + if (context.threw != null) + throw context.threw; + + return context.returned; + } + + /** + * If no results are examined via get_result(), this method can be used to manually throw the + * first seen Error from the operations. + */ + public void throw_first_exception() throws Error { + if (first_exception != null) + throw first_exception; + } + + private void on_context_completed(BatchContext context) { + if (first_exception == null && context.threw != null) + first_exception = context.threw; + + operation_completed(context.op, context.returned, context.threw); + + assert(completed_ops < contexts.size); + if (++completed_ops == contexts.size) { + try { + sem.notify(); + } catch (Error err) { + debug("Unable to notify NonblockingBatch semaphore: %s", err.message); + } + + completed(completed_ops, first_exception); + } + } +} + diff --git a/src/engine/sqlite/email/sqlite-message-row.vala b/src/engine/sqlite/email/sqlite-message-row.vala index a6e6f834..69d33878 100644 --- a/src/engine/sqlite/email/sqlite-message-row.vala +++ b/src/engine/sqlite/email/sqlite-message-row.vala @@ -29,6 +29,8 @@ public class Geary.Sqlite.MessageRow : Geary.Sqlite.Row { public string? body { get; set; } + public string? preview { get; set; } + public MessageRow(Table table) { base (table); } @@ -80,6 +82,9 @@ public class Geary.Sqlite.MessageRow : Geary.Sqlite.Row { if ((fields & Geary.Email.Field.BODY) != 0) body = fetch_string_for(result, MessageTable.Column.BODY); + + if ((fields & Geary.Email.Field.PREVIEW) != 0) + preview = fetch_string_for(result, MessageTable.Column.PREVIEW); } public Geary.Email to_email(int position, Geary.EmailIdentifier id) throws Error { @@ -114,6 +119,9 @@ public class Geary.Sqlite.MessageRow : Geary.Sqlite.Row { if (((fields & Geary.Email.Field.BODY) != 0) && (body != null)) email.set_message_body(new RFC822.Text(new Geary.Memory.StringBuffer(body))); + if (((fields & Geary.Email.Field.PREVIEW) != 0) && (preview != null)) + email.set_message_preview(new RFC822.Text(new Geary.Memory.StringBuffer(preview))); + return email; } @@ -206,6 +214,12 @@ public class Geary.Sqlite.MessageRow : Geary.Sqlite.Row { this.fields = this.fields.set(Geary.Email.Field.BODY); } + + if ((fields & Geary.Email.Field.PREVIEW) != 0) { + preview = (email.preview != null) ? email.preview.buffer.to_utf8() : null; + + this.fields = this.fields.set(Geary.Email.Field.PREVIEW); + } } private void unset_fields(Geary.Email.Field fields) { @@ -257,6 +271,12 @@ public class Geary.Sqlite.MessageRow : Geary.Sqlite.Row { this.fields = this.fields.clear(Geary.Email.Field.BODY); } + + if ((fields & Geary.Email.Field.PREVIEW) != 0) { + preview = null; + + this.fields = this.fields.clear(Geary.Email.Field.PREVIEW); + } } } diff --git a/src/engine/sqlite/email/sqlite-message-table.vala b/src/engine/sqlite/email/sqlite-message-table.vala index ffbf9295..75e73644 100644 --- a/src/engine/sqlite/email/sqlite-message-table.vala +++ b/src/engine/sqlite/email/sqlite-message-table.vala @@ -29,7 +29,9 @@ public class Geary.Sqlite.MessageTable : Geary.Sqlite.Table { HEADER, - BODY; + BODY, + + PREVIEW; } internal MessageTable(Geary.Sqlite.Database gdb, SQLHeavy.Table table) { @@ -44,8 +46,8 @@ public class Geary.Sqlite.MessageTable : Geary.Sqlite.Table { SQLHeavy.Query query = locked.prepare( "INSERT INTO MessageTable " + "(fields, date_field, date_time_t, from_field, sender, reply_to, to_field, cc, bcc, " - + "message_id, in_reply_to, reference_ids, subject, header, body) " - + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); + + "message_id, in_reply_to, reference_ids, subject, header, body, preview) " + + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); query.bind_int(0, row.fields); query.bind_string(1, row.date); query.bind_int64(2, row.date_time_t); @@ -61,6 +63,7 @@ public class Geary.Sqlite.MessageTable : Geary.Sqlite.Table { query.bind_string(12, row.subject); query.bind_string(13, row.header); query.bind_string(14, row.body); + query.bind_string(15, row.preview); int64 id = yield query.execute_insert_async(cancellable); locked.set_commit_required(); @@ -156,6 +159,15 @@ public class Geary.Sqlite.MessageTable : Geary.Sqlite.Table { yield query.execute_async(cancellable); } + if (row.fields.is_any_set(Geary.Email.Field.PREVIEW)) { + query = locked.prepare( + "UPDATE MessageTable SET preview=? WHERE id=?"); + query.bind_string(0, row.preview); + query.bind_int64(1, row.id); + + yield query.execute_async(cancellable); + } + // only commit if internally atomic if (transaction == null) yield locked.commit_async(cancellable); @@ -263,13 +275,15 @@ public class Geary.Sqlite.MessageTable : Geary.Sqlite.Table { case Geary.Email.Field.BODY: append = "body"; break; + + case Geary.Email.Field.PREVIEW: + append = "preview"; + break; } } if (append != null) { - if (!String.is_empty(builder.str)) - builder.append(", "); - + builder.append(", "); builder.append(append); } } diff --git a/src/wscript b/src/wscript index ab73d11f..1a92af6d 100644 --- a/src/wscript +++ b/src/wscript @@ -87,9 +87,10 @@ def build(bld): '../engine/impl/geary-remote-interfaces.vala', '../engine/impl/geary-replay-queue.vala', + '../engine/nonblocking/nonblocking-abstract-semaphore.vala', + '../engine/nonblocking/nonblocking-batch.vala', '../engine/nonblocking/nonblocking-mailbox.vala', '../engine/nonblocking/nonblocking-mutex.vala', - '../engine/nonblocking/nonblocking-abstract-semaphore.vala', '../engine/nonblocking/nonblocking-variants.vala', '../engine/rfc822/rfc822-error.vala',