diff --git a/icons/edit-symbolic.svg b/icons/edit-symbolic.svg new file mode 100644 index 00000000..294ca0b6 --- /dev/null +++ b/icons/edit-symbolic.svg @@ -0,0 +1,76 @@ + + + + + + + + Gnome Symbolic Icon Theme + + + + + image/svg+xml + + Gnome Symbolic Icon Theme + + + + + + diff --git a/sql/version-010.sql b/sql/version-010.sql index 0bfc39a1..120f6b81 100644 --- a/sql/version-010.sql +++ b/sql/version-010.sql @@ -1,5 +1,5 @@ -- --- Dummy database upgrade to add MessageSearchTable, whose parameters depend on --- things we need at run-time. See src/engine/imap-db/imap-db-database.vala in --- post_upgrade() for the code that runs the upgrade. +-- Add unread count column to the FolderTable -- + +ALTER TABLE FolderTable ADD COLUMN unread_count INTEGER DEFAULT 0; diff --git a/sql/version-011.sql b/sql/version-011.sql index e4aa2ddb..0bfc39a1 100644 --- a/sql/version-011.sql +++ b/sql/version-011.sql @@ -1,7 +1,5 @@ -- --- Add the internaldate column as a time_t value so we can sort on it. +-- Dummy database upgrade to add MessageSearchTable, whose parameters depend on +-- things we need at run-time. See src/engine/imap-db/imap-db-database.vala in +-- post_upgrade() for the code that runs the upgrade. -- - -ALTER TABLE MessageTable ADD COLUMN internaldate_time_t INTEGER; - -CREATE INDEX MessageTableInternalDateTimeTIndex ON MessageTable(internaldate_time_t); diff --git a/sql/version-012.sql b/sql/version-012.sql new file mode 100644 index 00000000..e4aa2ddb --- /dev/null +++ b/sql/version-012.sql @@ -0,0 +1,7 @@ +-- +-- Add the internaldate column as a time_t value so we can sort on it. +-- + +ALTER TABLE MessageTable ADD COLUMN internaldate_time_t INTEGER; + +CREATE INDEX MessageTableInternalDateTimeTIndex ON MessageTable(internaldate_time_t); diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a97bb5ab..5d784add 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -15,6 +15,7 @@ engine/abstract/geary-abstract-local-folder.vala engine/api/geary-account.vala engine/api/geary-account-information.vala +engine/api/geary-aggregated-folder-properties.vala engine/api/geary-attachment.vala engine/api/geary-base-object.vala engine/api/geary-composed-email.vala @@ -73,6 +74,7 @@ engine/imap/api/imap-email-identifier.vala engine/imap/api/imap-email-properties.vala engine/imap/api/imap-folder-properties.vala engine/imap/api/imap-folder.vala +engine/imap/command/imap-append-command.vala engine/imap/command/imap-capability-command.vala engine/imap/command/imap-close-command.vala engine/imap/command/imap-command.vala @@ -184,6 +186,16 @@ engine/imap-engine/replay-ops/imap-engine-replay-removal.vala engine/imap-engine/yahoo/imap-engine-yahoo-account.vala engine/imap-engine/yahoo/imap-engine-yahoo-folder.vala +engine/memory/memory-buffer.vala +engine/memory/memory-byte-buffer.vala +engine/memory/memory-empty-buffer.vala +engine/memory/memory-file-buffer.vala +engine/memory/memory-growable-buffer.vala +engine/memory/memory-string-buffer.vala +engine/memory/memory-unowned-byte-array-buffer.vala +engine/memory/memory-unowned-bytes-buffer.vala +engine/memory/memory-unowned-string-buffer.vala + engine/nonblocking/nonblocking-abstract-semaphore.vala engine/nonblocking/nonblocking-batch.vala engine/nonblocking/nonblocking-counting-semaphore.vala @@ -231,8 +243,8 @@ engine/util/util-generic-capabilities.vala engine/util/util-html.vala engine/util/util-imap-utf7.vala engine/util/util-inet.vala -engine/util/util-memory.vala engine/util/util-numeric.vala +engine/util/util-object.vala engine/util/util-reference-semantics.vala engine/util/util-scheduler.vala engine/util/util-single-item.vala diff --git a/src/client/composer/composer-window.vala b/src/client/composer/composer-window.vala index dec7096f..649f0451 100644 --- a/src/client/composer/composer-window.vala +++ b/src/client/composer/composer-window.vala @@ -80,6 +80,10 @@ public class ComposerWindow : Gtk.Window { """; + public const string ATTACHMENT_KEYWORDS_GENERIC = ".doc|.pdf|.xls|.ppt|.rtf|.pps"; + /// A list of keywords, separated by pipe ("|") characters, that suggest an attachment + public const string ATTACHMENT_KEYWORDS_LOCALIZED = _("attach|enclosed|enclosing|cover letter"); + // Signal sent when the "Send" button is clicked. public signal void send(ComposerWindow composer); @@ -304,7 +308,7 @@ public class ComposerWindow : Gtk.Window { try { body_html = referred.get_message().get_body(true); } catch (Error error) { - debug("Error getting messae body: %s", error.message); + debug("Error getting message body: %s", error.message); } add_attachments(referred.attachments); break; @@ -659,9 +663,56 @@ public class ComposerWindow : Gtk.Window { on_discard(); } + private bool email_contains_attachment_keywords() { + // Filter out all content contained in block quotes + string filtered = @"$subject\n"; + filtered += Util.DOM.get_text_representation(editor.get_dom_document(), "blockquote"); + + Regex url_regex = null; + try { + // Prepare to ignore urls later + url_regex = new Regex(URL_REGEX, RegexCompileFlags.CASELESS); + } catch (Error error) { + debug("Error building regex in keyword checker: %s", error.message); + } + + string[] keys = ATTACHMENT_KEYWORDS_GENERIC.casefold().split("|"); + foreach (string key in ATTACHMENT_KEYWORDS_LOCALIZED.casefold().split("|")) { + keys += key; + } + + string folded; + foreach (string line in filtered.split("\n")) { + // Stop looking once we hit forwarded content + if (line.has_prefix("--")) { + break; + } + + folded = line.casefold(); + foreach (string key in keys) { + if (key in folded) { + try { + // Make sure the match isn't coming from a url + if (key in url_regex.replace(folded, -1, 0, "")) { + return true; + } + } catch (Error error) { + debug("Regex replacement error in keyword checker: %s", error.message); + return true; + } + } + } + } + + return false; + } + private bool should_send() { bool has_subject = !Geary.String.is_empty(subject.strip()); - bool has_body_or_attachment = !Geary.String.is_empty(get_html()) || attachment_files.size > 0; + bool has_body = !Geary.String.is_empty(get_html()); + bool has_attachment = attachment_files.size > 0; + bool has_body_or_attachment = has_body || has_attachment; + string? confirmation = null; if (!has_subject && !has_body_or_attachment) { confirmation = _("Send message with an empty subject and body?"); @@ -669,6 +720,8 @@ public class ComposerWindow : Gtk.Window { confirmation = _("Send message with an empty subject?"); } else if (!has_body_or_attachment) { confirmation = _("Send message with an empty body?"); + } else if (!has_attachment && email_contains_attachment_keywords()) { + confirmation = _("Send message without an attachment?"); } if (confirmation != null) { ConfirmationDialog dialog = new ConfirmationDialog(this, diff --git a/src/client/folder-list/folder-list-folder-entry.vala b/src/client/folder-list/folder-list-folder-entry.vala index dac292de..0686a232 100644 --- a/src/client/folder-list/folder-list-folder-entry.vala +++ b/src/client/folder-list/folder-list-folder-entry.vala @@ -8,24 +8,30 @@ public class FolderList.FolderEntry : FolderList.AbstractFolderEntry, Sidebar.InternalDropTargetEntry, Sidebar.EmphasizableEntry { private bool has_new; - private int unread_count; public FolderEntry(Geary.Folder folder) { base(folder); has_new = false; - unread_count = 0; + folder.properties.notify[Geary.FolderProperties.PROP_NAME_EMAIL_UNDREAD].connect( + on_email_unread_count_changed); + } + + ~FolderEntry() { + folder.properties.notify[Geary.FolderProperties.PROP_NAME_EMAIL_UNDREAD].disconnect( + on_email_unread_count_changed); } public override string get_sidebar_name() { - return (unread_count == 0 ? folder.get_display_name() : + return (folder.properties.email_unread == 0 ? folder.get_display_name() : /// This string gets the folder name and the unread messages count, /// e.g. All Mail (5). - _("%s (%d)").printf(folder.get_display_name(), unread_count)); + _("%s (%d)").printf(folder.get_display_name(), folder.properties.email_unread)); } public override string? get_sidebar_tooltip() { - return (unread_count == 0 ? null : - ngettext("%d unread message", "%d unread messages", unread_count).printf(unread_count)); + return (folder.properties.email_unread == 0 ? null : + ngettext("%d unread message", "%d unread messages", folder.properties.email_unread). + printf(folder.properties.email_unread)); } public override Icon? get_sidebar_icon() { @@ -81,15 +87,6 @@ public class FolderList.FolderEntry : FolderList.AbstractFolderEntry, Sidebar.In is_emphasized_changed(has_new); } - public void set_unread_count(int unread_count) { - if (this.unread_count == unread_count) - return; - - this.unread_count = unread_count; - sidebar_name_changed(get_sidebar_name()); - sidebar_tooltip_changed(get_sidebar_tooltip()); - } - public bool internal_drop_received(Gdk.DragContext context, Gtk.SelectionData data) { // Copy or move? Gdk.ModifierType mask; @@ -104,4 +101,9 @@ public class FolderList.FolderEntry : FolderList.AbstractFolderEntry, Sidebar.In return true; } + + private void on_email_unread_count_changed() { + sidebar_name_changed(get_sidebar_name()); + sidebar_tooltip_changed(get_sidebar_tooltip()); + } } diff --git a/src/client/folder-list/folder-list-inbox-folder-entry.vala b/src/client/folder-list/folder-list-inbox-folder-entry.vala index 04d656c4..96f86504 100644 --- a/src/client/folder-list/folder-list-inbox-folder-entry.vala +++ b/src/client/folder-list/folder-list-inbox-folder-entry.vala @@ -16,7 +16,10 @@ public class FolderList.InboxFolderEntry : FolderList.FolderEntry { } public override string get_sidebar_name() { - return folder.account.information.nickname; + return (folder.properties.email_unread == 0 ? folder.account.information.nickname : + /// This string gets the account nickname and the unread messages count, + /// e.g. Work (5). + _("%s (%d)").printf(folder.account.information.nickname, folder.properties.email_unread)); } public Geary.AccountInformation get_account_information() { diff --git a/src/client/folder-list/folder-list-search-branch.vala b/src/client/folder-list/folder-list-search-branch.vala index d0c06d21..17c43464 100644 --- a/src/client/folder-list/folder-list-search-branch.vala +++ b/src/client/folder-list/folder-list-search-branch.vala @@ -36,7 +36,7 @@ public class FolderList.SearchEntry : FolderList.AbstractFolderEntry { } public override string? get_sidebar_tooltip() { - return _("%d results").printf(folder.get_properties().email_total); + return _("%d results").printf(folder.properties.email_total); } public override Icon? get_sidebar_icon() { diff --git a/src/client/util/util-date.vala b/src/client/util/util-date.vala index 2fb0cf1f..c0e4167c 100644 --- a/src/client/util/util-date.vala +++ b/src/client/util/util-date.vala @@ -122,7 +122,7 @@ public string pretty_print(DateTime datetime, ClockFormat clock_format) { if (diff < TimeSpan.HOUR) { return _("%dm ago").printf(diff / TimeSpan.MINUTE); } - if (diff < 6 * TimeSpan.HOUR) { + if (diff < 12 * TimeSpan.HOUR) { return _("%dh ago").printf(diff / TimeSpan.HOUR); } diff --git a/src/client/util/util-webkit.vala b/src/client/util/util-webkit.vala index 14f89abc..9679e4c1 100644 --- a/src/client/util/util-webkit.vala +++ b/src/client/util/util-webkit.vala @@ -42,6 +42,69 @@ namespace Util.DOM { class_list.remove(clas); } } + + // Returns the text contained in the DOM document, after ignoring tags of type "exclude" + // and padding newlines where appropriate. Used to scan for attachment keywords. + public string get_text_representation(WebKit.DOM.Document doc, string exclude) { + WebKit.DOM.HTMLElement? copy = Util.DOM.clone_node(doc.get_body()); + if (copy == null) { + return ""; + } + + // Keep deleting the next excluded element until there are none left + while (true) { + WebKit.DOM.HTMLElement? current = Util.DOM.select(copy, exclude); + if (current == null) { + break; + } + + WebKit.DOM.Node parent = current.get_parent_node(); + try { + parent.remove_child(current); + } catch (Error error) { + debug("Error removing blockquotes: %s", error.message); + break; + } + } + + WebKit.DOM.NodeList node_list; + try { + node_list = copy.query_selector_all("br"); + } catch (Error error) { + debug("Error finding
s: %s", error.message); + return copy.get_inner_text(); + } + + // Replace
tags with newlines + for (int i = 0; i < node_list.length; ++i) { + WebKit.DOM.Node br = node_list.item(i); + WebKit.DOM.Node parent = br.get_parent_node(); + try { + parent.replace_child(doc.create_text_node("\n"), br); + } catch (Error error) { + debug("Error replacing
: %s", error.message); + } + } + + try { + node_list = copy.query_selector_all("div"); + } catch (Error error) { + debug("Error finding
s: %s", error.message); + return copy.get_inner_text(); + } + + // Pad each
with newlines + for (int i = 0; i < node_list.length; ++i) { + WebKit.DOM.Node div = node_list.item(i); + try { + div.insert_before(doc.create_text_node("\n"), div.first_child); + div.append_child(doc.create_text_node("\n")); + } catch (Error error) { + debug("Error padding
with newlines: %s", error.message); + } + } + return copy.get_inner_text(); + } } public void bind_event(WebKit.WebView view, string selector, string event, Callback callback, diff --git a/src/client/views/conversation-viewer.vala b/src/client/views/conversation-viewer.vala index fead02e8..df30f259 100644 --- a/src/client/views/conversation-viewer.vala +++ b/src/client/views/conversation-viewer.vala @@ -219,6 +219,7 @@ public class ConversationViewer : Gtk.Box { current_conversation.appended.disconnect(on_conversation_appended); current_conversation.trimmed.disconnect(on_conversation_trimmed); current_conversation.email_flags_changed.disconnect(update_flags); + current_conversation = null; } // Disable message buttons until conversation loads. @@ -226,15 +227,13 @@ public class ConversationViewer : Gtk.Box { if (conversations == null || conversations.size == 0 || current_folder == null) { show_multiple_selected(0); - current_conversation = null; return; } - // Clear view before we yield, to make sure it happens. - clear(current_folder, current_folder.account.information); - web_view.scroll_reset(); - if (conversations.size == 1) { + clear(current_folder, current_folder.account.information); + web_view.scroll_reset(); + current_conversation = Geary.Collection.get_first(conversations); select_conversation_async.begin(current_conversation, current_folder, @@ -256,10 +255,13 @@ public class ConversationViewer : Gtk.Box { Geary.Folder current_folder) throws Error { Gee.Collection messages = conversation.get_emails(Geary.Conversation.Ordering.DATE_ASCENDING); + // Load this once, so if it's cancelled, we cancel the WHOLE load. + Cancellable cancellable = cancellable_fetch; + // Fetch full messages. Gee.Collection messages_to_add = new Gee.HashSet(); foreach (Geary.Email email in messages) - messages_to_add.add(yield fetch_full_message_async(email)); + messages_to_add.add(yield fetch_full_message_async(email, cancellable)); // Add messages. foreach (Geary.Email email in messages_to_add) @@ -316,17 +318,18 @@ public class ConversationViewer : Gtk.Box { } // Given an email, fetch the full version with all required fields. - private async Geary.Email fetch_full_message_async(Geary.Email email) throws Error { + private async Geary.Email fetch_full_message_async(Geary.Email email, + Cancellable? cancellable) throws Error { Geary.Email.Field required_fields = ConversationViewer.REQUIRED_FIELDS | Geary.ComposedEmail.REQUIRED_REPLY_FIELDS; Geary.Email full_email; if (email.id.get_folder_path() == null) { full_email = yield current_folder.account.local_fetch_email_async( - email.id, required_fields, cancellable_fetch); + email.id, required_fields, cancellable); } else { full_email = yield current_folder.fetch_email_async(email.id, - required_fields, Geary.Folder.ListFlags.NONE, cancellable_fetch); + required_fields, Geary.Folder.ListFlags.NONE, cancellable); } return full_email; @@ -345,7 +348,7 @@ public class ConversationViewer : Gtk.Box { } private async void on_conversation_appended_async(Geary.Email email) throws Error { - add_message(yield fetch_full_message_async(email)); + add_message(yield fetch_full_message_async(email, cancellable_fetch)); } private void on_conversation_appended_complete(Object? source, AsyncResult result) { @@ -1415,9 +1418,8 @@ public class ConversationViewer : Gtk.Box { continue; } else if (src.has_prefix("cid:")) { string mime_id = src.substring(4); - Geary.Memory.AbstractBuffer image_content = - message.get_content_by_mime_id(mime_id); - uint8[] image_data = image_content.get_array(); + Geary.Memory.Buffer image_content = message.get_content_by_mime_id(mime_id); + uint8[] image_data = image_content.get_bytes().get_data(); // Get the content type. bool uncertain_content_type; diff --git a/src/console/main.vala b/src/console/main.vala index 20f7fd0a..a0f91fec 100644 --- a/src/console/main.vala +++ b/src/console/main.vala @@ -83,6 +83,7 @@ class ImapConsole : Gtk.Window { "capabililties", "caps", "connect", + "unsecure", "disconnect", "login", "logout", @@ -94,6 +95,7 @@ class ImapConsole : Gtk.Window { "fetch", "uid-fetch", "fetch-fields", + "append", "help", "exit", "quit", @@ -138,6 +140,7 @@ class ImapConsole : Gtk.Window { break; case "connect": + case "unsecure": connect_cmd(cmd, args); break; @@ -181,6 +184,10 @@ class ImapConsole : Gtk.Window { fetch_fields(cmd, args); break; + case "append": + append(cmd, args); + break; + case "help": foreach (string cmdname in cmdnames) print_console_line(cmdname); @@ -285,10 +292,13 @@ class ImapConsole : Gtk.Window { check_args(cmd, args, 1, "hostname[:port]"); + Geary.Endpoint.Flags flags = Geary.Endpoint.Flags.GRACEFUL_DISCONNECT; + if (cmd != "unsecure") + flags |= Geary.Endpoint.Flags.SSL; + cx = new Geary.Imap.ClientConnection( new Geary.Endpoint(args[0], Geary.Imap.ClientConnection.DEFAULT_PORT_SSL, - Geary.Endpoint.Flags.SSL | Geary.Endpoint.Flags.GRACEFUL_DISCONNECT, - Geary.Imap.ClientConnection.DEFAULT_TIMEOUT_SEC)); + flags, Geary.Imap.ClientConnection.DEFAULT_TIMEOUT_SEC)); cx.sent_command.connect(on_sent_command); cx.received_status_response.connect(on_received_status_response); @@ -462,6 +472,25 @@ class ImapConsole : Gtk.Window { } } + private void append(string cmd, string[] args) throws Error { + check_connected(cmd, args, 2, " "); + + status("Appending %s to %s".printf(args[1], args[0])); + + cx.send_async.begin(new Geary.Imap.AppendCommand(new Geary.Imap.MailboxSpecifier(args[0]), + null, null, new Geary.Memory.FileBuffer(File.new_for_path(args[1]), true)), null, + on_appended); + } + + private void on_appended(Object? source, AsyncResult result) { + try { + cx.send_async.end(result); + status("Appended"); + } catch (Error err) { + exception(err); + } + } + private void close(string cmd, string[] args) throws Error { check_connected(cmd, args, 0, null); @@ -601,7 +630,7 @@ class ImapConsole : Gtk.Window { void main(string[] args) { Gtk.init(ref args); - Geary.Logging.set_flags(Geary.Logging.Flag.NETWORK); + Geary.Logging.enable_flags(Geary.Logging.Flag.NETWORK); Geary.Logging.log_to(stdout); ImapConsole console = new ImapConsole(); diff --git a/src/engine/abstract/geary-abstract-folder.vala b/src/engine/abstract/geary-abstract-folder.vala index fdc47be0..4620d386 100644 --- a/src/engine/abstract/geary-abstract-folder.vala +++ b/src/engine/abstract/geary-abstract-folder.vala @@ -50,9 +50,9 @@ public abstract class Geary.AbstractFolder : BaseObject, Geary.Folder { public abstract Geary.Account account { get; } - public abstract Geary.FolderPath get_path(); + public abstract Geary.FolderProperties properties { get; } - public abstract Geary.FolderProperties get_properties(); + public abstract Geary.FolderPath get_path(); public abstract Geary.SpecialFolderType get_special_folder_type(); diff --git a/src/engine/abstract/geary-abstract-local-folder.vala b/src/engine/abstract/geary-abstract-local-folder.vala index 1f65a564..0ba8d676 100644 --- a/src/engine/abstract/geary-abstract-local-folder.vala +++ b/src/engine/abstract/geary-abstract-local-folder.vala @@ -33,7 +33,7 @@ public abstract class Geary.AbstractLocalFolder : Geary.AbstractFolder { if (open_count++ > 0) return false; - notify_opened(Geary.Folder.OpenState.LOCAL, get_properties().email_total); + notify_opened(Geary.Folder.OpenState.LOCAL, properties.email_total); return true; } diff --git a/src/engine/api/geary-aggregated-folder-properties.vala b/src/engine/api/geary-aggregated-folder-properties.vala new file mode 100644 index 00000000..543b3557 --- /dev/null +++ b/src/engine/api/geary-aggregated-folder-properties.vala @@ -0,0 +1,54 @@ +/* Copyright 2013 Yorba Foundation + * + * This software is licensed under the GNU Lesser General Public License + * (version 2.1 or later). See the COPYING file in this distribution. + */ + +/** + * Aggregates multiple FolderProperties into one. This way a Geary.Folder can + * present one stable FolderProperties object that the client can register + * change listeners on, etc. despite most Geary.Folders having both a local + * and remote version of FolderProperties. + * + * The class relies on GObject bindings and the fact that FolderProperties + * contains only propertiess. + */ +private class Geary.AggregatedFolderProperties : Geary.FolderProperties { + // Map of child FolderProperties to their bindings. + private Gee.Map> child_bindings + = new Gee.HashMap>(); + + /** + * Creates an aggregate FolderProperties. + */ + public AggregatedFolderProperties() { + // Set defaults. + base(0, 0, Trillian.UNKNOWN, Trillian.UNKNOWN, Trillian.UNKNOWN); + } + + /** + * Adds a child FolderProperties. The child's property values will overwrite + * this class's property values. + */ + public void add(FolderProperties child) { + // Create a binding for all properties. + Gee.List? bindings = Geary.ObjectUtils.mirror_properties(child, this); + assert(bindings != null); + child_bindings.set(child, bindings); + } + + /** + * Removes a child FolderProperties. + */ + public bool remove(FolderProperties child) { + Gee.List bindings; + if (child_bindings.unset(child, out bindings)) { + Geary.ObjectUtils.unmirror_properties(bindings); + + return true; + } + + return false; + } +} + diff --git a/src/engine/api/geary-folder-path.vala b/src/engine/api/geary-folder-path.vala index 30e69bf2..e2938813 100644 --- a/src/engine/api/geary-folder-path.vala +++ b/src/engine/api/geary-folder-path.vala @@ -217,7 +217,7 @@ public class Geary.FolderPath : BaseObject, Gee.Hashable, * are less-than longer paths, assuming the path elements are equal up to the shorter path's * length. * - * Note that the {@ link FolderRoot.default_separator} has no bearing on comparisons, although + * Note that the {@link FolderRoot.default_separator} has no bearing on comparisons, although * {@link FolderRoot.case_sensitive} does. * * Returns -1 if this path is lexiographically before the other, 1 if its after, and 0 if they diff --git a/src/engine/api/geary-folder-properties.vala b/src/engine/api/geary-folder-properties.vala index 03047e77..c0615ad1 100644 --- a/src/engine/api/geary-folder-properties.vala +++ b/src/engine/api/geary-folder-properties.vala @@ -5,6 +5,12 @@ */ public abstract class Geary.FolderProperties : BaseObject { + public const string PROP_NAME_EMAIL_TOTAL = "email-total"; + public const string PROP_NAME_EMAIL_UNDREAD = "email-unread"; + public const string PROP_NAME_HAS_CHILDREN = "has-children"; + public const string PROP_NAME_SUPPORTS_CHILDREN = "supports-children"; + public const string PROP_NAME_IS_OPENABLE = "is-openable"; + /** * The total count of email in the Folder. */ diff --git a/src/engine/api/geary-folder.vala b/src/engine/api/geary-folder.vala index 148fffa5..4d29e321 100644 --- a/src/engine/api/geary-folder.vala +++ b/src/engine/api/geary-folder.vala @@ -109,6 +109,8 @@ public interface Geary.Folder : BaseObject { public abstract Geary.Account account { get; } + public abstract Geary.FolderProperties properties { get; } + /** * Fired when the folder is successfully opened by a caller. * @@ -236,19 +238,6 @@ public interface Geary.Folder : BaseObject { public abstract Geary.FolderPath get_path(); - /** - * Returns a FolderProperties that represents, if fully open, accurate values for this Folder, - * and if not, values that represent the last time the Folder was opened or examined by the - * Engine. - * - * The returned object is not guaranteed to be long-lived. If the Folder's state changes, it's - * possible a new FolderProperties will be set in its place. Instead of monitoring the fields - * of the FolderProperties for changes, use Account.folders_contents_changed() to be notified - * of changes and use the (potentially new) FolderProperties returned by this method at that - * point. - */ - public abstract Geary.FolderProperties get_properties(); - /** * Returns the special folder type of the folder. */ diff --git a/src/engine/api/geary-search-folder.vala b/src/engine/api/geary-search-folder.vala index 3d0e1ce3..324fc34f 100644 --- a/src/engine/api/geary-search-folder.vala +++ b/src/engine/api/geary-search-folder.vala @@ -30,11 +30,12 @@ public class Geary.SearchFolder : Geary.AbstractLocalFolder { public static const int MAX_RESULT_EMAILS = 1000; public override Account account { get { return _account; } } + public override FolderProperties properties { get { return _properties; } } private static FolderRoot? path = null; private weak Account _account; - private SearchFolderProperties properties = new SearchFolderProperties(0, 0); + private SearchFolderProperties _properties = new SearchFolderProperties(0, 0); private Gee.HashSet exclude_folders = new Gee.HashSet(); private Geary.SpecialFolderType[] exclude_types = { Geary.SpecialFolderType.SPAM, @@ -169,10 +170,6 @@ public class Geary.SearchFolder : Geary.AbstractLocalFolder { return Geary.SpecialFolderType.SEARCH; } - public override Geary.FolderProperties get_properties() { - return properties; - } - public override async Gee.List? list_email_async(int low, int count, Geary.Email.Field required_fields, Folder.ListFlags flags, Cancellable? cancellable = null) throws Error { diff --git a/src/engine/common/common-message-data.vala b/src/engine/common/common-message-data.vala index 95349346..ea79fb4e 100644 --- a/src/engine/common/common-message-data.vala +++ b/src/engine/common/common-message-data.vala @@ -142,15 +142,15 @@ public abstract class Geary.MessageData.Int64MessageData : AbstractMessageData, public abstract class Geary.MessageData.BlockMessageData : AbstractMessageData { public string data_name { get; private set; } - public Geary.Memory.AbstractBuffer buffer { get; private set; } + public Geary.Memory.Buffer buffer { get; private set; } - public BlockMessageData(string data_name, Geary.Memory.AbstractBuffer buffer) { + public BlockMessageData(string data_name, Geary.Memory.Buffer buffer) { this.data_name = data_name; this.buffer = buffer; } public override string to_string() { - return "%s (%lub)".printf(data_name, buffer.get_size()); + return "%s (%lub)".printf(data_name, buffer.size); } } diff --git a/src/engine/db/db-result.vala b/src/engine/db/db-result.vala index dd6f8bc8..b9ab679c 100644 --- a/src/engine/db/db-result.vala +++ b/src/engine/db/db-result.vala @@ -124,6 +124,18 @@ public class Geary.Db.Result : Geary.Db.Context { return s; } + /** + * column is zero-based. + */ + public Memory.Buffer string_buffer_at(int column) throws DatabaseError { + // Memory.StringBuffer is not entirely suited for this, as it can result in extra copies + // internally ... GrowableBuffer is better for large blocks + Memory.GrowableBuffer buffer = new Memory.GrowableBuffer(); + buffer.append(string_at(column).data); + + return buffer; + } + private void verify_at(int column) throws DatabaseError { if (finished) throw new DatabaseError.FINISHED("Query finished"); @@ -204,6 +216,14 @@ public class Geary.Db.Result : Geary.Db.Context { return string_at(convert_for(name)); } + /** + * name is the name of the column in the result set. See Statement.get_column_index() for name + * matching rules. + */ + public Memory.Buffer string_buffer_for(string name) throws DatabaseError { + return string_buffer_at(convert_for(name)); + } + private int convert_for(string name) throws DatabaseError { if (finished) throw new DatabaseError.FINISHED("Query finished"); diff --git a/src/engine/db/db-statement.vala b/src/engine/db/db-statement.vala index 4f7209f4..a9117a9a 100644 --- a/src/engine/db/db-statement.vala +++ b/src/engine/db/db-statement.vala @@ -31,6 +31,8 @@ public class Geary.Db.Statement : Geary.Db.Context { */ public signal void bindings_cleared(); + private Gee.HashSet held_buffers = new Gee.HashSet(); + internal Statement(Connection connection, string sql) throws DatabaseError { this.connection = connection; // save for logging in case prepare_v2() fails @@ -229,6 +231,38 @@ public class Geary.Db.Statement : Geary.Db.Context { return this; } + /** + * Binds the string representation of a {@link Memory.Buffer} to the replacement value + * in the {@link Statement}. + * + * If buffer supports {@link Memory.UnownedStringBuffer}, the unowned string will be used + * to avoid a memory copy. However, this means the Statement will hold a reference to the + * buffer until the Statement is destroyed. + * + * index is zero-based. + */ + public Statement bind_string_buffer(int index, Memory.Buffer? buffer) throws DatabaseError { + if (buffer == null) + return bind_string(index, null); + + Memory.UnownedStringBuffer? unowned_buffer = buffer as Memory.UnownedStringBuffer; + if (unowned_buffer == null) { + throw_on_error("Statement.bind_string_buffer", stmt.bind_text(index + 1, buffer.to_string())); + + return this; + } + + // hold on to buffer for lifetime of Statement, SQLite's callback isn't enough for us to + // selectively unref each Buffer as it's done with it + held_buffers.add(unowned_buffer); + + // note use of _bind_text, which is for static and other strings with their own memory + // management + stmt._bind_text(index + 1, unowned_buffer.to_unowned_string()); + + return this; + } + public override Statement? get_statement() { return this; } diff --git a/src/engine/imap-db/imap-db-account.vala b/src/engine/imap-db/imap-db-account.vala index 341c85da..2a67086e 100644 --- a/src/engine/imap-db/imap-db-account.vala +++ b/src/engine/imap-db/imap-db-account.vala @@ -135,7 +135,7 @@ private class Geary.ImapDB.Account : BaseObject { // create the folder object Db.Statement stmt = cx.prepare( "INSERT INTO FolderTable (name, parent_id, last_seen_total, last_seen_status_total, " - + "uid_validity, uid_next, attributes) VALUES (?, ?, ?, ?, ?, ?, ?)"); + + "uid_validity, uid_next, attributes, unread_count) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"); stmt.bind_string(0, path.basename); stmt.bind_rowid(1, parent_id); stmt.bind_int(2, Numeric.int_floor(properties.select_examine_messages, 0)); @@ -145,6 +145,7 @@ private class Geary.ImapDB.Account : BaseObject { stmt.bind_int64(5, (properties.uid_next != null) ? properties.uid_next.value : Imap.UID.INVALID); stmt.bind_string(6, properties.attrs.serialize()); + stmt.bind_int(7, properties.email_unread); stmt.exec(cancellable); @@ -174,15 +175,17 @@ private class Geary.ImapDB.Account : BaseObject { Db.Statement stmt; if (parent_id != Db.INVALID_ROWID) { stmt = cx.prepare( - "UPDATE FolderTable SET attributes=? WHERE parent_id=? AND name=?"); + "UPDATE FolderTable SET attributes=?, unread_count=? WHERE parent_id=? AND name=?"); stmt.bind_string(0, properties.attrs.serialize()); - stmt.bind_rowid(1, parent_id); - stmt.bind_string(2, path.basename); + stmt.bind_int(1, properties.email_unread); + stmt.bind_rowid(2, parent_id); + stmt.bind_string(3, path.basename); } else { stmt = cx.prepare( - "UPDATE FolderTable SET attributes=? WHERE parent_id IS NULL AND name=?"); + "UPDATE FolderTable SET attributes=?, unread_count=? WHERE parent_id IS NULL AND name=?"); stmt.bind_string(0, properties.attrs.serialize()); - stmt.bind_string(1, path.basename); + stmt.bind_int(1, properties.email_unread); + stmt.bind_string(2, path.basename); } stmt.exec(); @@ -200,7 +203,7 @@ private class Geary.ImapDB.Account : BaseObject { if (db_folder != null) { Imap.FolderProperties local_properties = db_folder.get_properties(); - local_properties.unseen = properties.unseen; + local_properties.set_status_unseen(properties.unseen); local_properties.recent = properties.recent; local_properties.attrs = properties.attrs; @@ -264,7 +267,7 @@ private class Geary.ImapDB.Account : BaseObject { if (db_folder != null) { Imap.FolderProperties local_properties = db_folder.get_properties(); - local_properties.unseen = properties.unseen; + local_properties.set_status_unseen(properties.unseen); local_properties.recent = properties.recent; local_properties.uid_validity = properties.uid_validity; local_properties.uid_next = properties.uid_next; @@ -349,7 +352,7 @@ private class Geary.ImapDB.Account : BaseObject { : new Geary.FolderRoot(basename, "/", Geary.Imap.Folder.CASE_SENSITIVE); Geary.Imap.FolderProperties properties = new Geary.Imap.FolderProperties( - result.int_for("last_seen_total"), 0, 0, + result.int_for("last_seen_total"), 0, new Imap.UIDValidity(result.int64_for("uid_validity")), new Imap.UID(result.int64_for("uid_next")), Geary.Imap.MailboxAttributes.deserialize(result.string_for("attributes"))); @@ -436,7 +439,7 @@ private class Geary.ImapDB.Account : BaseObject { Db.Result results = stmt.exec(cancellable); if (!results.finished) { - properties = new Imap.FolderProperties(results.int_for("last_seen_total"), 0, 0, + properties = new Imap.FolderProperties(results.int_for("last_seen_total"), 0, new Imap.UIDValidity(results.int64_for("uid_validity")), new Imap.UID(results.int64_for("uid_next")), Geary.Imap.MailboxAttributes.deserialize(results.string_for("attributes"))); diff --git a/src/engine/imap-db/imap-db-database.vala b/src/engine/imap-db/imap-db-database.vala index 28787d79..d44a8f97 100644 --- a/src/engine/imap-db/imap-db-database.vala +++ b/src/engine/imap-db/imap-db-database.vala @@ -35,11 +35,11 @@ private class Geary.ImapDB.Database : Geary.Db.VersionedDatabase { post_upgrade_encode_folder_names(); break; - case 10: + case 11: post_upgrade_add_search_table(); break; - case 11: + case 12: post_upgrade_populate_internal_date_time_t(); break; } @@ -87,7 +87,7 @@ private class Geary.ImapDB.Database : Geary.Db.VersionedDatabase { } } - // Version 10. + // Version 11. private void post_upgrade_add_search_table() { try { string stemmer = find_appropriate_search_stemmer(); @@ -149,7 +149,7 @@ private class Geary.ImapDB.Database : Geary.Db.VersionedDatabase { return "english"; } - // Version 11. + // Version 12. private void post_upgrade_populate_internal_date_time_t() { try { exec_transaction(Db.TransactionType.RW, (cx) => { diff --git a/src/engine/imap-db/imap-db-folder.vala b/src/engine/imap-db/imap-db-folder.vala index 3fb4fad4..fb32c5d9 100644 --- a/src/engine/imap-db/imap-db-folder.vala +++ b/src/engine/imap-db/imap-db-folder.vala @@ -920,8 +920,8 @@ private class Geary.ImapDB.Folder : BaseObject, Geary.ReferenceSemantics { stmt.bind_string(10, row.in_reply_to); stmt.bind_string(11, row.references); stmt.bind_string(12, row.subject); - stmt.bind_string(13, row.header); - stmt.bind_string(14, row.body); + stmt.bind_string_buffer(13, row.header); + stmt.bind_string_buffer(14, row.body); stmt.bind_string(15, row.preview); stmt.bind_string(16, row.email_flags); stmt.bind_string(17, row.internaldate); @@ -1256,7 +1256,7 @@ private class Geary.ImapDB.Folder : BaseObject, Geary.ReferenceSemantics { if (new_fields.is_any_set(Geary.Email.Field.HEADER)) { Db.Statement stmt = cx.prepare( "UPDATE MessageTable SET header=? WHERE id=?"); - stmt.bind_string(0, row.header); + stmt.bind_string_buffer(0, row.header); stmt.bind_rowid(1, row.id); stmt.exec(cancellable); @@ -1265,7 +1265,7 @@ private class Geary.ImapDB.Folder : BaseObject, Geary.ReferenceSemantics { if (new_fields.is_any_set(Geary.Email.Field.BODY)) { Db.Statement stmt = cx.prepare( "UPDATE MessageTable SET body=? WHERE id=?"); - stmt.bind_string(0, row.body); + stmt.bind_string_buffer(0, row.body); stmt.bind_rowid(1, row.id); stmt.exec(cancellable); diff --git a/src/engine/imap-db/imap-db-message-row.vala b/src/engine/imap-db/imap-db-message-row.vala index 235771e8..32bb370f 100644 --- a/src/engine/imap-db/imap-db-message-row.vala +++ b/src/engine/imap-db/imap-db-message-row.vala @@ -25,9 +25,9 @@ private class Geary.ImapDB.MessageRow { public string? subject { get; set; default = null; } - public string? header { get; set; default = null; } + public Memory.Buffer? header { get; set; default = null; } - public string? body { get; set; default = null; } + public Memory.Buffer? body { get; set; default = null; } public string? preview { get; set; default = null; } @@ -79,10 +79,10 @@ private class Geary.ImapDB.MessageRow { subject = results.string_for("subject"); if (fields.is_all_set(Geary.Email.Field.HEADER)) - header = results.string_for("header"); + header = results.string_buffer_for("header"); if (fields.is_all_set(Geary.Email.Field.BODY)) - body = results.string_for("body"); + body = results.string_buffer_for("body"); if (fields.is_all_set(Geary.Email.Field.PREVIEW)) preview = results.string_for("preview"); @@ -127,10 +127,10 @@ private class Geary.ImapDB.MessageRow { email.set_message_subject(new RFC822.Subject.decode(subject ?? "")); if (fields.is_all_set(Geary.Email.Field.HEADER)) - email.set_message_header(new RFC822.Header(new Geary.Memory.StringBuffer(header ?? ""))); + email.set_message_header(new RFC822.Header(header ?? Memory.EmptyBuffer.instance)); if (fields.is_all_set(Geary.Email.Field.BODY)) - email.set_message_body(new RFC822.Text(new Geary.Memory.StringBuffer(body ?? ""))); + email.set_message_body(new RFC822.Text(body ?? Memory.EmptyBuffer.instance)); if (fields.is_all_set(Geary.Email.Field.PREVIEW)) email.set_message_preview(new RFC822.PreviewText(new Geary.Memory.StringBuffer(preview ?? ""))); @@ -217,13 +217,13 @@ private class Geary.ImapDB.MessageRow { } if (email.fields.is_all_set(Geary.Email.Field.HEADER)) { - header = (email.header != null) ? email.header.buffer.to_string() : null; + header = (email.header != null) ? email.header.buffer : null; fields = fields.set(Geary.Email.Field.HEADER); } if (email.fields.is_all_set(Geary.Email.Field.BODY)) { - body = (email.body != null) ? email.body.buffer.to_string() : null; + body = (email.body != null) ? email.body.buffer : null; fields = fields.set(Geary.Email.Field.BODY); } diff --git a/src/engine/imap-db/outbox/smtp-outbox-folder.vala b/src/engine/imap-db/outbox/smtp-outbox-folder.vala index 1de27aee..e762b633 100644 --- a/src/engine/imap-db/outbox/smtp-outbox-folder.vala +++ b/src/engine/imap-db/outbox/smtp-outbox-folder.vala @@ -17,10 +17,10 @@ private class Geary.SmtpOutboxFolder : Geary.AbstractLocalFolder, Geary.FolderSu public int64 id; public int position; public int64 ordering; - public string? message; + public Memory.Buffer? message; public SmtpOutboxEmailIdentifier outbox_id; - public OutboxRow(int64 id, int position, int64 ordering, string? message) { + public OutboxRow(int64 id, int position, int64 ordering, Memory.Buffer? message) { assert(position >= 1); this.id = id; @@ -44,10 +44,12 @@ private class Geary.SmtpOutboxFolder : Geary.AbstractLocalFolder, Geary.FolderSu private weak Account _account; private Geary.Smtp.ClientSession smtp; private Nonblocking.Mailbox outbox_queue = new Nonblocking.Mailbox(); - private SmtpOutboxFolderProperties properties = new SmtpOutboxFolderProperties(0, 0); + private SmtpOutboxFolderProperties _properties = new SmtpOutboxFolderProperties(0, 0); public override Account account { get { return _account; } } + public override FolderProperties properties { get { return _properties; } } + // Requires the Database from the get-go because it runs a background task that access it // whether open or not public SmtpOutboxFolder(ImapDB.Database db, Account account) { @@ -81,7 +83,7 @@ private class Geary.SmtpOutboxFolder : Geary.AbstractLocalFolder, Geary.FolderSu int position = 1; while (!results.finished) { list.add(new OutboxRow(results.rowid_at(0), position++, results.int64_at(1), - results.string_at(2))); + results.string_buffer_at(2))); results.next(cancellable); } @@ -90,7 +92,7 @@ private class Geary.SmtpOutboxFolder : Geary.AbstractLocalFolder, Geary.FolderSu if (list.size > 0) { // set properties now (can't do yield in ctor) - properties.set_total(list.size); + _properties.set_total(list.size); debug("Priming outbox postman with %d stored messages", list.size); foreach (OutboxRow row in list) @@ -115,7 +117,7 @@ private class Geary.SmtpOutboxFolder : Geary.AbstractLocalFolder, Geary.FolderSu // Convert row into RFC822 message suitable for sending or framing RFC822.Message message; try { - message = new RFC822.Message.from_string(row.message); + message = new RFC822.Message.from_buffer(row.message); } catch (RFC822Error msg_err) { // TODO: This needs to be reported to the user debug("Outbox postman message error: %s", msg_err.message); @@ -177,7 +179,7 @@ private class Geary.SmtpOutboxFolder : Geary.AbstractLocalFolder, Geary.FolderSu // update properties try { - properties.set_total(yield get_email_count_async(null)); + _properties.set_total(yield get_email_count_async(null)); } catch (Error err) { debug("Outbox postman: Unable to fetch updated email count for properties: %s", err.message); @@ -197,10 +199,6 @@ private class Geary.SmtpOutboxFolder : Geary.AbstractLocalFolder, Geary.FolderSu return path; } - public override Geary.FolderProperties get_properties() { - return properties; - } - public override Geary.SpecialFolderType get_special_folder_type() { return Geary.SpecialFolderType.OUTBOX; } @@ -243,7 +241,7 @@ private class Geary.SmtpOutboxFolder : Geary.AbstractLocalFolder, Geary.FolderSu assert(!results.finished); int64 ordering = results.int64_at(0); - string message = results.string_at(1); + Memory.Buffer message = results.string_buffer_at(1); int position = do_get_position_by_ordering(cx, ordering, cancellable); @@ -257,7 +255,7 @@ private class Geary.SmtpOutboxFolder : Geary.AbstractLocalFolder, Geary.FolderSu assert(row != null); // update properties - properties.set_total(yield get_email_count_async(cancellable)); + _properties.set_total(yield get_email_count_async(cancellable)); // immediately add to outbox queue for delivery outbox_queue.send(row); @@ -309,7 +307,7 @@ private class Geary.SmtpOutboxFolder : Geary.AbstractLocalFolder, Geary.FolderSu int position = low; do { list.add(row_to_email(new OutboxRow(results.rowid_at(0), position++, results.int64_at(1), - results.string_at(2)))); + results.string_buffer_at(2)))); } while (results.next()); return Db.TransactionOutcome.DONE; @@ -354,7 +352,7 @@ private class Geary.SmtpOutboxFolder : Geary.AbstractLocalFolder, Geary.FolderSu } list.add(row_to_email(new OutboxRow(results.rowid_at(0), position++, ordering, - results.string_at(2)))); + results.string_buffer_at(2)))); } while (results.next()); return Db.TransactionOutcome.DONE; @@ -489,7 +487,7 @@ private class Geary.SmtpOutboxFolder : Geary.AbstractLocalFolder, Geary.FolderSu // Utility for getting an email object back from an outbox row. private Geary.Email row_to_email(OutboxRow row) throws Error { - RFC822.Message message = new RFC822.Message.from_string(row.message); + RFC822.Message message = new RFC822.Message.from_buffer(row.message); Geary.Email email = message.get_email(row.position, row.outbox_id); // TODO: Determine message's total size (header + body) to store in Properties. @@ -574,7 +572,7 @@ private class Geary.SmtpOutboxFolder : Geary.AbstractLocalFolder, Geary.FolderSu if (position < 1) return null; - return new OutboxRow(results.rowid_at(0), position, ordering, results.string_at(1)); + return new OutboxRow(results.rowid_at(0), position, ordering, results.string_buffer_at(1)); } private bool do_remove_email(Db.Connection cx, SmtpOutboxEmailIdentifier id, Cancellable? cancellable) diff --git a/src/engine/imap-engine/imap-engine-account-synchronizer.vala b/src/engine/imap-engine/imap-engine-account-synchronizer.vala index 604dfa2c..a4fc709d 100644 --- a/src/engine/imap-engine/imap-engine-account-synchronizer.vala +++ b/src/engine/imap-engine/imap-engine-account-synchronizer.vala @@ -227,7 +227,7 @@ private class Geary.ImapEngine.AccountSynchronizer : Geary.BaseObject { debug("Oldest local email in %s not old enough (%s vs. %s), synchronizing...", folder.to_string(), oldest_local.to_string(), epoch.to_string()); } - } else if (folder.get_properties().email_total == 0) { + } else if (folder.properties.email_total == 0) { // no local messages, no remote messages -- this is as good as having everything up // to the epoch return true; diff --git a/src/engine/imap-engine/imap-engine-generic-folder.vala b/src/engine/imap-engine/imap-engine-generic-folder.vala index af6658a5..1b7b1735 100644 --- a/src/engine/imap-engine/imap-engine-generic-folder.vala +++ b/src/engine/imap-engine/imap-engine-generic-folder.vala @@ -15,12 +15,14 @@ private class Geary.ImapEngine.GenericFolder : Geary.AbstractFolder, Geary.Folde Geary.Email.Field.PROPERTIES | ImapDB.Folder.REQUIRED_FOR_DUPLICATE_DETECTION; public override Account account { get { return _account; } } + public override FolderProperties properties { get { return _properties; } } internal ImapDB.Folder local_folder { get; protected set; } internal Imap.Folder? remote_folder { get; protected set; default = null; } internal EmailPrefetcher email_prefetcher { get; private set; } internal EmailFlagWatcher email_flag_watcher; private weak GenericAccount _account; + private Geary.AggregatedFolderProperties _properties = new Geary.AggregatedFolderProperties(); private Imap.Account remote; private ImapDB.Account local; private SpecialFolderType special_folder_type; @@ -37,6 +39,7 @@ private class Geary.ImapEngine.GenericFolder : Geary.AbstractFolder, Geary.Folde this.local = local; this.local_folder = local_folder; this.special_folder_type = special_folder_type; + _properties.add(local_folder.get_properties()); email_flag_watcher = new EmailFlagWatcher(this); email_flag_watcher.email_flags_changed.connect(on_email_flags_changed); @@ -53,16 +56,6 @@ private class Geary.ImapEngine.GenericFolder : Geary.AbstractFolder, Geary.Folde return local_folder.get_path(); } - public override Geary.FolderProperties get_properties() { - // Get properties in order of authoritativeness: - // - From open remote folder - // - Fetch from local store - if (remote_folder != null && get_open_state() == OpenState.BOTH) - return remote_folder.properties; - - return local_folder.get_properties(); - } - public override Geary.SpecialFolderType get_special_folder_type() { return special_folder_type; } @@ -380,7 +373,7 @@ private class Geary.ImapEngine.GenericFolder : Geary.AbstractFolder, Geary.Folde CreateLocalEmailOperation? create_op = null; if (to_create_or_merge.size > 0) { create_op = new CreateLocalEmailOperation(local_folder, to_create_or_merge, - NORMALIZATION_FIELDS); + normalization_fields); batch.add(create_op); } @@ -581,6 +574,8 @@ private class Geary.ImapEngine.GenericFolder : Geary.AbstractFolder, Geary.Folde return; } + _properties.add(remote_folder.properties); + // notify any subscribers with similar information notify_opened( (remote_folder != null) ? Geary.Folder.OpenState.BOTH : Geary.Folder.OpenState.LOCAL, @@ -591,6 +586,7 @@ private class Geary.ImapEngine.GenericFolder : Geary.AbstractFolder, Geary.Folde if (open_count == 0 || --open_count > 0) return; + _properties.remove(remote_folder.properties); yield close_internal_async(CloseReason.LOCAL_CLOSE, CloseReason.REMOTE_CLOSE, cancellable); } diff --git a/src/engine/imap/api/imap-folder-properties.vala b/src/engine/imap/api/imap-folder-properties.vala index cfcbb410..01a62be9 100644 --- a/src/engine/imap/api/imap-folder-properties.vala +++ b/src/engine/imap/api/imap-folder-properties.vala @@ -45,26 +45,33 @@ public class Geary.Imap.FolderProperties : Geary.FolderProperties { */ public int select_examine_messages { get; private set; } /** - * -1 if the FolderProperties were not obtained via a STATUS command + * -1 if the FolderProperties were not obtained or updated via a STATUS command */ public int status_messages { get; private set; } - public int unseen { get; internal set; } + /** + * -1 if the FolderProperties were not obtained or updated via a STATUS command + */ + public int unseen { get; private set; } public int recent { get; internal set; } public UIDValidity? uid_validity { get; internal set; } public UID? uid_next { get; internal set; } public MailboxAttributes attrs { get; internal set; } - // Note that unseen from SELECT/EXAMINE is the *position* of the first unseen message, - // not the total unseen count, so it should not be passed in here, but rather the unseen - // count from a STATUS command - public FolderProperties(int messages, int recent, int unseen, UIDValidity? uid_validity, + /** + * Note that unseen from SELECT/EXAMINE is the *position* of the first unseen message, + * not the total unseen count, so it's not be passed in here, but rather only from the unseen + * count from a STATUS command + */ + public FolderProperties(int messages, int recent, UIDValidity? uid_validity, UID? uid_next, MailboxAttributes attrs) { - base (messages, unseen, Trillian.UNKNOWN, Trillian.UNKNOWN, Trillian.UNKNOWN); + // give the base class a zero email_unread, as the notion of "unknown" doesn't exist in + // its contract + base (messages, 0, Trillian.UNKNOWN, Trillian.UNKNOWN, Trillian.UNKNOWN); select_examine_messages = messages; status_messages = -1; this.recent = recent; - this.unseen = unseen; + this.unseen = -1; this.uid_validity = uid_validity; this.uid_next = uid_next; this.attrs = attrs; @@ -159,5 +166,16 @@ public class Geary.Imap.FolderProperties : Geary.FolderProperties { // select/examine more authoritative than status email_total = messages; } + + public void set_status_unseen(int count) { + // drop unknown counts, especially if known is held here + if (count < 0) + return; + + unseen = count; + + // update base class value (which clients see) + email_unread = count; + } } diff --git a/src/engine/imap/api/imap-folder.vala b/src/engine/imap/api/imap-folder.vala index ad0c0ef2..ca44b7ae 100644 --- a/src/engine/imap/api/imap-folder.vala +++ b/src/engine/imap/api/imap-folder.vala @@ -63,7 +63,7 @@ private class Geary.Imap.Folder : BaseObject { this.info = info; path = info.mailbox.to_folder_path(info.delim); - properties = new Imap.FolderProperties(0, 0, 0, null, null, info.attrs); + properties = new Imap.FolderProperties(0, 0, null, null, info.attrs); } public async void open_async(Cancellable? cancellable) throws Error { @@ -206,7 +206,9 @@ private class Geary.Imap.Folder : BaseObject { break; case ResponseCodeType.UNSEEN: - properties.unseen = response_code.get_unseen(); + // do NOT update properties.unseen, as the UNSEEN response code (here) means + // the sequence number of the first unseen message, not the total count of + // unseen messages break; case ResponseCodeType.PERMANENT_FLAGS: diff --git a/src/engine/imap/command/imap-append-command.vala b/src/engine/imap/command/imap-append-command.vala new file mode 100644 index 00000000..10081ed9 --- /dev/null +++ b/src/engine/imap/command/imap-append-command.vala @@ -0,0 +1,31 @@ +/* Copyright 2013 Yorba Foundation + * + * This software is licensed under the GNU Lesser General Public License + * (version 2.1 or later). See the COPYING file in this distribution. + */ + +/** + * A representation of the IMAP APPEND command. + * + * See [[http://tools.ietf.org/html/rfc3501#section-6.3.11]] + */ + +public class Geary.Imap.AppendCommand : Command { + public const string NAME = "append"; + + public AppendCommand(MailboxSpecifier mailbox, MessageFlags? flags, InternalDate? internal_date, + Memory.Buffer message) { + base (NAME); + + add(mailbox.to_parameter()); + + if (flags != null && flags.size > 0) + add(flags.to_parameter()); + + if (internal_date != null) + add(internal_date.to_parameter()); + + add(new LiteralParameter(message)); + } +} + diff --git a/src/engine/imap/command/imap-command.vala b/src/engine/imap/command/imap-command.vala index 77c68d76..b86adee3 100644 --- a/src/engine/imap/command/imap-command.vala +++ b/src/engine/imap/command/imap-command.vala @@ -79,7 +79,7 @@ public class Geary.Imap.Command : RootParameters { if (stringp != null) add(stringp); else - error("Command continuations currently unsupported"); + add(new LiteralParameter(new Memory.StringBuffer(arg))); } } } @@ -110,10 +110,11 @@ public class Geary.Imap.Command : RootParameters { return this.name.down() == name.down(); } - public override async void serialize(Serializer ser) throws Error { + public override void serialize(Serializer ser, Tag tag) throws Error { assert(tag.is_assigned()); - yield base.serialize(ser); + base.serialize(ser, tag); + ser.push_end_of_message(); } } diff --git a/src/engine/imap/message/imap-flag.vala b/src/engine/imap/message/imap-flag.vala index 6f6d699a..ff8c7e99 100644 --- a/src/engine/imap/message/imap-flag.vala +++ b/src/engine/imap/message/imap-flag.vala @@ -32,6 +32,13 @@ public abstract class Geary.Imap.Flag : BaseObject, Gee.Hashable body_data_map { get; private set; - default = new Gee.HashMap(); } + public Gee.Map body_data_map { get; private set; + default = new Gee.HashMap(); } public FetchedData(SequenceNumber seq_num) { this.seq_num = seq_num; @@ -113,9 +113,9 @@ public class Geary.Imap.FetchedData : Object { FetchedData combined = new FetchedData(seq_num); Collection.map_set_all(combined.data_map, data_map); Collection.map_set_all(combined.data_map, other.data_map); - Collection.map_set_all(combined.body_data_map, + Collection.map_set_all(combined.body_data_map, body_data_map); - Collection.map_set_all(combined.body_data_map, + Collection.map_set_all(combined.body_data_map, other.body_data_map); return combined; @@ -130,7 +130,7 @@ public class Geary.Imap.FetchedData : Object { builder.append_printf("%s=%s ", data_type.to_string(), data_map.get(data_type).to_string()); foreach (FetchBodyDataIdentifier identifier in body_data_map.keys) - builder.append_printf("%s=%lu ", identifier.to_string(), body_data_map.get(identifier).get_size()); + builder.append_printf("%s=%lu ", identifier.to_string(), body_data_map.get(identifier).size); return builder.str; } diff --git a/src/engine/imap/response/imap-response-code.vala b/src/engine/imap/response/imap-response-code.vala index ddae183e..7a1ec5dc 100644 --- a/src/engine/imap/response/imap-response-code.vala +++ b/src/engine/imap/response/imap-response-code.vala @@ -94,9 +94,9 @@ public class Geary.Imap.ResponseCode : Geary.Imap.ListParameter { return "[%s]".printf(stringize_list()); } - public override async void serialize(Serializer ser) throws Error { + public override void serialize(Serializer ser, Tag tag) throws Error { ser.push_ascii('['); - yield serialize_list(ser); + serialize_list(ser, tag); ser.push_ascii(']'); } } diff --git a/src/engine/imap/transport/imap-client-connection.vala b/src/engine/imap/transport/imap-client-connection.vala index 892147bd..cc4561d6 100644 --- a/src/engine/imap/transport/imap-client-connection.vala +++ b/src/engine/imap/transport/imap-client-connection.vala @@ -38,6 +38,8 @@ public class Geary.Imap.ClientConnection : BaseObject { IDLING, IDLE, DEIDLING, + DEIDLING_SYNCHRONIZING, + SYNCHRONIZING, DISCONNECTED, COUNT @@ -56,6 +58,9 @@ public class Geary.Imap.ClientConnection : BaseObject { SEND, SEND_IDLE, + // To initiate a command continuation request + SYNCHRONIZE, + // RECVD_* will emit appropriate signals inside their transition handlers; do *not* use // issue_conditional_event() for these events RECVD_STATUS_RESPONSE, @@ -99,11 +104,14 @@ public class Geary.Imap.ClientConnection : BaseObject { private Serializer? ser = null; private Deserializer? des = null; private Geary.Nonblocking.Mutex send_mutex = new Geary.Nonblocking.Mutex(); + private Geary.Nonblocking.Spinlock synchronized_notifier = new Geary.Nonblocking.Spinlock(); private int tag_counter = 0; private char tag_prefix = 'a'; private uint flush_timeout_id = 0; private bool idle_when_quiet = false; private Gee.HashSet posted_idle_tags = new Gee.HashSet(); + private Tag? posted_synchronization_tag = null; + private StatusResponse? synchronization_status_response = null; private uint timeout_id = 0; private uint timeout_cmd_count = 0; @@ -173,10 +181,11 @@ public class Geary.Imap.ClientConnection : BaseObject { Geary.State.Mapping[] mappings = { new Geary.State.Mapping(State.UNCONNECTED, Event.CONNECTED, on_connected), - new Geary.State.Mapping(State.UNCONNECTED, Event.DISCONNECTED, Geary.State.nop), + new Geary.State.Mapping(State.UNCONNECTED, Event.DISCONNECTED, on_disconnected), new Geary.State.Mapping(State.CONNECTED, Event.SEND, on_proceed), new Geary.State.Mapping(State.CONNECTED, Event.SEND_IDLE, on_send_idle), + new Geary.State.Mapping(State.CONNECTED, Event.SYNCHRONIZE, on_synchronize), new Geary.State.Mapping(State.CONNECTED, Event.RECVD_STATUS_RESPONSE, on_status_response), new Geary.State.Mapping(State.CONNECTED, Event.RECVD_SERVER_DATA, on_server_data), new Geary.State.Mapping(State.CONNECTED, Event.RECVD_CONTINUATION_RESPONSE, on_continuation), @@ -198,11 +207,30 @@ public class Geary.Imap.ClientConnection : BaseObject { new Geary.State.Mapping(State.DEIDLING, Event.SEND, on_proceed), new Geary.State.Mapping(State.DEIDLING, Event.SEND_IDLE, on_send_idle), + new Geary.State.Mapping(State.DEIDLING, Event.SYNCHRONIZE, on_deidling_synchronize), new Geary.State.Mapping(State.DEIDLING, Event.RECVD_STATUS_RESPONSE, on_idle_status_response), new Geary.State.Mapping(State.DEIDLING, Event.RECVD_SERVER_DATA, on_server_data), new Geary.State.Mapping(State.DEIDLING, Event.RECVD_CONTINUATION_RESPONSE, on_idling_continuation), new Geary.State.Mapping(State.DEIDLING, Event.DISCONNECTED, on_disconnected), + new Geary.State.Mapping(State.DEIDLING_SYNCHRONIZING, Event.SEND, on_no_proceed), + new Geary.State.Mapping(State.DEIDLING_SYNCHRONIZING, Event.SEND_IDLE, on_no_proceed), + new Geary.State.Mapping(State.DEIDLING_SYNCHRONIZING, Event.RECVD_STATUS_RESPONSE, + on_deidling_synchronizing_status_response), + new Geary.State.Mapping(State.DEIDLING_SYNCHRONIZING, Event.RECVD_SERVER_DATA, on_server_data), + new Geary.State.Mapping(State.DEIDLING_SYNCHRONIZING, Event.RECVD_CONTINUATION_RESPONSE, + on_synchronize_continuation), + new Geary.State.Mapping(State.DEIDLING_SYNCHRONIZING, Event.DISCONNECTED, on_disconnected), + + new Geary.State.Mapping(State.SYNCHRONIZING, Event.SEND, on_no_proceed), + new Geary.State.Mapping(State.SYNCHRONIZING, Event.SEND_IDLE, on_no_proceed), + new Geary.State.Mapping(State.SYNCHRONIZING, Event.RECVD_STATUS_RESPONSE, + on_synchronize_status_response), + new Geary.State.Mapping(State.SYNCHRONIZING, Event.RECVD_SERVER_DATA, on_server_data), + new Geary.State.Mapping(State.SYNCHRONIZING, Event.RECVD_CONTINUATION_RESPONSE, + on_synchronize_continuation), + new Geary.State.Mapping(State.SYNCHRONIZING, Event.DISCONNECTED, on_disconnected), + // TODO: A DISCONNECTING state would be helpful here, allowing for responses and data // received from the server after a send error caused a disconnect to be signalled to // subscribers before moving to the DISCONNECTED state. That would require more work, @@ -210,6 +238,7 @@ public class Geary.Imap.ClientConnection : BaseObject { // everything to flush out before it shifted to a DISCONNECTED state as well. new Geary.State.Mapping(State.DISCONNECTED, Event.SEND, on_no_proceed), new Geary.State.Mapping(State.DISCONNECTED, Event.SEND_IDLE, on_no_proceed), + new Geary.State.Mapping(State.DISCONNECTED, Event.SYNCHRONIZE, on_no_proceed), new Geary.State.Mapping(State.DISCONNECTED, Event.RECVD_STATUS_RESPONSE, Geary.State.nop), new Geary.State.Mapping(State.DISCONNECTED, Event.RECVD_SERVER_DATA, Geary.State.nop), new Geary.State.Mapping(State.DISCONNECTED, Event.RECVD_CONTINUATION_RESPONSE, Geary.State.nop), @@ -388,9 +417,8 @@ public class Geary.Imap.ClientConnection : BaseObject { assert(ser == null); assert(des == null); - // not buffering the Serializer because it buffers using a MemoryOutputStream and not - // buffering the Deserializer because it uses a DataInputStream, which is buffered - ser = new Serializer(to_string(), ios.output_stream); + // Not buffering the Deserializer because it uses a DataInputStream, which is buffered + ser = new Serializer(to_string(), new BufferedOutputStream(ios.output_stream)); des = new Deserializer(to_string(), ios.input_stream); des.parameters_ready.connect(on_parameters_ready); @@ -415,7 +443,6 @@ public class Geary.Imap.ClientConnection : BaseObject { yield des.stop_async(); } - // TODO: May need to commit Serializer before disconnecting ser = null; des = null; } @@ -500,7 +527,6 @@ public class Geary.Imap.ClientConnection : BaseObject { recv_closed(); } - // TODO: Guard against reentrancy public async void send_async(Command cmd, Cancellable? cancellable = null) throws Error { check_for_connection(); @@ -511,8 +537,8 @@ public class Geary.Imap.ClientConnection : BaseObject { fsm.get_state_string(fsm.get_state())); } - // need to run this in critical section because OutputStreams can only be written to - // serially + // need to run this in critical section because Serializer requires it (don't want to be + // pushing data while a flush_async() is occurring) int token = yield send_mutex.claim_async(cancellable); // Always assign a new tag; Commands with pre-assigned Tags should not be re-sent. @@ -528,9 +554,7 @@ public class Geary.Imap.ClientConnection : BaseObject { try { // watch for disconnect while waiting for mutex if (ser != null) { - // TODO: Make serialize non-blocking; this would also remove the need for a send_mutex - // (although reentrancy should still be checked for) - yield cmd.serialize(ser); + cmd.serialize(ser, cmd.tag); } else { ser_err = new ImapError.NOT_CONNECTED("Send not allowed: connection in %s state", fsm.get_state_string(fsm.get_state())); @@ -582,7 +606,7 @@ public class Geary.Imap.ClientConnection : BaseObject { // need to signal when the IDLE command is sent, for completeness IdleCommand? idle_cmd = null; - // Like send_async(), need to use mutex when flushing as OutputStream must be accessed in + // Like send_async(), need to use mutex when flushing as Serializer must be accessed in // serialized fashion // // NOTE: Because this is happening in the background, it's possible for ser to go to null @@ -595,9 +619,54 @@ public class Geary.Imap.ClientConnection : BaseObject { // Dovecot will hang the connection (not send any replies) if IDLE is sent in the // same buffer as normal commands, so flush the buffer first, enqueue IDLE, and // flush that behind the first - if (ser != null) - yield ser.flush_async(); + bool is_synchronized = false; + while (ser != null) { + // prepare for upcoming synchronization point (continuation response could be + // recv'd before flush_async() completes) and reset prior synchronization response + posted_synchronization_tag = ser.next_synchronized_message(); + synchronization_status_response = null; + Tag? synchronize_tag; + yield ser.flush_async(is_synchronized, out synchronize_tag); + + // if no tag returned, all done, otherwise synchronization required + if (synchronize_tag == null) + break; + + // no longer synchronized + is_synchronized = false; + + // synchronization is not always possible + if (!issue_conditional_event(Event.SYNCHRONIZE)) { + debug("[%s] Unable to synchronize, exiting do_flush_async", to_string()); + + return; + } + + // wait for synchronization point to be reached + debug("[%s] Synchronizing...", to_string()); + yield synchronized_notifier.wait_async(); + + // watch for the synchronization request to be thwarted + if (synchronization_status_response != null) { + debug("[%s]: Failed to synchronize command continuation: %s", to_string(), + synchronization_status_response.to_string()); + + // skip pass current message, this one's done + if (ser != null) + ser.fast_forward_queue(); + } else { + debug("[%s] Synchronized, ready to continue", to_string()); + + // now synchronized, ready to continue + is_synchronized = true; + } + } + + // reset synchronization state + posted_synchronization_tag = null; + synchronization_status_response = null; + // as connection is "quiet" (haven't seen new command in n msec), go into IDLE state // if (a) allowed by owner and (b) allowed by state machine if (ser != null && idle_when_quiet && issue_conditional_event(Event.SEND_IDLE)) { @@ -611,11 +680,14 @@ public class Geary.Imap.ClientConnection : BaseObject { Logging.debug(Logging.Flag.NETWORK, "[%s] Initiating IDLE: %s", to_string(), idle_cmd.to_string()); - yield idle_cmd.serialize(ser); + idle_cmd.serialize(ser, idle_cmd.tag); + + Tag? synchronize_tag; + yield ser.flush_async(false, out synchronize_tag); + + // flushing IDLE should never require synchronization + assert(synchronize_tag == null); } - - if (ser != null) - yield ser.flush_async(); } catch (Error err) { idle_cmd = null; send_failure(err); @@ -779,6 +851,14 @@ public class Geary.Imap.ClientConnection : BaseObject { return do_proceed(State.IDLING, user); } + private uint on_synchronize(uint state, uint event, void *user) { + return do_proceed(State.SYNCHRONIZING, user); + } + + private uint on_deidling_synchronize(uint state, uint event, void *user) { + return do_proceed(State.DEIDLING_SYNCHRONIZING, user); + } + private uint on_status_response(uint state, uint event, void *user, Object? object) { fsm.do_post_transition(signal_status_response, user, object); @@ -812,7 +892,7 @@ public class Geary.Imap.ClientConnection : BaseObject { private uint on_idle_send(uint state, uint event, void *user) { Logging.debug(Logging.Flag.NETWORK, "[%s] Closing IDLE", to_string()); - // TODO: Because there is not DISCONNECTING state, need to watch for the Serializer + // TODO: Because there is no DISCONNECTING state, need to watch for the Serializer // disappearing during a disconnect while in a "normal" state if (ser == null) { debug("[%s] Unable to close IDLE: no serializer", to_string()); @@ -880,6 +960,55 @@ public class Geary.Imap.ClientConnection : BaseObject { return state; } + private uint on_deidling_synchronizing_status_response(uint state, uint event, void *user, + Object? object) { + // piggyback on on_idle_status_response, but instead of jumping to CONNECTED, jump to + // SYNCHRONIZING (because IDLE has completed) + return (on_idle_status_response(state, event, user, object) == State.CONNECTED) + ? State.SYNCHRONIZING : state; + } + + private uint on_synchronize_status_response(uint state, uint event, void *user, Object? object) { + StatusResponse status_response = (StatusResponse) object; + + // waiting for status response to synchronization message, treat others normally + if (posted_synchronization_tag == null || !posted_synchronization_tag.equal_to(status_response.tag)) { + fsm.do_post_transition(signal_status_response, user, object); + + return state; + } + + // receive status response while waiting for synchronization of a command; this means the + // server has rejected it + debug("[%s] Command continuation rejected: %s", to_string(), status_response.to_string()); + + // save result and notify sleeping flush_async() + synchronization_status_response = status_response; + synchronized_notifier.blind_notify(); + + return State.CONNECTED; + } + + private uint on_synchronize_continuation(uint state, uint event, void *user, Object? object) { + ContinuationResponse continuation = (ContinuationResponse) object; + + if (posted_synchronization_tag == null) { + debug("[%s] Bad command continuation received: %s", to_string(), + continuation.to_string()); + } else { + debug("[%s] Command continuation received for %s: %s", to_string(), + posted_synchronization_tag.to_string(), continuation.to_string()); + } + + // wake up the sleeping flush_async() call so it will continue + synchronization_status_response = null; + synchronized_notifier.blind_notify(); + + // There is no SYNCHRONIZED state, which is kind of fleeting; the moment the flush_async() + // call continues, no longer synchronized + return State.CONNECTED; + } + private uint on_bad_transition(uint state, uint event, void *user) { debug("[%s] Bad cx state transition %s", to_string(), fsm.get_event_issued_string(state, event)); diff --git a/src/engine/imap/transport/imap-deserializer.vala b/src/engine/imap/transport/imap-deserializer.vala index 592aa45e..643618dc 100644 --- a/src/engine/imap/transport/imap-deserializer.vala +++ b/src/engine/imap/transport/imap-deserializer.vala @@ -18,6 +18,9 @@ * that since Deserializer uses async I/O, this isn't technically possible unless the signals are * connected after the Idle loop has a chance to run; however, this is an implementation detail and * shouldn't be relied upon.) + * + * Internally Deserializer uses a DataInputStream to help decode the data. Since DataInputStream + * is buffered, there's no need to buffer the InputStream passed to Deserializer's constructor. */ public class Geary.Imap.Deserializer : BaseObject { @@ -320,7 +323,7 @@ public class Geary.Imap.Deserializer : BaseObject { bytes_received(bytes_read); // adjust the current buffer's size to the amount that was actually read in - block_buffer.adjust(current_buffer, bytes_read); + block_buffer.trim(current_buffer, bytes_read); push_data(bytes_read); } catch (Error err) { diff --git a/src/engine/imap/transport/imap-serializer.vala b/src/engine/imap/transport/imap-serializer.vala index 287d3f8d..d7c990d4 100644 --- a/src/engine/imap/transport/imap-serializer.vala +++ b/src/engine/imap/transport/imap-serializer.vala @@ -5,34 +5,54 @@ */ /** - * The Serializer asynchronously writes serialized IMAP commands to the supplied output stream. - * Since most IMAP commands are small in size (one line of data, often under 64 bytes), the - * Serializer writes them to a temporary buffer, only writing to the actual stream when literal data - * is written (which can often be large and coming off of disk) or commit_async() is called, which - * should be invoked when convenient, to prevent the buffer from growing too large. + * Serializer asynchronously writes serialized IMAP commands to the supplied output stream via a + * queue of buffers. * - * Because of this situation, the serialized commands will not necessarily reach the output stream - * unless commit_async() is called, which pushes the in-memory bytes to it. Since the - * output stream itself may be buffered, flush_async() should be called to verify the bytes have - * reached the wire. - * - * flush_async() implies commit_async(), but the reverse is not true. + * Since most IMAP commands are small in size (one line of data, often under 64 bytes), the + * Serializer writes them to a queue of temporary buffers (interspersed with user-supplied buffers + * that are intended to be literal data). The data is only written when {@link flush_async} is + * invoked. + * + * This means that if the caller wants some buffer beyond the steps described above, they should + * pass in a BufferedOutputStream (or one of its subclasses). flush_async() will flush the user's + * OutputStream after writing to it. + * + * Command continuation requires some synchronization between the Serializer and the + * {@link Deserializer}. It also requires some queue management. See {@link fast_forward_queue} + * and {@link next_synchronized_message}. + * + * @see Deserializer */ public class Geary.Imap.Serializer : BaseObject { + private class SerializedData { + public Memory.Buffer buffer; + public Tag? literal_data_tag; + + public SerializedData(Memory.Buffer buffer, Tag? literal_data_tag) { + this.buffer = buffer; + this.literal_data_tag = literal_data_tag; + } + } + private string identifier; private OutputStream outs; private ConverterOutputStream couts; private MemoryOutputStream mouts; private DataOutputStream douts; private Geary.Stream.MidstreamConverter midstream = new Geary.Stream.MidstreamConverter("Serializer"); + private Gee.Queue datastream = new Gee.LinkedList(); public Serializer(string identifier, OutputStream outs) { this.identifier = identifier; this.outs = outs; + // prepare the ConverterOutputStream (which wraps the caller's OutputStream and allows for + // midstream conversion) couts = new ConverterOutputStream(outs, midstream); couts.set_close_base_stream(false); + + // prepare the DataOutputStream (which generates buffers for the queue) mouts = new MemoryOutputStream(null, realloc, free); douts = new DataOutputStream(mouts); douts.set_close_base_stream(false); @@ -78,49 +98,135 @@ public class Geary.Imap.Serializer : BaseObject { douts.put_string("\r\n", null); } - public async void push_input_stream_literal_data_async(InputStream ins, - int priority = GLib.Priority.DEFAULT, Cancellable? cancellable = null) throws Error { - // commit the in-memory buffer to the output stream - yield commit_async(priority, cancellable); - - // splice the literal data directly to the output stream - yield couts.splice_async(ins, OutputStreamSpliceFlags.NONE, priority, cancellable); - } - - // commit_async() takes the stored (in-memory) serialized data and writes it asynchronously - // to the wrapped OutputStream. Note that this is *not* a flush, as it's possible the - // serialized data will be stored in a buffer in the OutputStream. Use flush_async() to force - // data onto the wire. - public async void commit_async(int priority = GLib.Priority.DEFAULT, Cancellable? cancellable = null) - throws Error { + private void enqueue_current_stream() throws IOError { size_t length = mouts.get_data_size(); - if (length == 0) + if (length <= 0) return; - if (Logging.are_all_flags_set(Logging.Flag.SERIALIZER)) { - StringBuilder builder = new StringBuilder(); - for (size_t ctr = 0; ctr < length; ctr++) - builder.append_c((char) mouts.get_data()[ctr]); - - Logging.debug(Logging.Flag.SERIALIZER, "[%s] send %s", to_string(), builder.str.strip()); - } + // close before converting to Memory.ByteBuffer + mouts.close(); - ssize_t index = 0; - do { - index += yield couts.write_async(mouts.get_data()[index:length], priority, cancellable); - } while (index < length); + SerializedData data = new SerializedData( + new Memory.ByteBuffer.from_memory_output_stream(mouts), null); + datastream.add(data); mouts = new MemoryOutputStream(null, realloc, free); douts = new DataOutputStream(mouts); + douts.set_close_base_stream(false); } - // This pushes all serialized data onto the wire. This calls commit_async() before - // flushing. - public async void flush_async(int priority = GLib.Priority.DEFAULT, Cancellable? cancellable = null) - throws Error { - yield commit_async(priority, cancellable); - yield couts.flush_async(priority, cancellable); - yield outs.flush_async(priority, cancellable); + /* + * Pushes an {link Memory.Buffer} to the serialized stream that must be synchronized + * with the server before transmission. + * + * Literal data may require synchronization with the server and so should only be used when + * necessary. See {link DataFormat.is_quoting_required} to test data. + * + * The supplied buffer must not be mutated once submitted to the {@link Serializer}. + * + * See [[http://tools.ietf.org/html/rfc3501#section-4.3]] and + * [[http://tools.ietf.org/html/rfc3501#section-7.5]] + */ + public void push_synchronized_literal_data(Tag tag, Memory.Buffer buffer) throws Error { + enqueue_current_stream(); + datastream.add(new SerializedData(buffer, tag)); + } + + /** + * Indicates that a complete message has been pushed to the {@link Serializer}. + * + * It's important to delineate messages for the Serializer, as it aids in queue management + * and command continuation (synchronization). + */ + public void push_end_of_message() throws Error { + enqueue_current_stream(); + datastream.add(null); + } + + /** + * Returns the {@link Tag} for the message with the next synchronization message Tag. + * + * This can be used to prepare for receiving a command continuation failure before sending + * the request via {@link flush_async}, as the response could return before that call completes. + */ + public Tag? next_synchronized_message() { + foreach (SerializedData? data in datastream) { + if (data != null && data.literal_data_tag != null) + return data.literal_data_tag; + } + + return null; + } + + /** + * Discards all buffers associated with the current message and moves the queue forward to the + * next one. + * + * This is useful when a command continuation is refused by the server and the command must be + * aborted. + * + * Any data currently in the buffer is *not* enqueued, as by definition it has not been marked + * with {@link push_end_of_message}. + */ + public void fast_forward_queue() { + while (!datastream.is_empty) { + if (datastream.poll() == null) + break; + } + } + + /** + * Push all serialized data and buffers onto the wire. + * + * Caller should pass is_synchronized=true if the connection has been synchronized for a command + * continuation. + * + * If synchronize_tag returns non-null, then the flush has not completed. The connection must + * wait for the server to send a continuation response before continuing. When ready, call + * flush_async() again with is_synchronized set to true. The tag is supplied to watch for + * an error condition from the server (which may reject the synchronization request). + */ + public async void flush_async(bool is_synchronized, out Tag? synchronize_tag, + Cancellable? cancellable = null) throws Error { + synchronize_tag = null; + + // commit the last buffer to the queue (although this is best done with push_end_message) + enqueue_current_stream(); + + // walk the SerializedData queue, pushing each out to the wire unless a synchronization + // point is encountered + while (!datastream.is_empty) { + // see if next data buffer is synchronized + SerializedData? data = datastream.peek(); + if (data != null && data.literal_data_tag != null && !is_synchronized) { + // report the Tag that is associated with the continuation + synchronize_tag = data.literal_data_tag; + + // break out to ensure pipe is flushed + break; + } + + // if not, remove and process + data = datastream.poll(); + if (data == null) { + // end of message, move on + continue; + } + + Logging.debug(Logging.Flag.SERIALIZER, "[%s] %s", to_string(), data.buffer.to_string()); + + // splice buffer's InputStream directly into OutputStream + yield couts.splice_async(data.buffer.get_input_stream(), OutputStreamSpliceFlags.NONE, + Priority.DEFAULT, cancellable); + + // if synchronized before, not any more + is_synchronized = false; + } + + // make sure everything is flushed out now ... some trouble with BufferedOutputStreams + // here, so flush ConverterOutputStream and its base stream + yield couts.flush_async(Priority.DEFAULT, cancellable); + yield couts.base_stream.flush_async(Priority.DEFAULT, cancellable); } public string to_string() { diff --git a/src/engine/memory/memory-buffer.vala b/src/engine/memory/memory-buffer.vala new file mode 100644 index 00000000..65250597 --- /dev/null +++ b/src/engine/memory/memory-buffer.vala @@ -0,0 +1,119 @@ +/* Copyright 2011-2013 Yorba Foundation + * + * This software is licensed under the GNU Lesser General Public License + * (version 2.1 or later). See the COPYING file in this distribution. + */ + +/** + * Represents an interface to a variety of backing buffers. + * + * A Buffer may be an in-memory or on-disk block of bytes. Buffer allows for a + * uniform interface to these blocks and makes it easy to move them around and avoiding copies. + * + * Questions of mutability are left to the implementation and users of Buffer. In general, + * AbstractBuffers should be built and modified before allowing other callers to access it. + * + * @see ByteBuffer + * @see EmptyBuffer + * @see GrowableBuffer + * @see StringBuffer + * @see UnownedStringBuffer + * @see UnownedBytesBuffer + */ + +public abstract class Geary.Memory.Buffer : BaseObject { + /** + * Returns the number of valid (usable) bytes in the buffer. + */ + public abstract size_t size { get; } + + /** + * Returns the number of bytes allocated (usable and unusable) for the buffer. + */ + public abstract size_t allocated_size { get; } + + /** + * Returns a Bytes object holding the buffer's contents. + * + * Since Bytes is immutable, the caller will need to make its own copy if it wants to modify + * the data. + */ + public abstract Bytes get_bytes(); + + /** + * Returns an InputStream that can read the buffer in its current entirety. + * + * Note that the InputStream may share its memory buffer(s) with the Buffer but does + * not hold references to them or the Buffer itself. Thus, the Buffer should + * only be destroyed after all InputStreams are destroyed or exhausted. + * + * The base class implementation uses {@link get_bytes} to create the InputStream. Subclasses + * should look for more optimal implementations. + */ + public virtual InputStream get_input_stream() { + return new MemoryInputStream.from_bytes(get_bytes()); + } + + /** + * Returns a ByteArray storing the buffer in its entirety. + * + * A copy of the backing buffer is returned. + * + * The base class implementation uses {@link get_bytes} to create the InputStream. Subclasses + * should look for more optimal implementations. + */ + public virtual ByteArray get_byte_array() { + ByteArray byte_array = new ByteArray(); + byte_array.append(get_bytes().get_data()); + + return byte_array; + } + + /** + * Returns an array of uint8 storing the buffer in its entirety. + * + * A copy of the backing buffer is returned. + * + * The base class implementation uses {@link get_bytes} to create the InputStream. Subclasses + * should look for more optimal implementations. + * + * @see UnownedBytesBuffer + */ + public virtual uint8[] get_uint8_array() { + return get_bytes().get_data(); + } + + /** + * Returns a copy of the contents of the buffer as though it was a null terminated string. + * + * The base class implementation uses {@link get_bytes} to create the InputStream. Subclasses + * should look for more optimal implementations. + * + * No validation is made on the string. See {@link get_valid_utf8}. + * + * @see UnownedStringBuffer + */ + public virtual string to_string() { + uint8[] buffer = get_uint8_array(); + buffer += (uint8) '\0'; + + return (string) buffer; + } + + /** + * Returns a copy of the contents of the buffer as though it was a UTF-8 string. + * + * The base class implementation uses {@link get_bytes} to create the InputStream. Subclasses + * should look for more optimal implementations. + * + * If the conversion fails or decodes as invalid UTF-8, an empty string is returned. + * + * @see UnownedStringBuffer.get_unowned_valid_utf8 + */ + public virtual string get_valid_utf8() { + string str = to_string(); + + return str.validate() ? str : ""; + } +} + diff --git a/src/engine/memory/memory-byte-buffer.vala b/src/engine/memory/memory-byte-buffer.vala new file mode 100644 index 00000000..72f094cd --- /dev/null +++ b/src/engine/memory/memory-byte-buffer.vala @@ -0,0 +1,98 @@ +/* Copyright 2011-2013 Yorba Foundation + * + * This software is licensed under the GNU Lesser General Public License + * (version 2.1 or later). See the COPYING file in this distribution. + */ + +/** + * Allows for a plain block of bytes to be represented as an {@link Buffer}. + */ + +public class Geary.Memory.ByteBuffer : Memory.Buffer, Memory.UnownedBytesBuffer { + /** + * {@inheritDoc} + */ + public override size_t size { + get { + return bytes.length; + } + } + + /** + * {@inheritDoc} + */ + public override size_t allocated_size { + get { + return allocated_bytes; + } + } + + private Bytes bytes; + private size_t allocated_bytes; + + /** + * filled is the number of usable bytes in the supplied buffer, allocated is the total size + * of the buffer. + * + * filled must be less than or equal to the allocated size of the buffer. + * + * A copy of the data buffer is made. See {@link ByteBuffer.ByteBuffer.take} for a no-copy + * alternative. + */ + public ByteBuffer(uint8[] data, size_t filled) { + assert(filled <= data.length); + + bytes = new Bytes(data[0:filled]); + allocated_bytes = bytes.length; + } + + /** + * filled is the number of usable bytes in the supplied buffer, allocated is the total size + * of the buffer. + * + * filled must be less than or equal to the allocated size of the buffer. + */ + public ByteBuffer.take(owned uint8[] data, size_t filled) { + assert(filled <= data.length); + + bytes = new Bytes.take(data[0:filled]); + allocated_bytes = data.length; + } + + /** + * Takes ownership and converts a ByteArray to a {@link ByteBuffer}. + * + * The ByteArray is freed after this call and should not be used. + */ + public ByteBuffer.from_byte_array(ByteArray byte_array) { + bytes = ByteArray.free_to_bytes(byte_array); + allocated_bytes = bytes.length; + } + + /** + * Takes ownership and converts a MemoryOutputStream to a {@link ByteBuffer}. + * + * The MemoryOutputStream ''must'' be closed before this call. + */ + public ByteBuffer.from_memory_output_stream(MemoryOutputStream mouts) { + assert(mouts.is_closed()); + + bytes = mouts.steal_as_bytes(); + allocated_bytes = bytes.length; + } + + /** + * {@inheritDoc} + */ + public override Bytes get_bytes() { + return bytes; + } + + /** + * {@inheritDoc} + */ + public unowned uint8[] to_unowned_uint8_array() { + return bytes.get_data(); + } +} + diff --git a/src/engine/memory/memory-empty-buffer.vala b/src/engine/memory/memory-empty-buffer.vala new file mode 100644 index 00000000..a45b092e --- /dev/null +++ b/src/engine/memory/memory-empty-buffer.vala @@ -0,0 +1,74 @@ +/* Copyright 2011-2013 Yorba Foundation + * + * This software is licensed under the GNU Lesser General Public License + * (version 2.1 or later). See the COPYING file in this distribution. + */ + +/** + * An EmptyBuffer fulfills the interface of {@link Buffer} for a zero-length block. + * + * Because all EmptyBuffers are the same and immutable, only a single may be used: {@link instance}. + */ + +public class Geary.Memory.EmptyBuffer : Memory.Buffer, Memory.UnownedStringBuffer, + Memory.UnownedBytesBuffer, Memory.UnownedByteArrayBuffer { + private static EmptyBuffer? _instance = null; + public static EmptyBuffer instance { + get { + return (_instance != null) ? _instance : _instance = new EmptyBuffer(); + } + } + + /** + * {@inheritDoc} + */ + public override size_t size { + get { + return 0; + } + } + + /** + * {@inheritDoc} + */ + public override size_t allocated_size { + get { + return 0; + } + } + + private Bytes bytes = new Bytes(new uint8[0]); + private ByteArray byte_array = new ByteArray(); + + private EmptyBuffer() { + } + + /** + * {@inheritDoc} + */ + public override Bytes get_bytes() { + return bytes; + } + + /** + * {@inheritDoc} + */ + public unowned uint8[] to_unowned_uint8_array() { + return bytes.get_data(); + } + + /** + * {@inheritDoc} + */ + public unowned string to_unowned_string() { + return ""; + } + + /** + * {@inheritDoc} + */ + public unowned ByteArray to_unowned_byte_array() { + return byte_array; + } +} + diff --git a/src/engine/memory/memory-file-buffer.vala b/src/engine/memory/memory-file-buffer.vala new file mode 100644 index 00000000..ba2e14d8 --- /dev/null +++ b/src/engine/memory/memory-file-buffer.vala @@ -0,0 +1,51 @@ +/* Copyright 2013 Yorba Foundation + * + * This software is licensed under the GNU Lesser General Public License + * (version 2.1 or later). See the COPYING file in this distribution. + */ + +//extern Bytes *g_bytes_new_with_free_func(void *data, size_t size, DestroyNotify destroy, void *user); + +/** + * Makes a file available as a {@link Memory.Buffer}. + */ + +public class Geary.Memory.FileBuffer : Memory.Buffer, Memory.UnownedBytesBuffer { + private File file; + private MappedFile mmap; + + public override size_t size { + get { + return mmap.get_length(); + } + } + + public override size_t allocated_size { + get { + return mmap.get_length(); + } + } + + /** + * The File is immediately opened when this is called. + */ + public FileBuffer(File file, bool readonly) throws Error { + if (file.get_path() == null) + throw new IOError.NOT_FOUND("File for Geary.Memory.FileBuffer not found"); + + this.file = file; + mmap = new MappedFile(file.get_path(), !readonly); + } + + public override Bytes get_bytes() { + return Bytes.new_with_owner(to_unowned_uint8_array(), mmap); + } + + public unowned uint8[] to_unowned_uint8_array() { + unowned uint8[] buffer = (uint8[]) mmap.get_contents(); + buffer.length = (int) mmap.get_length(); + + return buffer; + } +} + diff --git a/src/engine/memory/memory-growable-buffer.vala b/src/engine/memory/memory-growable-buffer.vala new file mode 100644 index 00000000..687a078a --- /dev/null +++ b/src/engine/memory/memory-growable-buffer.vala @@ -0,0 +1,237 @@ +/* Copyright 2011-2013 Yorba Foundation + * + * This software is licensed under the GNU Lesser General Public License + * (version 2.1 or later). See the COPYING file in this distribution. + */ + +/** + * An {@link Buffer} that can be grown by appending additional buffer fragments. + * + * A buffer can be grown by appending data to it ({@link append}), or by allocating additional space + * in the internal buffer ({@link allocate}) which can then be trimmed back if not entirely used + * ({@link trim}). + */ + +public class Geary.Memory.GrowableBuffer : Memory.Buffer, Memory.UnownedBytesBuffer, + Memory.UnownedStringBuffer { + private static uint8[] NUL_ARRAY = { '\0' }; + + private ByteArray? byte_array = new ByteArray(); + private Bytes? bytes = null; + + public override size_t size { + // account for trailing NUL, which is always kept in place for UnownedStringBuffer + get { + if (bytes != null) + return bytes.length - 1; + + assert(byte_array != null); + + return byte_array.len - 1; + } + } + + public override size_t allocated_size { + get { + return size; + } + } + + public GrowableBuffer() { + // add NUL for UnownedStringBuffer + byte_array.append(NUL_ARRAY); + } + + private Bytes to_bytes() { + if (bytes != null) { + assert(byte_array == null); + + return bytes; + } + + assert(byte_array != null); + + bytes = ByteArray.free_to_bytes(byte_array); + byte_array = null; + + return bytes; + } + + private unowned uint8[] get_bytes_no_nul() { + assert(bytes != null); + assert(bytes.get_size() > 0); + + return bytes.get_data()[0:bytes.get_size() - 1]; + } + + private ByteArray to_byte_array() { + if (byte_array != null) { + assert(bytes == null); + + return byte_array; + } + + assert(bytes != null); + + byte_array = Bytes.unref_to_array(bytes); + bytes = null; + + return byte_array; + } + + private unowned uint8[] get_byte_array_no_nul() { + assert(byte_array != null); + assert(byte_array.len > 0); + + return byte_array.data[0:byte_array.len - 1]; + } + + /** + * Appends the data to the existing GrowableBuffer. + * + * It's unwise to append to a GrowableBuffer while outstanding ByteArrays and InputStreams + * (from {@link get_byte_array} or {@link Buffer.get_input_stream}) are outstanding. + */ + public void append(uint8[] buffer) { + if (buffer.length <= 0) + return; + + to_byte_array(); + + // account for existing NUL + assert(byte_array.len > 0); + byte_array.set_size(byte_array.len - 1); + + // append buffer and new NUL for UnownedStringBuffer + byte_array.append(buffer); + byte_array.append(NUL_ARRAY); + } + + /** + * Allocate data within the backing buffer for writing. + * + * Any usused bytes in the returned buffer should be returned to the {@link GrowableBuffer} + * via {@link trim}. + * + * It's unwise to write to a GrowableBuffer while outstanding ByteArrays and InputStreams + * (from {@link get_byte_array} or {@link Buffer.get_input_stream}) are outstanding. Likewise, + * it's dangerous to be writing to a GrowableBuffer and in the process call get_bytes() and + * such. + */ + public unowned uint8[] allocate(size_t requested_bytes) { + to_byte_array(); + + // existing NUL must be there already + assert(byte_array.len > 0); + + uint original_bytes = byte_array.len; + uint new_size = original_bytes + (uint) requested_bytes; + + byte_array.set_size(new_size); + byte_array.data[new_size - 1] = String.EOS; + + // only return portion request, not including new NUL, but overwriting existing NUL + unowned uint8[] buffer = byte_array.data[(original_bytes - 1):(new_size - 1)]; + assert(buffer.length == requested_bytes); + + return buffer; + } + + /** + * Trim a previously allocated buffer. + * + * {@link allocate} returns an internal buffer that may be used for writing. If the entire + * buffer is not filled, it should be trimmed with this call. + * + * filled_bytes is the number of bytes in the supplied buffer that are used (filled). The + * remainder are to be trimmed. + * + * trim() can only trim back the last allocation; there is no facility for having multiple + * outstanding allocations and trimming each back randomly. + * + * WARNING: No other call to the GrowableBuffer should be made between allocate() and trim(). + * Requesting {@link get_bytes} and other calls may shift the buffer in memory. + */ + public void trim(uint8[] allocation, size_t filled_bytes) { + // TODO: pointer arithmetic to verify that this allocation actually belongs to the + // ByteArray + assert(byte_array != null); + assert(filled_bytes <= allocation.length); + + // don't need to worry about the NUL byte here (unless caller overran buffer, then we + // have bigger problems) + byte_array.set_size(byte_array.len - (uint) (allocation.length - filled_bytes)); + } + + /** + * {@inheritDoc} + */ + public override Bytes get_bytes() { + to_bytes(); + assert(bytes.get_size() > 0); + + // don't return trailing nul + return new Bytes.from_bytes(bytes, 0, bytes.get_size() - 1); + } + + /** + * {@inheritDoc} + */ + public override ByteArray get_byte_array() { + ByteArray copy = new ByteArray(); + + // don't copy trailing NUL + if (bytes != null) { + copy.append(get_bytes_no_nul()); + } else { + assert(byte_array != null); + copy.append(get_byte_array_no_nul()); + } + + return copy; + } + + /** + * {@inheritDoc} + */ + public override uint8[] get_uint8_array() { + // because returned array is not unowned, Vala will make a copy + return to_unowned_uint8_array(); + } + + /** + * {@inheritDoc} + */ + public unowned uint8[] to_unowned_uint8_array() { + // in any case, don't return trailing NUL + if (bytes != null) + return get_bytes_no_nul(); + + assert(byte_array != null); + + return get_byte_array_no_nul(); + } + + /** + * {@inheritDoc} + */ + public override string to_string() { + // because returned string is not unowned, Vala will make a copy + return to_unowned_string(); + } + + /** + * {@inheritDoc} + */ + public unowned string to_unowned_string() { + // because of how append() and allocate() ensure a trailing NUL, can convert data to a + // string without copy-and-append + if (bytes != null) + return (string) bytes.get_data(); + + assert(byte_array != null); + + return (string) byte_array.data; + } +} + diff --git a/src/engine/memory/memory-string-buffer.vala b/src/engine/memory/memory-string-buffer.vala new file mode 100644 index 00000000..c1b50140 --- /dev/null +++ b/src/engine/memory/memory-string-buffer.vala @@ -0,0 +1,69 @@ +/* Copyright 2011-2013 Yorba Foundation + * + * This software is licensed under the GNU Lesser General Public License + * (version 2.1 or later). See the COPYING file in this distribution. + */ + +/** + * Allows for a common string to be represented as an {@link Memory.Buffer}. + */ + +public class Geary.Memory.StringBuffer : Memory.Buffer, Memory.UnownedStringBuffer, + Memory.UnownedBytesBuffer { + public override size_t size { + get { + return length; + } + } + + public override size_t allocated_size { + get { + return length; + } + } + + private string str; + private size_t length; + private Bytes? bytes = null; + + public StringBuffer(string str) { + this.str = str; + length = str.data.length; + } + + /** + * {@inheritDoc} + */ + public override Bytes get_bytes() { + return (bytes != null) ? bytes : bytes = new Bytes(str.data); + } + + /** + * {@inheritDoc} + */ + public override string to_string() { + return str; + } + + /** + * {@inheritDoc} + */ + public override string get_valid_utf8() { + return str.validate() ? str : ""; + } + + /** + * {@inheritDoc} + */ + public unowned string to_unowned_string() { + return str; + } + + /** + * {@inheritDoc} + */ + public unowned uint8[] to_unowned_uint8_array() { + return str.data; + } +} + diff --git a/src/engine/memory/memory-unowned-byte-array-buffer.vala b/src/engine/memory/memory-unowned-byte-array-buffer.vala new file mode 100644 index 00000000..44d27d13 --- /dev/null +++ b/src/engine/memory/memory-unowned-byte-array-buffer.vala @@ -0,0 +1,25 @@ +/* Copyright 2013 Yorba Foundation + * + * This software is licensed under the GNU Lesser General Public License + * (version 2.1 or later). See the COPYING file in this distribution. + */ + +/** + * Interface which allows access to a backing ByteArray of {@link Memory.Buffer} without + * transferring ownership, i.e. copying the ByteArray. + * + * The presence of this interface indicates that obtaining access to the backing buffer is cheap. + * + * Not all AbstractBuffers can support this call, but if they can, this interface allows for it. + * The ByteArray that is returned should ''not'' be modified or freed by the caller. + */ + +public interface Geary.Memory.UnownedByteArrayBuffer : Memory.Buffer { + /** + * Returns an unowned pointer to the backing ByteArray. + * + * The returned ByteArray should not be modified or freed. + */ + public abstract unowned ByteArray to_unowned_byte_array(); +} + diff --git a/src/engine/memory/memory-unowned-bytes-buffer.vala b/src/engine/memory/memory-unowned-bytes-buffer.vala new file mode 100644 index 00000000..3fc12268 --- /dev/null +++ b/src/engine/memory/memory-unowned-bytes-buffer.vala @@ -0,0 +1,27 @@ +/* Copyright 2013 Yorba Foundation + * + * This software is licensed under the GNU Lesser General Public License + * (version 2.1 or later). See the COPYING file in this distribution. + */ + +/** + * Interface which allows access to backing data of {@link Memory.Buffer} without transferring + * ownership, i.e. copying the buffer. + * + * The presence of this interface indicates that obtaining access to the backing buffer is cheap + * (i.e. get_bytes().get_data() will do the same thing, but generating a Bytes object might have + * a cost). + * + * Not all AbstractBuffers can support this call, but if they can, this interface allows for it. + * The buffers that are returned should ''not'' be modified or freed by the caller. + */ + +public interface Geary.Memory.UnownedBytesBuffer : Memory.Buffer { + /** + * Returns an unowned pointer of the backing buffer. + * + * The returned array should not be modified or freed. + */ + public abstract unowned uint8[] to_unowned_uint8_array(); +} + diff --git a/src/engine/memory/memory-unowned-string-buffer.vala b/src/engine/memory/memory-unowned-string-buffer.vala new file mode 100644 index 00000000..33952998 --- /dev/null +++ b/src/engine/memory/memory-unowned-string-buffer.vala @@ -0,0 +1,34 @@ +/* Copyright 2013 Yorba Foundation + * + * This software is licensed under the GNU Lesser General Public License + * (version 2.1 or later). See the COPYING file in this distribution. + */ + +/** + * Interface which allows access to backing string of {@link Memory.Buffer} without transferring + * ownership, i.e. copying the string. + * + * The presence of this interface indicates that obtaining access to the backing string is cheap. + * + * Not all AbstractBuffers can support this call, but if they can, this interface allows for it. + * The buffers that are returned should ''not'' be modified or freed by the caller. + */ + +public interface Geary.Memory.UnownedStringBuffer : Memory.Buffer { + /** + * Returns an unowned string version of the backing buffer. + * + * The returned string should not be modified or freed. + */ + public abstract unowned string to_unowned_string(); + + /** + * An unowned version of {@link Memory.Buffer.get_valid_utf8}. + */ + public virtual unowned string get_unowned_valid_utf8() { + string str = to_unowned_string(); + + return str.validate() ? str : ""; + } +} + diff --git a/src/engine/rfc822/rfc822-message-data.vala b/src/engine/rfc822/rfc822-message-data.vala index 5a2e2178..a6a04be2 100644 --- a/src/engine/rfc822/rfc822-message-data.vala +++ b/src/engine/rfc822/rfc822-message-data.vala @@ -119,6 +119,13 @@ public class Geary.RFC822.Date : Geary.RFC822.MessageData, Geary.MessageData.Abs return value.equal(other.value); } + /** + * Returns the {@link Date} in ISO-8601 format. + */ + public virtual string serialize() { + return GMime.utils_header_format_date(as_time_t, 0); + } + public virtual uint hash() { return value.hash(); } @@ -181,7 +188,7 @@ public class Geary.RFC822.Header : Geary.MessageData.BlockMessageData, Geary.RFC private GMime.Message? message = null; private string[]? names = null; - public Header(Geary.Memory.AbstractBuffer buffer) { + public Header(Memory.Buffer buffer) { base ("RFC822.Header", buffer); } @@ -189,8 +196,7 @@ public class Geary.RFC822.Header : Geary.MessageData.BlockMessageData, Geary.RFC if (message != null) return message.get_header_list(); - GMime.Parser parser = new GMime.Parser.with_stream( - new GMime.StreamMem.with_buffer(buffer.get_array())); + GMime.Parser parser = new GMime.Parser.with_stream(Utils.create_stream_mem(buffer)); parser.set_respect_content_length(false); parser.set_scan_from(false); @@ -224,32 +230,30 @@ public class Geary.RFC822.Header : Geary.MessageData.BlockMessageData, Geary.RFC } public class Geary.RFC822.Text : Geary.MessageData.BlockMessageData, Geary.RFC822.MessageData { - public Text(Geary.Memory.AbstractBuffer buffer) { + public Text(Memory.Buffer buffer) { base ("RFC822.Text", buffer); } } public class Geary.RFC822.Full : Geary.MessageData.BlockMessageData, Geary.RFC822.MessageData { - public Full(Geary.Memory.AbstractBuffer buffer) { + public Full(Memory.Buffer buffer) { base ("RFC822.Full", buffer); } } // Used for decoding preview text. public class Geary.RFC822.PreviewText : Geary.RFC822.Text { - public PreviewText(Geary.Memory.AbstractBuffer _buffer) { + public PreviewText(Memory.Buffer _buffer) { base (_buffer); } - public PreviewText.with_header(Geary.Memory.AbstractBuffer buffer, Geary.Memory.AbstractBuffer - preview_header) { + public PreviewText.with_header(Memory.Buffer preview, Memory.Buffer preview_header) { string? charset = null; string? encoding = null; bool is_html = false; // Parse the header. - GMime.Stream header_stream = new GMime.StreamMem.with_buffer( - preview_header.get_array()); + GMime.Stream header_stream = Utils.create_stream_mem(preview_header); GMime.Parser parser = new GMime.Parser.with_stream(header_stream); GMime.Part? part = parser.construct_part() as GMime.Part; if (part != null) { @@ -259,8 +263,7 @@ public class Geary.RFC822.PreviewText : Geary.RFC822.Text { encoding = part.get_header("Content-Transfer-Encoding"); } - GMime.StreamMem input_stream = new GMime.StreamMem.with_buffer(buffer.get_array()); - + GMime.StreamMem input_stream = Utils.create_stream_mem(preview); ByteArray output = new ByteArray(); GMime.StreamMem output_stream = new GMime.StreamMem.with_byte_array(output); output_stream.set_owner(false); diff --git a/src/engine/rfc822/rfc822-message.vala b/src/engine/rfc822/rfc822-message.vala index 38fe73e7..060bc840 100644 --- a/src/engine/rfc822/rfc822-message.vala +++ b/src/engine/rfc822/rfc822-message.vala @@ -27,8 +27,7 @@ public class Geary.RFC822.Message : BaseObject { private GMime.Message message; public Message(Full full) throws RFC822Error { - GMime.Parser parser = new GMime.Parser.with_stream( - new GMime.StreamMem.with_buffer(full.buffer.get_array())); + GMime.Parser parser = new GMime.Parser.with_stream(Utils.create_stream_mem(full.buffer)); message = parser.construct_message(); if (message == null) @@ -42,8 +41,8 @@ public class Geary.RFC822.Message : BaseObject { stock_from_gmime(); } - public Message.from_string(string full_email) throws RFC822Error { - this(new Geary.RFC822.Full(new Geary.Memory.StringBuffer(full_email))); + public Message.from_buffer(Memory.Buffer full_email) throws RFC822Error { + this(new Geary.RFC822.Full(full_email)); } public Message.from_parts(Header header, Text body) throws RFC822Error { @@ -52,12 +51,12 @@ public class Geary.RFC822.Message : BaseObject { // http://redmine.yorba.org/issues/7034 // and // https://bugzilla.gnome.org/show_bug.cgi?id=701572 - // - // TODO: When fixed in GMime, return to original behavior of streaming each buffer in - uint8[] buffer = new uint8[header.buffer.get_size() + body.buffer.get_size()]; + // + // TODO: When fixed in GMime, return to original behavior of streaming each buffer in + uint8[] buffer = new uint8[header.buffer.size + body.buffer.size]; uint8* ptr = buffer; - GLib.Memory.copy(ptr, header.buffer.get_array(), header.buffer.get_size()); - GLib.Memory.copy(ptr + header.buffer.get_size(), body.buffer.get_array(), body.buffer.get_size()); + GLib.Memory.copy(ptr, header.buffer.get_bytes().get_data(), header.buffer.size); + GLib.Memory.copy(ptr + header.buffer.size, body.buffer.get_bytes().get_data(), body.buffer.size); GMime.Parser parser = new GMime.Parser.with_stream(new GMime.StreamMem.with_buffer(buffer)); message = parser.construct_message(); @@ -401,12 +400,11 @@ public class Geary.RFC822.Message : BaseObject { return (addrs.size > 0) ? addrs : null; } - public Geary.Memory.AbstractBuffer get_body_rfc822_buffer() { + public Memory.Buffer get_body_rfc822_buffer() { return new Geary.Memory.StringBuffer(message.to_string()); } - public Geary.Memory.AbstractBuffer get_first_mime_part_of_content_type(string content_type, - bool to_html = false) + public Memory.Buffer get_first_mime_part_of_content_type(string content_type, bool to_html = false) throws RFC822Error { // search for content type starting from the root GMime.Part? part = find_first_mime_part(message.get_mime_part(), content_type); @@ -534,7 +532,7 @@ public class Geary.RFC822.Message : BaseObject { return RFC822.MailboxAddress.list_to_string(recipients, "", (a) => a.to_searchable_string()); } - public Geary.Memory.AbstractBuffer get_content_by_mime_id(string mime_id) throws RFC822Error { + public Memory.Buffer get_content_by_mime_id(string mime_id) throws RFC822Error { GMime.Part? part = find_mime_part_by_mime_id(message.get_mime_part(), mime_id); if (part == null) { throw new RFC822Error.NOT_FOUND("Could not find a MIME part with content-id %s", @@ -613,7 +611,7 @@ public class Geary.RFC822.Message : BaseObject { } } - private Geary.Memory.AbstractBuffer mime_part_to_memory_buffer(GMime.Part part, + private Memory.Buffer mime_part_to_memory_buffer(GMime.Part part, bool to_utf8 = false, bool to_html = false) throws RFC822Error { GMime.DataWrapper? wrapper = part.get_content_object(); @@ -652,7 +650,7 @@ public class Geary.RFC822.Message : BaseObject { wrapper.write_to_stream(stream_filter); stream_filter.flush(); - return new Geary.Memory.Buffer(byte_array.data, byte_array.len); + return new Geary.Memory.ByteBuffer.from_byte_array(byte_array); } public string to_string() { diff --git a/src/engine/rfc822/rfc822-utils.vala b/src/engine/rfc822/rfc822-utils.vala index 4c911806..f0c70f59 100644 --- a/src/engine/rfc822/rfc822-utils.vala +++ b/src/engine/rfc822/rfc822-utils.vala @@ -21,6 +21,34 @@ public GMime.FilterCharset create_utf8_filter_charset(string from_charset) { return filter_charset; } +/** + * Uses the best-possible transfer of bytes from the Memory.Buffer to the GMime.StreamMem object. + * The StreamMem object should be destroyed *before* the Memory.Buffer object, since this method + * will use unowned variants whenever possible. + */ +public GMime.StreamMem create_stream_mem(Memory.Buffer buffer) { + Memory.UnownedByteArrayBuffer? unowned_bytes_array_buffer = buffer as Memory.UnownedByteArrayBuffer; + if (unowned_bytes_array_buffer != null) { + // set_byte_array doesn't do any copying and doesn't take ownership -- perfect, this is + // the best of all possible worlds, assuming the Memory.Buffer is not destroyed first + GMime.StreamMem stream = new GMime.StreamMem(); + stream.set_byte_array(unowned_bytes_array_buffer.to_unowned_byte_array()); + + return stream; + } + + Memory.UnownedBytesBuffer? unowned_bytes_buffer = buffer as Memory.UnownedBytesBuffer; + if (unowned_bytes_buffer != null) { + // StreamMem.with_buffer does do a buffer copy (there's not set_buffer() call like + // set_byte_array() for some reason), but don't do a buffer copy when it comes out of the + // Memory.Buffer + return new GMime.StreamMem.with_buffer(unowned_bytes_buffer.to_unowned_uint8_array()); + } + + // do plain-old buffer copy + return new GMime.StreamMem.with_buffer(buffer.get_uint8_array()); +} + public string create_subject_for_reply(Geary.Email email) { return (email.subject ?? new Geary.RFC822.Subject("")).create_reply().value; } diff --git a/src/engine/smtp/smtp-client-session.vala b/src/engine/smtp/smtp-client-session.vala index 690cea25..57ea5322 100644 --- a/src/engine/smtp/smtp-client-session.vala +++ b/src/engine/smtp/smtp-client-session.vala @@ -173,7 +173,8 @@ public class Geary.Smtp.ClientSession { // DATA Geary.RFC822.Message email_copy = new Geary.RFC822.Message.without_bcc(email); - response = yield cx.send_data_async(email_copy.get_body_rfc822_buffer().get_array(), cancellable); + response = yield cx.send_data_async(email_copy.get_body_rfc822_buffer().get_bytes().get_data(), + cancellable); if (!response.code.is_success_completed()) response.throw_error("Unable to send message"); diff --git a/src/engine/smtp/smtp-plain-authenticator.vala b/src/engine/smtp/smtp-plain-authenticator.vala index 50e4a1c3..d162d00a 100644 --- a/src/engine/smtp/smtp-plain-authenticator.vala +++ b/src/engine/smtp/smtp-plain-authenticator.vala @@ -31,7 +31,7 @@ public class Geary.Smtp.PlainAuthenticator : Geary.Smtp.AbstractAuthenticator { growable.append(nul); growable.append((credentials.pass ?? "").data); - return Base64.encode(growable.get_array()).data; + return Base64.encode(growable.get_bytes().get_data()).data; } } diff --git a/src/engine/util/util-collection.vala b/src/engine/util/util-collection.vala index 56dcb033..9e48d3fe 100644 --- a/src/engine/util/util-collection.vala +++ b/src/engine/util/util-collection.vala @@ -13,6 +13,11 @@ public Gee.ArrayList to_array_list(Gee.Collection c) { return list; } +public void add_all_array(Gee.Collection c, G[] ar) { + foreach (G g in ar) + c.add(g); +} + public G? get_first(Gee.Collection c) { Gee.Iterator iter = c.iterator(); diff --git a/src/engine/util/util-memory.vala b/src/engine/util/util-memory.vala deleted file mode 100644 index 0426f694..00000000 --- a/src/engine/util/util-memory.vala +++ /dev/null @@ -1,238 +0,0 @@ -/* Copyright 2011-2013 Yorba Foundation - * - * This software is licensed under the GNU Lesser General Public License - * (version 2.1 or later). See the COPYING file in this distribution. - */ - -public abstract class Geary.Memory.AbstractBuffer : BaseObject { - public abstract size_t get_size(); - - public abstract size_t get_allocated_size(); - - public abstract uint8[] get_array(); - - /** - * Returns an InputStream that can read the buffer in its current entirety. Note that the - * InputStream may share its memory buffer(s) with the AbstractBuffer but does not hold - * references to them or the AbstractBuffer itself. Thus, the AbstractBuffer should only be - * destroyed after all InputStreams are destroyed or exhausted. - */ - public abstract InputStream get_input_stream(); - - /** - * Returns the contents of the buffer as though it was a null terminated string. Note that this - * involves reading the entire buffer into memory. - * - * If the conversion fails or decodes as invalid UTF-8, an empty string is returned. - */ - public string to_string() { - uint8[] buffer = get_array(); - buffer += (uint8) '\0'; - return (string) buffer; - } - - /** - * Returns the contents of the buffer as though it was a UTF-8 string. Note that this involves - * reading the entire buffer into memory. - * - * If the conversion fails or decodes as invalid UTF-8, an empty string is returned. - */ - public string to_valid_utf8() { - string str = to_string(); - return str.validate() ? str : ""; - } -} - -public class Geary.Memory.EmptyBuffer : Geary.Memory.AbstractBuffer { - private static EmptyBuffer? _instance = null; - public static EmptyBuffer instance { - get { - if (_instance == null) - _instance = new EmptyBuffer(); - - return _instance; - } - } - - private uint8[]? empty = null; - - private EmptyBuffer() { - } - - public override size_t get_size() { - return 0; - } - - public override size_t get_allocated_size() { - return 0; - } - - public override uint8[] get_array() { - if (empty == null) - empty = new uint8[0]; - - return empty; - } - - public override InputStream get_input_stream() { - return new MemoryInputStream.from_data(get_array(), null); - } -} - -public class Geary.Memory.StringBuffer : Geary.Memory.AbstractBuffer { - private string str; - - public StringBuffer(string str) { - this.str = str; - } - - public override size_t get_size() { - return str.data.length; - } - - public override size_t get_allocated_size() { - return str.data.length; - } - - public override uint8[] get_array() { - return str.data; - } - - public override InputStream get_input_stream() { - return new MemoryInputStream.from_data(str.data, null); - } -} - -public class Geary.Memory.Buffer : Geary.Memory.AbstractBuffer { - private uint8[] buffer; - private size_t filled; - - public Buffer(uint8[] buffer, size_t filled) { - this.buffer = buffer; - this.filled = filled; - } - - public override size_t get_size() { - return filled; - } - - public override size_t get_allocated_size() { - return buffer.length; - } - - public override uint8[] get_array() { - return buffer[0:filled]; - } - - public override InputStream get_input_stream() { - return new MemoryInputStream.from_data(buffer[0:filled], null); - } -} - -public class Geary.Memory.GrowableBuffer : Geary.Memory.AbstractBuffer { - private class BufferFragment { - public uint8[] buffer; - public size_t reserved_bytes = 0; - public unowned uint8[]? active = null; - - public BufferFragment(size_t bytes) { - buffer = new uint8[bytes]; - } - - public unowned uint8[]? reserve(size_t requested_bytes) { - if((reserved_bytes + requested_bytes) > buffer.length) - return null; - - active = buffer[reserved_bytes:reserved_bytes + requested_bytes]; - reserved_bytes += requested_bytes; - - return active; - } - - public void adjust(uint8[] active, size_t adjusted_bytes) { - assert(this.active == active); - - assert(active.length >= adjusted_bytes); - size_t freed = active.length - adjusted_bytes; - - assert(reserved_bytes >= freed); - reserved_bytes -= freed; - - active = null; - } - } - - private size_t min_fragment_bytes; - private Gee.ArrayList fragments = new Gee.ArrayList(); - - public GrowableBuffer(size_t min_fragment_bytes = 1024) { - this.min_fragment_bytes = min_fragment_bytes; - } - - public unowned uint8[] allocate(size_t bytes) { - if (fragments.size > 0) { - unowned uint8[]? buffer = fragments[fragments.size - 1].reserve(bytes); - if (buffer != null) - return buffer; - } - - BufferFragment next = new BufferFragment( - (bytes < min_fragment_bytes) ? min_fragment_bytes : bytes); - fragments.add(next); - - unowned uint8[]? buffer = next.reserve(bytes); - assert(buffer != null); - - return buffer; - } - - public void adjust(uint8[] buffer, size_t adjusted_bytes) { - assert(fragments.size > 0); - - fragments[fragments.size - 1].adjust(buffer, adjusted_bytes); - } - - public void append(uint8[] buffer) { - unowned uint8[] dest = allocate(buffer.length); - assert(dest.length == buffer.length); - - GLib.Memory.copy(dest, buffer, buffer.length); - } - - public override size_t get_size() { - size_t size = 0; - foreach (BufferFragment fragment in fragments) - size += fragment.reserved_bytes; - - return size; - } - - public override size_t get_allocated_size() { - size_t size = 0; - foreach (BufferFragment fragment in fragments) - size += fragment.buffer.length; - - return size; - } - - public override uint8[] get_array() { - uint8[] buffer = new uint8[get_size()]; - uint8 *buffer_ptr = (uint8 *) buffer; - foreach (BufferFragment fragment in fragments) { - GLib.Memory.copy(buffer_ptr, fragment.buffer, fragment.reserved_bytes); - buffer_ptr += fragment.reserved_bytes; - } - - return buffer; - } - - public override InputStream get_input_stream() { - // TODO: add_data() copies the buffer, hence the optimization doesn't work yet. - MemoryInputStream mins = new MemoryInputStream(); - foreach (BufferFragment fragment in fragments) - mins.add_data(fragment.buffer[0:fragment.reserved_bytes], null); - - return mins; - } -} - diff --git a/src/engine/util/util-object.vala b/src/engine/util/util-object.vala new file mode 100644 index 00000000..66b7584f --- /dev/null +++ b/src/engine/util/util-object.vala @@ -0,0 +1,44 @@ +/* Copyright 2013 Yorba Foundation + * + * This software is licensed under the GNU Lesser General Public License + * (version 2.1 or later). See the COPYING file in this distribution. + */ + +namespace Geary.ObjectUtils { + +/** + * Creates a set of property bindings from source to dest with the given binding flags. + */ +public Gee.List? mirror_properties(Object source, Object dest, BindingFlags + flags = GLib.BindingFlags.DEFAULT | GLib.BindingFlags.SYNC_CREATE) { + // Make sets of both object's properties. + Gee.HashSet source_properties = new Gee.HashSet(); + Collection.add_all_array(source_properties, + source.get_class().list_properties()); + Gee.HashSet dest_properties = new Gee.HashSet(); + Collection.add_all_array(dest_properties, + dest.get_class().list_properties()); + + // Remove properties from source_properties that are not in both sets. + source_properties.retain_all(dest_properties); + + // Create all bindings. + Gee.List bindings = new Gee.ArrayList(); + foreach(ParamSpec ps in source_properties) + bindings.add(source.bind_property(ps.name, dest, ps.name, flags)); + + return bindings.size > 0 ? bindings : null; +} + +/** + * Removes a property mirror created by mirror_properties + */ +public void unmirror_properties(Gee.List bindings) { + foreach(Binding b in bindings) + b.unref(); + + bindings.clear(); +} + +} + diff --git a/ui/account_list.glade b/ui/account_list.glade index fa18a997..9a6ac2d0 100644 --- a/ui/account_list.glade +++ b/ui/account_list.glade @@ -14,7 +14,7 @@ - gtk-edit + edit-symbolic