nettika/geary
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Speed up search, add SearchEmailIds; fix #7372

Browse source 
This optimizes the search query itself, and also how we get the results:
instead of turning around and selecting the full row for each resulting
id, we simply pull out the id and internal date and that becomes the new
SearchEmailIdentifier.  This new class also lets us more properly order
EmailIdentifiers everywhere else.
This commit is contained in:
Charles Lindsay 2013-09-05 11:06:52 -07:00
parent 1d9ea39dd1
commit 9a13769907
23 changed files with 318 additions and 259 deletions
Download patch file Download diff file Expand all files Collapse all files

1
src/CMakeLists.txt
View file

@ -172,6 +172,7 @@ engine/imap-db/imap-db-email-identifier.vala
engine/imap-db/imap-db-folder.vala
engine/imap-db/imap-db-message-addresses.vala
engine/imap-db/imap-db-message-row.vala
engine/imap-db/imap-db-search-email-identifier.vala
engine/imap-db/outbox/smtp-outbox-email-identifier.vala
engine/imap-db/outbox/smtp-outbox-email-properties.vala
engine/imap-db/outbox/smtp-outbox-folder.vala

22
src/client/models/conversation-list-store.vala
View file

@ -101,28 +101,6 @@ public class ConversationListStore : Gtk.ListStore {
email_store = (current_folder == null ? null : new Geary.App.EmailStore(current_folder.account));
}
public Geary.EmailIdentifier? get_lowest_email_id() {
Gtk.TreeIter iter;
if (!get_iter_first(out iter))
return null;
Geary.EmailIdentifier? lowest_id = null;
do {
Geary.App.Conversation? conversation = get_conversation_at_iter(iter);
if (conversation == null)
continue;
Geary.EmailIdentifier? conversation_lowest = conversation.get_lowest_email_id();
if (conversation_lowest == null)
continue;
if (lowest_id == null || conversation_lowest.natural_sort_comparator(lowest_id) < 0)
lowest_id = conversation_lowest;
} while (iter_next(ref iter));
return lowest_id;
}
public Geary.App.Conversation? get_conversation_at_path(Gtk.TreePath path) {
Gtk.TreeIter iter;
if (!get_iter(out iter, path))

1
src/engine/abstract/geary-abstract-account.vala
View file

@ -109,7 +109,6 @@ public abstract class Geary.AbstractAccount : BaseObject, Geary.Account {
Geary.Email.Field required_fields, Cancellable? cancellable = null) throws Error;
public abstract async Gee.Collection<Geary.EmailIdentifier>? local_search_async(string query,
Geary.Email.Field requested_fields, bool partial_ok, Geary.FolderPath? email_id_folder_path,
int limit = 100, int offset = 0, Gee.Collection<Geary.FolderPath?>? folder_blacklist = null,
Gee.Collection<Geary.EmailIdentifier>? search_ids = null, Cancellable? cancellable = null) throws Error;

4
src/engine/abstract/geary-abstract-folder.vala
View file

@ -81,6 +81,10 @@ public abstract class Geary.AbstractFolder : BaseObject, Geary.Folder {
public abstract async void close_async(Cancellable? cancellable = null) throws Error;
public abstract async void find_boundaries_async(Gee.Collection<Geary.EmailIdentifier> ids,
out Geary.EmailIdentifier? low, out Geary.EmailIdentifier? high,
Cancellable? cancellable = null) throws Error;
public abstract async Gee.List<Geary.Email>? list_email_by_id_async(Geary.EmailIdentifier? initial_id,
int count, Geary.Email.Field required_fields, Folder.ListFlags flags, Cancellable? cancellable = null)
throws Error;

12
src/engine/api/geary-account.vala
View file

@ -274,16 +274,12 @@ public interface Geary.Account : BaseObject {
/**
* Performs a search with the given query string. Optionally, a list of folders not to search
* can be passed as well as a list of email identifiers to restrict the search to only those messages.
* Returns a list of email objects with the requested fields. If partial_ok is false, mail
* will only be returned if it includes all requested fields. The
* email_id_folder_path is used as the path when creating EmailIdentifiers.
* The list is ordered descending by Geary.EmailProperties.date_received,
* and is limited to a maximum number of results and starting offset, so
* you can walk the table. limit can be negative to mean "no limit" but
* Returns a list of EmailIdentifiers, or null if there are no results.
* The list is limited to a maximum number of results and starting offset,
* so you can walk the table. limit can be negative to mean "no limit" but
* offset must not be negative.
*/
public abstract async Gee.Collection<Geary.Email>? local_search_async(string query,
Geary.Email.Field requested_fields, bool partial_ok, Geary.FolderPath? email_id_folder_path,
public abstract async Gee.Collection<Geary.EmailIdentifier>? local_search_async(string query,
int limit = 100, int offset = 0, Gee.Collection<Geary.FolderPath?>? folder_blacklist = null,
Gee.Collection<Geary.EmailIdentifier>? search_ids = null, Cancellable? cancellable = null) throws Error;

16
src/engine/api/geary-email-identifier.vala
View file

@ -18,24 +18,20 @@
public abstract class Geary.EmailIdentifier : BaseObject, Gee.Hashable<Geary.EmailIdentifier> {
// Warning: only change this if you know what you are doing.
protected int64 unique;
protected string unique;
protected EmailIdentifier(int64 unique) {
protected EmailIdentifier(string unique) {
this.unique = unique;
}
public virtual uint hash() {
return Collection.int64_hash(unique) ^ get_type().name().hash();
return unique.hash();
}
public virtual bool equal_to(Geary.EmailIdentifier other) {
if (this == other)
return true;
// catch different subtypes
if (get_type() != other.get_type())
return false;
return unique == other.unique;
}
@ -49,11 +45,7 @@ public abstract class Geary.EmailIdentifier : BaseObject, Gee.Hashable<Geary.Ema
if (this == other)
return 0;
int cmp = strcmp(get_type().name(), other.get_type().name());
if (cmp != 0)
return cmp;
return (int) (unique - other.unique).clamp(-1, 1);
return strcmp(unique, other.unique);
}
/**

6
src/engine/api/geary-engine.vala
View file

@ -296,17 +296,17 @@ public class Geary.Engine : BaseObject {
Geary.Account account;
switch (account_information.service_provider) {
case ServiceProvider.GMAIL:
account = new ImapEngine.GmailAccount("Gmail account %s".printf(account_information.email),
account = new ImapEngine.GmailAccount("Gmail:%s".printf(account_information.email),
account_information, remote_account, local_account);
break;
case ServiceProvider.YAHOO:
account = new ImapEngine.YahooAccount("Yahoo account %s".printf(account_information.email),
account = new ImapEngine.YahooAccount("Yahoo:%s".printf(account_information.email),
account_information, remote_account, local_account);
break;
case ServiceProvider.OTHER:
account = new ImapEngine.OtherAccount("Other account %s".printf(account_information.email),
account = new ImapEngine.OtherAccount("Other:%s".printf(account_information.email),
account_information, remote_account, local_account);
break;

9
src/engine/api/geary-folder.vala
View file

@ -348,6 +348,15 @@ public interface Geary.Folder : BaseObject {
*/
public abstract async void close_async(Cancellable? cancellable = null) throws Error;
/**
* Find the lowest- and highest-ordered {@link EmailIdentifier}s in the
* folder, among the given set of EmailIdentifiers that may or may not be
* in the folder. If none of the given set are in the folder, return null.
*/
public abstract async void find_boundaries_async(Gee.Collection<Geary.EmailIdentifier> ids,
out Geary.EmailIdentifier? low, out Geary.EmailIdentifier? high,
Cancellable? cancellable = null) throws Error;
/**
* List emails from the {@link Folder} starting at a particular location within the vector
* and moving either direction along the mail stack.

235
src/engine/api/geary-search-folder.vala
View file

@ -56,7 +56,7 @@ public class Geary.SearchFolder : Geary.AbstractLocalFolder, Geary.FolderSupport
// Orphan emails (without a folder) are also excluded; see ctor.
};
private string? search_query = null;
private Gee.TreeSet<Geary.Email> search_results;
private Gee.TreeSet<ImapDB.SearchEmailIdentifier> search_results;
private Geary.Nonblocking.Mutex result_mutex = new Geary.Nonblocking.Mutex();
/**
@ -80,7 +80,7 @@ public class Geary.SearchFolder : Geary.AbstractLocalFolder, Geary.FolderSupport
}
~SearchFolder() {
account.folders_available_unavailable.disconnect(on_folders_available_unavailable);;
account.folders_available_unavailable.disconnect(on_folders_available_unavailable);
account.email_locally_complete.disconnect(on_email_locally_complete);
account.email_removed.disconnect(on_account_email_removed);
}
@ -96,31 +96,33 @@ public class Geary.SearchFolder : Geary.AbstractLocalFolder, Geary.FolderSupport
}
}
public override async void find_boundaries_async(Gee.Collection<Geary.EmailIdentifier> ids,
out Geary.EmailIdentifier? low, out Geary.EmailIdentifier? high,
Cancellable? cancellable = null) throws Error {
low = null;
high = null;
Gee.TreeSet<ImapDB.SearchEmailIdentifier> in_folder
= new Gee.TreeSet<ImapDB.SearchEmailIdentifier>();
foreach (Geary.EmailIdentifier id in ids) {
ImapDB.SearchEmailIdentifier? search_id = id as ImapDB.SearchEmailIdentifier;
// This shouldn't require a result_mutex lock since there's no yield.
if (search_id != null && search_id in search_results)
in_folder.add(search_id);
}
if (in_folder.size > 0) {
low = in_folder.first();
high = in_folder.last();
}
}
private async void append_new_email_async(string query, Geary.Folder folder,
Gee.Collection<Geary.EmailIdentifier> ids, Cancellable? cancellable) throws Error {
int result_mutex_token = yield result_mutex.claim_async();
Error? error = null;
try {
Gee.Collection<Geary.Email>? results = yield account.local_search_async(
query, Geary.Email.Field.PROPERTIES, false, path, MAX_RESULT_EMAILS, 0,
exclude_folders, ids, cancellable);
if (results != null) {
Gee.HashMap<Geary.EmailIdentifier, Geary.Email> to_add
= new Gee.HashMap<Geary.EmailIdentifier, Geary.Email>();
foreach(Geary.Email email in results)
if (!search_results.contains(email))
to_add.set(email.id, email);
if (to_add.size > 0) {
search_results.add_all(to_add.values);
_properties.set_total(search_results.size);
notify_email_appended(to_add.keys);
notify_email_count_changed(search_results.size, CountChangeReason.APPENDED);
}
}
yield do_search_async(query, ids, cancellable);
} catch(Error e) {
error = e;
}
@ -148,40 +150,17 @@ public class Geary.SearchFolder : Geary.AbstractLocalFolder, Geary.FolderSupport
private async void handle_removed_email_async(string query, Geary.Folder folder,
Gee.Collection<Geary.EmailIdentifier> ids, Cancellable? cancellable) throws Error {
int result_mutex_token = yield result_mutex.claim_async();
Error? error = null;
try {
Gee.HashMap<Geary.EmailIdentifier, Geary.Email> relevant_ids
= new Gee.HashMap<Geary.EmailIdentifier, Geary.Email>();
Gee.ArrayList<Geary.EmailIdentifier> relevant_ids = new Gee.ArrayList<Geary.EmailIdentifier>();
foreach (Geary.EmailIdentifier id in ids) {
// TODO: maybe we need to have a way of accessing search
// results indexed by id?
foreach (Geary.Email email in search_results) {
if (id.equal_to(email.id))
relevant_ids.set(id, email);
}
if (id in (Gee.Collection<Geary.EmailIdentifier>) search_results)
relevant_ids.add(id);
}
if (relevant_ids.size > 0) {
Gee.Collection<Geary.Email>? results = yield account.local_search_async(
query, Geary.Email.Field.PROPERTIES, false, path, MAX_RESULT_EMAILS, 0,
exclude_folders, relevant_ids.keys, cancellable);
Gee.HashMap<Geary.EmailIdentifier, Geary.Email> to_remove
= new Gee.HashMap<Geary.EmailIdentifier, Geary.Email>();
foreach (Geary.EmailIdentifier id in relevant_ids.keys) {
if (results == null || !(relevant_ids.get(id) in results))
to_remove.set(id, relevant_ids.get(id));
}
if (to_remove.size > 0) {
search_results.remove_all(to_remove.values);
_properties.set_total(search_results.size);
notify_email_removed(to_remove.keys);
notify_email_count_changed(search_results.size, CountChangeReason.APPENDED);
}
}
if (relevant_ids.size > 0)
yield do_search_async(query, relevant_ids, cancellable);
} catch(Error e) {
error = e;
}
@ -210,9 +189,9 @@ public class Geary.SearchFolder : Geary.AbstractLocalFolder, Geary.FolderSupport
* Clears the search query and results.
*/
public void clear() {
Gee.TreeSet<Geary.Email> local_results = search_results;
Gee.Collection<ImapDB.SearchEmailIdentifier> local_results = search_results;
clear_search_results();
notify_email_removed(email_collection_to_ids(local_results));
notify_email_removed(local_results);
notify_email_count_changed(0, Geary.Folder.CountChangeReason.REMOVED);
if (search_query != null) {
@ -238,68 +217,10 @@ public class Geary.SearchFolder : Geary.AbstractLocalFolder, Geary.FolderSupport
private async void set_search_query_async(string query, Cancellable? cancellable = null) throws Error {
int result_mutex_token = yield result_mutex.claim_async();
Error? error = null;
try {
// TODO: don't limit this to MAX_RESULT_EMAILS. Instead, we could
// be smarter about only fetching the search results in
// list_email_async() etc., but this leads to some more
// complications when redoing the search.
Gee.Collection<Geary.Email>? _new_results = yield account.local_search_async(
query, Geary.Email.Field.PROPERTIES, false, path, MAX_RESULT_EMAILS, 0,
exclude_folders, null, cancellable);
if (_new_results == null) {
// No results? Remove all existing results and return early. If there are no
// existing results, there's nothing to do.
if (search_results.size > 0) {
Gee.TreeSet<Geary.Email> local_results = search_results;
// Clear existing results.
clear_search_results();
// Note that we probably shouldn't be firing these signals
// from inside our mutex lock. We do it here, below, and
// in append_new_email_async(). Keep an eye on it, and if
// there's ever a case where it might cause problems, it
// shouldn't be too hard to move the firings outside.
notify_email_removed(email_collection_to_ids(local_results));
notify_email_count_changed(0, Geary.Folder.CountChangeReason.REMOVED);
}
} else {
// Move new search results into a hashset, using email ID for equality.
Gee.HashSet<Geary.Email> new_results = new Gee.HashSet<Geary.Email>(email_id_hash, email_id_equal);
new_results.add_all(_new_results);
// Match the new results up with the existing results.
Gee.HashSet<Geary.Email> to_add = new Gee.HashSet<Geary.Email>(email_id_hash, email_id_equal);
Gee.HashSet<Geary.Email> to_remove = new Gee.HashSet<Geary.Email>(email_id_hash, email_id_equal);
foreach(Geary.Email email in new_results)
if (!search_results.contains(email))
to_add.add(email);
foreach(Geary.Email email in search_results)
if (!new_results.contains(email))
to_remove.add(email);
search_results.remove_all(to_remove);
search_results.add_all(to_add);
_properties.set_total(search_results.size);
Geary.Folder.CountChangeReason reason = CountChangeReason.NONE;
if (to_add.size > 0) {
reason |= Geary.Folder.CountChangeReason.INSERTED;
}
if (to_remove.size > 0) {
notify_email_removed(email_collection_to_ids(to_remove));
reason |= Geary.Folder.CountChangeReason.REMOVED;
}
if (reason != CountChangeReason.NONE)
notify_email_count_changed(search_results.size, reason);
}
yield do_search_async(query, null, cancellable);
} catch(Error e) {
error = e;
}
@ -313,6 +234,67 @@ public class Geary.SearchFolder : Geary.AbstractLocalFolder, Geary.FolderSupport
throw error;
}
// NOTE: you must call this ONLY after locking result_mutex_token.
// If search_ids is null, the results of this search are considered to be
// the full new set. If non-null, the results are considered to be a delta
// and are added or subtracted from the full set.
private async void do_search_async(string query, Gee.Collection<Geary.EmailIdentifier>? search_ids,
Cancellable? cancellable) throws Error {
// TODO: don't limit this to MAX_RESULT_EMAILS. Instead, we could be
// smarter about only fetching the search results in list_email_async()
// etc., but this leads to some more complications when redoing the
// search.
Gee.ArrayList<ImapDB.SearchEmailIdentifier> results
= ImapDB.SearchEmailIdentifier.array_list_from_results(yield account.local_search_async(
query, MAX_RESULT_EMAILS, 0, exclude_folders, search_ids, cancellable));
Gee.ArrayList<ImapDB.SearchEmailIdentifier> added
= new Gee.ArrayList<ImapDB.SearchEmailIdentifier>();
Gee.ArrayList<Geary.EmailIdentifier> removed
= new Gee.ArrayList<ImapDB.SearchEmailIdentifier>();
Gee.Collection<Geary.EmailIdentifier> potentially_removed_ids
= (search_ids == null ? search_results : search_ids);
foreach (ImapDB.SearchEmailIdentifier id in results) {
if (!(id in search_results))
added.add(id);
}
foreach (Geary.EmailIdentifier id in potentially_removed_ids) {
if (!(id in (Gee.Collection<Geary.EmailIdentifier>) results))
removed.add(id);
}
bool first_results = search_results.size == 0;
search_results.remove_all(removed);
search_results.add_all(added);
_properties.set_total(search_results.size);
// Note that we probably shouldn't be firing these signals from inside
// our mutex lock. Keep an eye on it, and if there's ever a case where
// it might cause problems, it shouldn't be too hard to move the
// firings outside.
Geary.Folder.CountChangeReason reason = CountChangeReason.NONE;
if (added.size > 0) {
// We do a little optimization here. In the case we can trivially
// determine nothing was added "in the middle" of an existing
// result set, we say the results were appended rather than
// inserted because appends are handled more gracefully in the
// conversation monitor.
reason |= (first_results ? Geary.Folder.CountChangeReason.APPENDED
: Geary.Folder.CountChangeReason.INSERTED);
}
if (removed.size > 0) {
notify_email_removed(removed);
reason |= Geary.Folder.CountChangeReason.REMOVED;
}
if (reason != CountChangeReason.NONE)
notify_email_count_changed(search_results.size, reason);
}
public override async Gee.List<Geary.Email>? list_email_by_id_async(Geary.EmailIdentifier? initial_id,
int count, Geary.Email.Field required_fields, Folder.ListFlags flags, Cancellable? cancellable = null)
throws Error {
@ -325,10 +307,10 @@ public class Geary.SearchFolder : Geary.AbstractLocalFolder, Geary.FolderSupport
Geary.EmailIdentifier[] ids = new Geary.EmailIdentifier[search_results.size];
int initial_index = 0;
int i = 0;
foreach (Geary.Email email in search_results) {
if (initial_id != null && email.id.equal_to(initial_id))
foreach (ImapDB.SearchEmailIdentifier id in search_results) {
if (initial_id != null && id.equal_to(initial_id))
initial_index = i;
ids[i++] = email.id;
ids[i++] = id;
}
if (initial_id == null && flags.is_all_set(Folder.ListFlags.OLDEST_TO_NEWEST))
@ -455,26 +437,9 @@ public class Geary.SearchFolder : Geary.AbstractLocalFolder, Geary.FolderSupport
exclude_folders.add(null);
}
private uint email_id_hash(Geary.Email a) {
return a.id.hash();
}
private bool email_id_equal(Geary.Email a, Geary.Email b) {
return a.id.equal_to(b.id);
}
// Destroys existing results.
private void clear_search_results() {
search_results = new Gee.TreeSet<Geary.Email>(Geary.Email.compare_date_received_descending);
}
// Converts a collection of emails to a set of email ids.
private Gee.Set<Geary.EmailIdentifier> email_collection_to_ids(Gee.Collection<Geary.Email> collection) {
Gee.HashSet<Geary.EmailIdentifier> ids = new Gee.HashSet<Geary.EmailIdentifier>();
foreach(Geary.Email email in collection)
ids.add(email.id);
return ids;
search_results = new Gee.TreeSet<ImapDB.SearchEmailIdentifier>(
ImapDB.SearchEmailIdentifier.compare_descending);
}
}

34
src/engine/app/app-conversation-monitor.vala
View file

@ -34,7 +34,7 @@ public class Geary.App.ConversationMonitor : BaseObject {
public int min_window_count { get { return _min_window_count; }
set {
_min_window_count = value;
operation_queue.add(new FillWindowOperation(this, avoid_email_id_comparisons));
operation_queue.add(new FillWindowOperation(this, false));
}
}
@ -50,12 +50,6 @@ public class Geary.App.ConversationMonitor : BaseObject {
private bool reseed_notified = false;
private int _min_window_count = 0;
private ConversationOperationQueue operation_queue = new ConversationOperationQueue();
// TODO: this hack is a quick way to solve the problem of id-based loads
// not working for the SearchFolder because its EmailIdentifiers are just
// row ids. Really, the solution is to make all EmailIdentifiers ordered.
// If true, this treats all fill-window operations as inserts, which means
// the whole list gets reloaded.
private bool avoid_email_id_comparisons = false;
/**
* "monitoring-started" is fired when the Conversations folder has been opened for monitoring.
@ -196,10 +190,6 @@ public class Geary.App.ConversationMonitor : BaseObject {
_min_window_count = min_window_count;
folder.account.information.notify["imap-credentials"].connect(on_imap_credentials_notified);
// See the definition of this field; basically if it's the search
// folder, we can't do any id-based loading. This is a hack.
avoid_email_id_comparisons = (folder is Geary.SearchFolder);
}
~ConversationMonitor() {
@ -295,7 +285,7 @@ public class Geary.App.ConversationMonitor : BaseObject {
// the reseed has to wait until the folder's remote is opened (handled in on_folder_opened)
if (reseed_now)
operation_queue.add(new ReseedOperation(this, "already opened"));
operation_queue.add(new FillWindowOperation(this, avoid_email_id_comparisons));
operation_queue.add(new FillWindowOperation(this, false));
folder.email_appended.connect(on_folder_email_appended);
folder.email_removed.connect(on_folder_email_removed);
@ -636,7 +626,7 @@ public class Geary.App.ConversationMonitor : BaseObject {
private void on_folder_email_removed(Gee.Collection<Geary.EmailIdentifier> removed_ids) {
operation_queue.add(new RemoveOperation(this, removed_ids));
operation_queue.add(new FillWindowOperation(this, avoid_email_id_comparisons));
operation_queue.add(new FillWindowOperation(this, false));
}
private void on_account_email_locally_complete(Geary.Folder folder,
@ -657,7 +647,7 @@ public class Geary.App.ConversationMonitor : BaseObject {
Gee.Collection<Geary.App.Conversation> removed;
Gee.MultiMap<Geary.App.Conversation, Geary.Email> trimmed;
yield conversations.remove_emails_and_check_in_folder(removed_ids, folder.account,
yield conversations.remove_emails_and_check_in_folder_async(removed_ids, folder.account,
folder.path, out removed, out trimmed, null);
foreach (Conversation conversation in trimmed.get_keys())
@ -713,20 +703,20 @@ public class Geary.App.ConversationMonitor : BaseObject {
operation_queue.add(new FillWindowOperation(this, true));
}
private Geary.EmailIdentifier? get_lowest_email_id() {
private async Geary.EmailIdentifier? get_lowest_email_id_async(Cancellable? cancellable) {
Geary.EmailIdentifier? earliest_id = null;
foreach (Geary.App.Conversation conversation in conversations.conversations) {
Geary.EmailIdentifier? id = conversation.get_lowest_email_id();
if (id != null && (earliest_id == null || id.natural_sort_comparator(earliest_id) < 0))
earliest_id = id;
try {
yield folder.find_boundaries_async(conversations.get_email_identifiers(),
out earliest_id, null, cancellable);
} catch (Error e) {
debug("Error finding earliest email identifier: %s", e.message);
}
return earliest_id;
}
internal async void reseed_async(string why) {
Geary.EmailIdentifier? earliest_id = get_lowest_email_id();
Geary.EmailIdentifier? earliest_id = yield get_lowest_email_id_async(null);
try {
if (earliest_id != null) {
debug("ConversationMonitor (%s) reseeding starting from Email ID %s on opened %s", why,
@ -862,7 +852,7 @@ public class Geary.App.ConversationMonitor : BaseObject {
int initial_message_count = conversations.get_email_count();
Geary.EmailIdentifier? low_id = get_lowest_email_id();
Geary.EmailIdentifier? low_id = yield get_lowest_email_id_async(null);
if (low_id != null && !is_insert) {
// Load at least as many messages as remianing conversations.
int num_to_load = min_window_count - conversations.size;

20
src/engine/app/app-conversation.vala
View file

@ -43,7 +43,6 @@ public class Geary.App.Conversation : BaseObject {
private int convnum;
private weak Geary.App.ConversationMonitor? owner;
private Gee.HashMap<EmailIdentifier, Email> emails = new Gee.HashMap<EmailIdentifier, Email>();
private Geary.EmailIdentifier? lowest_id;
// this isn't ideal but the cost of adding an email to multiple sorted sets once versus
// the number of times they're accessed makes it worth it
@ -74,7 +73,6 @@ public class Geary.App.Conversation : BaseObject {
public Conversation(Geary.App.ConversationMonitor owner) {
convnum = next_convnum++;
this.owner = owner;
lowest_id = null;
owner.email_flags_changed.connect(on_email_flags_changed);
owner.folder.account.email_discovered.connect(on_email_discovered);
@ -226,8 +224,6 @@ public class Geary.App.Conversation : BaseObject {
foreach (Geary.FolderPath path in known_paths)
path_map.set(email.id, path);
check_lowest_id(email.id);
appended(email);
return true;
@ -252,10 +248,6 @@ public class Geary.App.Conversation : BaseObject {
}
}
lowest_id = null;
foreach (Email e in emails.values)
check_lowest_id(e.id);
trimmed(email);
return (removed_message_ids.size > 0) ? removed_message_ids : null;
@ -335,13 +327,6 @@ public class Geary.App.Conversation : BaseObject {
}
}
/**
* Return the EmailIdentifier with the lowest value.
*/
public Geary.EmailIdentifier? get_lowest_email_id() {
return lowest_id;
}
private bool check_flag(Geary.NamedFlag flag, bool contains) {
foreach (Geary.Email email in get_emails(Ordering.NONE)) {
if (email.email_flags != null && email.email_flags.contains(flag) == contains)
@ -359,11 +344,6 @@ public class Geary.App.Conversation : BaseObject {
return check_flag(flag, false);
}
private void check_lowest_id(EmailIdentifier id) {
if (lowest_id == null || id.natural_sort_comparator(lowest_id) < 0)
lowest_id = id;
}
private void on_email_flags_changed(Conversation conversation, Geary.Email email) {
if (conversation == this)
email_flags_changed(email);

21
src/engine/app/conversation-monitor/app-conversation-operation-queue.vala
View file

@ -20,8 +20,25 @@ private class Geary.App.ConversationOperationQueue : BaseObject {
public void add(ConversationOperation op) {
// There should only ever be one FillWindowOperation at a time.
if (op is FillWindowOperation)
mailbox.remove_matching((o) => { return (o is FillWindowOperation); });
FillWindowOperation? fill_op = op as FillWindowOperation;
if (fill_op != null) {
Gee.Collection<ConversationOperation> removed
= mailbox.remove_matching(o => o is FillWindowOperation);
// If there were any "insert" fill window ops, preserve that flag,
// as otherwise we might miss some data.
if (!fill_op.is_insert) {
foreach (ConversationOperation removed_op in removed) {
FillWindowOperation? removed_fill = removed_op as FillWindowOperation;
assert(removed_fill != null);
if (removed_fill.is_insert) {
fill_op.is_insert = true;
break;
}
}
}
}
mailbox.send(op);
}

10
src/engine/app/conversation-monitor/app-conversation-set.vala
View file

@ -29,6 +29,10 @@ private class Geary.App.ConversationSet : BaseObject {
return email_id_map.size;
}
public Gee.Collection<Geary.EmailIdentifier> get_email_identifiers() {
return email_id_map.keys;
}
public bool contains(Conversation conversation) {
return _conversations.contains(conversation);
}
@ -367,7 +371,7 @@ private class Geary.App.ConversationSet : BaseObject {
* Check a set of emails using check_conversation_in_folder_async(), return
* the set of emails that were removed due to not being in the folder.
*/
public async Gee.Collection<Conversation> check_conversations_in_folder(
public async Gee.Collection<Conversation> check_conversations_in_folder_async(
Gee.Collection<Conversation> conversations, Geary.Account account,
Geary.FolderPath required_folder_path, Cancellable? cancellable) {
Gee.ArrayList<Conversation> evaporated = new Gee.ArrayList<Conversation>();
@ -386,7 +390,7 @@ private class Geary.App.ConversationSet : BaseObject {
return evaporated;
}
public async void remove_emails_and_check_in_folder(
public async void remove_emails_and_check_in_folder_async(
Gee.Collection<Geary.EmailIdentifier> ids, Geary.Account account,
Geary.FolderPath required_folder_path, out Gee.Collection<Conversation> removed,
out Gee.MultiMap<Conversation, Geary.Email> trimmed, Cancellable? cancellable) {
@ -398,7 +402,7 @@ private class Geary.App.ConversationSet : BaseObject {
Gee.MultiMap<Conversation, Geary.Email> initial_trimmed;
remove_all_emails_by_identifier(ids, out initial_removed, out initial_trimmed);
Gee.Collection<Conversation> evaporated = yield check_conversations_in_folder(
Gee.Collection<Conversation> evaporated = yield check_conversations_in_folder_async(
initial_trimmed.get_keys(), account, required_folder_path, cancellable);
_removed.add_all(initial_removed);

2
src/engine/app/conversation-monitor/app-fill-window-operation.vala
View file

@ -5,7 +5,7 @@
*/
private class Geary.App.FillWindowOperation : ConversationOperation {
private bool is_insert;
public bool is_insert { get; internal set; }
public FillWindowOperation(ConversationMonitor monitor, bool is_insert) {
base(monitor);

40
src/engine/imap-db/imap-db-account.vala
View file

@ -668,22 +668,35 @@ private class Geary.ImapDB.Account : BaseObject {
return sql.str;
}
public async Gee.Collection<Geary.Email>? search_async(string prepared_query,
Geary.Email.Field requested_fields, bool partial_ok, Geary.FolderPath? email_id_folder_path,
public async Gee.Collection<Geary.EmailIdentifier>? search_async(string prepared_query,
int limit = 100, int offset = 0, Gee.Collection<Geary.FolderPath?>? folder_blacklist = null,
Gee.Collection<Geary.EmailIdentifier>? search_ids = null, Cancellable? cancellable = null) throws Error {
Gee.Collection<Geary.Email> search_results = new Gee.HashSet<Geary.Email>();
Gee.ArrayList<ImapDB.SearchEmailIdentifier> search_results
= new Gee.ArrayList<ImapDB.SearchEmailIdentifier>();
string? search_ids_sql = get_search_ids_sql(search_ids);
yield db.exec_transaction_async(Db.TransactionType.RO, (cx) => {
string blacklisted_ids_sql = do_get_blacklisted_message_ids_sql(
folder_blacklist, cx, cancellable);
// Every mutation of this query we could think of has been tried,
// and this version was found to minimize running time. We
// discovered that just doing a JOIN between the MessageTable and
// MessageSearchTable was causing a full table scan to order the
// results. When it's written this way, and we force SQLite to use
// the correct index (not sure why it can't figure it out on its
// own), it cuts the running time roughly in half of how it was
// before. The short version is: modify with extreme caution. See
// <http://redmine.yorba.org/issues/7372>.
string sql = """
SELECT id
FROM MessageSearchTable
JOIN MessageTable USING (id)
WHERE MessageSearchTable MATCH ?
SELECT id, internaldate_time_t
FROM MessageTable
INDEXED BY MessageTableInternalDateTimeTIndex
WHERE id IN (
SELECT id
FROM MessageSearchTable
WHERE MessageSearchTable MATCH ?
)
""";
if (blacklisted_ids_sql != "")
sql += " AND id NOT IN (%s)".printf(blacklisted_ids_sql);
@ -702,15 +715,10 @@ private class Geary.ImapDB.Account : BaseObject {
Db.Result result = stmt.exec(cancellable);
while (!result.finished) {
int64 id = result.int64_at(0);
Geary.Email.Field db_fields;
MessageRow row = Geary.ImapDB.Folder.do_fetch_message_row(
cx, id, requested_fields, out db_fields, cancellable);
if (partial_ok || row.fields.fulfills(requested_fields)) {
Geary.Email email = row.to_email(new Geary.ImapDB.EmailIdentifier(id, null));
Geary.ImapDB.Folder.do_add_attachments(cx, email, id, cancellable);
search_results.add(email);
}
int64 internaldate_time_t = result.int64_at(1);
DateTime? internaldate = (internaldate_time_t == -1
? null : new DateTime.from_unix_local(internaldate_time_t));
search_results.add(new ImapDB.SearchEmailIdentifier(id, internaldate));
result.next(cancellable);
}

6
src/engine/imap-db/imap-db-email-identifier.vala
View file

@ -11,7 +11,7 @@ private class Geary.ImapDB.EmailIdentifier : Geary.EmailIdentifier {
public EmailIdentifier(int64 message_id, Imap.UID? uid) {
assert(message_id != Db.INVALID_ROWID);
base (message_id);
base (message_id.to_string());
this.message_id = message_id;
this.uid = uid;
@ -20,7 +20,7 @@ private class Geary.ImapDB.EmailIdentifier : Geary.EmailIdentifier {
// Used when a new message comes off the wire and doesn't have a rowid associated with it (yet)
// Requires a UID in order to find or create such an association
public EmailIdentifier.no_message_id(Imap.UID uid) {
base (Db.INVALID_ROWID);
base (Db.INVALID_ROWID.to_string());
message_id = Db.INVALID_ROWID;
this.uid = uid;
@ -33,7 +33,7 @@ private class Geary.ImapDB.EmailIdentifier : Geary.EmailIdentifier {
public void promote_with_message_id(int64 message_id) {
assert(this.message_id == Db.INVALID_ROWID);
unique = message_id;
unique = message_id.to_string();
this.message_id = message_id;
}

58
src/engine/imap-db/imap-db-search-email-identifier.vala Normal file
View file

@ -0,0 +1,58 @@
/* Copyright 2013 Yorba Foundation
*
* This software is licensed under the GNU Lesser General Public License
* (version 2.1 or later). See the COPYING file in this distribution.
*/
private class Geary.ImapDB.SearchEmailIdentifier : ImapDB.EmailIdentifier,
Gee.Comparable<SearchEmailIdentifier> {
public static int compare_descending(SearchEmailIdentifier a, SearchEmailIdentifier b) {
return b.compare_to(a);
}
public DateTime? date_received { get; private set; }
public SearchEmailIdentifier(int64 message_id, DateTime? date_received) {
base(message_id, null);
this.date_received = date_received;
}
public override int natural_sort_comparator(Geary.EmailIdentifier o) {
ImapDB.SearchEmailIdentifier? other = o as ImapDB.SearchEmailIdentifier;
if (other == null)
return 1;
return compare_to(other);
}
public virtual int compare_to(SearchEmailIdentifier other) {
if (date_received != null && other.date_received != null)
return date_received.compare(other.date_received);
if (date_received == null && other.date_received == null)
return stable_sort_comparator(other);
return (date_received == null ? -1 : 1);
}
public override string to_string() {
return "[%s/null/%s]".printf(message_id.to_string(),
(date_received == null ? "null" : date_received.to_string()));
}
public static Gee.ArrayList<SearchEmailIdentifier> array_list_from_results(
Gee.Collection<Geary.EmailIdentifier>? results) {
Gee.ArrayList<SearchEmailIdentifier> r = new Gee.ArrayList<SearchEmailIdentifier>();
if (results != null) {
foreach (Geary.EmailIdentifier id in results) {
SearchEmailIdentifier? search_id = id as SearchEmailIdentifier;
assert(search_id != null);
r.add(search_id);
}
}
return r;
}
}

2
src/engine/imap-db/outbox/smtp-outbox-email-identifier.vala
View file

@ -8,7 +8,7 @@ private class Geary.SmtpOutboxEmailIdentifier : Geary.EmailIdentifier {
public int64 ordering { get; private set; }
public SmtpOutboxEmailIdentifier(int64 message_id, int64 ordering) {
base (message_id);
base ("%s:%lld".printf(get_type().name(), message_id));
this.ordering = ordering;
}

28
src/engine/imap-db/outbox/smtp-outbox-folder.vala
View file

@ -73,6 +73,34 @@ private class Geary.SmtpOutboxFolder : Geary.AbstractLocalFolder, Geary.FolderSu
do_postman_async.begin();
}
public override async void find_boundaries_async(Gee.Collection<Geary.EmailIdentifier> ids,
out Geary.EmailIdentifier? low, out Geary.EmailIdentifier? high,
Cancellable? cancellable = null) throws Error {
SmtpOutboxEmailIdentifier? outbox_low = null;
SmtpOutboxEmailIdentifier? outbox_high = null;
yield db.exec_transaction_async(Db.TransactionType.RO, (cx) => {
foreach (Geary.EmailIdentifier id in ids) {
SmtpOutboxEmailIdentifier? outbox_id = id as SmtpOutboxEmailIdentifier;
if (outbox_id == null)
continue;
OutboxRow? row = do_fetch_row_by_ordering(cx, outbox_id.ordering, cancellable);
if (row == null)
continue;
if (outbox_low == null || outbox_id.ordering < outbox_low.ordering)
outbox_low = outbox_id;
if (outbox_high == null || outbox_id.ordering > outbox_high.ordering)
outbox_high = outbox_id;
}
return Db.TransactionOutcome.DONE;
}, cancellable);
low = outbox_low;
high = outbox_high;
}
// Used solely for debugging, hence "(no subject)" not marked for translation
private static string message_subject(RFC822.Message message) {
return (message.subject != null && !String.is_empty(message.subject.to_string()))

6
src/engine/imap-engine/imap-engine-generic-account.vala
View file

@ -555,8 +555,7 @@ private abstract class Geary.ImapEngine.GenericAccount : Geary.AbstractAccount {
return yield local.fetch_email_async(check_id(email_id), required_fields, cancellable);
}
public override async Gee.Collection<Geary.Email>? local_search_async(string query,
Geary.Email.Field requested_fields, bool partial_ok, Geary.FolderPath? email_id_folder_path,
public override async Gee.Collection<Geary.EmailIdentifier>? local_search_async(string query,
int limit = 100, int offset = 0, Gee.Collection<Geary.FolderPath?>? folder_blacklist = null,
Gee.Collection<Geary.EmailIdentifier>? search_ids = null, Cancellable? cancellable = null) throws Error {
if (offset < 0)
@ -565,8 +564,7 @@ private abstract class Geary.ImapEngine.GenericAccount : Geary.AbstractAccount {
previous_prepared_search_query = local.prepare_search_query(query);
return yield local.search_async(previous_prepared_search_query,
requested_fields, partial_ok, email_id_folder_path, limit, offset,
folder_blacklist, search_ids, cancellable);
limit, offset, folder_blacklist, search_ids, cancellable);
}
public override async Gee.Collection<string>? get_search_matches_async(

25
src/engine/imap-engine/imap-engine-generic-folder.vala
View file

@ -606,6 +606,31 @@ private class Geary.ImapEngine.GenericFolder : Geary.AbstractFolder, Geary.Folde
remote_semaphore.notify_result(false, null);
}
public override async void find_boundaries_async(Gee.Collection<Geary.EmailIdentifier> ids,
out Geary.EmailIdentifier? low, out Geary.EmailIdentifier? high,
Cancellable? cancellable = null) throws Error {
low = null;
high = null;
Gee.MultiMap<Geary.EmailIdentifier, Geary.FolderPath>? map
= yield account.get_containing_folders_async(ids, cancellable);
if (map != null) {
Gee.ArrayList<Geary.EmailIdentifier> in_folder = new Gee.ArrayList<Geary.EmailIdentifier>();
foreach (Geary.EmailIdentifier id in map.get_keys()) {
if (path in map.get(id))
in_folder.add(id);
}
if (in_folder.size > 0) {
Gee.SortedSet<Geary.EmailIdentifier> sorted = Geary.EmailIdentifier.sort(in_folder);
low = sorted.first();
high = sorted.last();
}
}
}
private void on_email_complete(Gee.Collection<Geary.EmailIdentifier> email_ids) {
notify_email_locally_complete(email_ids);
}

11
src/engine/nonblocking/nonblocking-mailbox.vala
View file

@ -71,20 +71,19 @@ public class Geary.Nonblocking.Mailbox<G> : BaseObject {
}
/**
* Remove messages matching the given predicate. Return the number of
* removed messages.
* Remove messages matching the given predicate. Return the removed messages.
*/
public int remove_matching(owned Gee.Predicate<G> predicate) {
int count = 0;
public Gee.Collection<G> remove_matching(owned Gee.Predicate<G> predicate) {
Gee.ArrayList<G> removed = new Gee.ArrayList<G>();
// Iterate over a copy so we can modify the original.
foreach (G msg in queue.to_array()) {
if (predicate(msg)) {
queue.remove(msg);
++count;
removed.add(msg);
}
}
return count;
return removed;
}
public async G recv_async(Cancellable? cancellable = null) throws Error {

8
src/engine/util/util-collection.vala
View file

@ -13,6 +13,14 @@ public Gee.ArrayList<G> to_array_list<G>(Gee.Collection<G> c) {
return list;
}
public Gee.HashMap<Key, Value> to_hash_map<Key, Value>(
Gee.Collection<Value> c, Gee.MapFunc<Key, Value> key_selector) {
Gee.HashMap<Key, Value> map = new Gee.HashMap<Key, Value>();
foreach (Value v in c)
map.set(key_selector(v), v);
return map;
}
public void add_all_array<G>(Gee.Collection<G> c, G[] ar) {
foreach (G g in ar)
c.add(g);