Further work on detecting message removal when folder first selected: #3805

Needed to rethink storage strategies as I researched this and realized that a true scarce database -- where the database is sparsely populated both in columns and rows -- is not feasible due to IMAP's UID rules.  The strategy now means that the database rows are contiguous from the highest (newest) message to the oldest *requested by the user*.  This is a better situation than having to download the UID for the entire folder.

sql/Create.sql

@@ -51,11 +51,12 @@ CREATE TABLE MessageLocationTable (
     id INTEGER PRIMARY KEY,
     message_id INTEGER REFERENCES MessageTable ON DELETE CASCADE,
     folder_id INTEGER REFERENCES FolderTable ON DELETE CASCADE,
-    position INTEGER
+    ordering INTEGER
 );
 
 CREATE INDEX MessageLocationTableMessageIDIndex ON MessageLocationTable(message_id);
 CREATE INDEX MessageLocationTableFolderIDIndex ON MessageLocationTable(folder_id);
+CREATE INDEX MessageLocationTableOrderingIndex ON MessageLocationTable(ordering ASC);
 
 --
 -- IMAP-specific tables
@@ -68,7 +69,9 @@ CREATE INDEX MessageLocationTableFolderIDIndex ON MessageLocationTable(folder_id
 CREATE TABLE ImapFolderPropertiesTable (
     id INTEGER PRIMARY KEY,
     folder_id INTEGER UNIQUE REFERENCES FolderTable ON DELETE CASCADE,
+    last_seen_total INTEGER,
     uid_validity INTEGER,
+    uid_next INTEGER,
     attributes TEXT
 );
 
@@ -81,20 +84,10 @@ CREATE INDEX ImapFolderPropertiesTableFolderIDIndex ON ImapFolderPropertiesTable
 CREATE TABLE ImapMessagePropertiesTable (
     id INTEGER PRIMARY KEY,
     message_id INTEGER UNIQUE REFERENCES MessageTable ON DELETE CASCADE,
-    flags TEXT
+    flags TEXT,
+    internaldate TEXT,
+    rfc822_size INTEGER
 );
 
 CREATE INDEX ImapMessagePropertiesTableMessageIDIndex ON ImapMessagePropertiesTable(message_id);
 
---
--- ImapMessageLocationPropertiesTable
---
-
-CREATE TABLE ImapMessageLocationPropertiesTable (
-    id INTEGER PRIMARY KEY,
-    location_id INTEGER UNIQUE REFERENCES MessageLocationTable ON DELETE CASCADE,
-    uid INTEGER
-);
-
-CREATE INDEX ImapMessageLocationPropertiesTableLocationIDIndex ON ImapMessageLocationPropertiesTable(location_id);
-

src/client/ui/main-window.vala

@@ -214,7 +214,7 @@ public class MainWindow : Gtk.Window {
         
         yield current_folder.open_async(true);
         
-        current_folder.lazy_list_email_async(1, 1000, MessageListStore.REQUIRED_FIELDS,
+        current_folder.lazy_list_email_async(-1, 50, MessageListStore.REQUIRED_FIELDS,
             on_list_email_ready);
     }
     

src/client/ui/message-list-store.vala

@@ -61,7 +61,7 @@ public class MessageListStore : Gtk.TreeStore {
     
     // The Email should've been fetched with REQUIRED_FIELDS.
     public void append_envelope(Geary.Email envelope) {
-        assert(envelope.fields.fulfills(Geary.Email.Field.ENVELOPE));
+        assert(envelope.fields.fulfills(REQUIRED_FIELDS));
         
         Gtk.TreeIter iter;
         append(out iter, null);

src/common/common-arrays.vala

@@ -0,0 +1,49 @@
+/* Copyright 2011 Yorba Foundation
+ *
+ * This software is licensed under the GNU Lesser General Public License
+ * (version 2.1 or later).  See the COPYING file in this distribution. 
+ */
+
+namespace Arrays {
+
+public G[]? list_to_array<G>(Gee.List<G>? list) {
+    if (list == null)
+        return null;
+    
+    int length = list.size;
+    
+    G[] array = new G[length];
+    for (int ctr = 0; ctr < length; ctr++)
+        array[ctr] = list[ctr];
+    
+    return array;
+}
+
+public int int_find_low(int[] ar) {
+    assert(ar.length > 0);
+    
+    int low = int.MAX;
+    foreach (int i in ar) {
+        if (i < low)
+            low = i;
+    }
+    
+    return low;
+}
+
+public void int_find_high_low(int[] ar, out int low, out int high) {
+    assert(ar.length > 0);
+    
+    low = int.MAX;
+    high = int.MIN;
+    foreach (int i in ar) {
+        if (i < low)
+            low = i;
+        
+        if (i > high)
+            high = i;
+    }
+}
+
+}
+

src/engine/api/geary-abstract-folder.vala

@@ -84,6 +84,9 @@ public abstract class Geary.AbstractFolder : Object, Geary.Folder {
     public abstract async Geary.Email fetch_email_async(int position, Geary.Email.Field required_fields,
         Cancellable? cancellable = null) throws Error;
     
+    public abstract async void remove_email_async(Geary.Email email, Cancellable? cancellable = null)
+        throws Error;
+    
     public virtual string to_string() {
         return get_path().to_string();
     }

src/engine/api/geary-email.vala

@@ -36,7 +36,7 @@ public class Geary.Email : Object {
             return (this & required_fields) == required_fields;
         }
         
-        public inline bool is_set(Field required_fields) {
+        public inline bool is_any_set(Field required_fields) {
             return (this & required_fields) != 0;
         }
         

src/engine/api/geary-engine-error.vala

@@ -11,6 +11,7 @@ public errordomain Geary.EngineError {
     NOT_FOUND,
     READONLY,
     BAD_PARAMETERS,
-    INCOMPLETE_MESSAGE
+    INCOMPLETE_MESSAGE,
+    SERVER_UNAVAILABLE
 }
 

src/engine/api/geary-engine-folder.vala

@@ -7,11 +7,10 @@
 private class Geary.EngineFolder : Geary.AbstractFolder {
     private const int REMOTE_FETCH_CHUNK_COUNT = 10;
     
-    protected RemoteAccount remote;
-    protected LocalAccount local;
-    protected RemoteFolder? remote_folder = null;
-    protected LocalFolder local_folder;
-    
+    private RemoteAccount remote;
+    private LocalAccount local;
+    private RemoteFolder? remote_folder = null;
+    private LocalFolder local_folder;
     private bool opened = false;
     private Geary.Common.NonblockingSemaphore remote_semaphore =
         new Geary.Common.NonblockingSemaphore(true);
@@ -43,6 +42,22 @@ private class Geary.EngineFolder : Geary.AbstractFolder {
         throw new EngineError.READONLY("Engine currently read-only");
     }
     
+    /**
+     * This method is called by EngineFolder when the folder has been opened.  It allows for
+     * subclasses to examine either folder and cleanup any inconsistencies that have developed
+     * since the last time it was opened.
+     *
+     * Implementations should *not* use this as an opportunity to re-sync the entire database;
+     * EngineFolder does that automatically on-demand.  Rather, this should be used to re-sync
+     * inconsistencies that hamper or preclude fetching messages out of the database accurately.
+     *
+     * This will only be called if both the local and remote folder have been opened.
+     */
+    protected virtual async bool prepare_opened_folder(Geary.Folder local_folder, Geary.Folder remote_folder,
+        Cancellable? cancellable) throws Error {
+        return true;
+    }
+    
     public override async void open_async(bool readonly, Cancellable? cancellable = null) throws Error {
         if (opened)
             throw new EngineError.ALREADY_OPEN("Folder %s already open", to_string());
@@ -57,43 +72,53 @@ private class Geary.EngineFolder : Geary.AbstractFolder {
         // wait_for_remote_to_open(), which uses a NonblockingSemaphore to indicate that the remote
         // is open (or has failed to open).  This allows for early calls to list and fetch emails
         // can work out of the local cache until the remote is ready.
-        open_remote_async.begin(readonly, cancellable, on_open_remote_completed);
+        open_remote_async.begin(readonly, cancellable);
         
         opened = true;
     }
     
-    private async void open_remote_async(bool readonly, Cancellable? cancellable) throws Error {
-        RemoteFolder folder = (RemoteFolder) yield remote.fetch_folder_async(local_folder.get_path(),
-            cancellable);
-        yield folder.open_async(readonly, cancellable);
-        
-        remote_folder = folder;
-        remote_folder.updated.connect(on_remote_updated);
-        
-        remote_semaphore.notify();
-    }
-    
-    private void on_open_remote_completed(Object? source, AsyncResult result) {
+    private async void open_remote_async(bool readonly, Cancellable? cancellable) {
         try {
-            open_remote_async.end(result);
+            debug("Opening remote %s", local_folder.get_path().to_string());
+            RemoteFolder folder = (RemoteFolder) yield remote.fetch_folder_async(local_folder.get_path(),
+                cancellable);
+            yield folder.open_async(readonly, cancellable);
             
-            notify_opened(Geary.Folder.OpenState.BOTH);
-        } catch (Error err) {
-            debug("Unable to open remote folder %s: %s", to_string(), err.message);
-            
-            remote_folder = null;
-            try {
-                remote_semaphore.notify();
-            } catch (Error err) {
-                debug("Unable to notify remote folder ready: %s", err.message);
+            // allow subclasses to examine the opened folder and resolve any vital
+            // inconsistencies
+            if (yield prepare_opened_folder(local_folder, folder, cancellable)) {
+                // update flags, properties, etc.
+                yield local.update_folder_async(folder, cancellable);
+                
+                // all set; bless the remote folder as opened
+                remote_folder = folder;
+                remote_folder.updated.connect(on_remote_updated);
+            } else {
+                debug("Unable to prepare remote folder %s: prepare_opened_file() failed", to_string());
             }
-            
-            notify_opened(Geary.Folder.OpenState.LOCAL);
+        } catch (Error err) {
+            debug("Unable to open or prepare remote folder %s: %s", to_string(), err.message);
         }
+        
+        // notify any threads of execution waiting for the remote folder to open that the result
+        // of that operation is ready
+        try {
+            remote_semaphore.notify();
+        } catch (Error notify_err) {
+            debug("Unable to fire semaphore notifying remote folder ready/not ready: %s",
+                notify_err.message);
+        }
+        
+        // notify any subscribers with similar information
+        notify_opened(
+            (remote_folder != null) ? Geary.Folder.OpenState.BOTH : Geary.Folder.OpenState.LOCAL);
     }
     
-    private async void wait_for_remote_to_open() throws Error {
+    // Returns true if the remote folder is ready, false otherwise
+    private async bool wait_for_remote_to_open() throws Error {
         yield remote_semaphore.wait_async();
+        
+        return (remote_folder != null);
     }
     
     public override async void close_async(Cancellable? cancellable = null) throws Error {
@@ -118,8 +143,14 @@ private class Geary.EngineFolder : Geary.AbstractFolder {
     }
     
     public override async int get_email_count(Cancellable? cancellable = null) throws Error {
-        // TODO
-        return 0;
+        // TODO: Use monitoring to avoid round-trip to the server
+        if (!opened)
+            throw new EngineError.OPEN_REQUIRED("%s is not open", to_string());
+        
+        if (remote_folder != null)
+            return yield remote_folder.get_email_count(cancellable);
+        
+        return yield local_folder.get_email_count(cancellable);
     }
     
     public override async Gee.List<Geary.Email>? list_email_async(int low, int count,
@@ -144,8 +175,7 @@ private class Geary.EngineFolder : Geary.AbstractFolder {
     private async void do_list_email_async(int low, int count, Geary.Email.Field required_fields,
         Gee.List<Geary.Email>? accumulator, EmailCallback? cb, Cancellable? cancellable = null)
         throws Error {
-        assert(low >= 1);
-        assert(count >= 0 || count == -1);
+        check_span_specifiers(low, count);
         
         if (!opened)
             throw new EngineError.OPEN_REQUIRED("%s is not open", to_string());
@@ -158,6 +188,11 @@ private class Geary.EngineFolder : Geary.AbstractFolder {
             return;
         }
         
+        // normalize the position (ordering) of what's available locally with the situation on
+        // the server
+        int remote_count = yield normalize_email_positions_async(low, count, cancellable);
+        normalize_span_specifiers(ref low, ref count, remote_count);
+        
         Gee.List<Geary.Email>? local_list = null;
         try {
             local_list = yield local_folder.list_email_async(low, count, required_fields,
@@ -259,6 +294,12 @@ private class Geary.EngineFolder : Geary.AbstractFolder {
             return;
         }
         
+        // normalize the position (ordering) of what's available locally with the situation on
+        // the server
+        int low, high;
+        Arrays.int_find_high_low(by_position, out low, out high);
+        yield normalize_email_positions_async(low, high - low + 1, cancellable);
+        
         Gee.List<Geary.Email>? local_list = null;
         try {
             local_list = yield local_folder.list_email_sparse_async(by_position, required_fields,
@@ -346,17 +387,18 @@ private class Geary.EngineFolder : Geary.AbstractFolder {
     private async Gee.List<Geary.Email>? remote_list_email(int[] needed_by_position,
         Geary.Email.Field required_fields, EmailCallback? cb, Cancellable? cancellable) throws Error {
         // possible to call remote multiple times, wait for it to open once and go
-        yield wait_for_remote_to_open();
+        if (!yield wait_for_remote_to_open())
+            return null;
         
         debug("Background fetching %d emails for %s", needed_by_position.length, to_string());
         
         Gee.List<Geary.Email> full = new Gee.ArrayList<Geary.Email>();
         
         int index = 0;
-        while (index <= needed_by_position.length) {
+        while (index < needed_by_position.length) {
             // if a callback is specified, pull the messages down in chunks, so they can be reported
             // incrementally
-            unowned int[] list;
+            int[] list;
             if (cb != null) {
                 int list_count = int.min(REMOTE_FETCH_CHUNK_COUNT, needed_by_position.length - index);
                 list = needed_by_position[index:index + list_count];
@@ -443,7 +485,9 @@ private class Geary.EngineFolder : Geary.AbstractFolder {
             fields = fields.set(Geary.Email.Field.REFERENCES);
         
         // fetch from network
-        yield wait_for_remote_to_open();
+        if (!yield wait_for_remote_to_open())
+            throw new EngineError.SERVER_UNAVAILABLE("No connection to %s", remote.to_string());
+        
         Geary.Email email = yield remote_folder.fetch_email_async(num, fields, cancellable);
         
         // save to local store
@@ -452,10 +496,72 @@ private class Geary.EngineFolder : Geary.AbstractFolder {
         return email;
     }
     
+    public override async void remove_email_async(Geary.Email email, Cancellable? cancellable = null)
+        throws Error {
+        if (!opened)
+            throw new EngineError.OPEN_REQUIRED("Folder %s not opened", to_string());
+        
+        if (remote_folder == null) {
+            throw new EngineError.READONLY("Unable to delete from %s: remote unavailable",
+                to_string());
+        }
+        
+        yield remote_folder.remove_email_async(email, cancellable);
+        yield local_folder.remove_email_async(email, cancellable);
+    }
+    
     private void on_local_updated() {
     }
     
     private void on_remote_updated() {
     }
+    
+    // In order to maintain positions for all messages without storing all of them locally,
+    // the database stores entries for the lowest requested email to the highest (newest), which
+    // means there can be no gaps between the last in the database and the last on the server.
+    // This method takes care of that.
+    //
+    // Returns the email count on remote_folder.
+    private async int normalize_email_positions_async(int low, int count, Cancellable? cancellable)
+        throws Error {
+        if (!yield wait_for_remote_to_open())
+            throw new EngineError.SERVER_UNAVAILABLE("No connection to %s", remote.to_string());
+        
+        int local_count = yield local_folder.get_email_count(cancellable);
+        int remote_count = yield remote_folder.get_email_count(cancellable);
+        
+        // fixup span specifier
+        normalize_span_specifiers(ref low, ref count, remote_count);
+        
+        // Only prefetch properties for messages not being asked for by the user
+        // (any messages that may be between the user's high and the remote's high, assuming that
+        // all messages in local_count are contiguous from the highest email position, which is
+        // taken care of my prepare_opened_folder_async())
+        int local_low = remote_count - local_count + 1;
+        if (low >= local_low)
+            return remote_count;
+        
+        int prefetch_count = local_low - low;
+        
+        debug("prefetching %d (%d) for %s", low, prefetch_count, to_string());
+        
+        // Use PROPERTIES as they're the most useful information for certain actions (such as
+        // finding duplicates when we start using INTERNALDATE and RFC822.SIZE) and cheap to fetch
+        // TODO: Consider only fetching their UID; would need Geary.Email.Field.LOCATION (or\
+        // perhaps NONE is considered a call for just the UID).
+        Gee.List<Geary.Email>? list = yield remote_folder.list_email_async(low, prefetch_count,
+            Geary.Email.Field.PROPERTIES, cancellable);
+        if (list == null || list.size != prefetch_count) {
+            throw new EngineError.BAD_PARAMETERS("Unable to prefetch %d email starting at %d in %s",
+                count, low, to_string());
+        }
+        
+        foreach (Geary.Email email in list)
+            yield local_folder.create_email_async(email, cancellable);
+        
+        debug("prefetched %d for %s", prefetch_count, to_string());
+        
+        return remote_count;
+    }
 }
 

src/engine/api/geary-folder.vala

@@ -116,12 +116,12 @@ public interface Geary.Folder : Object {
      * from 1 to n.
      *
      * Note that this only returns the number of messages available to the backing medium.  In the
-     * case of the local store, this might be less than the number on the network server.  Folders
+     * case of the local store, this might differ from the number on the network server.  Folders
      * created by Engine are aggregating objects and will return the true count.  However, this
      * might require a round-trip to the server.
      *
-     * Also note that local folders may be sparsely populated.  get_count() returns the last position
-     * available, but not all emails from 1 to n may be available.
+     * Also note that local folders may be sparsely populated.  get_email_count() returns the last
+     * position available, but not all emails from 1 to n may be available.
      *
      * The Folder must be opened prior to attempting this operation.
      */
@@ -147,12 +147,20 @@ public interface Geary.Folder : Object {
     /**
      * Returns a list of messages that fulfill the required_fields flags starting at the low
      * position and moving up to (low + count).  If count is -1, the returned list starts at low
-     * and proceeds to all available emails.  The returned list is not guaranteed to be in any
-     * particular order.
+     * and proceeds to all available emails.  If low is -1, the *last* (most recent) 'count' emails
+     * are returned.  If both low and count are -1, it's no different than calling with low as
+     * 1 and count -1, that is, all emails are returned.  (See normalize_span_specifiers() for
+     * a utility function that handles all aspects of these requirements.)
+     *
+     * The returned list is not guaranteed to be in any particular order.  The position index
+     * (starting from low) *is* ordered, however, from oldest to newest (in terms of receipt by the 
+     * SMTP server, not necessarily the Sent: field), so if the caller wants the latest emails,
+     * they should calculate low by subtracting from get_email_count() or set low to -1 and use
+     * count to fetch the last n emails.
      *
      * If any position in low to (low + count) are out of range, only the email within range are
-     * reported.  No error is thrown.  This allows callers to blindly request the first n emails
-     * in a folder without determining the count first.
+     * reported.  No error is thrown.  This allows callers to blindly request the first or last n
+     * emails in a folder without determining the count first.
      *
      * Note that this only returns the emails with the required fields that are available to the
      * Folder's backing medium.  The local store may have fewer or incomplete messages, meaning that
@@ -163,13 +171,9 @@ public interface Geary.Folder : Object {
      * and fetch from the network only what it needs, so that the caller gets a full list.
      * Note that this means the call may require a round-trip to the server.
      *
-     * TODO: Delayed listing methods (where what's available are reported via a callback after the
-     * async method has completed) will be implemented in the future for more responsive behavior.
-     * These may be operations only available from Folders returned by Engine.
-     *
      * The Folder must be opened prior to attempting this operation.
      *
-     * low is one-based.
+     * low is one-based, unless -1 is specified, as explained above.
      */
     public abstract async Gee.List<Geary.Email>? list_email_async(int low, int count,
         Geary.Email.Field required_fields, Cancellable? cancellable = null) throws Error;
@@ -193,8 +197,7 @@ public interface Geary.Folder : Object {
      * only the emails within range are reported.  The list is not guaranteed to be in any
      * particular order.
      *
-     * See the notes in list_email_async() regarding issues about local versus remote stores and
-     * possible future additions to the API.
+     * See the notes in list_email_async() regarding issues about local versus remote stores.
      *
      * The Folder must be opened prior to attempting this operation.
      *
@@ -228,6 +231,56 @@ public interface Geary.Folder : Object {
     public abstract async Geary.Email fetch_email_async(int position, Geary.Email.Field required_fields,
         Cancellable? cancellable = null) throws Error;
     
+    /**
+     * Removes the email from the folder, determined by its EmailLocation.  If the email location
+     * is invalid for any reason, EngineError.NOT_FOUND is thrown.
+     *
+     * The Folder must be opened prior to attempting this operation.
+     */
+    public abstract async void remove_email_async(Geary.Email email, Cancellable? cancellable = null)
+        throws Error;
+    
+    /**
+     * check_span_specifiers() verifies that the span specifiers match the requirements set by
+     * list_email_async() and lazy_list_email_async().  If not, this method throws
+     * EngineError.BAD_PARAMETERS.
+     */
+    protected static void check_span_specifiers(int low, int count) throws EngineError {
+        if ((low < 1 && low != -1) || (count < 0 && count != -1))
+            throw new EngineError.BAD_PARAMETERS("low=%d count=%d", low, count);
+    }
+    
+    /**
+     * normalize_span_specifiers() deals with the varieties of span specifiers that can be passed
+     * to list_email_async() and lazy_list_email_async().  Note that this function is for
+     * implementations to convert 'low' and 'count' into positive values (1-based in the case of
+     * low) that are within an appropriate range.
+     *
+     * The caller should plug in 'low' and 'count' passed from the user as well as the total
+     * number of emails available (i.e. the complete span is 1..total).
+     */
+    protected static void normalize_span_specifiers(ref int low, ref int count, int total) 
+        throws EngineError {
+        check_span_specifiers(low, count);
+        
+        if (total < 0)
+            throw new EngineError.BAD_PARAMETERS("total=%d", total);
+        
+        // if both are -1, it's no different than low=1 count=-1 (that is, return all email)
+        if (low == -1 && count == -1)
+            low = 1;
+        
+        // if count is -1, it's like a globbed star (return everything starting at low)
+        if (count == -1 || total == 0)
+            count = total;
+            
+        if (low == -1)
+            low = (total - count).clamp(1, total);
+        
+        if ((low + count - 1) > total)
+            count = (total - low + 1).clamp(1, total);
+    }
+    
     /**
      * Used for debugging.  Should not be used for user-visible labels.
      */

src/engine/api/geary-generic-imap-folder.vala

@@ -8,4 +8,167 @@ private class Geary.GenericImapFolder : Geary.EngineFolder {
     public GenericImapFolder(RemoteAccount remote, LocalAccount local, LocalFolder local_folder) {
         base (remote, local, local_folder);
     }
+    
+    // Check if the remote folder's ordering has changed since last opened
+    protected override async bool prepare_opened_folder(Geary.Folder local_folder,
+        Geary.Folder remote_folder, Cancellable? cancellable) throws Error {
+        Geary.Imap.FolderProperties? local_properties =
+            (Geary.Imap.FolderProperties?) local_folder.get_properties();
+        Geary.Imap.FolderProperties? remote_properties =
+            (Geary.Imap.FolderProperties?) remote_folder.get_properties();
+        
+        // both sets of properties must be available
+        if (local_properties == null) {
+            debug("Unable to verify UID validity for %s: missing local properties", get_path().to_string());
+            
+            return false;
+        }
+        
+        if (remote_properties == null) {
+            debug("Unable to verify UID validity for %s: missing remote properties", get_path().to_string());
+            
+            return false;
+        }
+        
+        // and both must have their next UID's (it's possible they don't if it's a non-selectable
+        // folder)
+        if (local_properties.uid_next == null || local_properties.uid_validity == null) {
+            debug("Unable to verify UID next for %s: missing local UID next or validity",
+                get_path().to_string());
+            
+            return false;
+        }
+        
+        if (remote_properties.uid_next == null || remote_properties.uid_validity == null) {
+            debug("Unable to verify UID next for %s: missing remote UID next or validity",
+                get_path().to_string());
+            
+            return false;
+        }
+        
+        if (local_properties.uid_validity.value != remote_properties.uid_validity.value) {
+            // TODO: Don't deal with UID validity changes yet
+            debug("UID validity changed: %lld -> %lld", local_properties.uid_validity.value,
+                remote_properties.uid_validity.value);
+            breakpoint();
+        }
+        
+        Geary.Imap.Folder imap_remote_folder = (Geary.Imap.Folder) remote_folder;
+        Geary.Sqlite.Folder imap_local_folder = (Geary.Sqlite.Folder) local_folder;
+        
+        // if same, no problem-o
+        if (local_properties.uid_next.value != remote_properties.uid_next.value) {
+            debug("UID next changed: %lld -> %lld", local_properties.uid_next.value,
+                remote_properties.uid_next.value);
+            
+            // fetch everything from the last seen UID (+1) to the current next UID
+            // TODO: Could break this fetch up in chunks if it helps
+            Gee.List<Geary.Email>? newest = yield imap_remote_folder.list_email_uid_async(
+                local_properties.uid_next, null, Geary.Email.Field.PROPERTIES, cancellable);
+            
+            if (newest != null && newest.size > 0) {
+                debug("saving %d newest emails", newest.size);
+                foreach (Geary.Email email in newest) {
+                    try {
+                        yield local_folder.create_email_async(email, cancellable);
+                    } catch (Error newest_err) {
+                        debug("Unable to save new email in %s: %s", to_string(), newest_err.message);
+                    }
+                }
+            }
+        }
+        
+        // fetch email from earliest email to last to (a) remove any deletions and (b) update
+        // any flags that may have changed
+        Geary.Imap.UID last_uid = new Geary.Imap.UID(local_properties.uid_next.value - 1);
+        Geary.Imap.UID? earliest_uid = yield imap_local_folder.get_earliest_uid_async(cancellable);
+        
+        // if no earliest UID, that means no messages in local store, so nothing to update
+        if (earliest_uid == null || !earliest_uid.is_valid()) {
+            debug("No earliest UID in %s, nothing to update", to_string());
+            
+            return true;
+        }
+        
+        Gee.List<Geary.Email>? old_local = yield imap_local_folder.list_email_uid_async(earliest_uid,
+            last_uid, Geary.Email.Field.PROPERTIES, cancellable);
+        int local_length = (old_local != null) ? old_local.size : 0;
+        
+        // as before, if empty folder, nothing to update
+        if (local_length == 0) {
+            debug("Folder %s empty, nothing to update", to_string());
+            
+            return true;
+        }
+        
+        Gee.List<Geary.Email>? old_remote = yield imap_remote_folder.list_email_uid_async(earliest_uid,
+            last_uid, Geary.Email.Field.PROPERTIES, cancellable);
+        int remote_length = (old_remote != null) ? old_remote.size : 0;
+        
+        int remote_ctr = 0;
+        int local_ctr = 0;
+        for (;;) {
+            if (local_ctr >= local_length || remote_ctr >= remote_length)
+                break;
+            
+            Geary.Imap.UID remote_uid = 
+                ((Geary.Imap.EmailLocation) old_remote[remote_ctr].location).uid;
+            Geary.Imap.UID local_uid =
+                ((Geary.Imap.EmailLocation) old_local[local_ctr].location).uid;
+            
+            if (remote_uid.value == local_uid.value) {
+                // same, update flags and move on
+                try {
+                    yield imap_local_folder.update_email_async(old_remote[remote_ctr], true,
+                        cancellable);
+                } catch (Error update_err) {
+                    debug("Unable to update old email in %s: %s", to_string(), update_err.message);
+                }
+                
+                remote_ctr++;
+                local_ctr++;
+            } else if (remote_uid.value < local_uid.value) {
+                // one we'd not seen before is present, add and move to next remote
+                try {
+                    yield local_folder.create_email_async(old_remote[remote_ctr], cancellable);
+                } catch (Error add_err) {
+                    debug("Unable to add new email to %s: %s", to_string(), add_err.message);
+                }
+                
+                remote_ctr++;
+            } else {
+                assert(remote_uid.value > local_uid.value);
+                
+                // local's email on the server has been removed, remove locally
+                try {
+                    yield local_folder.remove_email_async(old_local[local_ctr], cancellable);
+                } catch (Error remove_err) {
+                    debug("Unable to remove discarded email from %s: %s", to_string(),
+                        remove_err.message);
+                }
+                
+                local_ctr++;
+            }
+        }
+        
+        // add newly-discovered emails to local store
+        for (; remote_ctr < remote_length; remote_ctr++) {
+            try {
+                yield local_folder.create_email_async(old_remote[remote_ctr], cancellable);
+            } catch (Error append_err) {
+                debug("Unable to append new email to %s: %s", to_string(), append_err.message);
+            }
+        }
+        
+        // remove anything left over
+        for (; local_ctr < local_length; local_ctr++) {
+            try {
+                yield local_folder.remove_email_async(old_local[local_ctr], cancellable);
+            } catch (Error discard_err) {
+                debug("Unable to discard email from %s: %s", to_string(), discard_err.message);
+            }
+        }
+        
+        return true;
+    }
 }

src/engine/api/geary-local-interfaces.vala

@@ -8,6 +8,9 @@ public interface Geary.LocalAccount : Object, Geary.Account {
     public abstract async void clone_folder_async(Geary.Folder folder, Cancellable? cancellable = null)
         throws Error;
     
+    public abstract async void update_folder_async(Geary.Folder folder, Cancellable? cancellable = null)
+        throws Error;
+    
     /**
      * Returns true if the email (identified by its Message-ID) already exists in the account's
      * local store, no matter the folder.

src/engine/imap/api/imap-account.vala

@@ -66,18 +66,17 @@ public class Geary.Imap.Account : Geary.AbstractAccount, Geary.RemoteAccount {
             if (processed == null)
                 delims.set(path.get_root().basename, mbox.delim);
             
-            UIDValidity? uid_validity = null;
+            StatusResults? status = null;
             if (!mbox.attrs.contains(MailboxAttribute.NO_SELECT)) {
                 try {
-                    StatusResults results = yield session_mgr.status_async(path.get_fullpath(),
-                        { StatusDataType.UIDVALIDITY }, cancellable);
-                    uid_validity = results.uidvalidity;
+                    status = yield session_mgr.status_async(path.get_fullpath(),
+                        StatusDataType.all(), cancellable);
                 } catch (Error status_err) {
-                    message("Unable to fetch UID Validity for %s: %s", path.to_string(), status_err.message);
+                    message("Unable to fetch status for %s: %s", path.to_string(), status_err.message);
                 }
             }
             
-            folders.add(new Geary.Imap.Folder(session_mgr, path, uid_validity, mbox));
+            folders.add(new Geary.Imap.Folder(session_mgr, path, status, mbox));
         }
         
         return folders;
@@ -104,14 +103,17 @@ public class Geary.Imap.Account : Geary.AbstractAccount, Geary.RemoteAccount {
             if (mbox == null)
                 throw_not_found(path);
             
-            UIDValidity? uid_validity = null;
+            StatusResults? status = null;
             if (!mbox.attrs.contains(MailboxAttribute.NO_SELECT)) {
-                StatusResults results = yield session_mgr.status_async(processed.get_fullpath(),
-                    { StatusDataType.UIDVALIDITY }, cancellable);
-                uid_validity = results.uidvalidity;
+                try {
+                    status = yield session_mgr.status_async(processed.get_fullpath(),
+                        StatusDataType.all(), cancellable);
+                } catch (Error status_err) {
+                    debug("Unable to get status for %s: %s", processed.to_string(), status_err.message);
+                }
             }
             
-            return new Geary.Imap.Folder(session_mgr, processed, uid_validity, mbox);
+            return new Geary.Imap.Folder(session_mgr, processed, status, mbox);
         } catch (ImapError err) {
             if (err is ImapError.SERVER_ERROR)
                 throw_not_found(path);

src/engine/imap/api/imap-email-properties.vala

@@ -12,9 +12,13 @@ public class Geary.Imap.EmailProperties : Geary.EmailProperties {
     public bool recent { get; private set; }
     public bool seen { get; private set; }
     public MessageFlags flags { get; private set; }
+    public InternalDate? internaldate { get; private set; }
+    public RFC822.Size? rfc822_size { get; private set; }
     
-    public EmailProperties(MessageFlags flags) {
+    public EmailProperties(MessageFlags flags, InternalDate? internaldate, RFC822.Size? rfc822_size) {
         this.flags = flags;
+        this.internaldate = internaldate;
+        this.rfc822_size = rfc822_size;
         
         answered = flags.contains(MessageFlag.ANSWERED);
         deleted = flags.contains(MessageFlag.DELETED);
@@ -24,10 +28,6 @@ public class Geary.Imap.EmailProperties : Geary.EmailProperties {
         seen = flags.contains(MessageFlag.SEEN);
     }
     
-    public bool is_empty() {
-        return (flags.size == 0);
-    }
-    
     public override bool is_unread() {
         return !flags.contains(MessageFlag.SEEN);
     }

src/engine/imap/api/imap-folder-extensions.vala

@@ -0,0 +1,24 @@
+/* Copyright 2011 Yorba Foundation
+ *
+ * This software is licensed under the GNU Lesser General Public License
+ * (version 2.1 or later).  See the COPYING file in this distribution. 
+ */
+
+public interface Geary.Imap.FolderExtensions : Geary.Folder {
+    /**
+     * Much like Geary.Folder.list_email_async(), but this list operation allows for a range of
+     * emails to be specified by their UID rather than position (message number).  If low is null
+     * that indicates to search from the lowest UID (1) to high.  Likewise, if high is null it
+     * indicates to search from low to the highest UID.  Setting both to null will return all
+     * emails in the folder.
+     *
+     * Unlike list_email_async(), this call guarantees that the messages will be returned in UID
+     * order, from lowest to highest.
+     *
+     * The folder must be open before making this call.
+     */
+    public abstract async Gee.List<Geary.Email>? list_email_uid_async(Geary.Imap.UID? low,
+        Geary.Imap.UID? high, Geary.Email.Field fields, Cancellable? cancellable = null)
+        throws Error;
+}
+

src/engine/imap/api/imap-folder-properties.vala

@@ -5,16 +5,40 @@
  */
 
 public class Geary.Imap.FolderProperties : Geary.FolderProperties {
+    public int messages { get; set; }
+    public int recent { get; set; }
+    public int unseen { get; set; }
     public UIDValidity? uid_validity { get; set; }
+    public UID? uid_next { get; set; }
     public MailboxAttributes attrs { get; private set; }
     public Trillian supports_children { get; private set; }
     public Trillian has_children { get; private set; }
     public Trillian is_openable { get; private set; }
     
-    public FolderProperties(UIDValidity? uid_validity, MailboxAttributes attrs) {
+    public FolderProperties(int messages, int recent, int unseen, UIDValidity? uid_validity,
+        UID? uid_next, MailboxAttributes attrs) {
+        this.messages = messages;
+        this.recent = recent;
+        this.unseen = unseen;
         this.uid_validity = uid_validity;
+        this.uid_next = uid_next;
         this.attrs = attrs;
         
+        init_flags();
+    }
+    
+    public FolderProperties.status(StatusResults status, MailboxAttributes attrs) {
+        messages = status.messages;
+        recent = status.recent;
+        unseen = status.unseen;
+        uid_validity = status.uid_validity;
+        uid_next = status.uid_next;
+        this.attrs = attrs;
+        
+        init_flags();
+    }
+    
+    private void init_flags() {
         supports_children = Trillian.from_boolean(!attrs.contains(MailboxAttribute.NO_INFERIORS));
         // \HasNoChildren is an optional attribute and lack of presence doesn't indiciate anything
         supports_children = attrs.contains(MailboxAttribute.HAS_NO_CHILDREN) ? Trillian.TRUE

src/engine/imap/api/imap-folder.vala

@@ -4,7 +4,7 @@
  * (version 2.1 or later).  See the COPYING file in this distribution. 
  */
 
-public class Geary.Imap.Folder : Geary.AbstractFolder, Geary.RemoteFolder {
+public class Geary.Imap.Folder : Geary.AbstractFolder, Geary.RemoteFolder, Geary.Imap.FolderExtensions {
     public const bool CASE_SENSITIVE = true;
     
     private ClientSessionManager session_mgr;
@@ -14,14 +14,17 @@ public class Geary.Imap.Folder : Geary.AbstractFolder, Geary.RemoteFolder {
     private Imap.FolderProperties properties;
     private Mailbox? mailbox = null;
     
-    internal Folder(ClientSessionManager session_mgr, Geary.FolderPath path, UIDValidity? uid_validity,
+    internal Folder(ClientSessionManager session_mgr, Geary.FolderPath path, StatusResults? status,
         MailboxInformation info) {
         this.session_mgr = session_mgr;
         this.info = info;
         this.path = path;
         
         readonly = Trillian.UNKNOWN;
-        properties = new Imap.FolderProperties(uid_validity, info.attrs);
+        
+        properties = (status != null)
+            ? new Imap.FolderProperties.status(status , info.attrs)
+            : new Imap.FolderProperties(0, 0, 0, null, null, info.attrs);
     }
     
     public override Geary.FolderPath get_path() {
@@ -44,8 +47,11 @@ public class Geary.Imap.Folder : Geary.AbstractFolder, Geary.RemoteFolder {
             cancellable);
         // TODO: hook up signals
         
+        // update with new information
         this.readonly = Trillian.from_boolean(readonly);
-        properties.uid_validity = mailbox.uid_validity;
+        
+        properties = new Imap.FolderProperties(mailbox.count, mailbox.recent, mailbox.unseen,
+            mailbox.uid_validity, mailbox.uid_next, properties.attrs);
         
         notify_opened(Geary.Folder.OpenState.REMOTE);
     }
@@ -65,6 +71,7 @@ public class Geary.Imap.Folder : Geary.AbstractFolder, Geary.RemoteFolder {
         if (mailbox == null)
             throw new EngineError.OPEN_REQUIRED("%s not opened", to_string());
         
+        // TODO: Need to monitor folder for updates to the message count
         return mailbox.count;
     }
     
@@ -80,11 +87,10 @@ public class Geary.Imap.Folder : Geary.AbstractFolder, Geary.RemoteFolder {
         if (mailbox == null)
             throw new EngineError.OPEN_REQUIRED("%s not opened", to_string());
         
-        MessageSet msg_set = (count != -1)
-            ? new MessageSet.range(low, count)
-            : new MessageSet.range_to_highest(low);
+        // TODO: Need to use a monitored count
+        normalize_span_specifiers(ref low, ref count, mailbox.count);
         
-        return yield mailbox.list_set_async(msg_set, fields, cancellable);
+        return yield mailbox.list_set_async(new MessageSet.range(low, count), fields, cancellable);
     }
     
     public override async Gee.List<Geary.Email>? list_email_sparse_async(int[] by_position,
@@ -95,6 +101,18 @@ public class Geary.Imap.Folder : Geary.AbstractFolder, Geary.RemoteFolder {
         return yield mailbox.list_set_async(new MessageSet.sparse(by_position), fields, cancellable);
     }
     
+    public async Gee.List<Geary.Email>? list_email_uid_async(Geary.Imap.UID? low,
+        Geary.Imap.UID? high, Geary.Email.Field fields, Cancellable? cancellable = null) throws Error {
+        if (mailbox == null)
+            throw new EngineError.OPEN_REQUIRED("%s not opened", to_string());
+        
+        MessageSet msg_set = (high != null)
+            ? new MessageSet.uid_range((low != null) ? low : new Geary.Imap.UID(1), high)
+            : new MessageSet.uid_range_to_highest(low);
+        
+        return yield mailbox.list_set_async(msg_set, fields, cancellable);
+    }
+    
     public override async Geary.Email fetch_email_async(int position, Geary.Email.Field fields,
         Cancellable? cancellable = null) throws Error {
         if (mailbox == null)
@@ -104,5 +122,17 @@ public class Geary.Imap.Folder : Geary.AbstractFolder, Geary.RemoteFolder {
         
         return yield mailbox.fetch_async(position, fields, cancellable);
     }
+    
+    public override async void remove_email_async(Geary.Email email, Cancellable? cancellable = null)
+        throws Error {
+        if (mailbox == null)
+            throw new EngineError.OPEN_REQUIRED("%s not opened", to_string());
+        
+        Geary.Imap.UID? uid = ((Geary.Imap.EmailLocation) email.location).uid;
+        if (uid == null)
+            throw new EngineError.NOT_FOUND("Removing email requires UID");
+        
+        throw new EngineError.READONLY("IMAP currently read-only");
+    }
 }
 

src/engine/imap/command/imap-command.vala

@@ -15,7 +15,7 @@ public class Geary.Imap.Command : RootParameters {
         this.args = args;
         
         add(tag);
-        add(new StringParameter(name));
+        add(new UnquotedStringParameter(name));
         if (args != null) {
             foreach (string arg in args)
                 add(new StringParameter(arg));

src/engine/imap/command/imap-commands.vala

@@ -90,11 +90,12 @@ public class Geary.Imap.CloseCommand : Command {
 
 public class Geary.Imap.FetchCommand : Command {
     public const string NAME = "fetch";
+    public const string UID_NAME = "uid fetch";
     
     public FetchCommand(Tag tag, MessageSet msg_set, FetchDataType[] data_items) {
-        base (tag, NAME);
+        base (tag, msg_set.is_uid ? UID_NAME : NAME);
         
-        add(new StringParameter(msg_set.value));
+        add(msg_set.to_parameter());
         
         assert(data_items.length > 0);
         if (data_items.length == 1) {

src/engine/imap/decoders/imap-fetch-results.vala

@@ -77,7 +77,7 @@ public class Geary.Imap.FetchResults : Geary.Imap.CommandResults {
         return map.keys;
     }
     
-    public void set_data(FetchDataType data_item, MessageData primitive) {
+    private void set_data(FetchDataType data_item, MessageData primitive) {
         map.set(data_item, primitive);
     }
     

src/engine/imap/decoders/imap-select-examine-results.vala

@@ -18,18 +18,20 @@ public class Geary.Imap.SelectExamineResults : Geary.Imap.CommandResults {
      */
     public int unseen { get; private set; }
     public UIDValidity? uid_validity { get; private set; }
+    public UID? uid_next { get; private set; }
     public Flags? flags { get; private set; }
     public Flags? permanentflags { get; private set; }
     public bool readonly { get; private set; }
     
     private SelectExamineResults(StatusResponse status_response, int exists, int recent, int unseen,
-        UIDValidity? uidvalidity, Flags? flags, Flags? permanentflags, bool readonly) {
+        UIDValidity? uid_validity, UID? uid_next, Flags? flags, Flags? permanentflags, bool readonly) {
         base (status_response);
         
         this.exists = exists;
         this.recent = recent;
         this.unseen = unseen;
         this.uid_validity = uid_validity;
+        this.uid_next = uid_next;
         this.flags = flags;
         this.permanentflags = permanentflags;
         this.readonly = readonly;
@@ -41,8 +43,8 @@ public class Geary.Imap.SelectExamineResults : Geary.Imap.CommandResults {
         int exists = -1;
         int recent = -1;
         int unseen = -1;
-        UIDValidity? uidvalidity = null;
-        UID? uidnext = null;
+        UIDValidity? uid_validity = null;
+        UID? uid_next = null;
         MessageFlags? flags = null;
         MessageFlags? permanentflags = null;
         
@@ -75,12 +77,12 @@ public class Geary.Imap.SelectExamineResults : Geary.Imap.CommandResults {
                             break;
                             
                             case ResponseCodeType.UIDVALIDITY:
-                                uidvalidity = new UIDValidity(
+                                uid_validity = new UIDValidity(
                                     ok_response.response_code.get_as_string(1).as_int());
                             break;
                             
                             case ResponseCodeType.UIDNEXT:
-                                uidnext = new UID(ok_response.response_code.get_as_string(1).as_int());
+                                uid_next = new UID(ok_response.response_code.get_as_string(1).as_int());
                             break;
                             
                             case ResponseCodeType.PERMANENT_FLAGS:
@@ -125,7 +127,7 @@ public class Geary.Imap.SelectExamineResults : Geary.Imap.CommandResults {
             throw new ImapError.PARSE_ERROR("Incomplete SELECT/EXAMINE Response: \"%s\"", response.to_string());
         
         return new SelectExamineResults(response.status_response, exists, recent, unseen,
-            uidvalidity, flags, permanentflags, readonly);
+            uid_validity, uid_next, flags, permanentflags, readonly);
     }
 }
 

src/engine/imap/decoders/imap-status-results.vala

@@ -14,22 +14,22 @@ public class Geary.Imap.StatusResults : Geary.Imap.CommandResults {
      * -1 if not set.
      */
     public int recent { get; private set; }
-    public UID? uidnext { get; private set; }
-    public UIDValidity? uidvalidity { get; private set; }
+    public UID? uid_next { get; private set; }
+    public UIDValidity? uid_validity { get; private set; }
     /**
      * -1 if not set.
      */
     public int unseen { get; private set; }
     
     public StatusResults(StatusResponse status_response, string mailbox, int messages, int recent,
-        UID? uidnext, UIDValidity? uidvalidity, int unseen) {
+        UID? uid_next, UIDValidity? uid_validity, int unseen) {
         base (status_response);
         
         this.mailbox = mailbox;
         this.messages = messages;
         this.recent = recent;
-        this.uidnext = uidnext;
-        this.uidvalidity = uidvalidity;
+        this.uid_next = uid_next;
+        this.uid_validity = uid_validity;
         this.unseen = unseen;
     }
     
@@ -53,8 +53,8 @@ public class Geary.Imap.StatusResults : Geary.Imap.CommandResults {
         
         int messages = -1;
         int recent = -1;
-        UID? uidnext = null;
-        UIDValidity? uidvalidity = null;
+        UID? uid_next = null;
+        UIDValidity? uid_validity = null;
         int unseen = -1;
         
         for (int ctr = 0; ctr < values.get_count(); ctr += 2) {
@@ -72,11 +72,11 @@ public class Geary.Imap.StatusResults : Geary.Imap.CommandResults {
                     break;
                     
                     case StatusDataType.UIDNEXT:
-                        uidnext = new UID(valuep.as_int());
+                        uid_next = new UID(valuep.as_int());
                     break;
                     
                     case StatusDataType.UIDVALIDITY:
-                        uidvalidity = new UIDValidity(valuep.as_int());
+                        uid_validity = new UIDValidity(valuep.as_int());
                     break;
                     
                     case StatusDataType.UNSEEN:
@@ -93,8 +93,8 @@ public class Geary.Imap.StatusResults : Geary.Imap.CommandResults {
             }
         }
         
-        return new StatusResults(response.status_response, mailbox.value, messages, recent, uidnext, 
-            uidvalidity, unseen);
+        return new StatusResults(response.status_response, mailbox.value, messages, recent, uid_next,
+            uid_validity, unseen);
     }
 }
 

src/engine/imap/message/imap-message-data.vala

@@ -23,6 +23,10 @@ public class Geary.Imap.UID : Geary.Common.Int64MessageData, Geary.Imap.MessageD
     public UID(int64 value) {
         base (value);
     }
+    
+    public bool is_valid() {
+        return value >= 1;
+    }
 }
 
 public class Geary.Imap.UIDValidity : Geary.Common.Int64MessageData, Geary.Imap.MessageData {

src/engine/imap/message/imap-message-set.vala

@@ -5,14 +5,23 @@
  */
 
 public class Geary.Imap.MessageSet {
-    public string value { get; private set; }
+    public bool is_uid { get; private set; default = false; }
+    
+    private string value { get; private set; }
     
     public MessageSet(int msg_num) {
-        assert(msg_num >= 0);
+        assert(msg_num > 0);
         
         value = "%d".printf(msg_num);
     }
     
+    public MessageSet.uid(UID uid) {
+        assert(uid.value > 0);
+        
+        value = "%lld".printf(uid.value);
+        is_uid = true;
+    }
+    
     public MessageSet.range(int low_msg_num, int count) {
         assert(low_msg_num > 0);
         assert(count > 0);
@@ -22,12 +31,27 @@ public class Geary.Imap.MessageSet {
             : "%d".printf(low_msg_num);
     }
     
+    public MessageSet.uid_range(UID low, UID high) {
+        assert(low.value > 0);
+        assert(high.value > 0);
+        
+        value = "%lld:%lld".printf(low.value, high.value);
+        is_uid = true;
+    }
+    
     public MessageSet.range_to_highest(int low_msg_num) {
         assert(low_msg_num > 0);
         
         value = "%d:*".printf(low_msg_num);
     }
     
+    public MessageSet.uid_range_to_highest(UID low) {
+        assert(low.value > 0);
+        
+        value = "%lld:*".printf(low.value);
+        is_uid = true;
+    }
+    
     public MessageSet.sparse(int[] msg_nums) {
         value = build_sparse_range(msg_nums);
     }
@@ -68,6 +92,11 @@ public class Geary.Imap.MessageSet {
         value = custom;
     }
     
+    public MessageSet.uid_custom(string custom) {
+        value = custom;
+        is_uid = true;
+    }
+    
     // TODO: It would be more efficient to look for runs in the numbers and form the set specifier
     // with them.
     private static string build_sparse_range(int[] msg_nums) {
@@ -86,5 +115,15 @@ public class Geary.Imap.MessageSet {
         
         return builder.str;
     }
+    
+    public Parameter to_parameter() {
+        // Message sets are not quoted, even if they use an atom-special character (this *might*
+        // be a Gmailism...)
+        return new UnquotedStringParameter(value);
+    }
+    
+    public string to_string() {
+        return value;
+    }
 }
 

src/engine/imap/message/imap-parameter.vala

@@ -82,6 +82,21 @@ public class Geary.Imap.StringParameter : Geary.Imap.Parameter {
     }
 }
 
+/**
+ * This delivers the string to the IMAP server with no quoting or formatting applied.  (Deserializer
+ * will never generate this Parameter.)  This can lead to server errors if misused.  Use only if
+ * absolutely necessary.
+ */
+public class Geary.Imap.UnquotedStringParameter : Geary.Imap.StringParameter {
+    public UnquotedStringParameter(string value) {
+        base (value);
+    }
+    
+    public override async void serialize(Serializer ser) throws Error {
+        ser.push_unquoted_string(value);
+    }
+}
+
 public class Geary.Imap.LiteralParameter : Geary.Imap.Parameter {
     private Geary.Memory.AbstractBuffer buffer;
     

src/engine/imap/transport/imap-client-session-manager.vala

@@ -161,8 +161,6 @@ public class Geary.Imap.ClientSessionManager {
         Gee.HashSet<SelectedContext> contexts) {
         SelectedContext context = (SelectedContext) semantics;
         
-        debug("Mailbox %s freed, closing select/examine", context.name);
-        
         // last reference to the Mailbox has been dropped, so drop the mailbox and move the
         // ClientSession back to the authorized state
         bool removed = contexts.remove(context);
@@ -174,8 +172,6 @@ public class Geary.Imap.ClientSessionManager {
     
     // This should only be called when sessions_mutex is locked.
     private async ClientSession create_new_authorized_session(Cancellable? cancellable) throws Error {
-        debug("Creating new session to %s", cred.server);
-        
         ClientSession new_session = new ClientSession(cred.server, default_port);
         new_session.disconnected.connect(on_disconnected);
         
@@ -187,8 +183,6 @@ public class Geary.Imap.ClientSessionManager {
         
         sessions.add(new_session);
         
-        debug("Created new session to %s, %d total", cred.server, sessions.size);
-        
         return new_session;
     }
     
@@ -231,8 +225,6 @@ public class Geary.Imap.ClientSessionManager {
     }
     
     private void on_disconnected(ClientSession session, ClientSession.DisconnectReason reason) {
-        debug("Client session %s disconnected: %s", session.to_string(), reason.to_string());
-        
         bool removed = sessions.remove(session);
         assert(removed);
     }

src/engine/imap/transport/imap-client-session.vala

@@ -398,8 +398,6 @@ public class Geary.Imap.ClientSession {
         
         if (params.err != null)
             throw params.err;
-        
-        debug("Connected to %s: %s", to_full_string(), connect_response.to_string());
     }
     
     private uint on_connect(uint state, uint event, void *user, Object? object) {
@@ -476,8 +474,6 @@ public class Geary.Imap.ClientSession {
     }
     
     private uint on_connect_denied(uint state, uint event, void *user) {
-        debug("Connection to %s denied by server", to_full_string());
-        
         return State.BROKEN;
     }
     
@@ -494,8 +490,6 @@ public class Geary.Imap.ClientSession {
         
         if (params.err != null)
             throw params.err;
-        
-        debug("Logged in to %s: %s", to_full_string(), params.cmd_response.to_string());
     }
     
     private uint on_login(uint state, uint event, void *user, Object? object) {
@@ -521,8 +515,6 @@ public class Geary.Imap.ClientSession {
     }
     
     private uint on_login_failed(uint state, uint event, void *user) {
-        debug("Login to %s failed", to_full_string());
-        
         return State.NOAUTH;
     }
     
@@ -815,8 +807,6 @@ public class Geary.Imap.ClientSession {
         
         if (params.err != null)
             throw params.err;
-        
-        debug("Logged out of %s: %s", to_string(), params.cmd_response.to_string());
     }
     
     private uint on_logout(uint state, uint event, void *user, Object? object) {

src/engine/imap/transport/imap-deserializer.vala

@@ -67,6 +67,7 @@ public class Geary.Imap.Deserializer {
     private unowned uint8[]? current_buffer = null;
     private bool flow_controlled = true;
     private int ins_priority = Priority.DEFAULT;
+    private char[] atom_specials_exceptions = { ' ', ' ', '\0' };
     
     public signal void flow_control(bool xon);
     
@@ -287,6 +288,10 @@ public class Geary.Imap.Deserializer {
         context = child;
     }
     
+    private char get_current_context_terminator() {
+        return (context is ResponseCode) ? ']' : ')';
+    }
+    
     private State pop() {
         ListParameter? parent = context.get_parent();
         if (parent == null) {
@@ -329,7 +334,8 @@ public class Geary.Imap.Deserializer {
     private uint on_first_param_char(uint state, uint event, void *user) {
         // look for opening characters to special parameter formats, otherwise jump to atom
         // handler (i.e. don't drop this character in the case of atoms)
-        switch (*((unichar *) user)) {
+        unichar ch = *((unichar *) user);
+        switch (ch) {
             case '[':
                 // open response code
                 ResponseCode response_code = new ResponseCode(context);
@@ -350,13 +356,13 @@ public class Geary.Imap.Deserializer {
                 
                 return State.START_PARAM;
             
-            case ')':
-            case ']':
-                // close list or response code
-                return pop();
-            
             default:
-                return on_tag_or_atom_char(State.ATOM, event, user);
+                // if current context's terminator, close the context, otherwise deserializer is
+                // now "in" an Atom
+                if (ch == get_current_context_terminator())
+                    return pop();
+                else
+                    return on_tag_or_atom_char(State.ATOM, event, user);
         }
     }
     
@@ -369,12 +375,18 @@ public class Geary.Imap.Deserializer {
         
         unichar ch = *((unichar *) user);
         
+        // get the terminator for this context and re-use the atom_special_exceptions array to
+        // pass to DataFormat.is_atom_special() (this means not allocating a new array on the heap
+        // for each call here, which isn't a problem because the FSM is non-reentrant)
+        char terminator = get_current_context_terminator();
+        atom_specials_exceptions[1] = terminator;
+        
         // Atom specials includes space and close-parens, but those are handled in particular ways
         // while in the ATOM state, so they're excluded here.  Like atom specials, the space is 
         // treated in a particular way for tags, but unlike atom, the close-parens character is not.
         if (state == State.TAG && DataFormat.is_tag_special(ch, " "))
             return state;
-        else if (state == State.ATOM && DataFormat.is_atom_special(ch, " )"))
+        else if (state == State.ATOM && DataFormat.is_atom_special(ch, (string) atom_specials_exceptions))
             return state;
         
         // message flag indicator is only legal at start of atom
@@ -390,7 +402,7 @@ public class Geary.Imap.Deserializer {
         
         // close-parens/close-square-bracket after an atom indicates end-of-list/end-of-response
         // code
-        if (state == State.ATOM && (ch == ')' || ch == ']')) {
+        if (state == State.ATOM && ch == terminator) {
             save_string_parameter();
             
             return pop();

src/engine/imap/transport/imap-mailbox.vala

@@ -7,8 +7,11 @@
 public class Geary.Imap.Mailbox : Geary.SmartReference {
     public string name { get; private set; }
     public int count { get; private set; }
+    public int recent { get; private set; }
+    public int unseen { get; private set; }
     public bool is_readonly { get; private set; }
-    public UIDValidity uid_validity { get; private set; }
+    public UIDValidity? uid_validity { get; private set; }
+    public UID? uid_next { get; private set; }
     
     private SelectedContext context;
     
@@ -26,8 +29,11 @@ public class Geary.Imap.Mailbox : Geary.SmartReference {
         
         name = context.name;
         count = context.exists;
+        recent = context.recent;
+        unseen = context.unseen;
         is_readonly = context.is_readonly;
         uid_validity = context.uid_validity;
+        uid_next = context.uid_next;
     }
     
     public async Gee.List<Geary.Email>? list_set_async(MessageSet msg_set, Geary.Email.Field fields,
@@ -36,20 +42,27 @@ public class Geary.Imap.Mailbox : Geary.SmartReference {
             throw new ImapError.NOT_SELECTED("Mailbox %s closed", name);
         
         if (fields == Geary.Email.Field.NONE)
-            throw new EngineError.BAD_PARAMETERS("No email fields specify for list");
+            throw new EngineError.BAD_PARAMETERS("No email fields specified");
         
-        CommandResponse resp = yield context.session.send_command_async(
-            new FetchCommand(context.session.generate_tag(), msg_set,
-                fields_to_fetch_data_types(fields)), cancellable);
+        Gee.List<FetchDataType> data_type_list = new Gee.ArrayList<FetchDataType>();
+        fields_to_fetch_data_types(fields, data_type_list);
         
-        if (resp.status_response.status != Status.OK)
-            throw new ImapError.SERVER_ERROR("Server error: %s", resp.to_string());
+        FetchCommand fetch_cmd = new FetchCommand(context.session.generate_tag(), msg_set,
+            Arrays.list_to_array<FetchDataType>(data_type_list));
+        
+        CommandResponse resp = yield context.session.send_command_async(fetch_cmd, cancellable);
+        
+        if (resp.status_response.status != Status.OK) {
+            throw new ImapError.SERVER_ERROR("Server error for %s: %s", fetch_cmd.to_string(),
+                resp.to_string());
+        }
         
         Gee.List<Geary.Email> msgs = new Gee.ArrayList<Geary.Email>();
         
         FetchResults[] results = FetchResults.decode(resp);
         foreach (FetchResults res in results) {
             UID? uid = res.get_data(FetchDataType.UID) as UID;
+            // see fields_to_fetch_data_types() for why this is guaranteed
             assert(uid != null);
             
             Geary.Email email = new Geary.Email(new Geary.Imap.EmailLocation(res.msg_num, uid));
@@ -65,12 +78,18 @@ public class Geary.Imap.Mailbox : Geary.SmartReference {
         if (context.is_closed())
             throw new ImapError.NOT_SELECTED("Mailbox %s closed", name);
         
-        CommandResponse resp = yield context.session.send_command_async(
-            new FetchCommand(context.session.generate_tag(), new MessageSet(msg_num),
-                fields_to_fetch_data_types(fields)), cancellable);
+        Gee.List<FetchDataType> data_type_list = new Gee.ArrayList<FetchDataType>();
+        fields_to_fetch_data_types(fields, data_type_list);
         
-        if (resp.status_response.status != Status.OK)
-            throw new ImapError.SERVER_ERROR("Server error: %s", resp.to_string());
+        FetchCommand fetch_cmd = new FetchCommand(context.session.generate_tag(), new MessageSet(msg_num),
+            Arrays.list_to_array<FetchDataType>(data_type_list));
+        
+        CommandResponse resp = yield context.session.send_command_async(fetch_cmd, cancellable);
+        
+        if (resp.status_response.status != Status.OK) {
+            throw new ImapError.SERVER_ERROR("Server error for %s: %s", fetch_cmd.to_string(),
+                resp.to_string());
+        }
         
         FetchResults[] results = FetchResults.decode(resp);
         if (results.length != 1)
@@ -102,8 +121,15 @@ public class Geary.Imap.Mailbox : Geary.SmartReference {
         disconnected(local);
     }
     
-    private static FetchDataType[] fields_to_fetch_data_types(Geary.Email.Field fields) {
+    private static void fields_to_fetch_data_types(Geary.Email.Field fields,
+        Gee.List<FetchDataType> data_type_list) {
+        // store FetchDataTypes in a set because the same data type may be requested multiple times
+        // by different fields (i.e. ENVELOPE)
         Gee.HashSet<FetchDataType> data_type_set = new Gee.HashSet<FetchDataType>();
+        
+        // UID is always fetched
+        data_type_set.add(FetchDataType.UID);
+        
         foreach (Geary.Email.Field field in Geary.Email.Field.all()) {
             switch (fields & field) {
                 case Geary.Email.Field.DATE:
@@ -123,7 +149,11 @@ public class Geary.Imap.Mailbox : Geary.SmartReference {
                 break;
                 
                 case Geary.Email.Field.PROPERTIES:
+                    // Gmail doesn't like using FAST when combined with other fetch types, so
+                    // do this manually
                     data_type_set.add(FetchDataType.FLAGS);
+                    data_type_set.add(FetchDataType.INTERNALDATE);
+                    data_type_set.add(FetchDataType.RFC822_SIZE);
                 break;
                 
                 case Geary.Email.Field.NONE:
@@ -135,20 +165,16 @@ public class Geary.Imap.Mailbox : Geary.SmartReference {
             }
         }
         
-        assert(data_type_set.size > 0);
-        FetchDataType[] data_types = new FetchDataType[data_type_set.size + 1];
-        int ctr = 0;
-        foreach (FetchDataType data_type in data_type_set)
-            data_types[ctr++] = data_type;
-        
-        // UID is always fetched, no matter what the caller requests
-        data_types[ctr] = FetchDataType.UID;
-        
-        return data_types;
+        data_type_list.add_all(data_type_set);
     }
     
     private static void fetch_results_to_email(FetchResults res, Geary.Email.Field fields,
         Geary.Email email) {
+        // accumulate these to submit Imap.EmailProperties all at once
+        Geary.Imap.MessageFlags? flags = null;
+        InternalDate? internaldate = null;
+        RFC822.Size? rfc822_size = null;
+        
         foreach (FetchDataType data_type in res.get_all_types()) {
             MessageData? data = res.get_data(data_type);
             if (data == null)
@@ -182,8 +208,16 @@ public class Geary.Imap.Mailbox : Geary.SmartReference {
                     email.set_message_body((RFC822.Text) data);
                 break;
                 
+                case FetchDataType.RFC822_SIZE:
+                    rfc822_size = (RFC822.Size) data;
+                break;
+                
                 case FetchDataType.FLAGS:
-                    email.set_email_properties(new Imap.EmailProperties((MessageFlags) data));
+                    flags = (MessageFlags) data;
+                break;
+                
+                case FetchDataType.INTERNALDATE:
+                    internaldate = (InternalDate) data;
                 break;
                 
                 default:
@@ -191,6 +225,9 @@ public class Geary.Imap.Mailbox : Geary.SmartReference {
                 break;
             }
         }
+        
+        if (flags != null && internaldate != null && rfc822_size != null)
+            email.set_email_properties(new Geary.Imap.EmailProperties(flags, internaldate, rfc822_size));
     }
 }
 
@@ -202,8 +239,10 @@ internal class Geary.Imap.SelectedContext : Object, Geary.ReferenceSemantics {
     public string name { get; protected set; }
     public int exists { get; protected set; }
     public int recent { get; protected set; }
+    public int unseen { get; protected set; }
     public bool is_readonly { get; protected set; }
-    public UIDValidity uid_validity { get; protected set; }
+    public UIDValidity? uid_validity { get; protected set; }
+    public UID? uid_next { get; protected set; }
     
     public signal void exists_changed(int exists);
     
@@ -220,7 +259,9 @@ internal class Geary.Imap.SelectedContext : Object, Geary.ReferenceSemantics {
         is_readonly = results.readonly;
         exists = results.exists;
         recent = results.recent;
+        unseen = results.unseen;
         uid_validity = results.uid_validity;
+        uid_next = results.uid_next;
         
         session.current_mailbox_changed.connect(on_session_mailbox_changed);
         session.unsolicited_exists.connect(on_unsolicited_exists);

src/engine/imap/transport/imap-serializer.vala

@@ -56,6 +56,13 @@ public class Geary.Imap.Serializer {
         }
     }
     
+    /**
+     * This will push the string to IMAP as-is.  Use only if you absolutely know what you're doing.
+     */
+    public void push_unquoted_string(string str) throws Error {
+        douts.put_string(str);
+    }
+    
     public void push_space() throws Error {
         douts.put_byte(' ', null);
     }

src/engine/sqlite/api/sqlite-account.vala

@@ -5,7 +5,7 @@
  */
 
 public class Geary.Sqlite.Account : Geary.AbstractAccount, Geary.LocalAccount {
-    private MailDatabase db;
+    private ImapDatabase db;
     private FolderTable folder_table;
     private ImapFolderPropertiesTable folder_properties_table;
     private MessageTable message_table;
@@ -14,7 +14,7 @@ public class Geary.Sqlite.Account : Geary.AbstractAccount, Geary.LocalAccount {
         base ("SQLite account for %s".printf(cred.to_string()));
         
         try {
-            db = new MailDatabase(cred.user);
+            db = new ImapDatabase(cred.user);
         } catch (Error err) {
             error("Unable to open database: %s", err.message);
         }
@@ -61,6 +61,27 @@ public class Geary.Sqlite.Account : Geary.AbstractAccount, Geary.LocalAccount {
                 imap_folder_properties));
     }
     
+    public async void update_folder_async(Geary.Folder folder, Cancellable? cancellable = null)
+        throws Error {
+        Geary.Imap.Folder imap_folder = (Geary.Imap.Folder) folder;
+        Geary.Imap.FolderProperties? imap_folder_properties = (Geary.Imap.FolderProperties?)
+            imap_folder.get_properties();
+        
+        // properties *must* be available
+        assert(imap_folder_properties != null);
+        
+        int64 parent_id = yield fetch_parent_id_async(folder.get_path(), cancellable);
+        
+        FolderRow? row = yield folder_table.fetch_async(parent_id, folder.get_path().basename,
+            cancellable);
+        if (row == null)
+            throw new EngineError.NOT_FOUND("Can't find in local store %s", folder.get_path().to_string());
+        
+        yield folder_properties_table.update_async(row.id,
+            new ImapFolderPropertiesRow.from_imap_properties(folder_properties_table, row.id,
+                imap_folder_properties));
+    }
+    
     public override async Gee.Collection<Geary.Folder> list_folders_async(Geary.FolderPath? parent,
         Cancellable? cancellable = null) throws Error {
         int64 parent_id = (parent != null)

src/engine/sqlite/api/sqlite-folder.vala

@@ -7,18 +7,17 @@
 // TODO: This class currently deals with generic email storage as well as IMAP-specific issues; in
 // the future, to support other email services, will need to break this up.
 
-public class Geary.Sqlite.Folder : Geary.AbstractFolder, Geary.LocalFolder {
+public class Geary.Sqlite.Folder : Geary.AbstractFolder, Geary.LocalFolder, Geary.Imap.FolderExtensions {
     private MailDatabase db;
     private FolderRow folder_row;
-    private Geary.FolderProperties? properties;
+    private Geary.Imap.FolderProperties? properties;
     private MessageTable message_table;
     private MessageLocationTable location_table;
-    private ImapMessageLocationPropertiesTable imap_location_table;
     private ImapMessagePropertiesTable imap_message_properties_table;
     private Geary.FolderPath path;
     private bool opened = false;
     
-    internal Folder(MailDatabase db, FolderRow folder_row, ImapFolderPropertiesRow? properties,
+    internal Folder(ImapDatabase db, FolderRow folder_row, ImapFolderPropertiesRow? properties,
         Geary.FolderPath path) throws Error {
         this.db = db;
         this.folder_row = folder_row;
@@ -27,7 +26,6 @@ public class Geary.Sqlite.Folder : Geary.AbstractFolder, Geary.LocalFolder {
         
         message_table = db.get_message_table();
         location_table = db.get_message_location_table();
-        imap_location_table = db.get_imap_message_location_table();
         imap_message_properties_table = db.get_imap_message_properties_table();
     }
     
@@ -65,8 +63,8 @@ public class Geary.Sqlite.Folder : Geary.AbstractFolder, Geary.LocalFolder {
     public override async int get_email_count(Cancellable? cancellable = null) throws Error {
         check_open();
         
-        // TODO
-        return 0;
+        // TODO: This can be cached and updated when changes occur
+        return yield location_table.fetch_count_for_folder_async(folder_row.id, cancellable);
     }
     
     public override async void create_email_async(Geary.Email email, Cancellable? cancellable = null)
@@ -78,8 +76,8 @@ public class Geary.Sqlite.Folder : Geary.AbstractFolder, Geary.LocalFolder {
         // See if it already exists; first by UID (which is only guaranteed to be unique in a folder,
         // not account-wide)
         int64 message_id;
-        if (yield imap_location_table.search_uid_in_folder(location.uid, folder_row.id, out message_id,
-            cancellable)) {
+        if (yield location_table.does_ordering_exist_async(folder_row.id, location.uid.value,
+            out message_id, cancellable)) {
             throw new EngineError.ALREADY_EXISTS("Email with UID %s already exists in %s",
                 location.uid.to_string(), to_string());
         }
@@ -89,18 +87,16 @@ public class Geary.Sqlite.Folder : Geary.AbstractFolder, Geary.LocalFolder {
             new MessageRow.from_email(message_table, email),
             cancellable);
         
+        // create the message location in the location lookup table using its UID for the ordering
+        // (which fulfills the requirements for the ordering column)
         MessageLocationRow location_row = new MessageLocationRow(location_table, Row.INVALID_ID,
-            message_id, folder_row.id, location.position);
-        int64 location_id = yield location_table.create_async(location_row, cancellable);
-        
-        ImapMessageLocationPropertiesRow imap_location_row = new ImapMessageLocationPropertiesRow(
-            imap_location_table, Row.INVALID_ID, location_id, location.uid);
-        yield imap_location_table.create_async(imap_location_row, cancellable);
+            message_id, folder_row.id, location.uid.value, location.position);
+        yield location_table.create_async(location_row, cancellable);
         
         // only write out the IMAP email properties if they're supplied and there's something to
         // write out -- no need to create an empty row
         Geary.Imap.EmailProperties? properties = (Geary.Imap.EmailProperties?) email.properties;
-        if (email.fields.fulfills(Geary.Email.Field.PROPERTIES) && properties != null && !properties.is_empty()) {
+        if (email.fields.fulfills(Geary.Email.Field.PROPERTIES) && properties != null) {
             ImapMessagePropertiesRow properties_row = new ImapMessagePropertiesRow.from_imap_properties(
                 imap_message_properties_table, message_id, properties);
             yield imap_message_properties_table.create_async(properties_row, cancellable);
@@ -109,11 +105,10 @@ public class Geary.Sqlite.Folder : Geary.AbstractFolder, Geary.LocalFolder {
     
     public override async Gee.List<Geary.Email>? list_email_async(int low, int count,
         Geary.Email.Field required_fields, Cancellable? cancellable) throws Error {
-        assert(low >= 1);
-        assert(count >= 0 || count == -1);
-        
         check_open();
         
+        normalize_span_specifiers(ref low, ref count, yield get_email_count(cancellable));
+        
         if (count == 0)
             return null;
         
@@ -133,6 +128,16 @@ public class Geary.Sqlite.Folder : Geary.AbstractFolder, Geary.LocalFolder {
         return yield list_email(list, required_fields, cancellable);
     }
     
+    public async Gee.List<Geary.Email>? list_email_uid_async(Geary.Imap.UID? low, Geary.Imap.UID? high,
+        Geary.Email.Field required_fields, Cancellable? cancellable = null) throws Error {
+        check_open();
+        
+        Gee.List<MessageLocationRow>? list = yield location_table.list_ordering_async(folder_row.id,
+            (low != null) ? low.value : 1, (high != null) ? high.value : -1, cancellable);
+        
+        return yield list_email(list, required_fields, cancellable);
+    }
+    
     private async Gee.List<Geary.Email>? list_email(Gee.List<MessageLocationRow>? list,
         Geary.Email.Field required_fields, Cancellable? cancellable) throws Error {
         check_open();
@@ -145,29 +150,24 @@ public class Geary.Sqlite.Folder : Geary.AbstractFolder, Geary.LocalFolder {
         // together when all the information is fetched
         Gee.List<Geary.Email> emails = new Gee.ArrayList<Geary.Email>();
         foreach (MessageLocationRow location_row in list) {
-            // fetch the IMAP message location properties that are associated with the generic
-            // message location
-            ImapMessageLocationPropertiesRow? imap_location_row = yield imap_location_table.fetch_async(
-                location_row.id, cancellable);
-            assert(imap_location_row != null);
-            
             // fetch the message itself
             MessageRow? message_row = yield message_table.fetch_async(location_row.message_id,
                 required_fields, cancellable);
             assert(message_row != null);
             
-            // only add to the list if the email contains all the required fields
-            if (!message_row.fields.is_set(required_fields))
+            // only add to the list if the email contains all the required fields (because
+            // properties comes out of a separate table, skip this if properties are requested)
+            if (!message_row.fields.fulfills(required_fields.clear(Geary.Email.Field.PROPERTIES)))
                 continue;
             
             ImapMessagePropertiesRow? properties = null;
-            if (required_fields.fulfills(Geary.Email.Field.PROPERTIES)) {
+            if (required_fields.is_all_set(Geary.Email.Field.PROPERTIES)) {
                 properties = yield imap_message_properties_table.fetch_async(location_row.message_id,
                     cancellable);
             }
             
             Geary.Email email = message_row.to_email(new Geary.Imap.EmailLocation(location_row.position,
-                imap_location_row.uid));
+                new Geary.Imap.UID(location_row.ordering)));
             if (properties != null)
                 email.set_email_properties(properties.get_imap_email_properties());
             
@@ -192,15 +192,6 @@ public class Geary.Sqlite.Folder : Geary.AbstractFolder, Geary.LocalFolder {
         
         assert(location_row.position == position);
         
-        ImapMessageLocationPropertiesRow? imap_location_row = yield imap_location_table.fetch_async(
-            location_row.id, cancellable);
-        if (imap_location_row == null) {
-            throw new EngineError.NOT_FOUND("No IMAP location properties at position %d in %s",
-                position, to_string());
-        }
-        
-        assert(imap_location_row.location_id == location_row.id);
-        
         MessageRow? message_row = yield message_table.fetch_async(location_row.message_id,
             required_fields, cancellable);
         if (message_row == null) {
@@ -221,13 +212,36 @@ public class Geary.Sqlite.Folder : Geary.AbstractFolder, Geary.LocalFolder {
         }
         
         Geary.Email email = message_row.to_email(new Geary.Imap.EmailLocation(location_row.position,
-            imap_location_row.uid));
+            new Geary.Imap.UID(location_row.ordering)));
         if (properties != null)
             email.set_email_properties(properties.get_imap_email_properties());
         
         return email;
     }
     
+    public async Geary.Imap.UID? get_earliest_uid_async(Cancellable? cancellable = null) throws Error {
+        check_open();
+        
+        int64 ordering = yield location_table.get_earliest_ordering_async(folder_row.id, cancellable);
+        
+        return (ordering >= 1) ? new Geary.Imap.UID(ordering) : null;
+    }
+    
+    public override async void remove_email_async(Geary.Email email, Cancellable? cancellable = null)
+        throws Error {
+        check_open();
+        
+        // TODO: Right now, deleting an email is merely detaching its association with a folder
+        // (since it may be located in multiple folders).  This means at some point in the future
+        // a vacuum will be required to remove emails that are completely unassociated with the
+        // account
+        Geary.Imap.UID? uid = ((Geary.Imap.EmailLocation) email.location).uid;
+        if (uid == null)
+            throw new EngineError.NOT_FOUND("UID required to delete local email");
+        
+        yield location_table.remove_by_ordering_async(folder_row.id, uid.value, cancellable);
+    }
+    
     public async bool is_email_present_at(int position, out Geary.Email.Field available_fields,
         Cancellable? cancellable = null) throws Error {
         check_open();
@@ -248,9 +262,8 @@ public class Geary.Sqlite.Folder : Geary.AbstractFolder, Geary.LocalFolder {
         check_open();
         
         int64 message_id;
-        return yield imap_location_table.search_uid_in_folder(
-            ((Geary.Imap.EmailLocation) email.location).uid, folder_row.id, out message_id,
-            cancellable);
+        return yield location_table.does_ordering_exist_async(folder_row.id,
+            ((Geary.Imap.EmailLocation) email.location).uid.value, out message_id, cancellable);
     }
     
     public async void update_email_async(Geary.Email email, bool duplicate_okay,
@@ -262,8 +275,8 @@ public class Geary.Sqlite.Folder : Geary.AbstractFolder, Geary.LocalFolder {
         // See if the message can be identified in the folder (which both reveals association and
         // a message_id that can be used for a merge; note that this works without a Message-ID)
         int64 message_id;
-        bool associated = yield imap_location_table.search_uid_in_folder(location.uid, folder_row.id,
-            out message_id, cancellable);
+        bool associated = yield location_table.does_ordering_exist_async(folder_row.id,
+            location.uid.value, out message_id, cancellable);
         
         // If working around the lack of a Message-ID and not associated with this folder, treat
         // this operation as a create; otherwise, since a folder-association is determined, do
@@ -316,13 +329,8 @@ public class Geary.Sqlite.Folder : Geary.AbstractFolder, Geary.LocalFolder {
             
             // insert email at supplied position
             location_row = new MessageLocationRow(location_table, Row.INVALID_ID, message_id,
-                folder_row.id, location.position);
-            int64 location_id = yield location_table.create_async(location_row, cancellable);
-            
-            // update position propeties
-            ImapMessageLocationPropertiesRow imap_location_row = new ImapMessageLocationPropertiesRow(
-                imap_location_table, Row.INVALID_ID, location_id, location.uid);
-            yield imap_location_table.create_async(imap_location_row, cancellable);
+                folder_row.id, location.uid.value, location.position);
+            yield location_table.create_async(location_row, cancellable);
         }
         
         // Merge any new information with the existing message in the local store
@@ -352,8 +360,14 @@ public class Geary.Sqlite.Folder : Geary.AbstractFolder, Geary.LocalFolder {
             
         // update IMAP properties
         if (email.fields.fulfills(Geary.Email.Field.PROPERTIES)) {
-            yield imap_message_properties_table.update_async(message_id,
-                ((Geary.Imap.EmailProperties) email.properties).flags.serialize(), cancellable);
+            Geary.Imap.EmailProperties properties = (Geary.Imap.EmailProperties) email.properties;
+            string? internaldate =
+                (properties.internaldate != null) ? properties.internaldate.original : null;
+            long rfc822_size =
+                (properties.rfc822_size != null) ? properties.rfc822_size.value : -1;
+            
+            yield imap_message_properties_table.update_async(message_id, properties.flags.serialize(),
+                internaldate, rfc822_size, cancellable);
         }
     }
 }

src/engine/sqlite/email/sqlite-mail-database.vala

@@ -39,36 +39,5 @@ public class Geary.Sqlite.MailDatabase : Geary.Sqlite.Database {
             ? location_table
             : (MessageLocationTable) add_table(new MessageLocationTable(this, heavy_table));
     }
-    
-    // TODO: This belongs in a subclass.
-    public Geary.Sqlite.ImapMessageLocationPropertiesTable get_imap_message_location_table() {
-        SQLHeavy.Table heavy_table;
-        ImapMessageLocationPropertiesTable? imap_location_table = get_table(
-            "ImapMessageLocationPropertiesTable", out heavy_table) as ImapMessageLocationPropertiesTable;
-        
-        return (imap_location_table != null)
-            ? imap_location_table
-            : (ImapMessageLocationPropertiesTable) add_table(new ImapMessageLocationPropertiesTable(this, heavy_table));
-    }
-    
-    // TODO: This belongs in a subclass.
-    public Geary.Sqlite.ImapFolderPropertiesTable get_imap_folder_properties_table() {
-        SQLHeavy.Table heavy_table;
-        ImapFolderPropertiesTable? imap_folder_properties_table = get_table(
-            "ImapFolderPropertiesTable", out heavy_table) as ImapFolderPropertiesTable;
-        
-        return imap_folder_properties_table 
-            ?? (ImapFolderPropertiesTable) add_table(new ImapFolderPropertiesTable(this, heavy_table));
-    }
-    
-    // TODO: This belongs in a subclass.
-    public Geary.Sqlite.ImapMessagePropertiesTable get_imap_message_properties_table() {
-        SQLHeavy.Table heavy_table;
-        ImapMessagePropertiesTable? imap_message_properties_table = get_table(
-            "ImapMessagePropertiesTable", out heavy_table) as ImapMessagePropertiesTable;
-        
-        return imap_message_properties_table 
-            ?? (ImapMessagePropertiesTable) add_table(new ImapMessagePropertiesTable(this, heavy_table));
-    }
 }
 

src/engine/sqlite/email/sqlite-message-location-row.vala

@@ -8,26 +8,34 @@ public class Geary.Sqlite.MessageLocationRow : Geary.Sqlite.Row {
     public int64 id { get; private set; }
     public int64 message_id { get; private set; }
     public int64 folder_id { get; private set; }
+    public int64 ordering { get; private set; }
+    /**
+     * Note that position is not stored in the database, but rather determined by its location
+     * determined by the sorted ordering.  If the database call is unable to easily determine the
+     * position of the message in the folder, this will be set to -1.
+     */
     public int position { get; private set; }
     
     public MessageLocationRow(MessageLocationTable table, int64 id, int64 message_id, int64 folder_id,
-        int position) {
+        int64 ordering, int position) {
         base (table);
         
         this.id = id;
         this.message_id = message_id;
         this.folder_id = folder_id;
+        this.ordering = ordering;
         this.position = position;
     }
     
-    public MessageLocationRow.from_query_result(MessageLocationTable table,
+    public MessageLocationRow.from_query_result(MessageLocationTable table, int position,
         SQLHeavy.QueryResult result) throws Error {
         base (table);
         
         id = fetch_int64_for(result, MessageLocationTable.Column.ID);
         message_id = fetch_int64_for(result, MessageLocationTable.Column.MESSAGE_ID);
         folder_id = fetch_int64_for(result, MessageLocationTable.Column.FOLDER_ID);
-        position = fetch_int_for(result, MessageLocationTable.Column.POSITION);
+        ordering = fetch_int64_for(result, MessageLocationTable.Column.ORDERING);
+        this.position = position;
     }
 }
 

src/engine/sqlite/email/sqlite-message-location-table.vala

@@ -10,7 +10,7 @@ public class Geary.Sqlite.MessageLocationTable : Geary.Sqlite.Table {
         ID,
         MESSAGE_ID,
         FOLDER_ID,
-        POSITION
+        ORDERING
     }
     
     public MessageLocationTable(Geary.Sqlite.Database db, SQLHeavy.Table table) {
@@ -20,10 +20,10 @@ public class Geary.Sqlite.MessageLocationTable : Geary.Sqlite.Table {
     public async int64 create_async(MessageLocationRow row, Cancellable? cancellable = null)
         throws Error {
         SQLHeavy.Query query = db.prepare(
-            "INSERT INTO MessageLocationTable (message_id, folder_id, position) VALUES (?, ?, ?)");
+            "INSERT INTO MessageLocationTable (message_id, folder_id, ordering) VALUES (?, ?, ?)");
         query.bind_int64(0, row.message_id);
         query.bind_int64(1, row.folder_id);
-        query.bind_int(2, row.position);
+        query.bind_int64(2, row.ordering);
         
         return yield query.execute_insert_async(cancellable);
     }
@@ -39,16 +39,16 @@ public class Geary.Sqlite.MessageLocationTable : Geary.Sqlite.Table {
         SQLHeavy.Query query;
         if (count >= 0) {
             query = db.prepare(
-                "SELECT id, message_id, position FROM MessageLocationTable WHERE folder_id = ? "
-                + "ORDER BY position LIMIT ? OFFSET ?");
+                "SELECT id, message_id, ordering FROM MessageLocationTable WHERE folder_id = ? "
+                + "ORDER BY ordering LIMIT ? OFFSET ?");
             query.bind_int64(0, folder_id);
             query.bind_int(1, count);
             query.bind_int(2, low - 1);
         } else {
             // count == -1
             query = db.prepare(
-                "SELECT id, message_id, position FROM MessageLocationTable WHERE folder_id = ? "
-                + "ORDER BY position OFFSET ?");
+                "SELECT id, message_id, ordering FROM MessageLocationTable WHERE folder_id = ? "
+                + "ORDER BY ordering OFFSET ?");
             query.bind_int64(0, folder_id);
             query.bind_int(1, low - 1);
         }
@@ -58,9 +58,11 @@ public class Geary.Sqlite.MessageLocationTable : Geary.Sqlite.Table {
             return null;
         
         Gee.List<MessageLocationRow> list = new Gee.ArrayList<MessageLocationRow>();
+        int position = low;
         do {
             list.add(new MessageLocationRow(this, results.fetch_int64(0), results.fetch_int64(1),
-                folder_id, results.fetch_int(2)));
+                folder_id, results.fetch_int64(2), position++));
+            
             yield results.next_async(cancellable);
         } while (!results.finished);
         
@@ -72,35 +74,72 @@ public class Geary.Sqlite.MessageLocationTable : Geary.Sqlite.Table {
      */
     public async Gee.List<MessageLocationRow>? list_sparse_async(int64 folder_id, int[] by_position,
         Cancellable? cancellable = null) throws Error {
-        // build a vector for the IN expression
-        StringBuilder vector = new StringBuilder("(");
-        for (int ctr = 0; ctr < by_position.length; ctr++) {
-            assert(by_position[ctr] >= 1);
-            
-            if (ctr < (by_position.length - 1))
-                vector.append_printf("%d, ", by_position[ctr]);
-            else
-                vector.append_printf("%d", by_position[ctr]);
-        }
-        vector.append(")");
-        
+        // reuse the query for each iteration
         SQLHeavy.Query query = db.prepare(
-            "SELECT id, message_id, position FROM MessageLocationTable WHERE folder_id = ? AND position IN ?");
-        query.bind_int64(0, folder_id);
-        query.bind_string(1, vector.str);
-        
-        SQLHeavy.QueryResult results = yield query.execute_async(cancellable);
-        if (results.finished)
-            return null;
+            "SELECT id, message_id, ordering FROM MessageLocationTable WHERE folder_id = ? "
+            + "ORDER BY ordering LIMIT 1 OFFSET ?");
         
         Gee.List<MessageLocationRow> list = new Gee.ArrayList<MessageLocationRow>();
-        do {
+        foreach (int position in by_position) {
+            assert(position >= 1);
+            
+            query.bind_int64(0, folder_id);
+            query.bind_int(1, position);
+            
+            SQLHeavy.QueryResult results = yield query.execute_async(cancellable);
+            if (results.finished)
+                continue;
+            
             list.add(new MessageLocationRow(this, results.fetch_int64(0), results.fetch_int64(1),
-                folder_id, results.fetch_int(2)));
-            yield results.next_async(cancellable);
-        } while (!results.finished);
+                folder_id, results.fetch_int64(2), position));
+            
+            query.clear();
+        }
         
-        return list;
+        return (list.size > 0) ? list : null;
+    }
+    
+    public async Gee.List<MessageLocationRow>? list_ordering_async(int64 folder_id, int64 low_ordering,
+        int64 high_ordering, Cancellable? cancellable = null) throws Error {
+        assert(low_ordering >= 0 || low_ordering == -1);
+        assert(high_ordering >= 0 || high_ordering == -1);
+        
+        SQLHeavy.Query query;
+        if (high_ordering != -1 && low_ordering != -1) {
+            query = db.prepare(
+                "SELECT id, message_id, ordering FROM MessageLocationTable WHERE folder_id = ? "
+                + "AND ordering >= ? AND ordering <= ? ORDER BY ordering ASC");
+            query.bind_int64(0, folder_id);
+            query.bind_int64(1, low_ordering);
+            query.bind_int64(2, high_ordering);
+        } else if (high_ordering == -1) {
+            query = db.prepare(
+                "SELECT id, message_id, ordering FROM MessageLocationTable WHERE folder_id = ? "
+                + "AND ordering >= ? ORDER BY ordering ASC");
+            query.bind_int64(0, folder_id);
+            query.bind_int64(1, low_ordering);
+        } else {
+            assert(low_ordering == -1);
+            query = db.prepare(
+                "SELECT id, message_id, ordering FROM MessageLocationTable WHERE folder_id = ? "
+                + "AND ordering <= ? ORDER BY ordering ASC");
+            query.bind_int64(0, folder_id);
+            query.bind_int64(1, high_ordering);
+        }
+        
+        SQLHeavy.QueryResult result = yield query.execute_async(cancellable);
+        if (result.finished)
+            return null;
+        
+        Gee.List<MessageLocationRow>? list = new Gee.ArrayList<MessageLocationRow>();
+        do {
+            list.add(new MessageLocationRow(this, result.fetch_int64(0), result.fetch_int64(1),
+                folder_id, result.fetch_int64(2), -1));
+            
+            yield result.next_async(cancellable);
+        } while (!result.finished);
+        
+        return (list.size > 0) ? list : null;
     }
     
     /**
@@ -111,8 +150,8 @@ public class Geary.Sqlite.MessageLocationTable : Geary.Sqlite.Table {
         assert(position >= 1);
         
         SQLHeavy.Query query = db.prepare(
-            "SELECT id, message_id, position FROM MessageLocationTable WHERE folder_id = ? "
-            + "AND position = ?");
+            "SELECT id, message_id, ordering FROM MessageLocationTable WHERE folder_id = ? "
+            + "ORDER BY ordering LIMIT 1 OFFSET ?");
         query.bind_int64(0, folder_id);
         query.bind_int(1, position);
         
@@ -121,7 +160,58 @@ public class Geary.Sqlite.MessageLocationTable : Geary.Sqlite.Table {
             return null;
         
         return new MessageLocationRow(this, results.fetch_int64(0), results.fetch_int64(1), folder_id,
-            results.fetch_int(2));
+            results.fetch_int64(2), position);
+    }
+    
+    public async int fetch_count_for_folder_async(int64 folder_id, Cancellable? cancellable = null)
+        throws Error {
+        SQLHeavy.Query query = db.prepare(
+            "SELECT COUNT(*) FROM MessageLocationTable WHERE folder_id = ?");
+        query.bind_int64(0, folder_id);
+        
+        SQLHeavy.QueryResult results = yield query.execute_async(cancellable);
+        
+        return (!results.finished) ? results.fetch_int(0) : 0;
+    }
+    
+    /**
+     * Find a row based on its ordering value in the folder.
+     */
+    public async bool does_ordering_exist_async(int64 folder_id, int64 ordering,
+        out int64 message_id, Cancellable? cancellable = null) throws Error {
+        SQLHeavy.Query query = db.prepare(
+            "SELECT message_id FROM MessageLocationTable WHERE folder_id = ? AND ordering = ?");
+        query.bind_int64(0, folder_id);
+        query.bind_int64(1, ordering);
+        
+        SQLHeavy.QueryResult results = yield query.execute_async(cancellable);
+        if (results.finished)
+            return false;
+        
+        message_id = results.fetch_int64(0);
+        
+        return true;
+    }
+    
+    public async int64 get_earliest_ordering_async(int64 folder_id, Cancellable? cancellable = null)
+        throws Error {
+        SQLHeavy.Query query = db.prepare(
+            "SELECT MIN(ordering) FROM MessageLocationTable WHERE folder_id = ?");
+        query.bind_int64(0, folder_id);
+        
+        SQLHeavy.QueryResult result = yield query.execute_async(cancellable);
+        
+        return (!result.finished) ? result.fetch_int64(0) : -1;
+    }
+    
+    public async void remove_by_ordering_async(int64 folder_id, int64 ordering,
+        Cancellable? cancellable = null) throws Error {
+        SQLHeavy.Query query = db.prepare(
+            "DELETE FROM MessageLocationTable WHERE folder_id = ? AND ordering = ?");
+        query.bind_int64(0, folder_id);
+        query.bind_int64(1, ordering);
+        
+        yield query.execute_async(cancellable);
     }
 }
 

src/engine/sqlite/email/sqlite-message-row.vala

@@ -50,7 +50,7 @@ public class Geary.Sqlite.MessageRow : Geary.Sqlite.Row {
         
         if ((fields & Geary.Email.Field.DATE) != 0) {
             date = fetch_string_for(result, MessageTable.Column.DATE_FIELD);
-            date_time_t = (time_t) fetch_int64_for(result, MessageTable.Column.DATE_INT64);
+            date_time_t = (time_t) fetch_int64_for(result, MessageTable.Column.DATE_TIME_T);
         }
         
         if ((fields & Geary.Email.Field.ORIGINATORS) != 0) {

src/engine/sqlite/email/sqlite-message-table.vala

@@ -11,7 +11,7 @@ public class Geary.Sqlite.MessageTable : Geary.Sqlite.Table {
         FIELDS,
         
         DATE_FIELD,
-        DATE_INT64,
+        DATE_TIME_T,
         
         FROM_FIELD,
         SENDER,
@@ -70,7 +70,7 @@ public class Geary.Sqlite.MessageTable : Geary.Sqlite.Table {
         
         query.execute();
         
-        if (row.fields.is_set(Geary.Email.Field.DATE)) {
+        if (row.fields.is_any_set(Geary.Email.Field.DATE)) {
             query = transaction.prepare(
                 "UPDATE MessageTable SET date_field=?, date_time_t=? WHERE id=?");
             query.bind_string(0, row.date);
@@ -80,7 +80,7 @@ public class Geary.Sqlite.MessageTable : Geary.Sqlite.Table {
             query.execute();
         }
         
-        if (row.fields.is_set(Geary.Email.Field.ORIGINATORS)) {
+        if (row.fields.is_any_set(Geary.Email.Field.ORIGINATORS)) {
             query = transaction.prepare(
                 "UPDATE MessageTable SET from_field=?, sender=?, reply_to=? WHERE id=?");
             query.bind_string(0, row.from);
@@ -91,7 +91,7 @@ public class Geary.Sqlite.MessageTable : Geary.Sqlite.Table {
             query.execute();
         }
         
-        if (row.fields.is_set(Geary.Email.Field.RECEIVERS)) {
+        if (row.fields.is_any_set(Geary.Email.Field.RECEIVERS)) {
             query = transaction.prepare(
                 "UPDATE MessageTable SET to_field=?, cc=?, bcc=? WHERE id=?");
             query.bind_string(0, row.to);
@@ -102,7 +102,7 @@ public class Geary.Sqlite.MessageTable : Geary.Sqlite.Table {
             query.execute();
         }
         
-        if (row.fields.is_set(Geary.Email.Field.REFERENCES)) {
+        if (row.fields.is_any_set(Geary.Email.Field.REFERENCES)) {
             query = transaction.prepare(
                 "UPDATE MessageTable SET message_id=?, in_reply_to=? WHERE id=?");
             query.bind_string(0, row.message_id);
@@ -112,7 +112,7 @@ public class Geary.Sqlite.MessageTable : Geary.Sqlite.Table {
             query.execute();
         }
         
-        if (row.fields.is_set(Geary.Email.Field.SUBJECT)) {
+        if (row.fields.is_any_set(Geary.Email.Field.SUBJECT)) {
             query = transaction.prepare(
                 "UPDATE MessageTable SET subject=? WHERE id=?");
             query.bind_string(0, row.subject);
@@ -121,7 +121,7 @@ public class Geary.Sqlite.MessageTable : Geary.Sqlite.Table {
             query.execute();
         }
         
-        if (row.fields.is_set(Geary.Email.Field.HEADER)) {
+        if (row.fields.is_any_set(Geary.Email.Field.HEADER)) {
             query = transaction.prepare(
                 "UPDATE MessageTable SET header=? WHERE id=?");
             query.bind_string(0, row.header);
@@ -130,7 +130,7 @@ public class Geary.Sqlite.MessageTable : Geary.Sqlite.Table {
             query.execute();
         }
         
-        if (row.fields.is_set(Geary.Email.Field.BODY)) {
+        if (row.fields.is_any_set(Geary.Email.Field.BODY)) {
             query = transaction.prepare(
                 "UPDATE MessageTable SET body=? WHERE id=?");
             query.bind_string(0, row.body);

src/engine/sqlite/imap/sqlite-imap-database.vala

@@ -0,0 +1,30 @@
+/* Copyright 2011 Yorba Foundation
+ *
+ * This software is licensed under the GNU Lesser General Public License
+ * (version 2.1 or later).  See the COPYING file in this distribution. 
+ */
+
+public class Geary.Sqlite.ImapDatabase : Geary.Sqlite.MailDatabase {
+    public ImapDatabase(string user) throws Error {
+        base (user);
+    }
+    
+    public Geary.Sqlite.ImapFolderPropertiesTable get_imap_folder_properties_table() {
+        SQLHeavy.Table heavy_table;
+        ImapFolderPropertiesTable? imap_folder_properties_table = get_table(
+            "ImapFolderPropertiesTable", out heavy_table) as ImapFolderPropertiesTable;
+        
+        return imap_folder_properties_table 
+            ?? (ImapFolderPropertiesTable) add_table(new ImapFolderPropertiesTable(this, heavy_table));
+    }
+    
+    public Geary.Sqlite.ImapMessagePropertiesTable get_imap_message_properties_table() {
+        SQLHeavy.Table heavy_table;
+        ImapMessagePropertiesTable? imap_message_properties_table = get_table(
+            "ImapMessagePropertiesTable", out heavy_table) as ImapMessagePropertiesTable;
+        
+        return imap_message_properties_table 
+            ?? (ImapMessagePropertiesTable) add_table(new ImapMessagePropertiesTable(this, heavy_table));
+    }
+}
+

src/engine/sqlite/imap/sqlite-imap-folder-properties-row.vala

@@ -7,16 +7,21 @@
 public class Geary.Sqlite.ImapFolderPropertiesRow : Geary.Sqlite.Row {
     public int64 id { get; private set; }
     public int64 folder_id { get; private set; }
+    public int last_seen_total { get; private set; }
     public Geary.Imap.UIDValidity? uid_validity { get; private set; }
+    public Geary.Imap.UID? uid_next { get; private set; }
     public string attributes { get; private set; }
     
     public ImapFolderPropertiesRow(ImapFolderPropertiesTable table, int64 id, int64 folder_id,
-        Geary.Imap.UIDValidity? uid_validity, string attributes) {
+        int last_seen_total, Geary.Imap.UIDValidity? uid_validity, Geary.Imap.UID? uid_next,
+        string attributes) {
         base (table);
         
         this.id = id;
         this.folder_id = folder_id;
+        this.last_seen_total = last_seen_total;
         this.uid_validity = uid_validity;
+        this.uid_next = uid_next;
         this.attributes = attributes;
     }
     
@@ -26,12 +31,14 @@ public class Geary.Sqlite.ImapFolderPropertiesRow : Geary.Sqlite.Row {
         
         id = Row.INVALID_ID;
         this.folder_id = folder_id;
+        last_seen_total = properties.messages;
         uid_validity = properties.uid_validity;
+        uid_next = properties.uid_next;
         attributes = properties.attrs.serialize();
     }
     
     public Geary.Imap.FolderProperties get_imap_folder_properties() {
-        return new Geary.Imap.FolderProperties(uid_validity,
+        return new Geary.Imap.FolderProperties(last_seen_total, 0, 0, uid_validity, uid_next,
             Geary.Imap.MailboxAttributes.deserialize(attributes));
     }
 }

src/engine/sqlite/imap/sqlite-imap-folder-properties-table.vala

@@ -9,8 +9,10 @@ public class Geary.Sqlite.ImapFolderPropertiesTable : Geary.Sqlite.Table {
     public enum Column {
         ID,
         FOLDER_ID,
+        LAST_SEEN_TOTAL,
         UID_VALIDITY,
-        FLAGS
+        UID_NEXT,
+        ATTRIBUTES
     }
     
     public ImapFolderPropertiesTable(Geary.Sqlite.Database gdb, SQLHeavy.Table table) {
@@ -20,18 +22,37 @@ public class Geary.Sqlite.ImapFolderPropertiesTable : Geary.Sqlite.Table {
     public async int64 create_async(ImapFolderPropertiesRow row, Cancellable? cancellable = null)
         throws Error {
         SQLHeavy.Query query = db.prepare(
-            "INSERT INTO ImapFolderPropertiesTable (folder_id, uid_validity, attributes) VALUES (?, ?, ?)");
+            "INSERT INTO ImapFolderPropertiesTable (folder_id, last_seen_total, uid_validity, uid_next, attributes) "
+            + "VALUES (?, ?, ?, ?, ?)");
         query.bind_int64(0, row.folder_id);
-        query.bind_int64(1, (row.uid_validity != null) ? row.uid_validity.value : -1);
-        query.bind_string(2, row.attributes);
+        query.bind_int(1, row.last_seen_total);
+        query.bind_int64(2, (row.uid_validity != null) ? row.uid_validity.value : -1);
+        query.bind_int64(3, (row.uid_next != null) ? row.uid_next.value : -1);
+        query.bind_string(4, row.attributes);
         
         return yield query.execute_insert_async(cancellable);
     }
     
+    public async void update_async(int64 folder_id, ImapFolderPropertiesRow row,
+        Cancellable? cancellable = null) throws Error {
+        SQLHeavy.Query query = db.prepare(
+            "UPDATE ImapFolderPropertiesTable "
+            + "SET last_seen_total = ?, uid_validity = ?, uid_next = ?, attributes = ? "
+            + "WHERE folder_id = ?");
+        query.bind_int(0, row.last_seen_total);
+        query.bind_int64(1, (row.uid_validity != null) ? row.uid_validity.value : -1);
+        query.bind_int64(2, (row.uid_next != null) ? row.uid_next.value : -1);
+        query.bind_string(3, row.attributes);
+        query.bind_int64(4, folder_id);
+        
+        yield query.execute_async(cancellable);
+    }
+    
     public async ImapFolderPropertiesRow? fetch_async(int64 folder_id, Cancellable? cancellable = null)
         throws Error {
         SQLHeavy.Query query = db.prepare(
-            "SELECT id, uid_validity, attributes FROM ImapFolderPropertiesTable WHERE folder_id = ?");
+            "SELECT id, last_seen_total, uid_validity, uid_next, attributes "
+            + "FROM ImapFolderPropertiesTable WHERE folder_id = ?");
         query.bind_int64(0, folder_id);
         
         SQLHeavy.QueryResult result = yield query.execute_async(cancellable);
@@ -39,11 +60,15 @@ public class Geary.Sqlite.ImapFolderPropertiesTable : Geary.Sqlite.Table {
             return null;
         
         Geary.Imap.UIDValidity? uid_validity = null;
-        if (result.fetch_int64(1) >= 0)
-            uid_validity = new Geary.Imap.UIDValidity(result.fetch_int64(1));
+        if (result.fetch_int64(2) >= 0)
+            uid_validity = new Geary.Imap.UIDValidity(result.fetch_int64(2));
         
-        return new ImapFolderPropertiesRow(this, result.fetch_int64(0), folder_id, uid_validity,
-            result.fetch_string(2));
+        Geary.Imap.UID? uid_next = null;
+        if (result.fetch_int64(3) >= 0)
+            uid_next = new Geary.Imap.UID(result.fetch_int64(3));
+        
+        return new ImapFolderPropertiesRow(this, result.fetch_int64(0), folder_id, result.fetch_int(1),
+            uid_validity, uid_next, result.fetch_string(4));
     }
 }
 

src/engine/sqlite/imap/sqlite-imap-message-location-properties-row.vala

@@ -1,21 +0,0 @@
-/* Copyright 2011 Yorba Foundation
- *
- * This software is licensed under the GNU Lesser General Public License
- * (version 2.1 or later).  See the COPYING file in this distribution. 
- */
-
-public class Geary.Sqlite.ImapMessageLocationPropertiesRow : Geary.Sqlite.Row {
-    public int64 id { get; private set; }
-    public int64 location_id { get; private set; }
-    public Geary.Imap.UID uid { get; private set; }
-    
-    public ImapMessageLocationPropertiesRow(ImapMessageLocationPropertiesTable table, int64 id,
-        int64 location_id, Geary.Imap.UID uid) {
-        base (table);
-        
-        this.id = id;
-        this.location_id = location_id;
-        this.uid = uid;
-    }
-}
-

src/engine/sqlite/imap/sqlite-imap-message-location-properties-table.vala

@@ -1,65 +0,0 @@
-/* Copyright 2011 Yorba Foundation
- *
- * This software is licensed under the GNU Lesser General Public License
- * (version 2.1 or later).  See the COPYING file in this distribution. 
- */
-
-public class Geary.Sqlite.ImapMessageLocationPropertiesTable : Geary.Sqlite.Table {
-    // This *must* be in the same order as the schema.
-    public enum Column {
-        ID,
-        LOCATION_ID,
-        UID
-    }
-    
-    public ImapMessageLocationPropertiesTable(Geary.Sqlite.Database gdb, SQLHeavy.Table table) {
-        base (gdb, table);
-    }
-    
-    public async int64 create_async(ImapMessageLocationPropertiesRow row,
-        Cancellable? cancellable = null) throws Error {
-        SQLHeavy.Query query = db.prepare(
-            "INSERT INTO ImapMessageLocationPropertiesTable (location_id, uid) VALUES (?, ?)");
-        query.bind_int64(0, row.location_id);
-        query.bind_int64(1, row.uid.value);
-        
-        return yield query.execute_insert_async(cancellable);
-    }
-    
-    public async ImapMessageLocationPropertiesRow? fetch_async(int64 location_id,
-        Cancellable? cancellable = null) throws Error {
-        SQLHeavy.Query query = db.prepare(
-            "SELECT id, uid FROM ImapMessageLocationPropertiesTable WHERE location_id = ?");
-        query.bind_int64(0, location_id);
-        
-        SQLHeavy.QueryResult result = yield query.execute_async(cancellable);
-        if (result.finished)
-            return null;
-        
-        return new ImapMessageLocationPropertiesRow(this, result.fetch_int64(0), location_id,
-            new Geary.Imap.UID(result.fetch_int64(1)));
-    }
-    
-    public async bool search_uid_in_folder(Geary.Imap.UID uid, int64 folder_id,
-        out int64 message_id, Cancellable? cancellable = null) throws Error {
-        message_id = Row.INVALID_ID;
-        
-        SQLHeavy.Query query = db.prepare(
-            "SELECT MessageLocationTable.message_id "
-            + "FROM ImapMessageLocationPropertiesTable "
-            + "INNER JOIN MessageLocationTable "
-            + "WHERE MessageLocationTable.folder_id=? "
-            + "AND ImapMessageLocationPropertiesTable.location_id=MessageLocationTable.id "
-            + "AND ImapMessageLocationPropertiesTable.uid=?");
-        query.bind_int64(0, folder_id);
-        query.bind_int64(1, uid.value);
-        
-        SQLHeavy.QueryResult result = yield query.execute_async(cancellable);
-        
-        if (!result.finished)
-            message_id = result.fetch_int64(0);
-        
-        return !result.finished;
-    }
-}
-

src/engine/sqlite/imap/sqlite-imap-message-properties-row.vala

@@ -8,14 +8,18 @@ public class Geary.Sqlite.ImapMessagePropertiesRow : Geary.Sqlite.Row {
     public int64 id { get; private set; }
     public int64 message_id { get; private set; }
     public string flags { get; private set; }
+    public string internaldate { get; private set; }
+    public long rfc822_size { get; private set; }
     
     public ImapMessagePropertiesRow(ImapMessagePropertiesTable table, int64 id, int64 message_id,
-        string flags) {
+        string flags, string internaldate, long rfc822_size) {
         base (table);
         
         this.id = id;
         this.message_id = message_id;
         this.flags = flags;
+        this.internaldate = internaldate;
+        this.rfc822_size = rfc822_size;
     }
     
     public ImapMessagePropertiesRow.from_imap_properties(ImapMessagePropertiesTable table,
@@ -25,10 +29,21 @@ public class Geary.Sqlite.ImapMessagePropertiesRow : Geary.Sqlite.Row {
         id = Row.INVALID_ID;
         this.message_id = message_id;
         flags = properties.flags.serialize();
+        internaldate = properties.internaldate.original;
+        rfc822_size = properties.rfc822_size.value;
     }
     
     public Geary.Imap.EmailProperties get_imap_email_properties() {
-        return new Geary.Imap.EmailProperties(Geary.Imap.MessageFlags.deserialize(flags));
+        Imap.InternalDate? constructed = null;
+        try {
+            constructed = new Imap.InternalDate(internaldate);
+        } catch (Error err) {
+            debug("Unable to construct internaldate object from \"%s\": %s", internaldate,
+                err.message);
+        }
+        
+        return new Geary.Imap.EmailProperties(Geary.Imap.MessageFlags.deserialize(flags),
+            constructed, new RFC822.Size(rfc822_size));
     }
 }
 

src/engine/sqlite/imap/sqlite-imap-message-properties-table.vala

@@ -8,7 +8,10 @@ public class Geary.Sqlite.ImapMessagePropertiesTable : Geary.Sqlite.Table {
     // This *must* be in the same order as the schema.
     public enum Column {
         ID,
-        FLAGS
+        MESSAGE_ID,
+        FLAGS,
+        INTERNALDATE,
+        RFC822_SIZE
     }
     
     public ImapMessagePropertiesTable(Geary.Sqlite.Database gdb, SQLHeavy.Table table) {
@@ -18,9 +21,12 @@ public class Geary.Sqlite.ImapMessagePropertiesTable : Geary.Sqlite.Table {
     public async int64 create_async(ImapMessagePropertiesRow row, Cancellable? cancellable = null)
         throws Error {
         SQLHeavy.Query query = db.prepare(
-            "INSERT INTO ImapMessagePropertiesTable (message_id, flags) VALUES (?, ?)");
+            "INSERT INTO ImapMessagePropertiesTable (message_id, flags, internaldate, rfc822_size) "
+            + "VALUES (?, ?, ?, ?)");
         query.bind_int64(0, row.message_id);
         query.bind_string(1, row.flags);
+        query.bind_string(2, row.internaldate);
+        query.bind_int64(3, row.rfc822_size);
         
         return yield query.execute_insert_async(cancellable);
     }
@@ -28,7 +34,8 @@ public class Geary.Sqlite.ImapMessagePropertiesTable : Geary.Sqlite.Table {
     public async ImapMessagePropertiesRow? fetch_async(int64 message_id, Cancellable? cancellable = null)
         throws Error {
         SQLHeavy.Query query = db.prepare(
-            "SELECT id, flags FROM ImapMessagePropertiesTable WHERE message_id = ?");
+            "SELECT id, flags internaldate, rfc822_size FROM ImapMessagePropertiesTable "
+            + "WHERE message_id = ?");
         query.bind_int64(0, message_id);
         
         SQLHeavy.QueryResult result = yield query.execute_async(cancellable);
@@ -36,15 +43,19 @@ public class Geary.Sqlite.ImapMessagePropertiesTable : Geary.Sqlite.Table {
             return null;
         
         return new ImapMessagePropertiesRow(this, result.fetch_int64(0), message_id,
-            result.fetch_string(1));
+            result.fetch_string(1), result.fetch_string(2), (long) result.fetch_int64(3));
     }
     
-    public async void update_async(int64 message_id, string flags, Cancellable? cancellable = null)
+    public async void update_async(int64 message_id, string? flags, string? internaldate, long rfc822_size,
+        Cancellable? cancellable = null)
         throws Error {
         SQLHeavy.Query query = db.prepare(
-            "UPDATE ImapMessagePropertiesTable SET flags = ? WHERE message_id = ?");
+            "UPDATE ImapMessagePropertiesTable SET flags = ?, internaldate = ?, rfc822_size = ? "
+            + "WHERE message_id = ?");
         query.bind_string(0, flags);
-        query.bind_int64(1, message_id);
+        query.bind_string(1, internaldate);
+        query.bind_int64(2, rfc822_size);
+        query.bind_int64(3, message_id);
         
         yield query.execute_async(cancellable);
     }

src/wscript

@@ -5,6 +5,7 @@
 
 def build(bld):
 	bld.common_src = [
+	'../common/common-arrays.vala',
 	'../common/common-date.vala',
 	'../common/common-intl.vala',
 	'../common/common-yorba-application.vala'
@@ -46,6 +47,7 @@ def build(bld):
 	'../engine/imap/api/imap-account.vala',
 	'../engine/imap/api/imap-email-location.vala',
 	'../engine/imap/api/imap-email-properties.vala',
+	'../engine/imap/api/imap-folder-extensions.vala',
 	'../engine/imap/api/imap-folder-properties.vala',
 	'../engine/imap/api/imap-folder.vala',
 	'../engine/imap/command/imap-command-response.vala',
@@ -100,10 +102,9 @@ def build(bld):
 	'../engine/sqlite/email/sqlite-message-location-table.vala',
 	'../engine/sqlite/email/sqlite-message-row.vala',
 	'../engine/sqlite/email/sqlite-message-table.vala',
+	'../engine/sqlite/imap/sqlite-imap-database.vala',
 	'../engine/sqlite/imap/sqlite-imap-folder-properties-row.vala',
 	'../engine/sqlite/imap/sqlite-imap-folder-properties-table.vala',
-	'../engine/sqlite/imap/sqlite-imap-message-location-properties-row.vala',
-	'../engine/sqlite/imap/sqlite-imap-message-location-properties-table.vala',
 	'../engine/sqlite/imap/sqlite-imap-message-properties-row.vala',
 	'../engine/sqlite/imap/sqlite-imap-message-properties-table.vala',
 	

wscript

@@ -55,6 +55,13 @@ def configure(conf):
 		mandatory=1,
 		args='--cflags --libs')
 	
+	conf.check_cfg(
+		package='sqlite3',
+		uselib_store='SQLITE',
+		atleast_version='3.7.5',
+		mandatory=1,
+		args='--cflags --libs')
+	
 	conf.check_cfg(
 		package='sqlheavy-0.1',
 		uselib_store='SQLHEAVY',