Properly decode Unicode folder names; fix #5217

Previously, we were taking folder names as they came off the wire.
Turns out IMAP specifies that folder names with 8 bit code points are
encoded in a crazy scheme unique to IMAP.  Now, we properly decode that
scheme to the correct UTF-8 folder names to be displayed to the user.

There's also now a database upgrade path that converts all existing
mailboxes to the decoded version, so your existing database should just
keep working.

sql/CMakeLists.txt

@@ -5,3 +5,4 @@ install(FILES version-002.sql DESTINATION ${SQL_DEST})
 install(FILES version-003.sql DESTINATION ${SQL_DEST})
 install(FILES version-004.sql DESTINATION ${SQL_DEST})
 install(FILES version-005.sql DESTINATION ${SQL_DEST})
+install(FILES version-006.sql DESTINATION ${SQL_DEST})

sql/version-006.sql

@@ -0,0 +1,7 @@
+--
+-- Dummy database upgrade to fix folder names being stored in encoded form.
+-- Before this version, all folder names are stored as they came off the wire.
+-- After this version, all folder names are stored in canonical UTF-8 form.
+-- See src/engine/imap-db/imap-db-database.vala in post_upgrade() for the code
+-- that runs the upgrade.
+--

src/CMakeLists.txt

@@ -81,6 +81,7 @@ engine/imap/message/imap-data-format.vala
 engine/imap/message/imap-fetch-data-type.vala
 engine/imap/message/imap-fetch-body-data-type.vala
 engine/imap/message/imap-flag.vala
+engine/imap/message/imap-mailbox-parameter.vala
 engine/imap/message/imap-message-data.vala
 engine/imap/message/imap-message-set.vala
 engine/imap/message/imap-parameter.vala
@@ -184,6 +185,7 @@ engine/util/util-converter.vala
 engine/util/util-files.vala
 engine/util/util-generic-capabilities.vala
 engine/util/util-html.vala
+engine/util/util-imap-utf7.vala
 engine/util/util-inet.vala
 engine/util/util-interfaces.vala
 engine/util/util-memory.vala

src/console/main.vala

@@ -390,8 +390,8 @@ class ImapConsole : Gtk.Window {
         check_connected(cmd, args, 2, "<reference> <mailbox>");
         
         status("Listing...");
-        cx.send_async.begin(new Geary.Imap.ListCommand.wildcarded(args[0], args[1], (cmd.down() == "xlist")),
+        cx.send_async.begin(new Geary.Imap.ListCommand.wildcarded(args[0],
-            null, on_list);
+            new Geary.Imap.MailboxParameter(args[1]), (cmd.down() == "xlist")), null, on_list);
     }
     
     private void on_list(Object? source, AsyncResult result) {
@@ -407,7 +407,8 @@ class ImapConsole : Gtk.Window {
         check_connected(cmd, args, 1, "<mailbox>");
         
         status("Opening %s read-only".printf(args[0]));
-        cx.send_async.begin(new Geary.Imap.ExamineCommand(args[0]), null, on_examine);
+        cx.send_async.begin(new Geary.Imap.ExamineCommand(new Geary.Imap.MailboxParameter(args[0])),
+            null, on_examine);
     }
     
     private void on_examine(Object? source, AsyncResult result) {
@@ -485,7 +486,8 @@ class ImapConsole : Gtk.Window {
         for (int ctr = 1; ctr < args.length; ctr++)
             data_items += Geary.Imap.StatusDataType.decode(args[ctr]);
         
-        cx.send_async.begin(new Geary.Imap.StatusCommand(args[0], data_items), null, on_get_status);
+        cx.send_async.begin(new Geary.Imap.StatusCommand(new Geary.Imap.MailboxParameter(args[0]),
+            data_items), null, on_get_status);
     }
     
     private void on_get_status(Object? source, AsyncResult result) {

src/engine/imap-db/imap-db-database.vala

@@ -25,19 +25,56 @@ private class Geary.ImapDB.Database : Geary.Db.VersionedDatabase {
     }
     
     protected override void post_upgrade(int version) {
-        if (version == 5) {
+        switch (version) {
-            try {
+            case 5:
-                Db.Result result = query("SELECT sender, from_field, to_field, cc, bcc FROM MessageTable");
+                post_upgrade_populate_autocomplete();
-                while (!result.finished) {
+            break;
-                    MessageAddresses message_addresses =
+            
-                        new MessageAddresses.from_result(account_owner_email, result);
+            case 6:
-                    foreach (Contact contact in message_addresses.contacts)
+                post_upgrade_encode_folder_names();
-                        do_update_contact_importance(get_master_connection(), contact);
+            break;
-                    result.next();
+        }
-                }
+    }
-            } catch (Error err) {
+    
-                debug("Error population autocompletion table during upgrade to database schema 5");
+    // Version 5.
+    private void post_upgrade_populate_autocomplete() {
+        try {
+            Db.Result result = query("SELECT sender, from_field, to_field, cc, bcc FROM MessageTable");
+            while (!result.finished) {
+                MessageAddresses message_addresses =
+                    new MessageAddresses.from_result(account_owner_email, result);
+                foreach (Contact contact in message_addresses.contacts)
+                    do_update_contact_importance(get_master_connection(), contact);
+                result.next();
             }
+        } catch (Error err) {
+            debug("Error populating autocompletion table during upgrade to database schema 5");
+        }
+    }
+    
+    // Version 6.
+    private void post_upgrade_encode_folder_names() {
+        try {
+            Db.Result select = query("SELECT id, name FROM FolderTable");
+            while (!select.finished) {
+                int64 id = select.int64_at(0);
+                string encoded_name = select.string_at(1);
+                
+                try {
+                    string canonical_name = Geary.ImapUtf7.imap_utf7_to_utf8(encoded_name);
+                    
+                    Db.Statement update = prepare("UPDATE FolderTable SET name=? WHERE id=?");
+                    update.bind_string(0, canonical_name);
+                    update.bind_int64(1, id);
+                    update.exec();
+                } catch (Error e) {
+                    debug("Error renaming folder %s to its canonical representation: %s", encoded_name, e.message);
+                }
+                
+                select.next();
+            }
+        } catch (Error e) {
+            debug("Error decoding folder names during upgrade to database schema 6: %s", e.message);
         }
     }
     

src/engine/imap/command/imap-commands.vala

@@ -62,28 +62,28 @@ public class Geary.Imap.ListCommand : Command {
     public const string NAME = "list";
     public const string XLIST_NAME = "xlist";
     
-    public ListCommand(string mailbox, bool use_xlist) {
+    public ListCommand(Geary.Imap.MailboxParameter mailbox, bool use_xlist) {
-        base (use_xlist ? XLIST_NAME : NAME, { "", mailbox });
+        base (use_xlist ? XLIST_NAME : NAME, { "", mailbox.value });
     }
     
-    public ListCommand.wildcarded(string reference, string mailbox, bool use_xlist) {
+    public ListCommand.wildcarded(string reference, Geary.Imap.MailboxParameter mailbox, bool use_xlist) {
-        base (use_xlist ? XLIST_NAME : NAME, { reference, mailbox });
+        base (use_xlist ? XLIST_NAME : NAME, { reference, mailbox.value });
     }
 }
 
 public class Geary.Imap.ExamineCommand : Command {
     public const string NAME = "examine";
     
-    public ExamineCommand(string mailbox) {
+    public ExamineCommand(Geary.Imap.MailboxParameter mailbox) {
-        base (NAME, { mailbox });
+        base (NAME, { mailbox.value });
     }
 }
 
 public class Geary.Imap.SelectCommand : Command {
     public const string NAME = "select";
     
-    public SelectCommand(string mailbox) {
+    public SelectCommand(Geary.Imap.MailboxParameter mailbox) {
-        base (NAME, { mailbox });
+        base (NAME, { mailbox.value });
     }
 }
 
@@ -98,10 +98,10 @@ public class Geary.Imap.CloseCommand : Command {
 public class Geary.Imap.StatusCommand : Command {
     public const string NAME = "status";
     
-    public StatusCommand(string mailbox, StatusDataType[] data_items) {
+    public StatusCommand(Geary.Imap.MailboxParameter mailbox, StatusDataType[] data_items) {
         base (NAME);
         
-        add(new StringParameter(mailbox));
+        add(mailbox);
         
         assert(data_items.length > 0);
         ListParameter data_item_list = new ListParameter(this);
@@ -161,11 +161,11 @@ public class Geary.Imap.CopyCommand : Command {
     public const string NAME = "copy";
     public const string UID_NAME = "uid copy";
 
-    public CopyCommand(MessageSet message_set, string destination) {
+    public CopyCommand(MessageSet message_set, Geary.Imap.MailboxParameter destination) {
         base (message_set.is_uid ? UID_NAME : NAME);
 
         add(message_set.to_parameter());
-        add(new StringParameter(destination));
+        add(destination);
     }
 }
 

src/engine/imap/decoders/imap-list-results.vala

@@ -58,7 +58,7 @@ public class Geary.Imap.ListResults : Geary.Imap.CommandResults {
     private Gee.List<MailboxInformation> list;
     private Gee.Map<string, MailboxInformation> map;
     
-    public ListResults(StatusResponse status_response, Gee.Map<string, MailboxInformation> map,
+    private ListResults(StatusResponse status_response, Gee.Map<string, MailboxInformation> map,
         Gee.List<MailboxInformation> list) {
         base (status_response);
         
@@ -76,7 +76,7 @@ public class Geary.Imap.ListResults : Geary.Imap.CommandResults {
                 StringParameter cmd = data.get_as_string(1);
                 ListParameter attrs = data.get_as_list(2);
                 StringParameter? delim = data.get_as_nullable_string(3);
-                StringParameter mailbox = data.get_as_string(4);
+                MailboxParameter mailbox = new MailboxParameter.from_string_parameter(data.get_as_string(4));
                 
                 if (!cmd.equals_ci(ListCommand.NAME) && !cmd.equals_ci(ListCommand.XLIST_NAME)) {
                     debug("Bad list response \"%s\": Not marked as list or xlist response",
@@ -105,11 +105,11 @@ public class Geary.Imap.ListResults : Geary.Imap.CommandResults {
                     info = new MailboxInformation(Geary.Imap.Account.INBOX_NAME,
                         (delim != null) ? delim.nullable_value : null, attributes);
                 } else {
-                    info = new MailboxInformation(mailbox.value,
+                    info = new MailboxInformation(mailbox.decode(),
                         (delim != null) ? delim.nullable_value : null, attributes);
                 }
                 
-                map.set(mailbox.value, info);
+                map.set(mailbox.decode(), info);
                 list.add(info);
             } catch (ImapError ierr) {
                 debug("Unable to decode \"%s\": %s", data.to_string(), ierr.message);

src/engine/imap/decoders/imap-status-results.vala

@@ -21,7 +21,7 @@ public class Geary.Imap.StatusResults : Geary.Imap.CommandResults {
      */
     public int unseen { get; private set; }
     
-    public StatusResults(StatusResponse status_response, string mailbox, int messages, int recent,
+    private StatusResults(StatusResponse status_response, string mailbox, int messages, int recent,
         UID? uid_next, UIDValidity? uid_validity, int unseen) {
         base (status_response);
         
@@ -43,7 +43,7 @@ public class Geary.Imap.StatusResults : Geary.Imap.CommandResults {
         
         ServerData data = response.server_data[0];
         StringParameter cmd = data.get_as_string(1);
-        StringParameter mailbox = data.get_as_string(2);
+        MailboxParameter mailbox = new MailboxParameter.from_string_parameter(data.get_as_string(2));
         ListParameter values = data.get_as_list(3);
         
         if (!cmd.equals_ci(StatusCommand.NAME)) {
@@ -93,7 +93,7 @@ public class Geary.Imap.StatusResults : Geary.Imap.CommandResults {
             }
         }
         
-        return new StatusResults(response.status_response, mailbox.value, messages, recent, uid_next,
+        return new StatusResults(response.status_response, mailbox.decode(), messages, recent, uid_next,
             uid_validity, unseen);
     }
 }

src/engine/imap/message/imap-mailbox-parameter.vala

@@ -0,0 +1,43 @@
+/* Copyright 2011-2012 Yorba Foundation
+ *
+ * This software is licensed under the GNU Lesser General Public License
+ * (version 2.1 or later).  See the COPYING file in this distribution.
+ */
+
+/**
+ * A StringParameter that holds a mailbox reference (can be wildcarded).  Used
+ * to juggle between our internal UTF-8 representation of mailboxes and IMAP's
+ * odd "modified UTF-7" representation.  The value is stored in IMAP's encoded
+ * format since that's how it comes across the wire.
+ */
+public class Geary.Imap.MailboxParameter : StringParameter {
+    private static string utf8_to_imap_utf7(string utf8) {
+        try {
+            return Geary.ImapUtf7.utf8_to_imap_utf7(utf8);
+        } catch (ConvertError e) {
+            debug("Error encoding mailbox name '%s': %s", utf8, e.message);
+            return utf8;
+        }
+    }
+    
+    private static string imap_utf7_to_utf8(string imap_utf7) {
+        try {
+            return Geary.ImapUtf7.imap_utf7_to_utf8(imap_utf7);
+        } catch (ConvertError e) {
+            debug("Invalid mailbox name '%s': %s", imap_utf7, e.message);
+            return imap_utf7;
+        }
+    }
+    
+    public MailboxParameter(string mailbox) {
+        base (utf8_to_imap_utf7(mailbox));
+    }
+    
+    public MailboxParameter.from_string_parameter(StringParameter string_parameter) {
+        base (string_parameter.value);
+    }
+    
+    public string decode() {
+        return imap_utf7_to_utf8(value);
+    }
+}

src/engine/imap/transport/imap-client-session-manager.vala

@@ -106,7 +106,8 @@ public class Geary.Imap.ClientSessionManager {
         ClientSession session = yield get_authorized_session_async(cancellable);
         
         ListResults results = ListResults.decode(yield session.send_command_async(
-            new ListCommand.wildcarded("", "%", session.get_capabilities().has_capability("XLIST")),
+            new ListCommand.wildcarded("", new Geary.Imap.MailboxParameter("%"),
+                session.get_capabilities().has_capability("XLIST")),
             cancellable));
         
         if (results.status_response.status != Status.OK)
@@ -124,7 +125,8 @@ public class Geary.Imap.ClientSessionManager {
         ClientSession session = yield get_authorized_session_async(cancellable);
         
         ListResults results = ListResults.decode(yield session.send_command_async(
-            new ListCommand(specifier, session.get_capabilities().has_capability("XLIST")),
+            new ListCommand(new Geary.Imap.MailboxParameter(specifier),
+                session.get_capabilities().has_capability("XLIST")),
             cancellable));
         
         if (results.status_response.status != Status.OK)
@@ -137,7 +139,8 @@ public class Geary.Imap.ClientSessionManager {
         ClientSession session = yield get_authorized_session_async(cancellable);
         
         ListResults results = ListResults.decode(yield session.send_command_async(
-            new ListCommand(path, session.get_capabilities().has_capability("XLIST")),
+            new ListCommand(new Geary.Imap.MailboxParameter(path),
+                session.get_capabilities().has_capability("XLIST")),
             cancellable));
         
         return (results.status_response.status == Status.OK) && (results.get_count() == 1);
@@ -148,7 +151,8 @@ public class Geary.Imap.ClientSessionManager {
         ClientSession session = yield get_authorized_session_async(cancellable);
         
         ListResults results = ListResults.decode(yield session.send_command_async(
-            new ListCommand(path, session.get_capabilities().has_capability("XLIST")),
+            new ListCommand(new Geary.Imap.MailboxParameter(path),
+                session.get_capabilities().has_capability("XLIST")),
             cancellable));
         
         if (results.status_response.status != Status.OK)
@@ -162,7 +166,7 @@ public class Geary.Imap.ClientSessionManager {
         ClientSession session = yield get_authorized_session_async(cancellable);
         
         StatusResults results = StatusResults.decode(yield session.send_command_async(
-            new StatusCommand(path, types), cancellable));
+            new StatusCommand(new Geary.Imap.MailboxParameter(path), types), cancellable));
         
         if (results.status_response.status != Status.OK)
             throw new ImapError.SERVER_ERROR("Server error: %s", results.to_string());

src/engine/imap/transport/imap-client-session.vala

@@ -87,10 +87,10 @@ public class Geary.Imap.ClientSession {
     }
     
     private class SelectParams : AsyncParams {
-        public string mailbox;
+        public Geary.Imap.MailboxParameter mailbox;
         public bool is_select;
         
-        public SelectParams(string mailbox, bool is_select, Cancellable? cancellable, SourceFunc cb) {
+        public SelectParams(Geary.Imap.MailboxParameter mailbox, bool is_select, Cancellable? cancellable, SourceFunc cb) {
             base (cancellable, cb);
             
             this.mailbox = mailbox;
@@ -908,8 +908,8 @@ public class Geary.Imap.ClientSession {
         Cancellable? cancellable) throws Error {
         string? old_mailbox = current_mailbox;
         
-        SelectParams params = new SelectParams(mailbox, is_select, cancellable,
+        SelectParams params = new SelectParams(new Geary.Imap.MailboxParameter(mailbox),
-            select_examine_async.callback);
+            is_select, cancellable, select_examine_async.callback);
         fsm.issue(Event.SELECT, null, params);
         
         if (params.do_yield)
@@ -932,7 +932,7 @@ public class Geary.Imap.ClientSession {
         
         SelectParams params = (SelectParams) object;
         
-        if (current_mailbox != null && current_mailbox == params.mailbox)
+        if (current_mailbox != null && current_mailbox == params.mailbox.decode())
             return state;
         
         // TODO: Currently don't handle situation where one mailbox is selected and another is
@@ -968,7 +968,7 @@ public class Geary.Imap.ClientSession {
         SelectParams params = (SelectParams) object;
         
         assert(current_mailbox == null);
-        current_mailbox = params.mailbox;
+        current_mailbox = params.mailbox.decode();
         current_mailbox_readonly = !params.is_select;
         
         return State.SELECTED;

src/engine/imap/transport/imap-mailbox.vala

@@ -604,7 +604,8 @@ public class Geary.Imap.Mailbox : Geary.SmartReference {
         if (context.is_closed())
             throw new ImapError.NOT_SELECTED("Mailbox %s closed", name);
 
-        yield context.session.send_command_async(new CopyCommand(msg_set, destination.to_string()),
+        yield context.session.send_command_async(
+            new CopyCommand(msg_set, new Geary.Imap.MailboxParameter(destination.to_string())),
             cancellable);
     }
 

src/engine/util/util-imap-utf7.vala

@@ -0,0 +1,269 @@
+/* Copyright 2013 Yorba Foundation
+ * Copyright (c) 2008-2012 Dovecot authors
+ *
+ * This software is licensed under the GNU Lesser General Public License
+ * (version 2.1 or later).  See the COPYING file in this distribution.
+ */
+
+namespace Geary.ImapUtf7 {
+
+/* This file was modified from Dovecot's LGPLv2.1-licensed implementation in
+ * dovecot-2.1.15/src/lib-imap/imap-utf7.c.
+ */
+
+/* These UTF16_* parts were modified from Dovecot's MIT-licensed Unicode
+ * library header in dovecot-2.1.15/src/lib/unichar.h.  I don't believe it's a
+ * substantial enough portion to warrant inclusion of the MIT license.
+ */
+
+/* Characters >= base require surrogates */
+private const unichar UTF16_SURROGATE_BASE = 0x10000;
+
+private const int UTF16_SURROGATE_SHIFT = 10;
+private const unichar UTF16_SURROGATE_MASK = 0x03ff;
+private const unichar UTF16_SURROGATE_HIGH_FIRST = 0xd800;
+private const unichar UTF16_SURROGATE_HIGH_LAST = 0xdbff;
+private const unichar UTF16_SURROGATE_HIGH_MAX = 0xdfff;
+private const unichar UTF16_SURROGATE_LOW_FIRST = 0xdc00;
+private const unichar UTF16_SURROGATE_LOW_LAST = 0xdfff;
+
+private unichar UTF16_SURROGATE_HIGH(unichar chr) {
+    return (UTF16_SURROGATE_HIGH_FIRST +
+        (((chr) - UTF16_SURROGATE_BASE) >> UTF16_SURROGATE_SHIFT));
+}
+private unichar UTF16_SURROGATE_LOW(unichar chr) {
+    return (UTF16_SURROGATE_LOW_FIRST +
+        (((chr) - UTF16_SURROGATE_BASE) & UTF16_SURROGATE_MASK));
+}
+
+private const string imap_b64enc =
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+,";
+
+private const uint8 XX = 0xff;
+private const uint8 imap_b64dec[256] = {
+    XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX,
+    XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX,
+    XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,62, 63,XX,XX,XX,
+    52,53,54,55, 56,57,58,59, 60,61,XX,XX, XX,XX,XX,XX,
+    XX, 0, 1, 2,  3, 4, 5, 6,  7, 8, 9,10, 11,12,13,14,
+    15,16,17,18, 19,20,21,22, 23,24,25,XX, XX,XX,XX,XX,
+    XX,26,27,28, 29,30,31,32, 33,34,35,36, 37,38,39,40,
+    41,42,43,44, 45,46,47,48, 49,50,51,XX, XX,XX,XX,XX,
+    XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX,
+    XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX,
+    XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX,
+    XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX,
+    XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX,
+    XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX,
+    XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX,
+    XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX
+};
+
+private void mbase64_encode(StringBuilder dest, uint8[] input) {
+    dest.append_c('&');
+    int pos = 0;
+    int len = input.length;
+    while (len >= 3) {
+        dest.append_c(imap_b64enc[input[pos + 0] >> 2]);
+        dest.append_c(imap_b64enc[((input[pos + 0] & 3) << 4) |
+                      (input[pos + 1] >> 4)]);
+        dest.append_c(imap_b64enc[((input[pos + 1] & 0x0f) << 2) |
+                      ((input[pos + 2] & 0xc0) >> 6)]);
+        dest.append_c(imap_b64enc[input[pos + 2] & 0x3f]);
+        pos += 3;
+        len -= 3;
+    }
+    if (len > 0) {
+        dest.append_c(imap_b64enc[input[pos + 0] >> 2]);
+        if (len == 1)
+            dest.append_c(imap_b64enc[(input[pos + 0] & 0x03) << 4]);
+        else {
+            dest.append_c(imap_b64enc[((input[pos + 0] & 0x03) << 4) |
+                          (input[pos + 1] >> 4)]);
+            dest.append_c(imap_b64enc[(input[pos + 1] & 0x0f) << 2]);
+        }
+    }
+    dest.append_c('-');
+}
+
+private int first_encode_index(string str) {
+    for (int p = 0; str[p] != '\0'; p++) {
+        if (str[p] == '&' || (uint8) str[p] >= 0x80)
+            return p;
+    }
+    return -1;
+}
+
+public string utf8_to_imap_utf7(string str) throws ConvertError {
+    int p = first_encode_index(str);
+    if (p < 0) {
+        /* no characters that need to be encoded */
+        return str;
+    }
+    
+    /* at least one encoded character */
+    StringBuilder dest = new StringBuilder();
+    dest.append_len(str, p);
+    while (p < str.length) {
+        if (str[p] == '&') {
+            dest.append("&-");
+            p++;
+            continue;
+        }
+        if ((uint8) str[p] < 0x80) {
+            dest.append_c(str[p]);
+            p++;
+            continue;
+        }
+        
+        uint8[] utf16 = {};
+        while ((uint8) str[p] >= 0x80) {
+            int next_p = p;
+            unichar chr;
+            // TODO: validate this conversion, throw ConvertError?
+            str.get_next_char(ref next_p, out chr);
+            if (chr < UTF16_SURROGATE_BASE) {
+                utf16 += (uint8) (chr >> 8);
+                utf16 += (uint8) (chr & 0xff);
+            } else {
+                unichar u16 = UTF16_SURROGATE_HIGH(chr);
+                utf16 += (uint8) (u16 >> 8);
+                utf16 += (uint8) (u16 & 0xff);
+                u16 = UTF16_SURROGATE_LOW(chr);
+                utf16 += (uint8) (u16 >> 8);
+                utf16 += (uint8) (u16 & 0xff);
+            }
+            p = next_p;
+        }
+        mbase64_encode(dest, utf16);
+    }
+    return dest.str;
+}
+
+private void utf16buf_to_utf8(StringBuilder dest, uint8[] output, ref int pos, int len) throws ConvertError {
+    if (len % 2 != 0)
+        throw new ConvertError.ILLEGAL_SEQUENCE("Odd number of bytes in UTF-16 data");
+    
+    uint16 high = (output[pos % 4] << 8) | output[(pos+1) % 4];
+    if (high < UTF16_SURROGATE_HIGH_FIRST ||
+        high > UTF16_SURROGATE_HIGH_MAX) {
+        /* single byte */
+        string? s = ((unichar) high).to_string();
+        if (s == null)
+            throw new ConvertError.ILLEGAL_SEQUENCE("Couldn't convert U+%04hx to UTF-8", high);
+        dest.append(s);
+        pos = (pos + 2) % 4;
+        return;
+    }
+    
+    if (high > UTF16_SURROGATE_HIGH_LAST)
+        throw new ConvertError.ILLEGAL_SEQUENCE("UTF-16 data out of range");
+    if (len != 4) {
+        /* missing the second character */
+        throw new ConvertError.ILLEGAL_SEQUENCE("Truncated UTF-16 data");
+    }
+    
+    uint16 low = (output[(pos+2)%4] << 8) | output[(pos+3) % 4];
+    if (low < UTF16_SURROGATE_LOW_FIRST || low > UTF16_SURROGATE_LOW_LAST)
+        throw new ConvertError.ILLEGAL_SEQUENCE("Illegal UTF-16 surrogate");
+    
+    unichar chr = UTF16_SURROGATE_BASE +
+        (((high & UTF16_SURROGATE_MASK) << UTF16_SURROGATE_SHIFT) |
+         (low & UTF16_SURROGATE_MASK));
+    string? s = chr.to_string();
+    if (s == null)
+        throw new ConvertError.ILLEGAL_SEQUENCE("Couldn't convert U+%04x to UTF-8", chr);
+    dest.append(s);
+}
+
+private void mbase64_decode_to_utf8(StringBuilder dest, string str, ref int p) throws ConvertError {
+    uint8 input[4], output[4];
+    int outstart = 0, outpos = 0;
+    
+    while (str[p] != '-') {
+        input[0] = imap_b64dec[(uint8) str[p + 0]];
+        input[1] = imap_b64dec[(uint8) str[p + 1]];
+        if (input[0] == 0xff || input[1] == 0xff)
+            throw new ConvertError.ILLEGAL_SEQUENCE("Illegal character in IMAP base-64 encoded sequence");
+        
+        output[outpos % 4] = (input[0] << 2) | (input[1] >> 4);
+        if (++outpos % 4 == outstart) {
+            utf16buf_to_utf8(dest, output, ref outstart, 4);
+        }
+        
+        input[2] = imap_b64dec[(uint8) str[p + 2]];
+        if (input[2] == 0xff) {
+            if (str[p + 2] != '-')
+                throw new ConvertError.ILLEGAL_SEQUENCE("Illegal character in IMAP base-64 encoded sequence");
+            
+            p += 2;
+            break;
+        }
+        
+        output[outpos % 4] = (input[1] << 4) | (input[2] >> 2);
+        if (++outpos % 4 == outstart) {
+            utf16buf_to_utf8(dest, output, ref outstart, 4);
+        }
+        
+        input[3] = imap_b64dec[(uint8) str[p + 3]];
+        if (input[3] == 0xff) {
+            if (str[p + 3] != '-')
+                throw new ConvertError.ILLEGAL_SEQUENCE("Illegal character in IMAP base-64 encoded sequence");
+            
+            p += 3;
+            break;
+        }
+        
+        output[outpos % 4] = ((input[2] << 6) & 0xc0) | input[3];
+        if (++outpos % 4 == outstart) {
+            utf16buf_to_utf8(dest, output, ref outstart, 4);
+        }
+        
+        p += 4;
+    }
+    if (outstart != outpos % 4) {
+        utf16buf_to_utf8(dest, output, ref outstart, (4 + outpos - outstart) % 4);
+    }
+    
+    /* found ending '-' */
+    p++;
+}
+
+public string imap_utf7_to_utf8(string str) throws ConvertError {
+    int p;
+    for (p = 0; str[p] != '\0'; p++) {
+        if (str[p] == '&' || (uint8) str[p] >= 0x80)
+            break;
+    }
+    if (str[p] == '\0') {
+        /* no IMAP-UTF-7 encoded characters */
+        return str;
+    }
+    if ((uint8) str[p] >= 0x80) {
+        /* 8bit characters - the input is broken */
+        throw new ConvertError.ILLEGAL_SEQUENCE("IMAP UTF-7 input string contains 8-bit data");
+    }
+    
+    /* at least one encoded character */
+    StringBuilder dest = new StringBuilder();
+    dest.append_len(str, p);
+    while (str[p] != '\0') {
+        if (str[p] == '&') {
+            if (str[++p] == '-') {
+                dest.append_c('&');
+                p++;
+            } else {
+                mbase64_decode_to_utf8(dest, str, ref p);
+                if (str[p + 0] == '&' && str[p + 1] != '-') {
+                    /* &...-& */
+                    throw new ConvertError.ILLEGAL_SEQUENCE("Illegal break in encoded text");
+                }
+            }
+        } else {
+            dest.append_c(str[p++]);
+        }
+    }
+    return dest.str;
+}
+
+}