diff --git a/src/client/conversation-list/conversation-list-view.vala b/src/client/conversation-list/conversation-list-view.vala
index ad6ec7f1..cc39ed95 100644
--- a/src/client/conversation-list/conversation-list-view.vala
+++ b/src/client/conversation-list/conversation-list-view.vala
@@ -564,7 +564,8 @@ public class ConversationList.View : Gtk.ScrolledWindow, Geary.BaseInterface {
         } else if (this.selection_mode_enabled) {
             this.selection_mode_enabled = false;
         // Set next conversation
-        } else if (this.config.autoselect) {
+        } else if (this.config.autoselect &&
+                   this.list.get_selected_rows().length() == 0) {
             restore_row();
         }
     }