diff --git a/sql/meson.build b/sql/meson.build index 126f2982..9a629380 100644 --- a/sql/meson.build +++ b/sql/meson.build @@ -24,6 +24,7 @@ sql_files = [ 'version-023.sql', 'version-024.sql', 'version-025.sql', + 'version-026.sql', ] install_data(sql_files, diff --git a/sql/version-026.sql b/sql/version-026.sql new file mode 100644 index 00000000..55ab4fb0 --- /dev/null +++ b/sql/version-026.sql @@ -0,0 +1,4 @@ +-- +-- Track when account storage was last cleaned. +-- +ALTER TABLE GarbageCollectionTable ADD COLUMN last_cleanup_time_t INTEGER DEFAULT NULL; diff --git a/src/client/application/application-controller.vala b/src/client/application/application-controller.vala index 2ff61277..6ba75960 100644 --- a/src/client/application/application-controller.vala +++ b/src/client/application/application-controller.vala @@ -18,6 +18,7 @@ internal class Application.Controller : Geary.BaseObject { private const uint MAX_AUTH_ATTEMPTS = 3; + private const uint CLEANUP_CHECK_AFTER_IDLE_BACKGROUND_MINUTES = 5; /** Determines if conversations can be trashed from the given folder. */ public static bool does_folder_support_trash(Geary.Folder? target) { @@ -91,6 +92,11 @@ internal class Application.Controller : Geary.BaseObject { // Requested mailto composers not yet fullfulled private Gee.List pending_mailtos = new Gee.ArrayList(); + // Timeout to do work in idle after all windows have been sent to the background + private Geary.TimeoutManager all_windows_backgrounded_timeout; + + private GLib.Cancellable? storage_cleanup_cancellable; + /** * Emitted when an account is added or is enabled. @@ -148,6 +154,9 @@ internal class Application.Controller : Geary.BaseObject { ConversationWebView.load_resources(); Accounts.SignatureWebView.load_resources(); + this.all_windows_backgrounded_timeout = + new Geary.TimeoutManager.seconds(CLEANUP_CHECK_AFTER_IDLE_BACKGROUND_MINUTES * 60, on_unfocused_idle); + this.folks = Folks.IndividualAggregator.dup(); if (!this.folks.is_prepared) { // Do this in the background since it can take a long time @@ -1413,6 +1422,35 @@ internal class Application.Controller : Geary.BaseObject { } } + /** + * Track a window receiving focus, for idle background work. + */ + public void window_focus_in() { + this.all_windows_backgrounded_timeout.reset(); + + if (this.storage_cleanup_cancellable != null) { + this.storage_cleanup_cancellable.cancel(); + + // Cleanup was still running and we don't know where we got to so + // we'll clear each of these so it runs next time we're in the + // background + foreach (AccountContext context in this.accounts.values) { + context.cancellable.cancelled.disconnect(this.storage_cleanup_cancellable.cancel); + + Geary.Account account = context.account; + account.last_storage_cleanup = null; + } + this.storage_cleanup_cancellable = null; + } + } + + /** + * Track a window going unfocused, for idle background work. + */ + public void window_focus_out() { + this.all_windows_backgrounded_timeout.start(); + } + /** Displays a composer on the last active main window. */ internal void show_composer(Composer.Widget composer) { var target = this.application.get_active_main_window(); @@ -1672,6 +1710,30 @@ internal class Application.Controller : Geary.BaseObject { } } + private void on_unfocused_idle() { + // Schedule later, catching cases where work should occur later while still in background + this.all_windows_backgrounded_timeout.reset(); + window_focus_out(); + + if (this.storage_cleanup_cancellable == null) + do_background_storage_cleanup.begin(); + } + + private async void do_background_storage_cleanup() { + debug("Checking for backgrounded idle work"); + this.storage_cleanup_cancellable = new GLib.Cancellable(); + + foreach (AccountContext context in this.accounts.values) { + Geary.Account account = context.account; + context.cancellable.cancelled.connect(this.storage_cleanup_cancellable.cancel); + yield account.cleanup_storage(this.storage_cleanup_cancellable); + if (this.storage_cleanup_cancellable.is_cancelled()) + break; + context.cancellable.cancelled.disconnect(this.storage_cleanup_cancellable.cancel); + } + this.storage_cleanup_cancellable = null; + } + } diff --git a/src/client/application/application-main-window.vala b/src/client/application/application-main-window.vala index 332b6847..0a1e2dc8 100644 --- a/src/client/application/application-main-window.vala +++ b/src/client/application/application-main-window.vala @@ -500,6 +500,15 @@ public class Application.MainWindow : // Window actions add_action_entries(MainWindow.WINDOW_ACTIONS, this); + this.focus_in_event.connect((w, e) => { + application.controller.window_focus_in(); + return false; + }); + this.focus_out_event.connect((w, e) => { + application.controller.window_focus_out(); + return false; + }); + setup_layout(application.config); on_change_orientation(); diff --git a/src/client/composer/composer-window.vala b/src/client/composer/composer-window.vala index c3d1db7b..f7d4fd78 100644 --- a/src/client/composer/composer-window.vala +++ b/src/client/composer/composer-window.vala @@ -47,6 +47,15 @@ public class Composer.Window : Gtk.ApplicationWindow, Container { set_titlebar(this.composer.header); } + this.focus_in_event.connect((w, e) => { + application.controller.window_focus_in(); + return false; + }); + this.focus_out_event.connect((w, e) => { + application.controller.window_focus_out(); + return false; + }); + show(); set_position(Gtk.WindowPosition.CENTER); } diff --git a/src/client/dialogs/upgrade-dialog.vala b/src/client/dialogs/upgrade-dialog.vala index 29d88864..e4871984 100644 --- a/src/client/dialogs/upgrade-dialog.vala +++ b/src/client/dialogs/upgrade-dialog.vala @@ -32,6 +32,11 @@ public class UpgradeDialog : Object { } private void on_start() { + // Disable main windows + foreach (Application.MainWindow window in this.application.get_main_windows()) { + window.sensitive = false; + } + Gtk.Builder builder = GioUtil.create_builder("upgrade_dialog.glade"); this.dialog = (Gtk.Dialog) builder.get_object("dialog"); this.dialog.set_transient_for( @@ -58,6 +63,11 @@ public class UpgradeDialog : Object { this.dialog.hide(); this.dialog = null; } + + // Enable main windows + foreach (Application.MainWindow window in this.application.get_main_windows()) { + window.sensitive = true; + } } /** diff --git a/src/engine/api/geary-account.vala b/src/engine/api/geary-account.vala index e08f9cc0..4a4850a5 100644 --- a/src/engine/api/geary-account.vala +++ b/src/engine/api/geary-account.vala @@ -145,6 +145,17 @@ public abstract class Geary.Account : BaseObject, Logging.Source { public ProgressMonitor db_upgrade_monitor { get; protected set; } public ProgressMonitor db_vacuum_monitor { get; protected set; } + /** + * The last time the account storage was cleaned. + * + * This does not imply that a full reap plus vacuum garbage + * collection (GC) is performed, merely that: + * 1. Any old messages are removed + * 2. If any old messages were removed, or the defined period + * (in ImapDB.GC) has past, a GC reap is performed + * 3. GC vacuum is run if recommended + */ + public GLib.DateTime? last_storage_cleanup { get; set; } public signal void opened(); @@ -236,6 +247,11 @@ public abstract class Geary.Account : BaseObject, Logging.Source { */ public signal void email_removed(Geary.Folder folder, Gee.Collection ids); + /** + * Fired when emails are removed from a local folder in this account. + */ + public signal void email_locally_removed(Geary.Folder folder, Gee.Collection ids); + /** * Fired when one or more emails have been locally saved to a folder with * the full set of Fields. @@ -511,6 +527,16 @@ public abstract class Geary.Account : BaseObject, Logging.Source { return new Logging.State(this, this.information.id); } + /** + * Perform cleanup of account storage. + * + * Work is performed if the appropriate interval has past since last + * execution. Alternatively if the interval has not past but vacuum GC + * has been flagged to run this will be executed. Designed to be run + * while the app is in the background and idle. + */ + public abstract async void cleanup_storage(GLib.Cancellable? cancellable); + /** Fires a {@link opened} signal. */ protected virtual void notify_opened() { opened(); @@ -558,6 +584,11 @@ public abstract class Geary.Account : BaseObject, Logging.Source { email_removed(folder, ids); } + /** Fires a {@link email_locally_removed} signal. */ + protected virtual void notify_email_locally_removed(Geary.Folder folder, Gee.Collection ids) { + email_locally_removed(folder, ids); + } + /** Fires a {@link email_locally_complete} signal. */ protected virtual void notify_email_locally_complete(Geary.Folder folder, Gee.Collection ids) { diff --git a/src/engine/api/geary-folder.vala b/src/engine/api/geary-folder.vala index 329bd520..89a940fd 100644 --- a/src/engine/api/geary-folder.vala +++ b/src/engine/api/geary-folder.vala @@ -425,6 +425,11 @@ public abstract class Geary.Folder : BaseObject, Logging.Source { */ public signal void email_removed(Gee.Collection ids); + /** + * Fired when emails are removed from the local folder. + */ + public signal void email_locally_removed(Gee.Collection ids); + /** * Fired when the total count of email in a folder has changed in any way. * diff --git a/src/engine/app/app-conversation-monitor.vala b/src/engine/app/app-conversation-monitor.vala index 7c9c4e2d..6272c7d8 100644 --- a/src/engine/app/app-conversation-monitor.vala +++ b/src/engine/app/app-conversation-monitor.vala @@ -321,6 +321,7 @@ public class Geary.App.ConversationMonitor : BaseObject, Logging.Source { this.base_folder.email_inserted.connect(on_folder_email_inserted); this.base_folder.email_locally_complete.connect(on_folder_email_complete); this.base_folder.email_removed.connect(on_folder_email_removed); + this.base_folder.email_locally_removed.connect(on_folder_email_removed); this.base_folder.opened.connect(on_folder_opened); this.base_folder.account.email_appended.connect(on_account_email_appended); this.base_folder.account.email_inserted.connect(on_account_email_inserted); @@ -669,6 +670,7 @@ public class Geary.App.ConversationMonitor : BaseObject, Logging.Source { this.base_folder.email_inserted.disconnect(on_folder_email_inserted); this.base_folder.email_locally_complete.disconnect(on_folder_email_complete); this.base_folder.email_removed.disconnect(on_folder_email_removed); + this.base_folder.email_locally_removed.disconnect(on_folder_email_removed); this.base_folder.opened.disconnect(on_folder_opened); this.base_folder.account.email_appended.disconnect(on_account_email_appended); this.base_folder.account.email_inserted.disconnect(on_account_email_inserted); diff --git a/src/engine/app/app-search-folder.vala b/src/engine/app/app-search-folder.vala index f161c9fc..85c8c75c 100644 --- a/src/engine/app/app-search-folder.vala +++ b/src/engine/app/app-search-folder.vala @@ -129,6 +129,7 @@ public class Geary.App.SearchFolder : account.folders_use_changed.connect(on_folders_use_changed); account.email_locally_complete.connect(on_email_locally_complete); account.email_removed.connect(on_account_email_removed); + account.email_locally_removed.connect(on_account_email_removed); new_contents(); @@ -142,6 +143,7 @@ public class Geary.App.SearchFolder : account.folders_use_changed.disconnect(on_folders_use_changed); account.email_locally_complete.disconnect(on_email_locally_complete); account.email_removed.disconnect(on_account_email_removed); + account.email_locally_removed.disconnect(on_account_email_removed); } /** diff --git a/src/engine/imap-db/imap-db-account.vala b/src/engine/imap-db/imap-db-account.vala index 2439df06..602b4c2d 100644 --- a/src/engine/imap-db/imap-db-account.vala +++ b/src/engine/imap-db/imap-db-account.vala @@ -382,6 +382,57 @@ private class Geary.ImapDB.Account : BaseObject { return create_local_folder(path, folder_id, properties); } + /** + * Fetch the last time the account cleanup was run. + */ + public async GLib.DateTime? fetch_last_cleanup_async(Cancellable? cancellable) + throws Error { + check_open(); + + int64 last_cleanup_time_t = -1; + yield db.exec_transaction_async(Db.TransactionType.RO, (cx) => { + Db.Result result = cx.query(""" + SELECT last_cleanup_time_t + FROM GarbageCollectionTable + WHERE id = 0 + """); + + if (result.finished) + return Db.TransactionOutcome.FAILURE; + + last_cleanup_time_t = !result.is_null_at(0) ? result.int64_at(0) : -1; + + return Db.TransactionOutcome.SUCCESS; + }, cancellable); + + return (last_cleanup_time_t >= 0) ? new DateTime.from_unix_local(last_cleanup_time_t) : null; + } + + /** + * Set the last time the account cleanup was run. + */ + public async void set_last_cleanup_async(GLib.DateTime? dt, Cancellable? cancellable) + throws Error { + check_open(); + + yield db.exec_transaction_async(Db.TransactionType.WO, (cx) => { + Db.Statement stmt = cx.prepare(""" + UPDATE GarbageCollectionTable + SET last_cleanup_time_t = ? + WHERE id = 0 + """); + if (dt != null) { + stmt.bind_int64(0, dt.to_unix()); + } else { + stmt.bind_null(0); + } + + stmt.exec(cancellable); + + return Db.TransactionOutcome.COMMIT; + }, cancellable); + } + private Geary.ImapDB.Folder? get_local_folder(Geary.FolderPath path) { FolderReference? folder_ref = folder_refs.get(path); if (folder_ref == null) diff --git a/src/engine/imap-db/imap-db-database.vala b/src/engine/imap-db/imap-db-database.vala index 91e2f01c..921189d1 100644 --- a/src/engine/imap-db/imap-db-database.vala +++ b/src/engine/imap-db/imap-db-database.vala @@ -18,6 +18,28 @@ private class Geary.ImapDB.Database : Geary.Db.VersionedDatabase { /** SQLite UTF-8 collation name. */ public const string UTF8_COLLATE = "UTF8COLL"; + /** Options to use when running garbage collection. */ + [Flags] + public enum GarbageCollectionOptions { + + /** Reaping will not be forced and vacuuming not permitted. */ + NONE, + + /** + * Reaping will be performed, regardless of recommendation. + */ + FORCE_REAP, + + /** + * Whether to permit database vacuum. + * + * Vacuuming is performed in the foreground. + */ + ALLOW_VACUUM; + } + + public bool want_background_vacuum { get; set; default = false; } + private static void utf8_transliterate_fold(Sqlite.Context context, Sqlite.Value[] values) { @@ -73,6 +95,26 @@ private class Geary.ImapDB.Database : Geary.Db.VersionedDatabase { throws Error { yield base.open(flags, cancellable); + Geary.ClientService services_to_pause[] = {}; + yield run_gc(NONE, services_to_pause, cancellable); + } + + /** + * Run garbage collection + * + * Reap should only be forced when there is known cleanup to perform and + * the interval based recommendation should be bypassed. + */ + public async void run_gc(GarbageCollectionOptions options, + Geary.ClientService[] services_to_pause, + GLib.Cancellable? cancellable) + throws Error { + + if (this.gc != null) { + debug("GC abandoned, possibly already running"); + return; + } + // Tie user-supplied Cancellable to internal Cancellable, which is used when close() is // called if (cancellable != null) @@ -89,26 +131,49 @@ private class Geary.ImapDB.Database : Geary.Db.VersionedDatabase { // VACUUM needs to execute in the foreground with the user given a busy prompt (and cannot // be run at the same time as REAP) if ((recommended & GC.RecommendedOperation.VACUUM) != 0) { - if (!vacuum_monitor.is_in_progress) - vacuum_monitor.notify_start(); + if (GarbageCollectionOptions.ALLOW_VACUUM in options) { + this.want_background_vacuum = false; + foreach (ClientService service in services_to_pause) { + yield service.stop(gc_cancellable); + } - try { - yield this.gc.vacuum_async(gc_cancellable); - } catch (Error err) { - message( - "Vacuum of IMAP database %s failed: %s", this.path, err.message - ); - throw err; - } finally { - if (vacuum_monitor.is_in_progress) - vacuum_monitor.notify_finish(); - } + if (!vacuum_monitor.is_in_progress) + vacuum_monitor.notify_start(); + + try { + yield this.gc.vacuum_async(gc_cancellable); + } catch (Error err) { + message( + "Vacuum of IMAP database %s failed: %s", this.path, err.message + ); + throw err; + } finally { + if (vacuum_monitor.is_in_progress) + vacuum_monitor.notify_finish(); + } + + foreach (ClientService service in services_to_pause) { + yield service.start(gc_cancellable); + } + } else { + // Flag a vacuum to run later when we've been idle in the background + debug("Flagging desire to GC vacuum"); + this.want_background_vacuum = true; + } + } + + // Abandon REAP if cancelled + if (cancellable != null && cancellable.is_cancelled()) { + cancellable.cancelled.disconnect(cancel_gc); + return; } // REAP can run in the background while the application is executing - if ((recommended & GC.RecommendedOperation.REAP) != 0) { + if (GarbageCollectionOptions.FORCE_REAP in options || (recommended & GC.RecommendedOperation.REAP) != 0) { // run in the background and allow application to continue running this.gc.reap_async.begin(gc_cancellable, on_reap_async_completed); + } else { + this.gc = null; } if (cancellable != null) @@ -123,6 +188,24 @@ private class Geary.ImapDB.Database : Geary.Db.VersionedDatabase { this.path, err.message); } + // Check if after reap we now want to schedule a background vacuum. The idea + // here is eg. if we've just reduced prefetch period, reap has detached a + // whole lot of messages and we want to vacuum. This check catches that + // vacuum recommendation, flagging it to run when in background. + this.gc.should_run_async.begin( + gc_cancellable, + (obj, res) => { + try { + GC.RecommendedOperation recommended = this.gc.should_run_async.end(res); + if ((recommended & GC.RecommendedOperation.VACUUM) != 0) + this.want_background_vacuum = true; + } catch (Error err) { + debug("Failed to run GC check on %s after REAP: %s", + this.path, err.message); + } + } + ); + this.gc = null; } diff --git a/src/engine/imap-db/imap-db-folder.vala b/src/engine/imap-db/imap-db-folder.vala index 9b854173..9bfd6c8e 100644 --- a/src/engine/imap-db/imap-db-folder.vala +++ b/src/engine/imap-db/imap-db-folder.vala @@ -41,6 +41,7 @@ private class Geary.ImapDB.Folder : BaseObject, Geary.ReferenceSemantics { private const int LIST_EMAIL_FIELDS_CHUNK_COUNT = 500; private const int REMOVE_COMPLETE_LOCATIONS_CHUNK_COUNT = 500; private const int CREATE_MERGE_EMAIL_CHUNK_COUNT = 25; + private const int OLD_MSG_DETACH_BATCH_SIZE = 1000; [Flags] public enum ListFlags { @@ -882,6 +883,107 @@ private class Geary.ImapDB.Folder : BaseObject, Geary.ReferenceSemantics { }, cancellable); } + public async Gee.Collection? detach_emails_before_timestamp(DateTime cutoff, + GLib.Cancellable? cancellable) throws Error { + debug("Detaching emails before %s for folder ID %s", cutoff.to_string(), this.folder_id.to_string()); + Gee.ArrayList? deleted_email_ids = null; + Gee.ArrayList deleted_primary_keys = null; + + yield db.exec_transaction_async(Db.TransactionType.RO, (cx) => { + // MessageLocationTable.ordering isn't relied on due to IMAP folder + // UIDs not guaranteed to be in order. + StringBuilder sql = new StringBuilder(); + sql.append(""" + SELECT id, message_id, ordering + FROM MessageLocationTable + WHERE folder_id = ? + AND message_id IN ( + SELECT id + FROM MessageTable + INDEXED BY MessageTableInternalDateTimeTIndex + WHERE internaldate_time_t < ? + ) + """); + + Db.Statement stmt = cx.prepare(sql.str); + stmt.bind_rowid(0, folder_id); + stmt.bind_int64(1, cutoff.to_unix()); + + Db.Result results = stmt.exec(cancellable); + + while (!results.finished) { + if (deleted_email_ids == null) { + deleted_email_ids = new Gee.ArrayList(); + deleted_primary_keys = new Gee.ArrayList(); + } + + deleted_email_ids.add( + new ImapDB.EmailIdentifier(results.int64_at(1), + new Imap.UID(results.int64_at(2))) + ); + deleted_primary_keys.add(results.rowid_at(0).to_string()); + + results.next(cancellable); + } + return Db.TransactionOutcome.DONE; + }, cancellable); + + if (deleted_email_ids != null) { + // Delete in batches to avoid hiting SQLite maximum query + // length (although quite unlikely) + int delete_index = 0; + while (delete_index < deleted_primary_keys.size) { + int batch_counter = 0; + + StringBuilder message_location_ids_sql_sublist = new StringBuilder(); + StringBuilder message_ids_sql_sublist = new StringBuilder(); + while (delete_index < deleted_primary_keys.size + && batch_counter < OLD_MSG_DETACH_BATCH_SIZE) { + if (batch_counter > 0) { + message_location_ids_sql_sublist.append(","); + message_ids_sql_sublist.append(","); + } + message_location_ids_sql_sublist.append( + deleted_primary_keys.get(delete_index) + ); + message_ids_sql_sublist.append( + deleted_email_ids.get(delete_index).message_id.to_string() + ); + delete_index++; + batch_counter++; + } + + yield db.exec_transaction_async(Db.TransactionType.WO, (cx) => { + StringBuilder sql = new StringBuilder(); + sql.append(""" + DELETE FROM MessageLocationTable + WHERE id IN ( + """); + sql.append(message_location_ids_sql_sublist.str); + sql.append(")"); + Db.Statement stmt = cx.prepare(sql.str); + + stmt.exec(cancellable); + + sql = new StringBuilder(); + sql.append(""" + DELETE FROM MessageSearchTable + WHERE docid IN ( + """); + sql.append(message_ids_sql_sublist.str); + sql.append(")"); + stmt = cx.prepare(sql.str); + + stmt.exec(cancellable); + + return Db.TransactionOutcome.COMMIT; + }, cancellable); + } + } + + return deleted_email_ids; + } + public async void mark_email_async(Gee.Collection to_mark, Geary.EmailFlags? flags_to_add, Geary.EmailFlags? flags_to_remove, Cancellable? cancellable) throws Error { diff --git a/src/engine/imap-engine/imap-engine-account-synchronizer.vala b/src/engine/imap-engine/imap-engine-account-synchronizer.vala index 21ea1c74..5b9b9c7e 100644 --- a/src/engine/imap-engine/imap-engine-account-synchronizer.vala +++ b/src/engine/imap-engine/imap-engine-account-synchronizer.vala @@ -25,6 +25,7 @@ private class Geary.ImapEngine.AccountSynchronizer : ); this.account.information.notify["prefetch-period-days"].connect(on_account_prefetch_changed); + this.account.old_messages_background_cleanup_request.connect(old_messages_background_cleanup); this.account.folders_available_unavailable.connect(on_folders_updated); this.account.folders_contents_altered.connect(on_folders_contents_altered); } @@ -44,7 +45,11 @@ private class Geary.ImapEngine.AccountSynchronizer : ); } - private void send_all(Gee.Collection folders, bool became_available) { + private void send_all(Gee.Collection folders, + bool became_available, + bool for_storage_clean=false, + IdleGarbageCollection? post_idle_detach_op=null) { + foreach (Folder folder in folders) { // Only sync folders that: // 1. Can actually be opened (i.e. are selectable) @@ -60,11 +65,19 @@ private class Geary.ImapEngine.AccountSynchronizer : !folder.properties.is_local_only && !folder.properties.is_virtual) { - AccountOperation op = became_available - ? new CheckFolderSync( - this.account, imap_folder, this.max_epoch - ) - : new RefreshFolderSync(this.account, imap_folder); + AccountOperation op; + if (became_available || for_storage_clean) { + CheckFolderSync check_op = new CheckFolderSync( + this.account, + imap_folder, + this.max_epoch, + for_storage_clean, + post_idle_detach_op + ); + op = check_op; + } else { + op = new RefreshFolderSync(this.account, imap_folder); + } try { this.account.queue_operation(op); @@ -84,6 +97,22 @@ private class Geary.ImapEngine.AccountSynchronizer : } } + private void old_messages_background_cleanup(GLib.Cancellable? cancellable) { + + if (this.account.is_open()) { + IdleGarbageCollection op = new IdleGarbageCollection(account); + + send_all(this.account.list_folders(), false, true, op); + + // Add GC operation after message removal during background cleanup + try { + this.account.queue_operation(op); + } catch (Error err) { + warning("Failed to queue sync operation: %s", err.message); + } + } + } + private void on_account_prefetch_changed() { this.prefetch_timer.start(); } @@ -214,15 +243,20 @@ private class Geary.ImapEngine.RefreshFolderSync : FolderOperation { */ private class Geary.ImapEngine.CheckFolderSync : RefreshFolderSync { - private DateTime sync_max_epoch; + private bool for_storage_clean; + private IdleGarbageCollection? post_idle_detach_op; internal CheckFolderSync(GenericAccount account, MinimalFolder folder, - DateTime sync_max_epoch) { + DateTime sync_max_epoch, + bool for_storage_clean, + IdleGarbageCollection? post_idle_detach_op) { base(account, folder); this.sync_max_epoch = sync_max_epoch; + this.for_storage_clean = for_storage_clean; + this.post_idle_detach_op = post_idle_detach_op; } protected override async void sync_folder(Cancellable cancellable) @@ -238,9 +272,34 @@ private class Geary.ImapEngine.CheckFolderSync : RefreshFolderSync { prefetch_max_epoch = this.sync_max_epoch; } + ImapDB.Folder local_folder = ((MinimalFolder) this.folder).local_folder; + + // Detach older emails outside the prefetch window + if (this.account.information.prefetch_period_days >= 0) { + Gee.Collection? detached_ids = + yield local_folder.detach_emails_before_timestamp(prefetch_max_epoch, + cancellable); + if (detached_ids != null) { + this.folder.email_locally_removed(detached_ids); + if (post_idle_detach_op != null) { + post_idle_detach_op.messages_detached(); + } + + if (!for_storage_clean) { + GenericAccount imap_account = (GenericAccount) account; + ForegroundGarbageCollection op = + new ForegroundGarbageCollection(imap_account); + try { + imap_account.queue_operation(op); + } catch (Error err) { + warning("Failed to queue sync operation: %s", err.message); + } + } + } + } + // get oldest local email and its time, as well as number // of messages in local store - ImapDB.Folder local_folder = ((MinimalFolder) this.folder).local_folder; Gee.List? list = yield local_folder.list_email_by_id_async( null, 1, @@ -366,3 +425,70 @@ private class Geary.ImapEngine.CheckFolderSync : RefreshFolderSync { } } + +/** + * Kicks off garbage collection after old messages have been removed. + * + * Queues a basic GC run which will run if old messages were detached + * after a folder became available. Not used for backgrounded account + * storage operations, which are handled instead by the + * {@link IdleGarbageCollection}. + */ +private class Geary.ImapEngine.ForegroundGarbageCollection: AccountOperation { + + internal ForegroundGarbageCollection(GenericAccount account) { + base(account); + } + + public override async void execute(GLib.Cancellable cancellable) + throws Error { + if (cancellable.is_cancelled()) + return; + + // Run basic GC + GenericAccount generic_account = (GenericAccount) account; + Geary.ClientService services_to_pause[] = {}; + yield generic_account.local.db.run_gc(NONE, services_to_pause, cancellable); + } + + public override bool equal_to(AccountOperation op) { + return (op != null + && (this == op || this.get_type() == op.get_type()) + && this.account == op.account); + } + +} + +/** + * Performs garbage collection after old messages have been removed during + * backgrounded idle cleanup. + * + * Queues a GC run after a cleanup of messages has occurred while the + * app is idle in the background. Vacuuming will be permitted and if + * messages have been removed a reap will be forced. + */ +private class Geary.ImapEngine.IdleGarbageCollection: AccountOperation { + + // Vacuum is allowed as we're running in the background + private Geary.ImapDB.Database.GarbageCollectionOptions options = ALLOW_VACUUM; + + internal IdleGarbageCollection(GenericAccount account) { + base(account); + } + + public override async void execute(GLib.Cancellable cancellable) + throws Error { + if (cancellable.is_cancelled()) + return; + + GenericAccount generic_account = (GenericAccount) this.account; + generic_account.local.db.run_gc.begin(this.options, + {generic_account.imap, generic_account.smtp}, + cancellable); + } + + public void messages_detached() { + // Reap is forced if messages were detached + this.options |= FORCE_REAP; + } +} diff --git a/src/engine/imap-engine/imap-engine-generic-account.vala b/src/engine/imap-engine/imap-engine-generic-account.vala index 6a83ff01..6c5f1d2d 100644 --- a/src/engine/imap-engine/imap-engine-generic-account.vala +++ b/src/engine/imap-engine/imap-engine-generic-account.vala @@ -17,6 +17,9 @@ private abstract class Geary.ImapEngine.GenericAccount : Geary.Account { // we don't need to double check. private const int REFRESH_FOLDER_LIST_SEC = 15 * 60; + /** Minimum interval between account storage cleanup work */ + private const uint APP_BACKGROUNDED_CLEANUP_WORK_INTERVAL_MINUTES = 60 * 24; + private const Folder.SpecialUse[] SUPPORTED_SPECIAL_FOLDERS = { DRAFTS, SENT, @@ -39,6 +42,8 @@ private abstract class Geary.ImapEngine.GenericAccount : Geary.Account { /** Local database for the account. */ public ImapDB.Account local { get; private set; } + public signal void old_messages_background_cleanup_request(GLib.Cancellable? cancellable); + private bool open = false; private Cancellable? open_cancellable = null; private Nonblocking.Semaphore? remote_ready_lock = null; @@ -135,6 +140,9 @@ private abstract class Geary.ImapEngine.GenericAccount : Geary.Account { throw err; } + this.last_storage_cleanup = yield this.local.fetch_last_cleanup_async(cancellable); + this.notify["last_storage_cleanup"].connect(on_last_storage_cleanup_notify); + this.open = true; notify_opened(); @@ -559,6 +567,24 @@ private abstract class Geary.ImapEngine.GenericAccount : Geary.Account { return (map.size == 0) ? null : map; } + /** {@inheritDoc} */ + public override async void cleanup_storage(GLib.Cancellable? cancellable) { + debug("Backgrounded storage cleanup check for %s account", this.information.display_name); + + DateTime now = new DateTime.now_local(); + DateTime? last_cleanup = this.last_storage_cleanup; + + if (last_cleanup == null || + (now.difference(last_cleanup) / TimeSpan.MINUTE > APP_BACKGROUNDED_CLEANUP_WORK_INTERVAL_MINUTES)) { + // Interval check is OK, start by detaching old messages + this.last_storage_cleanup = now; + this.old_messages_background_cleanup_request(cancellable); + } else if (local.db.want_background_vacuum) { + // Vacuum has been flagged as needed, run it + local.db.run_gc.begin(ALLOW_VACUUM, {this.imap, this.smtp}, cancellable); + } + } + /** * Constructs a set of folders and adds them to the account. * @@ -774,6 +800,7 @@ private abstract class Geary.ImapEngine.GenericAccount : Geary.Account { folder.email_appended.connect(notify_email_appended); folder.email_inserted.connect(notify_email_inserted); folder.email_removed.connect(notify_email_removed); + folder.email_locally_removed.connect(notify_email_locally_removed); folder.email_locally_complete.connect(notify_email_locally_complete); folder.email_flags_changed.connect(notify_email_flags_changed); } @@ -783,6 +810,7 @@ private abstract class Geary.ImapEngine.GenericAccount : Geary.Account { folder.email_appended.disconnect(notify_email_appended); folder.email_inserted.disconnect(notify_email_inserted); folder.email_removed.disconnect(notify_email_removed); + folder.email_locally_removed.disconnect(notify_email_locally_removed); folder.email_locally_complete.disconnect(notify_email_locally_complete); folder.email_flags_changed.disconnect(notify_email_flags_changed); } @@ -807,6 +835,11 @@ private abstract class Geary.ImapEngine.GenericAccount : Geary.Account { schedule_unseen_update(folder); } + /** {@inheritDoc} */ + protected override void notify_email_locally_removed(Geary.Folder folder, Gee.Collection ids) { + base.notify_email_locally_removed(folder, ids); + } + /** {@inheritDoc} */ protected override void notify_email_flags_changed(Geary.Folder folder, Gee.Map flag_map) { @@ -969,6 +1002,13 @@ private abstract class Geary.ImapEngine.GenericAccount : Geary.Account { } } + private void on_last_storage_cleanup_notify() { + this.local.set_last_cleanup_async.begin( + this.last_storage_cleanup, + this.open_cancellable + ); + } + } diff --git a/src/engine/imap-engine/imap-engine-minimal-folder.vala b/src/engine/imap-engine/imap-engine-minimal-folder.vala index 322dcfd8..20fdc2c2 100644 --- a/src/engine/imap-engine/imap-engine-minimal-folder.vala +++ b/src/engine/imap-engine/imap-engine-minimal-folder.vala @@ -1298,6 +1298,9 @@ private class Geary.ImapEngine.MinimalFolder : Geary.Folder, Geary.FolderSupport // open while processing first the flag updates then the // expunge from the remote yield this.replay_queue.checkpoint(cancellable); + + Geary.ClientService services_to_pause[] = {}; + yield this._account.local.db.run_gc(NONE, services_to_pause, cancellable); } private void check_open(string method) throws EngineError { diff --git a/test/engine/api/geary-account-mock.vala b/test/engine/api/geary-account-mock.vala index d2a6cae5..461a32fe 100644 --- a/test/engine/api/geary-account-mock.vala +++ b/test/engine/api/geary-account-mock.vala @@ -286,4 +286,7 @@ public class Geary.MockAccount : Account, MockObject { ); } + public override async void cleanup_storage(GLib.Cancellable? cancellable) { + + } } diff --git a/test/engine/imap-db/imap-db-database-test.vala b/test/engine/imap-db/imap-db-database-test.vala index 1fb73683..fd5046ae 100644 --- a/test/engine/imap-db/imap-db-database-test.vala +++ b/test/engine/imap-db/imap-db-database-test.vala @@ -106,7 +106,7 @@ class Geary.ImapDB.DatabaseTest : TestCase { ); db.open.end(async_result()); - assert_int(25, db.get_schema_version(), "Post-upgrade version"); + assert_int(26, db.get_schema_version(), "Post-upgrade version"); // Since schema v22 deletes the re-creates all attachments, // attachment 12 should no longer exist on the file system and diff --git a/test/engine/imap-db/imap-db-folder-test.vala b/test/engine/imap-db/imap-db-folder-test.vala index 17baabdc..eac4c0b8 100644 --- a/test/engine/imap-db/imap-db-folder-test.vala +++ b/test/engine/imap-db/imap-db-folder-test.vala @@ -26,6 +26,7 @@ class Geary.ImapDB.FolderTest : TestCase { //add_test("merge_existing_preview", merge_existing_preview); add_test("set_flags", set_flags); add_test("set_flags_on_deleted", set_flags_on_deleted); + add_test("detach_emails_before_timestamp", detach_emails_before_timestamp); } public override void set_up() throws GLib.Error { @@ -323,6 +324,70 @@ class Geary.ImapDB.FolderTest : TestCase { assert_flags(test, test_flags); } + public void detach_emails_before_timestamp() throws GLib.Error { + // Ensures that messages outside the folder and within the epoch aren't + // removed, and that messages meeting the criteria are removed. + + this.account.db.exec( + "INSERT INTO FolderTable (id, name) VALUES (2, 'other');" + ); + + GLib.DateTime threshold = new GLib.DateTime.local(2020, 1, 1, 0, 0, 0); + GLib.DateTime beyond_threshold = new GLib.DateTime.local(2019, 1, 1, 0, 0, 0); + GLib.DateTime within_threshold = new GLib.DateTime.local(2021, 1, 1, 0, 0, 0); + + Email.Field fixture_fields = Email.Field.RECEIVERS; + string fixture_to = "test1@example.com"; + this.account.db.exec( + "INSERT INTO MessageTable (id, fields, to_field, internaldate_time_t) " + + "VALUES (1, %d, '%s', %s);".printf(fixture_fields, + fixture_to, + within_threshold.to_unix().to_string()) + ); + this.account.db.exec( + "INSERT INTO MessageTable (id, fields, to_field, internaldate_time_t) " + + "VALUES (2, %d, '%s', %s);".printf(fixture_fields, + fixture_to, + within_threshold.to_unix().to_string()) + ); + this.account.db.exec( + "INSERT INTO MessageTable (id, fields, to_field, internaldate_time_t) " + + "VALUES (3, %d, '%s', %s);".printf(fixture_fields, + fixture_to, + beyond_threshold.to_unix().to_string()) + ); + + this.account.db.exec(""" + INSERT INTO MessageLocationTable + (id, message_id, folder_id, ordering, remove_marker) + VALUES + (1, 1, 1, 1, 1), + (2, 2, 2, 1, 1), + (3, 3, 1, 2, 1); + """); + + this.folder.detach_emails_before_timestamp.begin( + threshold, + null, + this.async_completion + ); + this.folder.detach_emails_before_timestamp.end(async_result()); + + int64[] expected = { 1, 2 }; + Db.Result result = this.account.db.query( + "SELECT id FROM MessageLocationTable" + ); + + int i = 0; + while (!result.finished) { + assert_true(i < expected.length, "Too many rows"); + assert_int64(expected[i], result.int64_at(0)); + i++; + result.next(); + } + assert_true(i == expected.length, "Not enough rows"); + } + private Email new_mock_remote_email(int64 uid, string? subject = null, Geary.EmailFlags? flags = null) {