Reenable converting plain text URLs to links in HTML documents.

* src/client/composer/composer-web-view.vala (ComposerWebView): Rename
  ::linkify_document since it really only applies to editor content. Make
  an asyc method so we can wait until its finished. Update call
  sites. Thunk call to JS.

* src/client/web-process/util-composer.vala,
  src/client/web-process/util-webkit.vala: Remove unused code.

* ui/composer-web-view.js (ComposerPageState):  Add ::linkifyContent
  method and ::linkify static method. Add unit tests.

src/CMakeLists.txt

@@ -413,7 +413,6 @@ client/util/util-webkit.vala
 set(WEB_PROCESS_SRC
 client/web-process/web-process-extension.vala
 client/web-process/util-composer.vala
-client/web-process/util-webkit.vala
 )
 
 set(CONSOLE_SRC

src/client/composer/composer-web-view.vala

@@ -407,6 +407,13 @@ public class ComposerWebView : ClientWebView {
         }
     }
 
+    /**
+     * Converts plain text URLs in the editor content into links.
+     */
+    public async void linkify_content() throws Error {
+        yield run_javascript("geary.linkifyContent();", null);
+    }
+
     /**
      * Returns the editor content as an HTML string.
      */
@@ -491,13 +498,6 @@ public class ComposerWebView : ClientWebView {
         return flowed.str;
     }
 
-    /**
-     * ???
-     */
-    public void linkify_document() {
-        // XXX
-    }
-
     public override bool button_release_event(Gdk.EventButton event) {
         // WebView seems to unconditionally consume button events, so
         // to show a link popopver after the view has processed one,

src/client/composer/composer-widget.vala

@@ -1260,16 +1260,15 @@ public class ComposerWidget : Gtk.EventBox {
     private async void on_send_async() {
         this.container.vanish();
         this.is_closing = true;
-        
+
-        this.editor.linkify_document();
-        
         // Perform send.
         try {
+            yield this.editor.linkify_content();
             yield this.account.send_email_async(yield get_composed_email());
         } catch (Error e) {
             GLib.message("Error sending email: %s", e.message);
         }
-        
+
         Geary.Nonblocking.Semaphore? semaphore = discard_draft();
         if (semaphore != null) {
             try {

src/client/web-process/util-composer.vala

@@ -19,11 +19,6 @@ namespace Util.Composer {
     private const string EDITING_DELETE_CONTAINER_ID = "WebKit-Editing-Delete-Container";
 
 
-    public void linkify_document(WebKit.WebPage page) {
-        Util.DOM.linkify_document(page.get_dom_document());
-    }
-
-
     /////////////////////// From WebEditorFixer ///////////////////////
 
     public bool on_should_insert_text(WebKit.WebPage page,

src/client/web-process/util-webkit.vala

@@ -1,124 +0,0 @@
-/* Copyright 2016 Software Freedom Conservancy Inc.
- *
- * This software is licensed under the GNU Lesser General Public License
- * (version 2.1 or later).  See the COPYING file in this distribution.
- */
-
-// Regex to determine if a URL has a known protocol.
-public const string PROTOCOL_REGEX = "^(aim|apt|bitcoin|cvs|ed2k|ftp|file|finger|git|gtalk|http|https|irc|ircs|irc6|lastfm|ldap|ldaps|magnet|news|nntp|rsync|sftp|skype|smb|sms|svn|telnet|tftp|ssh|webcal|xmpp):";
-
-namespace Util.DOM {
-
-    // Linkifies plain text links in an HTML document.
-    public void linkify_document(WebKit.DOM.Document document) {
-        linkify_recurse(document, document.get_body());
-    }
-
-    // Validates a URL.
-    // Ensures the URL begins with a valid protocol specifier.  (If not, we don't
-    // want to linkify it.)
-    // Note that the output of this will place '\01' chars before and after a link,
-    // which we use to split the string in linkify()
-    private bool pre_split_urls(MatchInfo match_info, StringBuilder result) {
-        try {
-            string? url = match_info.fetch(0);
-            Regex r = new Regex(PROTOCOL_REGEX, RegexCompileFlags.CASELESS);
-            result.append(r.match(url) ? "\01%s\01".printf(url) : url);
-        } catch (Error e) {
-            debug("URL parsing error: %s", e.message);
-        }
-        return false; // False to continue processing.
-    }
-
-    // Linkifies "plain text" links in the HTML doc.  If you want to do this
-    // for the entire document, use the get_dom_document().get_body() for the
-    // node param and leave _in_link as false.
-    private void linkify_recurse(WebKit.DOM.Document document, WebKit.DOM.Node node,
-        bool _in_link = false) {
-
-        bool in_link = _in_link;
-        if (node is WebKit.DOM.HTMLAnchorElement)
-            in_link = true;
-
-        string input = node.get_node_value();
-        if (!in_link && !Geary.String.is_empty(input)) {
-            try {
-                Regex r = new Regex(Geary.HTML.URL_REGEX, RegexCompileFlags.CASELESS);
-                string output = r.replace_eval(input, -1, 0, 0, pre_split_urls);
-                if (input != output) {
-                    // We got one!  Now split the text and swap out the node.
-                    Regex tester = new Regex(PROTOCOL_REGEX, RegexCompileFlags.CASELESS);
-                    string[] pieces = output.split("\01");
-                    Gee.ArrayList<WebKit.DOM.Node> new_nodes = new Gee.ArrayList<WebKit.DOM.Node>();
-
-                    for(int i = 0; i < pieces.length; i++) {
-                        //WebKit.DOM.Node new_node;
-                        if (tester.match(pieces[i])) {
-                            // Link part.
-                            WebKit.DOM.HTMLAnchorElement anchor = document.create_element("a")
-                                as WebKit.DOM.HTMLAnchorElement;
-                            anchor.href = pieces[i];
-                            anchor.set_inner_text(pieces[i]);
-                            new_nodes.add(anchor);
-                        } else {
-                            // Text part.
-                            WebKit.DOM.Node new_node = node.clone_node(false);
-                            new_node.set_node_value(pieces[i]);
-                            new_nodes.add(new_node);
-                        }
-                    }
-
-                    // Add our new nodes.
-                    WebKit.DOM.Node? sibling = node.get_next_sibling();
-                    for (int i = 0; i < new_nodes.size; i++) {
-                        WebKit.DOM.Node new_node = new_nodes.get(i);
-                        if (sibling == null)
-                            node.get_parent_node().append_child(new_node);
-                        else
-                            node.get_parent_node().insert_before(new_node, sibling);
-                    }
-
-                    // Remove the original node's text.
-                    node.set_node_value("");
-                }
-            } catch (Error e) {
-                debug("Error linkifying outgoing mail: %s", e.message);
-            }
-        }
-
-        // Visit children.
-        WebKit.DOM.NodeList list = node.get_child_nodes();
-        for (int i = 0; i < list.length; i++) {
-            linkify_recurse(document, list.item(i), in_link);
-        }
-    }
-
-    // Validates a URL.  Intended to be used as a RegexEvalCallback.
-    // Ensures the URL begins with a valid protocol specifier.  (If not, we don't
-    // want to linkify it.)
-    public bool is_valid_url(MatchInfo match_info, StringBuilder result) {
-        try {
-            string? url = match_info.fetch(0);
-            Regex r = new Regex(PROTOCOL_REGEX, RegexCompileFlags.CASELESS);
-
-            result.append(r.match(url) ? "<a href=\"%s\">%s</a>".printf(url, url) : url);
-        } catch (Error e) {
-            debug("URL parsing error: %s", e.message);
-        }
-        return false; // False to continue processing.
-    }
-
-    // Converts plain text emails to something safe and usable in HTML.
-    public string linkify_and_escape_plain_text(string input) throws Error {
-        // Convert < and > into non-printable characters, and change & to &amp;.
-        string output = input.replace("<", " \01 ").replace(">", " \02 ").replace("&", "&amp;");
-
-        // Converts text links into HTML hyperlinks.
-        Regex r = new Regex(Geary.HTML.URL_REGEX, RegexCompileFlags.CASELESS);
-
-        output = r.replace_eval(output, -1, 0, 0, is_valid_url);
-        return output.replace(" \01 ", "&lt;").replace(" \02 ", "&gt;");
-    }
-
-}
-

test/js/composer-page-state-test.vala

@@ -13,13 +13,15 @@ class ComposerPageStateTest : ClientWebViewTestCase<ComposerWebView> {
         add_test("edit_context_link", edit_context_link);
         add_test("indent_line", indent_line);
         add_test("contains_attachment_keywords", contains_attachment_keywords);
+        add_test("linkify_content", linkify_content);
         add_test("get_html", get_html);
         add_test("get_text", get_text);
         add_test("get_text_with_quote", get_text_with_quote);
         add_test("get_text_with_nested_quote", get_text_with_nested_quote);
-        add_test("resolve_nesting", resolve_nesting);
+
         add_test("contains_keywords", contains_keywords);
         add_test("quote_lines", quote_lines);
+        add_test("resolve_nesting", resolve_nesting);
         add_test("replace_non_breaking_space", replace_non_breaking_space);
     }
 
@@ -109,6 +111,45 @@ some text
         }
     }
 
+    public void linkify_content() {
+        // XXX split these up into multiple tests
+        load_body_fixture("""
+http://example1.com
+
+<p>http://example2.com</p>
+
+<p>http://example3.com http://example4.com</p>
+
+<a href="blarg">http://example5.com</a>
+
+unknown://example6.com
+""");
+
+        string expected = """
+<a href="http://example1.com">http://example1.com</a>
+
+<p><a href="http://example2.com">http://example2.com</a></p>
+
+<p><a href="http://example3.com">http://example3.com</a> <a href="http://example4.com">http://example4.com</a></p>
+
+<a href="blarg">http://example5.com</a>
+
+unknown://example6.com
+<br><br>""";
+
+        try {
+            run_javascript("geary.linkifyContent();");
+            assert(WebKitUtil.to_string(run_javascript("geary.messageBody.innerHTML;")) ==
+                   expected);
+        } catch (Geary.JS.Error err) {
+            print("Geary.JS.Error: %s\n", err.message);
+            assert_not_reached();
+        } catch (Error err) {
+            print("WKError: %s\n", err.message);
+            assert_not_reached();
+        }
+    }
+
     public void get_html() {
         string html = "<p>para</p>";
         load_body_fixture(html);

ui/composer-web-view.js

@@ -14,12 +14,13 @@ let ComposerPageState = function() {
 };
 ComposerPageState.BODY_ID = "message-body";
 ComposerPageState.KEYWORD_SPLIT_REGEX = /[\s]+/g;
-ComposerPageState.QUOTE_START = "‘";
+ComposerPageState.QUOTE_START = "\x91";  // private use one
-ComposerPageState.QUOTE_END = "’";
+ComposerPageState.QUOTE_END = "\x92";    // private use two
-ComposerPageState.QUOTE_MARKER = "\x7f";
+ComposerPageState.QUOTE_MARKER = "\x7f"; // delete
+ComposerPageState.PROTOCOL_REGEX = /^(aim|apt|bitcoin|cvs|ed2k|ftp|file|finger|git|gtalk|http|https|irc|ircs|irc6|lastfm|ldap|ldaps|magnet|news|nntp|rsync|sftp|skype|smb|sms|svn|telnet|tftp|ssh|webcal|xmpp):/i;
 // Taken from Geary.HTML.URL_REGEX, without the inline modifier (?x)
 // at the start, which is unsupported in JS
-ComposerPageState.URL_REGEX = new RegExp("\\b((?:[a-z][\\w-]+:(?:/{1,3}|[a-z0-9%])|www\\d{0,3}[.]|[a-z0-9.\\-]+[.][a-z]{2,4}/)(?:[^\\s()<>]+|\\(([^\\s()<>]+|(\\([^\\s()<>]+\\)))*\\))+(?:\\(([^\\s()<>]+|(\\([^\\s()<>]+\\)))*\\)|[^\\s`!()\\[\\]{};:'\".,<>?«»“”‘’]))", "i");
+ComposerPageState.URL_REGEX = new RegExp("\\b((?:[a-z][\\w-]+:(?:/{1,3}|[a-z0-9%])|www\\d{0,3}[.]|[a-z0-9.\\-]+[.][a-z]{2,4}/)(?:[^\\s()<>]+|\\(([^\\s()<>]+|(\\([^\\s()<>]+\\)))*\\))+(?:\\(([^\\s()<>]+|(\\([^\\s()<>]+\\)))*\\)|[^\\s`!()\\[\\]{};:'\".,<>?«»“”‘’]))", "gi");
 
 ComposerPageState.prototype = {
     __proto__: PageState.prototype,
@@ -264,6 +265,9 @@ ComposerPageState.prototype = {
             }
         }
     },
+    linkifyContent: function() {
+        ComposerPageState.linkify(this.messageBody);
+    },
     getHtml: function() {
         return this.messageBody.innerHTML;
     },
@@ -403,6 +407,54 @@ ComposerPageState.htmlToQuotedText = function(root) {
     return ComposerPageState.replaceNonBreakingSpace(text);
 };
 
+// Linkifies "plain text" link
+ComposerPageState.linkify = function(node) {
+    if (node.nodeType == Node.TEXT_NODE) {
+        // Examine text node for something that looks like a URL
+        let input = node.nodeValue;
+        if (input != null) {
+            let output = input.replace(ComposerPageState.URL_REGEX, function(url) {
+                if (url.match(ComposerPageState.PROTOCOL_REGEX) != null) {
+                    url = "\x01" + url + "\x01";
+                }
+                return url;
+            });
+
+            if (input != output) {
+                // We got one!  Now split the text and swap in a new anchor.
+                let parent = node.parentNode;
+                let sibling = node.nextSibling;
+                for (let part of output.split("\x01")) {
+                    let newNode = null;
+                    if (part.match(ComposerPageState.URL_REGEX) != null) {
+                        newNode = document.createElement("A");
+                        newNode.href = part;
+                        newNode.innerText = part;
+                    } else {
+                        newNode = document.createTextNode(part);
+                    }
+                    parent.insertBefore(newNode, sibling);
+                }
+                parent.removeChild(node);
+            }
+        }
+    } else {
+        // Recurse
+        let child = node.firstChild;
+        while (child != null) {
+            // Save the child and get its next sibling early since if
+            // it does actually contain a URL, it will be removed from
+            // the tree
+            let target = child;
+            child = child.nextSibling;
+            // Don't attempt to linkify existing links
+            if (target.nodeName != "A") {
+                ComposerPageState.linkify(target);
+            }
+        }
+    }
+};
+
 ComposerPageState.resolveNesting = function(text, values) {
     let tokenregex = new RegExp(
         "(.?)" +