Fix HTML, CSS and JS leaking into conversation list preview. Bug 714317

When generating the preview, only the first 128 bytes of the first MIME part is fetched and used. If this part is text/html with a significant amount of embedded CSS, then there is a good chance the string passed to Geary.HTML::remove_html_tags() will be invalid, or be missing closing elements. Since that function uses regexes that require balanced tags to remove script and style blocks, then it was very possible that in these cases this method will miss removing these blocks. To solve this, remove_html_tags() is removed and its call sites are replaced by calls to Geary.HTML::html_to_text(), which has been tidyied up to produce more human-readable result. Add unit tests to cover new html_to_text functionality and its call sites. * src/engine/util/util-html.vala: Remove remove_html_tags(). Update html_to_text() to not just insert line breaks, but also insert spaces and alt text, and ignore tags like HEAD, SCRIPT and STYLE, as appropriate. Add an optional param to also allow skipping BLOCKQUOTE elements, which we don't want in the preview.
2016-12-18 23:28:53 +11:00 · 2016-12-18 23:28:53 +11:00 · deb0c415d0
commit deb0c415d0
parent 330d263947
7 changed files with 780 additions and 102 deletions
--- a/src/engine/rfc822/rfc822-message-data.vala
+++ b/src/engine/rfc822/rfc822-message-data.vala
@ -390,14 +390,14 @@ public class Geary.RFC822.PreviewText : Geary.RFC822.Text {
        input_stream.write_to_stream(filter);
        uint8[] data = output.data;
        data += (uint8) '\0';
-        
+
        // Fix the preview up by removing HTML tags, redundant white space, common types of
        // message armor, text-based quotes, and various MIME fields.
        string preview_text = "";
-        string original_text = is_html ? Geary.HTML.remove_html_tags((string) data) : (string) data;
+        string original_text = is_html ? Geary.HTML.html_to_text((string) data, false) : (string) data;
        string[] all_lines = original_text.split("\r\n");
        bool in_header = false; // True after a header
-        
+
        foreach(string line in all_lines) {
            if (in_header && line.has_prefix(" ") || line.has_prefix("\t")) {
                continue; // Skip "folded" (multi-line) headers.
--- a/src/engine/rfc822/rfc822-message.vala
+++ b/src/engine/rfc822/rfc822-message.vala
@ -384,12 +384,12 @@ public class Geary.RFC822.Message : BaseObject {
            preview = get_plain_body(false, null);
        } catch (Error e) {
            try {
-                preview = Geary.HTML.remove_html_tags(get_html_body(null));
+                preview = Geary.HTML.html_to_text(get_html_body(null), false);
            } catch (Error error) {
                debug("Could not generate message preview: %s\n and: %s", e.message, error.message);
            }
        }
-        
+
        return Geary.String.safe_byte_substring((preview ?? "").chug(),
            Geary.Email.MAX_PREVIEW_BYTES);
    }
--- a/src/engine/util/util-html.vala
+++ b/src/engine/util/util-html.vala
@ -7,7 +7,10 @@
 namespace Geary.HTML {

 private int init_count = 0;
-private Gee.HashSet<string>? breaking_elements = null;
+private Gee.HashSet<string>? breaking_elements;
+private Gee.HashSet<string>? spacing_elements;
+private Gee.HashSet<string>? alt_text_elements;
+private Gee.HashSet<string>? ignored_elements;

 /**
 * Must be called before ''any'' call to the HTML namespace.
@ -18,58 +21,74 @@ private Gee.HashSet<string>? breaking_elements = null;
 public void init() {
    if (init_count++ != 0)
        return;
-    
-    init_breaking_elements();
+
+    init_element_sets();
 }

-private void init_breaking_elements() {
-    // Organized from <https://en.wikipedia.org/wiki/HTML_element>.  This is a
-    // list of block elements and some others that get special treatment.
-    // NOTE: this SHOULD be a const list, but due to
-    // <https://bugzilla.gnome.org/show_bug.cgi?id=646970>, it can't be.
-    string[] elements = {
+private void init_element_sets() {
+    // Organized from <https://en.wikipedia.org/wiki/HTML_element>,
+    // <https://html.spec.whatwg.org/multipage> and some custom
+    // inference.
+
+    // Block elements and some others that cause new lines to be
+    // inserted when converting to text. Not all block elements are
+    // included since some (e.g. lists) will have nested breaking
+    // children.
+    breaking_elements = new Gee.HashSet<string>(Ascii.stri_hash, Ascii.stri_equal);
+    breaking_elements.add_all_array({
        "address",
        "blockquote",
        "br", // [1]
        "caption", // [2]
        "center",
-        "dd",
-        "del", // [3]
-        "dir",
        "div",
-        "dl",
        "dt",
        "embed",
+        "form",
        "h1", "h2", "h3", "h4", "h5", "h6",
        "hr",
-        "img", // [1]
-        "ins", // [3]
+        "iframe", // [1]
        "li",
        "map", // [1]
        "menu",
        "noscript", // [2]
        "object", // [1]
-        "ol",
        "p",
        "pre",
-        "script", // [2]
-        "table",
-        "tbody",
-        "td",
-        "tfoot",
-        "th",
-        "thead",
        "tr",
-        "ul",
-        
+
        // [1]: Not block elements, but still break up the text
        // [2]: Some of these are oddities, but I figure they should break flow
-        // [3]: Can be used as either block or inline; we go for broke
-    };
-    
-    breaking_elements = new Gee.HashSet<string>(Ascii.stri_hash, Ascii.stri_equal);
-    foreach (string element in elements)
-        breaking_elements.add(element);
+    });
+
+    // Elements that cause spaces to be inserted afterwards when
+    // converting to text.
+    spacing_elements = new Gee.HashSet<string>(Ascii.stri_hash, Ascii.stri_equal);
+    spacing_elements.add_all_array({
+        "dt",
+        "dd",
+        "img",
+        "td",
+        "th",
+    });
+
+    // Elements that may have alt text
+    alt_text_elements = new Gee.HashSet<string>(Ascii.stri_hash, Ascii.stri_equal);
+    alt_text_elements.add_all_array({
+        "img",
+    });
+
+    // Elements that should not be included when converting to text
+    ignored_elements = new Gee.HashSet<string>(Ascii.stri_hash, Ascii.stri_equal);
+    ignored_elements.add_all_array({
+        "base",
+        "link",
+        "meta",
+        "head",
+        "script",
+        "style",
+        "template",
+    });
 }

 public inline string escape_markup(string? plain) {
@ -88,89 +107,61 @@ public string preserve_whitespace(string? text) {
    return output;
 }

-// Removes any text between < and >.  Additionally, if input terminates in the middle of a tag, 
-// the tag will be removed.
-// If the HTML is invalid, the original string will be returned.
-public string remove_html_tags(string input) {
-    try {
-        string output = input;
-        
-        // Count the number of < and > characters.
-        unichar c;
-        uint64 less_than = 0;
-        uint64 greater_than = 0;
-        for (int i = 0; output.get_next_char (ref i, out c);) {
-            if (c == '<')
-                less_than++;
-            else if (c == '>')
-                greater_than++;
-        }
-        
-        if (less_than == greater_than + 1) {
-            output += ">"; // Append an extra > so our regex works.
-            greater_than++;
-        }
-        
-        if (less_than != greater_than)
-            return input; // Invalid HTML.
-        
-        // Removes script tags and everything between them.
-        // Based on regex here: http://stackoverflow.com/questions/116403/im-looking-for-a-regular-expression-to-remove-a-given-xhtml-tag-from-a-string
-        Regex script = new Regex("<script[^>]*?>[\\s\\S]*?<\\/script>", RegexCompileFlags.CASELESS);
-        output = script.replace(output, -1, 0, "");
-        
-        // Removes style tags and everything between them.
-        // Based on regex above.
-        Regex style = new Regex("<style[^>]*?>[\\s\\S]*?<\\/style>", RegexCompileFlags.CASELESS);
-        output = style.replace(output, -1, 0, "");
-        
-        // Removes remaining tags.
-        Regex tags = new Regex("<[^>]*>", RegexCompileFlags.CASELESS);
-        return tags.replace(output, -1, 0, "");
-    } catch (Error e) {
-        debug("Error stripping HTML tags: %s", e.message);
-    }
-    
-    return input;
-}
-
 /**
 * Does a very approximate conversion from HTML to text.
 *
- * This does more than stripping tags -- it inserts line breaks where appropriate, decodes
- * entities, etc.  The layout of the text is largely lost.  This is primarily
- * useful for pulling out tokens for searching, not for presenting to the user.
+ * This does more than stripping tags -- it inserts line breaks where
+ * appropriate, decodes entities, etc. Note the full string is parsed
+ * by libxml's HTML parser to create a DOM-like tree representation,
+ * which is then walked, so this function can be somewhat
+ * computationally expensive.
 */
-public string html_to_text(string html, string encoding = Geary.RFC822.UTF8_CHARSET) {
+public string html_to_text(string html,
+                           bool include_blockquotes = true,
+                           string encoding = Geary.RFC822.UTF8_CHARSET) {
    Html.Doc *doc = Html.Doc.read_doc(html, "", encoding, Html.ParserOption.RECOVER |
        Html.ParserOption.NOERROR | Html.ParserOption.NOWARNING | Html.ParserOption.NOBLANKS |
        Html.ParserOption.NONET | Html.ParserOption.COMPACT);
-    
+
    StringBuilder text = new StringBuilder();
    if (doc != null) {
-        recurse_html_nodes_for_text(doc->get_root_element(), text);
+        recurse_html_nodes_for_text(doc->get_root_element(), include_blockquotes, text);
        delete doc;
    }
-    
+
    return text.str;
 }

-private void recurse_html_nodes_for_text(Xml.Node? node, StringBuilder text) {
-    // TODO: add alt text for things that have it?
-    
+private void recurse_html_nodes_for_text(Xml.Node? node,
+                                         bool include_blockquotes,
+                                         StringBuilder text) {
    for (unowned Xml.Node? n = node; n != null; n = n.next) {
-        if (n.type == Xml.ElementType.TEXT_NODE)
+        if (n.type == Xml.ElementType.TEXT_NODE) {
            text.append(n.content);
-        else if (n.type == Xml.ElementType.ELEMENT_NODE && element_needs_break(n.name))
-            text.append("\n");
-        
-        recurse_html_nodes_for_text(n.children, text);
+        } else if (n.type == Xml.ElementType.ELEMENT_NODE) {
+            string name = n.name;
+            if (include_blockquotes || name != "blockquote") {
+                if (name in alt_text_elements) {
+                    string? alt_text = node.get_prop("alt");
+                    if (alt_text != null) {
+                        text.append(alt_text);
+                    }
+                }
+
+                if (!(name in ignored_elements)) {
+                    recurse_html_nodes_for_text(n.children, include_blockquotes, text);
+                }
+
+                if (name in spacing_elements) {
+                    text.append(" ");
+                }
+
+                if (name in breaking_elements) {
+                    text.append("\n");
+                }
+            }
+        }
    }
 }

-// Determines if the named element should break the flow of text.
-private bool element_needs_break(string element) {
-    return breaking_elements.contains(element);
-}
-
 }
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@ -7,6 +7,8 @@ set(TEST_SRC
  testcase.vala # Taken as-is from libgee, courtesy Julien Peeters

  engine/rfc822-mailbox-address-test.vala
+  engine/rfc822-message-data-test.vala
+  engine/util-html-test.vala
 )

 # Vala
--- a/test/engine/rfc822-message-data-test.vala
+++ b/test/engine/rfc822-message-data-test.vala
@ -0,0 +1,602 @@
+/*
+ * Copyright 2016 Michael Gratton <mike@vee.net>
+ *
+ * This software is licensed under the GNU Lesser General Public License
+ * (version 2.1 or later). See the COPYING file in this distribution.
+ */
+
+class Geary.RFC822.MessageDataTest : Gee.TestCase {
+
+    public MessageDataTest() {
+        base("Geary.RFC822.MessageDataTest");
+        add_test("PreviewText.with_header", preview_text_with_header);
+    }
+
+    public void preview_text_with_header() {
+        string part_headers = "Content-Type: text/html; charset=utf-8\r\nContent-Transfer-Encoding: quoted-printable\r\n\r\n";
+
+        PreviewText preview1 = new PreviewText.with_header(
+            new Geary.Memory.StringBuffer(HTML_BODY1_ENCODED),
+            new Geary.Memory.StringBuffer(part_headers)
+        );
+        assert(preview1.buffer.to_string() == HTML_BODY1_EXPECTED);
+
+        PreviewText preview2 = new PreviewText.with_header(
+            new Geary.Memory.StringBuffer(HTML_BODY2_ENCODED),
+            new Geary.Memory.StringBuffer(part_headers)
+        );
+        assert(preview2.buffer.to_string() == HTML_BODY2_EXPECTED);
+    }
+
+    public static string HTML_BODY1_ENCODED = """<html><head>
+<meta http-equiv=3DContent-Type content=3D"text/html; charset=3Dutf-8">
+<style>
+.bodyblack { font-family: Verdana, Arial, Helvetica, sans-serif; font-size:=
+ 12px; }
+td { font-size: 12px; }
+.footer { font-family: Verdana, Arial, Helvetica, sans-serif; font-size: 10=
+px; }
+</style>
+</head>
+<body><table cellSpacing=3D"0" cellPadding=3D"0" width=3D"450" border=3D"0"=
+ class=3D"bodyblack"><tr><td>
+<p><br />Hi Kenneth, <br /> <br /> We xxxxx xxxx xx xxx xxx xx xxxx x xxxxx=
+xxx xxxxxxxx.=C2=A0<br /> <br /> <br /> <br />Thank you, <br /> <br />XXXXX=
+X XXXXXX<br /><br />You can reply directly to this message or click the fol=
+lowing link:<br /><a href=3D"https://app.foobar.com/xxxxxxxx752a0ab01641966=
+deff6c48623aba">https://app.foobar.com/xxxxxxxxxxxxxxxx1641966deff6c48623ab=
+a</a><br /><br />You can change your email preferences at:<br /><a href=3D"=
+https://app.foobar.com/xxxxxxxxxxxxx">https://app.foobar.com/xxxxxxxxxxx</a=
+></p></td></tr>
+</table></body></html>""";
+
+    public static string HTML_BODY1_EXPECTED = "Hi Kenneth, We xxxxx xxxx xx xxx xxx xx xxxx x xxxxxxxx xxxxxxxx.  Thank you, XXXXXX XXXXXX You can reply directly to this message or click the following link: https://app.foobar.com/xxxxxxxxxxxxxxxx1641966deff6c48623aba You can change your email preferences at: https://app.foobar.com/xxxxxxxxxxx";
+
+    public static string HTML_BODY2_ENCODED = """<!DOCTYPE html>
+<!--2c2a1c66-0638-7c87-5057-bff8be4291eb_v180-->
+<html>
+  <head>
+    <meta http-equiv=3D"Content-Type" content=3D"text/html; charset=3Dutf-8=
+"></meta><style type=3D"text/css">
+@media only screen and (max-width: 620px) {
+body[yahoo] .device-width {
+width: 450px !important
+}
+body[yahoo] .center {
+text-align: center !important
+}
+}
+@media only screen and (max-width: 479px) {
+body[yahoo] .device-width {
+width: 300px !important;
+padding: 0
+}
+body[yahoo] .mobile-full-width {
+width: 300px !important
+}
+}
+body[yahoo] .mobile-full-width {
+min-width: 103px;
+max-width: 300px;
+height: 38px;
+}
+body[yahoo] .mobile-full-width a {
+display: block;
+padding: 10px 0;
+}
+body[yahoo] .mobile-full-width td{
+padding: 0px !important
+}
+body { width: 100% !important; -webkit-text-size-adjust: 100% !important; -=
+ms-text-size-adjust: 100% !important; -webkit-font-smoothing: antialiased !=
+important; margin: 0 !important; padding: 0 0 100px !important; font-family=
+: Helvetica, Arial, sans-serif !important; background-color:#f9f9f9}
+.ReadMsgBody { width: 100% !important; background-color: #ffffff !important=
+; }
+.ExternalClass { width: 100% !important; }
+.ExternalClass { line-height: 100% !important; }
+img { display: block !important; outline: none !important; text-decoration:=
+ none !important; -ms-interpolation-mode: bicubic !important; }
+td{word-wrap: break-word;}
+.blueLinks a {
+color: #0654ba !important;
+text-decoration: none !important;
+}
+.whiteLinks a {
+color: #ffffff !important;
+text-decoration: none !important;
+font-weight: bold !important;
+}
+.wrapper {
+width: 100%;
+table-layout: fixed;
+-webkit-text-size-adjust: 100%;
+-ms-text-size-adjust: 100%;
+}
+.webkit {
+max-width: 100%;
+margin: 0 auto;
+}
+</style> <!--[if gte mso 9]>
+<style>td.product-details-block{word-break:break-all}.threeColumns{width:14=
+0px !important}.threeColumnsTd{padding:10px 20px !important}.fourColumns{wi=
+dth:158px !important}.fourColumnsPad{padding: 0 18px 0 0 !important}.fourCo=
+lumnsTd{padding:10px 0px !important}.twoColumnSixty{width:360px !important}=
+table{mso-table-lspace:0pt; mso-table-rspace:0pt;}</style>
+<![endif]-->
+<style type=3D"text/css">
+@media only screen and (max-width: 2000px) {
+*[class=3Dcta-block] {
+padding: 24px 0 24px 0px !important;
+}
+*[class=3Dcta-block-2] {
+padding: 24px 0 8px 0px !important;
+}
+*[class=3Dcta-block-3] {
+padding: 8px 0 24px 0px !important;
+}
+}
+@media only screen and (max-width: 620px) {
+*[class=3Dcta-block] {
+padding: 24px 0 24px 0px !important;
+}
+*[class=3Dcta-block-2] {
+padding: 24px 0 8px 0px !important;
+}
+*[class=3Dcta-block-3] {
+padding: 8px 0 24px 0px !important;
+}
+}
+@media screen and (max-width:480px) {
+*[class=3Dcta-block] {
+padding: 24px 0 24px !important;
+}
+*[class=3Dcta-block-2] {
+padding: 24px 0 8px !important;
+}
+*[class=3Dcta-block-3] {
+padding: 8px 0 24px !important;
+}
+*[class=3Dmobile-ebayLogo] {
+padding: 8px 0 8px !important;
+}
+*[class=3Dmobile-multi-item-left-image] {
+padding: 8px 15px 8px 0 !important;
+}
+*[class=3Dmobile-multi-item-right-image] {
+padding: 8px 0 8px 15px !important;
+}
+*[class=3Dmobile-dealmaker-headline] {
+font-size: 20px !important;
+line-height: 23px !important;
+}
+td.mobile-dealmaker-CTA1 {
+width: 303px !important;
+}
+}
+</style>
+  </head>
+  <body yahoo=3D"fix"> <center class=3D"wrapper" style=3D"background-color:=
+ #f9f9f9">
+        <div class=3D"webkit" style=3D"background-color: #f9f9f9"> <table i=
+d=3D"area2Container" width=3D"100%" border=3D"0" cellpadding=3D"0" cellspac=
+ing=3D"0" align=3D"center" style=3D"border-collapse: collapse !important; b=
+order-spacing: 0 !important; border: none; background-color:#f9f9f9">
+<tr>
+<td width=3D"100%" valign=3D"top" style=3D"border-collapse: collapse !impor=
+tant; border-spacing: 0 !important; border: none;">
+<table class=3D"device-width" style=3D"border-collapse: collapse !important=
+; border-spacing: 0 !important; border: none;" align=3D"center" bgcolor=3D"=
+#f9f9f9" border=3D"0" cellpadding=3D"0" cellspacing=3D"0" width=3D"600">
+<tbody>
+<tr>
+<td height=3D"1" valign=3D"top" style=3D"border-collapse: collapse !importa=
+nt; border-spacing: 0 !important; padding: 0; border: none; font-size: 1px;=
+ line-height: 1px; color: #f9f9f9">
+Buy It Now from US $1,750.00 to US $5,950.00.
+</td>
+</tr>
+</tbody>
+</table>
+</td>
+</tr>
+</table> <table id=3D"area4Container" width=3D"100%" border=3D"0" cellpaddi=
+ng=3D"0" cellspacing=3D"0" align=3D"center" style=3D"border-collapse: colla=
+pse !important; border-spacing: 0 !important; border: none; background-colo=
+r:#f9f9f9">
+<tr>
+<td width=3D"100%" valign=3D"top" style=3D"border-collapse: collapse !impor=
+tant; border-spacing: 0 !important; border: none;">
+<table width=3D"600" class=3D"device-width" border=3D"0" cellpadding=3D"0" =
+cellspacing=3D"0" align=3D"center" style=3D"border-collapse: collapse !impo=
+rtant; border-spacing: 0 !important; border: none;">
+<tr>
+<td class=3D"mobile-ebayLogo" valign=3D"top" style=3D"border-collapse: coll=
+apse !important; border-spacing: 0 !important; padding: 16px 0 16px; border=
+: none;"><a href=3D"http://rover.ebay.com/rover/0/e11021.m1831.l3127/7?euid=
+=3Dd9f42b5e860b4eabb98195c2888cba9e&bu=3D43210693952&loc=3Dhttp%3A%2F%2Fwww=
+.ebay.com.au%2Fulk%2Fstart%2Fshop&exe=3D15083&ext=3D38992&sojTags=3Dexe=3De=
+xe,ext=3Dext,bu=3Dbu" style=3D"text-decoration: none; color: #0654ba;"><img=
+ src=3D"http://p.ebaystatic.com/aw/email/eBayLogo.png" width=3D"133" border=
+=3D"0" alt=3D"eBay" align=3D"left" style=3D"display: inline block; outline:=
+ none; text-decoration: none; -ms-interpolation-mode: bicubic; border: none=
+;" /></a><img src=3D"http://rover.ebay.com/roveropen/0/e11021/7?euid=3Dd9f4=
+2b5e860b4eabb98195c2888cba9e&bu=3D43210693952&exe=3D15083&ext=3D38992&sojTa=
+gs=3Dexe=3Dexe,ext=3Dext,bu=3Dbu" alt=3D"" style=3D"border:0; height:1;"/><=
+/td>
+</tr>
+</table>
+</td>
+</tr>
+</table>   <table id=3D"area4Container" width=3D"100%" border=3D"0" cellpad=
+ding=3D"0" cellspacing=3D"0" align=3D"center" style=3D"border-collapse: col=
+lapse !important; border-spacing: 0 !important; border: none; background-co=
+lor:#f9f9f9">
+<tr>
+<td width=3D"100%" valign=3D"top" style=3D"border-collapse: collapse !impor=
+tant; border-spacing: 0 !important; border: none;">
+<table width=3D"600" cellspacing=3D"0" cellpadding=3D"0" border=3D"0" bgcol=
+or=3D"#f9f9f9" align=3D"center" style=3D"border-collapse: collapse !importa=
+nt; border-spacing: 0 !important; border: none;" class=3D"device-width">
+<tbody>
+<tr>
+<td valign=3D"top" style=3D"border-collapse: collapse !important; border-sp=
+acing: 0 !important; padding: 0;">
+<h1 align=3D"left" class=3D"mobile-dealmaker-headline" style=3D"font-family=
+: Helvetica, Arial, sans-serif; font-weight: 200; line-height: 29px; color:=
+ #333333; text-align: left; font-size: 24px; margin: 0;">
+Daccordi, Worldwide: <a href=3D'http://rover.ebay.com/rover/0/e11021.m3197.=
+l1150/7?euid=3Dd9f42b5e860b4eabb98195c2888cba9e&bu=3D43210693952&loc=3Dhttp=
+%3A%2F%2Fwww.ebay.com.au%2Fsch%2FCycling-%2F7294%2Fi.html%3FLH_PrefLoc%3D2%=
+26_sop%3D10%26_fln%3D1%26_nkw%3DDaccordi%26_trksid%3Dm194%26ssPageName%3DST=
+RK%253AMEFSRCHX%253ASRCH&exe=3D15083&ext=3D38992&sojTags=3Dexe=3Dexe,ext=3D=
+ext,bu=3Dbu' style=3D'text-decoration:none'>2 new</a> matches today
+</h1>
+</td>
+</tr>
+</tbody>
+</table>
+</td>
+</tr>
+</table>  <table width=3D"100%" border=3D"0" cellpadding=3D"0" cellspacing=
+=3D"0" align=3D"center" style=3D"border-collapse: collapse !important; bord=
+er-spacing: 0 !important; border: none;">
+<tr>
+<td width=3D"100%" valign=3D"top" bgcolor=3D"#f9f9f9" style=3D"border-colla=
+pse: collapse !important; border-spacing: 0 !important; border: none;">
+<table width=3D"600" border=3D"0" align=3D"center" cellspacing=3D"0" cellpa=
+dding=3D"0" style=3D"border-collapse: collapse !important; border-spacing: =
+0 !important; border: none;" class=3D"device-width">
+<tbody>
+<tr>
+<td valign=3D"top" style=3D"border-collapse: collapse !important; border-sp=
+acing: 0 !important; border: none; padding: 0; margin: 0;">
+<div align=3D"left" border=3D"0" cellspacing=3D"0" cellpadding=3D"0" width=
+=3D"146" style=3D"border-collapse: separate !important; border-spacing: 0 !=
+important; border: none; float:left; display:inline;">
+<table width=3D"146" border=3D"0" align=3D"left" cellspacing=3D"0" cellpadd=
+ing=3D"0" style=3D"border-collapse: collapse !important; border-spacing: 0 =
+!important; border: none; color: #333333">
+<tr>
+<td class=3D"mobile-multi-item-left-image" valign=3D"top" style=3D"border-c=
+ollapse: collapse !important; border-spacing: 0 !important; padding: 12px 1=
+2px 12px 0; border: none;">
+<table width=3D"100%" border=3D"0" cellspacing=3D"0" cellpadding=3D"0" styl=
+e=3D"border-collapse: collapse !important; border-spacing: 0 !important; bo=
+rder: none;">
+<tr>
+<td style=3D"border-collapse: collapse !important; border-spacing: 0 !impor=
+tant; border: none; padding: 0; margin: 0;">
+<table width=3D"132" height=3D"132" cellspacing=3D"0" cellpadding=3D"0" sty=
+le=3D"border-collapse: collapse !important; border-spacing: 0 !important; p=
+adding: 0; border: none;">
+<tbody>
+<tr>
+<td width=3D"132" valign=3D"center" height=3D"132" align=3D"center" style=
+=3D"max-width: 132px; border: 1px solid #dddddd;">
+<a href=3D"http://rover.ebay.com/rover/0/e11021.m43.l1120/7?euid=3Dd9f42b5e=
+860b4eabb98195c2888cba9e&bu=3D43210693952&loc=3Dhttp%3A%2F%2Fwww.ebay.com.a=
+u%2Fulk%2Fitm%2F391655221238&exe=3D15083&ext=3D38992&sojTags=3Dexe=3Dexe,ex=
+t=3Dext,bu=3Dbu">
+<span style=3D"display: block; outline: none; text-decoration: none; -ms-in=
+terpolation-mode: bicubic; border-radius: 3px; margin: 0; ">
+<img border=3D"0" src=3D"http://i.ebayimg.com/images/g/dxcAAOSwJ7RYVbhB/s-b=
+132x132.jpg" style=3D"max-width:100%; display: block; outline: none; text-d=
+ecoration: none; -ms-interpolation-mode: bicubic; margin: 0; border: none;"=
+ />
+</span>
+</a>
+</td>
+</tr>
+</tbody>
+</table>
+</td>
+</tr>
+<tr>
+<td valign=3D"top" style=3D"max-width: 132px; border-collapse: collapse !im=
+portant; border-spacing: 0 !important; padding: 12px 0 0; border: none;">
+<h3 align=3D"left" style=3D"font-family: Helvetica, Arial, sans-serif; font=
+-weight: normal; line-height: normal; color: #333333; text-align: left; fon=
+t-size: 12px; margin: 0 0 10px; word-break:break-all; height:31px;">
+<a style=3D"text-decoration: none; color: #0654ba;" href=3D"http://rover.eb=
+ay.com/rover/0/e11021.m43.l3160/7?euid=3Dd9f42b5e860b4eabb98195c2888cba9e&b=
+u=3D43210693952&loc=3Dhttp%3A%2F%2Fwww.ebay.com.au%2Fulk%2Fitm%2F3916552212=
+38&exe=3D15083&ext=3D38992&sojTags=3Dexe=3Dexe,ext=3Dext,bu=3Dbu">
+Daccordi 50th anniversary edition with...
+</a>
+</h3>
+</td>
+</tr>
+<tr>
+<td align=3D"left" style=3D"border-collapse: collapse !important; border-sp=
+acing: 0 !important; font-family: Helvetica, Arial, sans-serif; text-align:=
+ left; font-size: 12px; font-weight: bold; border: none; padding-bottom: 8p=
+x;">
+Buy it now: US $5,950.00
+</td>
+</tr>
+<tr>
+<td align=3D"left" style=3D"border-collapse: collapse !important; border-sp=
+acing: 0 !important; font-family: Helvetica, Arial, sans-serif; text-align:=
+ left; font-size: 12px; color: #E53238; font-weight: normal; border: none; =
+padding: 0; margin: 0;">
+100% positive feedback
+</td>
+</tr>
+</table>
+</td>
+</tr>
+</table>
+<table width=3D"146" border=3D"0" align=3D"left" cellspacing=3D"0" cellpadd=
+ing=3D"0" style=3D"border-collapse: collapse !important; border-spacing: 0 =
+!important; border: none; color: #333333">
+<tr>
+<td class=3D"mobile-multi-item-right-image" valign=3D"top" style=3D"border-=
+collapse: collapse !important; border-spacing: 0 !important; padding: 12px =
+12px 12px 0; border: none;">
+<table width=3D"100%" border=3D"0" cellspacing=3D"0" cellpadding=3D"0" styl=
+e=3D"border-collapse: collapse !important; border-spacing: 0 !important; bo=
+rder: none;">
+<tr>
+<td style=3D"border-collapse: collapse !important; border-spacing: 0 !impor=
+tant; border: none; padding: 0; margin: 0;">
+<table width=3D"132" height=3D"132" cellspacing=3D"0" cellpadding=3D"0" sty=
+le=3D"border-collapse: collapse !important; border-spacing: 0 !important; p=
+adding: 0; border: none;">
+<tbody>
+<tr>
+<td width=3D"132" valign=3D"center" height=3D"132" align=3D"center" style=
+=3D"max-width: 132px; border: 1px solid #dddddd;">
+<a href=3D"http://rover.ebay.com/rover/0/e11021.m43.l1120/7?euid=3Dd9f42b5e=
+860b4eabb98195c2888cba9e&bu=3D43210693952&loc=3Dhttp%3A%2F%2Fwww.ebay.com.a=
+u%2Fulk%2Fitm%2F132037720927&exe=3D15083&ext=3D38992&sojTags=3Dexe=3Dexe,ex=
+t=3Dext,bu=3Dbu">
+<span style=3D"display: block; outline: none; text-decoration: none; -ms-in=
+terpolation-mode: bicubic; border-radius: 3px; margin: 0; ">
+<img border=3D"0" src=3D"http://i.ebayimg.com/images/g/C3cAAOSwj85YOiHQ/s-b=
+132x132.jpg" style=3D"max-width:100%; display: block; outline: none; text-d=
+ecoration: none; -ms-interpolation-mode: bicubic; margin: 0; border: none;"=
+ />
+</span>
+</a>
+</td>
+</tr>
+</tbody>
+</table>
+</td>
+</tr>
+<tr>
+<td valign=3D"top" style=3D"max-width: 132px; border-collapse: collapse !im=
+portant; border-spacing: 0 !important; padding: 12px 0 0; border: none;">
+<h3 align=3D"left" style=3D"font-family: Helvetica, Arial, sans-serif; font=
+-weight: normal; line-height: normal; color: #333333; text-align: left; fon=
+t-size: 12px; margin: 0 0 10px; word-break:break-all; height:31px;">
+<a style=3D"text-decoration: none; color: #0654ba;" href=3D"http://rover.eb=
+ay.com/rover/0/e11021.m43.l3160/7?euid=3Dd9f42b5e860b4eabb98195c2888cba9e&b=
+u=3D43210693952&loc=3Dhttp%3A%2F%2Fwww.ebay.com.au%2Fulk%2Fitm%2F1320377209=
+27&exe=3D15083&ext=3D38992&sojTags=3Dexe=3Dexe,ext=3Dext,bu=3Dbu">
+Daccordi Griffe Campagnolo Croce D'Aune...
+</a>
+</h3>
+</td>
+</tr>
+<tr>
+<td align=3D"left" style=3D"border-collapse: collapse !important; border-sp=
+acing: 0 !important; font-family: Helvetica, Arial, sans-serif; text-align:=
+ left; font-size: 12px; font-weight: bold; border: none; padding-bottom: 8p=
+x;">
+Buy it now: US $1,750.00
+</td>
+</tr>
+<tr>
+<td align=3D"left" style=3D"border-collapse: collapse !important; border-sp=
+acing: 0 !important; font-family: Helvetica, Arial, sans-serif; text-align:=
+ left; font-size: 12px; color: #E53238; font-weight: normal; border: none; =
+padding: 0; margin: 0;">
+100% positive feedback
+</td>
+</tr>
+</table>
+</td>
+</tr>
+</table>
+</div>
+</td>
+</tr>
+</tbody>
+</table>
+</td>
+</tr>
+</table> <table id=3D"area5Container" width=3D"100%" border=3D"0" cellpaddi=
+ng=3D"0" cellspacing=3D"0" align=3D"center" style=3D"border-collapse: colla=
+pse !important; border-spacing: 0 !important; border: none; background-colo=
+r:#f9f9f9">
+<tr>
+<td>
+<table width=3D"600" class=3D"device-width" border=3D"0" cellpadding=3D"0" =
+cellspacing=3D"0" align=3D"center" bgcolor=3D"#f9f9f9" style=3D"border-coll=
+apse: collapse !important; border-spacing: 0 !important; border: none;">
+<tr>
+<td valign=3D"top" class=3D"cta-block-2" style=3D"border-collapse: collapse=
+ !important; border-spacing: 0 !important; border: none;">
+<table align=3D"left" cellpadding=3D"0" cellspacing=3D"0" border=3D"0" styl=
+e=3D"border-collapse: collapse !important; border-spacing: 0 !important; bo=
+rder: none; padding: 10px 0">
+<tr><td>
+<table align=3D"left" cellpadding=3D"0" cellspacing=3D"0" border=3D"0" clas=
+s=3D"mobile-full-width" style=3D"max-width: 320px; border-collapse: collaps=
+e !important; border-spacing: 0 !important;">
+<tr>
+<td width=3D"292" valign=3D"top" class=3D"center mobile-dealmaker-CTA1" ali=
+gn=3D"center" bgcolor=3D"#0654BA" style=3D"min-width: 290px;border-collapse=
+: collapse !important; border-spacing: 0 !important; font-size: 16px; line-=
+height: normal;background-color: 0654BA; padding: 11px 17px; ">
+<a href=3D"http://rover.ebay.com/rover/0/e11021.m4442.l1150/7?euid=3Dd9f42b=
+5e860b4eabb98195c2888cba9e&bu=3D43210693952&loc=3Dhttp%3A%2F%2Fwww.ebay.com=
+.au%2Fsch%2FCycling-%2F7294%2Fi.html%3FLH_PrefLoc%3D2%26_sop%3D10%26_fln%3D=
+1%26_nkw%3DDaccordi%26_trksid%3Dm194%26ssPageName%3DSTRK%253AMEFSRCHX%253AS=
+RCH&exe=3D15083&ext=3D38992&sojTags=3Dexe=3Dexe,ext=3Dext,bu=3Dbu" style=3D=
+"text-decoration: none; color: #ffffff; font-size: 16px; line-height: 18px;=
+ font-weight: 200; font-family: Helvetica, Arial, sans-serif; padding: 11px=
+ 17px;"> View all results</a>
+</td>
+</tr>
+</table>
+</td>
+<td style=3D"border-collapse: collapse !important;
+border-spacing: 0; !important; padding: 0"><img class=3D"collapse" src=3D"h=
+ttp://p.ebaystatic.com/aw/email/Welcome_Day_0/spacer.gif" width=3D"5" heigh=
+t=3D"1" alt=3D"" border=3D"0" style=3D"display:block; width: 5px !important=
+"></td>
+</tr>
+<![if ! gte mso 9]>
+<tr>
+<td style=3D"border-collapse: collapse !important;
+border-spacing: 0; !important; padding: 0"><img src=3D"http://p.ebaystatic.=
+com/aw/email/Welcome_Day_0/spacer.gif" width=3D"1" height=3D"5" alt=3D"" bo=
+rder=3D"0" style=3D"display:block; height: 5px !important"></td>
+</tr>
+<![endif]-->
+</table>
+<table align=3D"left" cellpadding=3D"0" cellspacing=3D"0" border=3D"0" styl=
+e=3D"border-collapse: collapse !important; border-spacing: 0 !important; bo=
+rder: none; padding: 10px 0">
+<tr><td>
+<table align=3D"left" cellpadding=3D"0" cellspacing=3D"0" border=3D"0" clas=
+s=3D"mobile-full-width" style=3D"max-width: 320px; border-collapse: collaps=
+e !important; border-spacing: 0 !important; border: 1px solid #dddddd;borde=
+r-radius: 3px;">
+<tr>
+<td width=3D"290" valign=3D"top" class=3D"center mobile-dealmaker-CTA1" ali=
+gn=3D"center" bgcolor=3D"#ffffff" style=3D"min-width: 290px; border-collaps=
+e: collapse !important; border-spacing: 0 !important; font-size: 16px; line=
+-height: normal;background-color: ffffff; padding: 10px 17px; ">
+<a href=3D"http://rover.ebay.com/rover/0/e11021.m4442.l1179/7?euid=3Dd9f42b=
+5e860b4eabb98195c2888cba9e&bu=3D43210693952&loc=3Dhttp%3A%2F%2Fwww.ebay.com=
+.au%2Fsch%2FCycling-%2F7294%2Fi.html%3FLH_PrefLoc%3D2%26_sop%3D10%26_fln%3D=
+1%26_nkw%3DDaccordi%26_trksid%3Dm194%26ssPageName%3DSTRK%253AMEFSRCHX%253AS=
+RCH%26replaceid%3D19105329025&exe=3D15083&ext=3D38992&sojTags=3Dexe=3Dexe,e=
+xt=3Dext,bu=3Dbu" style=3D"text-decoration: none; color: #0654BA; font-size=
+: 16px; line-height: 18px; font-weight: 200; font-family: Helvetica, Arial,=
+ sans-serif; padding: 10px 17px;">Refine this search</a>
+</td>
+</tr>
+</table>
+</td>
+<td style=3D"border-collapse: collapse !important;
+border-spacing: 0; !important; padding: 0"><img class=3D"collapse" src=3D"h=
+ttp://p.ebaystatic.com/aw/email/Welcome_Day_0/spacer.gif" width=3D"5" heigh=
+t=3D"1" alt=3D"" border=3D"0" style=3D"display:block; width: 5px !important=
+"></td>
+</tr>
+<![if ! gte mso 9]>
+<tr>
+<td style=3D"border-collapse: collapse !important;
+border-spacing: 0; !important; padding: 0"><img src=3D"http://p.ebaystatic.=
+com/aw/email/Welcome_Day_0/spacer.gif" width=3D"1" height=3D"5" alt=3D"" bo=
+rder=3D"0" style=3D"display:block; height: 5px !important"></td>
+</tr>
+<![endif]-->
+</table>
+</td>
+</tr>
+<tr>
+<td valign=3D"top" class=3D"cta-block-3" style=3D"border-collapse: collapse=
+ !important; border-spacing: 0 !important; padding: 0 0 8px 0px; border: no=
+ne;">
+<table width=3D"100%" align=3D"left" cellpadding=3D"0" cellspacing=3D"0" bo=
+rder=3D"0" style=3D"border-collapse: collapse !important; border-spacing: 0=
+ !important; border: none;">
+<tr>
+<td width=3D"100%" valign=3D"top" class=3D"center" align=3D"center" style=
+=3D"border-collapse: collapse !important; border-spacing: 0 !important; fon=
+t-size: 14px; line-height: normal; padding: 0px 17px;">
+<a href=3D"http://rover.ebay.com/rover/0/e11021.m4442.l1142/7?euid=3Dd9f42b=
+5e860b4eabb98195c2888cba9e&bu=3D43210693952&loc=3Dhttp%3A%2F%2Fcontact.ebay=
+.com.au%2Fws%2FeBayISAPI.dll%3FUnsubscribeEmailFavoriteSearch%26%26query%3D=
+3139313035333239303235-0db6b1b2ceaf88ebfc5edb9514cc5a36&exe=3D15083&ext=3D3=
+8992&sojTags=3Dexe=3Dexe,ext=3Dext,bu=3Dbu" style=3D"text-decoration: none;=
+ color: #0654BA; font-size: 14px; line-height: 18px; font-weight: normal; f=
+ont-family: Helvetica, Arial, sans-serif;"> Disable emails for this search<=
+/a>
+</td>
+</tr>
+</table>
+</td>
+</tr>
+</table>
+</td>
+</tr>
+</table> <table id=3D"area8Container" width=3D"100%" border=3D"0" cellpaddi=
+ng=3D"0" cellspacing=3D"0" align=3D"center" style=3D"border-collapse: colla=
+pse !important; border-spacing: 0 !important; border: none; border-top: sol=
+id 1px #dddddd; background-color: #ffffff"><tr><td style=3D"font-size:0px; =
+line-height:0px" height=3D"1">&nbsp;</td></tr></table> <table id=3D"area11C=
+ontainer" class=3D"whiteSection" width=3D"100%" border=3D"0" cellpadding=3D=
+"0" cellspacing=3D"0" align=3D"center" style=3D"border-collapse: collapse !=
+important; border-spacing: 0 !important; border: none; background-color: #f=
+fffff">
+<tr>
+<td width=3D"100%" valign=3D"top" style=3D"border-collapse: collapse !impor=
+tant; border-spacing: 0 !important; border: none;">
+<table width=3D"600" class=3D"device-width" border=3D"0" cellpadding=3D"0" =
+cellspacing=3D"0" align=3D"center" style=3D"border-collapse: collapse !impo=
+rtant; border-spacing: 0 !important; border: none;">
+<tr>
+<td class=3D"ebay-footer-block" style=3D"border-collapse: collapse !importa=
+nt; border-spacing: 0 !important; padding: 20px 0 60px; border: none;">
+<div id=3D"ReferenceId">
+<p style=3D"font-family: Helvetica, Arial, sans-serif; font-weight: normal;=
+ line-height: normal; color: #888888; text-align: left; font-size: 11px; ma=
+rgin: 0 0 10px;" align=3D"left"><strong>
+Email reference id: [#d9f42b5e860b4eabb98195c2888cba9e#]
+</strong></p></div>
+<p style=3D"font-family: Helvetica, Arial, sans-serif; font-weight: normal;=
+ line-height: normal; color: #888888; text-align: left; font-size: 11px; ma=
+rgin: 0 0 10px;" align=3D"left">
+We don't check this mailbox, so please don't reply to this message. If you =
+have a question, go to <a style=3D"text-decoration: none; color: #555555;" =
+href=3D"http://rover.ebay.com/rover/0/e11021.m1852.l6369/7?euid=3Dd9f42b5e8=
+60b4eabb98195c2888cba9e&bu=3D43210693952&loc=3Dhttp%3A%2F%2Focsnext.ebay.co=
+m.au%2Focs%2Fhome&exe=3D15083&ext=3D38992&sojTags=3Dexe=3Dexe,ext=3Dext,bu=
+=3Dbu" target=3D"_blank">Help & Contact</a>.
+</p>
+<p style=3D"font-family: Helvetica, Arial, sans-serif; font-weight: normal;=
+ line-height: normal; color: #888888; text-align: left; font-size: 11px; ma=
+rgin: 0 0 10px;" align=3D"left">
+&copy;2016 eBay Inc., eBay International AG Helvetiastrasse 15/17 - P.O. Bo=
+x 133, 3000 Bern 6, Switzerland
+</p>
+</td>
+</tr>
+</table>
+</td>
+</tr>
+</table>         </div>
+    </center></body>
+</html>
+
+""";
+
+    public static string HTML_BODY2_EXPECTED = "Buy It Now from US $1,750.00 to US $5,950.00. eBay Daccordi, Worldwide: 2 new matches today Daccordi 50th anniversary edition with... Buy it now: US $5,950.00 100% positive feedback Daccordi Griffe Campagnolo Croce D'Aune... Buy it now: US $1,750.00 100% positive feedback View all results Refine this search Disable emails for this search   Email reference id: [#d9f42b5e860b4eabb98195c2888cba9e#] We don't check this mailbox, so please don't reply to this message. If you have a question, go to Help & Contact. ©2016 eBay Inc., eBay International AG Helvetiastrasse 15/17 - P.O. Box 133, 3000 Bern 6, Switzerland";
+}
--- a/test/engine/util-html-test.vala
+++ b/test/engine/util-html-test.vala
@ -0,0 +1,76 @@
+/*
+ * Copyright 2016 Michael Gratton <mike@vee.net>
+ *
+ * This software is licensed under the GNU Lesser General Public License
+ * (version 2.1 or later). See the COPYING file in this distribution.
+ */
+
+class Geary.HTML.UtilTest : Gee.TestCase {
+
+    public UtilTest() {
+        base("Geary.HTML.Util");
+        add_test("remove_html_tags", remove_html_tags);
+    }
+
+    public void remove_html_tags() {
+        string blockquote_body = """<blockquote>hello</blockquote> <p>there</p>""";
+
+        string style_complete = """<style>
+.bodyblack { font-family: Verdana, Arial, Helvetica, sans-serif; font-size:
+ 12px; }
+td { font-size: 12px; }
+.footer { font-family: Verdana, Arial, Helvetica, sans-serif; font-size: 10
+px; }
+</style>""";
+
+        string style_truncated = """<html><head>
+<meta http-equiv=Content-Type content="text/html; charset=utf-8">
+<style>
+.bodyblack { font-family: Verdana, """;
+
+        assert(Geary.HTML.html_to_text(HTML_BODY_COMPLETE) == HTML_BODY_COMPLETE_EXPECTED);
+        assert(Geary.HTML.html_to_text(blockquote_body) == "hello\n there\n");
+        assert(Geary.HTML.html_to_text(blockquote_body, false) == " there\n");
+        assert(Geary.HTML.html_to_text(style_complete) == "");
+        assert(Geary.HTML.html_to_text(style_complete) == "");
+        assert(Geary.HTML.html_to_text(style_truncated) == "");
+    }
+
+    public static string HTML_BODY_COMPLETE = """<html><head>
+<meta http-equiv=Content-Type content="text/html; charset=utf-8">
+<style>
+.bodyblack { font-family: Verdana, Arial, Helvetica, sans-serif; font-size: 12px; }
+td { font-size: 12px; }
+.footer { font-family: Verdana, Arial, Helvetica, sans-serif; font-size: 10px; }
+</style>
+</head>
+<body><table cellSpacing="0" cellPadding="0" width="450" border="0" class="bodyblack"><tr><td>
+<p><br />Hi Kenneth, <br /> <br /> We xxxxx xxxx xx xxx xxx xx xxxx x xxxxxxxx xxxxxxxx.
+<br /> <br /> <br /> <br />Thank you, <br /> <br />XXXXX
+X XXXXXX<br /><br />You can reply directly to this message or click the following link:<br /><a href="https://app.foobar.com/xxxxxxxx752a0ab01641966deff6c48623aba">https://app.foobar.com/xxxxxxxxxxxxxxxx1641966deff6c48623aba</a><br /><br />You can change your email preferences at:<br /><a href="https://app.foobar.com/xxxxxxxxxxxxx">https://app.foobar.com/xxxxxxxxxxx</a></p></td></tr>
+</table></body></html>
+""";
+
+    public static string HTML_BODY_COMPLETE_EXPECTED = """
+
+Hi Kenneth, 
+
+ We xxxxx xxxx xx xxx xxx xx xxxx x xxxxxxxx xxxxxxxx.
+
+
+
+
+Thank you, 
+
+XXXXX
+X XXXXXX
+
+You can reply directly to this message or click the following link:
+https://app.foobar.com/xxxxxxxxxxxxxxxx1641966deff6c48623aba
+
+You can change your email preferences at:
+https://app.foobar.com/xxxxxxxxxxx
+ 
+""";
+
+}
--- a/test/main.vala
+++ b/test/main.vala
@ -7,9 +7,16 @@

 int main(string[] args) {
    Test.init(ref args);
+
+    Geary.RFC822.init();
+    Geary.HTML.init();
+
    TestSuite root = TestSuite.get_root();

+    // Engine tests
+    root.add_suite(new Geary.HTML.UtilTest().get_suite());
    root.add_suite(new Geary.RFC822.MailboxAddressTest().get_suite());
+    root.add_suite(new Geary.RFC822.MessageDataTest().get_suite());

    return Test.run();
 }