nettika/geary
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

FETCH BODY[section]<partial> support.

Browse source 
This adds support for retrieving partial header and body blocks straight from the email, and
therefore support to pull the References header from a message (which, for some reason, IMAP
doesn't support or include in the FETCH ENVELOPE command).  This is necessary for email conversations (#3808).

This required a change to the database schema, meaning old databases will need to be blown
away before starting.
This commit is contained in:
Jim Nelson 2011-10-17 18:56:17 -07:00
parent ab69b20b1c
commit db62ed5d93
13 changed files with 312 additions and 114 deletions
Download patch file Download diff file Expand all files Collapse all files

1
sql/Create.sql
View file

@ -33,6 +33,7 @@ CREATE TABLE MessageTable (
message_id TEXT,
in_reply_to TEXT,
reference_ids TEXT,
subject TEXT,

5
src/console/main.vala
View file

@ -408,9 +408,12 @@ class ImapConsole : Gtk.Window {
Geary.Imap.FetchBodyDataType fields = new Geary.Imap.FetchBodyDataType(
Geary.Imap.FetchBodyDataType.SectionPart.HEADER_FIELDS, args[1:args.length]);
Gee.List<Geary.Imap.FetchBodyDataType> list = new Gee.ArrayList<Geary.Imap.FetchBodyDataType>();
list.add(fields);
cx.send_async.begin(new Geary.Imap.FetchCommand(
new Geary.Imap.MessageSet.custom(args[0]), null, { fields }), null, on_fetch);
new Geary.Imap.MessageSet.custom(args[0]), null, list), null, on_fetch);
}
private void on_fetch(Object? source, AsyncResult result) {

5
src/engine/api/geary-email.vala
View file

@ -76,6 +76,7 @@ public class Geary.Email : Object {
// REFERENCES
public Geary.RFC822.MessageID? message_id { get; private set; default = null; }
public Geary.RFC822.MessageID? in_reply_to { get; private set; default = null; }
public Geary.RFC822.MessageIDList? references { get; private set; default = null; }
// SUBJECT
public Geary.RFC822.Subject? subject { get; private set; default = null; }
@ -126,9 +127,11 @@ public class Geary.Email : Object {
fields |= Field.RECEIVERS;
}
public void set_references(Geary.RFC822.MessageID? message_id, Geary.RFC822.MessageID? in_reply_to) {
public void set_full_references(Geary.RFC822.MessageID? message_id, Geary.RFC822.MessageID? in_reply_to,
Geary.RFC822.MessageIDList? references) {
this.message_id = message_id;
this.in_reply_to = in_reply_to;
this.references = references;
fields |= Field.REFERENCES;
}

76
src/engine/imap/command/imap-commands.vala
View file

@ -88,82 +88,6 @@ public class Geary.Imap.CloseCommand : Command {
}
}
public class Geary.Imap.FetchCommand : Command {
public const string NAME = "fetch";
public const string UID_NAME = "uid fetch";
public FetchCommand(MessageSet msg_set, FetchDataType[]? data_items,
FetchBodyDataType[]? body_data_items) {
base (msg_set.is_uid ? UID_NAME : NAME);
add(msg_set.to_parameter());
int data_items_length = (data_items != null) ? data_items.length : 0;
int body_data_items_length = (body_data_items != null) ? body_data_items.length : 0;
// if only one item being fetched, pass that as a singleton parameter, otherwise pass them
// all as a list
if (data_items_length == 1 && body_data_items_length == 0) {
add(data_items[0].to_parameter());
} else if (data_items_length == 0 && body_data_items_length == 1) {
add(body_data_items[0].to_parameter());
} else {
ListParameter list = new ListParameter(this);
if (data_items != null) {
foreach (FetchDataType data_item in data_items)
list.add(data_item.to_parameter());
}
if (body_data_items != null) {
foreach (FetchBodyDataType body_data_item in body_data_items)
list.add(body_data_item.to_parameter());
}
add(list);
}
}
public FetchCommand.from_collection(MessageSet msg_set, Gee.Collection<FetchDataType>? data_items,
Gee.Collection<FetchBodyDataType>? body_data_items) {
base (msg_set.is_uid ? UID_NAME : NAME);
add(msg_set.to_parameter());
int data_items_length = (data_items != null) ? data_items.size : 0;
int body_data_items_length = (body_data_items != null) ? body_data_items.size : 0;
// see note in unadorned ctor for reasoning here
if (data_items_length == 1 && body_data_items_length == 0) {
foreach (FetchDataType data_type in data_items) {
add(data_type.to_parameter());
break;
}
} else if (data_items_length == 0 && body_data_items_length == 1) {
foreach (FetchBodyDataType body_data_type in body_data_items) {
add(body_data_type.to_parameter());
break;
}
} else {
ListParameter data_item_list = new ListParameter(this);
if (data_items != null) {
foreach (FetchDataType data_item in data_items)
data_item_list.add(data_item.to_parameter());
}
if (body_data_items != null) {
foreach (FetchBodyDataType body_data_item in body_data_items)
data_item_list.add(body_data_item.to_parameter());
}
add(data_item_list);
}
}
}
public class Geary.Imap.StatusCommand : Command {
public const string NAME = "status";

74
src/engine/imap/command/imap-fetch-command.vala Normal file
View file

@ -0,0 +1,74 @@
/* Copyright 2011 Yorba Foundation
*
* This software is licensed under the GNU Lesser General Public License
* (version 2.1 or later). See the COPYING file in this distribution.
*/
public class Geary.Imap.FetchCommand : Command {
public const string NAME = "fetch";
public const string UID_NAME = "uid fetch";
public FetchCommand(MessageSet msg_set, FetchDataType[]? data_items,
Gee.List<FetchBodyDataType>? body_data_items) {
base (msg_set.is_uid ? UID_NAME : NAME);
add(msg_set.to_parameter());
int data_items_length = (data_items != null) ? data_items.length : 0;
int body_items_length = (body_data_items != null) ? body_data_items.size : 0;
// if only one item being fetched, pass that as a singleton parameter, otherwise pass them
// as a list
if (data_items_length == 1 && body_items_length == 0) {
add(data_items[0].to_parameter());
} else if (data_items_length == 0 && body_items_length == 1) {
add(body_data_items[0].to_parameter());
} else {
ListParameter list = new ListParameter(this);
if (data_items_length > 0) {
foreach (FetchDataType data_item in data_items)
list.add(data_item.to_parameter());
}
if (body_items_length > 0) {
foreach (FetchBodyDataType body_item in body_data_items)
list.add(body_item.to_parameter());
}
add(list);
}
}
public FetchCommand.from_collection(MessageSet msg_set, Gee.List<FetchDataType>? data_items,
Gee.List<FetchBodyDataType>? body_data_items) {
base (msg_set.is_uid ? UID_NAME : NAME);
add(msg_set.to_parameter());
int data_items_length = (data_items != null) ? data_items.size : 0;
int body_items_length = (body_data_items != null) ? body_data_items.size : 0;
// see note in unadorned ctor for reasoning here
if (data_items_length == 1 && body_items_length == 0) {
add(data_items[0].to_parameter());
} else if (data_items_length == 0 && body_items_length == 1) {
add(body_data_items[0].to_parameter());
} else {
ListParameter list = new ListParameter(this);
if (data_items_length > 0) {
foreach (FetchDataType data_item in data_items)
list.add(data_item.to_parameter());
}
if (body_items_length > 0) {
foreach (FetchBodyDataType body_item in body_data_items)
list.add(body_item.to_parameter());
}
add(list);
}
}
}

2
src/engine/imap/decoders/imap-fetch-data-decoder.vala
View file

@ -40,7 +40,7 @@ public abstract class Geary.Imap.FetchDataDecoder {
return decode_literal(literalp);
// bad news; this means this function isn't handling a Parameter type properly
assert_not_reached();;
assert_not_reached();
}
protected virtual MessageData decode_string(StringParameter param) throws ImapError {

30
src/engine/imap/decoders/imap-fetch-results.vala
View file

@ -16,6 +16,7 @@ public class Geary.Imap.FetchResults : Geary.Imap.CommandResults {
public int msg_num { get; private set; }
private Gee.Map<FetchDataType, MessageData> map = new Gee.HashMap<FetchDataType, MessageData>();
private Gee.List<Memory.AbstractBuffer> body_data = new Gee.ArrayList<Memory.AbstractBuffer>();
public FetchResults(StatusResponse status_response, int msg_num) {
base (status_response);
@ -40,16 +41,23 @@ public class Geary.Imap.FetchResults : Geary.Imap.CommandResults {
// and the structured data itself
for (int ctr = 0; ctr < list.get_count(); ctr += 2) {
StringParameter data_item_param = list.get_as_string(ctr);
FetchDataType data_item = FetchDataType.decode(data_item_param.value);
FetchDataDecoder? decoder = data_item.get_decoder();
if (decoder == null) {
debug("Unable to decode fetch response for \"%s\": No decoder available",
data_item.to_string());
continue;
}
results.set_data(data_item, decoder.decode(list.get_required(ctr + 1)));
if (FetchBodyDataType.is_fetch_body(data_item_param)) {
// FETCH body data items are merely a literal of all requested fields formatted
// in RFC822 header format
results.body_data.add(list.get_as_literal(ctr + 1).get_buffer());
} else {
FetchDataType data_item = FetchDataType.decode(data_item_param.value);
FetchDataDecoder? decoder = data_item.get_decoder();
if (decoder == null) {
debug("Unable to decode fetch response for \"%s\": No decoder available",
data_item.to_string());
continue;
}
results.set_data(data_item, decoder.decode(list.get_required(ctr + 1)));
}
}
return results;
@ -85,6 +93,10 @@ public class Geary.Imap.FetchResults : Geary.Imap.CommandResults {
return map.get(data_item);
}
public Gee.List<Memory.AbstractBuffer> get_body_data() {
return body_data.read_only_view;
}
public int get_count() {
return map.size;
}

6
src/engine/imap/message/imap-fetch-body-data-type.vala
View file

@ -105,6 +105,12 @@ public class Geary.Imap.FetchBodyDataType {
return builder.str;
}
public static bool is_fetch_body(StringParameter items) {
string strd = items.value.down();
return strd.has_prefix("body[") || strd.has_prefix("body.peek[");
}
public string to_string() {
return (!is_peek ? "body[%s%s]" : "body.peek[%s%s]").printf(section_part.serialize(),
serialize_field_names());

122
src/engine/imap/transport/imap-mailbox.vala
View file

@ -65,7 +65,6 @@ public class Geary.Imap.Mailbox : Geary.SmartReference {
body_data_type_list);
CommandResponse resp = yield context.session.send_command_async(fetch_cmd, cancellable);
if (resp.status_response.status != Status.OK) {
throw new ImapError.SERVER_ERROR("Server error for %s: %s", fetch_cmd.to_string(),
resp.to_string());
@ -103,7 +102,6 @@ public class Geary.Imap.Mailbox : Geary.SmartReference {
data_type_list, body_data_type_list);
CommandResponse resp = yield context.session.send_command_async(fetch_cmd, cancellable);
if (resp.status_response.status != Status.OK) {
throw new ImapError.SERVER_ERROR("Server error for %s: %s", fetch_cmd.to_string(),
resp.to_string());
@ -157,38 +155,47 @@ public class Geary.Imap.Mailbox : Geary.SmartReference {
// The assumption here is that because ENVELOPE is such a common fetch command, the
// server will have optimizations for it, whereas if we called for each header in the
// envelope separately, the server has to chunk harder parsing the RFC822 header
bool using_envelope = false;
if (fields.is_all_set(Geary.Email.Field.ENVELOPE)) {
data_types_list.add(FetchDataType.ENVELOPE);
// remove those flags and process any remaining
fields = fields.clear(Geary.Email.Field.ENVELOPE);
using_envelope = true;
}
// pack all the needed headers into a single FetchBodyDataType
string[] field_names = new string[0];
foreach (Geary.Email.Field field in Geary.Email.Field.all()) {
switch (fields & field) {
case Geary.Email.Field.DATE:
body_data_types_list.add(new FetchBodyDataType.peek(
FetchBodyDataType.SectionPart.HEADER_FIELDS, { "Date" }));
field_names += "Date";
break;
case Geary.Email.Field.ORIGINATORS:
body_data_types_list.add(new FetchBodyDataType.peek(
FetchBodyDataType.SectionPart.HEADER_FIELDS, { "From", "Sender", "Reply-To" }));
field_names += "From";
field_names += "Sender";
field_names += "Reply-To";
break;
case Geary.Email.Field.RECEIVERS:
body_data_types_list.add(new FetchBodyDataType.peek(
FetchBodyDataType.SectionPart.HEADER_FIELDS, { "To", "Cc", "Bcc" }));
field_names += "To";
field_names += "Cc";
field_names += "Bcc";
break;
case Geary.Email.Field.REFERENCES:
body_data_types_list.add(new FetchBodyDataType.peek(
FetchBodyDataType.SectionPart.HEADER_FIELDS, { "Message-ID", "In-Reply-To" }));
field_names += "References";
if (!using_envelope) {
field_names += "Message-ID";
field_names += "In-Reply-To";
}
break;
case Geary.Email.Field.SUBJECT:
body_data_types_list.add(new FetchBodyDataType.peek(
FetchBodyDataType.SectionPart.HEADER_FIELDS, { "Subject" }));
field_names += "Subject";
break;
case Geary.Email.Field.HEADER:
@ -215,15 +222,25 @@ public class Geary.Imap.Mailbox : Geary.SmartReference {
assert_not_reached();
}
}
if (field_names.length > 0) {
body_data_types_list.add(new FetchBodyDataType(
FetchBodyDataType.SectionPart.HEADER_FIELDS, field_names));
}
}
private static void fetch_results_to_email(FetchResults res, Geary.Email.Field fields,
Geary.Email email) {
Geary.Email email) throws Error {
// accumulate these to submit Imap.EmailProperties all at once
Geary.Imap.MessageFlags? flags = null;
InternalDate? internaldate = null;
RFC822.Size? rfc822_size = null;
// accumulate these to submit References all at once
RFC822.MessageID? message_id = null;
RFC822.MessageID? in_reply_to = null;
RFC822.MessageIDList? references = null;
foreach (FetchDataType data_type in res.get_all_types()) {
MessageData? data = res.get_data(data_type);
if (data == null)
@ -245,8 +262,10 @@ public class Geary.Imap.Mailbox : Geary.SmartReference {
if ((fields & Geary.Email.Field.RECEIVERS) != 0)
email.set_receivers(envelope.to, envelope.cc, envelope.bcc);
if ((fields & Geary.Email.Field.REFERENCES) != 0)
email.set_references(envelope.message_id, envelope.in_reply_to);
if ((fields & Geary.Email.Field.REFERENCES) != 0) {
message_id = envelope.message_id;
in_reply_to = envelope.in_reply_to;
}
break;
case FetchDataType.RFC822_HEADER:
@ -275,8 +294,81 @@ public class Geary.Imap.Mailbox : Geary.SmartReference {
}
}
// fields_to_fetch_data_types() will always generate a single FetchBodyDataType for all
// the header fields it needs
Gee.List<Memory.AbstractBuffer> body_data = res.get_body_data();
if (body_data.size > 0) {
assert(body_data.size == 1);
RFC822.Header headers = new RFC822.Header(body_data[0]);
// DATE
string? value = headers.get_header("Date");
if (!String.is_empty(value))
email.set_send_date(new RFC822.Date(value));
// ORIGINATORS
RFC822.MailboxAddresses? from = null;
RFC822.MailboxAddresses? sender = null;
RFC822.MailboxAddresses? reply_to = null;
value = headers.get_header("From");
if (!String.is_empty(value))
from = new RFC822.MailboxAddresses.from_rfc822_string(value);
value = headers.get_header("Sender");
if (!String.is_empty(value))
sender = new RFC822.MailboxAddresses.from_rfc822_string(value);
value = headers.get_header("Reply-To");
if (!String.is_empty(value))
reply_to = new RFC822.MailboxAddresses.from_rfc822_string(value);
email.set_originators(from, sender, reply_to);
// RECEIVERS
RFC822.MailboxAddresses? to = null;
RFC822.MailboxAddresses? cc = null;
RFC822.MailboxAddresses? bcc = null;
value = headers.get_header("To");
if (!String.is_empty(value))
to = new RFC822.MailboxAddresses.from_rfc822_string(value);
value = headers.get_header("Cc");
if (!String.is_empty(value))
cc = new RFC822.MailboxAddresses.from_rfc822_string(value);
value = headers.get_header("Bcc");
if (!String.is_empty(value))
bcc = new RFC822.MailboxAddresses.from_rfc822_string(value);
email.set_receivers(to, cc, bcc);
// REFERENCES
// (Note that it's possible the request used an IMAP ENVELOPE, in which case only the
// References header will be present if REFERENCES were required.)
value = headers.get_header("Message-ID");
if (!String.is_empty(value))
message_id = new RFC822.MessageID(value);
value = headers.get_header("In-Reply-To");
if (!String.is_empty(value))
in_reply_to = new RFC822.MessageID(value);
value = headers.get_header("References");
if (!String.is_empty(value))
references = new RFC822.MessageIDList(value);
// SUBJECT
value = headers.get_header("Subject");
if (!String.is_empty(value))
email.set_message_subject(new RFC822.Subject(value));
}
if (flags != null && internaldate != null && rfc822_size != null)
email.set_email_properties(new Geary.Imap.EmailProperties(flags, internaldate, rfc822_size));
email.set_full_references(message_id, in_reply_to, references);
}
}

78
src/engine/rfc822/rfc822-message-data.vala
View file

@ -13,10 +13,46 @@
public interface Geary.RFC822.MessageData : Geary.Common.MessageData {
}
public class Geary.RFC822.MessageID : Geary.Common.StringMessageData, Geary.RFC822.MessageData {
public class Geary.RFC822.MessageID : Geary.Common.StringMessageData, Geary.RFC822.MessageData,
Geary.Equalable {
public MessageID(string value) {
base (value);
}
public bool equals(Equalable e) {
MessageID? message_id = e as MessageID;
if (message_id == null)
return false;
if (this == message_id)
return true;
return value == message_id.value;
}
}
public class Geary.RFC822.MessageIDList : Geary.Common.StringMessageData, Geary.RFC822.MessageData {
private Gee.List<MessageID>? list = null;
public MessageIDList(string value) {
base (value);
}
public Gee.List<MessageID> decoded() {
if (list != null)
return list;
list = new Gee.ArrayList<MessageID>(Equalable.equal_func);
string[] ids = value.split(" ");
foreach (string id in ids) {
id = id.strip();
if (!String.is_empty(id))
list.add(new MessageID(id));
}
return list;
}
}
public class Geary.RFC822.Date : Geary.RFC822.MessageData, Geary.Common.MessageData {
@ -55,9 +91,49 @@ public class Geary.RFC822.Subject : Geary.Common.StringMessageData, Geary.RFC822
}
public class Geary.RFC822.Header : Geary.Common.BlockMessageData, Geary.RFC822.MessageData {
private GMime.Message? message = null;
private string[]? names = null;
public Header(Geary.Memory.AbstractBuffer buffer) {
base ("RFC822.Header", buffer);
}
private unowned GMime.HeaderList get_headers() throws RFC822Error {
if (message != null)
return message.get_header_list();
GMime.Parser parser = new GMime.Parser.with_stream(
new GMime.StreamMem.with_buffer(buffer.get_array()));
parser.set_respect_content_length(false);
parser.set_scan_from(false);
message = parser.construct_message();
if (message == null)
throw new RFC822Error.INVALID("Unable to parse RFC 822 headers");
return message.get_header_list();
}
public string get_header(string name) throws RFC822Error {
return get_headers().get(name);
}
public string[] get_header_names() throws RFC822Error {
if (names != null)
return names;
names = new string[0];
unowned GMime.HeaderIter iter;
if (!get_headers().get_iter(out iter))
return names;
do {
names += iter.get_name();
} while (iter.next());
return names;
}
}
public class Geary.RFC822.Text : Geary.Common.BlockMessageData, Geary.RFC822.MessageData {

7
src/engine/sqlite/email/sqlite-message-row.vala
View file

@ -21,6 +21,7 @@ public class Geary.Sqlite.MessageRow : Geary.Sqlite.Row {
public string? message_id { get; set; }
public string? in_reply_to { get; set; }
public string? references { get; set; }
public string? subject { get; set; }
@ -97,8 +98,10 @@ public class Geary.Sqlite.MessageRow : Geary.Sqlite.Row {
}
if ((fields & Geary.Email.Field.REFERENCES) != 0) {
email.set_references((message_id != null) ? new RFC822.MessageID(message_id) : null,
(in_reply_to != null) ? new RFC822.MessageID(in_reply_to) : null);
email.set_full_references(
(message_id != null) ? new RFC822.MessageID(message_id) : null,
(in_reply_to != null) ? new RFC822.MessageID(in_reply_to) : null,
(references != null) ? new RFC822.MessageIDList(references) : null);
}
if (((fields & Geary.Email.Field.SUBJECT) != 0) && (subject != null))

19
src/engine/sqlite/email/sqlite-message-table.vala
View file

@ -23,6 +23,7 @@ public class Geary.Sqlite.MessageTable : Geary.Sqlite.Table {
MESSAGE_ID,
IN_REPLY_TO,
REFERENCES,
SUBJECT,
@ -43,8 +44,8 @@ public class Geary.Sqlite.MessageTable : Geary.Sqlite.Table {
SQLHeavy.Query query = locked.prepare(
"INSERT INTO MessageTable "
+ "(fields, date_field, date_time_t, from_field, sender, reply_to, to_field, cc, bcc, "
+ "message_id, in_reply_to, subject, header, body) "
+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
+ "message_id, in_reply_to, reference_ids, subject, header, body) "
+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
query.bind_int(0, row.fields);
query.bind_string(1, row.date);
query.bind_int64(2, row.date_time_t);
@ -56,9 +57,10 @@ public class Geary.Sqlite.MessageTable : Geary.Sqlite.Table {
query.bind_string(8, row.bcc);
query.bind_string(9, row.message_id);
query.bind_string(10, row.in_reply_to);
query.bind_string(11, row.subject);
query.bind_string(12, row.header);
query.bind_string(13, row.body);
query.bind_string(11, row.references);
query.bind_string(12, row.subject);
query.bind_string(13, row.header);
query.bind_string(14, row.body);
int64 id = yield query.execute_insert_async(cancellable);
locked.set_commit_required();
@ -116,10 +118,11 @@ public class Geary.Sqlite.MessageTable : Geary.Sqlite.Table {
if (row.fields.is_any_set(Geary.Email.Field.REFERENCES)) {
query = locked.prepare(
"UPDATE MessageTable SET message_id=?, in_reply_to=? WHERE id=?");
"UPDATE MessageTable SET message_id=?, in_reply_to=?, reference_ids = ? WHERE id=?");
query.bind_string(0, row.message_id);
query.bind_string(1, row.in_reply_to);
query.bind_int64(2, row.id);
query.bind_string(2, row.references);
query.bind_int64(3, row.id);
yield query.execute_async(cancellable);
}
@ -242,7 +245,7 @@ public class Geary.Sqlite.MessageTable : Geary.Sqlite.Table {
break;
case Geary.Email.Field.REFERENCES:
append = "message_id, in_reply_to";
append = "message_id, in_reply_to, reference_ids";
break;
case Geary.Email.Field.SUBJECT:

1
src/wscript
View file

@ -42,6 +42,7 @@ def build(bld):
'../engine/imap/command/imap-command-response.vala',
'../engine/imap/command/imap-commands.vala',
'../engine/imap/command/imap-command.vala',
'../engine/imap/command/imap-fetch-command.vala',
'../engine/imap/decoders/imap-command-results.vala',
'../engine/imap/decoders/imap-fetch-data-decoder.vala',
'../engine/imap/decoders/imap-fetch-results.vala',