IMAP: work in progress of downloading the mail body.

* Most things are in place now, we just try to download the body to the
  wrong file, as the final location is currently unknown.
* Added local only kPartialMessage flag for mails, but it's not being
  used yet.

src/add-ons/mail_daemon/inbound_protocols/imap/IMAPConnectionWorker.cpp

@@ -59,6 +59,11 @@ public:
 		return fWorker._MailboxFor(folder);
 	}
 
+	int32 BodyFetchLimit()
+	{
+		return fWorker.fSettings.BodyFetchLimit();
+	}
+
 	status_t EnqueueCommand(WorkerCommand* command)
 	{
 		return fWorker._EnqueueCommand(command);
@@ -125,15 +130,78 @@ public:
 };
 
 
+class FetchBodiesCommand : public SyncCommand, public IMAP::FetchListener {
+public:
+	FetchBodiesCommand(IMAPFolder& folder, IMAPMailbox& mailbox,
+		std::vector<uint32>& entries)
+		:
+		fFolder(folder),
+		fMailbox(mailbox),
+		fEntries(entries)
+	{
+	}
+
+	virtual status_t Process(IMAPConnectionWorker& worker)
+	{
+		IMAP::Protocol& protocol = WorkerPrivate(worker).Protocol();
+
+		if (fEntries.empty())
+			return B_OK;
+
+		fUID = *fEntries.begin();
+		fEntries.erase(fEntries.begin());
+
+		// TODO: check nextUID if we get one
+		status_t status = WorkerPrivate(worker).SelectMailbox(fFolder);
+		if (status != B_OK)
+			return status;
+
+		printf("IMAP: fetch body for %lu\n", fUID);
+		// TODO: combine smaller messages together for faster retrieval
+		IMAP::FetchCommand fetch(fUID, fUID, IMAP::kFetchBody);
+		fetch.SetListener(this);
+
+		return protocol.ProcessCommand(fetch);
+	}
+
+	virtual bool IsDone() const
+	{
+		return fEntries.empty();
+	}
+
+	virtual bool FetchData(uint32 fetchFlags, BDataIO& stream, size_t& length)
+	{
+		fFetchStatus = fFolder.StoreBody(fUID, stream, length, fRef, fFile);
+		return true;
+	}
+
+	virtual void FetchedData(uint32 fetchFlags, uint32 uid, uint32 flags)
+	{
+		if (fFetchStatus == B_OK)
+			fFolder.BodyStored(fRef, fFile, uid);
+	}
+
+private:
+	IMAPFolder&				fFolder;
+	IMAPMailbox&			fMailbox;
+	std::vector<uint32>		fEntries;
+	uint32					fUID;
+	entry_ref				fRef;
+	BFile					fFile;
+	status_t				fFetchStatus;
+};
+
+
 class FetchHeadersCommand : public SyncCommand, public IMAP::FetchListener {
 public:
 	FetchHeadersCommand(IMAPFolder& folder, IMAPMailbox& mailbox,
-		uint32 from, uint32 to)
+		uint32 from, uint32 to, int32 bodyFetchLimit)
 		:
 		fFolder(folder),
 		fMailbox(mailbox),
 		fFrom(from),
-		fTo(to)
+		fTo(to),
+		fBodyFetchLimit(bodyFetchLimit)
 	{
 	}
 
@@ -152,8 +220,6 @@ public:
 		if (to - fFrom >= kMaxFetchEntries)
 			to = fFrom + kMaxFetchEntries - 1;
 
-		// TODO: trigger download of mails for all messages below the
-		// body fetch limit
 		printf("IMAP: fetch headers from %lu to %lu\n", fFrom, to);
 		IMAP::FetchCommand fetch(fFrom, to,
 			IMAP::kFetchHeader | IMAP::kFetchFlags);
@@ -164,6 +230,13 @@ public:
 			return status;
 
 		fFrom = to + 1;
+
+		if (IsDone() && !fFetchBodies.empty()) {
+			// Enqueue command to fetch the message bodies
+			WorkerPrivate(worker).EnqueueCommand(new FetchBodiesCommand(fFolder,
+				fMailbox, fFetchBodies));
+		}
+
 		return B_OK;
 	}
 
@@ -174,15 +247,20 @@ public:
 
 	virtual bool FetchData(uint32 fetchFlags, BDataIO& stream, size_t& length)
 	{
-		fFetchStatus = fFolder.StoreMessage(fFile, fetchFlags, stream,
+		fFetchStatus = fFolder.StoreMessage(fetchFlags, stream, length,
-			length, fRef);
+			fRef, fFile);
 		return true;
 	}
 
 	virtual void FetchedData(uint32 fetchFlags, uint32 uid, uint32 flags)
 	{
-		if (fFetchStatus == B_OK)
+		if (fFetchStatus == B_OK) {
 			fFolder.MessageStored(fRef, fFile, fetchFlags, uid, flags);
+
+			uint32 size = fMailbox.MessageSize(uid);
+			if (fBodyFetchLimit < 0 || size < fBodyFetchLimit)
+				fFetchBodies.push_back(uid);
+		}
 	}
 
 private:
@@ -190,6 +268,8 @@ private:
 	IMAPMailbox&			fMailbox;
 	uint32					fFrom;
 	uint32					fTo;
+	uint32					fBodyFetchLimit;
+	std::vector<uint32>		fFetchBodies;
 	entry_ref				fRef;
 	BFile					fFile;
 	status_t				fFetchStatus;
@@ -269,9 +349,13 @@ public:
 				return status;
 
 			// Determine how much we need to download
+			// TODO: also retrieve the header size, and only take the body
+			// size into account if it's below the limit
 			for (size_t i = 0; i < entries.size(); i++) {
 				printf("%10lu %8lu bytes, flags: %#lx\n", entries[i].uid,
 					entries[i].size, entries[i].flags);
+				fMailbox->AddMessageEntry(entries[i].uid, entries[i].flags,
+					entries[i].size);
 				fTotalBytes += entries[i].size;
 			}
 			fTotalEntries += entries.size();
@@ -283,7 +367,8 @@ public:
 				if (fMailboxEntries > 0) {
 					// Add pending command to fetch the message headers
 					WorkerCommand* command = new FetchHeadersCommand(*fFolder,
-						*fMailbox, fFirstUID, fNextUID);
+						*fMailbox, fFirstUID, fNextUID,
+						WorkerPrivate(worker).BodyFetchLimit());
 					if (!fFetchCommands.AddItem(command))
 						delete command;
 				}

src/add-ons/mail_daemon/inbound_protocols/imap/IMAPFolder.cpp

@@ -132,7 +132,7 @@ IMAPFolder::IMAPFolder(IMAPProtocol& protocol, const BString& mailboxName,
 			uint32 uid = B_BENDIAN_TO_HOST_INT32(entries[i].uid);
 			uint32 flags = B_BENDIAN_TO_HOST_INT32(entries[i].flags);
 
-			fUIDMap.insert(std::make_pair(uid, flags));
+			fFlagsMap.insert(std::make_pair(uid, flags));
 		}
 	}
 
@@ -167,6 +167,29 @@ IMAPFolder::SetUIDValidity(uint32 uidValidity)
 }
 
 
+status_t
+IMAPFolder::GetMessageEntryRef(uint32 uid, entry_ref& ref) const
+{
+	UIDToRefMap::const_iterator found = fRefMap.find(uid);
+	if (found == fRefMap.end())
+		return B_ENTRY_NOT_FOUND;
+
+	ref = found->second;
+	return B_OK;
+}
+
+
+uint32
+IMAPFolder::MessageFlags(uint32 uid) const
+{
+	UIDToFlagsMap::const_iterator found = fFlagsMap.find(uid);
+	if (found == fFlagsMap.end())
+		return 0;
+
+	return found->second;
+}
+
+
 /*!	Stores the given \a stream into a temporary file using the provided
 	BFile object. A new file will be created, and the \a ref object will
 	point to it. The file will remain open when this method exits without
@@ -176,8 +199,8 @@ IMAPFolder::SetUIDValidity(uint32 uidValidity)
 	were an error.
 */
 status_t
-IMAPFolder::StoreMessage(BFile& file, uint32 fetchFlags, BDataIO& stream,
+IMAPFolder::StoreMessage(uint32 fetchFlags, BDataIO& stream,
-	size_t& length, entry_ref& ref)
+	size_t& length, entry_ref& ref, BFile& file)
 {
 	BPath path;
 	status_t status = path.SetTo(&fRef);
@@ -189,26 +212,11 @@ IMAPFolder::StoreMessage(BFile& file, uint32 fetchFlags, BDataIO& stream,
 	if (status != B_OK)
 		return status;
 
-	char buffer[65535];
+	status = _WriteStream(file, stream, length);
-	while (length > 0) {
+	if (status == B_OK)
-		ssize_t bytesRead = stream.Read(buffer,
+		temporaryFile.KeepFile();
-			std::min(sizeof(buffer), length));
-		if (bytesRead < 0)
-			return bytesRead;
-		if (bytesRead <= 0)
-			break;
 
-		length -= bytesRead;
+	return status;
-
-		ssize_t bytesWritten = file.Write(buffer, bytesRead);
-		if (bytesWritten < 0)
-			return bytesWritten;
-		if (bytesWritten != bytesRead)
-			return B_IO_ERROR;
-	}
-
-	temporaryFile.KeepFile();
-	return B_OK;
 }
 
 
@@ -223,29 +231,73 @@ IMAPFolder::MessageStored(entry_ref& ref, BFile& file, uint32 fetchFlags,
 	if ((fetchFlags & IMAP::kFetchFlags) != 0)
 		_WriteFlags(file, flags);
 
-	// TODO: the call below may move/rename the file
+	// TODO: the call below may move/rename the file - this prevents downloading
-	fProtocol.MessageStored(ref, file, fetchFlags);
+	// the body to the correct file!
+	fProtocol.MessageStored(*this, ref, file, fetchFlags);
 	file.Unset();
 
+	fRefMap.insert(std::make_pair(uid, ref));
+
 	if (uid > fLastUID) {
 		// Update last known UID
 		fLastUID = uid;
 
 		BNode directory(&fRef);
 		status_t status = _WriteUInt32(directory, kLastUIDAttribute, uid);
-		if (status != B_OK)
+		if (status != B_OK) {
 			fprintf(stderr, "IMAP: Could not write last UID for mailbox "
 				"%s: %s\n", fMailboxName.String(), strerror(status));
+		}
 	}
 }
 
 
+/*!	Appends the given \a stream as body to the message file for the
+	specified unique ID. The file will remain open when this method exits
+	without an error.
+
+	\a length will reflect how many bytes are left to read in case there
+	were an error.
+*/
+status_t
+IMAPFolder::StoreBody(uint32 uid, BDataIO& stream, size_t& length,
+	entry_ref& ref, BFile& file)
+{
+	status_t status = GetMessageEntryRef(uid, ref);
+	if (status != B_OK)
+		return status;
+
+	status = file.SetTo(&ref, B_OPEN_AT_END | B_WRITE_ONLY);
+	if (status != B_OK)
+		return status;
+
+	BPath path(&ref);
+	printf("IMAP: write body to %s\n", path.Path());
+
+	return _WriteStream(file, stream, length);
+}
+
+
+/*!	Notifies the protocol that a body has been fetched.
+	This method also closes the \a file passed in.
+*/
+void
+IMAPFolder::BodyStored(entry_ref& ref, BFile& file, uint32 uid)
+{
+	fProtocol.MessageStored(*this, ref, file, IMAP::kFetchBody);
+	file.Unset();
+}
+
+
 void
 IMAPFolder::DeleteMessage(uint32 uid)
 {
 }
 
 
+/*!	Called when the flags of a message changed on the server. This will update
+	the flags for the local file.
+*/
 void
 IMAPFolder::SetMessageFlags(uint32 uid, uint32 flags)
 {
@@ -264,8 +316,8 @@ IMAPFolder::_InitializeFolderState()
 	// Create set of the last known UID state - if an entry is found, it
 	// is being removed from the list. The remaining entries were deleted.
 	std::set<uint32> lastUIDs;
-	UIDToFlagsMap::iterator iterator = fUIDMap.begin();
+	UIDToFlagsMap::iterator iterator = fFlagsMap.begin();
-	for (; iterator != fUIDMap.end(); iterator++)
+	for (; iterator != fFlagsMap.end(); iterator++)
 		lastUIDs.insert(iterator->first);
 
 	BDirectory directory(&fRef);
@@ -286,12 +338,11 @@ IMAPFolder::_InitializeFolderState()
 			// The message is still around
 			lastUIDs.erase(found);
 
-			UIDToFlagsMap::iterator flagsFound = fUIDMap.find(uid);
+			uint32 flagsFound = MessageFlags(uid);
-			ASSERT(flagsFound != fUIDMap.end());
+			if (flagsFound != flags) {
-			if (flagsFound->second != flags) {
 				// Its flags have changed locally, and need to be updated
 				fListener->MessageFlagsChanged(_Token(uid), ref,
-					flagsFound->second, flags);
+					flagsFound, flags);
 			}
 		} else {
 			// This is a new message
@@ -376,3 +427,28 @@ IMAPFolder::_WriteUInt32(BNode& node, const char* attribute, uint32 value)
 
 	return bytesWritten < 0 ? bytesWritten : B_IO_ERROR;
 }
+
+
+status_t
+IMAPFolder::_WriteStream(BFile& file, BDataIO& stream, size_t& length)
+{
+	char buffer[65535];
+	while (length > 0) {
+		ssize_t bytesRead = stream.Read(buffer,
+			std::min(sizeof(buffer), length));
+		if (bytesRead < 0)
+			return bytesRead;
+		if (bytesRead <= 0)
+			break;
+
+		length -= bytesRead;
+
+		ssize_t bytesWritten = file.Write(buffer, bytesRead);
+		if (bytesWritten < 0)
+			return bytesWritten;
+		if (bytesWritten != bytesRead)
+			return B_IO_ERROR;
+	}
+
+	return B_OK;
+}

src/add-ons/mail_daemon/inbound_protocols/imap/IMAPFolder.h

@@ -24,6 +24,11 @@ struct MessageToken {
 	uint32	uid;
 };
 
+// Additional local only message flags
+enum FolderMessageFlags {
+	kPartialMessage = 0x00010000,
+};
+
 
 class FolderListener {
 public:
@@ -51,13 +56,23 @@ public:
 
 			uint32				LastUID() const { return fLastUID; }
 
-			status_t			StoreMessage(BFile& file, uint32 fetchFlags,
+			status_t			GetMessageEntryRef(uint32 uid,
-									BDataIO& stream, size_t& length,
+									entry_ref& ref) const;
-									entry_ref& ref);
+			uint32				MessageFlags(uint32 uid) const;
+
+			status_t			StoreMessage(uint32 fetchFlags, BDataIO& stream,
+									size_t& length, entry_ref& ref,
+									BFile& file);
 			void				MessageStored(entry_ref& ref, BFile& file,
 									uint32 fetchFlags, uint32 uid,
 									uint32 flags);
 
+			status_t			StoreBody(uint32 uid, BDataIO& stream,
+									size_t& length, entry_ref& ref,
+									BFile& file);
+			void				BodyStored(entry_ref& ref, BFile& file,
+									uint32 uid);
+
 			void				DeleteMessage(uint32 uid);
 			void				SetMessageFlags(uint32 uid, uint32 flags);
 
@@ -75,6 +90,9 @@ private:
 			status_t			_WriteUInt32(BNode& node,
 									const char* attribute, uint32 value);
 
+			status_t			_WriteStream(BFile& file, BDataIO& stream,
+									size_t& length);
+
 private:
 	typedef std::hash_map<uint32, uint32> UIDToFlagsMap;
 	typedef std::hash_map<uint32, entry_ref> UIDToRefMap;
@@ -86,7 +104,7 @@ private:
 			uint32				fLastUID;
 			FolderListener*		fListener;
 			UIDToRefMap			fRefMap;
-			UIDToFlagsMap		fUIDMap;
+			UIDToFlagsMap		fFlagsMap;
 };
 
 

src/add-ons/mail_daemon/inbound_protocols/imap/IMAPMailbox.cpp

@@ -24,6 +24,36 @@ IMAPMailbox::~IMAPMailbox()
 }
 
 
+void
+IMAPMailbox::AddMessageEntry(uint32 uid, uint32 flags, uint32 size)
+{
+	fMessageEntries.insert(
+		std::make_pair(uid, MessageFlagsAndSize(flags, size)));
+}
+
+
+uint32
+IMAPMailbox::MessageFlags(uint32 uid) const
+{
+	MessageEntryMap::const_iterator found = fMessageEntries.find(uid);
+	if (found == fMessageEntries.end())
+		return 0;
+
+	return found->second.flags;
+}
+
+
+uint32
+IMAPMailbox::MessageSize(uint32 uid) const
+{
+	MessageEntryMap::const_iterator found = fMessageEntries.find(uid);
+	if (found == fMessageEntries.end())
+		return 0;
+
+	return found->second.size;
+}
+
+
 uint32
 IMAPMailbox::MessageAdded(const MessageToken& fromToken, const entry_ref& ref)
 {

src/add-ons/mail_daemon/inbound_protocols/imap/IMAPMailbox.h

@@ -22,6 +22,11 @@ public:
 
 			const BString&		MailboxName() const { return fMailboxName; }
 
+			void				AddMessageEntry(uint32 uid, uint32 flags,
+									uint32 size);
+			uint32				MessageFlags(uint32 uid) const;
+			uint32				MessageSize(uint32 uid) const;
+
 	// FolderListener interface
 	virtual	uint32				MessageAdded(const MessageToken& fromToken,
 									const entry_ref& ref);
@@ -32,8 +37,22 @@ public:
 									uint32 newFlags);
 
 protected:
+	struct MessageFlagsAndSize {
+		MessageFlagsAndSize(uint32 _flags, uint32 _size)
+			:
+			flags(_flags),
+			size(_size)
+		{
+		}
+
+		uint32	flags;
+		uint32	size;
+	};
+	typedef std::hash_map<uint32, MessageFlagsAndSize> MessageEntryMap;
+
 			IMAP::Protocol&		fProtocol;
 			BString				fMailboxName;
+			MessageEntryMap		fMessageEntries;
 };
 
 

src/add-ons/mail_daemon/inbound_protocols/imap/IMAPProtocol.cpp

@@ -123,7 +123,8 @@ IMAPProtocol::WorkerQuit(IMAPConnectionWorker* worker)
 
 
 void
-IMAPProtocol::MessageStored(entry_ref& ref, BFile& stream, uint32 fetchFlags)
+IMAPProtocol::MessageStored(IMAPFolder& folder, entry_ref& ref, BFile& stream,
+	uint32 fetchFlags)
 {
 	if ((fetchFlags & IMAP::kFetchHeader) != 0)
 		NotifyHeaderFetched(ref, &stream);

src/add-ons/mail_daemon/inbound_protocols/imap/IMAPProtocol.h

@@ -34,7 +34,8 @@ public:
 									IMAP::Protocol& protocol, bool idle);
 			void				WorkerQuit(IMAPConnectionWorker* worker);
 
-			void				MessageStored(entry_ref& ref, BFile& stream,
+			void				MessageStored(IMAPFolder& folder,
+									entry_ref& ref, BFile& stream,
 									uint32 fetchFlags);
 
 	virtual	status_t			SyncMessages();

src/add-ons/mail_daemon/inbound_protocols/imap/imap_lib/Commands.h

@@ -34,12 +34,14 @@ struct MessageEntry {
 typedef std::vector<MessageEntry> MessageEntryList;
 
 enum MessageFlags {
-	kSeen		= 0x01,
+	kSeen				= 0x01,
-	kAnswered	= 0x02,
+	kAnswered			= 0x02,
-	kFlagged	= 0x04,
+	kFlagged			= 0x04,
-	kDeleted	= 0x08,
+	kDeleted			= 0x08,
-	kDraft		= 0x10
+	kDraft				= 0x10,
 	// \Recent doesn't really have any useful meaning, so we just ignore it
+
+	kServerFlagsMask	= 0x0000ffff
 };