diff --git a/src/add-ons/mail_daemon/inbound_protocols/imap/IMAPConnectionWorker.cpp b/src/add-ons/mail_daemon/inbound_protocols/imap/IMAPConnectionWorker.cpp index 79ef13bf1e..97add7b291 100644 --- a/src/add-ons/mail_daemon/inbound_protocols/imap/IMAPConnectionWorker.cpp +++ b/src/add-ons/mail_daemon/inbound_protocols/imap/IMAPConnectionWorker.cpp @@ -59,6 +59,11 @@ public: return fWorker._MailboxFor(folder); } + int32 BodyFetchLimit() + { + return fWorker.fSettings.BodyFetchLimit(); + } + status_t EnqueueCommand(WorkerCommand* command) { return fWorker._EnqueueCommand(command); @@ -125,15 +130,78 @@ public: }; +class FetchBodiesCommand : public SyncCommand, public IMAP::FetchListener { +public: + FetchBodiesCommand(IMAPFolder& folder, IMAPMailbox& mailbox, + std::vector& entries) + : + fFolder(folder), + fMailbox(mailbox), + fEntries(entries) + { + } + + virtual status_t Process(IMAPConnectionWorker& worker) + { + IMAP::Protocol& protocol = WorkerPrivate(worker).Protocol(); + + if (fEntries.empty()) + return B_OK; + + fUID = *fEntries.begin(); + fEntries.erase(fEntries.begin()); + + // TODO: check nextUID if we get one + status_t status = WorkerPrivate(worker).SelectMailbox(fFolder); + if (status != B_OK) + return status; + + printf("IMAP: fetch body for %lu\n", fUID); + // TODO: combine smaller messages together for faster retrieval + IMAP::FetchCommand fetch(fUID, fUID, IMAP::kFetchBody); + fetch.SetListener(this); + + return protocol.ProcessCommand(fetch); + } + + virtual bool IsDone() const + { + return fEntries.empty(); + } + + virtual bool FetchData(uint32 fetchFlags, BDataIO& stream, size_t& length) + { + fFetchStatus = fFolder.StoreBody(fUID, stream, length, fRef, fFile); + return true; + } + + virtual void FetchedData(uint32 fetchFlags, uint32 uid, uint32 flags) + { + if (fFetchStatus == B_OK) + fFolder.BodyStored(fRef, fFile, uid); + } + +private: + IMAPFolder& fFolder; + IMAPMailbox& fMailbox; + std::vector fEntries; + uint32 fUID; + entry_ref fRef; + BFile fFile; + status_t fFetchStatus; +}; + + class FetchHeadersCommand : public SyncCommand, public IMAP::FetchListener { public: FetchHeadersCommand(IMAPFolder& folder, IMAPMailbox& mailbox, - uint32 from, uint32 to) + uint32 from, uint32 to, int32 bodyFetchLimit) : fFolder(folder), fMailbox(mailbox), fFrom(from), - fTo(to) + fTo(to), + fBodyFetchLimit(bodyFetchLimit) { } @@ -152,8 +220,6 @@ public: if (to - fFrom >= kMaxFetchEntries) to = fFrom + kMaxFetchEntries - 1; - // TODO: trigger download of mails for all messages below the - // body fetch limit printf("IMAP: fetch headers from %lu to %lu\n", fFrom, to); IMAP::FetchCommand fetch(fFrom, to, IMAP::kFetchHeader | IMAP::kFetchFlags); @@ -164,6 +230,13 @@ public: return status; fFrom = to + 1; + + if (IsDone() && !fFetchBodies.empty()) { + // Enqueue command to fetch the message bodies + WorkerPrivate(worker).EnqueueCommand(new FetchBodiesCommand(fFolder, + fMailbox, fFetchBodies)); + } + return B_OK; } @@ -174,15 +247,20 @@ public: virtual bool FetchData(uint32 fetchFlags, BDataIO& stream, size_t& length) { - fFetchStatus = fFolder.StoreMessage(fFile, fetchFlags, stream, - length, fRef); + fFetchStatus = fFolder.StoreMessage(fetchFlags, stream, length, + fRef, fFile); return true; } virtual void FetchedData(uint32 fetchFlags, uint32 uid, uint32 flags) { - if (fFetchStatus == B_OK) + if (fFetchStatus == B_OK) { fFolder.MessageStored(fRef, fFile, fetchFlags, uid, flags); + + uint32 size = fMailbox.MessageSize(uid); + if (fBodyFetchLimit < 0 || size < fBodyFetchLimit) + fFetchBodies.push_back(uid); + } } private: @@ -190,6 +268,8 @@ private: IMAPMailbox& fMailbox; uint32 fFrom; uint32 fTo; + uint32 fBodyFetchLimit; + std::vector fFetchBodies; entry_ref fRef; BFile fFile; status_t fFetchStatus; @@ -269,9 +349,13 @@ public: return status; // Determine how much we need to download + // TODO: also retrieve the header size, and only take the body + // size into account if it's below the limit for (size_t i = 0; i < entries.size(); i++) { printf("%10lu %8lu bytes, flags: %#lx\n", entries[i].uid, entries[i].size, entries[i].flags); + fMailbox->AddMessageEntry(entries[i].uid, entries[i].flags, + entries[i].size); fTotalBytes += entries[i].size; } fTotalEntries += entries.size(); @@ -283,7 +367,8 @@ public: if (fMailboxEntries > 0) { // Add pending command to fetch the message headers WorkerCommand* command = new FetchHeadersCommand(*fFolder, - *fMailbox, fFirstUID, fNextUID); + *fMailbox, fFirstUID, fNextUID, + WorkerPrivate(worker).BodyFetchLimit()); if (!fFetchCommands.AddItem(command)) delete command; } diff --git a/src/add-ons/mail_daemon/inbound_protocols/imap/IMAPFolder.cpp b/src/add-ons/mail_daemon/inbound_protocols/imap/IMAPFolder.cpp index 6cb060ad2c..a7abf30751 100644 --- a/src/add-ons/mail_daemon/inbound_protocols/imap/IMAPFolder.cpp +++ b/src/add-ons/mail_daemon/inbound_protocols/imap/IMAPFolder.cpp @@ -132,7 +132,7 @@ IMAPFolder::IMAPFolder(IMAPProtocol& protocol, const BString& mailboxName, uint32 uid = B_BENDIAN_TO_HOST_INT32(entries[i].uid); uint32 flags = B_BENDIAN_TO_HOST_INT32(entries[i].flags); - fUIDMap.insert(std::make_pair(uid, flags)); + fFlagsMap.insert(std::make_pair(uid, flags)); } } @@ -167,6 +167,29 @@ IMAPFolder::SetUIDValidity(uint32 uidValidity) } +status_t +IMAPFolder::GetMessageEntryRef(uint32 uid, entry_ref& ref) const +{ + UIDToRefMap::const_iterator found = fRefMap.find(uid); + if (found == fRefMap.end()) + return B_ENTRY_NOT_FOUND; + + ref = found->second; + return B_OK; +} + + +uint32 +IMAPFolder::MessageFlags(uint32 uid) const +{ + UIDToFlagsMap::const_iterator found = fFlagsMap.find(uid); + if (found == fFlagsMap.end()) + return 0; + + return found->second; +} + + /*! Stores the given \a stream into a temporary file using the provided BFile object. A new file will be created, and the \a ref object will point to it. The file will remain open when this method exits without @@ -176,8 +199,8 @@ IMAPFolder::SetUIDValidity(uint32 uidValidity) were an error. */ status_t -IMAPFolder::StoreMessage(BFile& file, uint32 fetchFlags, BDataIO& stream, - size_t& length, entry_ref& ref) +IMAPFolder::StoreMessage(uint32 fetchFlags, BDataIO& stream, + size_t& length, entry_ref& ref, BFile& file) { BPath path; status_t status = path.SetTo(&fRef); @@ -189,26 +212,11 @@ IMAPFolder::StoreMessage(BFile& file, uint32 fetchFlags, BDataIO& stream, if (status != B_OK) return status; - char buffer[65535]; - while (length > 0) { - ssize_t bytesRead = stream.Read(buffer, - std::min(sizeof(buffer), length)); - if (bytesRead < 0) - return bytesRead; - if (bytesRead <= 0) - break; + status = _WriteStream(file, stream, length); + if (status == B_OK) + temporaryFile.KeepFile(); - length -= bytesRead; - - ssize_t bytesWritten = file.Write(buffer, bytesRead); - if (bytesWritten < 0) - return bytesWritten; - if (bytesWritten != bytesRead) - return B_IO_ERROR; - } - - temporaryFile.KeepFile(); - return B_OK; + return status; } @@ -223,29 +231,73 @@ IMAPFolder::MessageStored(entry_ref& ref, BFile& file, uint32 fetchFlags, if ((fetchFlags & IMAP::kFetchFlags) != 0) _WriteFlags(file, flags); - // TODO: the call below may move/rename the file - fProtocol.MessageStored(ref, file, fetchFlags); + // TODO: the call below may move/rename the file - this prevents downloading + // the body to the correct file! + fProtocol.MessageStored(*this, ref, file, fetchFlags); file.Unset(); + fRefMap.insert(std::make_pair(uid, ref)); + if (uid > fLastUID) { // Update last known UID fLastUID = uid; BNode directory(&fRef); status_t status = _WriteUInt32(directory, kLastUIDAttribute, uid); - if (status != B_OK) + if (status != B_OK) { fprintf(stderr, "IMAP: Could not write last UID for mailbox " "%s: %s\n", fMailboxName.String(), strerror(status)); + } } } +/*! Appends the given \a stream as body to the message file for the + specified unique ID. The file will remain open when this method exits + without an error. + + \a length will reflect how many bytes are left to read in case there + were an error. +*/ +status_t +IMAPFolder::StoreBody(uint32 uid, BDataIO& stream, size_t& length, + entry_ref& ref, BFile& file) +{ + status_t status = GetMessageEntryRef(uid, ref); + if (status != B_OK) + return status; + + status = file.SetTo(&ref, B_OPEN_AT_END | B_WRITE_ONLY); + if (status != B_OK) + return status; + + BPath path(&ref); + printf("IMAP: write body to %s\n", path.Path()); + + return _WriteStream(file, stream, length); +} + + +/*! Notifies the protocol that a body has been fetched. + This method also closes the \a file passed in. +*/ +void +IMAPFolder::BodyStored(entry_ref& ref, BFile& file, uint32 uid) +{ + fProtocol.MessageStored(*this, ref, file, IMAP::kFetchBody); + file.Unset(); +} + + void IMAPFolder::DeleteMessage(uint32 uid) { } +/*! Called when the flags of a message changed on the server. This will update + the flags for the local file. +*/ void IMAPFolder::SetMessageFlags(uint32 uid, uint32 flags) { @@ -264,8 +316,8 @@ IMAPFolder::_InitializeFolderState() // Create set of the last known UID state - if an entry is found, it // is being removed from the list. The remaining entries were deleted. std::set lastUIDs; - UIDToFlagsMap::iterator iterator = fUIDMap.begin(); - for (; iterator != fUIDMap.end(); iterator++) + UIDToFlagsMap::iterator iterator = fFlagsMap.begin(); + for (; iterator != fFlagsMap.end(); iterator++) lastUIDs.insert(iterator->first); BDirectory directory(&fRef); @@ -286,12 +338,11 @@ IMAPFolder::_InitializeFolderState() // The message is still around lastUIDs.erase(found); - UIDToFlagsMap::iterator flagsFound = fUIDMap.find(uid); - ASSERT(flagsFound != fUIDMap.end()); - if (flagsFound->second != flags) { + uint32 flagsFound = MessageFlags(uid); + if (flagsFound != flags) { // Its flags have changed locally, and need to be updated fListener->MessageFlagsChanged(_Token(uid), ref, - flagsFound->second, flags); + flagsFound, flags); } } else { // This is a new message @@ -376,3 +427,28 @@ IMAPFolder::_WriteUInt32(BNode& node, const char* attribute, uint32 value) return bytesWritten < 0 ? bytesWritten : B_IO_ERROR; } + + +status_t +IMAPFolder::_WriteStream(BFile& file, BDataIO& stream, size_t& length) +{ + char buffer[65535]; + while (length > 0) { + ssize_t bytesRead = stream.Read(buffer, + std::min(sizeof(buffer), length)); + if (bytesRead < 0) + return bytesRead; + if (bytesRead <= 0) + break; + + length -= bytesRead; + + ssize_t bytesWritten = file.Write(buffer, bytesRead); + if (bytesWritten < 0) + return bytesWritten; + if (bytesWritten != bytesRead) + return B_IO_ERROR; + } + + return B_OK; +} diff --git a/src/add-ons/mail_daemon/inbound_protocols/imap/IMAPFolder.h b/src/add-ons/mail_daemon/inbound_protocols/imap/IMAPFolder.h index 6c9100e127..997b65cc10 100644 --- a/src/add-ons/mail_daemon/inbound_protocols/imap/IMAPFolder.h +++ b/src/add-ons/mail_daemon/inbound_protocols/imap/IMAPFolder.h @@ -24,6 +24,11 @@ struct MessageToken { uint32 uid; }; +// Additional local only message flags +enum FolderMessageFlags { + kPartialMessage = 0x00010000, +}; + class FolderListener { public: @@ -51,13 +56,23 @@ public: uint32 LastUID() const { return fLastUID; } - status_t StoreMessage(BFile& file, uint32 fetchFlags, - BDataIO& stream, size_t& length, - entry_ref& ref); + status_t GetMessageEntryRef(uint32 uid, + entry_ref& ref) const; + uint32 MessageFlags(uint32 uid) const; + + status_t StoreMessage(uint32 fetchFlags, BDataIO& stream, + size_t& length, entry_ref& ref, + BFile& file); void MessageStored(entry_ref& ref, BFile& file, uint32 fetchFlags, uint32 uid, uint32 flags); + status_t StoreBody(uint32 uid, BDataIO& stream, + size_t& length, entry_ref& ref, + BFile& file); + void BodyStored(entry_ref& ref, BFile& file, + uint32 uid); + void DeleteMessage(uint32 uid); void SetMessageFlags(uint32 uid, uint32 flags); @@ -75,6 +90,9 @@ private: status_t _WriteUInt32(BNode& node, const char* attribute, uint32 value); + status_t _WriteStream(BFile& file, BDataIO& stream, + size_t& length); + private: typedef std::hash_map UIDToFlagsMap; typedef std::hash_map UIDToRefMap; @@ -86,7 +104,7 @@ private: uint32 fLastUID; FolderListener* fListener; UIDToRefMap fRefMap; - UIDToFlagsMap fUIDMap; + UIDToFlagsMap fFlagsMap; }; diff --git a/src/add-ons/mail_daemon/inbound_protocols/imap/IMAPMailbox.cpp b/src/add-ons/mail_daemon/inbound_protocols/imap/IMAPMailbox.cpp index 0703147ff3..2c07be5140 100644 --- a/src/add-ons/mail_daemon/inbound_protocols/imap/IMAPMailbox.cpp +++ b/src/add-ons/mail_daemon/inbound_protocols/imap/IMAPMailbox.cpp @@ -24,6 +24,36 @@ IMAPMailbox::~IMAPMailbox() } +void +IMAPMailbox::AddMessageEntry(uint32 uid, uint32 flags, uint32 size) +{ + fMessageEntries.insert( + std::make_pair(uid, MessageFlagsAndSize(flags, size))); +} + + +uint32 +IMAPMailbox::MessageFlags(uint32 uid) const +{ + MessageEntryMap::const_iterator found = fMessageEntries.find(uid); + if (found == fMessageEntries.end()) + return 0; + + return found->second.flags; +} + + +uint32 +IMAPMailbox::MessageSize(uint32 uid) const +{ + MessageEntryMap::const_iterator found = fMessageEntries.find(uid); + if (found == fMessageEntries.end()) + return 0; + + return found->second.size; +} + + uint32 IMAPMailbox::MessageAdded(const MessageToken& fromToken, const entry_ref& ref) { diff --git a/src/add-ons/mail_daemon/inbound_protocols/imap/IMAPMailbox.h b/src/add-ons/mail_daemon/inbound_protocols/imap/IMAPMailbox.h index 46c1a04b0b..352b2de4c9 100644 --- a/src/add-ons/mail_daemon/inbound_protocols/imap/IMAPMailbox.h +++ b/src/add-ons/mail_daemon/inbound_protocols/imap/IMAPMailbox.h @@ -22,6 +22,11 @@ public: const BString& MailboxName() const { return fMailboxName; } + void AddMessageEntry(uint32 uid, uint32 flags, + uint32 size); + uint32 MessageFlags(uint32 uid) const; + uint32 MessageSize(uint32 uid) const; + // FolderListener interface virtual uint32 MessageAdded(const MessageToken& fromToken, const entry_ref& ref); @@ -32,8 +37,22 @@ public: uint32 newFlags); protected: + struct MessageFlagsAndSize { + MessageFlagsAndSize(uint32 _flags, uint32 _size) + : + flags(_flags), + size(_size) + { + } + + uint32 flags; + uint32 size; + }; + typedef std::hash_map MessageEntryMap; + IMAP::Protocol& fProtocol; BString fMailboxName; + MessageEntryMap fMessageEntries; }; diff --git a/src/add-ons/mail_daemon/inbound_protocols/imap/IMAPProtocol.cpp b/src/add-ons/mail_daemon/inbound_protocols/imap/IMAPProtocol.cpp index 6b1c5e56d8..5499c06e19 100644 --- a/src/add-ons/mail_daemon/inbound_protocols/imap/IMAPProtocol.cpp +++ b/src/add-ons/mail_daemon/inbound_protocols/imap/IMAPProtocol.cpp @@ -123,7 +123,8 @@ IMAPProtocol::WorkerQuit(IMAPConnectionWorker* worker) void -IMAPProtocol::MessageStored(entry_ref& ref, BFile& stream, uint32 fetchFlags) +IMAPProtocol::MessageStored(IMAPFolder& folder, entry_ref& ref, BFile& stream, + uint32 fetchFlags) { if ((fetchFlags & IMAP::kFetchHeader) != 0) NotifyHeaderFetched(ref, &stream); diff --git a/src/add-ons/mail_daemon/inbound_protocols/imap/IMAPProtocol.h b/src/add-ons/mail_daemon/inbound_protocols/imap/IMAPProtocol.h index c3c43a339e..531b5c9b17 100644 --- a/src/add-ons/mail_daemon/inbound_protocols/imap/IMAPProtocol.h +++ b/src/add-ons/mail_daemon/inbound_protocols/imap/IMAPProtocol.h @@ -34,7 +34,8 @@ public: IMAP::Protocol& protocol, bool idle); void WorkerQuit(IMAPConnectionWorker* worker); - void MessageStored(entry_ref& ref, BFile& stream, + void MessageStored(IMAPFolder& folder, + entry_ref& ref, BFile& stream, uint32 fetchFlags); virtual status_t SyncMessages(); diff --git a/src/add-ons/mail_daemon/inbound_protocols/imap/imap_lib/Commands.h b/src/add-ons/mail_daemon/inbound_protocols/imap/imap_lib/Commands.h index b47d194680..634d2ecf29 100644 --- a/src/add-ons/mail_daemon/inbound_protocols/imap/imap_lib/Commands.h +++ b/src/add-ons/mail_daemon/inbound_protocols/imap/imap_lib/Commands.h @@ -34,12 +34,14 @@ struct MessageEntry { typedef std::vector MessageEntryList; enum MessageFlags { - kSeen = 0x01, - kAnswered = 0x02, - kFlagged = 0x04, - kDeleted = 0x08, - kDraft = 0x10 + kSeen = 0x01, + kAnswered = 0x02, + kFlagged = 0x04, + kDeleted = 0x08, + kDraft = 0x10, // \Recent doesn't really have any useful meaning, so we just ignore it + + kServerFlagsMask = 0x0000ffff };