diff --git a/backend/pop3/src/main/java/com/fsck/k9/backend/pop3/CommandSetFlag.java b/backend/pop3/src/main/java/com/fsck/k9/backend/pop3/CommandSetFlag.java deleted file mode 100644 index 8d0f06b668d..00000000000 --- a/backend/pop3/src/main/java/com/fsck/k9/backend/pop3/CommandSetFlag.java +++ /dev/null @@ -1,47 +0,0 @@ -package com.fsck.k9.backend.pop3; - - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import com.fsck.k9.mail.Flag; -import com.fsck.k9.mail.MessagingException; -import com.fsck.k9.mail.store.pop3.Pop3Folder; -import com.fsck.k9.mail.store.pop3.Pop3Message; -import com.fsck.k9.mail.store.pop3.Pop3Store; -import org.jetbrains.annotations.NotNull; - - -class CommandSetFlag { - private final Pop3Store pop3Store; - - - CommandSetFlag(Pop3Store pop3Store) { - this.pop3Store = pop3Store; - } - - void setFlag(@NotNull String folderServerId, @NotNull List messageServerIds, @NotNull Flag flag, - boolean newState) throws MessagingException { - - Pop3Folder remoteFolder = pop3Store.getFolder(folderServerId); - if (!remoteFolder.isFlagSupported(flag)) { - return; - } - - try { - remoteFolder.open(); - List messages = new ArrayList<>(); - for (String uid : messageServerIds) { - messages.add(remoteFolder.getMessage(uid)); - } - - if (messages.isEmpty()) { - return; - } - remoteFolder.setFlags(messages, Collections.singleton(flag), newState); - } finally { - remoteFolder.close(); - } - } -} diff --git a/backend/pop3/src/main/java/com/fsck/k9/backend/pop3/CommandSetFlag.kt b/backend/pop3/src/main/java/com/fsck/k9/backend/pop3/CommandSetFlag.kt new file mode 100644 index 00000000000..daf236ad822 --- /dev/null +++ b/backend/pop3/src/main/java/com/fsck/k9/backend/pop3/CommandSetFlag.kt @@ -0,0 +1,34 @@ +package com.fsck.k9.backend.pop3 + +import com.fsck.k9.mail.Flag +import com.fsck.k9.mail.MessagingException +import com.fsck.k9.mail.store.pop3.Pop3Store + +internal class CommandSetFlag(private val pop3Store: Pop3Store) { + + @Throws(MessagingException::class) + fun setFlag( + folderServerId: String, + messageServerIds: List, + flag: Flag, + newState: Boolean, + ) { + val folder = pop3Store.getFolder(folderServerId) + if (!folder.isFlagSupported(flag)) { + return + } + + try { + folder.open() + + val messages = messageServerIds.map { folder.getMessage(it) } + if (messages.isEmpty()) { + return + } + + folder.setFlags(messages, setOf(flag), newState) + } finally { + folder.close() + } + } +} diff --git a/backend/pop3/src/main/java/com/fsck/k9/backend/pop3/Pop3Sync.java b/backend/pop3/src/main/java/com/fsck/k9/backend/pop3/Pop3Sync.java deleted file mode 100644 index 8eae32f22b3..00000000000 --- a/backend/pop3/src/main/java/com/fsck/k9/backend/pop3/Pop3Sync.java +++ /dev/null @@ -1,593 +0,0 @@ -package com.fsck.k9.backend.pop3; - - -import com.fsck.k9.backend.api.BackendFolder; -import com.fsck.k9.backend.api.BackendFolder.MoreMessages; -import com.fsck.k9.backend.api.BackendStorage; -import com.fsck.k9.backend.api.SyncConfig; -import com.fsck.k9.backend.api.SyncListener; -import com.fsck.k9.helper.ExceptionHelper; -import com.fsck.k9.logging.Timber; -import com.fsck.k9.mail.AuthenticationFailedException; -import com.fsck.k9.mail.FetchProfile; -import com.fsck.k9.mail.Flag; -import com.fsck.k9.mail.MessageDownloadState; -import com.fsck.k9.mail.MessageRetrievalListener; -import com.fsck.k9.mail.MessagingException; -import com.fsck.k9.mail.store.pop3.Pop3Folder; -import com.fsck.k9.mail.store.pop3.Pop3Message; -import com.fsck.k9.mail.store.pop3.Pop3Store; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; - - -class Pop3Sync { - private static final String EXTRA_LATEST_OLD_MESSAGE_SEEN_TIME = "latestOldMessageSeenTime"; - - private final String accountName; - private final BackendStorage backendStorage; - private final Pop3Store remoteStore; - - - Pop3Sync(String accountName, BackendStorage backendStorage, Pop3Store pop3Store) { - this.accountName = accountName; - this.backendStorage = backendStorage; - this.remoteStore = pop3Store; - } - - void sync(String folder, SyncConfig syncConfig, SyncListener listener) { - synchronizeMailboxSynchronous(folder, syncConfig, listener); - } - - void synchronizeMailboxSynchronous(String folder, SyncConfig syncConfig, SyncListener listener) { - Pop3Folder remoteFolder = null; - - Timber.i("Synchronizing folder %s:%s", accountName, folder); - - BackendFolder backendFolder = null; - try { - Timber.d("SYNC: About to process pending commands for account %s", accountName); - - Timber.v("SYNC: About to get local folder %s", folder); - backendFolder = backendStorage.getFolder(folder); - - listener.syncStarted(folder); - - /* - * Get the message list from the local store and create an index of - * the uids within the list. - */ - - Map localUidMap = backendFolder.getAllMessagesAndEffectiveDates(); - - Timber.v("SYNC: About to get remote folder %s", folder); - remoteFolder = remoteStore.getFolder(folder); - - /* - * Synchronization process: - * - Open the folder - Upload any local messages that are marked as PENDING_UPLOAD (Drafts, Sent, Trash) - Get the message count - Get the list of the newest K9.DEFAULT_VISIBLE_LIMIT messages - getMessages(messageCount - K9.DEFAULT_VISIBLE_LIMIT, messageCount) - See if we have each message locally, if not fetch it's flags and envelope - Get and update the unread count for the folder - Update the remote flags of any messages we have locally with an internal date newer than the remote message. - Get the current flags for any messages we have locally but did not just download - Update local flags - For any message we have locally but not remotely, delete the local message to keep cache clean. - Download larger parts of any new messages. - (Optional) Download small attachments in the background. - */ - - /* - * Open the remote folder. This pre-loads certain metadata like message count. - */ - Timber.v("SYNC: About to open remote folder %s", folder); - - remoteFolder.open(); - - listener.syncAuthenticationSuccess(); - - /* - * Get the remote message count. - */ - int remoteMessageCount = remoteFolder.getMessageCount(); - - int visibleLimit = backendFolder.getVisibleLimit(); - - if (visibleLimit < 0) { - visibleLimit = syncConfig.getDefaultVisibleLimit(); - } - - final List remoteMessages = new ArrayList<>(); - Map remoteUidMap = new HashMap<>(); - - Timber.v("SYNC: Remote message count for folder %s is %d", folder, remoteMessageCount); - - final Date earliestDate = syncConfig.getEarliestPollDate(); - long earliestTimestamp = earliestDate != null ? earliestDate.getTime() : 0L; - - - int remoteStart = 1; - if (remoteMessageCount > 0) { - /* Message numbers start at 1. */ - if (visibleLimit > 0) { - remoteStart = Math.max(0, remoteMessageCount - visibleLimit) + 1; - } else { - remoteStart = 1; - } - - Timber.v("SYNC: About to get messages %d through %d for folder %s", - remoteStart, remoteMessageCount, folder); - - final AtomicInteger headerProgress = new AtomicInteger(0); - listener.syncHeadersStarted(folder); - - - List remoteMessageArray = - remoteFolder.getMessages(remoteStart, remoteMessageCount, null); - - int messageCount = remoteMessageArray.size(); - - for (Pop3Message thisMess : remoteMessageArray) { - headerProgress.incrementAndGet(); - listener.syncHeadersProgress(folder, headerProgress.get(), messageCount); - - Long localMessageTimestamp = localUidMap.get(thisMess.getUid()); - if (localMessageTimestamp == null || localMessageTimestamp >= earliestTimestamp) { - remoteMessages.add(thisMess); - remoteUidMap.put(thisMess.getUid(), thisMess); - } - } - - Timber.v("SYNC: Got %d messages for folder %s", remoteUidMap.size(), folder); - - listener.syncHeadersFinished(folder, headerProgress.get(), remoteUidMap.size()); - } else if (remoteMessageCount < 0) { - throw new Exception("Message count " + remoteMessageCount + " for folder " + folder); - } - - /* - * Remove any messages that are in the local store but no longer on the remote store or are too old - */ - MoreMessages moreMessages = backendFolder.getMoreMessages(); - if (syncConfig.getSyncRemoteDeletions()) { - List destroyMessageUids = new ArrayList<>(); - for (String localMessageUid : localUidMap.keySet()) { - if (remoteUidMap.get(localMessageUid) == null) { - destroyMessageUids.add(localMessageUid); - } - } - - if (!destroyMessageUids.isEmpty()) { - moreMessages = MoreMessages.UNKNOWN; - - backendFolder.destroyMessages(destroyMessageUids); - for (String uid : destroyMessageUids) { - listener.syncRemovedMessage(folder, uid); - } - } - - } - // noinspection UnusedAssignment, free memory early? (better break up the method!) - localUidMap = null; - - if (moreMessages == MoreMessages.UNKNOWN) { - updateMoreMessages(remoteFolder, backendFolder, remoteStart); - } - - /* - * Now we download the actual content of messages. - */ - int newMessages = downloadMessages(syncConfig, remoteFolder, backendFolder, remoteMessages, - listener); - - listener.folderStatusChanged(folder); - - /* Notify listeners that we're finally done. */ - - backendFolder.setLastChecked(System.currentTimeMillis()); - backendFolder.setStatus(null); - - Timber.d("Done synchronizing folder %s:%s @ %tc with %d new messages", - accountName, - folder, - System.currentTimeMillis(), - newMessages); - - listener.syncFinished(folder); - - Timber.i("Done synchronizing folder %s:%s", accountName, folder); - - } catch (AuthenticationFailedException e) { - listener.syncFailed(folder, "Authentication failure", e); - } catch (Exception e) { - Timber.e(e, "synchronizeMailbox"); - // If we don't set the last checked, it can try too often during - // failure conditions - String rootMessage = ExceptionHelper.getRootCauseMessage(e); - if (backendFolder != null) { - try { - backendFolder.setStatus(rootMessage); - backendFolder.setLastChecked(System.currentTimeMillis()); - } catch (Exception e1) { - Timber.e(e1, "Could not set last checked on folder %s:%s", accountName, folder); - } - } - - listener.syncFailed(folder, rootMessage, e); - - Timber.e("Failed synchronizing folder %s:%s @ %tc", accountName, folder, - System.currentTimeMillis()); - - } finally { - if (remoteFolder != null) { - remoteFolder.close(); - } - } - } - - private void updateMoreMessages(Pop3Folder remoteFolder, BackendFolder backendFolder, - int remoteStart) { - - if (remoteStart == 1) { - backendFolder.setMoreMessages(MoreMessages.FALSE); - } else { - boolean moreMessagesAvailable = remoteFolder.areMoreMessagesAvailable(remoteStart); - - MoreMessages newMoreMessages = (moreMessagesAvailable) ? MoreMessages.TRUE : MoreMessages.FALSE; - backendFolder.setMoreMessages(newMoreMessages); - } - } - - private int downloadMessages(final SyncConfig syncConfig, final Pop3Folder remoteFolder, - final BackendFolder backendFolder, List inputMessages, - final SyncListener listener) throws MessagingException { - - final Date earliestDate = syncConfig.getEarliestPollDate(); - Date downloadStarted = new Date(); // now - - if (earliestDate != null) { - Timber.d("Only syncing messages after %s", earliestDate); - } - final String folder = remoteFolder.getServerId(); - - List syncFlagMessages = new ArrayList<>(); - List unsyncedMessages = new ArrayList<>(); - final AtomicInteger newMessages = new AtomicInteger(0); - - List messages = new ArrayList<>(inputMessages); - - for (Pop3Message message : messages) { - evaluateMessageForDownload(message, folder, backendFolder, unsyncedMessages, syncFlagMessages, listener); - } - - final AtomicInteger progress = new AtomicInteger(0); - final int todo = unsyncedMessages.size() + syncFlagMessages.size(); - listener.syncProgress(folder, progress.get(), todo); - - Timber.d("SYNC: Have %d unsynced messages", unsyncedMessages.size()); - - messages.clear(); - final List largeMessages = new ArrayList<>(); - final List smallMessages = new ArrayList<>(); - if (!unsyncedMessages.isEmpty()) { - int visibleLimit = backendFolder.getVisibleLimit(); - int listSize = unsyncedMessages.size(); - - if ((visibleLimit > 0) && (listSize > visibleLimit)) { - unsyncedMessages = unsyncedMessages.subList(0, visibleLimit); - } - - FetchProfile fp = new FetchProfile(); - fp.add(FetchProfile.Item.ENVELOPE); - - Timber.d("SYNC: About to fetch %d unsynced messages for folder %s", unsyncedMessages.size(), folder); - - fetchUnsyncedMessages(syncConfig, remoteFolder, unsyncedMessages, smallMessages, largeMessages, progress, - todo, fp, listener); - - Timber.d("SYNC: Synced unsynced messages for folder %s", folder); - } - - Timber.d("SYNC: Have %d large messages and %d small messages out of %d unsynced messages", - largeMessages.size(), smallMessages.size(), unsyncedMessages.size()); - - unsyncedMessages.clear(); - /* - * Grab the content of the small messages first. This is going to - * be very fast and at very worst will be a single up of a few bytes and a single - * download of 625k. - */ - FetchProfile fp = new FetchProfile(); - //TODO: Only fetch small and large messages if we have some - fp.add(FetchProfile.Item.BODY); - // fp.add(FetchProfile.Item.FLAGS); - // fp.add(FetchProfile.Item.ENVELOPE); - downloadSmallMessages(remoteFolder, backendFolder, smallMessages, progress, newMessages, todo, fp, listener); - smallMessages.clear(); - /* - * Now do the large messages that require more round trips. - */ - fp = new FetchProfile(); - fp.add(FetchProfile.Item.STRUCTURE); - downloadLargeMessages(syncConfig, remoteFolder, backendFolder, largeMessages, progress, newMessages, todo, fp, listener); - largeMessages.clear(); - - Timber.d("SYNC: Synced remote messages for folder %s, %d new messages", folder, newMessages.get()); - - // If the oldest message seen on this sync is newer than the oldest message seen on the previous sync, then - // we want to move our high-water mark forward. - Date oldestMessageTime = backendFolder.getOldestMessageDate(); - if (oldestMessageTime != null) { - if (oldestMessageTime.before(downloadStarted) && - oldestMessageTime.after(getLatestOldMessageSeenTime(backendFolder))) { - setLatestOldMessageSeenTime(backendFolder, oldestMessageTime); - } - } - - return newMessages.get(); - } - - private Date getLatestOldMessageSeenTime(BackendFolder backendFolder) { - Long latestOldMessageSeenTime = backendFolder.getFolderExtraNumber(EXTRA_LATEST_OLD_MESSAGE_SEEN_TIME); - long timestamp = latestOldMessageSeenTime != null ? latestOldMessageSeenTime : 0L; - return new Date(timestamp); - } - - private void setLatestOldMessageSeenTime(BackendFolder backendFolder, Date oldestMessageTime) { - backendFolder.setFolderExtraNumber(EXTRA_LATEST_OLD_MESSAGE_SEEN_TIME, oldestMessageTime.getTime()); - } - - private void evaluateMessageForDownload( - final Pop3Message message, - final String folder, - final BackendFolder backendFolder, - final List unsyncedMessages, - final List syncFlagMessages, - SyncListener listener) { - - String messageServerId = message.getUid(); - if (message.isSet(Flag.DELETED)) { - Timber.v("Message with uid %s is marked as deleted", messageServerId); - - syncFlagMessages.add(message); - return; - } - - boolean messagePresentLocally = backendFolder.isMessagePresent(messageServerId); - - if (!messagePresentLocally) { - if (!message.isSet(Flag.X_DOWNLOADED_FULL) && !message.isSet(Flag.X_DOWNLOADED_PARTIAL)) { - Timber.v("Message with uid %s has not yet been downloaded", messageServerId); - - unsyncedMessages.add(message); - } else { - Timber.v("Message with uid %s is partially or fully downloaded", messageServerId); - - // Store the updated message locally - boolean completeMessage = message.isSet(Flag.X_DOWNLOADED_FULL); - if (completeMessage) { - backendFolder.saveMessage(message, MessageDownloadState.FULL); - } else { - backendFolder.saveMessage(message, MessageDownloadState.PARTIAL); - } - - boolean isOldMessage = isOldMessage(backendFolder, message); - listener.syncNewMessage(folder, messageServerId, isOldMessage); - } - return; - } - - Set messageFlags = backendFolder.getMessageFlags(messageServerId); - if (!messageFlags.contains(Flag.DELETED)) { - Timber.v("Message with uid %s is present in the local store", messageServerId); - - if (!messageFlags.contains(Flag.X_DOWNLOADED_FULL) && !messageFlags.contains(Flag.X_DOWNLOADED_PARTIAL)) { - Timber.v("Message with uid %s is not downloaded, even partially; trying again", messageServerId); - - unsyncedMessages.add(message); - } else { - syncFlagMessages.add(message); - } - } else { - Timber.v("Local copy of message with uid %s is marked as deleted", messageServerId); - } - } - - private void fetchUnsyncedMessages(final SyncConfig syncConfig, final Pop3Folder remoteFolder, - List unsyncedMessages, - final List smallMessages, - final List largeMessages, - final AtomicInteger progress, - final int todo, - FetchProfile fp, - final SyncListener listener) throws MessagingException { - final String folder = remoteFolder.getServerId(); - - final Date earliestDate = syncConfig.getEarliestPollDate(); - remoteFolder.fetch(unsyncedMessages, fp, - new MessageRetrievalListener() { - @Override - public void messageFinished(Pop3Message message) { - try { - if (message.isSet(Flag.DELETED) || message.olderThan(earliestDate)) { - if (message.isSet(Flag.DELETED)) { - Timber.v("Newly downloaded message %s:%s:%s was marked deleted on server, " + - "skipping", accountName, folder, message.getUid()); - } else { - Timber.d("Newly downloaded message %s is older than %s, skipping", - message.getUid(), earliestDate); - } - - progress.incrementAndGet(); - - //TODO: This might be the source of poll count errors in the UI. Is todo always the same as ofTotal - listener.syncProgress(folder, progress.get(), todo); - return; - } - - if (syncConfig.getMaximumAutoDownloadMessageSize() > 0 && - message.getSize() > syncConfig.getMaximumAutoDownloadMessageSize()) { - largeMessages.add(message); - } else { - smallMessages.add(message); - } - } catch (Exception e) { - Timber.e(e, "Error while storing downloaded message."); - } - } - }, - syncConfig.getMaximumAutoDownloadMessageSize()); - } - - private void downloadSmallMessages( - final Pop3Folder remoteFolder, - final BackendFolder backendFolder, - List smallMessages, - final AtomicInteger progress, - final AtomicInteger newMessages, - final int todo, - FetchProfile fp, - final SyncListener listener) throws MessagingException { - final String folder = remoteFolder.getServerId(); - - Timber.d("SYNC: Fetching %d small messages for folder %s", smallMessages.size(), folder); - - remoteFolder.fetch(smallMessages, - fp, new MessageRetrievalListener() { - @Override - public void messageFinished(final Pop3Message message) { - try { - - // Store the updated message locally - backendFolder.saveMessage(message, MessageDownloadState.FULL); - progress.incrementAndGet(); - - // Increment the number of "new messages" if the newly downloaded message is - // not marked as read. - if (!message.isSet(Flag.SEEN)) { - newMessages.incrementAndGet(); - } - - String messageServerId = message.getUid(); - Timber.v("About to notify listeners that we got a new small message %s:%s:%s", - accountName, folder, messageServerId); - - // Update the listener with what we've found - listener.syncProgress(folder, progress.get(), todo); - - boolean isOldMessage = isOldMessage(backendFolder, message); - listener.syncNewMessage(folder, messageServerId, isOldMessage); - } catch (Exception e) { - Timber.e(e, "SYNC: fetch small messages"); - } - } - }, - -1); - - Timber.d("SYNC: Done fetching small messages for folder %s", folder); - } - - private boolean isOldMessage(BackendFolder backendFolder, Pop3Message message) { - return message.olderThan(getLatestOldMessageSeenTime(backendFolder)); - } - - private void downloadLargeMessages( - final SyncConfig syncConfig, - final Pop3Folder remoteFolder, - final BackendFolder backendFolder, - List largeMessages, - final AtomicInteger progress, - final AtomicInteger newMessages, - final int todo, - FetchProfile fp, - SyncListener listener) throws MessagingException { - final String folder = remoteFolder.getServerId(); - - Timber.d("SYNC: Fetching large messages for folder %s", folder); - - int maxDownloadSize = syncConfig.getMaximumAutoDownloadMessageSize(); - remoteFolder.fetch(largeMessages, fp, null, maxDownloadSize); - for (Pop3Message message : largeMessages) { - - downloadSaneBody(syncConfig, remoteFolder, backendFolder, message); - - String messageServerId = message.getUid(); - Timber.v("About to notify listeners that we got a new large message %s:%s:%s", - accountName, folder, messageServerId); - - // Update the listener with what we've found - progress.incrementAndGet(); - - // TODO do we need to re-fetch this here? - Set flags = backendFolder.getMessageFlags(messageServerId); - // Increment the number of "new messages" if the newly downloaded message is - // not marked as read. - if (!flags.contains(Flag.SEEN)) { - newMessages.incrementAndGet(); - } - - listener.syncProgress(folder, progress.get(), todo); - - boolean isOldMessage = isOldMessage(backendFolder, message); - listener.syncNewMessage(folder, messageServerId, isOldMessage); - } - - Timber.d("SYNC: Done fetching large messages for folder %s", folder); - } - - private void downloadSaneBody(SyncConfig syncConfig, Pop3Folder remoteFolder, BackendFolder backendFolder, - Pop3Message message) throws MessagingException { - /* - * The provider was unable to get the structure of the message, so - * we'll download a reasonable portion of the message and mark it as - * incomplete so the entire thing can be downloaded later if the user - * wishes to download it. - */ - FetchProfile fp = new FetchProfile(); - fp.add(FetchProfile.Item.BODY_SANE); - /* - * TODO a good optimization here would be to make sure that all Stores set - * the proper size after this fetch and compare the before and after size. If - * they equal we can mark this SYNCHRONIZED instead of PARTIALLY_SYNCHRONIZED - */ - - int maxDownloadSize = syncConfig.getMaximumAutoDownloadMessageSize(); - remoteFolder.fetch(Collections.singletonList(message), fp, null, maxDownloadSize); - - boolean completeMessage = false; - // Certain (POP3) servers give you the whole message even when you ask for only the first x Kb - if (!message.isSet(Flag.X_DOWNLOADED_FULL)) { - /* - * Mark the message as fully downloaded if the message size is smaller than - * the account's autodownload size limit, otherwise mark as only a partial - * download. This will prevent the system from downloading the same message - * twice. - * - * If there is no limit on autodownload size, that's the same as the message - * being smaller than the max size - */ - if (syncConfig.getMaximumAutoDownloadMessageSize() == 0 - || message.getSize() < syncConfig.getMaximumAutoDownloadMessageSize()) { - completeMessage = true; - } - } - - // Store the updated message locally - if (completeMessage) { - backendFolder.saveMessage(message, MessageDownloadState.FULL); - } else { - backendFolder.saveMessage(message, MessageDownloadState.PARTIAL); - } - } -} diff --git a/backend/pop3/src/main/java/com/fsck/k9/backend/pop3/Pop3Sync.kt b/backend/pop3/src/main/java/com/fsck/k9/backend/pop3/Pop3Sync.kt new file mode 100644 index 00000000000..1517d2e0f4b --- /dev/null +++ b/backend/pop3/src/main/java/com/fsck/k9/backend/pop3/Pop3Sync.kt @@ -0,0 +1,659 @@ +package com.fsck.k9.backend.pop3 + +import com.fsck.k9.backend.api.BackendFolder +import com.fsck.k9.backend.api.BackendStorage +import com.fsck.k9.backend.api.SyncConfig +import com.fsck.k9.backend.api.SyncListener +import com.fsck.k9.helper.ExceptionHelper +import com.fsck.k9.logging.Timber +import com.fsck.k9.mail.AuthenticationFailedException +import com.fsck.k9.mail.FetchProfile +import com.fsck.k9.mail.Flag +import com.fsck.k9.mail.MessageDownloadState +import com.fsck.k9.mail.MessageRetrievalListener +import com.fsck.k9.mail.MessagingException +import com.fsck.k9.mail.store.pop3.Pop3Folder +import com.fsck.k9.mail.store.pop3.Pop3Message +import com.fsck.k9.mail.store.pop3.Pop3Store +import java.lang.Exception +import java.util.ArrayList +import java.util.Date +import java.util.HashMap +import java.util.concurrent.atomic.AtomicInteger + +@Suppress("TooManyFunctions") +internal class Pop3Sync( + private val accountName: String, + private val backendStorage: BackendStorage, + private val remoteStore: Pop3Store, +) { + + fun sync(folder: String, syncConfig: SyncConfig, listener: SyncListener) { + synchronizeMailboxSynchronous(folder, syncConfig, listener) + } + + @Suppress( + "TooGenericExceptionCaught", + "TooGenericExceptionThrown", + "LongMethod", + "CyclomaticComplexMethod", + "NestedBlockDepth", + ) + fun synchronizeMailboxSynchronous(folder: String, syncConfig: SyncConfig, listener: SyncListener) { + var remoteFolder: Pop3Folder? = null + + Timber.i("Synchronizing folder %s:%s", accountName, folder) + + var backendFolder: BackendFolder? = null + try { + Timber.d("SYNC: About to process pending commands for account %s", accountName) + + Timber.v("SYNC: About to get local folder %s", folder) + backendFolder = backendStorage.getFolder(folder) + + listener.syncStarted(folder) + + /* + * Get the message list from the local store and create an index of + * the uids within the list. + */ + var localUidMap: Map = backendFolder.getAllMessagesAndEffectiveDates() + + Timber.v("SYNC: About to get remote folder %s", folder) + remoteFolder = remoteStore.getFolder(folder) + + /* + * Synchronization process: + * + Open the folder + Upload any local messages that are marked as PENDING_UPLOAD (Drafts, Sent, Trash) + Get the message count + Get the list of the newest K9.DEFAULT_VISIBLE_LIMIT messages + getMessages(messageCount - K9.DEFAULT_VISIBLE_LIMIT, messageCount) + See if we have each message locally, if not fetch it's flags and envelope + Get and update the unread count for the folder + Update the remote flags of any messages we have locally with an internal date newer than the remote message. + Get the current flags for any messages we have locally but did not just download + Update local flags + For any message we have locally but not remotely, delete the local message to keep cache clean. + Download larger parts of any new messages. + (Optional) Download small attachments in the background. + */ + + /* + * Open the remote folder. This pre-loads certain metadata like message count. + */ + Timber.v("SYNC: About to open remote folder %s", folder) + + remoteFolder.open() + + listener.syncAuthenticationSuccess() + + /* + * Get the remote message count. + */ + val remoteMessageCount = remoteFolder.messageCount + + var visibleLimit = backendFolder.visibleLimit + + if (visibleLimit < 0) { + visibleLimit = syncConfig.defaultVisibleLimit + } + + val remoteMessages: MutableList = ArrayList() + val remoteUidMap: MutableMap = HashMap() + + Timber.v("SYNC: Remote message count for folder %s is %d", folder, remoteMessageCount) + + val earliestDate = syncConfig.earliestPollDate + val earliestTimestamp = if (earliestDate != null) earliestDate.time else 0L + + /* Message numbers start at 1. */ + var remoteStart = 1 + if (remoteMessageCount > 0) { + // Adjust the starting message number based on the visible limit + if (visibleLimit > 0) { + remoteStart += (remoteMessageCount - visibleLimit).coerceAtLeast(0) + } + + Timber.v( + "SYNC: About to get messages %d through %d for folder %s", + remoteStart, + remoteMessageCount, + folder, + ) + + val headerProgress = AtomicInteger(0) + listener.syncHeadersStarted(folder) + + val remoteMessageArray = remoteFolder.getMessages(remoteStart, remoteMessageCount, null) + + val messageCount = remoteMessageArray.size + + for (thisMess in remoteMessageArray) { + headerProgress.incrementAndGet() + listener.syncHeadersProgress(folder, headerProgress.get(), messageCount) + + val localMessageTimestamp = localUidMap[thisMess.uid] + if (localMessageTimestamp == null || localMessageTimestamp >= earliestTimestamp) { + remoteMessages.add(thisMess) + remoteUidMap.put(thisMess.uid, thisMess) + } + } + + Timber.v("SYNC: Got %d messages for folder %s", remoteUidMap.size, folder) + + listener.syncHeadersFinished(folder, headerProgress.get(), remoteUidMap.size) + } else if (remoteMessageCount < 0) { + throw Exception("Message count $remoteMessageCount for folder $folder") + } + + /* + * Remove any messages that are in the local store but no longer on the remote store or are too old + */ + var moreMessages = backendFolder.getMoreMessages() + if (syncConfig.syncRemoteDeletions) { + val destroyMessageUids: MutableList = ArrayList() + for (localMessageUid in localUidMap.keys) { + if (remoteUidMap[localMessageUid] == null) { + destroyMessageUids.add(localMessageUid) + } + } + + if (!destroyMessageUids.isEmpty()) { + moreMessages = BackendFolder.MoreMessages.UNKNOWN + + backendFolder.destroyMessages(destroyMessageUids) + for (uid in destroyMessageUids) { + listener.syncRemovedMessage(folder, uid) + } + } + } + + if (moreMessages == BackendFolder.MoreMessages.UNKNOWN) { + updateMoreMessages(remoteFolder, backendFolder, remoteStart) + } + + /* + * Now we download the actual content of messages. + */ + val newMessages = downloadMessages( + syncConfig, + remoteFolder, + backendFolder, + remoteMessages, + listener, + ) + + listener.folderStatusChanged(folder) + + /* Notify listeners that we're finally done. */ + backendFolder.setLastChecked(System.currentTimeMillis()) + backendFolder.setStatus(null) + + Timber.d( + "Done synchronizing folder %s:%s @ %tc with %d new messages", + accountName, + folder, + System.currentTimeMillis(), + newMessages, + ) + + listener.syncFinished(folder) + + Timber.i("Done synchronizing folder %s:%s", accountName, folder) + } catch (e: AuthenticationFailedException) { + listener.syncFailed(folder, "Authentication failure", e) + } catch (e: Exception) { + Timber.e(e, "synchronizeMailbox") + // If we don't set the last checked, it can try too often during + // failure conditions + val rootMessage = ExceptionHelper.getRootCauseMessage(e) + if (backendFolder != null) { + try { + backendFolder.setStatus(rootMessage) + backendFolder.setLastChecked(System.currentTimeMillis()) + } catch (e1: Exception) { + Timber.e(e1, "Could not set last checked on folder %s:%s", accountName, folder) + } + } + + listener.syncFailed(folder, rootMessage, e) + + Timber.e( + "Failed synchronizing folder %s:%s @ %tc", + accountName, + folder, + System.currentTimeMillis(), + ) + } finally { + remoteFolder?.close() + } + } + + private fun updateMoreMessages( + remoteFolder: Pop3Folder, + backendFolder: BackendFolder, + remoteStart: Int, + ) { + if (remoteStart == 1) { + backendFolder.setMoreMessages(BackendFolder.MoreMessages.FALSE) + } else { + val moreMessagesAvailable = remoteFolder.areMoreMessagesAvailable(remoteStart) + + val newMoreMessages = + if ((moreMessagesAvailable)) BackendFolder.MoreMessages.TRUE else BackendFolder.MoreMessages.FALSE + backendFolder.setMoreMessages(newMoreMessages) + } + } + + @Suppress("TooGenericExceptionCaught", "LongMethod") + @Throws(MessagingException::class) + private fun downloadMessages( + syncConfig: SyncConfig, + remoteFolder: Pop3Folder, + backendFolder: BackendFolder, + inputMessages: MutableList, + listener: SyncListener, + ): Int { + val earliestDate = syncConfig.earliestPollDate + val downloadStarted = Date() // now + + if (earliestDate != null) { + Timber.d("Only syncing messages after %s", earliestDate) + } + val folder = remoteFolder.serverId + + val syncFlagMessages: MutableList = ArrayList() + var unsyncedMessages: MutableList = ArrayList() + val newMessages = AtomicInteger(0) + + val messages: MutableList = ArrayList(inputMessages) + + for (message in messages) { + evaluateMessageForDownload(message, folder, backendFolder, unsyncedMessages, syncFlagMessages, listener) + } + + val progress = AtomicInteger(0) + val todo = unsyncedMessages.size + syncFlagMessages.size + listener.syncProgress(folder, progress.get(), todo) + + Timber.d("SYNC: Have %d unsynced messages", unsyncedMessages.size) + + messages.clear() + val largeMessages: MutableList = ArrayList() + val smallMessages: MutableList = ArrayList() + if (!unsyncedMessages.isEmpty()) { + val visibleLimit = backendFolder.visibleLimit + val listSize = unsyncedMessages.size + + if ((visibleLimit > 0) && (listSize > visibleLimit)) { + unsyncedMessages = unsyncedMessages.subList(0, visibleLimit) + } + + val fp = FetchProfile() + fp.add(FetchProfile.Item.ENVELOPE) + + Timber.d("SYNC: About to fetch %d unsynced messages for folder %s", unsyncedMessages.size, folder) + + fetchUnsyncedMessages( + syncConfig, remoteFolder, unsyncedMessages, smallMessages, largeMessages, progress, + todo, fp, listener, + ) + + Timber.d("SYNC: Synced unsynced messages for folder %s", folder) + } + + Timber.d( + "SYNC: Have %d large messages and %d small messages out of %d unsynced messages", + largeMessages.size, + smallMessages.size, + unsyncedMessages.size, + ) + + unsyncedMessages.clear() + /* + * Grab the content of the small messages first. This is going to + * be very fast and at very worst will be a single up of a few bytes and a single + * download of 625k. + */ + var fp = FetchProfile() + // TODO: Only fetch small and large messages if we have some + fp.add(FetchProfile.Item.BODY) + // fp.add(FetchProfile.Item.FLAGS); + // fp.add(FetchProfile.Item.ENVELOPE); + downloadSmallMessages(remoteFolder, backendFolder, smallMessages, progress, newMessages, todo, fp, listener) + smallMessages.clear() + /* + * Now do the large messages that require more round trips. + */ + fp = FetchProfile() + fp.add(FetchProfile.Item.STRUCTURE) + downloadLargeMessages( + syncConfig, + remoteFolder, + backendFolder, + largeMessages, + progress, + newMessages, + todo, + fp, + listener, + ) + largeMessages.clear() + + Timber.d("SYNC: Synced remote messages for folder %s, %d new messages", folder, newMessages.get()) + + // If the oldest message seen on this sync is newer than the oldest message seen on the previous sync, then + // we want to move our high-water mark forward. + val oldestMessageTime = backendFolder.getOldestMessageDate() + if (oldestMessageTime != null) { + if (oldestMessageTime.before(downloadStarted) && + oldestMessageTime.after(getLatestOldMessageSeenTime(backendFolder)) + ) { + setLatestOldMessageSeenTime(backendFolder, oldestMessageTime) + } + } + + return newMessages.get() + } + + private fun getLatestOldMessageSeenTime(backendFolder: BackendFolder): Date { + val latestOldMessageSeenTime = backendFolder.getFolderExtraNumber(EXTRA_LATEST_OLD_MESSAGE_SEEN_TIME) + val timestamp = if (latestOldMessageSeenTime != null) latestOldMessageSeenTime else 0L + return Date(timestamp) + } + + private fun setLatestOldMessageSeenTime(backendFolder: BackendFolder, oldestMessageTime: Date) { + backendFolder.setFolderExtraNumber(EXTRA_LATEST_OLD_MESSAGE_SEEN_TIME, oldestMessageTime.time) + } + + private fun evaluateMessageForDownload( + message: Pop3Message, + folder: String, + backendFolder: BackendFolder, + unsyncedMessages: MutableList, + syncFlagMessages: MutableList, + listener: SyncListener, + ) { + val messageServerId = message.uid + if (message.isSet(Flag.DELETED)) { + Timber.v("Message with uid %s is marked as deleted", messageServerId) + + syncFlagMessages.add(message) + return + } + + val messagePresentLocally = backendFolder.isMessagePresent(messageServerId) + + if (!messagePresentLocally) { + if (!message.isSet(Flag.X_DOWNLOADED_FULL) && !message.isSet(Flag.X_DOWNLOADED_PARTIAL)) { + Timber.v("Message with uid %s has not yet been downloaded", messageServerId) + + unsyncedMessages.add(message) + } else { + Timber.v("Message with uid %s is partially or fully downloaded", messageServerId) + + // Store the updated message locally + val completeMessage = message.isSet(Flag.X_DOWNLOADED_FULL) + if (completeMessage) { + backendFolder.saveMessage(message, MessageDownloadState.FULL) + } else { + backendFolder.saveMessage(message, MessageDownloadState.PARTIAL) + } + + val isOldMessage = isOldMessage(backendFolder, message) + listener.syncNewMessage(folder, messageServerId, isOldMessage) + } + return + } + + val messageFlags: Set = backendFolder.getMessageFlags(messageServerId) + if (!messageFlags.contains(Flag.DELETED)) { + Timber.v("Message with uid %s is present in the local store", messageServerId) + + if (!messageFlags.contains(Flag.X_DOWNLOADED_FULL) && !messageFlags.contains(Flag.X_DOWNLOADED_PARTIAL)) { + Timber.v("Message with uid %s is not downloaded, even partially; trying again", messageServerId) + + unsyncedMessages.add(message) + } else { + syncFlagMessages.add(message) + } + } else { + Timber.v("Local copy of message with uid %s is marked as deleted", messageServerId) + } + } + + @Suppress("LongParameterList") + @Throws(MessagingException::class) + private fun fetchUnsyncedMessages( + syncConfig: SyncConfig, + remoteFolder: Pop3Folder, + unsyncedMessages: MutableList?, + smallMessages: MutableList, + largeMessages: MutableList, + progress: AtomicInteger, + todo: Int, + fp: FetchProfile?, + listener: SyncListener, + ) { + val folder = remoteFolder.serverId + + val earliestDate = syncConfig.earliestPollDate + remoteFolder.fetch( + unsyncedMessages, + fp, + object : MessageRetrievalListener { + + @Suppress("TooGenericExceptionCaught") + override fun messageFinished(message: Pop3Message) { + try { + if (message.isSet(Flag.DELETED) || message.olderThan(earliestDate)) { + if (message.isSet(Flag.DELETED)) { + Timber.v( + "Newly downloaded message %s:%s:%s was marked deleted on server, " + + "skipping", + accountName, + folder, + message.uid, + ) + } else { + Timber.d( + "Newly downloaded message %s is older than %s, skipping", + message.uid, + earliestDate, + ) + } + + progress.incrementAndGet() + + // TODO: This might be the source of poll count errors in the UI. + // Is todo always the same as ofTotal + listener.syncProgress(folder, progress.get(), todo) + return + } + + if (syncConfig.maximumAutoDownloadMessageSize > 0 && + message.size > syncConfig.maximumAutoDownloadMessageSize + ) { + largeMessages.add(message) + } else { + smallMessages.add(message) + } + } catch (e: Exception) { + Timber.e(e, "Error while storing downloaded message.") + } + } + }, + syncConfig.maximumAutoDownloadMessageSize, + ) + } + + @Suppress("LongParameterList") + @Throws(MessagingException::class) + private fun downloadSmallMessages( + remoteFolder: Pop3Folder, + backendFolder: BackendFolder, + smallMessages: MutableList, + progress: AtomicInteger, + newMessages: AtomicInteger, + todo: Int, + fp: FetchProfile?, + listener: SyncListener, + ) { + val folder = remoteFolder.serverId + + Timber.d("SYNC: Fetching %d small messages for folder %s", smallMessages.size, folder) + + remoteFolder.fetch( + smallMessages, + fp, + object : MessageRetrievalListener { + + @Suppress("TooGenericExceptionCaught") + override fun messageFinished(message: Pop3Message) { + try { + // Store the updated message locally + + backendFolder.saveMessage(message, MessageDownloadState.FULL) + progress.incrementAndGet() + + // Increment the number of "new messages" if the newly downloaded message is + // not marked as read. + if (!message.isSet(Flag.SEEN)) { + newMessages.incrementAndGet() + } + + val messageServerId = message.uid + Timber.v( + "About to notify listeners that we got a new small message %s:%s:%s", + accountName, + folder, + messageServerId, + ) + + // Update the listener with what we've found + listener.syncProgress(folder, progress.get(), todo) + + val isOldMessage = isOldMessage(backendFolder, message) + listener.syncNewMessage(folder, messageServerId, isOldMessage) + } catch (e: Exception) { + Timber.e(e, "SYNC: fetch small messages") + } + } + }, + -1, + ) + + Timber.d("SYNC: Done fetching small messages for folder %s", folder) + } + + private fun isOldMessage(backendFolder: BackendFolder, message: Pop3Message): Boolean { + return message.olderThan(getLatestOldMessageSeenTime(backendFolder)) + } + + @Suppress("LongParameterList") + @Throws(MessagingException::class) + private fun downloadLargeMessages( + syncConfig: SyncConfig, + remoteFolder: Pop3Folder, + backendFolder: BackendFolder, + largeMessages: MutableList, + progress: AtomicInteger, + newMessages: AtomicInteger, + todo: Int, + fp: FetchProfile?, + listener: SyncListener, + ) { + val folder = remoteFolder.serverId + + Timber.d("SYNC: Fetching large messages for folder %s", folder) + + val maxDownloadSize = syncConfig.maximumAutoDownloadMessageSize + remoteFolder.fetch(largeMessages, fp, null, maxDownloadSize) + for (message in largeMessages) { + downloadSaneBody(syncConfig, remoteFolder, backendFolder, message) + + val messageServerId = message.uid + Timber.v( + "About to notify listeners that we got a new large message %s:%s:%s", + accountName, + folder, + messageServerId, + ) + + // Update the listener with what we've found + progress.incrementAndGet() + + // TODO do we need to re-fetch this here? + val flags: Set = backendFolder.getMessageFlags(messageServerId) + // Increment the number of "new messages" if the newly downloaded message is + // not marked as read. + if (!flags.contains(Flag.SEEN)) { + newMessages.incrementAndGet() + } + + listener.syncProgress(folder, progress.get(), todo) + + val isOldMessage = isOldMessage(backendFolder, message) + listener.syncNewMessage(folder, messageServerId, isOldMessage) + } + + Timber.d("SYNC: Done fetching large messages for folder %s", folder) + } + + @Throws(MessagingException::class) + private fun downloadSaneBody( + syncConfig: SyncConfig, + remoteFolder: Pop3Folder, + backendFolder: BackendFolder, + message: Pop3Message, + ) { + /* + * The provider was unable to get the structure of the message, so + * we'll download a reasonable portion of the message and mark it as + * incomplete so the entire thing can be downloaded later if the user + * wishes to download it. + */ + val fp = FetchProfile() + fp.add(FetchProfile.Item.BODY_SANE) + + /* + * TODO a good optimization here would be to make sure that all Stores set + * the proper size after this fetch and compare the before and after size. If + * they equal we can mark this SYNCHRONIZED instead of PARTIALLY_SYNCHRONIZED + */ + val maxDownloadSize = syncConfig.maximumAutoDownloadMessageSize + remoteFolder.fetch(mutableListOf(message), fp, null, maxDownloadSize) + + var completeMessage = false + // Certain (POP3) servers give you the whole message even when you ask for only the first x Kb + if (!message.isSet(Flag.X_DOWNLOADED_FULL)) { + /* + * Mark the message as fully downloaded if the message size is smaller than + * the account's autodownload size limit, otherwise mark as only a partial + * download. This will prevent the system from downloading the same message + * twice. + * + * If there is no limit on autodownload size, that's the same as the message + * being smaller than the max size + */ + if (syncConfig.maximumAutoDownloadMessageSize == 0 || + message.size < syncConfig.maximumAutoDownloadMessageSize + ) { + completeMessage = true + } + } + + // Store the updated message locally + if (completeMessage) { + backendFolder.saveMessage(message, MessageDownloadState.FULL) + } else { + backendFolder.saveMessage(message, MessageDownloadState.PARTIAL) + } + } + + companion object { + private const val EXTRA_LATEST_OLD_MESSAGE_SEEN_TIME = "latestOldMessageSeenTime" + } +} diff --git a/mail/protocols/pop3/src/main/java/com/fsck/k9/mail/store/pop3/Pop3Folder.java b/mail/protocols/pop3/src/main/java/com/fsck/k9/mail/store/pop3/Pop3Folder.java index 5570da63767..78c24b4b15d 100644 --- a/mail/protocols/pop3/src/main/java/com/fsck/k9/mail/store/pop3/Pop3Folder.java +++ b/mail/protocols/pop3/src/main/java/com/fsck/k9/mail/store/pop3/Pop3Folder.java @@ -16,6 +16,7 @@ import com.fsck.k9.mail.K9MailLib; import com.fsck.k9.mail.MessageRetrievalListener; import com.fsck.k9.mail.MessagingException; +import org.jetbrains.annotations.NotNull; import static com.fsck.k9.mail.K9MailLib.DEBUG_PROTOCOL_POP3; import static com.fsck.k9.mail.store.pop3.Pop3Commands.*; @@ -101,6 +102,7 @@ public int getMessageCount() { return messageCount; } + @NotNull public Pop3Message getMessage(String uid) { Pop3Message message = uidToMsgMap.get(uid); if (message == null) {