Add support for RFCs 5530 and 6855.

5530 (ENABLE) lets a client tell the server which extensions it would like
to use, and lets the server tell the client which extensions is has
enabled. Most IMAP extensions don't need to be enabled, but UTF8=ACCEPT
does.

6855 (UTF8=ACCEPT) requires three things of clients:
1. The client uses ENABLE (added with tests)
2. The client accepts UTF8 strings (worked already, this adds testing)
3. The client cannot use certain SEARCH syntax (tb already did not)

6855 also allows some optimisations that this change does not contain:
1. A client can send UTF8 quoted-strings instead of some literals
2. A client can send UTF8 folder names instead of mUTF7

This change doesn't do either of those, because they adds complexity (and
unit tests) but no functionality. Some IMAP commands become three or seven
or even ten bytes smaller. Who cares?

RFC 6855 differs modestly from its replacement, currently in the
RFC-editor's queue. This implementation sides with the newer RFC when
there's any difference.

This commit contains a couple of unit tests that do nothing right now,
they merely guard against possible future breakage.

mail/protocols/imap/src/main/java/com/fsck/k9/mail/store/imap/Capabilities.java

@@ -19,4 +19,6 @@ class Capabilities {
     public static final String UID_PLUS = "UIDPLUS";
     public static final String LIST_EXTENDED = "LIST-EXTENDED";
     public static final String MOVE = "MOVE";
+    public static final String ENABLE = "ENABLE";
+    public static final String UTF8_ACCEPT = "UTF8=ACCEPT";
 }

mail/protocols/imap/src/main/java/com/fsck/k9/mail/store/imap/Commands.java

@@ -21,4 +21,5 @@ class Commands {
     public static final String UID_COPY = "UID COPY";
     public static final String UID_MOVE = "UID MOVE";
     public static final String UID_EXPUNGE = "UID EXPUNGE";
+    public static final String ENABLE = "ENABLE UTF8=ACCEPT";
 }

mail/protocols/imap/src/main/java/com/fsck/k9/mail/store/imap/EnabledResponse.java

@@ -0,0 +1,52 @@
+package com.fsck.k9.mail.store.imap;
+
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Set;
+
+import static com.fsck.k9.mail.store.imap.ImapResponseParser.equalsIgnoreCase;
+
+
+class EnabledResponse {
+    private final Set<String> capabilities;
+
+
+    private EnabledResponse(Set<String> capabilities) {
+        this.capabilities = Collections.unmodifiableSet(capabilities);
+    }
+
+    public static EnabledResponse parse(List<ImapResponse> responses) {
+        EnabledResponse result = null;
+        for (ImapResponse response : responses)
+            if (result == null && response.getTag() == null)
+                result = parse(response);
+        return result;
+    }
+
+    static EnabledResponse parse(ImapList capabilityList) {
+        if (capabilityList.isEmpty() || !equalsIgnoreCase(capabilityList.get(0), Responses.ENABLED)) {
+            return null;
+        }
+
+        int size = capabilityList.size();
+        HashSet<String> capabilities = new HashSet<>(size - 1);
+
+        for (int i = 1; i < size; i++) {
+            if (!capabilityList.isString(i)) {
+                return null;
+            }
+
+            String uppercaseCapability = capabilityList.getString(i).toUpperCase(Locale.US);
+            capabilities.add(uppercaseCapability);
+        }
+
+        return new EnabledResponse(capabilities);
+    }
+
+    public Set<String> getCapabilities() {
+        return capabilities;
+    }
+}

mail/protocols/imap/src/main/java/com/fsck/k9/mail/store/imap/ImapConnection.kt

@@ -11,6 +11,7 @@ internal interface ImapConnection {
     val isConnected: Boolean
     val outputStream: OutputStream
     val isUidPlusCapable: Boolean
+    val isUtf8AcceptCapable: Boolean
     val isIdleCapable: Boolean
 
     @Throws(IOException::class, MessagingException::class)

mail/protocols/imap/src/main/java/com/fsck/k9/mail/store/imap/RealImapConnection.kt

@@ -55,6 +55,7 @@ internal class RealImapConnection(
     private var responseParser: ImapResponseParser? = null
     private var nextCommandTag = 0
     private var capabilities = emptySet<String>()
+    private var enabled = emptySet<String>()
     private var stacktraceForClose: Exception? = null
     private var open = false
     private var retryOAuthWithNewToken = true
@@ -98,6 +99,7 @@ internal class RealImapConnection(
 
             enableCompressionIfRequested()
             sendClientInfoIfSupported()
+            enableCapabilitiesIfSupported()
 
             retrievePathPrefixIfNecessary()
             retrievePathDelimiterIfNecessary()
@@ -254,6 +256,20 @@ internal class RealImapConnection(
         }
     }
 
+    private fun enableCapabilitiesIfSupported() {
+        if (!hasCapability(Capabilities.ENABLE)) {
+	    return;
+	}
+
+        try {
+            val responses = executeSimpleCommand(Commands.ENABLE)
+	    val enabledResponse = EnabledResponse.parse(responses) ?: return
+	    enabled = enabledResponse.capabilities
+        } catch (e: NegativeImapResponseException) {
+            Timber.d(e, "Ignoring negative response to ENABLE command")
+        }
+    }
+
     private fun upgradeToTlsIfNecessary() {
         if (settings.connectionSecurity == ConnectionSecurity.STARTTLS_REQUIRED) {
             upgradeToTls()
@@ -681,6 +697,9 @@ internal class RealImapConnection(
     override val isUidPlusCapable: Boolean
         get() = capabilities.contains(Capabilities.UID_PLUS)
 
+    override val isUtf8AcceptCapable: Boolean
+        get() = enabled.contains(Capabilities.UTF8_ACCEPT)
+
     @Synchronized
     override fun close() {
         if (!open) return

mail/protocols/imap/src/main/java/com/fsck/k9/mail/store/imap/Responses.java

@@ -17,4 +17,5 @@ class Responses {
     public static final String COPYUID = "COPYUID";
     public static final String SEARCH = "SEARCH";
     public static final String UIDVALIDITY = "UIDVALIDITY";
+    public static final String ENABLED = "ENABLED";
 }

mail/protocols/imap/src/test/java/com/fsck/k9/mail/store/imap/ImapResponseParserTest.kt

@@ -434,6 +434,17 @@ class ImapResponseParserTest {
         assertThatAllInputWasConsumed()
     }
 
+    @Test
+    fun `readResponse() with LIST response containing folder name with UTF8`() {
+        val parser = createParserWithResponses("""* LIST (\HasNoChildren) "." "萬里長城"""")
+
+        val response = parser.readResponse()
+
+        assertThat(response).hasSize(4)
+        assertThat(response).index(3).isEqualTo("萬里長城")
+        assertThatAllInputWasConsumed()
+    }
+
     @Test
     fun `readResponse() with LIST response containing folder name with parentheses should throw`() {
         val parser = createParserWithResponses("""* LIST (\NoInferiors) "/" Root/Folder/Subfolder()""")

mail/protocols/imap/src/test/java/com/fsck/k9/mail/store/imap/RealImapConnectionTest.kt

@@ -774,6 +774,42 @@ class RealImapConnectionTest {
         server.verifyInteractionCompleted()
     }
 
+    @Test
+    fun `open() with ENABLE capability should try to enable UTF8=ACCEPT`() {
+        val server = MockImapServer().apply {
+            simplePreAuthAndLoginDialog(postAuthCapabilities = "ENABLE")
+            expect("3 ENABLE UTF8=ACCEPT")
+            output("* ENABLED")
+            output("3 OK")
+            simplePostAuthenticationDialog(tag = 4)
+        }
+        val imapConnection = startServerAndCreateImapConnection(server, useCompression = true)
+
+        imapConnection.open()
+        assertThat(imapConnection.isUtf8AcceptCapable).isFalse()
+
+        server.verifyConnectionStillOpen()
+        server.verifyInteractionCompleted()
+    }
+
+    @Test
+    fun `open() with ENABLE and UTF8=ACCEPT capabilities should enable UTF8=ACCEPT`() {
+        val server = MockImapServer().apply {
+            simplePreAuthAndLoginDialog(postAuthCapabilities = "ENABLE UTF8=ACCEPT")
+            expect("3 ENABLE UTF8=ACCEPT")
+            output("* ENABLED UTF8=ACCEPT")
+            output("3 OK")
+            simplePostAuthenticationDialog(tag = 4)
+        }
+        val imapConnection = startServerAndCreateImapConnection(server, useCompression = true)
+
+        imapConnection.open()
+        assertThat(imapConnection.isUtf8AcceptCapable).isTrue()
+
+        server.verifyConnectionStillOpen()
+        server.verifyInteractionCompleted()
+    }
+
     @Test
     fun `open() with COMPRESS=DEFLATE capability should enable compression`() {
         val server = MockImapServer().apply {

mail/protocols/imap/src/test/java/com/fsck/k9/mail/store/imap/RealImapFolderTest.kt

@@ -841,6 +841,57 @@ class RealImapFolderTest {
         verify(messages[0]).setHeader(MimeHeader.HEADER_CONTENT_TYPE, "text/plain;\r\n CHARSET=US-ASCII")
     }
 
+    @Test
+    fun fetch_withStructureFetchProfile_shouldNotBreakOnUnicodeAddresses() {
+        val folder = createFolder("Folder")
+        prepareImapFolderForOpen(OpenMode.READ_ONLY)
+        folder.open(OpenMode.READ_ONLY)
+        val bodyStructure = // from RFC 3501 via Arnt and Abhijit
+            "* 1 FETCH (BODYSTRUCTURE ((\"text\" \"plain\" NIL " +
+                "NIL \"Part number 1\" \"7BIT\" 9 1 NIL NIL NIL NIL" +
+                ")(\"application\" \"octet-stream\" NIL NIL \"Part " +
+                "number 2\" \"BASE64\" 14 \"qWXKy9s0ny8E1/5/uzNhpg=" +
+                "=\" (\"attachment\" (\"filename\" \"foo.bar\" \"si" +
+                "ze\" \"8\")) NIL NIL)(\"message\" \"rfc822\" NIL N" +
+                "IL \"Part number 3\" \"7BIT\" 540 (\"Thu, 20 May 2" +
+                "004 14:28:50 +0200\" \"embedded rfc822 message\" (" +
+                "(\"Arnt Gulbrandsen\" NIL \"arnt\" \"ø.example\"))" +
+                " NIL NIL ((\"Abhijit Menon-Sen\" NIL \"ams\" \"ø.e" +
+                "xample\")) NIL NIL NIL NIL) ((\"text\" \"plain\" N" +
+                "IL NIL \"Part number 3.1\" \"7BIT\" 9 1 NIL (\"inl" +
+                "ine\" NIL) (\"en\" \"no\" \"de\") NIL)(\"applicati" +
+                "on\" \"octet-stream\" NIL NIL \"Part number 3.2\" " +
+                "\"BASE64\" 14 NIL NIL NIL NIL) \"mixed\" (\"bounda" +
+                "ry\" \"Y\") NIL NIL NIL) 24 NIL NIL NIL NIL)((\"im" +
+                "age\" \"gif\" NIL NIL \"Part number 4.1\" \"BASE64" +
+                "\" 0 NIL NIL NIL NIL)(\"message\" \"rfc822\" NIL N" +
+                "IL \"Part number 4.2\" \"7BIT\" 658 (\"Thu, 20 May" +
+                " 2004 14:28:50 +0200\" \"second embedded rfc822 me" +
+                "ssage\" ((\"Abhijit Menon-Sen\" NIL \"ams\" \"ø.ex" +
+                "ample\")) NIL NIL ((\"Arnt Gulbrandsen\" NIL \"arn" +
+                "t\" \"ø.example\")) NIL NIL NIL NIL) ((\"text\" \"" +
+                "plain\" NIL NIL \"Part number 4.2.1\" \"7BIT\" 9 1" +
+                " NIL NIL NIL NIL)((\"text\" \"plain\" NIL NIL \"Pa" +
+                "rt number 4.2.2.1\" \"7BIT\" 9 1 NIL NIL \"en\" NI" +
+                "L)(\"text\" \"richtext\" NIL NIL \"Part number 4.2" +
+                ".2.2\" \"7BIT\" 9 1 NIL NIL NIL NIL) \"alternative" +
+                "\" (\"boundary\" \"B\") NIL NIL NIL) \"mixed\" (\"" +
+                "boundary\" \"A\") NIL NIL NIL) 34 NIL NIL NIL NIL)" +
+                " \"mixed\" (\"boundary\" \"Z\") NIL NIL NIL) \"mix" +
+                "ed\" (\"boundary\" \"X\") NIL NIL NIL) UID 1)"
+        whenever(imapConnection.readResponse(anyOrNull()))
+            .thenReturn(createImapResponse(bodyStructure))
+            .thenReturn(createImapResponse("x OK"))
+        val messages = createImapMessages("1")
+        val fetchProfile = createFetchProfile(FetchProfile.Item.STRUCTURE)
+
+        folder.fetch(messages, fetchProfile, null, MAX_DOWNLOAD_SIZE)
+        // We don't really care what happens; the tests in Address and
+        // MIME4J take care of that. At this point we just care that
+        // it doesn't break parsing, cause an exception or anything
+        // like that.
+    }
+
     @Test
     fun `fetch() with simple content type parameter`() {
         testHeaderFromBodyStructure(

mail/protocols/imap/src/test/java/com/fsck/k9/mail/store/imap/TestImapConnection.kt

@@ -13,6 +13,7 @@ internal open class TestImapConnection(val timeout: Long, override val connectio
     override val outputStream: OutputStream
         get() = TODO("Not yet implemented")
     override val isUidPlusCapable: Boolean = true
+    override val isUtf8AcceptCapable: Boolean = false
     override var isIdleCapable: Boolean = true
         protected set