New network protocol version v4 - filestream

Ticket: ENT-12414
Changelog: Title
Signed-off-by: Lars Erik Wik <[email protected]>
Co-authored-by: Craig Comstock <[email protected]>

Author: larsewi

cf-agent/verify_files_utils.c

@@ -1551,7 +1551,7 @@ bool CopyRegularFile(EvalContext *ctx, const char *source, const char *dest, con
             return false;
         }
 
-        if (!CopyRegularFileNet(source, ToChangesPath(new),
+        if (!CopyRegularFileNet(source, dest, ToChangesPath(new),
                                 sstat->st_size, attr->copy.encrypt, conn, sstat->st_mode))
         {
             RecordFailure(ctx, pp, attr, "Failed to copy file '%s' from '%s'",

cf-serverd/server_common.c

@@ -54,6 +54,7 @@ static const int CF_NOSIZE = -1;
 #include <mutex.h>                                 /* ThreadLock */
 #include <stat_cache.h>                            /* struct Stat */
 #include <unix.h>                                  /* GetUserID() */
+#include <file_stream.h>
 #include "server_access.h"
 
 
@@ -402,6 +403,9 @@ static void FailedTransfer(ConnectionInfo *connection)
 
 void CfGetFile(ServerFileGetState *args)
 {
+    assert(args != NULL);
+    assert(args->conn != NULL);
+
     int fd;
     off_t n_read, total = 0, sendlen = 0, count = 0;
     char sendbuffer[CF_BUFSIZE + 256], filename[CF_BUFSIZE - 128];
@@ -421,12 +425,23 @@ void CfGetFile(ServerFileGetState *args)
 
     if (!TransferRights(args->conn, filename, &sb))
     {
+        const ProtocolVersion version = ConnectionInfoProtocolVersion(conn_info);
+        assert(ProtocolIsKnown(version));
+
         Log(LOG_LEVEL_INFO, "REFUSE access to file: %s", filename);
+
+        if (ProtocolSupportsFileStream(version)) {
+            Log(LOG_LEVEL_VERBOSE, "REFUSAL to user='%s' of request: %s",
+                NULL_OR_EMPTY(args->conn->username) ? "?" : args->conn->username,
+                args->replyfile);
+            FileStreamRefuse(args->conn->conn_info->ssl);
+            return;
+        }
+        /* Else then handle older protocols */
+
         RefuseAccess(args->conn, args->replyfile);
         snprintf(sendbuffer, CF_BUFSIZE, "%s", CF_FAILEDSTR);
 
-        const ProtocolVersion version = ConnectionInfoProtocolVersion(conn_info);
-        assert(ProtocolIsKnown(version));
         if (ProtocolIsClassic(version))
         {
             SendSocketStream(ConnectionInfoSocket(conn_info), sendbuffer, args->buf_size);
@@ -440,6 +455,12 @@ void CfGetFile(ServerFileGetState *args)
 
 /* File transfer */
 
+    const ProtocolVersion version = ConnectionInfoProtocolVersion(conn_info);
+    if (ProtocolSupportsFileStream(version)) {
+        FileStreamServe(conn_info->ssl, filename);
+        return;
+    }
+
     if ((fd = safe_open(filename, O_RDONLY)) == -1)
     {
         Log(LOG_LEVEL_ERR, "Open error of file '%s'. (open: %s)",

libcfnet/README.md

@@ -15,6 +15,7 @@ Names of protocol versions:
 1. `"classic"` - Legacy, pre-TLS, protocol. Not enabled or allowed by default.
 2. `"tls"` - TLS Protocol using OpenSSL. Encrypted and 2-way authentication.
 3. `"cookie"` - TLS Protocol with cookie command for duplicate host detection.
+3. `"filestream"` - Introduces a new streaming API for get file request (powered by librsync).
 
 Wanted protocol version can be specified from policy:
 
@@ -59,3 +60,30 @@ Both server and client will then set `conn_info->protocol` to `2`, and use proto
 There is currently no way to require a specific version number (only allow / disallow version 1).
 This is because version 2 and 3 are practically identical.
 Downgrade from version 3 to 2 happens seamlessly, but crucially, it doesn't downgrade to version 1 inside the TLS code.
+
+## Commands
+
+### `GET <FILENAME>` (protocol v4)
+
+The following is a description of the `GET <FILENAME>` command, modified in
+protocol version v4 (introduced in CFEngine 3.25).
+
+The initial motivation for creating a new protocol version `filestream` was
+due to a race condition found in the `GET <FILENAME>` request. It relied on the
+file size aquired by `STAT <FILENAME>`. However, if the file size increased
+between the two requests, the client would think that the remaining data at the
+offset of the aquired file size is a new protocol header. This situation would lead
+to undefined behaviour. Hence, we needed a new protocol to send files. Instead
+of reinventing the wheel, we decided to use librsync which utilizes the RSYNC
+protocol to transmit files.
+
+The server implementation is found in function
+[CfGet()](../cf-serverd/server_common.c). Client impementations are found in
+[CopyRegularFileNet()](client_code.c) and [ProtocolGet()](protocol.c)
+
+Similar to before, the client issues a `GET <FILENAME>` request. However,
+instead of continuing to execute the old protocol, the client immediately calls
+`FileStreamFetch()` from the "File Stream API". Upon receiving such a request,
+the server calls either `FileStreamRefuse()` (to refuse the request) or
+`FileStreamServe()` (to comply with the request). The internal workings of the
+File Stream API is well explained in [file_stream.h](file_stream.h).

libcfnet/client_code.c

@@ -45,6 +45,7 @@
 #include <misc_lib.h>                                   /* ProgrammingError */
 #include <printsize.h>                                         /* PRINTSIZE */
 #include <lastseen.h>                                            /* LastSaw */
+#include <file_stream.h>
 
 
 #define CFENGINE_SERVICE "cfengine"
@@ -749,9 +750,11 @@ static void FlushFileStream(int sd, int toget)
 
 /* TODO finalise socket or TLS session in all cases that this function fails
  * and the transaction protocol is out of sync. */
-bool CopyRegularFileNet(const char *source, const char *dest, off_t size,
+bool CopyRegularFileNet(const char *source, const char *basis, const char *dest, off_t size,
                         bool encrypt, AgentConnection *conn, mode_t mode)
 {
+    assert(conn != NULL);
+
     char *buf, workbuf[CF_BUFSIZE], cfchangedstr[265];
     const int buf_size = 2048;
 
@@ -774,23 +777,12 @@ bool CopyRegularFileNet(const char *source, const char *dest, off_t size,
 
     unlink(dest);                /* To avoid link attacks */
 
-    int dd = safe_open_create_perms(dest, O_WRONLY | O_CREAT | O_TRUNC | O_EXCL | O_BINARY, mode);
-    if (dd == -1)
-    {
-        Log(LOG_LEVEL_ERR,
-            "Copy from server '%s' to destination '%s' failed (open: %s)",
-            conn->this_server, dest, GetErrorStr());
-        unlink(dest);
-        return false;
-    }
-
     workbuf[0] = '\0';
     int tosend = snprintf(workbuf, CF_BUFSIZE, "GET %d %s", buf_size, source);
     if (tosend <= 0 || tosend >= CF_BUFSIZE)
     {
         Log(LOG_LEVEL_ERR, "Failed to compose GET command for file %s",
             source);
-        close(dd);
         return false;
     }
 
@@ -799,7 +791,21 @@ bool CopyRegularFileNet(const char *source, const char *dest, off_t size,
     if (SendTransaction(conn->conn_info, workbuf, tosend, CF_DONE) == -1)
     {
         Log(LOG_LEVEL_ERR, "Couldn't send GET command");
-        close(dd);
+        return false;
+    }
+
+    const ProtocolVersion version = ConnectionInfoProtocolVersion(conn->conn_info);
+    if (ProtocolSupportsFileStream(version)) {
+        return FileStreamFetch(conn->conn_info->ssl, basis, dest, mode);
+    }
+
+    int dd = safe_open_create_perms(dest, O_WRONLY | O_CREAT | O_TRUNC | O_EXCL | O_BINARY, mode);
+    if (dd == -1)
+    {
+        Log(LOG_LEVEL_ERR,
+            "Copy from server '%s' to destination '%s' failed (open: %s)",
+            conn->this_server, dest, GetErrorStr());
+        unlink(dest);
         return false;
     }
 

libcfnet/client_code.h

@@ -47,7 +47,7 @@ AgentConnection *ServerConnection(const char *server, const char *port, const Rl
 void DisconnectServer(AgentConnection *conn);
 
 bool CompareHashNet(const char *file1, const char *file2, bool encrypt, AgentConnection *conn);
-bool CopyRegularFileNet(const char *source, const char *dest, off_t size,
+bool CopyRegularFileNet(const char *source, const char *basis, const char *dest, off_t size,
                         bool encrypt, AgentConnection *conn, mode_t mode);
 Item *RemoteDirList(const char *dirname, bool encrypt, AgentConnection *conn);
 

libcfnet/protocol.c

@@ -31,6 +31,7 @@
 #include <stat_cache.h>
 #include <string_lib.h>
 #include <tls_generic.h>
+#include <file_stream.h>
 
 Seq *ProtocolOpenDir(AgentConnection *conn, const char *path)
 {
@@ -124,6 +125,38 @@ bool ProtocolGet(AgentConnection *conn, const char *remote_path,
         return false;
     }
 
+    /* Use file stream API if it is available */
+    const ProtocolVersion version = ConnectionInfoProtocolVersion(conn->conn_info);
+    if (ProtocolSupportsFileStream(version))
+    {
+        fclose(file_ptr);
+
+        char dest[PATH_MAX];
+        ret = snprintf(dest, sizeof(dest), "%s.cfnew", local_path);
+        if (ret < 0 || (size_t)ret >= sizeof(dest))
+        {
+            Log(LOG_LEVEL_ERR, "Truncation error: Path too long (%d >= %zu)", ret, sizeof(dest));
+            return false;
+        }
+
+        if (!FileStreamFetch(conn->conn_info->ssl, local_path, dest, perms))
+        {
+            /* Error is already logged */
+            return false;
+        }
+
+        Log(LOG_LEVEL_VERBOSE, "Replacing file '%s' with '%s'...", dest, local_path);
+        if (rename(dest, local_path) == -1)
+        {
+            Log(LOG_LEVEL_ERR, "Failed to replace destination file '%s' with basis file '%s': %s", dest, local_path, GetErrorStr());
+            return false;
+        }
+
+        return true;
+    }
+
+    /* Otherwise, use old protocol */
+
     char cfchangedstr[sizeof(CF_CHANGEDSTR1 CF_CHANGEDSTR2)];
     snprintf(cfchangedstr, sizeof(cfchangedstr), "%s%s",
              CF_CHANGEDSTR1, CF_CHANGEDSTR2);

libcfnet/protocol_version.c

@@ -33,6 +33,10 @@ ProtocolVersion ParseProtocolVersionPolicy(const char *const s)
     {
         return CF_PROTOCOL_COOKIE;
     }
+    else if (StringEqual(s, "4") || StringEqual(s, "filestream"))
+    {
+        return CF_PROTOCOL_FILESTREAM;
+    }
     else if (StringEqual(s, "latest"))
     {
         return CF_PROTOCOL_LATEST;

libcfnet/protocol_version.h

@@ -39,10 +39,11 @@ typedef enum
     /* --- Greater versions use TLS as secure communications layer --- */
     CF_PROTOCOL_TLS = 2,
     CF_PROTOCOL_COOKIE = 3,
+    CF_PROTOCOL_FILESTREAM = 4,
 } ProtocolVersion;
 
 /* We use CF_PROTOCOL_LATEST as the default for new connections. */
-#define CF_PROTOCOL_LATEST CF_PROTOCOL_COOKIE
+#define CF_PROTOCOL_LATEST CF_PROTOCOL_FILESTREAM
 
 static inline const char *ProtocolVersionString(const ProtocolVersion p)
 {
@@ -54,6 +55,8 @@ static inline const char *ProtocolVersionString(const ProtocolVersion p)
         return "tls";
     case CF_PROTOCOL_CLASSIC:
         return "classic";
+    case CF_PROTOCOL_FILESTREAM:
+        return "filestream";
     default:
         return "undefined";
     }
@@ -84,6 +87,11 @@ static inline bool ProtocolIsClassic(const ProtocolVersion p)
     return (p == CF_PROTOCOL_CLASSIC);
 }
 
+static inline bool ProtocolSupportsFileStream(const ProtocolVersion p)
+{
+    return (p >= CF_PROTOCOL_FILESTREAM);
+}
+
 static inline bool ProtocolTerminateCSV(const ProtocolVersion p)
 {
     return (p < CF_PROTOCOL_COOKIE);