OT Frontend Changes - Stage 1 (#298)

* Added stage 1 of OT frontend changes - operation capture

* code cleanup + refactor

* trying to resolve broken pipeline

* review changes

Author: Varun-Sethu

frontend/package-lock.json

@@ -11,6 +11,8 @@
     "@mui/material": "5.8.5",
     "@mui/styles": "5.8.4",
     "@reduxjs/toolkit": "1.8.4",
+    "@types/socket.io": "^3.0.2",
+    "@types/socket.io-client": "^3.0.0",
     "react": "17.0.2",
     "react-dom": "17.0.2",
     "react-icons": "4.3.1",
@@ -20,6 +22,8 @@
     "redux-saga": "1.1.3",
     "slate": "0.78.0",
     "slate-react": "0.79.0",
+    "socket.io": "^4.5.3",
+    "socket.io-client": "^4.5.3",
     "styled-components": "5.3.5",
     "uuid": "8.3.2",
     "web-vitals": "1.1.2"

frontend/package.json

@@ -0,0 +1,3 @@
+# CMS OT Client
+
+TODO: there really needs to be a better way of doing this, currently this code is copied directly from the backend folder. In the future there should maybe be some shared folder between frontend and backend JUST for stuff like API clients?

frontend/src/packages/editor/api/OTClient/README.md

@@ -0,0 +1,106 @@
+/**
+ * Client server implementation
+ */
+
+import { io, Socket } from "socket.io-client";
+import { CMSOperation } from "./operation";
+import { OperationQueue } from "./operationQueue";
+import { bind } from "./util";
+
+const ACK_TIMEOUT_DURATION = 10_000;
+
+/*
+	The Client-Server protocol
+		- The general outline of our client-server protocol is as follows:
+			- Client wants to send an operation (it applies it locally)
+			- If there are any operations in the op buffer it pushes it to the end
+			- If there aren't it sends it directly to the server
+
+			- The client then awaits for an acknowledgment
+			- While it waits of an acknowledgement it queues everything in the buffer
+			- All incoming operations from the server are transformed against buffer operations (As they haven't been applied yet)
+			- When at acknowledgement is received the client then sends the next queued operation to the server
+*/
+
+export default class Client {
+  // TODO: Handle destruction / closing of the websocket
+  constructor(opCallback: (op: CMSOperation) => void) {
+    this.socket = io(`ws://localhost:8080/edit?document=${document}`);
+
+    this.socket.on("connect", this.handleConnection);
+    this.socket.on("ack", this.handleAck);
+    this.socket.on("op", this.handleOperation(opCallback));
+  }
+
+  /**
+   * Handles an incoming acknowledgement operation
+   */
+  private handleAck = () => {
+    clearTimeout(this.timeoutID);
+    this.pendingAcknowledgement = false;
+
+    // dequeue the current operation and send a new one if required
+    this.queuedOperations.dequeueOperation();
+    bind((op) => this.sendToServer(op), this.queuedOperations.peekHead());
+  };
+
+  /**
+   * Handles an incoming operation from the server
+   */
+  private handleOperation =
+    (opCallback: (op: CMSOperation) => void) => (operation: CMSOperation) => {
+      const transformedOp =
+        this.queuedOperations.applyAndTransformIncomingOperation(operation);
+      opCallback(transformedOp);
+
+      this.appliedOperations += 1;
+    };
+
+  /**
+   * Handles the even when the connection opens
+   */
+  private handleConnection = () => {
+    console.log(`Socket ${this.socket.id} connected: ${this.socket.connected}`);
+  };
+
+  /**
+   * Send an operation from client to centralised server through websocket
+   *
+   * @param operation the operation the client wants to send
+   */
+  public pushOperation = (operation: CMSOperation) => {
+    // Note that if there aren't any pending acknowledgements then the operation queue will be empty
+    this.queuedOperations.enqueueOperation(operation);
+
+    if (!this.pendingAcknowledgement) {
+      this.sendToServer(operation);
+    }
+  };
+
+  /**
+   * Pushes an operation to the server
+   */
+  private sendToServer = (operation: CMSOperation) => {
+    this.pendingAcknowledgement = true;
+
+    this.socket.send(
+      JSON.stringify({ operation, appliedOperations: this.appliedOperations })
+    );
+    this.timeoutID = setTimeout(
+      () => {
+        throw Error(`Did not receive ACK after ${ACK_TIMEOUT_DURATION} ms!`);
+      },
+      ACK_TIMEOUT_DURATION,
+      "finish"
+    );
+  };
+
+  private socket: Socket;
+
+  private queuedOperations: OperationQueue = new OperationQueue();
+  private pendingAcknowledgement = false;
+  private appliedOperations = 0;
+
+  // distinct types between node and the browser confuses typescript
+  private timeoutID: any = 0;
+}

frontend/src/packages/editor/api/OTClient/client.ts

@@ -0,0 +1,168 @@
+// Represents atomic operations that can be applied to a piece of data of a specific type
+// TODO: in the future update object operation to strictly contain CMS operation data
+type stringOperation = { rangeStart: number; rangeEnd: number, newValue: string };
+type integerOperation = { newValue: number };
+type booleanOperation = { newValue: boolean };
+// eslint-disable-next-line @typescript-eslint/ban-types
+type objectOperation = { newValue: object }; 
+// eslint-disable-next-line @typescript-eslint/ban-types
+type arrayOperation = { newValue: object };
+type noop = Record<string, never>;
+
+// atomicOperation is a single operation that can be applied in our system
+type atomicOperation = 
+| { "$type": "stringOperation", stringOperation: stringOperation }
+| { "$type": "integerOperation", integerOperation: integerOperation }
+| { "$type": "booleanOperation", booleanOperation: booleanOperation }
+| { "$type": "objectOperation", objectOperation: objectOperation }
+| { "$type": "arrayOperation", arrayOperation: arrayOperation }
+| { "$type": "noop", noop: noop}
+
+// operation is the atomic operation that is sent between clients and servers
+export type CMSOperation = {
+    Path: number[],
+    OperationType: "insert" | "delete",
+
+    IsNoOp: boolean
+    Operation: atomicOperation
+}
+
+export const noop: CMSOperation = { 
+    Path: [], 
+    OperationType: "insert", 
+    IsNoOp: true, 
+    Operation:  {
+        "$type": "noop",
+        noop: {}
+    }
+};
+
+// Actual OT transformation functions
+export const transform = (
+    a: CMSOperation,
+    b: CMSOperation
+  ): [CMSOperation, CMSOperation] => {
+    const transformedPaths = transformPaths(a, b);
+    [a.Path, b.Path] = transformedPaths;
+  
+    return [normalise(a), normalise(b)];
+  };
+  
+  /**
+   * Takes in two operations and transforms them accordingly, note that it only
+   * returns the updated paths
+   */
+  const transformPaths = (a: CMSOperation, b: CMSOperation): [number[], number[]] => {
+    const tp = transformationPoint(a.Path, b.Path);
+    if (!effectIndependent(a.Path, b.Path, tp)) {
+      switch (true) {
+        case a.OperationType === "insert" && b.OperationType === "insert":
+          return transformInserts(a.Path, b.Path, tp);
+        case a.OperationType === "delete" && b.OperationType === "delete":
+          return transformDeletes(a.Path, b.Path, tp);
+        case a.OperationType === "insert" && b.OperationType === "delete":
+          return transformInsertDelete(a.Path, b.Path, tp);
+        default: {
+          const result = transformInsertDelete(b.Path, a.Path, tp);
+          result.reverse();
+          return result;
+        }
+      }
+    }
+  
+    return [a.Path, b.Path];
+  };
+  
+  /**
+   * Takes 2 paths and their transformation point and transforms them as if they
+   * were insertion functions
+   */
+  const transformInserts = (
+    a: number[],
+    b: number[],
+    tp: number
+  ): [number[], number[]] => {
+    switch (true) {
+      case a[tp] > b[tp]:
+        return [update(a, tp, 1), b];
+      case a[tp] < b[tp]:
+        return [a, update(b, tp, 1)];
+      default:
+        return a.length > b.length
+          ? [update(a, tp, 1), b]
+          : (a.length < b.length
+            ? [a, update(b, tp, 1)]
+            : [a, b]);
+    }
+  };
+  
+  /**
+   * Takes 2 paths and transforms them as if they were deletion operations
+   */
+  const transformDeletes = (
+    a: number[],
+    b: number[],
+    tp: number
+  ): [number[], number[]] => {
+    switch (true) {
+      case a[tp] > b[tp]:
+        return [update(a, tp, -1), b];
+      case a[tp] < b[tp]:
+        return [a, update(b, tp, -1)];
+      default:
+        return a.length > b.length
+          ? [[], b]
+          : (a.length < b.length
+            ? [a, []]
+            : [[], []]);
+    }
+  };
+  
+  /**
+   * Takes an insertion operation and a deletion operation and transforms them
+   */
+  const transformInsertDelete = (
+    insertOp: number[],
+    deleteOp: number[],
+    tp: number
+  ): [number[], number[]] => {
+    switch (true) {
+      case insertOp[tp] > deleteOp[tp]:
+        return [update(insertOp, tp, -1), deleteOp];
+      case insertOp[tp] < deleteOp[tp]:
+        return [insertOp, update(deleteOp, tp, 1)];
+      default:
+        return insertOp.length > deleteOp.length
+          ? [[], deleteOp]
+          : [insertOp, update(deleteOp, tp, 1)];
+    }
+  };
+  
+  /**
+   * Updates a specific index in a path
+   */
+  const update = (pos: number[], toChange: number, change: number) => {
+    pos[toChange] += change;
+    return pos;
+  };
+  
+  /**
+   * Takes in two paths and computes their transformation point
+   */
+  const transformationPoint = (a: number[], b: number[]): number =>
+    [...Array(Math.min(a.length, b.length)).keys()].find(
+      (i) => a[i] != b[i]
+    ) ?? Math.min(a.length, b.length);
+  
+  /**
+   * Takes two paths and determines if their effect is independent or not
+   */
+  const effectIndependent = (a: number[], b: number[], tp: number): boolean =>
+    (a.length > tp + 1 && b.length > tp + 1) ||
+    (a[tp] > b[tp] && a.length < b.length) ||
+    (a[tp] < b[tp] && a.length > b.length);
+  
+  /**
+   * Normalise turns an empty operation into a noop
+   */
+  const normalise = (a: CMSOperation): CMSOperation => (a.Path.length === 0 ? noop : a);

frontend/src/packages/editor/api/OTClient/operation.ts

@@ -0,0 +1,60 @@
+import { CMSOperation, transform } from "./operation";
+
+/**
+ * OperationQueue is a simple data structure of the maintenance of outgoing
+ * operations, new operations are pushed to this queue and when and incoming
+ * operation from the server applies that operation is transformed against all
+ * elements of this queue
+ */
+export class OperationQueue {
+
+  /**
+   * Push an operation to the end of the operation queue
+   *
+   * @param operation - the new operation to add
+   * @returns the new length of the queue
+   */
+  public enqueueOperation = (operation: CMSOperation): number =>
+    this.operationQueue.push(operation);
+
+  /**
+   * Takes an incoming operation from the server and applies it to all
+   * elements of the queue
+   *
+   * @param serverOp - the incoming operation from the server
+   * @returns serverOp transformed against all operations in the operation queue
+   */
+  public applyAndTransformIncomingOperation = (
+    serverOp: CMSOperation
+  ): CMSOperation => {
+    const { newQueue, newOp } = this.operationQueue.reduce(
+      (prevSet, op) => {
+        const newOp = transform(op, prevSet.newOp);
+        return { newQueue: prevSet.newQueue.concat(newOp), newOp: newOp[1] };
+      },
+      { newQueue: [] as CMSOperation[], newOp: serverOp }
+    );
+
+    this.operationQueue = newQueue;
+    return newOp;
+  };
+
+  /**
+   * @returns if are any operations queued
+   */
+  public isEmpty = (): boolean => this.operationQueue.length === 0;
+
+  /**
+   * @returns the operation at the head of the operation queue and removes it
+   */
+  public dequeueOperation = (): CMSOperation | undefined =>
+    this.operationQueue.shift();
+
+  /**
+   * @returns the operation at the head of the operation queue
+   */
+  public peekHead = (): CMSOperation | undefined => this.operationQueue[0];
+
+  // operationQueue is our internal operation queue
+  private operationQueue = [] as CMSOperation[];
+}

frontend/src/packages/editor/api/OTClient/operationQueue.ts

@@ -0,0 +1,9 @@
+/**
+ * Bind operation in functional programming
+ *
+ * @param f - the function to apply on the value
+ * @param y - the value
+ * @returns the return value of the function with y passed in if y is not undefined else undefined
+ */
+export const bind = <T, V>(f: (x: T) => V, y: T | undefined): V | undefined =>
+  y !== undefined ? f(y) : undefined;

frontend/src/packages/editor/api/OTClient/util.ts

@@ -0,0 +1,13 @@
+// TODO: remove this and replace with API client once thats complete, see:
+// https://github.com/csesoc/website/pull/238
+export const publishDocument = (documentId: string) => {
+    fetch("/api/filesystem/publish-document", {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/x-www-form-urlencoded",
+        },
+        body: new URLSearchParams({
+          DocumentID: `${documentId}`,
+        }),
+    });
+}