NR-142002 Containerize api server (R&D) (#384)

* Allow dev mode launch from IDE by reading vault secrets at runtime.
* Add instructions for pre-req faker-service-gateway
* Add docker compose for launching mongodb

Author: dsellarsnr

.gitignore

@@ -1,3 +1,4 @@
 .DS_Store
 .sandbox-config.sh
 log/
+certs/

.idea/jsLibraryMappings.xml

@@ -1,6 +1,49 @@
 {
     "version": "0.2.0",
     "configurations": [
+    {
+        "name": "api_ser.js local",
+        "program": "${workspaceFolder}/api_server/bin/api_server.js",
+        "args": ["--one_worker", "--dev_secrets"],
+        "env": {
+            "AWS_REGION": "us-east-1",
+            "BROADCAST_ENGINE_PUBNUB_PREFERRED_KEY_COLOR": "blue",
+            "BROADCAST_ENGINE_PUBNUB_UUID": "CodeStreamServer",
+            "CS_API_AUTH_ORIGIN": "https://auth.cdstrm.dev/no-auth",
+            "CS_API_CALLBACK_ENV": "local",
+            "CS_API_DONT_WANT_AWS": "true",
+            "CS_API_LOGS": "${workspaceFolder}/log",
+            "CS_API_MARKETING_SITE_URL": "https://teamcodestream.webflow.io",
+            "CS_API_PORT": "12078",
+            "CS_API_PUBLIC_API_URL": "https://localhost.newrelic.com:12079",
+            "CS_API_SANDBOX": "${workspaceFolder}",
+            "CSSVC_BACKEND_ROOT": "${workspaceFolder}",
+            "CSSVC_CFG_FILE": "./codestream-docker.json",
+            "CSSVC_EMAIL_SUPPRESS_EMAILS": "true",
+            "CSSVC_ENV": "local",
+            "INTEGRATIONS_NEW_RELIC_IDENTITY_CLOUD_CREDENTIALS_SERVICE_HOST": "https://staging-credential-service.nr-ops.net",
+            "INTEGRATIONS_NEW_RELIC_IDENTITY_CLOUD_GRAPHQL_HOST": "https://nerd-graph.staging-service.nr-ops.net",
+            "INTEGRATIONS_NEW_RELIC_IDENTITY_CLOUD_IDP_SERVICE_HOST": "https://idp-service.staging-service.nr-ops.net",
+            "INTEGRATIONS_NEW_RELIC_IDENTITY_CLOUD_LOGIN_SERVICE_HOST": "https://staging-login.newrelic.com",
+            "INTEGRATIONS_NEW_RELIC_IDENTITY_CLOUD_NEW_RELIC_REGION": "us01",
+            "INTEGRATIONS_NEW_RELIC_IDENTITY_CLOUD_ORG_SERVICE_HOST": "https://staging-organization-service.nr-ops.net",
+            "INTEGRATIONS_NEW_RELIC_IDENTITY_CLOUD_SIGNUP_SERVICE_HOST": "https://signup-processor.staging-service.newrelic.com",
+            "INTEGRATIONS_NEW_RELIC_IDENTITY_CLOUD_USER_SERVICE_HOST": "https://staging-user-service.nr-ops.net",
+            "INTEGRATIONS_NEWRELICGROK_CLOUD_API_URL": "https://nerd-completion.staging-service.nr-ops.net/v1/chat/completions",
+            "NODE_PATH": "${workspaceFolder}/broadcaster/node_modules:${workspaceFolder}/api_server/node_modules",
+            "SERVICE_ENV": "staging",
+            "SHARED_GENERAL_NEW_RELIC_API_URL": "https://staging-api.newrelic.com",
+            "SHARED_GENERAL_NEW_RELIC_LANDING_SERVICE_URL": "https://landing.staging-service.newrelic.com",
+            "SHARED_GENERAL_NEW_RELIC_SEC_API_URL": "https://nrsec-workflow-api.staging-service.newrelic.com",
+            "SHARED_GENERAL_RUN_TIME_ENVIRONMENT": "local",
+            "STORAGE_MONGO_URL": "mongodb://localhost:27017/codestream"
+        },
+        "request": "launch",
+        "skipFiles": [
+            "<node_internals>/**"
+        ],
+        "type": "node"
+    },
         {
             "type": "node",
             "request": "attach",

.idea/runConfigurations/api_server_js_local.xml

@@ -0,0 +1,52 @@
+FROM cf-registry.nr-ops.net/nd-ma/base-node:16
+
+RUN apt-get update && \
+    apt-get upgrade && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN mkdir /opt/config
+WORKDIR /opt/api
+RUN mkdir log tmp pid
+
+WORKDIR /opt/api/codestream-server
+ADD api_server/config /opt/api/codestream-server/api_server/config
+ADD api_server/lib /opt/api/codestream-server/api_server/lib
+ADD api_server/modules /opt/api/codestream-server/api_server/modules
+ADD shared /opt/api/codestream-server/shared
+ADD api_server/etc/webmail_companies.js /opt/api/codestream-server/api_server/etc/webmail_companies.js
+ADD api_server/etc/capabilities.js /opt/api/codestream-server/api_server/etc/capabilities.js
+ADD api_server/etc/version_matrix.json /opt/api/codestream-server/api_server/etc/version_matrix.json
+ADD api_server/etc/configs /opt/api/codestream-server/api_server/etc/configs
+ADD api_server/bin/cs_api-start-docker /opt/api/codestream-server/api_server/bin/cs_api-start-docker
+ADD api_server/bin/*.js /opt/api/codestream-server/api_server/bin/
+ADD api_server/package.json /opt/api/codestream-server/api_server/package.json
+ADD api_server/package-lock.json /opt/api/codestream-server/api_server/package-lock.json
+ADD codestream-docker.json /opt/config/codestream-services-config.json
+ADD certs/out/localhost.newrelic.com.* /opt/api/codestream-server/api_server/etc/certs/
+
+WORKDIR /opt/api/codestream-server/api_server
+RUN npm install --no-save
+
+EXPOSE 8443
+
+RUN chmod +700 /opt/api/codestream-server/api_server/etc/certs && \
+    chmod +700 /opt/api/codestream-server/api_server/etc/certs/* && \
+    chmod +777 /opt/config && \
+    chmod +777 /opt/api/log && \
+    chmod +777 /opt/config/codestream-services-config.json
+
+ENV CSSVC_BACKEND_ROOT=/opt/api/codestream-server
+ENV CSSVC_CFG_FILE=/opt/config/codestream-services-config.json
+ENV CS_API_PORT=8443
+#ENV CSSVC_CFG_URL=mongodb://host/codestream
+ENV NODE_PATH=/opt/api/codestream-server/api_server/node_modules
+ENV CS_API_TOP=/opt/api/codestream-server/api_server
+ENV CS_API_LOGS=/opt/api/log
+ENV CS_API_TMP=/opt/api/tmp
+ENV CS_API_ASSET_ENV=docker
+ENV CS_API_SANDBOX=/opt/api
+#ENV SSL_CA_FILE=/opt/api/codestream-server/api_server/etc/certs/localhost.newrelic.com.csr
+#ENV SSL_CERT_FILE=/opt/api/codestream-server/api_server/etc/certs/localhost.newrelic.com.crt
+#ENV SSL_KEY_FILE=/opt/api/codestream-server/api_server/etc/certs/localhost.newrelic.com.key
+
+CMD [ "/opt/api/codestream-server/api_server/bin/cs_api-start-docker" ]

.vscode/launch.json

@@ -2,108 +2,74 @@
 
 On the backend (aka. the server-side), CodeStream runs a number of services to
 provide all the functionality needed for the clients. The default development
-environment will use the codestream broadcaster and rabbitMQ with outbound mail
+environment will use the codestream broadcaster with outbound mail
 disabled.
 
-## Development Setup with the devtools Framework
+## Development Setup with docker-compose
 
-_Note: CodeStream employees should use the dev_tools sandbox as it will provide
-most of the ancillary resources you'll need, most notably our development
-configuration which includes secrets for pubnub, integration providers, etc...
-Details [here](docs/codestream-sandbox-setup.md). Supplemental documentation is
-[here](docs/README.md)._
+## Development Setup
 
-For everyone else, read on...
-
-## Development Setup without the devtools Framework
 ### Prerequisites
 
 1. Mac or Linux computer using zsh or bash.
 
-1. Official CodeStream builds (CI) use Nodejs 16.13.2 with npm 8.1.2
-
 1. [Docker Desktop](https://www.docker.com/products/docker-desktop) which we'll
-   use to provide MongoDB and a pre-configured RabbitMQ.
-
-If you do not wish to use docker, you'll need to provide both of these services:
+   use to provide MongoDB.
 
-1. MongoDB 3.4.9 with `mongodb://localhost/codestream` providing full access to
-   create collections and indexes in the `codestream` database. If you're
-   willing to run docker, the instructions below will show you how to install a
-   MongoDB docker container.
+### Installation
 
-1. RabbitMQ 3.7.x with the delayed message exchange plugin. You'll also need to
-   create a codestream user with access. [Notes here](api_server/docs/rabbitmq.md).
+For local development we use docker compose to only run mongodb. You will need to run the 
+api-server locally via your IDE or command line.
 
-### Installation
+1. Clone and setup [faker-service-gateway](https://source.datanerd.us/codestream/faker-service-gateway). Faker service gateway will handle SSL and proxy requests to the api-server. 
 
-1. Fork the
-   [codestream-server](https://github.com/teamcodestream/codestream-server) repo
-   and clone it.
+1. Clone the [codestream-server](https://github.com/teamcodestream/codestream-server) repo.
 
-1. Setup your shell's environment
+1. Start up the docker container for MongoDB via docker compose.
    ```
-   cd codestream-server
-   source dev-env.sh     # custom settings go in .sandbox-config.sh
+   docker compose up mongodb -d
    ```
-
-1. Install all the node modules
+1. Install dependencies
    ```
    npm run install:all
    ```
 
-1. Install the rabbitmq docker container pre-configured for codestream (the
-   container name will be csrabbitmq)
-   ```
-   npm run run:docker:csrabbitmq
-   ```
+1. Make sure you are authenticated with vault!, i.e.
+  
+  `newrelic-vault us login -method=okta username=<username> totp=<otp>`
 
-1. Create a docker volume for mongo and launch the mongodb docker container.
-   The docker volume will ensure the data persists beyond the lifespan of the
-   container.
-   ```
-   npm run run:docker:csmongo
-   ```
+### Method 1 - launch from shell
 
-1. In a separate shell, source in the `dev-env.sh` environment and start up the
-   api service. It will repeatedly try to connect to the broadcaster. That's ok.
-   Move on once you've started it.
+1. Source the secrets into your current shell
    ```
-   source dev-env.sh
-   npm run start:api
+   . ./devSecrets.sh
    ```
 
-1. In a another separate shell, source in the `dev-env.sh` environment and start
-   up the broadcaster service.
+1. Setup and start up the api-server without docker
    ```
-   source dev-env.sh
-   npm run start:broadcaster
+    ./start-api-server.sh
    ```
 
-1. In yet another shell, source in the `dev-env.sh` environment and start the
-   onprem admin UI. This will first run webpack to build **public/bundle.js**
-   (which contains the client-side code).
-   ```
-   source dev-env.sh
-   npm run start:opadm
-   ```
-   If your intention is to work on the admin_server, you'll want another shell
-   to run `npm run dev` which will run webpack in watch mode to keep bundle.js
-   updated WRT client-side code which is stored in **src/**.
+### Method 2 - launch from IDE
 
-1. The inbound email service is disabled in the default config.
+1. Run the `api_server.js local` run config from vscode or jetbrains
+
+Point your CodeStream extension to https://localhost.newrelic.com:12079. You should be 
+able to login and see o11y. 
+
+Develop to your heart's content!!!!  We _love_ pull-requests.
+
+## Run everything in docker
+
+If you want to just run the api_server locally with fewer commands. 
+
+1. Make sure you are logged into vault
+
+1. Source the secrets into your current shell
    ```
-   source dev-env.sh
-   npm run start:mailin
+   . ./devSecrets.sh
    ```
-
-1. The outbound email service is also disabled in the default config.
+1. Start docker compose
    ```
-   source dev-env.sh
-   npm run start:mailout
+   docker compose up
    ```
-
-Point your CodeStream extension to http://localhost:12000. You should be able to
-register and create codemarks. The onprem admin console is at http://localhost:12002
-
-Develop to your heart's content!!!!  We _love_ pull-requests.

Dockerfile

@@ -4,5 +4,6 @@
 
 cd $CS_API_TOP
 echo "API docker start script is running"
+bin/set-globals.js
 bin/ensure-indexes.js build
 bin/api_server.js --one_worker

README.md

@@ -0,0 +1,20 @@
+#!/usr/bin/env node
+
+const shellescape = require('shell-escape');
+const { readVaultDevSecrets } = require('../../shared/server_utils/dev_secrets');
+
+async function main() {
+	const env = await readVaultDevSecrets();
+	for (const key in env) {
+		console.log(`export ${key}="${shellescape([env[key]])}"`);
+	}
+}
+
+(async () => {
+	try {
+		await main();
+	} catch (e) {
+		console.error('Error', e);
+	}
+	process.exit();
+})();

api_server/bin/cs_api-start-docker

@@ -0,0 +1,36 @@
+#!/usr/bin/env node
+/* eslint no-console: 0 */
+const ApiConfig = require(process.env.CSSVC_BACKEND_ROOT + '/api_server/config/config');
+const MongoClient = require('mongodb').MongoClient;
+
+const globalsToSet = [ "serviceGatewayAuth" ];
+
+(async function () {
+	let mongoClient, db;
+	try {
+		console.log("set-globals...");
+		const mongoUrl = ApiConfig.configIsMongo()
+			? ApiConfig.options.mongoUrl
+			: (await ApiConfig.loadPreferredConfig()).storage.mongo.url;
+		const mongoTlsOpts = ApiConfig.configIsMongo()
+			? ApiConfig.options.mongoTlsOpts
+			: (await ApiConfig.loadPreferredConfig()).storage.mongo.tlsOptions;
+		mongoClient = await MongoClient.connect(mongoUrl, Object.assign({ useNewUrlParser: true }, mongoTlsOpts));
+		db = mongoClient.db();
+		for (const global of globalsToSet) {
+			const item = await db.collection('globals').findOne({ tag: global });
+			if (!item) {
+				await db.collection('globals').insertOne({ enabled: true, tag: global });
+			} else {
+				if (!item.enabled) {
+					await db.collection('globals').updateOne({ tag: global }, { $set: { enabled: true } });
+				}
+			}
+			console.log(`${global} set to true`);
+		}
+		process.exit(0);
+	} catch (error) {
+		console.log('mongo connect error', error);
+		process.exit(1);
+	}
+})();