Skip to content
This repository was archived by the owner on Nov 15, 2024. It is now read-only.

Commit 55619fd

Browse files
dsellarsnrdsellarsnr
authored
NR-142002 Containerize api server (R&D) (#384)
* Allow dev mode launch from IDE by reading vault secrets at runtime. * Add instructions for pre-req faker-service-gateway * Add docker compose for launching mongodb
1 parent cd5bfd0 commit 55619fd

18 files changed

+1563
-75
lines changed

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
.DS_Store
22
.sandbox-config.sh
33
log/
4+
certs/

‎.idea/jsLibraryMappings.xml

+6
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎.idea/runConfigurations/api_server_js_local.xml

+38
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎.vscode/launch.json

+43
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,49 @@
11
{
22
"version": "0.2.0",
33
"configurations": [
4+
{
5+
"name": "api_ser.js local",
6+
"program": "${workspaceFolder}/api_server/bin/api_server.js",
7+
"args": ["--one_worker", "--dev_secrets"],
8+
"env": {
9+
"AWS_REGION": "us-east-1",
10+
"BROADCAST_ENGINE_PUBNUB_PREFERRED_KEY_COLOR": "blue",
11+
"BROADCAST_ENGINE_PUBNUB_UUID": "CodeStreamServer",
12+
"CS_API_AUTH_ORIGIN": "https://auth.cdstrm.dev/no-auth",
13+
"CS_API_CALLBACK_ENV": "local",
14+
"CS_API_DONT_WANT_AWS": "true",
15+
"CS_API_LOGS": "${workspaceFolder}/log",
16+
"CS_API_MARKETING_SITE_URL": "https://teamcodestream.webflow.io",
17+
"CS_API_PORT": "12078",
18+
"CS_API_PUBLIC_API_URL": "https://localhost.newrelic.com:12079",
19+
"CS_API_SANDBOX": "${workspaceFolder}",
20+
"CSSVC_BACKEND_ROOT": "${workspaceFolder}",
21+
"CSSVC_CFG_FILE": "./codestream-docker.json",
22+
"CSSVC_EMAIL_SUPPRESS_EMAILS": "true",
23+
"CSSVC_ENV": "local",
24+
"INTEGRATIONS_NEW_RELIC_IDENTITY_CLOUD_CREDENTIALS_SERVICE_HOST": "https://staging-credential-service.nr-ops.net",
25+
"INTEGRATIONS_NEW_RELIC_IDENTITY_CLOUD_GRAPHQL_HOST": "https://nerd-graph.staging-service.nr-ops.net",
26+
"INTEGRATIONS_NEW_RELIC_IDENTITY_CLOUD_IDP_SERVICE_HOST": "https://idp-service.staging-service.nr-ops.net",
27+
"INTEGRATIONS_NEW_RELIC_IDENTITY_CLOUD_LOGIN_SERVICE_HOST": "https://staging-login.newrelic.com",
28+
"INTEGRATIONS_NEW_RELIC_IDENTITY_CLOUD_NEW_RELIC_REGION": "us01",
29+
"INTEGRATIONS_NEW_RELIC_IDENTITY_CLOUD_ORG_SERVICE_HOST": "https://staging-organization-service.nr-ops.net",
30+
"INTEGRATIONS_NEW_RELIC_IDENTITY_CLOUD_SIGNUP_SERVICE_HOST": "https://signup-processor.staging-service.newrelic.com",
31+
"INTEGRATIONS_NEW_RELIC_IDENTITY_CLOUD_USER_SERVICE_HOST": "https://staging-user-service.nr-ops.net",
32+
"INTEGRATIONS_NEWRELICGROK_CLOUD_API_URL": "https://nerd-completion.staging-service.nr-ops.net/v1/chat/completions",
33+
"NODE_PATH": "${workspaceFolder}/broadcaster/node_modules:${workspaceFolder}/api_server/node_modules",
34+
"SERVICE_ENV": "staging",
35+
"SHARED_GENERAL_NEW_RELIC_API_URL": "https://staging-api.newrelic.com",
36+
"SHARED_GENERAL_NEW_RELIC_LANDING_SERVICE_URL": "https://landing.staging-service.newrelic.com",
37+
"SHARED_GENERAL_NEW_RELIC_SEC_API_URL": "https://nrsec-workflow-api.staging-service.newrelic.com",
38+
"SHARED_GENERAL_RUN_TIME_ENVIRONMENT": "local",
39+
"STORAGE_MONGO_URL": "mongodb://localhost:27017/codestream"
40+
},
41+
"request": "launch",
42+
"skipFiles": [
43+
"<node_internals>/**"
44+
],
45+
"type": "node"
46+
},
447
{
548
"type": "node",
649
"request": "attach",

‎Dockerfile

+52
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
FROM cf-registry.nr-ops.net/nd-ma/base-node:16
2+
3+
RUN apt-get update && \
4+
apt-get upgrade && \
5+
rm -rf /var/lib/apt/lists/*
6+
7+
RUN mkdir /opt/config
8+
WORKDIR /opt/api
9+
RUN mkdir log tmp pid
10+
11+
WORKDIR /opt/api/codestream-server
12+
ADD api_server/config /opt/api/codestream-server/api_server/config
13+
ADD api_server/lib /opt/api/codestream-server/api_server/lib
14+
ADD api_server/modules /opt/api/codestream-server/api_server/modules
15+
ADD shared /opt/api/codestream-server/shared
16+
ADD api_server/etc/webmail_companies.js /opt/api/codestream-server/api_server/etc/webmail_companies.js
17+
ADD api_server/etc/capabilities.js /opt/api/codestream-server/api_server/etc/capabilities.js
18+
ADD api_server/etc/version_matrix.json /opt/api/codestream-server/api_server/etc/version_matrix.json
19+
ADD api_server/etc/configs /opt/api/codestream-server/api_server/etc/configs
20+
ADD api_server/bin/cs_api-start-docker /opt/api/codestream-server/api_server/bin/cs_api-start-docker
21+
ADD api_server/bin/*.js /opt/api/codestream-server/api_server/bin/
22+
ADD api_server/package.json /opt/api/codestream-server/api_server/package.json
23+
ADD api_server/package-lock.json /opt/api/codestream-server/api_server/package-lock.json
24+
ADD codestream-docker.json /opt/config/codestream-services-config.json
25+
ADD certs/out/localhost.newrelic.com.* /opt/api/codestream-server/api_server/etc/certs/
26+
27+
WORKDIR /opt/api/codestream-server/api_server
28+
RUN npm install --no-save
29+
30+
EXPOSE 8443
31+
32+
RUN chmod +700 /opt/api/codestream-server/api_server/etc/certs && \
33+
chmod +700 /opt/api/codestream-server/api_server/etc/certs/* && \
34+
chmod +777 /opt/config && \
35+
chmod +777 /opt/api/log && \
36+
chmod +777 /opt/config/codestream-services-config.json
37+
38+
ENV CSSVC_BACKEND_ROOT=/opt/api/codestream-server
39+
ENV CSSVC_CFG_FILE=/opt/config/codestream-services-config.json
40+
ENV CS_API_PORT=8443
41+
#ENV CSSVC_CFG_URL=mongodb://host/codestream
42+
ENV NODE_PATH=/opt/api/codestream-server/api_server/node_modules
43+
ENV CS_API_TOP=/opt/api/codestream-server/api_server
44+
ENV CS_API_LOGS=/opt/api/log
45+
ENV CS_API_TMP=/opt/api/tmp
46+
ENV CS_API_ASSET_ENV=docker
47+
ENV CS_API_SANDBOX=/opt/api
48+
#ENV SSL_CA_FILE=/opt/api/codestream-server/api_server/etc/certs/localhost.newrelic.com.csr
49+
#ENV SSL_CERT_FILE=/opt/api/codestream-server/api_server/etc/certs/localhost.newrelic.com.crt
50+
#ENV SSL_KEY_FILE=/opt/api/codestream-server/api_server/etc/certs/localhost.newrelic.com.key
51+
52+
CMD [ "/opt/api/codestream-server/api_server/bin/cs_api-start-docker" ]

‎README.md

+38-72
Original file line numberDiff line numberDiff line change
@@ -2,108 +2,74 @@
22

33
On the backend (aka. the server-side), CodeStream runs a number of services to
44
provide all the functionality needed for the clients. The default development
5-
environment will use the codestream broadcaster and rabbitMQ with outbound mail
5+
environment will use the codestream broadcaster with outbound mail
66
disabled.
77

8-
## Development Setup with the devtools Framework
8+
## Development Setup with docker-compose
99

10-
_Note: CodeStream employees should use the dev_tools sandbox as it will provide
11-
most of the ancillary resources you'll need, most notably our development
12-
configuration which includes secrets for pubnub, integration providers, etc...
13-
Details [here](docs/codestream-sandbox-setup.md). Supplemental documentation is
14-
[here](docs/README.md)._
10+
## Development Setup
1511

16-
For everyone else, read on...
17-
18-
## Development Setup without the devtools Framework
1912
### Prerequisites
2013

2114
1. Mac or Linux computer using zsh or bash.
2215

23-
1. Official CodeStream builds (CI) use Nodejs 16.13.2 with npm 8.1.2
24-
2516
1. [Docker Desktop](https://www.docker.com/products/docker-desktop) which we'll
26-
use to provide MongoDB and a pre-configured RabbitMQ.
27-
28-
If you do not wish to use docker, you'll need to provide both of these services:
17+
use to provide MongoDB.
2918

30-
1. MongoDB 3.4.9 with `mongodb://localhost/codestream` providing full access to
31-
create collections and indexes in the `codestream` database. If you're
32-
willing to run docker, the instructions below will show you how to install a
33-
MongoDB docker container.
19+
### Installation
3420

35-
1. RabbitMQ 3.7.x with the delayed message exchange plugin. You'll also need to
36-
create a codestream user with access. [Notes here](api_server/docs/rabbitmq.md).
21+
For local development we use docker compose to only run mongodb. You will need to run the
22+
api-server locally via your IDE or command line.
3723

38-
### Installation
24+
1. Clone and setup [faker-service-gateway](https://source.datanerd.us/codestream/faker-service-gateway). Faker service gateway will handle SSL and proxy requests to the api-server.
3925

40-
1. Fork the
41-
[codestream-server](https://github.com/teamcodestream/codestream-server) repo
42-
and clone it.
26+
1. Clone the [codestream-server](https://github.com/teamcodestream/codestream-server) repo.
4327

44-
1. Setup your shell's environment
28+
1. Start up the docker container for MongoDB via docker compose.
4529
```
46-
cd codestream-server
47-
source dev-env.sh # custom settings go in .sandbox-config.sh
30+
docker compose up mongodb -d
4831
```
49-
50-
1. Install all the node modules
32+
1. Install dependencies
5133
```
5234
npm run install:all
5335
```
5436

55-
1. Install the rabbitmq docker container pre-configured for codestream (the
56-
container name will be csrabbitmq)
57-
```
58-
npm run run:docker:csrabbitmq
59-
```
37+
1. Make sure you are authenticated with vault!, i.e.
38+
39+
`newrelic-vault us login -method=okta username=<username> totp=<otp>`
6040

61-
1. Create a docker volume for mongo and launch the mongodb docker container.
62-
The docker volume will ensure the data persists beyond the lifespan of the
63-
container.
64-
```
65-
npm run run:docker:csmongo
66-
```
41+
### Method 1 - launch from shell
6742

68-
1. In a separate shell, source in the `dev-env.sh` environment and start up the
69-
api service. It will repeatedly try to connect to the broadcaster. That's ok.
70-
Move on once you've started it.
43+
1. Source the secrets into your current shell
7144
```
72-
source dev-env.sh
73-
npm run start:api
45+
. ./devSecrets.sh
7446
```
7547

76-
1. In a another separate shell, source in the `dev-env.sh` environment and start
77-
up the broadcaster service.
48+
1. Setup and start up the api-server without docker
7849
```
79-
source dev-env.sh
80-
npm run start:broadcaster
50+
./start-api-server.sh
8151
```
8252

83-
1. In yet another shell, source in the `dev-env.sh` environment and start the
84-
onprem admin UI. This will first run webpack to build **public/bundle.js**
85-
(which contains the client-side code).
86-
```
87-
source dev-env.sh
88-
npm run start:opadm
89-
```
90-
If your intention is to work on the admin_server, you'll want another shell
91-
to run `npm run dev` which will run webpack in watch mode to keep bundle.js
92-
updated WRT client-side code which is stored in **src/**.
53+
### Method 2 - launch from IDE
9354

94-
1. The inbound email service is disabled in the default config.
55+
1. Run the `api_server.js local` run config from vscode or jetbrains
56+
57+
Point your CodeStream extension to https://localhost.newrelic.com:12079. You should be
58+
able to login and see o11y.
59+
60+
Develop to your heart's content!!!! We _love_ pull-requests.
61+
62+
## Run everything in docker
63+
64+
If you want to just run the api_server locally with fewer commands.
65+
66+
1. Make sure you are logged into vault
67+
68+
1. Source the secrets into your current shell
9569
```
96-
source dev-env.sh
97-
npm run start:mailin
70+
. ./devSecrets.sh
9871
```
99-
100-
1. The outbound email service is also disabled in the default config.
72+
1. Start docker compose
10173
```
102-
source dev-env.sh
103-
npm run start:mailout
74+
docker compose up
10475
```
105-
106-
Point your CodeStream extension to http://localhost:12000. You should be able to
107-
register and create codemarks. The onprem admin console is at http://localhost:12002
108-
109-
Develop to your heart's content!!!! We _love_ pull-requests.

‎api_server/bin/cs_api-start-docker

+1
Original file line numberDiff line numberDiff line change
@@ -4,5 +4,6 @@
44

55
cd $CS_API_TOP
66
echo "API docker start script is running"
7+
bin/set-globals.js
78
bin/ensure-indexes.js build
89
bin/api_server.js --one_worker

‎api_server/bin/cs_dev_secrets.js

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
#!/usr/bin/env node
2+
3+
const shellescape = require('shell-escape');
4+
const { readVaultDevSecrets } = require('../../shared/server_utils/dev_secrets');
5+
6+
async function main() {
7+
const env = await readVaultDevSecrets();
8+
for (const key in env) {
9+
console.log(`export ${key}="${shellescape([env[key]])}"`);
10+
}
11+
}
12+
13+
(async () => {
14+
try {
15+
await main();
16+
} catch (e) {
17+
console.error('Error', e);
18+
}
19+
process.exit();
20+
})();

‎api_server/bin/set-globals.js

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#!/usr/bin/env node
2+
/* eslint no-console: 0 */
3+
const ApiConfig = require(process.env.CSSVC_BACKEND_ROOT + '/api_server/config/config');
4+
const MongoClient = require('mongodb').MongoClient;
5+
6+
const globalsToSet = [ "serviceGatewayAuth" ];
7+
8+
(async function () {
9+
let mongoClient, db;
10+
try {
11+
console.log("set-globals...");
12+
const mongoUrl = ApiConfig.configIsMongo()
13+
? ApiConfig.options.mongoUrl
14+
: (await ApiConfig.loadPreferredConfig()).storage.mongo.url;
15+
const mongoTlsOpts = ApiConfig.configIsMongo()
16+
? ApiConfig.options.mongoTlsOpts
17+
: (await ApiConfig.loadPreferredConfig()).storage.mongo.tlsOptions;
18+
mongoClient = await MongoClient.connect(mongoUrl, Object.assign({ useNewUrlParser: true }, mongoTlsOpts));
19+
db = mongoClient.db();
20+
for (const global of globalsToSet) {
21+
const item = await db.collection('globals').findOne({ tag: global });
22+
if (!item) {
23+
await db.collection('globals').insertOne({ enabled: true, tag: global });
24+
} else {
25+
if (!item.enabled) {
26+
await db.collection('globals').updateOne({ tag: global }, { $set: { enabled: true } });
27+
}
28+
}
29+
console.log(`${global} set to true`);
30+
}
31+
process.exit(0);
32+
} catch (error) {
33+
console.log('mongo connect error', error);
34+
process.exit(1);
35+
}
36+
})();

0 commit comments

Comments
 (0)