15.1.2

.vscode/launch.json

@@ -33,9 +33,8 @@
 				"CSSVC_CFG_FILE": "${workspaceFolder}/codestream-docker.json",
 				"NODE_PATH": "${workspaceFolder}/api_server/node_modules:${workspaceFolder}/broadcaster/node_modules",
 				"_comment": "These vars enable the api to queue outbound emails via SQS",
-				"_AWS_PROFILE": "nrdev-super",
 				"CSSVC_SUPPRESS_EMAILS": "false",
-				"CSSVC_OUTBOUND_EMAIL_QUEUE_NAME": "local_${env:USER}_outboundEmail",
+				"CSSVC_OUTBOUND_EMAIL_QUEUE_NAME": "dev_${env:USER}_outboundEmail",
 				"CS_API_DONT_WANT_AWS": "false"
 			},
 			"request": "launch",

README.md

@@ -7,8 +7,6 @@ disabled.
 
 ## Development Setup with docker-compose
 
-## Development Setup
-
 ### Prerequisites
 
 1. Mac or Linux computer using zsh or bash.
@@ -46,13 +44,27 @@ api-server locally via your IDE or command line.
    npm run install:all
    ```
 
+### Mongo upgrade caveat
+
+If you have been running mongodb 4 in docker compose you will need to delete the mongodb volume to get a clean start 
+for mongodb 5, otherwise mongo will exit shortly after startup. This will delete all the data in your local docker
+mongodb instance. 
+
+```
+docker compose down --volumes
+docker compose up mongodb -d
+```
+
+With a fresh database you will need to run `./start-api-server.sh -init-db-only` before being able to run api-server from the IDE. 
+
+
 ### Method 1 - launch from shell and run natively
 
 1. Setup and start up the api-server without docker. The default
    behavior is to initialize the database and disable outbound
    email queueing.
    ```
-   ./start-api-server.sh [-init-db-only | -no-db | -enable-mailout ]
+   ./start-api-server.sh [-init-db-only | -no-db | -enable-mailout | -mock-mode ]
    ```
 
 ### Method 2 - launch from IDE
@@ -72,3 +84,24 @@ Develop to your heart's content!!!!  We _love_ pull-requests.
 ## Run everything in docker
 
 Not working right now :(. Check back later.
+
+## Running Tests
+
+As usual, make sure you are authenticated with vault. 
+
+Start the api server - use `-mock-mode` flag if needed
+```bash
+./start-api-server.sh [-init-db-only | -no-db | -enable-mailout | -mock-mode ]
+```
+
+In a separate terminal, source testMode.sh (this will also set CS_API_MOCK_MODE=1)
+
+```bash
+. ./testMode.sh
+```
+
+then run the tests
+```bash
+cd api_server
+npm run test
+```

api_server/bin/cs_dev_secrets.js

@@ -6,7 +6,7 @@ const { readVaultDevSecrets } = require('../../shared/server_utils/dev_secrets')
 async function main() {
 	const env = await readVaultDevSecrets();
 	for (const key in env) {
-		console.log(`export ${key}="${shellescape([env[key]])}"`);
+		console.log(`export ${key}=${shellescape([env[key]])}`);
 	}
 }
 

api_server/config/dev-secrets.json

@@ -1,4 +1,6 @@
 [
+	"AWS_ACCESS_KEY_ID",
+	"AWS_SECRET_ACCESS_KEY",
    	"BROADCAST_ENGINE_PUBNUB_BLUE_KEY_PUBLISH_KEY",
 	"BROADCAST_ENGINE_PUBNUB_BLUE_KEY_SECRET_KEY",
 	"BROADCAST_ENGINE_PUBNUB_BLUE_KEY_SUBSCRIBE_KEY",
@@ -44,5 +46,14 @@
 	"STORAGE_MONGO_URL",
 	"TELEMETRY_SEGMENT_TOKEN",
 	"TELEMETRY_SEGMENT_WEB_TOKEN",
-	"UNIVERSAL_SECRETS_TELEMETRY"
+	"UNIVERSAL_SECRETS_TELEMETRY",
+	"UNIVERSAL_SECRETS_TELEMETRY",
+	"SHARED_SECRETS_AUTH",
+	"SHARED_SECRETS_COMMENT_ENGINE",
+	"SHARED_SECRETS_COMMENT_ENGINE_ALT",
+	"SHARED_SECRETS_CONFIRMATION_CHEAT",
+	"SHARED_SECRETS_COOKIE",
+	"SHARED_SECRETS_MAIL",
+	"SHARED_SECRETS_SIGNUP_FLOW_JWT",
+	"SHARED_SECRETS_SUBSCRIPTION_CHEAT"
 ]

api_server/etc/capabilities.js

@@ -145,15 +145,5 @@ module.exports = {
 		description: 'Enable code analysis',
 		version: '14.24.1',
 		restricted: true
-	},
-	'showLogSearch': {
-		description: 'Enable log search for entities',
-		version: '14.24.1',
-		restricted: true
-	},	
-	'showNrqlBuilder': {
-		description: 'Enable NRQL Builder',
-		version: '15.0.2',
-		restricted: true
 	}
 };

api_server/lib/test_base/api_request_test.js

@@ -178,6 +178,14 @@ class APIRequestTest extends GenericTest {
 	// make header options to go out with the API request
 	makeHeaderOptions (options, requestOptions) {
 		requestOptions.headers = Object.assign({}, requestOptions.headers || {});
+
+		// IMPORTANT!
+		// set to explicitly close and reopen the connection every time when running test suites
+		// running through Faker Service Gateway (per local dev in the New Relic world) _sometimes_ runs into
+		// timeouts on keep-alive connections, resulting in ECONNRESETs in response to the test requests (eventually)
+		// disable this at your own peril!
+		requestOptions.headers['Connection'] = 'close';
+
 		if (options.token) {
 			// use this token in the request
 			requestOptions.headers.Authorization = 'Bearer ' + options.token;

api_server/modules/access_logger/access_logger.js

@@ -49,6 +49,7 @@ class AccessLogger extends APIServerModule {
 		const ide = request.headers['x-cs-plugin-ide'] || '???';
 		const pluginVersion = request.headers['x-cs-plugin-version'] || '???';
 		const ideVersion = request.headers['x-cs-ide-version'] || '???';
+		const clientMachineId = request.headers['x-cs-client-machine-id'] || '???';
 		let ip = request.headers['x-forwarded-for'];
 		if (!ip && request.connection) {
 			const addr = request.connection.remoteAddress;
@@ -81,7 +82,8 @@ class AccessLogger extends APIServerModule {
 			ide,
 			ver: pluginVersion,
 			idev: ideVersion,
-			tnum: testNum
+			tnum: testNum,
+			cmi: clientMachineId
 		};
 
 		const text = 
@@ -98,7 +100,8 @@ class AccessLogger extends APIServerModule {
 			request.headers['user-agent']  + '" "' + 
 			ide                            + '" "' +
 			pluginVersion                  + '" "' + 
-			ideVersion                     + '" '  +
+			ideVersion                     + '" "'  +
+			clientMachineId                + '" '  +
 			testNum;				
 
 		this.api.log(text, request.id, 'info', {}/*, json*/);

api_server/modules/broadcaster/test/codestream_message_test.js

@@ -316,7 +316,14 @@ class CodeStreamMessageTest extends CodeStreamAPITest {
 			Assert(message.message.messageId, 'received message has no messageId');
 			this.message.messageId = message.message.messageId;	// don't care what it is
 		}
-		Assert.deepEqual(message.message, this.message, 'received message doesn\'t match');
+		try {
+			Assert.deepStrictEqual(message.message, this.message, 'received message doesn\'t match');
+		}
+		catch (e) {
+			// I have no clue why I need to do this, shouldn't Assert just throw???
+			console.warn('Message assertion failed:', e);
+			throw e;
+		}
 		return true;
 	}
 

api_server/modules/companies/post_company_request.js

@@ -76,7 +76,7 @@ class PostCompanyRequest extends PostRequest {
 		// a response to the client with a valid access token, but knowing the refresh token
 		// isn't valid ... but we'll fetch a new refresh token after a generous period of time 
 		// to allow the race condition to clear
-		await this.updateRefreshToken();
+		//await this.updateRefreshToken();
 	}
 
 	// evidently there is some kind of race condition in the Azure B2C API which causes

api_server/modules/companies/test/post_company/clear_first_session_test.js

@@ -87,6 +87,7 @@ class ClearFirstSessionTest extends PostCompanyTest {
 				if (error) { return callback(error); }
 				this.path = '/users/me';
 				this.token = response.accessToken;
+				delete this.data;
 				callback();
 			}
 		);

api_server/modules/companies/test/post_company/test.js

@@ -40,10 +40,11 @@ class PostCompanyRequestTester {
 		new FirstCompanyOneUserPerOrgTest().test();
 		new FirstCompanyOneUserPerOrgTest({ unifiedIdentityEnabled: true }).test();
 		new ClearFirstSessionTest().test();
-		new IDPSignupTest().test();
-		new IDPSignupBehindSGTest().test();
+		//new IDPSignupTest().test();
+		//new IDPSignupBehindSGTest().test();
 		new LinkedNROrgIdTest().test();
 		new CompanyNameFromRegistrationTest().test();
+		/*
 		// serialize these tests because they are time-dependent, and fail on the
 		// default setup-then-run methodology
 		SerializeTests([
@@ -56,6 +57,7 @@ class PostCompanyRequestTester {
 			FirstCompanyRefreshTokenFetchTest,
 			FirstCompanyRefreshTokenBehindSGFetchTest
 		]);
+		*/
 	}
 }
 

api_server/modules/users/errors.js

@@ -172,5 +172,10 @@ module.exports = {
 		code: 'USRC-1032',
 		message: 'Client must switch to using a different server URL',
 		description: 'The client is directed to switch to using a different server URL, the URL is given in the error info'
+	},
+	'clientBlacklisted': {
+		code: 'USRC-1033',
+		message: 'Requests are being denied to this client',
+		description: 'This client was found to be on the client black list, all requests will be denied'
 	}
 };

api_server/modules/users/join_company_helper.js

@@ -352,7 +352,7 @@ class JoinCompanyHelper {
 		// a response to the client with a valid access token, but knowing the refresh token
 		// isn't valid ... but we'll fetch a new refresh token after a generous period of time 
 		// to allow the race condition to clear
-		await this.updateRefreshToken();
+		//await this.updateRefreshToken();
 	}
 
 	// evidently there is some kind of race condition in the Azure B2C API which causes

api_server/modules/users/test/idp_sync/change_email_test.js

@@ -29,12 +29,17 @@ class ChangeEmailTest extends Aggregation(CodeStreamAPITest, CommonInit) {
 
 	// before the test runs...
 	before (callback) {
-		this.init(callback);
+		this.init(error => {
+			if (error) { return callback(error); }
+			this.inputEmail = this.data.email;
+			delete this.data;
+			callback();
+		});
 	}
 
 	// validate the response to the test request
 	validateResponse (data) {
-		Assert.strictEqual(data.user.email, this.data.email, 'fetched user\'s email does not match');
+		Assert.deepStrictEqual(data.user.email, this.inputEmail, 'fetched user\'s email does not match');
 	}
 }
 

api_server/modules/users/test/idp_sync/change_name_message_test.js

@@ -28,7 +28,7 @@ class ChangeNameMessageTest extends Aggregation(CodeStreamMessageTest, CommonIni
 		this.data = {
 			name: this.userFactory.randomFullName()
 		};
-		const expectedVersion = 7;
+		const expectedVersion = 6;
 		this.message = {
 			user: {
 				id: this.currentUser.user.id,

api_server/modules/users/test/idp_sync/change_name_test.js

@@ -29,12 +29,17 @@ class ChangeNameTest extends Aggregation(CodeStreamAPITest, CommonInit) {
 
 	// before the test runs...
 	before (callback) {
-		this.init(callback);
+		this.init(error => {
+			if (error) { return callback(error); }
+			this.inputName = this.data.name;
+			delete this.data;
+			callback();
+		});
 	}
 
 	// validate the response to the test request
 	validateResponse (data) {
-		Assert.strictEqual(data.user.fullName, this.data.name, 'fetched user\'s name does not match');
+		Assert.deepStrictEqual(data.user.fullName, this.inputName, 'fetched user\'s name does not match');
 	}
 }
 

api_server/modules/users/test/idp_sync/change_org_name_test.js

@@ -27,12 +27,17 @@ class ChangeOrgNameTest extends Aggregation(CodeStreamAPITest, CommonInit) {
 
 	// before the test runs...
 	before (callback) {
-		this.init(callback);
+		this.init(error => {
+			if (error) { return callback(error); }
+			this.inputName = this.data.name;
+			delete this.data;
+			callback();
+		});
 	}
 
 	// validate the response to the test request
 	validateResponse (data) {
-		Assert.strictEqual(data.company.name, this.data.name, 'fetched company\'s name does not match');
+		Assert.deepStrictEqual(data.company.name, this.inputName, 'fetched company\'s name does not match');
 	}
 }
 

api_server/modules/users/test/idp_sync/deactivate_org_test.js

@@ -41,6 +41,7 @@ class DeactivateOrgTest extends Aggregation(CodeStreamAPITest, CommonInit) {
 	doLogin (callback) {
 		super.doLogin(error => {
 			Assert(error, 'no error thrown during login');
+			delete this.data;
 			callback();
 		});
 	}

api_server/modules/users/test/idp_sync/deactivate_user_test.js

@@ -43,6 +43,7 @@ class DeactivateUserTest extends Aggregation(CodeStreamAPITest, CommonInit) {
 	doLogin (callback) {
 		super.doLogin(error => {
 			Assert(error, 'no error thrown during login');
+			delete this.data;
 			callback();
 		});
 	}

api_server/modules/users/users.js

@@ -278,16 +278,40 @@ class Users extends Restful {
 	middlewares () {
 		return async (request, response, next) => {
 
-			if (request.url === '/no-auth/capabilities') {
-				next();
-				return;
+			const now = Date.now();
+
+			// read globals, but only once a minute
+			let globalMaintenanceMode, clientBlackList;
+			if (!this.lastGlobalRead || this.lastGlobalRead < now - 60 * 1000) {
+				globalMaintenanceMode = await this.api.data.globals.getOneByQuery(
+					{ tag: 'inMaintenanceMode' }, 
+					{ overrideHintRequired: true }
+				);
+				clientBlackList = await this.api.data.globals.getOneByQuery(
+					{ tag: 'clientBlackList' },
+					{ overrideHintRequired: true }
+				);
+				this.lastGlobalRead = now;
+				this.cachedGLobalMaintenanceMode = globalMaintenanceMode;
+				this.cachedClientBlackList = clientBlackList;
+			} else {
+				globalMaintenanceMode = this.cachedGlobalMaintenanceMode;
+				clientBlackList = this.cachedClientBlackList;
 			}
 
-			// look for global maintenance mode set
-			const globalMaintenanceMode = await this.api.data.globals.getOneByQuery(
-				{ tag: 'inMaintenanceMode' }, 
-				{ overrideHintRequired: true }
-			);
+			// determine whether this client's client-generated machine ID is blacklisted,
+			// and if so, deny this request
+			const clientMachineId = request.headers['x-cs-client-machine-id'];
+			if (clientMachineId && clientBlackList && clientBlackList.enabled) {
+				clientBlackList = clientBlackList.list?.split(',');
+				if (clientBlackList.includes(clientMachineId)) {
+					request.abortWith = {
+						status: 403,
+						error: this.errorHandler.error('clientBlacklisted')
+					};
+					return next();
+				}
+			}
 
 			// look for override maintenance mode header, to allow for internal testing
 			const overrideMaintenanceMode = (