Updates for Feb 2021 bug fix challenge

jmgasper · jmgasper · commit 5618842f1933 · 2021-02-08T12:36:40.000+11:00
https://software.topcoder.com/review/actions/ViewProjectDetails?method=viewProjectDetails&pid=30165062
diff --git a/README.md b/README.md
@@ -93,4 +93,12 @@ Direct ID migration can be done with script located in **topcoder-x-processor**
 ```shell
 npm run direct-connect-migration
 ```
+By default it takes 15 projects at time, but you can change this by specifying BATCH_SIZE environment variable.
+
+## Repository url collisions scan
+
+To scan and log which projects has colliding (the same) repository urls run
+```shell
+npm run log-repository-collisions
+```
 By default it takes 15 projects at time, but you can change this by specifying BATCH_SIZE environment variable.
diff --git a/TeamsVsGroups.md b/TeamsVsGroups.md
@@ -0,0 +1,40 @@
+
+## Github's Teams and Gitlab's Groups differences
+
+Github's Teams are groups of organization members whereas Gitlab's Groups are just groups of Gitlab users (Groups are more like Github' organizations).
+Each Github organization can have repositories and assign teams to them and each team can have subteams.
+Gitlab groups can have repositories and create subgroups (nested).
+
+## Setup guide
+
+### Github
+
+1. Go to 'https://github.com/settings/organizations' and click 'New organization' button
+2. Select your plan.
+3. Enter organization's name, contact email address, solve the captcha and click Next
+4. Click 'Skip this step'
+5. If you receive a survey, you can just go to the bottom and click Submit without filling anything.
+6. On your new organization page go to 'Teams' tab and click 'New team' button.
+7. Fill in your team's name, description (optional) and visibility. Submit by clicking 'Create team'.
+Now you have your team created and you should get redirect to its page.
+You can assign it to an organization's repository by clicking 'Add Repository' and entering repository's name, in 'Repositories' tab of a team's page.
+
+### Gitlab
+
+1. Go to 'https://gitlab.com/dashboard/groups' and click on 'New group' button.
+2. Enter group's name and set visibility level. Finish by clicking 'Create group'.
+You can now create repositories for this group or subgroups.
+
+## Roles
+
+In Topcoder X you can select role which user who joins via specific invitation link receives.
+
+### Github
+
+For github team you can set two roles: Member and Maintainer.
+You can read about them here: https://docs.github.com/en/github/setting-up-and-managing-organizations-and-teams/permission-levels-for-an-organization and https://docs.github.com/en/github/setting-up-and-managing-organizations-and-teams/giving-team-maintainer-permissions-to-an-organization-member
+
+### Gitlab
+
+For gitlab group you can set five roles: Guest, Reporter, Developer, Maintainer, Owner
+You can read about them here: https://docs.gitlab.com/ee/user/permissions.html
diff --git a/package.json b/package.json
@@ -21,7 +21,8 @@
         "test:fe": "gulp protractor",
         "lint": "gulp lint",
         "heroku-postbuild": "gulp build",
-        "create-tables": "CREATE_DB=true node scripts/create-update-tables.js"
+        "create-tables": "CREATE_DB=true node scripts/create-update-tables.js",
+        "log-repository-collisions": "node scripts/log-repository-collisions.js"
     },
     "dependencies": {
         "angular": "~1.8.0",
diff --git a/scripts/log-repository-collisions.js b/scripts/log-repository-collisions.js
@@ -0,0 +1,43 @@
+const logger = require('../src/common/logger')
+const models = require('../src/models')
+const _ = require('lodash')
+
+async function main() {
+    const BATCH_SIZE = process.env.BATCH_SIZE || 15;
+    let previousSize = BATCH_SIZE;
+    let previousKey = null;
+    // Array containing project ids already found colliding
+    const collidingUrls = []
+    let batch = 1;
+    // Run this loop as long as there can be more objects in a database
+    while (previousSize === BATCH_SIZE) {
+        logger.debug(`Running batch no. ${batch}`)
+        // Go over all active projects, limiting to BATCH_SIZE
+        const projects = await models.Project.scan({
+            archived: 'false'
+        }).consistent().limit(BATCH_SIZE).startAt(previousKey).exec()
+        for (const project of projects) {
+            // If url was already found colliding go to a next iteration
+            if (collidingUrls.includes(project.repoUrl)) continue;
+            const collisions = await models.Project.scan({
+                repoUrl: project.repoUrl,
+                archived: 'false'
+            }).exec()
+            // If scan found only this project go to a next interation
+            if (collisions.length < 2) continue;
+            logger.info(`Repository ${project.repoUrl} has ${collisions.length} collisions`);
+            _.forEach(collisions, collision => {
+                logger.info(`--- ID: ${collision.id}`)
+            })
+            collidingUrls.push(project.repoUrl)
+        }
+        previousKey = projects.lastKey
+        previousSize = projects.scannedCount
+        batch++
+    }
+}
+main().then(() => {
+   logger.info('Collision scan completed')
+}).catch(err => {
+    logger.logFullError(err, 'collision scan')
+})
diff --git a/src/config.js b/src/config.js
@@ -22,7 +22,6 @@ module.exports = {
 
   // used as base to construct various URLs
   WEBSITE: process.env.WEBSITE || 'http://topcoderx.topcoder-dev.com',
-  WEBSITE_SECURE: process.env.WEBSITE_SECURE || 'https://topcoderx.topcoder-dev.com',
   GITLAB_API_BASE_URL: process.env.GITLAB_API_BASE_URL || 'https://gitlab.com',
 
   // kafka configuration
@@ -50,7 +49,6 @@ module.exports = {
   OPEN_FOR_PICKUP_ISSUE_LABEL: process.env.OPEN_FOR_PICKUP_ISSUE_LABEL || 'tcx_OpenForPickup',
   ALLOWED_TOPCODER_ROLES: process.env.ALLOWED_TOPCODER_ROLES || ['administrator', 'admin', 'connect manager', 'connect admin', 'copilot', 'connect copilot'],
   COPILOT_ROLE: process.env.COPILOT_ROLE || 'copilot',
-  HELP_LINK: process.env.HELP_LINK || 'https://github.com/topcoder-platform/topcoder-x-ui/wiki',
   ADMINISTRATOR_ROLES: process.env.ADMINISTRATOR_ROLES || ['administrator', 'admin'],
   DYNAMODB: {
     AWS_ACCESS_KEY_ID: process.env.AWS_ACCESS_KEY_ID,
@@ -69,100 +67,20 @@ module.exports = {
   },
 };
 
-const frontendConfigs = {
-  local: {
-    JWT_V3_NAME: 'v3jwt',
-    JWT_V2_NAME: 'tcjwt',
-    COOKIES_SECURE: false,
-    TC_LOGIN_URL: 'https://accounts-auth0.topcoder-dev.com/',
-    API_URL: 'https://127.0.0.1:8443',
-    ADMIN_TOOL_URL: 'http://localhost:8080/api/v2',
-    ACCOUNTS_CONNECTOR_URL: 'https://accounts.topcoder-dev.com/connector.html',
-    CONNECT_URL_BASE: 'https://connect.topcoder-dev/projects/',
-    OWNER_LOGIN_GITHUB_URL: '/api/v1/github/owneruser/login',
-    OWNER_LOGIN_GITLAB_URL: '/api/v1/gitlab/owneruser/login',
-    TOPCODER_URL: 'https://topcoder-dev.com',
-    GITHUB_TEAM_URL: 'https://github.com/orgs/',
-    GITLAB_GROUP_URL: 'https://gitlab.com/groups/',
-  },
-  heroku: {
-    JWT_V3_NAME: 'v3jwt',
-    JWT_V2_NAME: 'tcjwt',
-    COOKIES_SECURE: false,
-    TC_LOGIN_URL: 'https://accounts-auth0.topcoder-dev.com/',
-    API_URL: 'https://api.topcoder-dev.com',
-    ADMIN_TOOL_URL: 'https://api.topcoder-dev.com/v2',
-    ACCOUNTS_CONNECTOR_URL: 'https://accounts.topcoder-dev.com/connector.html',
-    CONNECT_URL_BASE: 'https://connect.topcoder-dev.com/projects/',
-    OWNER_LOGIN_GITHUB_URL: '/api/v1/github/owneruser/login',
-    OWNER_LOGIN_GITLAB_URL: '/api/v1/gitlab/owneruser/login',
-    TOPCODER_URL: 'https://topcoder-dev.com',
-    GITHUB_TEAM_URL: 'https://github.com/orgs/',
-    GITLAB_GROUP_URL: 'https://gitlab.com/groups/',
-  },
-  dev: {
-    JWT_V3_NAME: 'v3jwt',
-    JWT_V2_NAME: 'tcjwt',
-    COOKIES_SECURE: false,
-    TC_LOGIN_URL: 'https://accounts-auth0.topcoder-dev.com/',
-    API_URL: 'https://api.topcoder-dev.com',
-    ADMIN_TOOL_URL: 'https://api.topcoder-dev.com/v2',
-    ACCOUNTS_CONNECTOR_URL: 'https://accounts.topcoder-dev.com/connector.html',
-    CONNECT_URL_BASE: 'https://connect.topcoder-dev.com/projects/',
-    OWNER_LOGIN_GITHUB_URL: '/api/v1/github/owneruser/login',
-    OWNER_LOGIN_GITLAB_URL: '/api/v1/gitlab/owneruser/login',
-    TOPCODER_URL: 'https://topcoder-dev.com',
-    GITHUB_TEAM_URL: 'https://github.com/orgs/',
-    GITLAB_GROUP_URL: 'https://gitlab.com/groups/',
-  },
-  qa: {
-    JWT_V3_NAME: 'v3jwt',
-    JWT_V2_NAME: 'tcjwt',
-    COOKIES_SECURE: false,
-    TC_LOGIN_URL: 'https://accounts-auth0.topcoder-dev.com/',
-    API_URL: 'https://api.topcoder-qa.com',
-    ADMIN_TOOL_URL: 'https://api.topcoder-qa.com/v2',
-    ACCOUNTS_CONNECTOR_URL: 'https://accounts.topcoder-qa.com/connector.html',
-    CONNECT_URL_BASE: 'https://connect.topcoder-dev.com/projects/',
-    OWNER_LOGIN_GITHUB_URL: '/api/v1/github/owneruser/login',
-    OWNER_LOGIN_GITLAB_URL: '/api/v1/gitlab/owneruser/login',
-    TOPCODER_URL: 'https://topcoder-dev.com',
-    GITHUB_TEAM_URL: 'https://github.com/orgs/',
-    GITLAB_GROUP_URL: 'https://gitlab.com/groups/',
-  },
-  prod: {
-    JWT_V3_NAME: 'v3jwt',
-    JWT_V2_NAME: 'tcjwt',
-    COOKIES_SECURE: false,
-    TC_LOGIN_URL: 'https://accounts.topcoder.com/member',
-    API_URL: 'https://api.topcoder.com',
-    ADMIN_TOOL_URL: 'https://api.topcoder.com/v2',
-    ACCOUNTS_CONNECTOR_URL: 'https://accounts.topcoder.com/connector.html',
-    CONNECT_URL_BASE: 'https://connect.topcoder.com/projects/',
-    OWNER_LOGIN_GITHUB_URL: '/api/v1/github/owneruser/login',
-    OWNER_LOGIN_GITLAB_URL: '/api/v1/gitlab/owneruser/login',
-    TOPCODER_URL: 'https://topcoder-dev.com',
-    GITHUB_TEAM_URL: 'https://github.com/orgs/',
-    GITLAB_GROUP_URL: 'https://gitlab.com/groups/',
-  },
-};
-
-const activeEnv = module.exports.TOPCODER_ENV;
 module.exports.frontendConfigs = {
-  helpLink: module.exports.HELP_LINK,
   copilotRole: module.exports.COPILOT_ROLE,
   administratorRoles: module.exports.ADMINISTRATOR_ROLES,
-  JWT_V3_NAME: process.env.JWT_V3_NAME || frontendConfigs[activeEnv].JWT_V3_NAME,
-  JWT_V2_NAME: process.env.JWT_V2_NAME || frontendConfigs[activeEnv].JWT_V2_NAME,
-  COOKIES_SECURE: process.env.COOKIES_SECURE || frontendConfigs[activeEnv].COOKIES_SECURE,
-  TC_LOGIN_URL: process.env.TC_LOGIN_URL || frontendConfigs[activeEnv].TC_LOGIN_URL,
-  API_URL: process.env.API_URL || frontendConfigs[activeEnv].API_URL,
-  ADMIN_TOOL_URL: process.env.ADMIN_TOOL_URL || frontendConfigs[activeEnv].ADMIN_TOOL_URL,
-  ACCOUNTS_CONNECTOR_URL: process.env.ACCOUNTS_CONNECTOR_URL || frontendConfigs[activeEnv].ACCOUNTS_CONNECTOR_URL,
-  CONNECT_URL_BASE: process.env.CONNECT_URL_BASE || frontendConfigs[activeEnv].CONNECT_URL_BASE,
-  OWNER_LOGIN_GITHUB_URL: process.env.OWNER_LOGIN_GITHUB_URL || frontendConfigs[activeEnv].OWNER_LOGIN_GITHUB_URL,
-  OWNER_LOGIN_GITLAB_URL: process.env.OWNER_LOGIN_GITLAB_URL || frontendConfigs[activeEnv].OWNER_LOGIN_GITLAB_URL,
-  TOPCODER_URL: process.env.TOPCODER_URL || frontendConfigs[activeEnv].TOPCODER_URL,
-  GITHUB_TEAM_URL: process.env.GITHUB_TEAM_URL || frontendConfigs[activeEnv].GITHUB_TEAM_URL,
-  GITLAB_GROUP_URL: process.env.GITLAB_GROUP_URL || frontendConfigs[activeEnv].GITLAB_GROUP_URL,
+  helpLink: process.env.HELP_LINK || 'https://github.com/topcoder-platform/topcoder-x-ui/wiki',
+  JWT_V3_NAME: process.env.JWT_V3_NAME || 'v3jwt',
+  JWT_V2_NAME: process.env.JWT_V2_NAME || 'tcjwt',
+  COOKIES_SECURE: process.env.COOKIES_SECURE || false,
+  TC_LOGIN_URL: process.env.TC_LOGIN_URL || 'https://accounts-auth0.topcoder-dev.com/',
+  ADMIN_TOOL_URL: process.env.ADMIN_TOOL_URL || 'https://api.topcoder-dev.com/v2',
+  ACCOUNTS_CONNECTOR_URL: process.env.ACCOUNTS_CONNECTOR_URL || 'https://accounts.topcoder-dev.com/connector.html',
+  CONNECT_URL_BASE: process.env.CONNECT_URL_BASE || 'https://connect.topcoder-dev.com/projects/',
+  OWNER_LOGIN_GITHUB_URL: process.env.OWNER_LOGIN_GITHUB_URL || '/api/v1/github/owneruser/login',
+  OWNER_LOGIN_GITLAB_URL: process.env.OWNER_LOGIN_GITLAB_URL || '/api/v1/gitlab/owneruser/login',
+  TOPCODER_URL: process.env.TOPCODER_URL || 'https://topcoder-dev.com',
+  GITHUB_TEAM_URL: process.env.GITHUB_TEAM_URL || 'https://github.com/orgs/',
+  GITLAB_GROUP_URL: process.env.GITLAB_GROUP_URL || 'https://gitlab.com/groups/',
 };
diff --git a/src/controllers/GithubController.js b/src/controllers/GithubController.js
@@ -151,6 +151,10 @@ async function addUserToTeamCallback(req, res) {
     .query({client_id: config.GITHUB_CLIENT_ID, client_secret: config.GITHUB_CLIENT_SECRET, code})
     .set('Accept', 'application/json')
     .end();
+  // Throw error if github access token was not returned (e.g. invalid code)
+  if (!result.body.access_token) {
+    throw new errors.UnauthorizedError('Github authorization failed.', result.body.error_description);
+  }
   const token = result.body.access_token;
   // add user to team
   console.log(`adding ${token} to ${team.teamId} with ${team.ownerToken}`); /* eslint-disable-line no-console */
diff --git a/src/controllers/GitlabController.js b/src/controllers/GitlabController.js
@@ -188,6 +188,10 @@ async function addUserToGroupCallback(req, res) {
       redirect_uri: `${config.WEBSITE}/api/${config.API_VERSION}/gitlab/normaluser/callback`,
     })
     .end();
+  // Throw error if github access token was not returned (ex. invalid code)
+  if (!result.body.access_token) {
+    throw new errors.UnauthorizedError('Gitlab authorization failed.', result.body.error_description);
+  }
   const token = result.body.access_token;
 
   // get group name
@@ -235,7 +239,8 @@ async function addUserToGroupCallback(req, res) {
     });
   }
   // redirect to success page
-  res.redirect(`${constants.USER_ADDED_TO_TEAM_SUCCESS_URL}/gitlab/${currentGroup.full_path}`);
+  // For gitlab subgroups we need to replace / with something different. Default encoding doesn't work as angular route fails to match %2F
+  res.redirect(`${constants.USER_ADDED_TO_TEAM_SUCCESS_URL}/gitlab/${currentGroup.full_path.replace('/', '@!2F')}`);
 }
 
 
diff --git a/src/front/src/app/less/custom.less b/src/front/src/app/less/custom.less
@@ -20,7 +20,10 @@
   color: green;
   font-size: 16px;
 }
-
+.red-times-icon {
+  color: red;
+  font-size: 16px;
+}
 .orange-warning-icon {
   color: orange;
   font-size: 16px;
diff --git a/src/front/src/app/members/member.controller.js b/src/front/src/app/members/member.controller.js
@@ -11,8 +11,9 @@ angular.module('topcoderX')
         const org = params[0];
         const team = url.replace(org, '').substring(1);
         $scope.link = $rootScope.appConfig.GITHUB_TEAM_URL + org + '/teams/' + team;
-      } else if (provider === 'github') {
-        $scope.link = $rootScope.appConfig.GITLAB_GROUP_URL + url;
+      } else if (provider === 'gitlab') {
+        // For gitlab subgroups we can't just pass encoded link to this route because anguler doesn't match %2F, so we use @!2F as a workaround
+        $scope.link = $rootScope.appConfig.GITLAB_GROUP_URL + url.replace('@!2F', '/');
       }
     };
     _getUrl($scope.provider, $stateParams.url);
diff --git a/src/front/src/app/projects/projects.html b/src/front/src/app/projects/projects.html
@@ -61,6 +61,7 @@ <h4>You don't have active projects right now. Please
                                                         <th class="col-lg-2">Service Provider</th>
                                                         <th class="col-lg-2" ng-show="isAdminUser">Owner</th>
                                                         <th class="col-lg-2">Copilot</th>
+                                                        <th class="col-lg-2">Copilot payments</th>
                                                         <th class="col-lg-3" data-sort-ignore="true"></th>
                                                     </tr>
                                                 </thead>
@@ -78,6 +79,14 @@ <h4>You don't have active projects right now. Please
                                                         </td>
                                                         <td class="col-lg-2" ng-show="isAdminUser">{{project.owner}}</td>
                                                         <td class="col-lg-2">{{project.copilot}}</td>
+                                                        <td class="col-lg-2" data-value="{{project.createCopilotPayments}}">
+                                                            <div ng-if="project.createCopilotPayments === 'true'">
+                                                                <i class="fa fa-check green-check-icon"/>
+                                                            </div>
+                                                            <div ng-if="project.createCopilotPayments === 'false'">
+                                                                <i class="fa fa-times red-times-icon"/>
+                                                            </div>
+                                                        </td>
                                                         <td class="col-lg-2">
                                                             <button class="btn btn-sm btn-success" ng-click="goProject(project)">
                                                                 <strong>Manage</strong>
diff --git a/src/services/GithubService.js b/src/services/GithubService.js
@@ -178,7 +178,7 @@ getTeamRegistrationUrl.schema = Joi.object().keys({
   token: Joi.string().required(),
   ownerUsername: Joi.string().required(),
   teamId: Joi.string().required(),
-  accessLevel: Joi.string().required(),
+  accessLevel: Joi.string().valid('member', 'maintainer').required(),
 });
 
 /**
diff --git a/src/services/ProjectService.js b/src/services/ProjectService.js
@@ -72,6 +72,20 @@ const createProjectSchema = {
  * @private
  */
 async function _validateProjectData(project) {
+  const filter = {
+    repoUrl: project.repoUrl,
+    archived: 'false'
+  }
+  if (project.id) {
+    filter.id = {
+      ne: project.id
+    }
+  }
+  const existsInDatabase = await dbHelper.scanOne(models.Project, filter)
+  if (existsInDatabase) {
+    throw new errors.ValidationError(`This repo already has a Topcoder-X project associated with it.
+    Copilot: ${existsInDatabase.copilot}, Owner: ${existsInDatabase.owner}`)
+  }
   const provider = await helper.getProviderType(project.repoUrl);
   const userRole = project.copilot ? project.copilot : project.owner;
   const setting = await userService.getUserSetting(userRole);
