Fix #2 and #3; improve debugging a bit

README.rst

@@ -38,7 +38,7 @@ were never merged.
        --gitlab-url http://workbench.dachary.org \
        --gitlab-token sxQJ67SQKihMrGWVf \
        --gitlab-repo ceph/ceph-backports \
-       --github-token 64933d355fda9844aadd4e224d \
+       --github-auth githubusername:64933d355fda9844aadd4e224d \
        --github-repo ceph/ceph \
        --ignore-closed
 
@@ -90,7 +90,7 @@ Hacking
     --gitlab-url http://workbench.dachary.org \
     --gitlab-token XXXXXXXXX \
     --gitlab-repo dachary/testrepo2 \
-    --github-token XXXXXXXXX \
+    --github-auth XXXXXXXXX \
     --github-repo dachary/testrepo \
     --ssh-public-key ~/.ssh/id_rsa.pub \
     --verbose  

github2gitlab/main.py

@@ -21,6 +21,7 @@
 import git
 import gitdb
 import hashlib
+from http.client import HTTPConnection
 import json
 import logging
 import os
@@ -57,15 +58,18 @@ def __init__(self, args):
 
         if not self.args.gitlab_repo:
             self.args.gitlab_repo = self.args.github_repo
-        (self.args.gitlab_namespace,
-         self.args.gitlab_name) = self.args.gitlab_repo.split('/')
+
+        repo_parts = self.args.gitlab_repo.split('/')
+        self.args.gitlab_name = repo_parts.pop()
+        self.args.gitlab_namespace = '/'.join(repo_parts)
+
         self.args.gitlab_repo = parse.quote_plus(self.args.gitlab_repo)
 
         self.github = {
             'url': "https://api.github.com",
             'git': "https://github.com",
             'repo': self.args.github_repo,
-            'token': self.args.github_token,
+            'auth': self.args.github_auth or None
         }
         if self.args.branches:
             self.github['branches'] = self.args.branches.split(',')
@@ -74,18 +78,29 @@ def __init__(self, args):
             'host': self.args.gitlab_url,
             'name': self.args.gitlab_name,
             'namespace': self.args.gitlab_namespace,
+            'group': None,
             'url': self.args.gitlab_url + "/api/v4",
             'repo': self.args.gitlab_repo,
             'token': self.args.gitlab_token,
         }
 
         if self.args.verbose:
             level = logging.DEBUG
+            HTTPConnection.debuglevel = 1
         else:
             level = logging.INFO
 
+        logging.getLogger("urllib3").setLevel(level)
         logging.getLogger('github2gitlab').setLevel(level)
 
+        g = self.gitlab
+        url = g['url'] + "/groups"
+        query = {'private_token': g['token'], 'all_available': 'true', 'per_page': 10000}
+        groups = requests.get(url, params=query).json()
+        matches = list(filter(lambda group: group['full_path'] == g['namespace'].lower(), groups))
+        if any(matches):
+            g['group_id'] = matches[0]['id']
+
         self.tmpdir = "/tmp"
 
     @staticmethod
@@ -101,8 +116,8 @@ def get_parser():
                             required=True)
         parser.add_argument('--gitlab-repo',
                             help='Gitlab repo (for instance ceph/ceph)')
-        parser.add_argument('--github-token',
-                            help='GitHub authentication token')
+        parser.add_argument('--github-auth',
+                            help='GitHub auth credentials, in the form username:token')
         parser.add_argument('--github-repo',
                             help='GitHub repo (for instance ceph/ceph)',
                             required=True)
@@ -115,6 +130,9 @@ def get_parser():
         parser.add_argument('--ignore-closed', action='store_const',
                             const=True,
                             help='ignore pull requests closed and not merged')
+        parser.add_argument('--skip-add-key', action='store_const',
+                            const=True,
+                            help='do not attempt to add local SSH key to Gitlab')
         parser.add_argument('--skip-pull-requests', action='store_const',
                             const=True,
                             help='do not mirror PR to MR')
@@ -127,14 +145,18 @@ def get_parser():
         parser.add_argument('--clean', action='store_const',
                             const=True,
                             help='Remove the repo after sync')
+        parser.add_argument('--visibility',
+                            help='Visbility of created repos (public, internal, private)',
+                            default='public')
         return parser
 
     @staticmethod
     def factory(argv):
         return GitHub2GitLab(GitHub2GitLab.get_parser().parse_args(argv))
 
     def run(self):
-        self.add_key()
+        if not self.args.skip_add_key:
+            self.add_key()
         if self.add_project():
             self.unprotect_branches()
         self.git_mirror()
@@ -153,8 +175,7 @@ def sh(self, command):
             args=command,
             stdout=subprocess.PIPE,
             stderr=subprocess.STDOUT,
-            shell=True,
-            bufsize=1)
+            shell=True)
         lines = []
         with proc.stdout:
             for line in iter(proc.stdout.readline, b''):
@@ -179,9 +200,15 @@ def gitlab_create_remote(self, repo):
 
     def git_mirror(self):
         name = self.gitlab['name']
+        url = self.github['git']
+
+        if(self.github['auth']):
+            url = self.github['git'].replace('https://', 'https://{}@'.format(self.github['auth']))
+
         if not os.path.exists(name):
-            self.sh("git clone --bare " + self.github['git'] +
+            self.sh("git clone --bare " + url +
                     "/" + self.github['repo'] + " " + name)
+
         repo = git.Repo(name)
         os.chdir(name)
         if not hasattr(repo.remotes, 'gitlab'):
@@ -219,7 +246,7 @@ def git_mirror(self):
     def git_mirror_optimize(self, repo):
         self.sh("git fetch origin +refs/pull/*:refs/remotes/origin/pull/*")
         for head in repo.refs:
-            pr = re.search('^origin/pull/(\d+)/head$', head.name)
+            pr = re.search('^origin/pull/(\\d+)/head$', head.name)
             if not pr:
                 continue
             pr = pr.group(1)
@@ -285,15 +312,21 @@ def add_project(self):
         g = self.gitlab
         url = g['url'] + "/projects/" + g['repo']
         query = {'private_token': g['token']}
+        if g['group_id']:
+            query['group_id'] = g['group_id']
+
         if (requests.get(url, params=query).status_code == requests.codes.ok):
             log.debug("project " + url + " already exists")
             return None
         else:
-            log.info("add project " + g['repo'])
             url = g['url'] + "/projects"
-            query['public'] = 'true'
-            query['namespace'] = g['namespace']
+            query['visibility'] = self.args.visibility
             query['name'] = g['name']
+            if g['group_id']:
+                query['namespace_id'] = g['group_id']
+            else:
+                query['namespace'] = g['namespace']
+            log.info("add project " + g['repo'])
             result = requests.post(url, params=query)
             if result.status_code != requests.codes.created:
                 raise ValueError(result.text)
@@ -484,8 +517,8 @@ def get_pull_requests(self):
         "https://developer.github.com/v3/pulls/#list-pull-requests"
         g = self.github
         query = {'state': 'all'}
-        if self.args.github_token:
-            query['access_token'] = g['token']
+        if g['auth']:
+            query['access_token'] = g['auth'].split(':')[1]
 
         def f(pull):
             if self.args.ignore_closed:
@@ -511,7 +544,7 @@ def create_merge_request(self, query):
         g = self.gitlab
         query['private_token'] = g['token']
         url = g['url'] + "/projects/" + g['repo'] + "/merge_requests"
-        log.info('create_merge_request: ' + str(query))
+        log.debug('create_merge_request: ' + str(query))
         result = requests.post(url, params=query)
         if result.status_code != requests.codes.created:
             raise ValueError(result.text)
@@ -547,7 +580,7 @@ def put_merge_request(self, merge_request, updates):
         updates['private_token'] = g['token']
         url = (g['url'] + "/projects/" + g['repo'] + "/merge_requests/" +
                str(merge_request['iid']))
-        log.info('update_merge_request: ' + url + ' <= ' + str(updates))
+        log.debug('update_merge_request: ' + url + ' <= ' + str(updates))
         return requests.put(url, params=updates).json()
 
     def verify_merge_update(self, updates, result):