Major fork with Github Issues Import API support

gh-issues-import.py

@@ -6,6 +6,7 @@
 import sys, os
 import datetime
 import argparse, configparser
+import time
 
 import query
 
@@ -28,7 +29,6 @@ class state:
 
 http_error_messages = {}
 http_error_messages[401] = "ERROR: There was a problem during authentication.\nDouble check that your username and password are correct, and that you have permission to read from or write to the specified repositories."
-http_error_messages[403] = http_error_messages[401]; # Basically the same problem. GitHub returns 403 instead to prevent abuse.
 http_error_messages[404] = "ERROR: Unable to find the specified repository.\nDouble check the spelling for the source and target repositories. If either repository is private, make sure the specified user is allowed access to it."
 
 
@@ -54,6 +54,8 @@ def init_config():
 	arg_parser.add_argument('--ignore-comments',  dest='ignore_comments',  action='store_true', help="Do not import comments in the issue.")		
 	arg_parser.add_argument('--ignore-milestone', dest='ignore_milestone', action='store_true', help="Do not import the milestone attached to the issue.")
 	arg_parser.add_argument('--ignore-labels',    dest='ignore_labels',    action='store_true', help="Do not import labels attached to the issue.")
+	arg_parser.add_argument('--ignore-pullrequests',    dest='ignore_pullrequests',    action='store_true', help="Do not import issues that are pull requests.")
+	arg_parser.add_argument('--use-github-import-api', dest='use_github_import_api', action='store_true', help="Use the new github import api")
 
 	arg_parser.add_argument('--issue-template', help="Specify a template file for use with issues.")
 	arg_parser.add_argument('--comment-template', help="Specify a template file for use with comments.")
@@ -63,7 +65,7 @@ def init_config():
 	include_group.add_argument("--all", dest='import_all', action='store_true', help="Import all issues, regardless of state.")
 	include_group.add_argument("--open", dest='import_open', action='store_true', help="Import only open issues.")
 	include_group.add_argument("--closed", dest='import_closed', action='store_true', help="Import only closed issues.")
-	include_group.add_argument("-i", "--issues", type=int, nargs='+', help="The list of issues to import.");
+	include_group.add_argument("-i", "--issues", type=int, nargs='+', help="The list of issues to import.")
 
 	args = arg_parser.parse_args()
 
@@ -106,8 +108,10 @@ def load_config_file(config_file_name):
 	config.set('settings', 'import-milestone', str(not args.ignore_milestone))
 	config.set('settings', 'import-labels',    str(not args.ignore_labels))
 
-	config.set('settings', 'import-open-issues',   str(args.import_all or args.import_open));
-	config.set('settings', 'import-closed-issues', str(args.import_all or args.import_closed));
+	config.set('settings', 'import-open-issues',   str(args.import_all or args.import_open))
+	config.set('settings', 'import-closed-issues', str(args.import_all or args.import_closed))
+	config.set('settings', 'import-pullrequests',  str(not args.ignore_pullrequests))
+	config.set('settings', 'use-github-import-api', str(args.use_github_import_api))
 
 
 	# Make sure no required config values are missing
@@ -187,45 +191,63 @@ def format_comment(template_data):
 	template = config.get('format', 'comment_template', fallback=default_template)
 	return format_from_template(template, template_data)
 
-def send_request(which, url, post_data=None):
+def send_request(which, url, post_data=None, method=None, accept=None):
 
 	if post_data is not None:
 		post_data = json.dumps(post_data).encode("utf-8")
 
 	full_url = "%s/%s" % (config.get(which, 'url'), url)
-	req = urllib.request.Request(full_url, post_data)
+	req = urllib.request.Request(full_url, post_data, method=method)
 
 	username = config.get(which, 'username')
 	password = config.get(which, 'password')
 	req.add_header("Authorization", b"Basic " + base64.urlsafe_b64encode(username.encode("utf-8") + b":" + password.encode("utf-8")))
 
 	req.add_header("Content-Type", "application/json")
-	req.add_header("Accept", "application/json")
+	if accept is not None:
+		req.add_header("Accept", accept)
+	else:
+		req.add_header("Accept", "application/json")
 	req.add_header("User-Agent", "IQAndreas/github-issues-import")
-
-	try:
-		response = urllib.request.urlopen(req)
-		json_data = response.read()
-	except urllib.error.HTTPError as error:
-
-		error_details = error.read();
-		error_details = json.loads(error_details.decode("utf-8"))
-
-		if error.code in http_error_messages:
-			sys.exit(http_error_messages[error.code])
-		else:
-			error_message = "ERROR: There was a problem importing the issues.\n%s %s" % (error.code, error.reason)
-			if 'message' in error_details:
-				error_message += "\nDETAILS: " + error_details['message']
-			sys.exit(error_message)
+
+	retry = True
+	while retry:
+		retry = False
+		try:
+			response = urllib.request.urlopen(req)
+			json_data = response.read()
+		except urllib.error.HTTPError as error:
+
+			error_details = error.read();
+			error_details = json.loads(error_details.decode("utf-8"))
+
+			if error.code == 403:
+				print("Got 403 [%s], assuming rate limit error and waiting for 1 minute..." % error_details['message'])
+				time.sleep(60)
+				retry = True
+			elif error.code in http_error_messages:
+				sys.exit(http_error_messages[error.code])
+			else:
+				error_message = "ERROR: There was a problem importing the issues.\n%s %s" % (error.code, error.reason)
+				if 'message' in error_details:
+					error_message += "\nDETAILS: " + error_details['message']
+				sys.exit(error_message)
 
 	return json.loads(json_data.decode("utf-8"))
 
 def get_milestones(which):
 	return send_request(which, "milestones?state=open")
 
 def get_labels(which):
-	return send_request(which, "labels")
+	page = 1
+	labels = []
+	while True:
+		next_labels = send_request(which, "labels?page=%s" % page)
+		if next_labels:
+			labels.extend(next_labels)
+			page += 1
+		else:
+			return labels
 
 def get_issue_by_id(which, issue_id):
 	return send_request(which, "issues/%d" % issue_id)
@@ -242,11 +264,15 @@ def get_issues_by_id(which, issue_ids):
 def get_issues_by_state(which, state):
 	issues = []
 	page = 1
+	import_pullrequests = config.getboolean('settings', 'import-pullrequests')
 	while True:
 		new_issues = send_request(which, "issues?state=%s&direction=asc&page=%d" % (state, page))
 		if not new_issues:
 			break
-		issues.extend(new_issues)
+		if import_pullrequests:
+			issues.extend(new_issues)
+		else:
+			issues.extend(filter(lambda issue:'pull_request' not in issue ,new_issues))
 		page += 1
 	return issues
 
@@ -264,7 +290,7 @@ def import_milestone(source):
 		"due_on": source['due_on']
 	}
 
-	result_milestone = send_request('target', "milestones", source)
+	result_milestone = send_request('target', "milestones", data)
 	print("Successfully created milestone '%s'" % result_milestone['title'])
 	return result_milestone
 
@@ -274,7 +300,7 @@ def import_label(source):
 		"color": source['color']
 	}
 
-	result_label = send_request('target', "labels", source)
+	result_label = send_request('target', "labels", data)
 	print("Successfully created label '%s'" % result_label['name'])
 	return result_label
 
@@ -297,6 +323,141 @@ def import_comments(comments, issue_number):
 
 	return result_comments
 
+def import_issues_golden_comet(issues):
+	state.current = state.GENERATING
+
+	known_milestones = get_milestones('target')
+	def get_milestone_by_title(title):
+		for milestone in known_milestones:
+			if milestone['title'] == title : return milestone
+		return None
+
+	known_labels = get_labels('target')
+	def get_label_by_name(name):
+		for label in known_labels:
+			if label['name'] == name : return label
+		return None
+
+	issue_migrations = []
+	new_milestones = []
+	num_new_comments = 0
+	new_labels = []
+
+	for issue in issues:
+		issue_migration = {}
+		new_issue = {}
+		new_issue['title'] = issue['title']
+		if issue['closed_at'] is not None:
+			new_issue['closed_at'] = issue['closed_at']
+			new_issue['closed'] = True
+		new_issue['created_at'] = issue['created_at']
+		new_issue['updated_at'] = issue['updated_at']
+		# TODO new_issue['assignee'] = issue['assignee']
+
+		if config.getboolean('settings', 'import-milestone') and 'milestone' in issue and issue['milestone'] is not None:
+			# Since the milestones' ids are going to differ, we will compare them by title instead
+			found_milestone = get_milestone_by_title(issue['milestone']['title'])
+			if found_milestone:
+				new_issue['milestone_object'] = found_milestone
+			else:
+				new_milestone = issue['milestone']
+				new_issue['milestone_object'] = new_milestone
+				known_milestones.append(new_milestone) # Allow it to be found next time
+				new_milestones.append(new_milestone)   # Put it in a queue to add it later
+
+		if config.getboolean('settings', 'import-labels') and 'labels' in issue and issue['labels'] is not None:
+			new_issue['label_objects'] = []
+			for issue_label in issue['labels']:
+				found_label = get_label_by_name(issue_label['name'])
+				if found_label:
+					new_issue['label_objects'].append(found_label)
+				else:
+					new_issue['label_objects'].append(issue_label)
+					known_labels.append(issue_label) # Allow it to be found next time
+					new_labels.append(issue_label)   # Put it in a queue to add it later
+
+		comments = []
+		if config.getboolean('settings', 'import-comments') and 'comments' in issue and issue['comments'] != 0:
+			num_new_comments += int(issue['comments'])
+			original_comments = get_comments_on_issue('source', issue)
+			for original_comment in original_comments:
+				comment = {}
+				ctemplate_data = {}
+				ctemplate_data['user_name'] = original_comment['user']['login']
+				ctemplate_data['user_url'] = original_comment['user']['html_url']
+				ctemplate_data['user_avatar'] = original_comment['user']['avatar_url']
+				ctemplate_data['date'] = format_date(original_comment['created_at'])
+				ctemplate_data['url'] =  original_comment['html_url']
+				ctemplate_data['body'] = original_comment['body']
+
+				comment['body'] = format_comment(ctemplate_data)
+				comment['created_at'] = original_comment['created_at']
+				comments.append(comment)
+
+		template_data = {}
+		template_data['user_name'] = issue['user']['login']
+		template_data['user_url'] = issue['user']['html_url']
+		template_data['user_avatar'] = issue['user']['avatar_url']
+		template_data['date'] = format_date(issue['created_at'])
+		template_data['url'] =  issue['html_url']
+		template_data['body'] = issue['body']
+
+		if "pull_request" in issue and issue['pull_request']['html_url'] is not None:
+			new_issue['body'] = format_pull_request(template_data)
+		else:
+			new_issue['body'] = format_issue(template_data)
+
+		issue_migration['issue'] = new_issue
+		issue_migration['comments'] = comments
+
+		issue_migrations.append(issue_migration)
+
+	state.current = state.IMPORT_CONFIRMATION
+
+	print("You are about to add to '" + config.get('target', 'repository') + "':")
+	print(" *", len(issue_migrations), "new issues") 
+	print(" *", num_new_comments, "new comments") 
+	print(" *", len(new_milestones), "new milestones") 
+	print(" *", len(new_labels), "new labels") 
+	if not query.yes_no("Are you sure you wish to continue?"):
+		sys.exit()
+
+	state.current = state.IMPORTING
+
+	for milestone in new_milestones:
+		result_milestone = import_milestone(milestone)
+		milestone['number'] = result_milestone['number']
+		milestone['url'] = result_milestone['url']
+
+	for label in new_labels:
+		result_label = import_label(label)
+
+	migration_results = []
+	for issue_migration in issue_migrations:
+		issue = issue_migration['issue']
+		if 'milestone_object' in issue:
+			issue['milestone'] = issue['milestone_object']['number']
+			del issue['milestone_object']
+
+		if 'label_objects' in issue:
+			issue_labels = []
+			for label in issue['label_objects']:
+				issue_labels.append(label['name'])
+			issue['labels'] = issue_labels
+			del issue['label_objects']
+
+		migration_result = send_request('target', "import/issues", issue_migration, "POST", "application/vnd.github.golden-comet-preview+json")
+		print("Sent import request. Status: '%s'. Status url: '%s'" % (migration_result['status'], migration_result['url']))
+
+		migration_results.append(migration_result)
+
+	state.current = state.IMPORT_COMPLETE
+
+	# TODO wait for import requests to complete
+
+	return issue_migrations
+
+
 # Will only import milestones and issues that are in use by the imported issues, and do not exist in the target repository
 def import_issues(issues):
 
@@ -326,7 +487,7 @@ def get_label_by_name(name):
 
 		# Temporary fix for marking closed issues
 		if issue['closed_at']:
-			new_issue['title'] = "[CLOSED] " + new_issue['title']
+			new_issue['state'] = "closed"
 
 		if config.getboolean('settings', 'import-comments') and 'comments' in issue and issue['comments'] != 0:
 			num_new_comments += int(issue['comments'])
@@ -411,6 +572,12 @@ def get_label_by_name(name):
 			print(" > Successfully added", len(result_comments), "comments.")
 
 		result_issues.append(result_issue)
+
+		# add issue closing
+		if 'state' in issue and issue['state'] == 'closed':
+			number = result_issue['number']
+			result_issue = send_request('target', 'issues/%s' % number, issue, 'PATCH')
+			print(" > Successfully closed issue '%s'" % result_issue['title'])
 
 	state.current = state.IMPORT_COMPLETE
 
@@ -442,7 +609,10 @@ def get_label_by_name(name):
 
 	# Further states defined within the function
 	# Finally, add these issues to the target repository
-	import_issues(issues)
+	if config.getboolean('settings', 'use-github-import-api'):
+		import_issues_golden_comet(issues)
+	else:
+		import_issues(issues)
 
 	state.current = state.COMPLETE