Added support for version 4 of the Gitlab API (#100)

* Added support for version 4 of the Gitlab API

* Defaulting to APIv4

* Updated documentation

Author: zyronix

README.md

@@ -7,7 +7,7 @@ gitlab-mirrors creates read only copies of remote repositories in gitlab.  It
 provides a CLI management interface for managing the mirrored repositories (e.g.
 add, delete, update) so that an admin may regularly update all mirrors using
 `crontab`.  It operates by interacting with the [GitLab API][gitlab-api] using
-[python-gitlab3][python-gitlab3].
+[python-gitlab][python-gitlab].
 
 ## Features
 
@@ -87,4 +87,4 @@ Created by Sam Gleske under [MIT License](LICENSE).
 [gm-puppet]: https://github.com/logicminds/gitlab_mirrors
 [issues]: https://github.com/samrocketman/gitlab-mirrors/issues
 [mirror-missing]: http://feedback.gitlab.com/forums/176466-general/suggestions/4286666-mirror-git-svn-into-repo-
-[python-gitlab3]: https://github.com/alexvh/python-gitlab3
+[python-gitlab]: https://github.com/python-gitlab/python-gitlab

add_mirror.sh

@@ -18,8 +18,11 @@ if [ ! -f "${git_mirrors_dir}/config.sh" ];then
   exit 1
 fi
 
+#check if api version is set
+[ -z $gitlab_api_version ] && gitlab_api_version=4
+
 #export env vars for python script
-export gitlab_user_token_secret gitlab_url gitlab_namespace gitlab_user ssl_verify
+export gitlab_user_token_secret gitlab_url gitlab_namespace gitlab_user ssl_verify gitlab_api_version
 
 PROGNAME="${0##*/}"
 PROGVERSION="${VERSION}"

config.sh.SAMPLE

@@ -42,6 +42,8 @@ gitlab_url="https://gitlab.example.com"
 gitlab_user="gitmirror"
 #Generate a token for your $gitlab_user and set it here.
 gitlab_user_token_secret="$(head -n1 "${user_home}/private_token" 2> /dev/null || echo "")"
+#Sets the Gitlab API version, either 3 or 4
+gitlab_api_version=4
 #Verify signed SSL certificates?
 ssl_verify=true
 #Push to GitLab over http?  Otherwise will push projects via SSH.

delete_mirror.sh

@@ -16,8 +16,11 @@ if [ ! -f "${git_mirrors_dir}/config.sh" ];then
   exit 1
 fi
 
+#check if api version is set
+[ -z $gitlab_api_version ] && gitlab_api_version=4
+
 #export env vars for python script
-export gitlab_user_token_secret gitlab_url gitlab_namespace gitlab_user ssl_verify
+export gitlab_user_token_secret gitlab_url gitlab_namespace gitlab_user ssl_verify gitlab_api_version
 
 cd "${git_mirrors_dir}"
 

docs/prerequisites.md

@@ -2,8 +2,8 @@
 
 ### Required software
 
-* [Tested with GitLab 6.x/7.x/8.x][gitlab]
-* [pyapi-gitlab3 @ v0.5.4][python-gitlab3]
+* [Tested with GitLab 8.x/9.x/10/x][gitlab]
+* [python-gitlab @ v1.1.0][python-gitlab]
 * [GNU coreutils][coreutils]
 * [git 1.8.0][git] or later
 
@@ -24,14 +24,10 @@ aditional options.
 
 ### Required software install snippets
 
-#### python-gitlab3
-
-    yum install python-setuptools
-    git clone https://github.com/alexvh/python-gitlab3.git
-    cd python-gitlab3
-    git checkout v0.5.4
-    python setup.py install
+#### python-gitlab
 
+    yum install python-pip
+    pip install python-gitlab
 
 #### Installing git
 
@@ -93,4 +89,4 @@ Next up is [Installation and Setup](installation.md).
 [gitlab]: https://about.gitlab.com/
 [git-src]: http://code.google.com/p/git-core/
 [git-svn]: https://www.kernel.org/pub/software/scm/git/docs/git-svn.html
-[python-gitlab3]: https://github.com/alexvh/python-gitlab3
+[python-gitlab]: https://github.com/python-gitlab/python-gitlab

docs/upgrade/upgrade_0.5.x-0.6.x.md

@@ -0,0 +1,36 @@
+# Upgrade Notes
+
+This documentation outlines steps for you to upgrade from gitlab-mirrors `0.5.x`
+to gitlab-mirrors `0.6.x`.  It is assumed you'll be working on a test instance
+of gitlab in a production environment.  If you only have a single gitlab
+instance then follow these steps with care and at your own risk.
+
+gitlab-mirrors has been certified to use a new prerequisite library called
+[python-gitlab](https://github.com/python-gitlab/python-gitlab).  Therefore 
+you must install `python-gitlab` before upgrading `gitlab-mirrors` to the latest 
+edition.
+
+# 1. Disable any cron jobs
+
+If you have cron jobs set up then you'll need to disable them to avoid them
+launching gitlab-mirrors during your upgrade.
+
+# 2. Update python-gitlab3 python-gitlab
+
+I'll ouline the steps here real quick.
+
+    yum install python-pip
+    pip uninstall gitlab3
+    pip install python-gitlab
+
+# 3. Update your gitlab-mirrors
+
+    su - gitmirror
+    cd gitlab-mirrors
+    git checkout master
+    git fetch
+    git pull origin master
+    git checkout v0.6.x
+
+Test on a dummy project to ensure your new setup works.  Once you have verified
+everything works then you can re-enable the cron jobs.

lib/manage_gitlab_project.py

@@ -7,9 +7,9 @@
 from optparse import OptionParser
 import os
 try:
-  import gitlab3 as gitlab
+  import gitlab
 except ImportError:
-  raise ImportError("python-gitlab3 module is not installed.  You probably didn't read the install instructions closely enough.  See docs/prerequisites.md.")
+  raise ImportError("python-gitlab module is not installed.  You probably didn't read the install instructions closely enough.  See docs/prerequisites.md.")
 
 
 
@@ -19,6 +19,7 @@
   gitlab_namespace=os.environ['gitlab_namespace']
   gitlab_user=os.environ['gitlab_user']
   ssl_verify=os.environ['ssl_verify']
+  gitlab_api_version=os.environ['gitlab_api_version']
 except KeyError:
   print >> stderr, "Environment config missing.  Do not run this script standalone."
   exit(1)
@@ -44,14 +45,44 @@
 project_name=args[0]
 
 if not eval(ssl_verify.capitalize()):
-  git=gitlab.GitLab(gitlab_url=gitlab_url,token=token_secret,ssl_verify=False)
+  git=gitlab.Gitlab(gitlab_url,token_secret,ssl_verify=False,api_version=gitlab_api_version)
 else:
-  git=gitlab.GitLab(gitlab_url=gitlab_url,token=token_secret,ssl_verify=True)
+  git=gitlab.Gitlab(gitlab_url,token_secret,ssl_verify=True,api_version=gitlab_api_version)
+
+def find_group(**kwargs):
+  groups = git.groups.list()
+  return _find_matches(groups, kwargs, False)
+
+def find_project(**kwargs):
+  projects = git.projects.list(as_list=True)
+  return _find_matches(projects, kwargs, False)
+
+def _find_matches(objects, kwargs, find_all):
+  """Helper function for _add_find_fn. Find objects whose properties
+  match all key, value pairs in kwargs.
+  Source: https://github.com/doctormo/python-gitlab3/blob/master/gitlab3/__init__.py
+  """
+  ret = []
+  for obj in objects:
+    match = True
+    # Match all supplied parameters
+    for param, val in kwargs.items():
+      if not getattr(obj, param) == val:
+        match = False
+        break
+      if match:
+        if find_all:
+          ret.append(obj)
+        else:
+          return obj
+  if not find_all:
+    return None
+  return ret
 
 # transfer the project from the source namespace to the specified group namespace
 def transfer_project(src_project, group):
   value = group.transfer_project(src_project.id)
-  dest_project = git.find_project(name=src_project.name)
+  dest_project = find_project(name=src_project.name)
   return dest_project
 
 def createproject(pname):
@@ -69,14 +100,16 @@ def createproject(pname):
     'wiki_enabled': options.wiki,
     'snippets_enabled': options.snippets,
     'public': options.public,
-    'namespace_id': git.find_group(name=gitlab_namespace).id,
+    'namespace_id': find_group(name=gitlab_namespace).id,
   }
   #make all project options lowercase boolean strings i.e. true instead of True
   for x in project_options.keys():
     project_options[x] = str(project_options[x]).lower()
   print >> stderr, "Creating new project %s" % pname
-  git.add_project(pname,description=description,**project_options)
-  found_project = git.find_project(name=pname)
+  project_options['name'] = pname
+  project_options['description'] = description
+  git.projects.create(project_options)
+  found_project = find_project(name=pname)
   if needs_transfer(gitlab_user, gitlab_namespace, found_project):
      found_project = transfer_project(found_project, found_group)
   return found_project
@@ -88,13 +121,19 @@ def needs_transfer(user, groupname, project):
     namespace = groupname
   else:
     namespace = user
-  return project.namespace['name'] != namespace
+  if type(project.namespace) == gitlab.v3.objects.Group:
+    return project.namespace.name != namespace
+  else:
+    return project.namespace['name'] != namespace
+  
 
 if options.create:
-  found_group=git.find_group(name=gitlab_namespace)
+  found_group = find_group(name=gitlab_namespace)
   found_project = None
-  # search the group namespace first
-  found_project=git.find_project(name=project_name)
+
+  found_project= find_project(name=project_name)
+  #exit()
+  
   if found_project:
     if needs_transfer(gitlab_user, gitlab_namespace, found_project):
       found_project = transfer_project(found_project, found_group)
@@ -112,7 +151,7 @@ def needs_transfer(user, groupname, project):
     print found_project.ssh_url_to_repo
 elif options.delete:
   try:
-    deleted_project=git.find_project(name=project_name).delete()
+    deleted_project=find_project(name=project_name).delete()
   except Exception as e:
     print >> stderr, e
     exit(1)

requirements.txt

@@ -0,0 +1 @@
+python-gitlab>=1.0.2,<2