Merge branch 'development' release 0.6.0

samrocketman · samrocketman · commit dd28d50faf46 · 2018-08-10T18:12:44.000-07:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,13 @@
+---
+## gitlab-mirrors v0.6.0
+
+* Consolidate user options and deps inclusion. See [PR #104][#104]
+* Added support for version 4 of the Gitlab API. See [PR #100][#100]
+
+[#100]: https://github.com/samrocketman/gitlab-mirrors/pull/100
+[#104]: https://github.com/samrocketman/gitlab-mirrors/pull/104
+
+
 ---
 ## gitlab-mirrors v0.5.3
 
diff --git a/README.md b/README.md
@@ -7,7 +7,7 @@ gitlab-mirrors creates read only copies of remote repositories in gitlab.  It
 provides a CLI management interface for managing the mirrored repositories (e.g.
 add, delete, update) so that an admin may regularly update all mirrors using
 `crontab`.  It operates by interacting with the [GitLab API][gitlab-api] using
-[python-gitlab3][python-gitlab3].
+[python-gitlab][python-gitlab].
 
 ## Features
 
@@ -81,10 +81,11 @@ Created by Sam Gleske under [MIT License](LICENSE).
 * Docs #54  [Martijn Vermaat](https://github.com/martijnvermaat)
 * Better logging #57 [Loic Dachary](https://github.com/dachary)
 * Fixed project transfer bug #78 [Corey Osman](https://github.com/logicminds)
+* Added support for GitLab API v4 #100 [Romke van Dijk](https://github.com/zyronix)
 
 [gitlab-api]: http://api.gitlab.org/
 [gm-docker]: https://github.com/Klowner/docker-gitlab-mirrors
 [gm-puppet]: https://github.com/logicminds/gitlab_mirrors
 [issues]: https://github.com/samrocketman/gitlab-mirrors/issues
 [mirror-missing]: http://feedback.gitlab.com/forums/176466-general/suggestions/4286666-mirror-git-svn-into-repo-
-[python-gitlab3]: https://github.com/alexvh/python-gitlab3
+[python-gitlab]: https://github.com/python-gitlab/python-gitlab
diff --git a/add_mirror.sh b/add_mirror.sh
@@ -10,16 +10,13 @@ set -e
 
 #Include all user options and dependencies
 git_mirrors_dir="${0%/*}"
-[ -f "${git_mirrors_dir}/config.sh" ] && . "${git_mirrors_dir}/config.sh"
-. "${git_mirrors_dir}/lib/VERSION"
-. "${git_mirrors_dir}/lib/functions.sh"
-if [ ! -f "${git_mirrors_dir}/config.sh" ];then
-  red_echo "config.sh missing!  Copy and customize from config.sh.SAMPLE.  Aborting." 1>&2
-  exit 1
-fi
+source ${git_mirrors_dir}/includes.sh
+
+#check if api version is set
+[ -z $gitlab_api_version ] && gitlab_api_version=4
 
 #export env vars for python script
-export gitlab_user_token_secret gitlab_url gitlab_namespace gitlab_user ssl_verify
+export gitlab_user_token_secret gitlab_url gitlab_namespace gitlab_user ssl_verify gitlab_api_version
 
 PROGNAME="${0##*/}"
 PROGVERSION="${VERSION}"
diff --git a/config.sh.SAMPLE b/config.sh.SAMPLE
@@ -42,6 +42,8 @@ gitlab_url="https://gitlab.example.com"
 gitlab_user="gitmirror"
 #Generate a token for your $gitlab_user and set it here.
 gitlab_user_token_secret="$(head -n1 "${user_home}/private_token" 2> /dev/null || echo "")"
+#Sets the Gitlab API version, either 3 or 4
+gitlab_api_version=4
 #Verify signed SSL certificates?
 ssl_verify=true
 #Push to GitLab over http?  Otherwise will push projects via SSH.
diff --git a/delete_mirror.sh b/delete_mirror.sh
@@ -8,16 +8,13 @@ set -e
 
 #Include all user options and dependencies
 git_mirrors_dir="${0%/*}"
-[ -f "${git_mirrors_dir}/config.sh" ] && . "${git_mirrors_dir}/config.sh"
-. "${git_mirrors_dir}/lib/VERSION"
-. "${git_mirrors_dir}/lib/functions.sh"
-if [ ! -f "${git_mirrors_dir}/config.sh" ];then
-  red_echo "config.sh missing!  Copy and customize from config.sh.SAMPLE.  Aborting." 1>&2
-  exit 1
-fi
+source ${git_mirrors_dir}/includes.sh
+
+#check if api version is set
+[ -z $gitlab_api_version ] && gitlab_api_version=4
 
 #export env vars for python script
-export gitlab_user_token_secret gitlab_url gitlab_namespace gitlab_user ssl_verify
+export gitlab_user_token_secret gitlab_url gitlab_namespace gitlab_user ssl_verify gitlab_api_version
 
 cd "${git_mirrors_dir}"
 
diff --git a/docs/prerequisites.md b/docs/prerequisites.md
@@ -2,8 +2,8 @@
 
 ### Required software
 
-* [Tested with GitLab 6.x/7.x/8.x][gitlab]
-* [pyapi-gitlab3 @ v0.5.4][python-gitlab3]
+* [Tested with GitLab 8.x/9.x/10/x][gitlab]
+* [python-gitlab @ v1.1.0][python-gitlab]
 * [GNU coreutils][coreutils]
 * [git 1.8.0][git] or later
 
@@ -24,14 +24,10 @@ aditional options.
 
 ### Required software install snippets
 
-#### python-gitlab3
-
-    yum install python-setuptools
-    git clone https://github.com/alexvh/python-gitlab3.git
-    cd python-gitlab3
-    git checkout v0.5.4
-    python setup.py install
+#### python-gitlab
 
+    yum install python-pip
+    pip install python-gitlab
 
 #### Installing git
 
@@ -93,4 +89,4 @@ Next up is [Installation and Setup](installation.md).
 [gitlab]: https://about.gitlab.com/
 [git-src]: http://code.google.com/p/git-core/
 [git-svn]: https://www.kernel.org/pub/software/scm/git/docs/git-svn.html
-[python-gitlab3]: https://github.com/alexvh/python-gitlab3
+[python-gitlab]: https://github.com/python-gitlab/python-gitlab
diff --git a/docs/upgrade/upgrade_0.5.x-0.6.x.md b/docs/upgrade/upgrade_0.5.x-0.6.x.md
@@ -0,0 +1,47 @@
+# Upgrade Notes
+
+This documentation outlines steps for you to upgrade from gitlab-mirrors `0.5.x`
+to gitlab-mirrors `0.6.x`.  It is assumed you'll be working on a test instance
+of gitlab in a production environment.  If you only have a single gitlab
+instance then follow these steps with care and at your own risk.
+
+gitlab-mirrors has been certified to use a new prerequisite library called
+[python-gitlab](https://github.com/python-gitlab/python-gitlab).  Therefore you
+must install `python-gitlab` before upgrading `gitlab-mirrors` to the latest
+edition.
+
+# 1. Disable any cron jobs
+
+If you have cron jobs set up then you'll need to disable them to avoid them
+launching gitlab-mirrors during your upgrade.
+
+# 2. Update python-gitlab3 python-gitlab
+
+I'll ouline the steps here real quick.
+
+    yum install python-pip
+    pip uninstall gitlab3
+    pip install python-gitlab
+
+# 3. Update your gitlab-mirrors
+
+    su - gitmirror
+    cd gitlab-mirrors
+    git checkout master
+    git fetch
+    git pull origin master
+    git checkout v0.6.x
+
+Test on a dummy project to ensure your new setup works.  Once you have verified
+everything works then you can re-enable the cron jobs.
+
+# Alternatively
+
+Rely on virtualenv.
+
+    virtualenv -p python2.7 .venv
+    source ./.venv/bin/activate
+    pip install python-gitlab
+
+Be sure to source ./.venv/bin/activate before running gitlab-mirrors shell
+scripts.
diff --git a/git-mirrors.sh b/git-mirrors.sh
@@ -5,13 +5,7 @@
 
 #Include all user options and dependencies
 git_mirrors_dir="${0%/*}"
-[ -f "${git_mirrors_dir}/config.sh" ] && . "${git_mirrors_dir}/config.sh"
-. "${git_mirrors_dir}/lib/VERSION"
-. "${git_mirrors_dir}/lib/functions.sh"
-if [ ! -f "${git_mirrors_dir}/config.sh" ];then
-  red_echo "config.sh missing!  Copy and customize from config.sh.SAMPLE.  Aborting." 1>&2
-  exit 1
-fi
+source ${git_mirrors_dir}/includes.sh
 
 cd "${git_mirrors_dir}"
 
diff --git a/includes.sh b/includes.sh
@@ -0,0 +1,11 @@
+#!/bin/bash -e
+
+#Include all user options and dependencies
+git_mirrors_dir="${0%/*}"
+[ -f "${git_mirrors_dir}/config.sh" ] && . "${git_mirrors_dir}/config.sh"
+. "${git_mirrors_dir}/lib/VERSION"
+. "${git_mirrors_dir}/lib/functions.sh"
+if [ ! -f "${git_mirrors_dir}/config.sh" ];then
+  red_echo "config.sh missing!  Copy and customize from config.sh.SAMPLE.  Aborting." 1>&2
+  exit 1
+fi
diff --git a/lib/VERSION b/lib/VERSION
@@ -1 +1 @@
-VERSION="v0.5.3"
+VERSION="v0.6.0"
diff --git a/lib/manage_gitlab_project.py b/lib/manage_gitlab_project.py
@@ -7,9 +7,9 @@
 from optparse import OptionParser
 import os
 try:
-  import gitlab3 as gitlab
+  import gitlab
 except ImportError:
-  raise ImportError("python-gitlab3 module is not installed.  You probably didn't read the install instructions closely enough.  See docs/prerequisites.md.")
+  raise ImportError("python-gitlab module is not installed.  You probably didn't read the install instructions closely enough.  See docs/prerequisites.md.")
 
 
 
@@ -19,6 +19,7 @@
   gitlab_namespace=os.environ['gitlab_namespace']
   gitlab_user=os.environ['gitlab_user']
   ssl_verify=os.environ['ssl_verify']
+  gitlab_api_version=os.environ['gitlab_api_version']
 except KeyError:
   print >> stderr, "Environment config missing.  Do not run this script standalone."
   exit(1)
@@ -44,14 +45,44 @@
 project_name=args[0]
 
 if not eval(ssl_verify.capitalize()):
-  git=gitlab.GitLab(gitlab_url=gitlab_url,token=token_secret,ssl_verify=False)
+  git=gitlab.Gitlab(gitlab_url,token_secret,ssl_verify=False,api_version=gitlab_api_version)
 else:
-  git=gitlab.GitLab(gitlab_url=gitlab_url,token=token_secret,ssl_verify=True)
+  git=gitlab.Gitlab(gitlab_url,token_secret,ssl_verify=True,api_version=gitlab_api_version)
+
+def find_group(**kwargs):
+  groups = git.groups.list()
+  return _find_matches(groups, kwargs, False)
+
+def find_project(**kwargs):
+  projects = git.projects.list(as_list=True)
+  return _find_matches(projects, kwargs, False)
+
+def _find_matches(objects, kwargs, find_all):
+  """Helper function for _add_find_fn. Find objects whose properties
+  match all key, value pairs in kwargs.
+  Source: https://github.com/doctormo/python-gitlab3/blob/master/gitlab3/__init__.py
+  """
+  ret = []
+  for obj in objects:
+    match = True
+    # Match all supplied parameters
+    for param, val in kwargs.items():
+      if not getattr(obj, param) == val:
+        match = False
+        break
+      if match:
+        if find_all:
+          ret.append(obj)
+        else:
+          return obj
+  if not find_all:
+    return None
+  return ret
 
 # transfer the project from the source namespace to the specified group namespace
 def transfer_project(src_project, group):
   value = group.transfer_project(src_project.id)
-  dest_project = git.find_project(name=src_project.name)
+  dest_project = find_project(name=src_project.name)
   return dest_project
 
 def createproject(pname):
@@ -69,14 +100,16 @@ def createproject(pname):
     'wiki_enabled': options.wiki,
     'snippets_enabled': options.snippets,
     'public': options.public,
-    'namespace_id': git.find_group(name=gitlab_namespace).id,
+    'namespace_id': find_group(name=gitlab_namespace).id,
   }
   #make all project options lowercase boolean strings i.e. true instead of True
   for x in project_options.keys():
     project_options[x] = str(project_options[x]).lower()
   print >> stderr, "Creating new project %s" % pname
-  git.add_project(pname,description=description,**project_options)
-  found_project = git.find_project(name=pname)
+  project_options['name'] = pname
+  project_options['description'] = description
+  git.projects.create(project_options)
+  found_project = find_project(name=pname)
   if needs_transfer(gitlab_user, gitlab_namespace, found_project):
      found_project = transfer_project(found_project, found_group)
   return found_project
@@ -88,13 +121,19 @@ def needs_transfer(user, groupname, project):
     namespace = groupname
   else:
     namespace = user
-  return project.namespace['name'] != namespace
+  if type(project.namespace) == gitlab.v3.objects.Group:
+    return project.namespace.name != namespace
+  else:
+    return project.namespace['name'] != namespace
+  
 
 if options.create:
-  found_group=git.find_group(name=gitlab_namespace)
+  found_group = find_group(name=gitlab_namespace)
   found_project = None
-  # search the group namespace first
-  found_project=git.find_project(name=project_name)
+
+  found_project= find_project(name=project_name)
+  #exit()
+  
   if found_project:
     if needs_transfer(gitlab_user, gitlab_namespace, found_project):
       found_project = transfer_project(found_project, found_group)
@@ -112,7 +151,7 @@ def needs_transfer(user, groupname, project):
     print found_project.ssh_url_to_repo
 elif options.delete:
   try:
-    deleted_project=git.find_project(name=project_name).delete()
+    deleted_project=find_project(name=project_name).delete()
   except Exception as e:
     print >> stderr, e
     exit(1)
diff --git a/ls-mirrors.sh b/ls-mirrors.sh
@@ -10,13 +10,7 @@ set -e
 
 #Include all user options and dependencies
 git_mirrors_dir="${0%/*}"
-[ -f "${git_mirrors_dir}/config.sh" ] && . "${git_mirrors_dir}/config.sh"
-. "${git_mirrors_dir}/lib/VERSION"
-. "${git_mirrors_dir}/lib/functions.sh"
-if [ ! -f "${git_mirrors_dir}/config.sh" ];then
-  red_echo "config.sh missing!  Copy and customize from config.sh.SAMPLE.  Aborting." 1>&2
-  exit 1
-fi
+source ${git_mirrors_dir}/includes.sh
 
 PROGNAME="${0##*/}"
 PROGVERSION="${VERSION}"
diff --git a/requirements.txt b/requirements.txt
@@ -0,0 +1 @@
+python-gitlab>=1.0.2,<2
diff --git a/update_mirror.sh b/update_mirror.sh
@@ -10,13 +10,7 @@ set -e
 
 #Include all user options and dependencies
 git_mirrors_dir="${0%/*}"
-[ -f "${git_mirrors_dir}/config.sh" ] && . "${git_mirrors_dir}/config.sh"
-. "${git_mirrors_dir}/lib/VERSION"
-. "${git_mirrors_dir}/lib/functions.sh"
-if [ ! -f "${git_mirrors_dir}/config.sh" ];then
-  red_echo "config.sh missing!  Copy and customize from config.sh.SAMPLE.  Aborting." 1>&2
-  exit 1
-fi
+source ${git_mirrors_dir}/includes.sh
 
 #sane update defaults that are backwards compatible
 force_update="${force_update:-false}"
