From d853f5032256b1d9b0b798d90ff8549f8a50e297 Mon Sep 17 00:00:00 2001 From: Guy Kisel Date: Wed, 1 Feb 2017 10:32:17 -0800 Subject: [PATCH 01/17] get docker build working and add uwsgi config --- Dockerfile | 7 ++++--- uwsgi.ini | 3 +++ 2 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 uwsgi.ini diff --git a/Dockerfile b/Dockerfile index c6327b6..e6f32f4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,9 @@ FROM tiangolo/uwsgi-nginx-flask:flask -COPY * /app +RUN apt-get update && apt-get install -y ruby-full haskell-platform shellcheck nodejs build-essential -RUN curl -sL https://deb.nodesource.com/setup_6.x | sudo -E bash - -RUN apt-get update && apt-get install -y ruby-full haskell-platform shellcheck nodejs build-essential nodejs-legacy +RUN pip install -U pip + +COPY * /app/ RUN pip install -r /app/requirements.txt diff --git a/uwsgi.ini b/uwsgi.ini new file mode 100644 index 0000000..4e79fbd --- /dev/null +++ b/uwsgi.ini @@ -0,0 +1,3 @@ +[uwsgi] +module = main +callable = app \ No newline at end of file From b5790660afca6eea2267e321d70dc3081edb5162 Mon Sep 17 00:00:00 2001 From: Guy Kisel Date: Wed, 1 Feb 2017 15:27:52 -0800 Subject: [PATCH 02/17] move files into app dir --- Dockerfile | 2 +- Procfile | 2 +- main.py => app/main.py | 0 uwsgi.ini => app/uwsgi.ini | 0 4 files changed, 2 insertions(+), 2 deletions(-) rename main.py => app/main.py (100%) rename uwsgi.ini => app/uwsgi.ini (100%) diff --git a/Dockerfile b/Dockerfile index e6f32f4..74761ce 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,6 @@ RUN apt-get update && apt-get install -y ruby-full haskell-platform shellcheck n RUN pip install -U pip -COPY * /app/ +COPY ./app /app RUN pip install -r /app/requirements.txt diff --git a/Procfile b/Procfile index 629b83a..0aeea5f 100644 --- a/Procfile +++ b/Procfile @@ -1 +1 @@ -web: python main.py +web: python app/main.py diff --git a/main.py b/app/main.py similarity index 100% rename from main.py rename to app/main.py diff --git a/uwsgi.ini b/app/uwsgi.ini similarity index 100% rename from uwsgi.ini rename to app/uwsgi.ini From c3e2bcd00b7d8e80243cf143dd08e0cd0f2679db Mon Sep 17 00:00:00 2001 From: Guy Kisel Date: Fri, 3 Feb 2017 14:14:06 -0800 Subject: [PATCH 03/17] copy requirements file to app dir --- Dockerfile | 1 + app/uwsgi.ini | 3 --- 2 files changed, 1 insertion(+), 3 deletions(-) delete mode 100644 app/uwsgi.ini diff --git a/Dockerfile b/Dockerfile index 74761ce..e1cf855 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,5 +5,6 @@ RUN apt-get update && apt-get install -y ruby-full haskell-platform shellcheck n RUN pip install -U pip COPY ./app /app +COPY requirements.txt /app RUN pip install -r /app/requirements.txt diff --git a/app/uwsgi.ini b/app/uwsgi.ini deleted file mode 100644 index 4e79fbd..0000000 --- a/app/uwsgi.ini +++ /dev/null @@ -1,3 +0,0 @@ -[uwsgi] -module = main -callable = app \ No newline at end of file From c6d7966c49b9e233fa366ae154394148f4b85685 Mon Sep 17 00:00:00 2001 From: Guy Kisel Date: Fri, 3 Feb 2017 15:35:13 -0800 Subject: [PATCH 04/17] install nvm and stuff --- Dockerfile | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e1cf855..303a801 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,30 @@ FROM tiangolo/uwsgi-nginx-flask:flask -RUN apt-get update && apt-get install -y ruby-full haskell-platform shellcheck nodejs build-essential +RUN apt-get update && apt-get install -y \ + curl git bzr mercurial build-essential \ + zip ruby-full haskell-platform shellcheck \ + python-pip python-dev \ + nodejs build-essential + +ENV NVM_DIR /usr/local/nvm +ENV NODE_VERSION node + +# Install nvm with node and npm +RUN curl https://raw.githubusercontent.com/creationix/nvm/v0.33.0/install.sh | bash \ + && . $NVM_DIR/nvm.sh \ + && nvm install $NODE_VERSION \ + && nvm alias default $NODE_VERSION \ + && nvm use default \ + && npm install -g jsonlint jscs eslint jshint + +ENV NODE_PATH $NVM_DIR/versions/node/v$NODE_VERSION/lib/node_modules +ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH + +RUN echo '. "$NVM_DIR/nvm.sh"' >> /etc/profile +RUN echo NODE_VERSION=$NODE_VERSION >> /etc/environment +RUN echo NVM_DIR=$NVM_DIR >> /etc/environment +RUN echo NODE_PATH=$NVM_DIR/versions/node/v$NODE_VERSION/lib/node_modules >> /etc/environment +RUN echo PATH=$NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH >> /etc/environment RUN pip install -U pip From 1364b9bf5f892da802ca77f158a31baa55bcb4ba Mon Sep 17 00:00:00 2001 From: Guy Kisel Date: Sun, 5 Feb 2017 16:15:20 -0800 Subject: [PATCH 05/17] add ssh keys as needed --- app/main.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/app/main.py b/app/main.py index 12bfd33..9b9dc01 100644 --- a/app/main.py +++ b/app/main.py @@ -21,6 +21,7 @@ DOTFILES = 'dotfiles' STOP_FILE_NAME = '.inlineplzstop' REVIEWS_IN_PROGRESS = dict() +SSH_LOCK = threading.Lock() @app.errorhandler(Exception) @@ -63,6 +64,14 @@ def clone_dotfiles(url, org, tempdir, token): return clone(clone_url, dotfile_path, token) +def ssh_keygen(url): + with SSH_LOCK: + output = subprocess.check_output(['ssh-keygen', '-F', url]) + if output.strip(): + return + subprocess.check_call(['ssh-keyscan', '-t', 'rsa', url, '>>', '~/.ssh/known_hosts']) + + def lint(data): try: pull_request = data['pull_request']['number'] @@ -70,7 +79,7 @@ def lint(data): name = data['repository']['name'] token = os.environ.get('TOKEN') interface = 'github' - url = os.environ.get('URL', 'https://github.com') + url = data['repository']['ssh_url'].split('@')[1].split(':')[0] event_type = data['action'] sha = data['pull_request']['head']['sha'] ref = data['pull_request']['head']['ref'] @@ -92,6 +101,8 @@ def lint(data): if event_type not in ['opened', 'synchronize']: return + ssh_keygen(url) + # make temp dirs tempdir = tempfile.mkdtemp() dotfile_dir = tempfile.mkdtemp() From 190aaf5548ed28a7d91eb11e7bb79901f07df8c9 Mon Sep 17 00:00:00 2001 From: Guy Kisel Date: Sun, 5 Feb 2017 17:07:14 -0800 Subject: [PATCH 06/17] install pyenv and rvm --- Dockerfile | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 303a801..8b13902 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,10 @@ FROM tiangolo/uwsgi-nginx-flask:flask -RUN apt-get update && apt-get install -y \ - curl git bzr mercurial build-essential \ - zip ruby-full haskell-platform shellcheck \ - python-pip python-dev \ - nodejs build-essential +RUN apt-get update && apt-get upgrade --fix-missing -y +RUN apt-get update && apt-get install -y curl git bzr mercurial build-essential +RUN apt-get install -y zip ruby-full haskell-platform shellcheck +RUN apt-get install -y python-pip python-dev +RUN apt-get install -y nodejs build-essential golang ENV NVM_DIR /usr/local/nvm ENV NODE_VERSION node @@ -27,8 +27,24 @@ RUN echo NODE_PATH=$NVM_DIR/versions/node/v$NODE_VERSION/lib/node_modules >> /et RUN echo PATH=$NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH >> /etc/environment RUN pip install -U pip - +RUN curl -L https://raw.githubusercontent.com/yyuu/pyenv-installer/master/bin/pyenv-installer | bash +RUN export PATH="/root/.pyenv/bin:$PATH" +ENV PATH /root/.pyenv/bin:$PATH +RUN eval "$(pyenv init -)" +RUN eval "$(pyenv virtualenv-init -)" +RUN pyenv update +RUN pyenv install 2.7.13 +RUN pyenv install 3.6.0 COPY ./app /app COPY requirements.txt /app - RUN pip install -r /app/requirements.txt + +# Install rvm (https://github.com/vallard/docker/blob/master/rails/Dockerfile) +RUN apt-get install -y curl patch gawk g++ gcc make libc6-dev patch libreadline6-dev zlib1g-dev libssl-dev libyaml-dev libsqlite3-dev sqlite3 autoconf libgdbm-dev libncurses5-dev automake libtool bison pkg-config libffi-dev +RUN useradd -ms /bin/bash app +USER app +RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 +RUN /bin/bash -l -c "curl -L get.rvm.io | bash -s stable --rails" +RUN /bin/bash -l -c "rvm install 2.1" +RUN /bin/bash -l -c "echo 'gem: --no-ri --no-rdoc' > ~/.gemrc" +RUN /bin/bash -l -c "gem install bundler --no-ri --no-rdoc" From e78f634b126a1d7e4df711ffce8f933b4f844bd1 Mon Sep 17 00:00:00 2001 From: Guy Kisel Date: Mon, 6 Feb 2017 12:46:37 -0800 Subject: [PATCH 07/17] don't switch users --- Dockerfile | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8b13902..48097a0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,16 +35,15 @@ RUN eval "$(pyenv virtualenv-init -)" RUN pyenv update RUN pyenv install 2.7.13 RUN pyenv install 3.6.0 +RUN pyenv global 2.7.13 3.6.0 COPY ./app /app COPY requirements.txt /app RUN pip install -r /app/requirements.txt # Install rvm (https://github.com/vallard/docker/blob/master/rails/Dockerfile) RUN apt-get install -y curl patch gawk g++ gcc make libc6-dev patch libreadline6-dev zlib1g-dev libssl-dev libyaml-dev libsqlite3-dev sqlite3 autoconf libgdbm-dev libncurses5-dev automake libtool bison pkg-config libffi-dev -RUN useradd -ms /bin/bash app -USER app RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 -RUN /bin/bash -l -c "curl -L get.rvm.io | bash -s stable --rails" -RUN /bin/bash -l -c "rvm install 2.1" -RUN /bin/bash -l -c "echo 'gem: --no-ri --no-rdoc' > ~/.gemrc" -RUN /bin/bash -l -c "gem install bundler --no-ri --no-rdoc" +RUN curl -L https://get.rvm.io | bash -s stable +RUN bash -l -c "rvm requirements" +RUN bash -l -c "rvm install 2.0" +RUN bash -l -c "gem install bundler --no-ri --no-rdoc" From c544e2f34ba9f9cd1572de2b04cb23eae086906c Mon Sep 17 00:00:00 2001 From: Guy Kisel Date: Mon, 6 Feb 2017 12:57:03 -0800 Subject: [PATCH 08/17] setup known hosts --- Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Dockerfile b/Dockerfile index 48097a0..3db7f08 100644 --- a/Dockerfile +++ b/Dockerfile @@ -47,3 +47,8 @@ RUN curl -L https://get.rvm.io | bash -s stable RUN bash -l -c "rvm requirements" RUN bash -l -c "rvm install 2.0" RUN bash -l -c "gem install bundler --no-ri --no-rdoc" + +RUN mkdir -p /root/.ssh +RUN touch /root/.ssh/known_hosts +RUN chmod 0700 /root/.ssh +RUN chmod 0600 /root/.ssh/known_hosts \ No newline at end of file From 6adafa4669b5693632b76be3a046edc439aa2ad4 Mon Sep 17 00:00:00 2001 From: Guy Kisel Date: Mon, 6 Feb 2017 13:10:38 -0800 Subject: [PATCH 09/17] handle keygen errors --- app/main.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/app/main.py b/app/main.py index 9b9dc01..9d98249 100644 --- a/app/main.py +++ b/app/main.py @@ -66,9 +66,12 @@ def clone_dotfiles(url, org, tempdir, token): def ssh_keygen(url): with SSH_LOCK: - output = subprocess.check_output(['ssh-keygen', '-F', url]) - if output.strip(): - return + try: + output = subprocess.check_output(['ssh-keygen', '-F', url], stderr=subprocess.STDOUT) + if output.strip(): + return + except subprocess.CalledProcessError: + pass subprocess.check_call(['ssh-keyscan', '-t', 'rsa', url, '>>', '~/.ssh/known_hosts']) From 45a153a617da39c109a66ecb21a3d599482f05dd Mon Sep 17 00:00:00 2001 From: Guy Kisel Date: Mon, 6 Feb 2017 13:25:03 -0800 Subject: [PATCH 10/17] properly append to known hosts --- app/main.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/app/main.py b/app/main.py index 9d98249..ce8ba49 100644 --- a/app/main.py +++ b/app/main.py @@ -71,8 +71,13 @@ def ssh_keygen(url): if output.strip(): return except subprocess.CalledProcessError: - pass - subprocess.check_call(['ssh-keyscan', '-t', 'rsa', url, '>>', '~/.ssh/known_hosts']) + traceback.print_exc() + try: + output = subprocess.check_call(['ssh-keyscan', '-t', 'rsa', url]) + with open(os.path.join(os.path.expanduser('~'), '.ssh', 'known_hosts'), 'a') as known_hosts: + known_hosts.write(output) + except Exception: + traceback.print_exc() def lint(data): From 92890024ce82a6d97a81362e9c1fcd76c55943a1 Mon Sep 17 00:00:00 2001 From: Guy Kisel Date: Mon, 6 Feb 2017 13:34:07 -0800 Subject: [PATCH 11/17] capture keyscan stderr --- app/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/main.py b/app/main.py index ce8ba49..572b358 100644 --- a/app/main.py +++ b/app/main.py @@ -73,7 +73,7 @@ def ssh_keygen(url): except subprocess.CalledProcessError: traceback.print_exc() try: - output = subprocess.check_call(['ssh-keyscan', '-t', 'rsa', url]) + output = subprocess.check_call(['ssh-keyscan', '-t', 'rsa', url], stderr=subprocess.STDOUT) with open(os.path.join(os.path.expanduser('~'), '.ssh', 'known_hosts'), 'a') as known_hosts: known_hosts.write(output) except Exception: From 3d53f5af881341917e8ed13b31cc628d7674f3a8 Mon Sep 17 00:00:00 2001 From: Guy Kisel Date: Mon, 6 Feb 2017 13:43:38 -0800 Subject: [PATCH 12/17] actually capture output --- app/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/main.py b/app/main.py index 572b358..e28a490 100644 --- a/app/main.py +++ b/app/main.py @@ -73,7 +73,7 @@ def ssh_keygen(url): except subprocess.CalledProcessError: traceback.print_exc() try: - output = subprocess.check_call(['ssh-keyscan', '-t', 'rsa', url], stderr=subprocess.STDOUT) + output = subprocess.check_output(['ssh-keyscan', '-t', 'rsa', url], stderr=subprocess.STDOUT) with open(os.path.join(os.path.expanduser('~'), '.ssh', 'known_hosts'), 'a') as known_hosts: known_hosts.write(output) except Exception: From bd346201a6dd43da00f909d735d7cebea9a5b698 Mon Sep 17 00:00:00 2001 From: Guy Kisel Date: Wed, 8 Feb 2017 16:06:04 -0800 Subject: [PATCH 13/17] handle SSH key generation --- app/main.py | 51 ++++++++++++++++++++++++++++++++++++++++++------ requirements.txt | 2 +- 2 files changed, 46 insertions(+), 7 deletions(-) diff --git a/app/main.py b/app/main.py index e28a490..6f5501c 100644 --- a/app/main.py +++ b/app/main.py @@ -4,7 +4,9 @@ from __future__ import print_function from __future__ import unicode_literals +import hashlib import os +import random import shutil import subprocess import tempfile @@ -12,6 +14,7 @@ import time import traceback from flask import Flask, request, redirect +import github3 app = Flask(__name__) @@ -20,9 +23,19 @@ SAFE_ENV['TOKEN'] = '' DOTFILES = 'dotfiles' STOP_FILE_NAME = '.inlineplzstop' +SSH_FILE_NAME = 'inline_plz_rsa' +SSH_FILE_PATH = os.path.join(os.path.expanduser('~'), '.ssh', SSH_FILE_NAME) REVIEWS_IN_PROGRESS = dict() SSH_LOCK = threading.Lock() +TRUSTED = os.environ.get('TRUSTED', '').lower().strip() in ['true', 'yes', '1'] +if TRUSTED: + ssh_keygen() + +SSH_KEY_HASH = hashlib.md5() +SSH_KEY_HASH.update(open(SSH_FILE_PATH).read()) +SSH_KEY_HASH = SSH_KEY_HASH.hexdigest()[-6:] + @app.errorhandler(Exception) def all_exception_handler(): @@ -64,16 +77,41 @@ def clone_dotfiles(url, org, tempdir, token): return clone(clone_url, dotfile_path, token) -def ssh_keygen(url): +def ssh_keygen(): + time.sleep(random.randint(1, 10)) + while not os.path.exists(SSH_FILE_PATH): + try: + subprocess.check_call(['ssh-keygen', '-t', 'rsa', '-b', '2048', '-f', SSH_FILE_PATH]) + except Exception: + traceback.print_exc() + time.sleep(random.randint(1, 10)) + + +def ssh_setup(url, token): with SSH_LOCK: + if not url or url in ['http://github.com', 'https://github.com']: + github = github3.GitHub(token=token) + else: + github = github3.GitHubEnterprise(url, token=token) + + key_found = False + for key in github.iter_keys(): + if SSH_FILE_NAME in key.title and not SSH_KEY_HASH in key.title: + github.delete_key(key.id) + elif key.title == '{}_{}'.format(SSH_FILE_NAME, SSH_KEY_HASH): + key_found = True + if not key_found: + github.create_key('{}_{}'.format(SSH_FILE_NAME, SSH_KEY_HASH), SSH_FILE_PATH + '.pub') + + keygen_url = url.split('//')[-1] try: - output = subprocess.check_output(['ssh-keygen', '-F', url], stderr=subprocess.STDOUT) + output = subprocess.check_output(['ssh-keygen', '-F', keygen_url], stderr=subprocess.STDOUT) if output.strip(): return except subprocess.CalledProcessError: traceback.print_exc() try: - output = subprocess.check_output(['ssh-keyscan', '-t', 'rsa', url], stderr=subprocess.STDOUT) + output = subprocess.check_output(['ssh-keyscan', '-t', 'rsa', keygen_url], stderr=subprocess.STDOUT) with open(os.path.join(os.path.expanduser('~'), '.ssh', 'known_hosts'), 'a') as known_hosts: known_hosts.write(output) except Exception: @@ -96,7 +134,7 @@ def lint(data): except KeyError: traceback.print_exc() return 'Invalid pull request data.' - trusted = os.environ.get('TRUSTED', '').lower().strip() in ['true', 'yes', '1'] + print('Starting inline-plz:') print('Event: {}'.format(event_type)) @@ -109,7 +147,8 @@ def lint(data): if event_type not in ['opened', 'synchronize']: return - ssh_keygen(url) + if TRUSTED: + ssh_setup(url, token) # make temp dirs tempdir = tempfile.mkdtemp() @@ -149,7 +188,7 @@ def lint(data): '--interface={}'.format(interface), '--zero-exit' ] - if trusted: + if TRUSTED: args.append('--trusted') if clone_dotfiles(url, org, dotfile_dir, token): args.append('--config-dir={}'.format( diff --git a/requirements.txt b/requirements.txt index 8f2968d..a1c32eb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ configparser==3.5.0 docutils==0.12 dodgy==0.1.9 Flask==0.11.1 -github3.py==0.9.5 +github3.py==0.9.6 inlineplz==0.23.1 isort==4.2.5 itsdangerous==0.24 From a947e9eadd396b0d004ff65370a2b80b9e8e8411 Mon Sep 17 00:00:00 2001 From: Guy Kisel Date: Wed, 8 Feb 2017 16:24:30 -0800 Subject: [PATCH 14/17] define ssh_keygen earlier --- app/main.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/app/main.py b/app/main.py index 6f5501c..755d127 100644 --- a/app/main.py +++ b/app/main.py @@ -28,6 +28,17 @@ REVIEWS_IN_PROGRESS = dict() SSH_LOCK = threading.Lock() + +def ssh_keygen(): + time.sleep(random.randint(1, 10)) + while not os.path.exists(SSH_FILE_PATH): + try: + subprocess.check_call(['ssh-keygen', '-t', 'rsa', '-b', '2048', '-f', SSH_FILE_PATH]) + except Exception: + traceback.print_exc() + time.sleep(random.randint(1, 10)) + + TRUSTED = os.environ.get('TRUSTED', '').lower().strip() in ['true', 'yes', '1'] if TRUSTED: ssh_keygen() @@ -77,16 +88,6 @@ def clone_dotfiles(url, org, tempdir, token): return clone(clone_url, dotfile_path, token) -def ssh_keygen(): - time.sleep(random.randint(1, 10)) - while not os.path.exists(SSH_FILE_PATH): - try: - subprocess.check_call(['ssh-keygen', '-t', 'rsa', '-b', '2048', '-f', SSH_FILE_PATH]) - except Exception: - traceback.print_exc() - time.sleep(random.randint(1, 10)) - - def ssh_setup(url, token): with SSH_LOCK: if not url or url in ['http://github.com', 'https://github.com']: From c95d7d2b79a2b8765b0bbba512f7ff0d6c35db50 Mon Sep 17 00:00:00 2001 From: Guy Kisel Date: Wed, 8 Feb 2017 16:36:05 -0800 Subject: [PATCH 15/17] send blank passphrase and fix url --- app/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/main.py b/app/main.py index 755d127..b5d8be4 100644 --- a/app/main.py +++ b/app/main.py @@ -33,7 +33,7 @@ def ssh_keygen(): time.sleep(random.randint(1, 10)) while not os.path.exists(SSH_FILE_PATH): try: - subprocess.check_call(['ssh-keygen', '-t', 'rsa', '-b', '2048', '-f', SSH_FILE_PATH]) + subprocess.check_call(['ssh-keygen', '-t', 'rsa', '-b', '2048', '-f', SSH_FILE_PATH, '-q', '-N', '""']) except Exception: traceback.print_exc() time.sleep(random.randint(1, 10)) @@ -126,7 +126,7 @@ def lint(data): name = data['repository']['name'] token = os.environ.get('TOKEN') interface = 'github' - url = data['repository']['ssh_url'].split('@')[1].split(':')[0] + url = 'https://' + data['repository']['ssh_url'].split('@')[1].split(':')[0] event_type = data['action'] sha = data['pull_request']['head']['sha'] ref = data['pull_request']['head']['ref'] From 2eeb2aa61253a72da3774b6e438c1da642175d3f Mon Sep 17 00:00:00 2001 From: Guy Kisel Date: Wed, 8 Feb 2017 16:48:23 -0800 Subject: [PATCH 16/17] actually send empty passphrase --- app/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/main.py b/app/main.py index b5d8be4..9e66ba3 100644 --- a/app/main.py +++ b/app/main.py @@ -33,7 +33,7 @@ def ssh_keygen(): time.sleep(random.randint(1, 10)) while not os.path.exists(SSH_FILE_PATH): try: - subprocess.check_call(['ssh-keygen', '-t', 'rsa', '-b', '2048', '-f', SSH_FILE_PATH, '-q', '-N', '""']) + subprocess.check_call(['ssh-keygen', '-t', 'rsa', '-b', '2048', '-f', SSH_FILE_PATH, '-q', '-N', '']) except Exception: traceback.print_exc() time.sleep(random.randint(1, 10)) From fc167a4e81b0547c210febaa05ef7a2d50253cf5 Mon Sep 17 00:00:00 2001 From: Guy Kisel Date: Thu, 9 Feb 2017 15:25:18 -0800 Subject: [PATCH 17/17] set up ssh agent --- Dockerfile | 4 ++-- app/main.py | 18 +++++++++++++++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3db7f08..26deb42 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM tiangolo/uwsgi-nginx-flask:flask RUN apt-get update && apt-get upgrade --fix-missing -y RUN apt-get update && apt-get install -y curl git bzr mercurial build-essential -RUN apt-get install -y zip ruby-full haskell-platform shellcheck +RUN apt-get install -y zip ruby-full haskell-platform shellcheck ssh RUN apt-get install -y python-pip python-dev RUN apt-get install -y nodejs build-essential golang @@ -51,4 +51,4 @@ RUN bash -l -c "gem install bundler --no-ri --no-rdoc" RUN mkdir -p /root/.ssh RUN touch /root/.ssh/known_hosts RUN chmod 0700 /root/.ssh -RUN chmod 0600 /root/.ssh/known_hosts \ No newline at end of file +RUN chmod 0600 /root/.ssh/known_hosts diff --git a/app/main.py b/app/main.py index 9e66ba3..217e5f8 100644 --- a/app/main.py +++ b/app/main.py @@ -7,6 +7,7 @@ import hashlib import os import random +import re import shutil import subprocess import tempfile @@ -34,6 +35,14 @@ def ssh_keygen(): while not os.path.exists(SSH_FILE_PATH): try: subprocess.check_call(['ssh-keygen', '-t', 'rsa', '-b', '2048', '-f', SSH_FILE_PATH, '-q', '-N', '']) + ssh_output = subprocess.check_output('ssh-agent -s', shell=True, stderr=subprocess.STDOUT) + # http://code.activestate.com/recipes/533143-set-environment-variables-for-using-ssh-in-python-/ + for sh_line in ssh_output.splitlines(): + matches=re.search("(\S+)\=(\S+)\;", sh_line) + if matches: + os.environ[matches.group(1)]=matches.group(2) + SAFE_ENV[matches.group(1)]=matches.group(2) + subprocess.check_call('ssh-add {}'.format(SSH_FILE_PATH), shell=True) except Exception: traceback.print_exc() time.sleep(random.randint(1, 10)) @@ -90,6 +99,13 @@ def clone_dotfiles(url, org, tempdir, token): def ssh_setup(url, token): with SSH_LOCK: + try: + with open(os.path.join(os.path.expanduser('~'), '.ssh', 'config'), 'ar+') as sshconfig: + contents = sshconfig.read() + if not 'HostName {}'.format(url) in contents: + sshconfig.write('\nHost {0}\n\tHostName {0}\n\tIdentityFile {1}'.format(url, SSH_FILE_PATH)) + except Exception: + traceback.print_exc() if not url or url in ['http://github.com', 'https://github.com']: github = github3.GitHub(token=token) else: @@ -102,7 +118,7 @@ def ssh_setup(url, token): elif key.title == '{}_{}'.format(SSH_FILE_NAME, SSH_KEY_HASH): key_found = True if not key_found: - github.create_key('{}_{}'.format(SSH_FILE_NAME, SSH_KEY_HASH), SSH_FILE_PATH + '.pub') + github.create_key('{}_{}'.format(SSH_FILE_NAME, SSH_KEY_HASH), open(SSH_FILE_PATH + '.pub').read()) keygen_url = url.split('//')[-1] try: