From 4d84f3458b17d0fa421f740cf94790d749876c18 Mon Sep 17 00:00:00 2001 From: Assiya Date: Fri, 3 Aug 2018 14:02:50 +0600 Subject: [PATCH 1/4] WIP: Add a script to upload reports to GitHub --- coverage_crawler/github.py | 31 +++++++++++++++++++++++++++++++ run_crawler.py | 2 ++ 2 files changed, 33 insertions(+) create mode 100644 coverage_crawler/github.py diff --git a/coverage_crawler/github.py b/coverage_crawler/github.py new file mode 100644 index 0000000..586b550 --- /dev/null +++ b/coverage_crawler/github.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- + +import os +import shutil +import subprocess + + +def upload_to_github(report_path): + # Clone the repository if doesn't exist + if not os.path.isdir('coverage-crawler-reports'): + subprocess.run(['git', 'clone', 'https://github.com/coverage-crawler-updater/coverage-crawler-reports']) + os.chdir('coverage-crawler-reports') + + # Remove the content of repository except of README + prev_files = os.listdir(os.getcwd()) + for f in prev_files: + if f != 'README.md' and f.startswith('.') is False: + print(f) + subprocess.run(['git', 'pull', 'https://github.com/rhcu/coverage-crawler-reports', 'master']) + + # Push the new content + files = os.listdir(os.path.abspath(report_path)) + print(files) + for f in files: + print('Im here') + shutil.move(os.path.join(report_path, f), os.getcwd()) + + # Commit the content + subprocess.run(['git', 'add', '*']) + subprocess.run(['git', 'commit', '-m', 'Coverage crawler reports upload']) + subprocess.run(['git', 'push', 'https://coverage-crawler-updater:ACCESS_TOKEN@github.com/coverage-crawler-updater/coverage-crawler-reports', 'master', '--force']) diff --git a/run_crawler.py b/run_crawler.py index b15cbc8..50edc52 100644 --- a/run_crawler.py +++ b/run_crawler.py @@ -1,7 +1,9 @@ # -*- coding: utf-8 -*- from coverage_crawler import crawler +from coverage_crawler import github with open('websites.txt') as f: for website in f: report = crawler.run(website) + github.upload_to_github(report) From 17c23e4f1ef20111af92df1e0a69619c2661dfd6 Mon Sep 17 00:00:00 2001 From: Assiya Date: Fri, 3 Aug 2018 19:10:53 +0600 Subject: [PATCH 2/4] Add changes to github.py --- coverage_crawler/github.py | 15 ++++++++------- run_crawler.py | 2 -- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/coverage_crawler/github.py b/coverage_crawler/github.py index 586b550..888636b 100644 --- a/coverage_crawler/github.py +++ b/coverage_crawler/github.py @@ -5,27 +5,28 @@ import subprocess -def upload_to_github(report_path): +def upload_to_github(report_path, git_user_name, git_password): # Clone the repository if doesn't exist if not os.path.isdir('coverage-crawler-reports'): - subprocess.run(['git', 'clone', 'https://github.com/coverage-crawler-updater/coverage-crawler-reports']) + subprocess.run(['git', 'clone', 'https://github.com/rhcu/coverage-crawler-reports']) os.chdir('coverage-crawler-reports') # Remove the content of repository except of README prev_files = os.listdir(os.getcwd()) for f in prev_files: - if f != 'README.md' and f.startswith('.') is False: - print(f) + if f != 'home' and f.startswith('.') is False: + os.remove(f) + if f == 'home': + shutil.rmtree('home') subprocess.run(['git', 'pull', 'https://github.com/rhcu/coverage-crawler-reports', 'master']) # Push the new content files = os.listdir(os.path.abspath(report_path)) - print(files) + for f in files: - print('Im here') shutil.move(os.path.join(report_path, f), os.getcwd()) # Commit the content subprocess.run(['git', 'add', '*']) subprocess.run(['git', 'commit', '-m', 'Coverage crawler reports upload']) - subprocess.run(['git', 'push', 'https://coverage-crawler-updater:ACCESS_TOKEN@github.com/coverage-crawler-updater/coverage-crawler-reports', 'master', '--force']) + subprocess.run(['git', 'push', 'https://{}:{}@github.com/rhcu/coverage-crawler-reports'.format(git_user_name, git_password), 'master', '--force']) diff --git a/run_crawler.py b/run_crawler.py index 50edc52..b15cbc8 100644 --- a/run_crawler.py +++ b/run_crawler.py @@ -1,9 +1,7 @@ # -*- coding: utf-8 -*- from coverage_crawler import crawler -from coverage_crawler import github with open('websites.txt') as f: for website in f: report = crawler.run(website) - github.upload_to_github(report) From fe82ec6efd85278771244753caf73e6b6d956751 Mon Sep 17 00:00:00 2001 From: Assiya Date: Tue, 7 Aug 2018 17:03:29 +0600 Subject: [PATCH 3/4] Created index.html that points to other reports --- coverage_crawler/crawler.py | 2 +- coverage_crawler/github.py | 34 +++++++++++++++++++++++++--------- run_crawler.py | 6 ++++++ 3 files changed, 32 insertions(+), 10 deletions(-) diff --git a/coverage_crawler/crawler.py b/coverage_crawler/crawler.py index 87449ff..6bd2fe0 100644 --- a/coverage_crawler/crawler.py +++ b/coverage_crawler/crawler.py @@ -290,4 +290,4 @@ def run(website): driver.quit() - return '{}/report'.format(data_folder) + return os.path.abspath(os.path.join(os.getcwd(), data_folder)) diff --git a/coverage_crawler/github.py b/coverage_crawler/github.py index 888636b..65a32e6 100644 --- a/coverage_crawler/github.py +++ b/coverage_crawler/github.py @@ -5,7 +5,7 @@ import subprocess -def upload_to_github(report_path, git_user_name, git_password): +def upload_to_github(reports_list, git_user_name, git_password): # Clone the repository if doesn't exist if not os.path.isdir('coverage-crawler-reports'): subprocess.run(['git', 'clone', 'https://github.com/rhcu/coverage-crawler-reports']) @@ -14,19 +14,35 @@ def upload_to_github(report_path, git_user_name, git_password): # Remove the content of repository except of README prev_files = os.listdir(os.getcwd()) for f in prev_files: - if f != 'home' and f.startswith('.') is False: + if os.path.isdir(f): + shutil.rmtree(f) + elif f.startswith('.') is False: os.remove(f) - if f == 'home': - shutil.rmtree('home') subprocess.run(['git', 'pull', 'https://github.com/rhcu/coverage-crawler-reports', 'master']) - # Push the new content - files = os.listdir(os.path.abspath(report_path)) - - for f in files: - shutil.move(os.path.join(report_path, f), os.getcwd()) + with open('index.html', 'w') as f: + f.write(""" + + + Reports + + +

The list of available reports for websites:

+
+ """) + for website, report in reports_list.items(): + # Push the new content + shutil.move(report, os.getcwd()) + name_of_folder = report.rsplit('/', 1)[-1] + f.write('{}'.format(os.path.join(name_of_folder, 'report/index.html'), website)) + f.write('
') + f.write(""" + + + """) # Commit the content + subprocess.run(['git', 'init']) subprocess.run(['git', 'add', '*']) subprocess.run(['git', 'commit', '-m', 'Coverage crawler reports upload']) subprocess.run(['git', 'push', 'https://{}:{}@github.com/rhcu/coverage-crawler-reports'.format(git_user_name, git_password), 'master', '--force']) diff --git a/run_crawler.py b/run_crawler.py index b15cbc8..bbc28ed 100644 --- a/run_crawler.py +++ b/run_crawler.py @@ -1,7 +1,13 @@ # -*- coding: utf-8 -*- from coverage_crawler import crawler +from coverage_crawler import github + +reports = {} with open('websites.txt') as f: for website in f: report = crawler.run(website) + reports[website] = report + +github.upload_to_github(reports, 'GIT_ACCOUNT', 'GIT_PASSWORD') From 83fd144290212112aaba841e6ae8cff56eebbdb5 Mon Sep 17 00:00:00 2001 From: Assiya Khuzyakhmetova Date: Tue, 7 Aug 2018 17:06:49 +0600 Subject: [PATCH 4/4] Removed spaces --- coverage_crawler/crawler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/coverage_crawler/crawler.py b/coverage_crawler/crawler.py index eea5570..6bd2fe0 100644 --- a/coverage_crawler/crawler.py +++ b/coverage_crawler/crawler.py @@ -289,5 +289,5 @@ def run(website): generatehtml.generate_html(data_folder) driver.quit() - - return os.path.abspath(os.path.join(os.getcwd(), data_folder)) \ No newline at end of file + + return os.path.abspath(os.path.join(os.getcwd(), data_folder))