diff --git a/data_providers/alexandria/integrations/ingest_alexandria.py b/data_providers/alexandria/integrations/ingest_alexandria.py index 34e2ad2..511de37 100644 --- a/data_providers/alexandria/integrations/ingest_alexandria.py +++ b/data_providers/alexandria/integrations/ingest_alexandria.py @@ -59,12 +59,30 @@ def parse_line(line): default = Sentiment - args = (Timestamp, default, StoryId, Country, Sentiment, \ - Confidence, Novelty, Subjects, Relevance, Ticker, Timestamp, \ - Ticker, Timestamp, default, StoryId, Timestamp, Country, \ - Sentiment, Confidence, Novelty, Subjects, Relevance) - - return args + return ( + Timestamp, + default, + StoryId, + Country, + Sentiment, + Confidence, + Novelty, + Subjects, + Relevance, + Ticker, + Timestamp, + Ticker, + Timestamp, + default, + StoryId, + Timestamp, + Country, + Sentiment, + Confidence, + Novelty, + Subjects, + Relevance, + ) def parse(fname): diff --git a/data_providers/thomson_reuters/api/Knowledge-Direct-API/fundamentals/financialstatements.py b/data_providers/thomson_reuters/api/Knowledge-Direct-API/fundamentals/financialstatements.py index 53240a0..02b664c 100644 --- a/data_providers/thomson_reuters/api/Knowledge-Direct-API/fundamentals/financialstatements.py +++ b/data_providers/thomson_reuters/api/Knowledge-Direct-API/fundamentals/financialstatements.py @@ -12,8 +12,8 @@ def convertJsonTuple(jsonData): - data = json.loads(jsonData, object_hook=lambda d: namedtuple('data', d.keys())(*d.values())) - return data + return json.loads( + jsonData, object_hook=lambda d: namedtuple('data', d.keys())(*d.values())) ## Company Identifiers Conversion ## -- Convert the jsonData to a Company Identifier diff --git a/data_providers/xignite/api/xignite/xignite_economic_calendar.py b/data_providers/xignite/api/xignite/xignite_economic_calendar.py index c5d6b55..ee94ae0 100644 --- a/data_providers/xignite/api/xignite/xignite_economic_calendar.py +++ b/data_providers/xignite/api/xignite/xignite_economic_calendar.py @@ -28,7 +28,11 @@ def construct_url(self, country, date_range): """ if date_range: url_parameters = {"ReleasedOnStart": date_range[0], "ReleasedOnEnd": date_range[1], "CountryCode": country} - formatted_parameters = '&'.join('{0}={1}'.format(key, ''.join([str(var) for var in val])) for key, val in sorted(url_parameters.items())) + formatted_parameters = '&'.join( + '{0}={1}'.format(key, ''.join(str(var) for var in val)) + for key, val in sorted(url_parameters.items()) + ) + return "%s?_token=%s&%s" % (self.base_url, self.access_token, formatted_parameters) def query(self, country, date_range): diff --git a/data_providers/xignite/api/xignite/xignite_global_quotes.py b/data_providers/xignite/api/xignite/xignite_global_quotes.py index 901651b..aa548cf 100644 --- a/data_providers/xignite/api/xignite/xignite_global_quotes.py +++ b/data_providers/xignite/api/xignite/xignite_global_quotes.py @@ -29,7 +29,11 @@ def construct_url(self, symbols): """ if symbols: url_parameters = {"Identifiers": symbols,"IdentifierType": ["Symbol"]} - formatted_parameters = '&'.join('{0}={1}'.format(key, ','.join([str(var) for var in val])) for key, val in sorted(url_parameters.items())) + formatted_parameters = '&'.join( + '{0}={1}'.format(key, ','.join(str(var) for var in val)) + for key, val in sorted(url_parameters.items()) + ) + return "%s?_token=%s&%s" % (self.base_url, self.access_token, formatted_parameters) diff --git a/fintech_sandbox/generate_data_partner_list.py b/fintech_sandbox/generate_data_partner_list.py index 6b1ff5c..9d1730c 100644 --- a/fintech_sandbox/generate_data_partner_list.py +++ b/fintech_sandbox/generate_data_partner_list.py @@ -25,9 +25,9 @@ def naming_convention(full_name): def process_article(html): import shutil images_dir = WIKI_REPO_DIR + "images/" - + soup = BeautifulSoup(html, 'lxml') - + elts = soup.findAll('article') profiles = [] for elt in elts: @@ -36,13 +36,9 @@ def process_article(html): text = elt.get_text().strip().splitlines() if len(text): text = text[0] - - logo = img.attrs.get('src', None) - if logo: - img_name = urlparse.urlparse(logo).path.split('/')[-1] - else: - img_name = None + logo = img.attrs.get('src', None) + img_name = urlparse.urlparse(logo).path.split('/')[-1] if logo else None profile = { 'logo': logo, 'img_name': img_name, @@ -54,13 +50,13 @@ def process_article(html): if profile['id'][-1] == '_': profile['id'] = profile['id'][:-1] profiles.append(profile) - + for profile in profiles: # Process bio rs = requests.get('http://fintechsandbox.org' + profile['link']) html = rs.content soup = BeautifulSoup(html, 'lxml') - + elts = soup.findAll('article') title_elt = soup.find(name='h1', attrs={'class':'title'}) if title_elt: @@ -68,12 +64,14 @@ def process_article(html): desc_elt = soup.find(name='div', attrs={'class':'content'}) if desc_elt: profile.update({'title':desc_elt.text.strip().splitlines()[0]}) - + # Process External Links sidebar_elt = soup.find(name='div', attrs={'class':'sidebar'}) profile.update({'external_links': [{'url':elt.attrs['href'].strip(), 'text': elt.text.strip() } for elt in sidebar_elt.findAll('a')]}) - + for l in profile['external_links']: + if l['url'].find('fintechsandbox') > -1: + continue if l['url'].find('mailto:') == 0: profile['email_link'] = l['url'] profile['email_text'] = l['text'] @@ -84,12 +82,10 @@ def process_article(html): profile['linkedin'] = l['url'] elif l['url'].find('twitter') > -1: profile['twitter'] = l['url'] - elif l['url'].find('fintechsandbox') > -1: - pass else: profile['website'] = l['url'] - - + + # Download logo rs = requests.get(profile['logo'], stream=True) logo_fname = '{}{}.{}'.format(images_dir, profile['id'], profile['logo_type']) @@ -99,7 +95,7 @@ def process_article(html): with open(logo_fname, 'wb') as f: rs.raw.decode_content = True shutil.copyfileobj(rs.raw, f) - + import pandas as pd df = pd.DataFrame(profiles) df['name'] = df['name'].apply(lambda x: x.strip()) @@ -222,7 +218,7 @@ def create_markup_table(df, profile_category): assert profile_category in ['Sandbox-Members','Data-Providers'], "profile_category must be either 'Sandbox-Members' or 'Data-Providers'" #[![Fintech Sandbox Profile](images/thomson_reuters.png)](http://fintechsandbox.org/partner/thomson-reuters) - + headings = ['Provider', 'Resource Page', 'External Profile', 'Website', 'Email', 'Phone','LinkedIn','Twitter'] @@ -235,19 +231,22 @@ def create_markup_table(df, profile_category): '[![{linkedin}](images/icons/linkedin.png)]({linkedin})', '[![{twitter}](images/icons/twitter.png)]({twitter})' ] - - - - - table = ['| ' + ' | '.join(headings) +' |'] - table.append('|' + '|'.join(['-'*len(h) for h in headings]) +'|') - for _, data in df.iterrows(): - row = '| ' + ' | '.join([_process_template(t,data) for t in template]) +' |' + + + + + table = [ + '| ' + ' | '.join(headings) + ' |', + '|' + '|'.join('-' * len(h) for h in headings) + '|', + ] + + for _, data in df.iterrows(): + row = '| ' + ' | '.join(_process_template(t,data) for t in template) + ' |' table.append(row) markup = '\n'.join(table) - + with open(WIKI_REPO_DIR + '{}.md'.format(profile_category), 'w') as fh: fh.writelines(markup) diff --git a/member_sandbox/databases/postgres.py b/member_sandbox/databases/postgres.py index bc0945a..ab87c7f 100644 --- a/member_sandbox/databases/postgres.py +++ b/member_sandbox/databases/postgres.py @@ -27,7 +27,7 @@ def nothing_to_return(sql): """ Evaluate if sql statement will return data or not """ - return True if 'CREATE' in sql or 'UPDATE' in sql or 'INSERT' in sql or 'DELETE' in sql else False + return 'CREATE' in sql or 'UPDATE' in sql or 'INSERT' in sql or 'DELETE' in sql def execute(info, sql, args=None, return_columns=False): @@ -46,7 +46,11 @@ def execute(info, sql, args=None, return_columns=False): rows = c.fetchall() # return the column names with the data - return tuple([desc[0] for desc in c.description]) + rows if return_columns else rows + return ( + tuple(desc[0] for desc in c.description) + rows + if return_columns + else rows + ) def executemany(info, sql, args=None):