From f0e5ccfe1fb0c9d9ff3beb6d0b3573aa539c2897 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Sun, 6 Jul 2025 18:52:12 +0200 Subject: [PATCH 1/6] Add bulk_export and adjustments needed to make it work. Changed to requests.session object for persistent connection recycling --- CHANGELOG.md | 16 ++++++++++++ src/xurrent/core.py | 63 ++++++++++++++++++++++++++++++++++++--------- 2 files changed, 67 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2848c5c..8f3790e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,22 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [0.9.2] - 2025-07-06 + +### Added + +- Core: Added bulk_export() function to dowload bulk record data + +### Changed + +- Core: Switched to a requests.session object to enable persistent connection recycling. +- Core: Provide options to disable pagination and prevent api result's JSON parsing (in association with bulk_export). + +### Bug Fixes + +- Core: do not prepend the base_url to the uri of an api_call if a protocol is already included (i.e. uri is already fully-formed). + ## [0.9.1] - 2025-06-13 ### Changed diff --git a/src/xurrent/core.py b/src/xurrent/core.py index 9ad836f..1cc78e4 100644 --- a/src/xurrent/core.py +++ b/src/xurrent/core.py @@ -63,6 +63,12 @@ def __init__(self, base_url, api_key, api_account,resolve_user=True, logger: Log self.logger = logger else: self.logger = self.create_logger(False) + #Create a requests session to maintain persistent connections, with preset headers + self.__requests = requests.session() + self.__requests.headers.update({ + 'Authorization': f'Bearer {self.api_key}', + 'x-xurrent-account': self.api_account + }) if resolve_user: # Import Person lazily from .people import Person @@ -130,31 +136,27 @@ def set_log_level(self, level: LogLevel): handler.setLevel(level) - def api_call(self, uri: str, method='GET', data=None, per_page=100): + def api_call(self, uri: str, method='GET', data=None, per_page=100, raw=False): """ Make a call to the Xurrent API with support for rate limiting and pagination. :param uri: URI to call - :param method: HTTP method to use + :param method: HTTP method to use (default: GET) :param data: Data to send with the request (optional) - :param per_page: Number of records per page for GET requests (default: 100) + :param per_page: Number of records per page for GET requests, setting to 0/None disables pagination (default: 100) + :param raw: Do not process the request result, e.g. in the case of non-JSON data (default: False) :return: JSON response from the API or aggregated data for paginated GET """ - # Ensure the base URL is included in the URI - if not uri.startswith(self.base_url): + # Ensure the base URL is included in the URI, if no protocol (https://) specified + if not uri.startswith(self.base_url) and "://" not in uri[:10]: uri = f'{self.base_url}{uri}' - headers = { - 'Authorization': f'Bearer {self.api_key}', - 'x-xurrent-account': self.api_account - } - aggregated_data = [] next_page_url = uri while next_page_url: try: # Append pagination parameters for GET requests - if method == 'GET': + if per_page and method == 'GET': # if contains ? or does not end with /, append per_page next_page_url = self.__append_per_page(next_page_url, per_page) @@ -162,7 +164,7 @@ def api_call(self, uri: str, method='GET', data=None, per_page=100): self.logger.debug(f'{method} {next_page_url} {data if method != "GET" else ""}') # Make the HTTP request - response = requests.request(method, next_page_url, headers=headers, json=data) + response = self.__requests.request(method, next_page_url, json=data) if response.status_code == 204: return None @@ -179,6 +181,10 @@ def api_call(self, uri: str, method='GET', data=None, per_page=100): self.logger.error(f'Error in request: {response.status_code} - {response.text}') response.raise_for_status() + #Stop here if we shall not process or interperet the returned data + if raw: + return response.content + # Process response response_data = response.json() @@ -206,6 +212,39 @@ def api_call(self, uri: str, method='GET', data=None, per_page=100): # Return aggregated results for paginated GET return aggregated_data + def bulk_export(self, type: str, export_format='csv', save_as=None, poll_timeout=5): + """ + Make a call to the Xurrent API to perform a bulk export + :param type: Resource type(s) to download, comma-delimited + :param export_format: either 'csv' or 'xlsx' (Default: csv) + :param save_as: Save the results to a file instead of returning the raw result + :param poll_timeout: Seconds to wait between export result polls (Default: 5 seconds) + :return: CSV or ZIP data from the export + """ + + #Initiate an export and get the polling token + export = self.api_call('/export', method = 'POST', data = dict(type = type, export_format = export_format)) + + #Begin export results poll waiting loop + while True: + self.logger.debug('Export poll wait.') + time.sleep(poll_timeout) + result = self.api_call(f"/export/{export['token']}", per_page = None) + if result['state'] in ("queued","processing"): + continue + if result['state'] == "done": + break + self.logger.error(f'Export request failed: {result=}') + raise + + #Save or Return the exported data + result = self.api_call(result["url"], per_page = None, raw = True) + if save_as: + with open(save_as, 'wb') as file: + file.write(result) + return True + return result + def custom_fields_to_object(self, custom_fields): """ Convert a list of custom fields to a dictionary. From ca316895a4b161e9cd399fc5e395459322751aa6 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 15 Jul 2025 01:28:17 +0200 Subject: [PATCH 2/6] Call the session for what it is --- src/xurrent/core.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/xurrent/core.py b/src/xurrent/core.py index 1cc78e4..e9c72a7 100644 --- a/src/xurrent/core.py +++ b/src/xurrent/core.py @@ -64,8 +64,8 @@ def __init__(self, base_url, api_key, api_account,resolve_user=True, logger: Log else: self.logger = self.create_logger(False) #Create a requests session to maintain persistent connections, with preset headers - self.__requests = requests.session() - self.__requests.headers.update({ + self.__session = requests.session() + self.__session.headers.update({ 'Authorization': f'Bearer {self.api_key}', 'x-xurrent-account': self.api_account }) @@ -164,7 +164,7 @@ def api_call(self, uri: str, method='GET', data=None, per_page=100, raw=False): self.logger.debug(f'{method} {next_page_url} {data if method != "GET" else ""}') # Make the HTTP request - response = self.__requests.request(method, next_page_url, json=data) + response = self.__session.request(method, next_page_url, json=data) if response.status_code == 204: return None From b52fbe894dad48f5612cd8457a94ea74e434d969 Mon Sep 17 00:00:00 2001 From: Fabian Franz Steiner <75947402+fasteiner@users.noreply.github.com> Date: Wed, 6 Aug 2025 11:51:16 +0200 Subject: [PATCH 3/6] Update CHANGELOG.md change last header to unreleased --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f3790e..be28653 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [0.9.2] - 2025-07-06 +## [Unreleased] ### Added From 2dfbec340abf166bf9904de97087ce558be7535c Mon Sep 17 00:00:00 2001 From: Fabian Franz Steiner <75947402+fasteiner@users.noreply.github.com> Date: Wed, 6 Aug 2025 12:02:12 +0200 Subject: [PATCH 4/6] Update CHANGELOG.md bug fix --> fixed, to be compliant with Keep a ChangeLog standard --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index be28653..a4d47db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Core: Switched to a requests.session object to enable persistent connection recycling. - Core: Provide options to disable pagination and prevent api result's JSON parsing (in association with bulk_export). -### Bug Fixes +### Fixed - Core: do not prepend the base_url to the uri of an api_call if a protocol is already included (i.e. uri is already fully-formed). From 9cc4dd48f7effe4bb27c8802066cb770995402ca Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Sat, 9 Aug 2025 20:02:00 +0200 Subject: [PATCH 5/6] Readme update, export fucntion format standardization --- README.md | 25 +++++++++++++++---------- src/xurrent/core.py | 8 ++++---- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index cfe8fd6..4c9bdea 100644 --- a/README.md +++ b/README.md @@ -34,12 +34,11 @@ This module is used to interact with the Xurrent API. It provides a set of class # Plain API Call uri = "/requests?subject=Example Subject" - connection_object.api_call(uri, 'GET') + x_api_helper.api_call(uri, 'GET') # Convert node ID - helper.decode_api_id('ZmFiaWFuc3RlaW5lci4yNDEyMTAxMDE0MTJANG1lLWRlbW8uY29tL1JlcS83MDU3NTU') # fabiansteiner.241210101412@4me-demo.com/Req/705755 + x_api_helper.decode_api_id('ZmFiaWFuc3RlaW5lci4yNDEyMTAxMDE0MTJANG1lLWRlbW8uY29tL1JlcS83MDU3NTU') # fabiansteiner.241210101412@4me-demo.com/Req/705755 # this can be used to derive the ID from the nodeID - ``` #### Configuration Items @@ -68,7 +67,7 @@ This module is used to interact with the Xurrent API. It provides a set of class # creating without specifying the label, takes the last ci of the product and increments the label # example: "wdc-02" -> "wdc-03" data = {"name": "New CI", "type": "software", "status": "in_production", "product_id": ""} - new_ci = ConfigurationItem.create(api_helper, data) + new_ci = ConfigurationItem.create(x_api_helper, data) print(new_ci) # Archive a Configuration Item (must be in an allowed state) @@ -115,7 +114,6 @@ This module is used to interact with the Xurrent API. It provides a set of class people.trash() #restore people.restore() - ``` #### Requests @@ -142,7 +140,6 @@ This module is used to interact with the Xurrent API. It provides a set of class #restore request.restore() - ``` @@ -212,7 +209,6 @@ This module is used to interact with the Xurrent API. It provides a set of class "text": "This is a test note", "internal": True }) - ``` #### Tasks @@ -240,8 +236,6 @@ This module is used to interact with the Xurrent API. It provides a set of class task.reject() #approve task.approve() - - ``` #### Teams @@ -271,7 +265,6 @@ This module is used to interact with the Xurrent API. It provides a set of class #### Workflows ```python - from xurrent.workflows import Workflow workflow = Workflow.get_by_id(x_api_helper, ) @@ -284,3 +277,15 @@ This module is used to interact with the Xurrent API. It provides a set of class workflow.close(completion_reason="withdrawn", note="This is a test note") ``` + +### Bulk Export +```python + import csv + + #Request a bulk export of "people" + csvdata = x_api_helper.bulk_export("people") + + #Iterate fetched export rows with the csv library, where row 1 defines the column names + for row in csv.reader(csvdata): + print(row["Employee Number"]) +``` diff --git a/src/xurrent/core.py b/src/xurrent/core.py index e9c72a7..d97061a 100644 --- a/src/xurrent/core.py +++ b/src/xurrent/core.py @@ -64,7 +64,7 @@ def __init__(self, base_url, api_key, api_account,resolve_user=True, logger: Log else: self.logger = self.create_logger(False) #Create a requests session to maintain persistent connections, with preset headers - self.__session = requests.session() + self.__session = requests.Session() self.__session.headers.update({ 'Authorization': f'Bearer {self.api_key}', 'x-xurrent-account': self.api_account @@ -219,7 +219,7 @@ def bulk_export(self, type: str, export_format='csv', save_as=None, poll_timeout :param export_format: either 'csv' or 'xlsx' (Default: csv) :param save_as: Save the results to a file instead of returning the raw result :param poll_timeout: Seconds to wait between export result polls (Default: 5 seconds) - :return: CSV or ZIP data from the export + :return: CSV or XSLX data from the export, ZIP if multiple types supplied """ #Initiate an export and get the polling token @@ -230,9 +230,9 @@ def bulk_export(self, type: str, export_format='csv', save_as=None, poll_timeout self.logger.debug('Export poll wait.') time.sleep(poll_timeout) result = self.api_call(f"/export/{export['token']}", per_page = None) - if result['state'] in ("queued","processing"): + if result['state'] in ('queued','processing'): continue - if result['state'] == "done": + if result['state'] == 'done': break self.logger.error(f'Export request failed: {result=}') raise From 6bf59f0d157a8e05e7f7dc0ef67035034973335d Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Sat, 9 Aug 2025 21:50:33 +0200 Subject: [PATCH 6/6] Example fix --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4c9bdea..e971d57 100644 --- a/README.md +++ b/README.md @@ -281,11 +281,12 @@ This module is used to interact with the Xurrent API. It provides a set of class ### Bulk Export ```python import csv + import io #Request a bulk export of "people" csvdata = x_api_helper.bulk_export("people") #Iterate fetched export rows with the csv library, where row 1 defines the column names - for row in csv.reader(csvdata): + for row in csv.DictReader(io.StringIO(csvdata)): print(row["Employee Number"]) ```