From f9a0f65d81ceba8c8391d3bcae68c92231c85e0b Mon Sep 17 00:00:00 2001 From: bio-boris Date: Tue, 21 Oct 2025 15:21:51 -0500 Subject: [PATCH 01/21] Add authorization header to Elasticsearch query --- src/es_client/query.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/es_client/query.py b/src/es_client/query.py index 65d4be3..36f132c 100644 --- a/src/es_client/query.py +++ b/src/es_client/query.py @@ -86,10 +86,14 @@ def search(params, meta): headers = {'Content-Type': 'application/json'} + auth_header_value = config['authorization_header_value'] + if auth_header_value: + headers['Authorization'] = auth_header_value + # Allows index exclusion; otherwise there is an error params = {'allow_no_indices': 'true'} - resp = requests.post(url, data=json.dumps(options), params=params, headers=headers) + resp = requests.post(url, data=json.dumps(options), params=params, headers=headers) # nosec B113 if not resp.ok: _handle_es_err(resp) From 5e7547df0fcc821de2b75c2ec512f2b617746d89 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Tue, 21 Oct 2025 15:23:19 -0500 Subject: [PATCH 02/21] Refactor headers setup for Elasticsearch request --- src/search2_rpc/service.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/search2_rpc/service.py b/src/search2_rpc/service.py index 4f54b77..fdf8cbd 100644 --- a/src/search2_rpc/service.py +++ b/src/search2_rpc/service.py @@ -26,9 +26,14 @@ def show_indexes(params, meta): """List all index names for our prefix""" prefix = config['index_prefix'] + headers = {'Content-Type': 'application/json'} + auth_header_value = config['authorization_header_value'] + if auth_header_value: + headers['Authorization'] = auth_header_value resp = requests.get( config['elasticsearch_url'] + '/_cat/indices/' + prefix + '*?format=json', - headers={'Content-Type': 'application/json'}, + headers=headers, + timeout=120 ) if not resp.ok: raise ElasticsearchError(resp.text) From 6d8af834cf8150f50d32ac466b98b6700e7453ff Mon Sep 17 00:00:00 2001 From: bio-boris Date: Tue, 21 Oct 2025 15:24:28 -0500 Subject: [PATCH 03/21] Update __main__.py --- src/server/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/__main__.py b/src/server/__main__.py index f30a9a4..b1d4fc1 100644 --- a/src/server/__main__.py +++ b/src/server/__main__.py @@ -136,7 +136,7 @@ def _get_status_code(result: dict) -> int: # Wait for dependencies to start logger.info('Checking connection to elasticsearch') -wait_for_service(config['elasticsearch_url'], 'Elasticsearch') +wait_for_service(config['elasticsearch_url'], 'Elasticsearch', auth_token=config['authorization_header_value']) # Start the server app.run( host='0.0.0.0', # nosec From 8831845e5fe09229b8a590efaf6a81b98647b40c Mon Sep 17 00:00:00 2001 From: bio-boris Date: Tue, 21 Oct 2025 15:25:44 -0500 Subject: [PATCH 04/21] Implement Basic Authentication header encoder Added a function to encode username and password for Basic Authentication headers. --- src/utils/config.py | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/src/utils/config.py b/src/utils/config.py index f5bae4f..d73cc10 100644 --- a/src/utils/config.py +++ b/src/utils/config.py @@ -1,18 +1,30 @@ import yaml import urllib.request import os +import base64 + + +def auth_header_encoder(username, password): + """ + Encodes username and password for a Basic Authentication header. + Returns None if either username or password is not provided. + """ + if not (username and password): + return None + credentials = f"{username}:{password}" + credentials_bytes = credentials.encode('utf-8') + base64_credentials = base64.b64encode(credentials_bytes).decode('utf-8') + return f"Basic {base64_credentials}" def init_config(): """ - Initialize configuration data for the whole app + Initialize configuration data for the whole app. """ - # TODO: it might be better to NOT default to testing configuration, - # but rather explicitly set the test environment. - # Reason? A failure to configure one of these in prod could lead to - # confusing failure conditions. ws_url = os.environ.get('WORKSPACE_URL', 'https://ci.kbase.us/services/ws').strip('/') es_url = os.environ.get('ELASTICSEARCH_URL', 'http://localhost:9200').strip('/') + es_auth_username = os.environ.get('ELASTICSEARCH_AUTH_USERNAME') + es_auth_password = os.environ.get('ELASTICSEARCH_AUTH_PASSWORD') index_prefix = os.environ.get('INDEX_PREFIX', 'test') prefix_delimiter = os.environ.get('INDEX_PREFIX_DELIMITER', '.') suffix_delimiter = os.environ.get('INDEX_SUFFIX_DELIMITER', '_') @@ -24,6 +36,9 @@ def init_config(): 'USER_PROFILE_URL', 'https://ci.kbase.us/services/user_profile/rpc/' ) + + auth_header_value = auth_header_encoder(es_auth_username, es_auth_password) + # Load the global configuration release (non-environment specific, public config) allowed_protocols = ('https://', 'http://', 'file://') matches_protocol = (config_url.startswith(prot) for prot in allowed_protocols) @@ -33,10 +48,12 @@ def init_config(): global_config = yaml.safe_load(res) with open('VERSION') as fd: app_version = fd.read().replace('\n', '') + return { 'dev': bool(os.environ.get('DEVELOPMENT')), 'global': global_config, 'elasticsearch_url': es_url, + 'authorization_header_value': auth_header_value, 'index_prefix': index_prefix, 'prefix_delimiter': prefix_delimiter, 'suffix_delimiter': suffix_delimiter, From 17b149276ae6f27078dccd3310fb723cad1aedef Mon Sep 17 00:00:00 2001 From: bio-boris Date: Tue, 21 Oct 2025 15:33:20 -0500 Subject: [PATCH 05/21] Update wait_for_service.py --- src/utils/wait_for_service.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/utils/wait_for_service.py b/src/utils/wait_for_service.py index 6eff310..80144aa 100644 --- a/src/utils/wait_for_service.py +++ b/src/utils/wait_for_service.py @@ -7,12 +7,15 @@ WAIT_POLL_INTERVAL = 5 -def wait_for_service(url, name, timeout=DEFAULT_TIMEOUT): +def wait_for_service(url, name, timeout=DEFAULT_TIMEOUT, auth_token=None): start = time.time() + headers = {} + if auth_token: + headers['Authorization'] = auth_token while True: logger.info(f'Attempting to connect to {name} at {url}') try: - requests.get(url, timeout=timeout).raise_for_status() + requests.get(url, timeout=timeout, headers=headers).raise_for_status() logger.info(f'{name} is online!') break except Exception: From 54058f665a9889f1f82417f3799b336b75c9c2fd Mon Sep 17 00:00:00 2001 From: bio-boris Date: Tue, 21 Oct 2025 16:33:46 -0500 Subject: [PATCH 06/21] add headers to init --- docker-compose.yaml | 5 ++- tests/helpers/init_elasticsearch.py | 69 +++++++---------------------- 2 files changed, 20 insertions(+), 54 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index e8b4e5e..574c409 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -17,6 +17,8 @@ services: - DEVELOPMENT=1 - PYTHONUNBUFFERED=true - ELASTICSEARCH_URL=http://elasticsearch:9200 + - ELASTICSEARCH_AUTH_USERNAME=elastic + - ELASTICSEARCH_AUTH_PASSWORD=changeme - WORKERS=2 elasticsearch: @@ -25,7 +27,8 @@ services: - "ES_JAVA_OPTS=-Xms512m -Xmx512m" - bootstrap.memory_lock=true - discovery.type=single-node - - xpack.security.enabled=false + - xpack.security.enabled=true + - ELASTIC_PASSWORD=changeme ports: - "127.0.0.1:9200:9200" - "127.0.0.1:9300:9300" diff --git a/tests/helpers/init_elasticsearch.py b/tests/helpers/init_elasticsearch.py index dea80f0..1199a35 100644 --- a/tests/helpers/init_elasticsearch.py +++ b/tests/helpers/init_elasticsearch.py @@ -1,59 +1,22 @@ import requests import json - from src.utils.config import config -# TODO use a util for creating index names -narrative_index_name = ''.join([ - config['index_prefix'], - config['prefix_delimiter'], - config['global']['ws_type_to_indexes']['KBaseNarrative.Narrative'], -]) - -index_names = [ - config['index_prefix'] + config['prefix_delimiter'] + 'index1', - config['index_prefix'] + config['prefix_delimiter'] + 'index2', -] - -_ES_URL = 'http://localhost:9200' +# Define headers at module level +_BASE_HEADERS = {'Content-Type': 'application/json'} -# Simple run once semaphore -_COMPLETED = False -# -# For the test docs, note the workspace must match the doc's idea of permissions. -# See data.py for the workspace definitions in which: -# 0 - public workspace, refdata -# 1 - public workspace, narrative -# 100 - private workspace, narrative -# 101 - private, inaccessible workspace, narrative -test_docs = [ - # Public doc, refdata - {'name': 'public-doc1', 'access_group': '0', 'is_public': True, 'timestamp': 10}, - # Public doc, narrative - {'name': 'public-doc2', 'access_group': '1', 'is_public': True, 'timestamp': 12}, - # Private but accessible doc - {'name': 'private-doc1', 'is_public': False, 'access_group': '100', 'timestamp': 7}, - # Private but inaccessible doc - {'name': 'private-doc2', 'is_public': False, 'access_group': '101', 'timestamp': 9}, -] - -narrative_docs = [ - { - 'name': 'narrative1', - 'narrative_title': 'narrative1', - 'is_public': True, - 'obj_id': 123, - 'access_group': '1', - 'timestamp': 1, - }, -] +def _get_headers(): + """Get headers with optional authorization.""" + headers = _BASE_HEADERS.copy() + if config.get('authorization_header_value'): + headers['Authorization'] = config['authorization_header_value'] + return headers +# Then use it in your functions: def init_elasticsearch(): - """ - Initialize the indexes and documents on elasticsearch before running tests. - """ + """Initialize the indexes and documents on elasticsearch before running tests.""" global _COMPLETED if _COMPLETED: return @@ -65,6 +28,7 @@ def init_elasticsearch(): create_doc(index_name, doc) for doc in narrative_docs: create_doc(narrative_index_name, doc) + # create default_search alias for all fields. url = f"{_ES_URL}/_aliases" alias_name = config['index_prefix'] + config['prefix_delimiter'] + "default_search" @@ -73,7 +37,7 @@ def init_elasticsearch(): {"add": {"indices": index_names, "alias": alias_name}} ] } - resp = requests.post(url, data=json.dumps(body), headers={'Content-Type': 'application/json'}) + resp = requests.post(url, data=json.dumps(body), headers=_get_headers()) if not resp.ok: raise RuntimeError("Error creating aliases on ES:", resp.text) _COMPLETED = True @@ -91,7 +55,7 @@ def create_index(index_name): 'index': {'number_of_shards': 2, 'number_of_replicas': 1} } }), - headers={'Content-Type': 'application/json'}, + headers=_get_headers(), ) if not resp.ok and resp.json()['error']['type'] != 'index_already_exists_exception': raise RuntimeError('Error creating index on ES:', resp.text) @@ -99,14 +63,13 @@ def create_index(index_name): def create_doc(index_name, data): # Wait for doc to sync - url = '/'.join([ # type: ignore + url = '/'.join([ _ES_URL, index_name, '_doc', data['name'], '?refresh=wait_for' ]) - headers = {'Content-Type': 'application/json'} - resp = requests.put(url, data=json.dumps(data), headers=headers) + resp = requests.put(url, data=json.dumps(data), headers=_get_headers()) if not resp.ok: - raise RuntimeError(f"Error creating test doc:\n{resp.text}") + raise RuntimeError(f"Error creating test doc:\n{resp.text}") \ No newline at end of file From b6f0d7bea17da4d92f6f305a5ae50ef2d60c9497 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Tue, 21 Oct 2025 16:37:54 -0500 Subject: [PATCH 07/21] add headers to init --- tests/helpers/init_elasticsearch.py | 104 +++++++++++++++------------- 1 file changed, 57 insertions(+), 47 deletions(-) diff --git a/tests/helpers/init_elasticsearch.py b/tests/helpers/init_elasticsearch.py index 1199a35..62932d3 100644 --- a/tests/helpers/init_elasticsearch.py +++ b/tests/helpers/init_elasticsearch.py @@ -1,22 +1,69 @@ import requests import json -from src.utils.config import config -# Define headers at module level -_BASE_HEADERS = {'Content-Type': 'application/json'} +from src.utils.config import config def _get_headers(): - """Get headers with optional authorization.""" - headers = _BASE_HEADERS.copy() - if config.get('authorization_header_value'): - headers['Authorization'] = config['authorization_header_value'] + """Get HTTP headers for Elasticsearch requests, including auth if configured.""" + headers = {'Content-Type': 'application/json'} + auth_header_value = config.get('authorization_header_value') + if auth_header_value: + headers['Authorization'] = auth_header_value return headers -# Then use it in your functions: +# TODO use a util for creating index names +narrative_index_name = ''.join([ + config['index_prefix'], + config['prefix_delimiter'], + config['global']['ws_type_to_indexes']['KBaseNarrative.Narrative'], +]) + +index_names = [ + config['index_prefix'] + config['prefix_delimiter'] + 'index1', + config['index_prefix'] + config['prefix_delimiter'] + 'index2', +] + +_ES_URL = 'http://localhost:9200' + +# Simple run once semaphore +_COMPLETED = False + +# +# For the test docs, note the workspace must match the doc's idea of permissions. +# See data.py for the workspace definitions in which: +# 0 - public workspace, refdata +# 1 - public workspace, narrative +# 100 - private workspace, narrative +# 101 - private, inaccessible workspace, narrative +test_docs = [ + # Public doc, refdata + {'name': 'public-doc1', 'access_group': '0', 'is_public': True, 'timestamp': 10}, + # Public doc, narrative + {'name': 'public-doc2', 'access_group': '1', 'is_public': True, 'timestamp': 12}, + # Private but accessible doc + {'name': 'private-doc1', 'is_public': False, 'access_group': '100', 'timestamp': 7}, + # Private but inaccessible doc + {'name': 'private-doc2', 'is_public': False, 'access_group': '101', 'timestamp': 9}, +] + +narrative_docs = [ + { + 'name': 'narrative1', + 'narrative_title': 'narrative1', + 'is_public': True, + 'obj_id': 123, + 'access_group': '1', + 'timestamp': 1, + }, +] + + def init_elasticsearch(): - """Initialize the indexes and documents on elasticsearch before running tests.""" + """ + Initialize the indexes and documents on elasticsearch before running tests. + """ global _COMPLETED if _COMPLETED: return @@ -28,7 +75,6 @@ def init_elasticsearch(): create_doc(index_name, doc) for doc in narrative_docs: create_doc(narrative_index_name, doc) - # create default_search alias for all fields. url = f"{_ES_URL}/_aliases" alias_name = config['index_prefix'] + config['prefix_delimiter'] + "default_search" @@ -36,40 +82,4 @@ def init_elasticsearch(): "actions": [ {"add": {"indices": index_names, "alias": alias_name}} ] - } - resp = requests.post(url, data=json.dumps(body), headers=_get_headers()) - if not resp.ok: - raise RuntimeError("Error creating aliases on ES:", resp.text) - _COMPLETED = True - - -def create_index(index_name): - # Check if exists - resp = requests.head(_ES_URL + '/' + index_name) - if resp.status_code == 200: - return - resp = requests.put( - _ES_URL + '/' + index_name, - data=json.dumps({ - 'settings': { - 'index': {'number_of_shards': 2, 'number_of_replicas': 1} - } - }), - headers=_get_headers(), - ) - if not resp.ok and resp.json()['error']['type'] != 'index_already_exists_exception': - raise RuntimeError('Error creating index on ES:', resp.text) - - -def create_doc(index_name, data): - # Wait for doc to sync - url = '/'.join([ - _ES_URL, - index_name, - '_doc', - data['name'], - '?refresh=wait_for' - ]) - resp = requests.put(url, data=json.dumps(data), headers=_get_headers()) - if not resp.ok: - raise RuntimeError(f"Error creating test doc:\n{resp.text}") \ No newline at end of file + } \ No newline at end of file From 655b92d5dda9aa4a0c97f5e95de19c83a501f4c0 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Tue, 21 Oct 2025 16:38:58 -0500 Subject: [PATCH 08/21] add headers to init --- tests/helpers/init_elasticsearch.py | 40 ++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/tests/helpers/init_elasticsearch.py b/tests/helpers/init_elasticsearch.py index 62932d3..b9c5e9a 100644 --- a/tests/helpers/init_elasticsearch.py +++ b/tests/helpers/init_elasticsearch.py @@ -60,6 +60,40 @@ def _get_headers(): ] +def create_index(index_name): + """Create an Elasticsearch index if it does not already exist.""" + # Check if exists + resp = requests.head(_ES_URL + '/' + index_name) + if resp.status_code == 200: + return + resp = requests.put( + _ES_URL + '/' + index_name, + data=json.dumps({ + 'settings': { + 'index': {'number_of_shards': 2, 'number_of_replicas': 1} + } + }), + headers=_get_headers(), + ) + if not resp.ok and resp.json()['error']['type'] != 'index_already_exists_exception': + raise RuntimeError('Error creating index on ES:', resp.text) + + +def create_doc(index_name, data): + """Create a document in the specified index.""" + # Wait for doc to sync + url = '/'.join([ + _ES_URL, + index_name, + '_doc', + data['name'], + '?refresh=wait_for' + ]) + resp = requests.put(url, data=json.dumps(data), headers=_get_headers()) + if not resp.ok: + raise RuntimeError(f"Error creating test doc:\n{resp.text}") + + def init_elasticsearch(): """ Initialize the indexes and documents on elasticsearch before running tests. @@ -82,4 +116,8 @@ def init_elasticsearch(): "actions": [ {"add": {"indices": index_names, "alias": alias_name}} ] - } \ No newline at end of file + } + resp = requests.post(url, data=json.dumps(body), headers=_get_headers()) + if not resp.ok: + raise RuntimeError("Error creating aliases on ES:", resp.text) + _COMPLETED = True \ No newline at end of file From 3d1d4e8f8538d5711da36877d8e416d4e2e02f37 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Tue, 21 Oct 2025 16:39:23 -0500 Subject: [PATCH 09/21] add headers to init --- tests/helpers/init_elasticsearch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/helpers/init_elasticsearch.py b/tests/helpers/init_elasticsearch.py index b9c5e9a..eb5ee38 100644 --- a/tests/helpers/init_elasticsearch.py +++ b/tests/helpers/init_elasticsearch.py @@ -120,4 +120,4 @@ def init_elasticsearch(): resp = requests.post(url, data=json.dumps(body), headers=_get_headers()) if not resp.ok: raise RuntimeError("Error creating aliases on ES:", resp.text) - _COMPLETED = True \ No newline at end of file + _COMPLETED = True From b8349ac42f3511e0e17b9f03a579b6bce7fc8ff6 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Tue, 21 Oct 2025 16:45:57 -0500 Subject: [PATCH 10/21] Integrate! --- .github/workflows/test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f58827d..fd629ea 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,6 +1,3 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - name: Run search_api2 tests on: @@ -21,6 +18,9 @@ on: jobs: build: runs-on: ubuntu-latest + env: + ELASTICSEARCH_AUTH_USERNAME: elastic + ELASTICSEARCH_AUTH_PASSWORD: changeme steps: - name: Check out GitHub repo @@ -47,4 +47,4 @@ jobs: uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} - fail_ci_if_error: true + fail_ci_if_error: true \ No newline at end of file From 3586cbc150b6e90528a2deaf3b75525e23130a09 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Tue, 21 Oct 2025 16:55:56 -0500 Subject: [PATCH 11/21] Removed magic number --- src/search2_rpc/service.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/search2_rpc/service.py b/src/search2_rpc/service.py index fdf8cbd..b98931f 100644 --- a/src/search2_rpc/service.py +++ b/src/search2_rpc/service.py @@ -33,7 +33,6 @@ def show_indexes(params, meta): resp = requests.get( config['elasticsearch_url'] + '/_cat/indices/' + prefix + '*?format=json', headers=headers, - timeout=120 ) if not resp.ok: raise ElasticsearchError(resp.text) From 73c6bfaecc31c00835158a926afa8be86dc65f2f Mon Sep 17 00:00:00 2001 From: bio-boris Date: Tue, 21 Oct 2025 19:43:03 -0500 Subject: [PATCH 12/21] Update test.yml --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fd629ea..871cbb4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,7 +20,7 @@ jobs: runs-on: ubuntu-latest env: ELASTICSEARCH_AUTH_USERNAME: elastic - ELASTICSEARCH_AUTH_PASSWORD: changeme + ELASTICSEARCH_AUTH_PASSWORD: changeme123 steps: - name: Check out GitHub repo From 6bd49916953d36c31c2956c8180b522666232968 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Tue, 21 Oct 2025 19:48:25 -0500 Subject: [PATCH 13/21] Update test.yml --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 871cbb4..fd629ea 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,7 +20,7 @@ jobs: runs-on: ubuntu-latest env: ELASTICSEARCH_AUTH_USERNAME: elastic - ELASTICSEARCH_AUTH_PASSWORD: changeme123 + ELASTICSEARCH_AUTH_PASSWORD: changeme steps: - name: Check out GitHub repo From 2268345197c1e40fc2b5e5b475653dc88486fcbe Mon Sep 17 00:00:00 2001 From: bio-boris Date: Thu, 23 Oct 2025 15:31:52 -0500 Subject: [PATCH 14/21] require auth --- src/es_client/query.py | 8 +------- src/search2_rpc/service.py | 6 +----- src/server/__main__.py | 2 +- src/utils/config.py | 11 ++++++++--- src/utils/wait_for_service.py | 5 +---- tests/helpers/init_elasticsearch.py | 15 +++------------ tests/unit/utils/test_config.py | 16 +++++++++++++++- 7 files changed, 30 insertions(+), 33 deletions(-) diff --git a/src/es_client/query.py b/src/es_client/query.py index 36f132c..358a0a5 100644 --- a/src/es_client/query.py +++ b/src/es_client/query.py @@ -84,16 +84,10 @@ def search(params, meta): if params.get('track_total_hits'): options['track_total_hits'] = params.get('track_total_hits') - headers = {'Content-Type': 'application/json'} - - auth_header_value = config['authorization_header_value'] - if auth_header_value: - headers['Authorization'] = auth_header_value - # Allows index exclusion; otherwise there is an error params = {'allow_no_indices': 'true'} - resp = requests.post(url, data=json.dumps(options), params=params, headers=headers) # nosec B113 + resp = requests.post(url, data=json.dumps(options), params=params, headers=config['elasticsearch_headers']) # nosec B113 if not resp.ok: _handle_es_err(resp) diff --git a/src/search2_rpc/service.py b/src/search2_rpc/service.py index b98931f..ddd564d 100644 --- a/src/search2_rpc/service.py +++ b/src/search2_rpc/service.py @@ -26,13 +26,9 @@ def show_indexes(params, meta): """List all index names for our prefix""" prefix = config['index_prefix'] - headers = {'Content-Type': 'application/json'} - auth_header_value = config['authorization_header_value'] - if auth_header_value: - headers['Authorization'] = auth_header_value resp = requests.get( config['elasticsearch_url'] + '/_cat/indices/' + prefix + '*?format=json', - headers=headers, + headers=config['elasticsearch_headers'], ) if not resp.ok: raise ElasticsearchError(resp.text) diff --git a/src/server/__main__.py b/src/server/__main__.py index b1d4fc1..601585d 100644 --- a/src/server/__main__.py +++ b/src/server/__main__.py @@ -136,7 +136,7 @@ def _get_status_code(result: dict) -> int: # Wait for dependencies to start logger.info('Checking connection to elasticsearch') -wait_for_service(config['elasticsearch_url'], 'Elasticsearch', auth_token=config['authorization_header_value']) +wait_for_service(config['elasticsearch_url'], 'Elasticsearch', config['elasticsearch_headers']) # Start the server app.run( host='0.0.0.0', # nosec diff --git a/src/utils/config.py b/src/utils/config.py index d73cc10..f183baa 100644 --- a/src/utils/config.py +++ b/src/utils/config.py @@ -7,10 +7,10 @@ def auth_header_encoder(username, password): """ Encodes username and password for a Basic Authentication header. - Returns None if either username or password is not provided. + Raises RuntimeError if either username or password is not provided. """ if not (username and password): - return None + raise RuntimeError("Elasticsearch authentication credentials are required. Set ELASTICSEARCH_AUTH_USERNAME and ELASTICSEARCH_AUTH_PASSWORD environment variables.") credentials = f"{username}:{password}" credentials_bytes = credentials.encode('utf-8') base64_credentials = base64.b64encode(credentials_bytes).decode('utf-8') @@ -38,6 +38,11 @@ def init_config(): ) auth_header_value = auth_header_encoder(es_auth_username, es_auth_password) + # Store the complete headers dict with all required keys + elasticsearch_headers = { + 'Content-Type': 'application/json', + 'Authorization': auth_header_value + } # Load the global configuration release (non-environment specific, public config) allowed_protocols = ('https://', 'http://', 'file://') @@ -53,7 +58,7 @@ def init_config(): 'dev': bool(os.environ.get('DEVELOPMENT')), 'global': global_config, 'elasticsearch_url': es_url, - 'authorization_header_value': auth_header_value, + 'elasticsearch_headers': elasticsearch_headers, 'index_prefix': index_prefix, 'prefix_delimiter': prefix_delimiter, 'suffix_delimiter': suffix_delimiter, diff --git a/src/utils/wait_for_service.py b/src/utils/wait_for_service.py index 80144aa..a21ad87 100644 --- a/src/utils/wait_for_service.py +++ b/src/utils/wait_for_service.py @@ -7,11 +7,8 @@ WAIT_POLL_INTERVAL = 5 -def wait_for_service(url, name, timeout=DEFAULT_TIMEOUT, auth_token=None): +def wait_for_service(url, name, headers, timeout=DEFAULT_TIMEOUT): start = time.time() - headers = {} - if auth_token: - headers['Authorization'] = auth_token while True: logger.info(f'Attempting to connect to {name} at {url}') try: diff --git a/tests/helpers/init_elasticsearch.py b/tests/helpers/init_elasticsearch.py index eb5ee38..fbdae22 100644 --- a/tests/helpers/init_elasticsearch.py +++ b/tests/helpers/init_elasticsearch.py @@ -4,15 +4,6 @@ from src.utils.config import config -def _get_headers(): - """Get HTTP headers for Elasticsearch requests, including auth if configured.""" - headers = {'Content-Type': 'application/json'} - auth_header_value = config.get('authorization_header_value') - if auth_header_value: - headers['Authorization'] = auth_header_value - return headers - - # TODO use a util for creating index names narrative_index_name = ''.join([ config['index_prefix'], @@ -73,7 +64,7 @@ def create_index(index_name): 'index': {'number_of_shards': 2, 'number_of_replicas': 1} } }), - headers=_get_headers(), + headers=config['elasticsearch_headers'], ) if not resp.ok and resp.json()['error']['type'] != 'index_already_exists_exception': raise RuntimeError('Error creating index on ES:', resp.text) @@ -89,7 +80,7 @@ def create_doc(index_name, data): data['name'], '?refresh=wait_for' ]) - resp = requests.put(url, data=json.dumps(data), headers=_get_headers()) + resp = requests.put(url, data=json.dumps(data), headers=config['elasticsearch_headers']) if not resp.ok: raise RuntimeError(f"Error creating test doc:\n{resp.text}") @@ -117,7 +108,7 @@ def init_elasticsearch(): {"add": {"indices": index_names, "alias": alias_name}} ] } - resp = requests.post(url, data=json.dumps(body), headers=_get_headers()) + resp = requests.post(url, data=json.dumps(body), headers=config['elasticsearch_headers']) if not resp.ok: raise RuntimeError("Error creating aliases on ES:", resp.text) _COMPLETED = True diff --git a/tests/unit/utils/test_config.py b/tests/unit/utils/test_config.py index d859499..1f2b5e5 100644 --- a/tests/unit/utils/test_config.py +++ b/tests/unit/utils/test_config.py @@ -1,4 +1,4 @@ -from src.utils.config import init_config +from src.utils.config import init_config, auth_header_encoder import os import pytest @@ -13,3 +13,17 @@ def test_init_config_invalid_config_url(): os.environ['GLOBAL_CONFIG_URL'] = original_url else: os.environ.pop('GLOBAL_CONFIG_URL') + + +@pytest.mark.parametrize("username,password", [ + (None, None), + (None, 'password'), + ('username', None), + ('', ''), + ('', 'password'), + ('username', ''), +]) +def test_auth_header_encoder_missing_credentials(username, password): + with pytest.raises(RuntimeError) as rte: + auth_header_encoder(username, password) + assert 'Elasticsearch authentication credentials are required' in str(rte) From c883797a45b6578f106288b49ceb1fef39e27402 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Thu, 23 Oct 2025 15:34:59 -0500 Subject: [PATCH 15/21] require auth --- tests/helpers/integration_setup.py | 2 +- tests/helpers/unit_setup.py | 3 ++- tests/unit/utils/test_wait_for_service.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/helpers/integration_setup.py b/tests/helpers/integration_setup.py index 81b888e..1e24965 100644 --- a/tests/helpers/integration_setup.py +++ b/tests/helpers/integration_setup.py @@ -28,7 +28,7 @@ def start_service(app_url): stdout=container_out, stderr=container_err, cwd=cwd) - wait_for_service(app_url, "search2") + wait_for_service(app_url, "search2", {}) def stop_service(): diff --git a/tests/helpers/unit_setup.py b/tests/helpers/unit_setup.py index c82b37e..ad6bf90 100644 --- a/tests/helpers/unit_setup.py +++ b/tests/helpers/unit_setup.py @@ -1,5 +1,6 @@ import subprocess from src.utils.wait_for_service import wait_for_service +from src.utils.config import config from src.utils.logger import logger import json import os @@ -29,7 +30,7 @@ def start_service(wait_for_url, wait_for_name): container_out = open("container.out", "w") container_err = open("container.err", "w") container_process = subprocess.Popen(cmd, shell=True, stdout=container_out, stderr=container_err) - wait_for_service(wait_for_url, wait_for_name) + wait_for_service(wait_for_url, wait_for_name, config['elasticsearch_headers']) def stop_service(): diff --git a/tests/unit/utils/test_wait_for_service.py b/tests/unit/utils/test_wait_for_service.py index 43913c0..d17e98b 100644 --- a/tests/unit/utils/test_wait_for_service.py +++ b/tests/unit/utils/test_wait_for_service.py @@ -16,7 +16,7 @@ def bad_url_with_timeout(name, url, timeout, caplog): with caplog.at_level(logging.INFO, logger='search2'): start = time.time() with pytest.raises(SystemExit) as se: - wait_for_service(url, 'foo', timeout=timeout) + wait_for_service(url, 'foo', {}, timeout=timeout) # Ensure it is attempting to exit. assert se.type == SystemExit From 4a40c6ccf6f8bb3837ff3b206d332ba1446b6eed Mon Sep 17 00:00:00 2001 From: bio-boris Date: Thu, 23 Oct 2025 15:36:21 -0500 Subject: [PATCH 16/21] Update config.py --- src/utils/config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/utils/config.py b/src/utils/config.py index f183baa..85b8675 100644 --- a/src/utils/config.py +++ b/src/utils/config.py @@ -38,7 +38,6 @@ def init_config(): ) auth_header_value = auth_header_encoder(es_auth_username, es_auth_password) - # Store the complete headers dict with all required keys elasticsearch_headers = { 'Content-Type': 'application/json', 'Authorization': auth_header_value From 6954f9993ca7b25763fb673d09badbf0675011cc Mon Sep 17 00:00:00 2001 From: bio-boris Date: Thu, 23 Oct 2025 15:39:34 -0500 Subject: [PATCH 17/21] lint --- src/es_client/query.py | 4 +++- src/utils/config.py | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/es_client/query.py b/src/es_client/query.py index 358a0a5..1015f78 100644 --- a/src/es_client/query.py +++ b/src/es_client/query.py @@ -87,7 +87,9 @@ def search(params, meta): # Allows index exclusion; otherwise there is an error params = {'allow_no_indices': 'true'} - resp = requests.post(url, data=json.dumps(options), params=params, headers=config['elasticsearch_headers']) # nosec B113 + resp = requests.post( + url, data=json.dumps(options), params=params, headers=config['elasticsearch_headers'] + ) # nosec B113 if not resp.ok: _handle_es_err(resp) diff --git a/src/utils/config.py b/src/utils/config.py index 85b8675..66da593 100644 --- a/src/utils/config.py +++ b/src/utils/config.py @@ -10,7 +10,10 @@ def auth_header_encoder(username, password): Raises RuntimeError if either username or password is not provided. """ if not (username and password): - raise RuntimeError("Elasticsearch authentication credentials are required. Set ELASTICSEARCH_AUTH_USERNAME and ELASTICSEARCH_AUTH_PASSWORD environment variables.") + raise RuntimeError( + "Elasticsearch authentication credentials are required. " + "Set ELASTICSEARCH_AUTH_USERNAME and ELASTICSEARCH_AUTH_PASSWORD environment variables." + ) credentials = f"{username}:{password}" credentials_bytes = credentials.encode('utf-8') base64_credentials = base64.b64encode(credentials_bytes).decode('utf-8') From 2092636f438b5dab2b4d4accd7c58d6ba7bc0662 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Thu, 23 Oct 2025 16:14:33 -0500 Subject: [PATCH 18/21] Update CHANGELOG.md --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f74a0c..840f275 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,12 +11,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Upgraded Python to version 3.9.19 in test workflows and Dockerfile - Updated integration tests README file +- Required tests to now use authentication against elasticsearch +- ### Fixed - Container/service shutdown issues; all unit and integration tests now pass locally ### Security - Vendored `kbase-jsonrpcbase` 0.3.0a6 and `jsonrpc11base` to resolve dependency conflicts +- Now requires credentials for connection to elasticsearch + ## [1.0.0] - 2021-04-20 ### Fixed From 63662a2ca637796c68be2cfbe58c6c94e3852d2e Mon Sep 17 00:00:00 2001 From: bio-boris Date: Mon, 27 Oct 2025 11:54:37 -0500 Subject: [PATCH 19/21] Update CHANGELOG.md Co-authored-by: MrCreosote --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 840f275..bb2436d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Upgraded Python to version 3.9.19 in test workflows and Dockerfile - Updated integration tests README file - Required tests to now use authentication against elasticsearch -- ### Fixed - Container/service shutdown issues; all unit and integration tests now pass locally From 6cfd89a45d62264f04ec7a140006a443ba463943 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Mon, 27 Oct 2025 11:54:53 -0500 Subject: [PATCH 20/21] Update .github/workflows/test.yml Co-authored-by: MrCreosote --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fd629ea..103f1a9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -47,4 +47,4 @@ jobs: uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} - fail_ci_if_error: true \ No newline at end of file + fail_ci_if_error: true From f91467a3dee0ddf6751bdefe26b0f99379923d43 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Mon, 27 Oct 2025 11:55:15 -0500 Subject: [PATCH 21/21] Update tests/unit/utils/test_config.py Co-authored-by: MrCreosote --- tests/unit/utils/test_config.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/unit/utils/test_config.py b/tests/unit/utils/test_config.py index 1f2b5e5..a6888e7 100644 --- a/tests/unit/utils/test_config.py +++ b/tests/unit/utils/test_config.py @@ -24,6 +24,5 @@ def test_init_config_invalid_config_url(): ('username', ''), ]) def test_auth_header_encoder_missing_credentials(username, password): - with pytest.raises(RuntimeError) as rte: + with pytest.raises(RuntimeError, match="Elasticsearch authentication credentials are required"): auth_header_encoder(username, password) - assert 'Elasticsearch authentication credentials are required' in str(rte)