diff --git a/metecho/api/sf_run_flow.py b/metecho/api/sf_run_flow.py index 2884e9f40..408f38097 100644 --- a/metecho/api/sf_run_flow.py +++ b/metecho/api/sf_run_flow.py @@ -2,11 +2,12 @@ import json import logging import os +from socket import gaierror +import requests import shutil import subprocess import time from datetime import datetime - from cumulusci.core.config import OrgConfig, TaskConfig from cumulusci.core.runtime import BaseCumulusCI from cumulusci.oauth.client import OAuth2Client, OAuth2ClientConfig @@ -265,27 +266,60 @@ def mutate_scratch_org(*, scratch_org_config, org_result, email): ) +def is_network_error(exception) -> bool: + """Helper function to determine if a network error, + such as a dns propagation delay is occuring.""" + + # gai error stands for GetAddressInfo Error + if isinstance(exception, gaierror): + return True + subexception = exception.__context__ or exception.__cause__ + if subexception: + return is_network_error(subexception) + else: + return False + + def get_access_token(*, org_result, scratch_org_config): - """Trades the AuthCode from a ScratchOrgInfo for an org access token, - and stores it in the org config. + """Trades the AuthCode from a ScratchOrgInfo for an + org access token,and stores it in the org config. - The AuthCode is short-lived so this is only useful immediately after - the scratch org is created. This must be completed once in order for future + The AuthCode is short-lived so this is only useful + immediately after the scratch org is created. + This must be completed once in order for future access tokens to be obtained using the JWT token flow. """ - oauth_config = OAuth2ClientConfig( - client_id=SF_CLIENT_ID, - client_secret=SF_CLIENT_SECRET, - redirect_uri=SF_CALLBACK_URL, - auth_uri=f"{scratch_org_config.instance_url}/services/oauth2/authorize", - token_uri=f"{scratch_org_config.instance_url}/services/oauth2/token", - scope="web full refresh_token", + total_wait_time = 0 + while total_wait_time < settings.MAXIMUM_JOB_LENGTH: + oauth_config = OAuth2ClientConfig( + client_id=SF_CLIENT_ID, + client_secret=SF_CLIENT_SECRET, + redirect_uri=SF_CALLBACK_URL, + auth_uri=f"{scratch_org_config.instance_url}/services/oauth2/authorize", + token_uri=f"{scratch_org_config.instance_url}/services/oauth2/token", + scope="web full refresh_token", + ) + oauth = OAuth2Client(oauth_config) + try: + auth_result = oauth.auth_code_grant(org_result["AuthCode"]).json() + scratch_org_config.config[ + "access_token" + ] = scratch_org_config._scratch_info["access_token"] = auth_result[ + "access_token" + ] + return + except requests.exceptions.ConnectionError as exception: + if is_network_error(exception): + actual_exception = exception.__cause__ or exception.__context__ + logger.info(actual_exception) + total_wait_time += 10 + time.sleep(10) + else: + raise + + raise ScratchOrgError( + f"Failed to build your scratch org after {settings.MAXIMUM_JOB_LENGTH} seconds." ) - oauth = OAuth2Client(oauth_config) - auth_result = oauth.auth_code_grant(org_result["AuthCode"]).json() - scratch_org_config.config["access_token"] = scratch_org_config._scratch_info[ - "access_token" - ] = auth_result["access_token"] def deploy_org_settings( @@ -352,7 +386,9 @@ def create_org( ) org_result = poll_for_scratch_org_completion(devhub_api, org_result) mutate_scratch_org( - scratch_org_config=scratch_org_config, org_result=org_result, email=email + scratch_org_config=scratch_org_config, + org_result=org_result, + email=email, ) get_access_token(org_result=org_result, scratch_org_config=scratch_org_config) org_config = deploy_org_settings( @@ -418,7 +454,9 @@ def run_flow(*, cci, org_config, flow_name, project_path, user): orig_stdout, _ = p.communicate() if p.returncode: p = subprocess.run( - [command, "error", "info"], capture_output=True, env={"HOME": project_path} + [command, "error", "info"], + capture_output=True, + env={"HOME": project_path}, ) traceback = p.stdout.decode("utf-8") logger.warning(traceback) diff --git a/metecho/api/tests/sf_run_flow.py b/metecho/api/tests/sf_run_flow.py index 23f4ef245..7c019a9c8 100644 --- a/metecho/api/tests/sf_run_flow.py +++ b/metecho/api/tests/sf_run_flow.py @@ -3,13 +3,15 @@ to change substantially, and as it stands, it's full of implicit external calls, so this would be mock-heavy anyway. """ - from contextlib import ExitStack from unittest.mock import MagicMock, patch - +from requests.exceptions import InvalidSchema import pytest from requests.exceptions import HTTPError +from cumulusci.core.config.scratch_org_config import ScratchOrgConfig +from cumulusci.oauth.client import OAuth2Client +from requests.exceptions import ConnectionError from metecho.exceptions import SubcommandException from ..sf_run_flow import ( @@ -196,6 +198,114 @@ def test_get_access_token(mocker): assert OAuth2Client.called +@pytest.mark.django_db +@patch("metecho.api.sf_run_flow.time.sleep") +@patch("metecho.api.sf_run_flow.settings.MAXIMUM_JOB_LENGTH", 9) +def test_get_access_token_dns_delay_garbage_url(sleep, mocker): + scratch_org_config = ScratchOrgConfig( + name="dev", + config={ + "access_token": 123, + "instance_url": "garbage://tesdfgfdsfg54w36st.co345654356tm", + }, + ) + real_auth_code_grant = OAuth2Client.auth_code_grant + call_count = 0 + mocker.patch("metecho.api.sf_run_flow.is_network_error", False) + + def fake_auth_code_grant(self, config): + nonlocal call_count + call_count += 1 + return real_auth_code_grant(self, config) + + mocker.patch.object( + OAuth2Client, + "auth_code_grant", + fake_auth_code_grant, + ) + mocker.auth_code_grant = "123" + auth_token_endpoint = f"'{scratch_org_config.instance_url}/services/oauth2/token'" + expected_result = f"No connection adapters were found for {auth_token_endpoint}" + with pytest.raises( + InvalidSchema, + match=expected_result, + ): + get_access_token( + org_result={"AuthCode": "123"}, + scratch_org_config=scratch_org_config, + ) + + assert call_count == 1 + + +@pytest.mark.django_db +@patch("metecho.api.sf_run_flow.time.sleep") +@patch("metecho.api.sf_run_flow.settings.MAXIMUM_JOB_LENGTH", 9) +def test_get_access_token_dns_delay_raises_error(sleep, mocker): + scratch_org_config = ScratchOrgConfig( + name="dev", + config={ + "access_token": 123, + "instance_url": "https://test.com", + }, + ) + call_count = 0 + + def fake_auth_code_grant(self, config): + nonlocal call_count + call_count += 1 + raise ConnectionError("FooBar") + + mocker.patch.object( + OAuth2Client, + "auth_code_grant", + fake_auth_code_grant, + ) + mocker.auth_code_grant = "123" + with pytest.raises(ConnectionError, match="FooBar"): + get_access_token( + org_result={"AuthCode": "123"}, + scratch_org_config=scratch_org_config, + ) + + assert call_count == 1 + + +@pytest.mark.django_db +@patch("metecho.api.sf_run_flow.time.sleep") +@patch("metecho.api.sf_run_flow.settings.MAXIMUM_JOB_LENGTH", 11) +def test_get_access_token_dns_delay(sleep, mocker): + """Prove test is looping for DNS delays""" + scratch_org_config = ScratchOrgConfig( + name="dev", + config={ + "access_token": 123, + "instance_url": "https://tesdfgfdsfg54w36st.co345654356tm", + }, + ) + real_auth_code_grant = OAuth2Client.auth_code_grant + call_count = 0 + + def fake_auth_code_grant(self, config): + nonlocal call_count + call_count += 1 + return real_auth_code_grant(self, config) + + mocker.patch.object( + OAuth2Client, + "auth_code_grant", + fake_auth_code_grant, + ) + mocker.auth_code_grant = "123" + with pytest.raises(ScratchOrgError): + + get_access_token( + org_result={"AuthCode": "123"}, + scratch_org_config=scratch_org_config, + ) + assert call_count == 2 + + class TestDeployOrgSettings: def test_org_preference_settings(self): with ExitStack() as stack: