@W-11993356 Metecho: handle scratch org DNS propagation delays

metecho/api/sf_run_flow.py

@@ -2,11 +2,12 @@
 import json
 import logging
 import os
+from socket import gaierror
+import requests
 import shutil
 import subprocess
 import time
 from datetime import datetime
-
 from cumulusci.core.config import OrgConfig, TaskConfig
 from cumulusci.core.runtime import BaseCumulusCI
 from cumulusci.oauth.client import OAuth2Client, OAuth2ClientConfig
@@ -265,27 +266,60 @@ def mutate_scratch_org(*, scratch_org_config, org_result, email):
     )
 
 
+def is_network_error(exception) -> bool:
+    """Helper function to determine if a network error,
+    such as a dns propagation delay is occuring."""
+
+    # gai error stands for GetAddressInfo Error
+    if isinstance(exception, gaierror):
+        return True
+    subexception = exception.__context__ or exception.__cause__
+    if subexception:
+        return is_network_error(subexception)
+    else:
+        return False
+
+
 def get_access_token(*, org_result, scratch_org_config):
-    """Trades the AuthCode from a ScratchOrgInfo for an org access token,
-    and stores it in the org config.
+    """Trades the AuthCode from a ScratchOrgInfo for an
+    org access token,and stores it in the org config.
 
-    The AuthCode is short-lived so this is only useful immediately after
-    the scratch org is created. This must be completed once in order for future
+    The AuthCode is short-lived so this is only useful
+    immediately after the scratch org is created.
+    This must be completed once in order for future
     access tokens to be obtained using the JWT token flow.
     """
-    oauth_config = OAuth2ClientConfig(
-        client_id=SF_CLIENT_ID,
-        client_secret=SF_CLIENT_SECRET,
-        redirect_uri=SF_CALLBACK_URL,
-        auth_uri=f"{scratch_org_config.instance_url}/services/oauth2/authorize",
-        token_uri=f"{scratch_org_config.instance_url}/services/oauth2/token",
-        scope="web full refresh_token",
+    total_wait_time = 0
+    while total_wait_time < settings.MAXIMUM_JOB_LENGTH:
+        oauth_config = OAuth2ClientConfig(
+            client_id=SF_CLIENT_ID,
+            client_secret=SF_CLIENT_SECRET,
+            redirect_uri=SF_CALLBACK_URL,
+            auth_uri=f"{scratch_org_config.instance_url}/services/oauth2/authorize",
+            token_uri=f"{scratch_org_config.instance_url}/services/oauth2/token",
+            scope="web full refresh_token",
+        )
+        oauth = OAuth2Client(oauth_config)
+        try:
+            auth_result = oauth.auth_code_grant(org_result["AuthCode"]).json()
+            scratch_org_config.config[
+                "access_token"
+            ] = scratch_org_config._scratch_info["access_token"] = auth_result[
+                "access_token"
+            ]
+            return
+        except requests.exceptions.ConnectionError as exception:
+            if is_network_error(exception):
+                actual_exception = exception.__cause__ or exception.__context__
+                logger.info(actual_exception)
+                total_wait_time += 10
+                time.sleep(10)
+            else:
+                raise
+
+    raise ScratchOrgError(
+        f"Failed to build your scratch org after {settings.MAXIMUM_JOB_LENGTH} seconds."
     )
-    oauth = OAuth2Client(oauth_config)
-    auth_result = oauth.auth_code_grant(org_result["AuthCode"]).json()
-    scratch_org_config.config["access_token"] = scratch_org_config._scratch_info[
-        "access_token"
-    ] = auth_result["access_token"]
 
 
 def deploy_org_settings(
@@ -352,7 +386,9 @@ def create_org(
     )
     org_result = poll_for_scratch_org_completion(devhub_api, org_result)
     mutate_scratch_org(
-        scratch_org_config=scratch_org_config, org_result=org_result, email=email
+        scratch_org_config=scratch_org_config,
+        org_result=org_result,
+        email=email,
     )
     get_access_token(org_result=org_result, scratch_org_config=scratch_org_config)
     org_config = deploy_org_settings(
@@ -418,7 +454,9 @@ def run_flow(*, cci, org_config, flow_name, project_path, user):
     orig_stdout, _ = p.communicate()
     if p.returncode:
         p = subprocess.run(
-            [command, "error", "info"], capture_output=True, env={"HOME": project_path}
+            [command, "error", "info"],
+            capture_output=True,
+            env={"HOME": project_path},
         )
         traceback = p.stdout.decode("utf-8")
         logger.warning(traceback)

metecho/api/tests/sf_run_flow.py

@@ -3,13 +3,15 @@
 to change substantially, and as it stands, it's full of implicit
 external calls, so this would be mock-heavy anyway.
 """
-
 from contextlib import ExitStack
 from unittest.mock import MagicMock, patch
-
+from requests.exceptions import InvalidSchema
 import pytest
 from requests.exceptions import HTTPError
+from cumulusci.core.config.scratch_org_config import ScratchOrgConfig
+from cumulusci.oauth.client import OAuth2Client
 
+from requests.exceptions import ConnectionError
 from metecho.exceptions import SubcommandException
 
 from ..sf_run_flow import (
@@ -196,6 +198,114 @@ def test_get_access_token(mocker):
     assert OAuth2Client.called
 
 
+@pytest.mark.django_db
+@patch("metecho.api.sf_run_flow.time.sleep")
+@patch("metecho.api.sf_run_flow.settings.MAXIMUM_JOB_LENGTH", 9)
+def test_get_access_token_dns_delay_garbage_url(sleep, mocker):
+    scratch_org_config = ScratchOrgConfig(
+        name="dev",
+        config={
+            "access_token": 123,
+            "instance_url": "garbage://tesdfgfdsfg54w36st.co345654356tm",
+        },
+    )
+    real_auth_code_grant = OAuth2Client.auth_code_grant
+    call_count = 0
+    mocker.patch("metecho.api.sf_run_flow.is_network_error", False)
+
+    def fake_auth_code_grant(self, config):
+        nonlocal call_count
+        call_count += 1
+        return real_auth_code_grant(self, config)
+
+    mocker.patch.object(
+        OAuth2Client,
+        "auth_code_grant",
+        fake_auth_code_grant,
+    )
+    mocker.auth_code_grant = "123"
+    auth_token_endpoint = f"'{scratch_org_config.instance_url}/services/oauth2/token'"
+    expected_result = f"No connection adapters were found for {auth_token_endpoint}"
+    with pytest.raises(
+        InvalidSchema,
+        match=expected_result,
+    ):
+        get_access_token(
+            org_result={"AuthCode": "123"},
+            scratch_org_config=scratch_org_config,
+        )
+
+    assert call_count == 1
+
+
+@pytest.mark.django_db
+@patch("metecho.api.sf_run_flow.time.sleep")
+@patch("metecho.api.sf_run_flow.settings.MAXIMUM_JOB_LENGTH", 9)
+def test_get_access_token_dns_delay_raises_error(sleep, mocker):
+    scratch_org_config = ScratchOrgConfig(
+        name="dev",
+        config={
+            "access_token": 123,
+            "instance_url": "https://test.com",
+        },
+    )
+    call_count = 0
+
+    def fake_auth_code_grant(self, config):
+        nonlocal call_count
+        call_count += 1
+        raise ConnectionError("FooBar")
+
+    mocker.patch.object(
+        OAuth2Client,
+        "auth_code_grant",
+        fake_auth_code_grant,
+    )
+    mocker.auth_code_grant = "123"
+    with pytest.raises(ConnectionError, match="FooBar"):
+        get_access_token(
+            org_result={"AuthCode": "123"},
+            scratch_org_config=scratch_org_config,
+        )
+
+    assert call_count == 1
+
+
+@pytest.mark.django_db
+@patch("metecho.api.sf_run_flow.time.sleep")
+@patch("metecho.api.sf_run_flow.settings.MAXIMUM_JOB_LENGTH", 11)
+def test_get_access_token_dns_delay(sleep, mocker):
+    """Prove test is looping for DNS delays"""
+    scratch_org_config = ScratchOrgConfig(
+        name="dev",
+        config={
+            "access_token": 123,
+            "instance_url": "https://tesdfgfdsfg54w36st.co345654356tm",
+        },
+    )
+    real_auth_code_grant = OAuth2Client.auth_code_grant
+    call_count = 0
+
+    def fake_auth_code_grant(self, config):
+        nonlocal call_count
+        call_count += 1
+        return real_auth_code_grant(self, config)
+
+    mocker.patch.object(
+        OAuth2Client,
+        "auth_code_grant",
+        fake_auth_code_grant,
+    )
+    mocker.auth_code_grant = "123"
+    with pytest.raises(ScratchOrgError):
+
+        get_access_token(
+            org_result={"AuthCode": "123"},
+            scratch_org_config=scratch_org_config,
+        )
+    assert call_count == 2
+
+
 class TestDeployOrgSettings:
     def test_org_preference_settings(self):
         with ExitStack() as stack: