feat(aws): AWSX-1891 Migrate route53 to logs-backend (#978)

Signed-off-by: Vincent Boutour <vincent.boutour@datadoghq.com>

Author: ViBiOh

aws/logs_monitoring/steps/enrichment.py

@@ -1,14 +1,10 @@
-import logging
 import json
+import logging
 import os
 import re
-from settings import (
-    DD_SOURCE,
-    DD_SERVICE,
-    DD_HOST,
-    DD_CUSTOM_TAGS,
-)
+
 from enhanced_lambda_metrics import parse_lambda_tags_from_arn
+from settings import DD_CUSTOM_TAGS, DD_HOST, DD_SERVICE, DD_SOURCE
 from steps.enums import AwsEventSource
 
 HOST_IDENTITY_REGEXP = re.compile(
@@ -30,7 +26,6 @@ def enrich(events, cache_layer):
         extract_ddtags_from_message(event)
         extract_host_from_cloudtrails(event)
         extract_host_from_guardduty(event)
-        extract_host_from_route53(event)
 
     return events
 
@@ -220,19 +215,3 @@ def extract_host_from_guardduty(event):
             host = host.get("instanceDetails", {}).get("instanceId")
             if host is not None:
                 event[DD_HOST] = host
-
-
-def extract_host_from_route53(event):
-    if event is not None and event.get(DD_SOURCE) == str(AwsEventSource.ROUTE53):
-        message = event.get("message", {})
-        if isinstance(message, str):
-            try:
-                message = json.loads(message)
-            except json.JSONDecodeError:
-                logger.debug("Failed to decode Route53 message")
-                return
-
-        if isinstance(message, dict):
-            host = message.get("srcids", {}).get("instance")
-            if host is not None:
-                event[DD_HOST] = host

aws/logs_monitoring/steps/enums.py

@@ -14,7 +14,6 @@ class AwsEventSource(Enum):
     MSK = "msk"
     MYSQL = "mysql"
     POSTGRESQL = "postgresql"
-    ROUTE53 = "route53"
     S3 = "s3"
     SNS = "sns"
     STEPFUNCTION = "stepfunction"
@@ -30,7 +29,6 @@ def cloudwatch_sources():
             AwsEventSource.ELASTICSEARCH,
             AwsEventSource.FARGATE,
             AwsEventSource.MSK,
-            AwsEventSource.ROUTE53,
         ]
 
 
@@ -46,8 +44,6 @@ def __init__(self, string, event_source):
     GUARDDUTY = ("guardduty", AwsEventSource.GUARDDUTY)
     KINESIS = ("amazon_kinesis", AwsEventSource.KINESIS)
     MSK = ("amazon_msk", AwsEventSource.MSK)
-    # e.g. AWSLogs/123456779121/vpcdnsquerylogs/vpc-********/2021/05/11/vpc-********_vpcdnsquerylogs_********_20210511T0910Z_71584702.log.gz
-    ROUTE53 = ("vpcdnsquerylogs", AwsEventSource.ROUTE53)
 
     def __str__(self):
         return f"{self.string}"

aws/logs_monitoring/tests/test_enrichment.py

@@ -1,16 +1,15 @@
 import json
-
 import unittest
 from unittest.mock import MagicMock
 
-from caching.cache_layer import CacheLayer
 from approvaltests.approvals import verify_as_json
+
+from caching.cache_layer import CacheLayer
 from steps.enrichment import (
     add_metadata_to_lambda_log,
+    extract_ddtags_from_message,
     extract_host_from_cloudtrails,
     extract_host_from_guardduty,
-    extract_host_from_route53,
-    extract_ddtags_from_message,
 )
 
 
@@ -169,14 +168,6 @@ def test_parse_source_guardduty(self):
         extract_host_from_guardduty(event)
         self.assertEqual(event["host"], "i-99999999")
 
-    def test_parse_source_route53(self):
-        event = {
-            "ddsource": "route53",
-            "message": {"srcids": {"instance": "i-99999999"}},
-        }
-        extract_host_from_route53(event)
-        self.assertEqual(event["host"], "i-99999999")
-
 
 class TestLambdaMetadataEnrichment(unittest.TestCase):
     def test_empty_event(self):

aws/logs_monitoring/tests/test_parsing.py

@@ -134,24 +134,6 @@ def test_redshift_gov_event(self):
             str(AwsEventSource.S3),
         )
 
-    def test_route53_event(self):
-        self.assertEqual(
-            parse_event_source(
-                {"awslogs": "logs"},
-                "my-route53-loggroup123",
-            ),
-            str(AwsEventSource.ROUTE53),
-        )
-
-    def test_vpcdnsquerylogs_event(self):
-        self.assertEqual(
-            parse_event_source(
-                {"Records": ["logs-from-s3"]},
-                "AWSLogs/123456779121/vpcdnsquerylogs/vpc-********/2021/05/11/vpc-********_vpcdnsquerylogs_********_20210511T0910Z_71584702.log.gz",
-            ),
-            str(AwsEventSource.ROUTE53),
-        )
-
     def test_fargate_event(self):
         self.assertEqual(
             parse_event_source(

aws/logs_monitoring/tools/integration_tests/snapshots/cloudwatch_log_route53.json

@@ -159,11 +159,6 @@ def test_cloudwatch_log_lambda_invocation(self):
         snapshot_filename = f"{input_filename}~snapshot"
         self.compare_snapshot(input_filename, snapshot_filename)
 
-    def test_cloudwatch_log_route53(self):
-        input_filename = f"{snapshot_dir}/cloudwatch_log_route53.json"
-        snapshot_filename = f"{input_filename}~snapshot"
-        self.compare_snapshot(input_filename, snapshot_filename)
-
     def test_cloudwatch_log_timeout(self):
         input_filename = f"{snapshot_dir}/cloudwatch_log_timeout.json"
         snapshot_filename = f"{input_filename}~snapshot"