Release v2.1.3

Author: PeterBengtson

.env.test.example

@@ -0,0 +1,56 @@
+# SOAR Testing Environment Configuration
+# Copy this file to .env.test and modify as needed for your testing environment.
+# .env.test is in .gitignore so your local changes won't be committed.
+
+# AWS Configuration for Testing
+AWS_DEFAULT_REGION=us-east-1
+AWS_ACCESS_KEY_ID=test
+AWS_SECRET_ACCESS_KEY=test
+AWS_SESSION_TOKEN=test
+
+# LocalStack Configuration (for integration tests)
+LOCALSTACK_ENDPOINT=http://localhost:4566
+SERVICES=s3,lambda,dynamodb,iam,sts,rds,ec2,securityhub,stepfunctions
+
+# Python Path Configuration
+PYTHONPATH=${PYTHONPATH}:$(pwd)
+
+# Test Account Configuration
+TEST_ACCOUNT_ID=123456789012
+TEST_ORGANIZATION_ID=o-example123456
+
+# Test Region Configuration
+TEST_REGIONS=us-east-1,us-west-2
+
+# Security Hub Test Configuration
+TEST_SECURITY_HUB_ACCOUNT=123456789012
+TEST_FINDING_PROVIDER_FIELDS=AWS/Inspector,AWS/GuardDuty,AWS/Config
+
+# Database Test Configuration
+TEST_DYNAMODB_TABLE_PREFIX=soar-test-
+
+# Email Test Configuration
+TEST_EMAIL_FROM=test@example.com
+TEST_EMAIL_TO=security-team@example.com
+
+# AI/Bedrock Test Configuration (for real AWS tests)
+# Uncomment and configure for real AWS testing
+# AWS_BEDROCK_REGION=us-east-1
+# AWS_BEDROCK_MODEL_ID=anthropic.claude-3-sonnet-20240229-v1:0
+
+# Ticketing System Test Configuration
+# JIRA_URL=https://your-company.atlassian.net
+# JIRA_USERNAME=test-user
+# JIRA_API_TOKEN=your-test-token
+# SERVICENOW_INSTANCE=your-test-instance
+# SERVICENOW_USERNAME=test-user
+# SERVICENOW_PASSWORD=test-password
+
+# Test Execution Flags
+RUN_INTEGRATION_TESTS=false
+RUN_REAL_AWS_TESTS=false
+SKIP_SLOW_TESTS=true
+
+# Logging Configuration
+LOG_LEVEL=INFO
+TEST_LOG_LEVEL=DEBUG

.gitignore

@@ -198,6 +198,7 @@ celerybeat-schedule.*
 
 # Environments
 .env
+.env.test
 .venv
 env/
 venv/

CHANGELOG.md

@@ -1,9 +1,20 @@
 # Change Log
 
+## v2.1.3
+    * Comprehensive testing infrastructure implementation with 53% auto-remediation coverage (16/30 functions, 236 tests)
+    * Complete EC2 auto-remediation testing (8/8 functions, 134 tests) with ASFF standardization patterns
+    * Complete RDS auto-remediation testing (7/7 controls, 88 tests) with comprehensive edge case coverage
+    * Established documentation-first testing methodology for efficient test development
+    * Added centralized test data management via fixtures/asff_data.py for consistent ASFF structures
+    * Implemented critical pytest module import isolation to prevent cross-contamination between test suites
+    * Enhanced testing documentation with LocalStack Docker integration and contributor guidelines
+    * Added bug handling protocol for test development to ensure proper production code review processes
+    * Comprehensive documentation added to core SOAR functions and auto-remediation components
+    * Testing strategy now serves as template for expanding coverage across all OpenSecOps repositories
+
 ## v2.1.2
     * Set incident AI Query to 600 (was 120) as Claude Sonnet 4 keeps timing out - it's new.
 
-
 ## v2.1.1
     * Added existence check to the auto-remediation for ELB.5.
 

functions/accounts/clear_account_data/app.py

@@ -1,3 +1,20 @@
+"""
+AWS Account Data Cache Management: Clear Account Data
+
+This Lambda function clears all cached account data from the DynamoDB table used 
+for account information caching. This is typically used for maintenance operations
+or when a full refresh of account data is needed across the SOAR system.
+
+Operations:
+1. Scan the entire cached account data table to get all account IDs
+2. Create batch delete requests for all found accounts
+3. Process deletions in batches of 25 (DynamoDB batch limit)
+4. Handle unprocessed items with retry logic
+
+Target Resources: DynamoDB cached account data table
+Purpose: Complete cache invalidation for account data refresh
+"""
+
 import os
 import boto3
 
@@ -9,40 +26,70 @@
 
 
 def lambda_handler(_event, _context):
-    # Scan the table to get all items with only the 'id' attribute projected
+    """
+    Main Lambda handler for clearing all cached account data.
+    
+    Args:
+        _event: Lambda event data (unused)
+        _context: Lambda context (unused)
+        
+    Returns:
+        None (implicit)
+        
+    Process:
+        1. Scan table for all account IDs using pagination
+        2. Create delete requests in batches of 25 items
+        3. Process each batch with retry handling for unprocessed items
+        4. Continue until all cached account data is cleared
+    """
+    # STEP 1: Scan entire table to collect all account IDs
+    # Use ProjectionExpression to minimize data transfer by only retrieving IDs
     response = table.scan(ProjectionExpression="id")
     result = response['Items']
 
-    # If there are more items to scan, continue scanning and adding to the result list
+    # Handle pagination - continue scanning if more items exist
     while 'LastEvaluatedKey' in response:
         response = table.scan(ExclusiveStartKey=response['LastEvaluatedKey'])
         result.extend(response['Items'])
 
-    # Create a list of delete requests for each item in the result list
+    # STEP 2: Create delete requests for all discovered accounts
     remaining_delete_requests = [delete_request(x['id']) for x in result]
 
-    # Process delete requests in batches of 25
+    # STEP 3: Process deletions in batches of 25 (DynamoDB limit)
     while len(remaining_delete_requests) > 0:
         delete_requests = remaining_delete_requests[:25]
         remaining_delete_requests = remaining_delete_requests[25:]
 
-        # Process each batch of delete requests
+        # STEP 4: Execute batch delete with retry handling for unprocessed items
         while len(delete_requests) > 0:
             print(delete_requests)
-            # Use the batch_write_item method to delete the items in the batch
+            # Execute batch delete operation
             response = dynamodb.batch_write_item(
                 RequestItems={
                     CACHED_ACCOUNT_DATA_TABLE_NAME: delete_requests
                 }
             )
             print(response)
-            # Get any unprocessed items and add them back to the delete_requests list
+            
+            # Handle unprocessed items - retry them in the next iteration
             delete_requests = response.get('UnprocessedItems', {}).get(
                 CACHED_ACCOUNT_DATA_TABLE_NAME, [])
 
 
 def delete_request(account_id):
-    # Create a delete request for a given account_id
+    """
+    Create a DynamoDB delete request structure for a given account ID.
+    
+    Args:
+        account_id: AWS account ID to delete from cache
+        
+    Returns:
+        dict: DynamoDB delete request structure for batch operations
+        
+    Format:
+        Returns the standard DynamoDB batch delete request format with
+        the account ID as the primary key.
+    """
     return {
         'DeleteRequest': {
             'Key': {

functions/accounts/get_account_data/app.py

@@ -1,61 +1,121 @@
+"""
+AWS Account Data Management: Get Account Data with Caching
+
+This Lambda function retrieves comprehensive account information including metadata,
+tags, organizational structure, and team assignments. Uses DynamoDB caching to 
+improve performance and reduce API calls to AWS Organizations.
+
+Account Data Includes:
+- Basic account information (ID, name, email, join date)
+- Organizational unit placement and hierarchy
+- Tag-based metadata (team, environment, client, project)
+- Contact information and team email assignments
+- Ticketing system integration details (Jira/ServiceNow)
+- Account age and classification (new vs established)
+
+Caching Strategy:
+- Check DynamoDB cache first for existing account data
+- Fetch fresh data from AWS Organizations if cache miss
+- Store fresh data in cache for future requests
+- Cache improves performance and reduces Organizations API throttling
+
+Target Resources: AWS Organizations accounts and DynamoDB cache
+Purpose: Centralized account metadata retrieval with performance optimization
+"""
+
 import os
 import datetime as dt
 import json
 import boto3
 from botocore.config import Config
 from dateutil import parser
 
-# Get environment variables
+# SOAR Configuration
 PRODUCT_NAME = os.environ['PRODUCT_NAME']
+
+# Team Contact Configuration
 ACCOUNT_TEAM_EMAIL_TAG = os.environ['ACCOUNT_TEAM_EMAIL_TAG']         # soar:team:email
 ACCOUNT_TEAM_EMAIL_TAG_APP = os.environ['ACCOUNT_TEAM_EMAIL_TAG_APP'] # soar:team:email:app
 DEFAULT_TEAM_EMAIL = os.environ['DEFAULT_TEAM_EMAIL']
+
+# Account Classification Tags
 ENVIRONMENT_TAG = os.environ['ENVIRONMENT_TAG']                       # soar:environment
 CLIENT_TAG = os.environ['CLIENT_TAG']                                 # soar:client
 PROJECT_TAG = os.environ['PROJECT_TAG']                               # soar:project
 TEAM_TAG = os.environ['TEAM_TAG']                                     # soar:team
 
-TICKETING_SYSTEM = os.environ['TICKETING_SYSTEM'] # 
+# Ticketing System Integration
+TICKETING_SYSTEM = os.environ['TICKETING_SYSTEM']                    # JIRA or ServiceNow
 
+# Jira Integration Configuration
 JIRA_PROJECT_KEY_TAG = os.environ['JIRA_PROJECT_KEY_TAG']             # soar:jira:project-key
 JIRA_PROJECT_KEY_TAG_APP = os.environ['JIRA_PROJECT_KEY_TAG_APP']     # soar:jira:project-key:app
-JIRA_DEFAULT_PROJECT_KEY = os.environ['JIRA_DEFAULT_PROJECT_KEY']     # XXX
+JIRA_DEFAULT_PROJECT_KEY = os.environ['JIRA_DEFAULT_PROJECT_KEY']     # Default project key
 
+# ServiceNow Integration Configuration
 SERVICE_NOW_PROJECT_QUEUE_TAG = os.environ['SERVICE_NOW_PROJECT_QUEUE_TAG']         # soar:service-now:project-queue
 SERVICE_NOW_PROJECT_QUEUE_TAG_APP = os.environ['SERVICE_NOW_PROJECT_QUEUE_TAG_APP'] # soar:service-now:project-queue:app
-SERVICE_NOW_DEFAULT_PROJECT_QUEUE = os.environ['SERVICE_NOW_DEFAULT_PROJECT_QUEUE'] # XXX
+SERVICE_NOW_DEFAULT_PROJECT_QUEUE = os.environ['SERVICE_NOW_DEFAULT_PROJECT_QUEUE'] # Default queue
 
+# Cache Configuration
 CACHED_ACCOUNT_DATA_TABLE_NAME = os.environ['CACHED_ACCOUNT_DATA_TABLE_NAME']
-
 MIN_AGE_HOURS = int(os.environ['MIN_AGE_HOURS'])
 
-
-# Configure Boto3
+# Configure Boto3 with minimal retries - let Step Functions handle retry logic
 config = Config(
     retries={
         'total_max_attempts': 1  # Let Step Functions handle the retries
     }
 )
 
-# Create Boto3 clients
+# AWS Service Clients
 client = boto3.client('organizations', config=config)
 dynamodb = boto3.client('dynamodb')
 
-# Lambda handler function
+
 def lambda_handler(account_id, _context):
-    # Check if account data is cached
+    """
+    Main Lambda handler for retrieving account data with caching.
+    
+    Args:
+        account_id: AWS account ID to retrieve data for
+        _context: Lambda context (unused)
+        
+    Returns:
+        dict: Comprehensive account data including metadata, contacts, and classification
+        
+    Process:
+        1. Check DynamoDB cache for existing account data
+        2. If cache hit, return cached data immediately
+        3. If cache miss, fetch fresh data from AWS Organizations
+        4. Store fresh data in cache and return to caller
+    """
+    # STEP 1: Check cache first for performance optimization
     cached_account_data = get_cached_account_data(account_id)
     if cached_account_data:
         print(f"Account {account_id}: Using cache")
         return cached_account_data
 
-    # Fetch fresh account data
+    # STEP 2: Cache miss - fetch fresh data and update cache
     account_data = get_fresh_account_data(account_id)
     put_cached_account_data(account_id, account_data)
     return account_data
 
-# Get cached account data from DynamoDB
+
 def get_cached_account_data(account_id):
+    """
+    Retrieve account data from DynamoDB cache.
+    
+    Args:
+        account_id: AWS account ID to look up
+        
+    Returns:
+        dict: Cached account data if found and valid, False otherwise
+        
+    Error Handling:
+        Returns False for any cache misses, corruption, or JSON parsing errors.
+        This ensures graceful fallback to fresh data fetching.
+    """
     response = dynamodb.get_item(
         TableName=CACHED_ACCOUNT_DATA_TABLE_NAME,
         Key={
@@ -76,8 +136,18 @@ def get_cached_account_data(account_id):
 
     return data
 
-# Put account data into DynamoDB cache
 def put_cached_account_data(account_id, account_data):
+    """
+    Store account data in DynamoDB cache for future requests.
+    
+    Args:
+        account_id: AWS account ID as cache key
+        account_data: Complete account data dictionary to cache
+        
+    Cache Format:
+        Stores JSON-serialized account data with account ID as primary key.
+        This enables fast lookup and reduces Organizations API calls.
+    """
     response = dynamodb.put_item(
         TableName=CACHED_ACCOUNT_DATA_TABLE_NAME,
         Item={
@@ -87,8 +157,29 @@ def put_cached_account_data(account_id, account_data):
     )
     print(response)
 
-# Fetch fresh account data
+
 def get_fresh_account_data(account_id):
+    """
+    Fetch comprehensive account data from AWS Organizations and compile metadata.
+    
+    Args:
+        account_id: AWS account ID to retrieve information for
+        
+    Returns:
+        dict: Complete account data including:
+            - Basic account info (ID, name, email, join date)
+            - Organizational structure (OU placement)
+            - Tag-based metadata (team, environment, client, project)
+            - Contact information (team emails)
+            - Ticketing integration (Jira/ServiceNow project mappings)
+            - Account classification (new vs established)
+            - Tallies for reporting and aggregation
+            
+    Data Sources:
+        - AWS Organizations: Account details, tags, OU structure
+        - Environment variables: Default values and tag mappings
+        - Calculated fields: Account age, team assignments, project mappings
+    """
     print(f"Account {account_id}: Fetching fresh data")
 
     # Get account details