Skip to content

Commit 09fc0ab

Browse files
farskifarski
committed
Add FTP NLB logs
1 parent 23fae20 commit 09fc0ab

File tree

3 files changed

+152
-0
lines changed

3 files changed

+152
-0
lines changed

‎spire/templates/apps-200A.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@ Parameters:
7777
ClickhouseLegacyClientSecurityGroupId: { Type: AWS::EC2::SecurityGroup::Id }
7878
X8664AsgCapacityProviderName: { Type: String }
7979
Aarch64AsgCapacityProviderName: { Type: String }
80+
SharedGlueDatabaseName: { Type: String }
8081

8182
CastleSharedAlbListenerRulePriorityPrefix: { Type: String }
8283

@@ -221,6 +222,7 @@ Resources:
221222
TheCountHostname: !Ref TheCountHostname
222223
TheCastleHostname: !Ref TheCastleHostname
223224
Aarch64AsgCapacityProviderName: !Ref Aarch64AsgCapacityProviderName
225+
SharedGlueDatabaseName: !Ref SharedGlueDatabaseName
224226
Tags:
225227
- { Key: prx:meta:tagging-version, Value: "2021-04-07" }
226228
- { Key: prx:cloudformation:stack-name, Value: !Ref AWS::StackName }

‎spire/templates/apps/exchange.yml‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ Parameters:
7373
TheCountHostname: { Type: String }
7474
TheCastleHostname: { Type: String }
7575
Aarch64AsgCapacityProviderName: { Type: String }
76+
SharedGlueDatabaseName: { Type: String }
7677

7778
Conditions:
7879
IsProduction: !Equals [!Ref EnvironmentType, Production]
@@ -1729,10 +1730,158 @@ Resources:
17291730
# AWS Transfer Family does not allow public servers with insecure FTP
17301731
# enabled. Putting the server behind an NLB allows forwarding of public FTP
17311732
# requests to a private server.
1733+
NlbAccessLogsBucket:
1734+
Type: AWS::S3::Bucket
1735+
DeletionPolicy: Retain
1736+
UpdateReplacePolicy: Retain
1737+
Properties:
1738+
PublicAccessBlockConfiguration:
1739+
BlockPublicAcls: true
1740+
BlockPublicPolicy: true
1741+
IgnorePublicAcls: true
1742+
RestrictPublicBuckets: true
1743+
LifecycleConfiguration:
1744+
Rules:
1745+
- ExpirationInDays: !If [IsProduction, 14, 3]
1746+
Status: Enabled
1747+
Tags:
1748+
- { Key: prx:meta:tagging-version, Value: "2021-04-07" }
1749+
- { Key: prx:cloudformation:stack-name, Value: !Ref AWS::StackName }
1750+
- { Key: prx:cloudformation:stack-id, Value: !Ref AWS::StackId }
1751+
- { Key: prx:cloudformation:root-stack-name, Value: !Ref RootStackName }
1752+
- { Key: prx:cloudformation:root-stack-id, Value: !Ref RootStackId }
1753+
- { Key: prx:ops:environment, Value: !Ref EnvironmentType }
1754+
- { Key: prx:dev:application, Value: Common }
1755+
NlbAccessLogsBucketPolicy:
1756+
# https://docs.aws.amazon.com/elasticloadbalancing/latest/network/enable-access-logs.html#access-logging-bucket-requirements
1757+
Type: AWS::S3::BucketPolicy
1758+
Properties:
1759+
Bucket: !Ref NlbAccessLogsBucket
1760+
PolicyDocument:
1761+
Statement:
1762+
- Action: s3:GetBucketAcl
1763+
Effect: Allow
1764+
Principal:
1765+
Service: delivery.logs.amazonaws.com
1766+
Resource: !GetAtt NlbAccessLogsBucket.Arn
1767+
- Action: s3:PutObject
1768+
Condition:
1769+
StringEquals:
1770+
s3:x-amz-acl: bucket-owner-full-control
1771+
Effect: Allow
1772+
Principal:
1773+
Service: delivery.logs.amazonaws.com
1774+
Resource: !Sub ${NlbAccessLogsBucket.Arn}/AWSLogs/${AWS::AccountId}/*
1775+
- Action: s3:PutObject
1776+
Effect: Allow
1777+
Principal:
1778+
Service: logdelivery.elasticloadbalancing.amazonaws.com
1779+
Resource: !Sub ${NlbAccessLogsBucket.Arn}/AWSLogs/${AWS::AccountId}/*
1780+
Version: "2012-10-17"
1781+
NlbAccessLogsGlueTable:
1782+
Type: AWS::Glue::Table
1783+
Properties:
1784+
CatalogId: !Ref AWS::AccountId
1785+
DatabaseName: !Ref SharedGlueDatabaseName
1786+
TableInput:
1787+
Description: !Sub >-
1788+
${EnvironmentType} Exchange FTP server NLB logs
1789+
Name: exchange-ftp-nlb
1790+
Parameters:
1791+
projection.enabled: "true"
1792+
projection.date.type: date
1793+
projection.date.range: 2021/01/01,NOW
1794+
projection.date.format: yyyy/MM/dd
1795+
projection.date.interval: "1"
1796+
projection.date.interval.unit: DAYS
1797+
storage.location.template: !Sub s3://${NlbAccessLogsBucket}/AWSLogs/${AWS::AccountId}/elasticloadbalancing/${AWS::Region}/${!date}
1798+
PartitionKeys:
1799+
- Name: date
1800+
Type: string
1801+
StorageDescriptor:
1802+
Columns:
1803+
- Name: type
1804+
Type: string
1805+
- Name: version
1806+
Type: string
1807+
- Name: time
1808+
Type: string
1809+
- Name: elb
1810+
Type: string
1811+
- Name: listener
1812+
Type: string
1813+
- Name: client_ip
1814+
Type: string
1815+
- Name: client_port
1816+
Type: int
1817+
- Name: target_ip
1818+
Type: string
1819+
- Name: target_port
1820+
Type: int
1821+
- Name: tcp_connection_time_ms
1822+
Type: double
1823+
- Name: tls_handshake_time_ms
1824+
Type: double
1825+
- Name: received_bytes
1826+
Type: bigint
1827+
- Name: sent_bytes
1828+
Type: bigint
1829+
- Name: incoming_tls_alert
1830+
Type: int
1831+
- Name: chosen_cert_arn
1832+
Type: string
1833+
- Name: chosen_cert_serial
1834+
Type: string
1835+
- Name: tls_cipher_suite
1836+
Type: string
1837+
- Name: tls_protocol_version
1838+
Type: string
1839+
- Name: tls_named_group
1840+
Type: string
1841+
- Name: domain_name
1842+
Type: string
1843+
- Name: alpn_fe_protocol
1844+
Type: string
1845+
- Name: alpn_be_protocol
1846+
Type: string
1847+
- Name: alpn_client_preference_list
1848+
Type: string
1849+
- Name: tls_connection_creation_time
1850+
Type: string
1851+
InputFormat: org.apache.hadoop.mapred.TextInputFormat
1852+
Location: !Sub s3://${NlbAccessLogsBucket}/AWSLogs/${AWS::AccountId}/elasticloadbalancing/${AWS::Region}/
1853+
OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
1854+
SerdeInfo:
1855+
Parameters:
1856+
input.regex: >-
1857+
([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*):([0-9]*) ([^ ]*):([0-9]*) ([-.0-9]*) ([-.0-9]*) ([-0-9]*) ([-0-9]*) ([-0-9]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ?([^ ]*)?( .*)?
1858+
serialization.format: "1"
1859+
SerializationLibrary: org.apache.hadoop.hive.serde2.RegexSerDe
1860+
TableType: EXTERNAL_TABLE
1861+
AccessLogsRecentQuery:
1862+
Type: AWS::Athena::NamedQuery
1863+
Properties:
1864+
Database: !Ref SharedGlueDatabaseName
1865+
Description: >-
1866+
View recent FTP NLB requests using the table's date partitions
1867+
Name: !Sub ${EnvironmentType} FTP NLB Recent Requests
1868+
QueryString: !Sub >-
1869+
SELECT *
1870+
FROM "${SharedGlueDatabaseName}"."${NlbAccessLogsGlueTable}"
1871+
WHERE "date" > date_format(current_date - interval '1' day, '%Y/%m/%d')
1872+
ORDER BY time ASC
17321873
FtpServerNlb:
17331874
Type: AWS::ElasticLoadBalancingV2::LoadBalancer
17341875
Properties:
17351876
IpAddressType: ipv4
1877+
LoadBalancerAttributes:
1878+
# https://docs.aws.amazon.com/elasticloadbalancing/latest/network/load-balancer-access-logs.html
1879+
- Key: idle_timeout.timeout_seconds
1880+
Value: "150"
1881+
- Key: access_logs.s3.enabled
1882+
Value: "true"
1883+
- Key: access_logs.s3.bucket
1884+
Value: !Ref NlbAccessLogsBucket
17361885
Scheme: internet-facing
17371886
SubnetMappings:
17381887
- AllocationId: !GetAtt FtpServerNblElasticIp.AllocationId

‎spire/templates/root.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -810,6 +810,7 @@ Resources:
810810
ClickhouseLegacyClientSecurityGroupId: !GetAtt SharedClickhouseSecurityGroupStack.Outputs.LegacyClientSecurityGroupId
811811
X8664AsgCapacityProviderName: !GetAtt SharedEcsAsgStack.Outputs.CapacityProviderName
812812
Aarch64AsgCapacityProviderName: !GetAtt SharedEcsAsgAarch64Stack.Outputs.CapacityProviderName
813+
SharedGlueDatabaseName: !GetAtt SharedGlueDatabaseStack.Outputs.SharedGlueDatabaseName
813814

814815
# App-specific parameters
815816

0 commit comments

Comments
 (0)