Merge branch 'master' into simplify-update-item-call

Author: switch180

.github/scripts/build-assets.py

@@ -24,13 +24,13 @@
 
 #Move static assets
 Path(dest_root, 'assets').mkdir(parents=True, exist_ok=False)
-data_files = ['design-patterns/cloudformation/lab.yaml',
-    'design-patterns/cloudformation/C9.yaml',
-    'design-patterns/cloudformation/UserData.sh',
+data_files = ['design-patterns/cloudformation/C9.yaml',
     'design-patterns/cloudformation/UserDataC9.sh',
     'event-driven/event-driven-cfn.yaml',
     'static/files/hands-on-labs/migration-env-setup.yaml',
-    'static/files/hands-on-labs/migration-dms-setup.yaml']
+    'static/files/hands-on-labs/migration-dms-setup.yaml',
+    'static/files/dynamodb-opensearch-zetl/dynamodb-opensearch-setup.yaml'
+]
 for inp_file in data_files:
     src_file = os.path.join(pkg_root, inp_file)
     head, tail = ntpath.split(src_file)
@@ -60,6 +60,31 @@
 		workshop_zip.write(scenario2)
 shutil.move(os.path.join(os.getcwd(), 'scenario-solutions.zip'), os.path.join(dest_root, 'assets', 'scenario-solutions.zip'))
 
+#Create LHOL zETL ZIP
+os.chdir(os.path.join(pkg_root, 'static', 'files', 'dynamodb-opensearch-zetl'))
+with ZipFile('OpenSearchPipeline.zip', 'w') as workshop_zip:
+	for pyscript in glob.glob('./OpenSearchPipeline/*'):
+		workshop_zip.write(pyscript)
+shutil.move(os.path.join(os.getcwd(), 'OpenSearchPipeline.zip'), os.path.join(dest_root, 'assets', 'OpenSearchPipeline.zip'))
+
+
+#Create Game-Player-Data Python Scripts ZIP
+os.chdir(os.path.join(pkg_root, 'game-player-data'))
+with ZipFile('battle-royale.zip', 'w') as workshop_zip:
+	for pyscript in glob.glob('./scripts/*.py'):
+		workshop_zip.write(pyscript)
+	for js_script in glob.glob('./scripts/*.json'):
+		workshop_zip.write(js_script)
+shutil.move(os.path.join(os.getcwd(), 'battle-royale.zip'), os.path.join(dest_root, 'assets', 'battle-royale.zip'))
+
+#Create Global Serverless ZIP
+os.chdir(os.path.join(pkg_root, 'global-serverless'))
+with ZipFile('global-serverless.zip', 'w') as workshop_zip:
+	for data_file in glob.glob('global-serverless/*'):
+		workshop_zip.write(data_file)
+	workshop_zip.write('global-serverless/.chalice/config.json')
+shutil.move(os.path.join(os.getcwd(), 'global-serverless.zip'), os.path.join(dest_root, 'assets', 'global-serverless.zip'))
+
 
 #Create Event Driven ZIPs
 zips_to_make = ['MapLambdaPackage', 'ReduceLambdaPackage', 'StateLambdaPackage', 'GeneratorLambdaPackage']

.github/workflows/main.yml

@@ -27,14 +27,18 @@ jobs:
       with:
         aws-region: us-east-1
         role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
+    - name: S3SyncStaticWeb
+      run: aws s3 sync ./global-serverless/web/ s3://amazon-dynamodb-labs-static/static/global-serverless-application/web/
+    - name: StaticYearReplacer
+      run: sed -i "s/<<PRESENTYEAR>>/$(date +%Y)/" ./static/files/visualizer/index.html
+    - name: S3SyncVisualizer
+      run: aws s3 sync ./static/files/visualizer/ s3://amazon-dynamodb-labs-static/static/visualizer/
     - name: Pull preview build
       run: aws s3 sync s3://amazon-dynamodb-labs-static/build/ . && chmod +x preview_build
     - name: Build Assets
       run: python3 ./.github/scripts/build-assets.py
     - name: S3Sync
       run: aws s3 sync public/assets/ s3://$STEP_S3_BUCKET/assets/ --delete
-    - name: SetS3Acl
-      run: aws s3api put-object-acl --grant-read uri=http://acs.amazonaws.com/groups/global/AllUsers --bucket $STEP_S3_BUCKET --key assets/lab.yaml
     - name: SetS3Acl
       run: aws s3api put-object-acl --grant-read uri=http://acs.amazonaws.com/groups/global/AllUsers --bucket $STEP_S3_BUCKET --key assets/C9.yaml
     - name: SetS3AclED1
@@ -51,3 +55,11 @@ jobs:
       run: aws s3api put-object-acl --grant-read uri=http://acs.amazonaws.com/groups/global/AllUsers --bucket $STEP_S3_BUCKET --key assets/migration-env-setup.yaml
     - name: SetS3AclLHOLDMS
       run: aws s3api put-object-acl --grant-read uri=http://acs.amazonaws.com/groups/global/AllUsers --bucket $STEP_S3_BUCKET --key assets/migration-dms-setup.yaml
+    - name: SetS3AclLHOLzETL
+      run: aws s3api put-object-acl --grant-read uri=http://acs.amazonaws.com/groups/global/AllUsers --bucket $STEP_S3_BUCKET --key assets/OpenSearchPipeline.zip
+    - name: SetS3AclLHOLzETLYaml
+      run: aws s3api put-object-acl --grant-read uri=http://acs.amazonaws.com/groups/global/AllUsers --bucket $STEP_S3_BUCKET --key assets/dynamodb-opensearch-setup.yaml
+    - name: SetS3AclLGME1
+      run: aws s3api put-object-acl --grant-read uri=http://acs.amazonaws.com/groups/global/AllUsers --bucket $STEP_S3_BUCKET --key assets/battle-royale.zip
+    - name: SetS3AclLMR
+      run: aws s3api put-object-acl --grant-read uri=http://acs.amazonaws.com/groups/global/AllUsers --bucket $STEP_S3_BUCKET --key assets/global-serverless.zip

.gitignore

@@ -5,3 +5,4 @@ __pycache__/
 .python-version
 build
 .hugo_build.lock
+preview_build

README.md

@@ -1,4 +1,4 @@
-# Amazon DynamoDB Labs
+# Amazon DynamoDB Labs / Amazon DynamoDB Immersion Day
 The repo for https://catalog.workshops.aws/dynamodb-labs/en-US , formerly https://amazon-dynamodb-labs.com
 
 ### Dev:
@@ -23,5 +23,10 @@ Make a pull request with changes. PRs will be automatically checked to make sure
 
 On merge to master, a GitHub action will deploy the assets to amazon-dynamodb-labs.com and verify the build to ensure the markdown and other files are correctly formatted. From there, a maintainer must manually pull the changes and push to https://catalog.workshops.aws/dynamodb-labs/en-US
 
+#### Internal maintainer: sync changes to internal copy of this repo
+This public repo is pushed to the internal repo in workshop studio using a combination of rsync (we assume you are on macOS) and git. The file `sync.sh` copies the source files to the WS repo folder, and after that you follow the internal README.md to complete the sync.
+1. Run sync.sh to sync the public repo to the amazon-dynamodb-immersion-day repo. e.g. `./sync.sh  -d /Users/$USER/workspace/amazon-dynamodb-immersion-day` . Choose y to sync the files.
+2. Change into the directory for amazon-dynamodb-immersion-day (the workshop studio version), open README.md, and follow the instructions there to git add and push the changes internally. Note that some assets, specifically the LEDA central account resources, are only authored on the internal repo, and they have a separate set of commands to push updates because they are assets that must be in a special S3 bucket owned by WS.
+
 ## License
 This project is licensed under the Apache-2.0 License.

content/authors.en.md

@@ -1,5 +1,5 @@
 ---
-title: "Contributors to Amazon DynamoDB Labs"
+title: "Contributors to the Immersion Day"
 hidden: false
 chapter: true
 description: "Our editors and hall of fame."
@@ -13,8 +13,17 @@ weight: 100
 1. Sean Shriver ([switch180](https://github.com/switch180)) - Ported the whole lab to amazon-dynamodb-labs.com with a custom Hugo theme. Made the "bullet-proof" CloudFormation template for the lab. Updated the hands on lab to Python3
 1. Daniel Yoder ([danielsyoder](https://github.com/danielsyoder)) - The brains behind amazon-dynamodb-labs.com and the co-creator of the design scenarios
 
-### 2023 additions
+### 2024 additions
+The Generative AI workshop LBED was released in early 2024:
+1. John Terhune - ([@terhunej](https://github.com/terhunej)) - Primary author
+1. Zhang Xin - ([@SEZ9](https://github.com/SEZ9)) - Content contributor and original author of a lab that John used as the basis of LBED
+1. Sean Shriver - ([@switch180](https://github.com/switch180)) - Editor, tech reviewer, and merger
+
+The LSQL relational migration lab was released in late 2024:
+1. Robert McCauley - ([robm26](https://github.com/robm26)) - Primary author
+1. Sean Shriver - ([@switch180](https://github.com/switch180)) - Editor, tech reviewer, and merger
 
+### 2023 additions
 The serverless event driven architecture lab was added in 2023:
 
 1. Lucas Rettenmeier ([@rettenls](https://github.com/rettenls)) - Workshop creator for re\:Invent 2021

content/change-data-capture/clean-up/index.en.md

@@ -0,0 +1,62 @@
+---
+title: "5. Summary and Clean Up"
+date: 2023-12-01T00:00:00-00:00
+weight: 20
+chapter: true
+---
+
+Congratulations! You have made it to the end of the workshop.
+
+In this workshop you explored capturing item level changes on a DynamoDB table using DynamoDB Streams and Kinesis Data Streams. In this instance, you wrote the previous version of updated items to a different DynamoDB table. By applying these same techniques, you can build complex event driven solutions that are triggered by changes to items you have stored on DynamoDB.
+
+If you used an account provided by Workshop Studio, you do not need to do any cleanup. The account terminates when the event is over.
+
+If you used your own account, please remove the following resources:
+
+* The Lambda Function Event Source Mappings:
+
+```bash
+UUID_1=`aws lambda list-event-source-mappings --function-name create-order-history-kds --query 'EventSourceMappings[].UUID' --output text`
+UUID_2=`aws lambda list-event-source-mappings --function-name create-order-history-ddbs --query 'EventSourceMappings[].UUID' --output text`
+aws lambda delete-event-source-mapping --uuid ${UUID_1}
+aws lambda delete-event-source-mapping --uuid ${UUID_2}
+```
+
+* The AWS Lambda functions created during the labs:
+
+```bash
+aws lambda delete-function --function-name create-order-history-ddbs
+aws lambda delete-function --function-name create-order-history-kds
+```  
+
+* The AWS Kinesis data stream created during the labs:
+
+```bash
+aws kinesis delete-stream --stream-name Orders
+```
+
+* The Amazon DynamoDB tables created in the Getting Started section of the lab:
+
+```bash
+aws dynamodb delete-table --table-name Orders
+aws dynamodb delete-table --table-name OrdersHistory
+```
+
+* The Amazon SQS queues created during the labs:
+
+```bash
+aws sqs delete-queue --queue-url https://sqs.${REGION}.amazonaws.com/${ACCOUNT_ID}/orders-ddbs-dlq
+aws sqs delete-queue --queue-url https://sqs.${REGION}.amazonaws.com/${ACCOUNT_ID}/orders-kds-dlq
+```
+
+* The IAM policies attached to the IAM execution roles you created:
+
+![Delete IAM Policies](/static/images/change-data-capture/cleanup/delete-policies-one.png)
+
+![Delete IAM Policies](/static/images/change-data-capture/cleanup/delete-policies-two.png)
+
+* The AWS IAM execution roles created for the lambda functions:
+
+![Delete IAM Roles](/static/images/change-data-capture/cleanup/delete-roles.png)
+
+This should wrap up the cleanup process.

content/change-data-capture/ex1/configure-lambda.en.md

@@ -0,0 +1,114 @@
+---
+title: "Configure Lambda Function"
+date: 2023-12-01T00:00:00-00:00
+weight: 120
+chapter: true
+---
+
+Configure your lambda function to copy changed records from the Orders DynamoDB streams to the OrdersHistory table by doing the following.
+
+1. Go to the IAM dashboard on the AWS Management Console and inspect the IAM policy, i.e. **AWSLambdaMicroserviceExecutionRole...**, created when you created the create-order-history-ddbs lambda function.
+
+![AWS Lambda function console](/static/images/change-data-capture/ex1/iam-edit-policy.png)
+
+```
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": [
+                "dynamodb:DeleteItem",
+                "dynamodb:GetItem",
+                "dynamodb:PutItem",
+                "dynamodb:Scan",
+                "dynamodb:UpdateItem"
+            ],
+            "Resource": "arn:aws:dynamodb:{aws-region}:{aws-account-id}:table/*"
+        }
+    ]
+}
+```
+
+2. Update the policy statement by editing and replacing the existing policy using the following IAM policy statement. 
+
+```json
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": [
+                "dynamodb:DescribeStream",
+                "dynamodb:GetRecords",
+                "dynamodb:GetShardIterator",
+                "dynamodb:ListStreams"
+            ],
+            "Resource": "arn:aws:dynamodb:{aws-region}:{aws-account-id}:table/Orders/stream/*"
+        },
+        {
+            "Effect": "Allow",
+            "Action": "dynamodb:PutItem",
+            "Resource": "arn:aws:dynamodb:{aws-region}:{aws-account-id}:table/OrdersHistory"
+        },
+        {
+            "Effect": "Allow",
+            "Action": "sqs:SendMessage",
+            "Resource": "arn:aws:sqs:{aws-region}:{aws-account-id}:orders-ddbs-dlq"
+        }
+    ]
+}
+```
+
+The updated IAM policy gives the create-order-history-ddbs lambda function the permissions required to read events from the Orders DynamoDB stream, write new items to the OrdersHistory DynamoDB table and send messages to the orders-ddbs-dlq SQS queue.
+
+::alert[Replace **{aws-region}** and **{aws-account-id}** in the policy statement above with the correct value for your AWS region and your AWS account ID.]
+
+3. Go to the lambda function console editor. Select **Layers** then select **Add a Layer**.
+
+![AWS Lambda function console](/static/images/change-data-capture/ex1/select-layer.png)
+
+![AWS Lambda function console](/static/images/change-data-capture/ex1/add-layer.png)
+
+4. Select **Specify an ARN**, enter the Lambda Layer ARN below.
+
+```bash
+arn:aws:lambda:{aws-region}:017000801446:layer:AWSLambdaPowertoolsPythonV2:58
+```
+
+5. Click **Verify** then select **Add**. 
+
+![AWS Lambda function console](/static/images/change-data-capture/ex1/specify-layer.png)
+
+::alert[Replace {aws-region} with ID for the AWS region that you are currently working on.]
+
+6. Go to the configuration section of the lambda console editor. Select **Environment variables** then select **Edit**.
+
+![AWS Lambda function console](/static/images/change-data-capture/ex1/edit-env-var.png) 
+
+7. Add a new environment variable called **ORDERS_HISTORY_DB** and set its value to **OrdersHistory**.
+
+![AWS Lambda function console](/static/images/change-data-capture/ex1/new-env-var.png) 
+
+8. Select **Triggers** then select **Add trigger**.
+
+![AWS Lambda function console](/static/images/change-data-capture/ex1/triggers.png) 
+
+9. Select **DynamoDB** as the trigger source.
+10. Select the **Orders** DynamoDB table.
+11. Set the **Batch size** to **10** and leave all other values unchanged.
+
+![AWS Lambda function console](/static/images/change-data-capture/ex1/trigger-config.png) 
+
+12. Click **Additional settings** to expand the section.
+13. Provide the ARN of the **orders-ddbs-dlq** SQS queue you created earlier.
+
+```bash
+arn:aws:sqs:{aws-region}:{aws-account-id}:orders-ddbs-dlq
+```
+
+14. Set the Retry attempts to 3.
+
+![AWS Lambda function console](/static/images/change-data-capture/ex1/trigger-settings.png) 
+
+15. Select **Add**.

content/change-data-capture/ex1/create-dlq.en.md

@@ -0,0 +1,29 @@
+---
+title: "Create Dead Letter Queue"
+date: 2023-12-01T00:00:00-00:00
+weight: 110
+chapter: true
+---
+
+If the lambda function is not able to successfully process a record it receives from DynamoDB stream, the Lambda service should write the metadata for the error record to a dead letter queue (DLQ) so the reason for the failure can be investigated and resolved. 
+
+So create an [Amazon SQS Dead Letter Queue](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-dead-letter-queues.html) named **orders-ddbs-dlq** for your lambda function trigger using the AWS CLI command below.
+
+```bash
+aws sqs create-queue --queue-name orders-ddbs-dlq
+```
+
+Sample output:
+
+```
+{
+    "QueueUrl": "https://sqs.{aws-region}.amazonaws.com/{aws-account-id}/orders-ddbs-dlq"
+}
+```
+
+Later you will need the queue ARN. Use the below command, modifying the queue URL after *--queue-url* to match the result of the previous command, and then save the ARN for later use.
+
+```bash
+aws sqs get-queue-attributes --attribute-names "QueueArn" --query 'Attributes.QueueArn' --output text \
+--queue-url "https://sqs.{aws-region}.amazonaws.com/{aws-account-id}/orders-ddbs-dlq"
+```