From 698ce546f7f0dda2908c416e22dae6ce57b2a150 Mon Sep 17 00:00:00 2001 From: Rafael Sales Date: Wed, 24 Jun 2026 06:38:23 -0300 Subject: [PATCH 1/2] feat: implement AWS Glue integration and GlueStudioPanel --- package.json | 1 + pnpm-lock.yaml | 208 ++++++- src/app/page.tsx | 16 + src/components/diagram/PropertiesPanel.tsx | 13 +- src/components/layout/UnifiedToolbar.tsx | 10 +- src/components/ministack/GlueStudioPanel.tsx | 197 +++++++ .../ministack/MiniConsoleDialog.tsx | 2 + src/components/ministack/glue/CatalogTab.tsx | 133 +++++ src/components/ministack/glue/JobsTab.tsx | 212 ++++++++ .../ministack/glue/PySparkEditor.tsx | 143 +++++ src/components/ministack/glue/RunsTab.tsx | 238 ++++++++ src/components/ministack/glue/TablesTab.tsx | 218 ++++++++ src/lib/ministack/browser-actions.ts | 10 +- src/lib/ministack/client.ts | 4 + src/lib/ministack/glue-actions.ts | 508 ++++++++++++++++++ src/lib/ministack/service-map.ts | 5 +- src/lib/store.ts | 60 +++ 17 files changed, 1960 insertions(+), 18 deletions(-) create mode 100644 src/components/ministack/GlueStudioPanel.tsx create mode 100644 src/components/ministack/glue/CatalogTab.tsx create mode 100644 src/components/ministack/glue/JobsTab.tsx create mode 100644 src/components/ministack/glue/PySparkEditor.tsx create mode 100644 src/components/ministack/glue/RunsTab.tsx create mode 100644 src/components/ministack/glue/TablesTab.tsx create mode 100644 src/lib/ministack/glue-actions.ts diff --git a/package.json b/package.json index e2ef192..a78811e 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "@aws-sdk/client-elastic-load-balancing-v2": "^3.1073.0", "@aws-sdk/client-elasticache": "^3.1073.0", "@aws-sdk/client-eventbridge": "^3.1073.0", + "@aws-sdk/client-glue": "^3.1075.0", "@aws-sdk/client-iam": "^3.1073.0", "@aws-sdk/client-kinesis": "^3.1073.0", "@aws-sdk/client-kms": "^3.1073.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 924bb70..110eb54 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -47,6 +47,9 @@ importers: '@aws-sdk/client-eventbridge': specifier: ^3.1073.0 version: 3.1073.0 + '@aws-sdk/client-glue': + specifier: ^3.1075.0 + version: 3.1075.0 '@aws-sdk/client-iam': specifier: ^3.1073.0 version: 3.1073.0 @@ -368,6 +371,10 @@ packages: resolution: {integrity: sha512-h37GDGdaWifY7MxGDdgN9byhMZ6qqwxZ8KuE6/l/NYeFYv7LsXXa3FJp/EruiH+U78EQdOFYbfyO/mMxCZJewQ==} engines: {node: '>=20.0.0'} + '@aws-sdk/client-glue@3.1075.0': + resolution: {integrity: sha512-10849e8XW/iMgvGjo8EIsisFWetzMzgUuDl42Gdu64AtVNfRU0WSEvRxMyzBkaM1xy7czgs7s5d7BvB25RXB4A==} + engines: {node: '>=20.0.0'} + '@aws-sdk/client-iam@3.1073.0': resolution: {integrity: sha512-TEovDL6IPOiThcSWj0xVJe0Mffc06xFD1xEry+kAREra0d8ArxGFnYeCRBOxQkGpw0H3z0T8STWxw73ibkm/Mw==} engines: {node: '>=20.0.0'} @@ -432,38 +439,74 @@ packages: resolution: {integrity: sha512-YofH63shc6YRdXjz80BJkpJW+Bkn0Cuu2dn4Rv7s9G2Idt58tgtzQEWxrR2xVljlVfIBeUjPuULnSVYLke3sUQ==} engines: {node: '>=20.0.0'} + '@aws-sdk/core@3.974.23': + resolution: {integrity: sha512-MiWR/uWjxjFXGzrE0Ghc5lWxUxzHsUWFhV+OX7M4cR9SrmrnZs6TXavnCWnzzdwJeFri34xQo81rvGNzK3c4BQ==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-env@3.972.48': resolution: {integrity: sha512-h6FEC95fbexUd6zxm4PdgS82bTcI2PRtUb2ZwMipb/Xr8bPwtf0G8rBo2jp7NA24Mbx2JA8/WingiYpA9RCCyw==} engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-env@3.972.49': + resolution: {integrity: sha512-liB3yQNHCM9k/gu/w36XHMKPluT7HTlnGUhRbBGSISDQkcr/Sy1zsZabiuvQj8WG5yW573u9RehrBvvnIQ9OEQ==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-http@3.972.50': resolution: {integrity: sha512-lJO3OLpjvz5m/RSBQmsG/CEUGsvCy5ruxKwPQaOCqxqCMuyYT2BZwQUTDZVVwqQ9LrZKuK24JSa6r31hL/tvkg==} engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-http@3.972.51': + resolution: {integrity: sha512-XET0H2oofciJ5lMRWNIvRjAP7Q3wv2XT+JtJJEdhPWUMwe3TvQ9qcxonpu7vXmNngncvFpi4E2It+Tamas/naA==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-ini@3.972.55': resolution: {integrity: sha512-TBoF4buBGYhXjdZAryayY2TrkQj2B2KfE/msG4V53XCt+w0EhEwM2JRjx8p2grJ2C6gtH5++SAwEvGMRdi0yyw==} engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-ini@3.972.56': + resolution: {integrity: sha512-IAmc61hbgQiHht9U3x0tnRwz0lzdwOwD/i9voRgdJrKamF+JtmrBOsW9GwB7mfFonNWOWL4qARWYrF8veEMe3w==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-login@3.972.54': resolution: {integrity: sha512-hBWI3wZTdTGiuMfmPts6AWbAjFfRniOQnqx68tc2cQvRKWawFbN9wkLOVPWM1FAOyowZU73mC6Fi+rHSHNyLFw==} engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-login@3.972.55': + resolution: {integrity: sha512-hBBkANo3cDn+h2qxxzER4a+J8JCO9o9Z/YYmU7iky6AcaarX5RRdRcHNC6SLdwY0vAXQygn6soUbDqPn3GghaA==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-node@3.972.57': resolution: {integrity: sha512-u6dClpzNdWf1HGWz4wwhdXi1wiOofCLniM9S4BQQGlLAN9TW7VB+ld5V533GdKrYMaFeBGFqKnj0JCYvynLqwQ==} engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-node@3.972.58': + resolution: {integrity: sha512-OyCLVmSI7pZO8hxwNVX6pXhTVlJqRBTp+ijdEfJSUj0RyjHnF602OfAarOzGq6wkGodeFkYBt8MmJ6A6ycRgWw==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-process@3.972.48': resolution: {integrity: sha512-w6VZwojPt12WnEkAUy6Nu4K6sWCbBmR7QX390b0nE6vRvkXbrYr9Lq9VySGkfjiMjpUA87op+J4EgvRmtWIDoQ==} engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-process@3.972.49': + resolution: {integrity: sha512-C8h36lBuC/RnBSsjlO+dn6xZm3KbAl5vpJaVPAfQnMmz2/OISmKOc8XZcqMQgO2ADwBYNRMM6Kf3vz9G/TulMQ==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-sso@3.972.54': resolution: {integrity: sha512-23uZpIpF2SIFDCa1fcWa202tK4gGeyvX6GIIAjiB8WBsvsVRBMnJ/7dCxHzxf7eZT7GToJg837LDIBnZsl/VUg==} engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-sso@3.972.55': + resolution: {integrity: sha512-1FkOz74Ea5QGS9jtIoXp55T/IkSS3spv+nLTT07fRY/+T5xmEOqaYBVIaEmX4zTNvbV6g2lrtlaVKWEoNyJt3w==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-web-identity@3.972.54': resolution: {integrity: sha512-0Iv5QttS6wcATlodYKgvQj6B9Db51rx7NU9fqu0PoLeS4BIgdYMc/QK4smwLwpm5RFrs02V/eLyEFp3FklvlNQ==} engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-web-identity@3.972.55': + resolution: {integrity: sha512-g2BoECD1q01kTPByi56+VLVvdWDzMkKIcr77qixpqH0okw2t0U5CoPv+6S8v/D1Y2Wa6QKKtn6XAtDzP+Kfpvg==} + engines: {node: '>=20.0.0'} + '@aws-sdk/dynamodb-codec@3.973.22': resolution: {integrity: sha512-QBs7/nWHsPA0wTEqhU5iPgaDjeZzQHhmwJr6Q0f56rW3Fk8ExJQgXFzEg/l48iDwJVUIMLjPqhw7dNP3cShUxA==} engines: {node: '>=20.0.0'} @@ -520,6 +563,10 @@ packages: resolution: {integrity: sha512-4IwtcYSxEIVw5hcp8ogq0CMbFNZFw7jJUetpfFUhFFeqsa1K8j2Ihg2hnxLyOp3stMZnXda6VzOmPi1AFZQXcg==} engines: {node: '>=20.0.0'} + '@aws-sdk/nested-clients@3.997.23': + resolution: {integrity: sha512-gO93ZPsI2bxeFZD42f1/qjDw6FAZkNZcKRO94LIiT03fzOmcJ9e/tunxjVjA1Rl69ClmVJzz8H3G9CdKef10PA==} + engines: {node: '>=20.0.0'} + '@aws-sdk/signature-v4-multi-region@3.996.35': resolution: {integrity: sha512-6L/VWs+Wch2stHemCGTmUNqKLMzURxQDK5boNG3Jn3kAOp71meDUuS5sbObpEvFxHDq0uWeSLFDNSYsjNt+Dlg==} engines: {node: '>=20.0.0'} @@ -532,6 +579,10 @@ packages: resolution: {integrity: sha512-Tolawuc3I9Q6pElcqoBQMLCiCOfKn3eqG4oNIRci4BurhsrJmzXkhF3N+6LRXJrWYFtJKfTkBuLbYCLr8+pwig==} engines: {node: '>=20.0.0'} + '@aws-sdk/token-providers@3.1074.0': + resolution: {integrity: sha512-pv80IzgGW4RnXWtft692chZOM9i6PhebVsLCcnaM4dBEPZva2fE6FXAHs76G7Rc7s3yGyX/68G0nZMrUy+Vmpg==} + engines: {node: '>=20.0.0'} + '@aws-sdk/types@3.973.13': resolution: {integrity: sha512-pEHZqRkAlHfnfAU9tK+WpKv/gBNjGJrHMgA3A0iYRGyswBS2t0pfez+lWlwktb3Bqa0ovh7w/QJTFwp3fDxLNg==} engines: {node: '>=20.0.0'} @@ -548,6 +599,10 @@ packages: resolution: {integrity: sha512-StElZPEoBquWwNqw1AcfpzEyZqJvFxouG+mpDNYlcH6ZOrqd2CuIryv+8LV8gNHZUOyKyJF3Dq9vxaXEmDR9TQ==} engines: {node: '>=20.0.0'} + '@aws-sdk/xml-builder@3.972.31': + resolution: {integrity: sha512-SzE4Pgyl+hDF+BuyuzxUSpwnuUu9lJuO1YGgteG89/4Qv0+2IQiVQqdbPV32IozLvXWQChPQcdkk/sKvb1QHiQ==} + engines: {node: '>=20.0.0'} + '@aws/lambda-invoke-store@0.2.4': resolution: {integrity: sha512-iY8yvjE0y651BixKNPgmv1WrQc+GZ142sb0z4gYnChDDY2YqI4P/jsSopBWrKfAt7LOJAkOXt7rC/hms+WclQQ==} engines: {node: '>=18.0.0'} @@ -1471,10 +1526,6 @@ packages: resolution: {integrity: sha512-X9rVls3En0z3NtrmguTmpRM0/NqtWUxBjal6fcAkwtsub+gOdLZ6kD+V7xhUgFMGdG14bHbZ7M5QjaRI1+DatQ==} engines: {node: '>=18.0.0'} - '@smithy/types@4.14.2': - resolution: {integrity: sha512-P+otAxbV4CqBybp7EkcJCrig63yE2E7PuNVOmilVMRcx/O+QDzGULTrKsq4DV13gSfak9ObPrWaHl/9bL5YcWw==} - engines: {node: '>=18.0.0'} - '@smithy/types@4.15.0': resolution: {integrity: sha512-Z5TAOxygoFvybJV3igo5SloFflSokHx2hu1eFA+DxDTcn+FtKxUSui+rbTRG1pAafMA888Z3MVvCWUuvCrTXjg==} engines: {node: '>=18.0.0'} @@ -4240,7 +4291,7 @@ snapshots: '@aws-crypto/crc32c@5.2.0': dependencies: '@aws-crypto/util': 5.2.0 - '@aws-sdk/types': 3.973.8 + '@aws-sdk/types': 3.973.13 tslib: 2.8.1 '@aws-crypto/sha1-browser@5.2.0': @@ -4257,7 +4308,7 @@ snapshots: '@aws-crypto/sha256-js': 5.2.0 '@aws-crypto/supports-web-crypto': 5.2.0 '@aws-crypto/util': 5.2.0 - '@aws-sdk/types': 3.973.8 + '@aws-sdk/types': 3.973.13 '@aws-sdk/util-locate-window': 3.965.5 '@smithy/util-utf8': 2.3.0 tslib: 2.8.1 @@ -4265,7 +4316,7 @@ snapshots: '@aws-crypto/sha256-js@5.2.0': dependencies: '@aws-crypto/util': 5.2.0 - '@aws-sdk/types': 3.973.8 + '@aws-sdk/types': 3.973.13 tslib: 2.8.1 '@aws-crypto/supports-web-crypto@5.2.0': @@ -4274,7 +4325,7 @@ snapshots: '@aws-crypto/util@5.2.0': dependencies: - '@aws-sdk/types': 3.973.8 + '@aws-sdk/types': 3.973.13 '@smithy/util-utf8': 2.3.0 tslib: 2.8.1 @@ -4468,6 +4519,19 @@ snapshots: '@smithy/types': 4.15.0 tslib: 2.8.1 + '@aws-sdk/client-glue@3.1075.0': + dependencies: + '@aws-crypto/sha256-browser': 5.2.0 + '@aws-crypto/sha256-js': 5.2.0 + '@aws-sdk/core': 3.974.23 + '@aws-sdk/credential-provider-node': 3.972.58 + '@aws-sdk/types': 3.973.13 + '@smithy/core': 3.25.1 + '@smithy/fetch-http-handler': 5.5.1 + '@smithy/node-http-handler': 4.8.1 + '@smithy/types': 4.15.0 + tslib: 2.8.1 + '@aws-sdk/client-iam@3.1073.0': dependencies: '@aws-crypto/sha256-browser': 5.2.0 @@ -4680,6 +4744,17 @@ snapshots: bowser: 2.14.1 tslib: 2.8.1 + '@aws-sdk/core@3.974.23': + dependencies: + '@aws-sdk/types': 3.973.13 + '@aws-sdk/xml-builder': 3.972.31 + '@aws/lambda-invoke-store': 0.2.4 + '@smithy/core': 3.25.1 + '@smithy/signature-v4': 5.5.1 + '@smithy/types': 4.15.0 + bowser: 2.14.1 + tslib: 2.8.1 + '@aws-sdk/credential-provider-env@3.972.48': dependencies: '@aws-sdk/core': 3.974.22 @@ -4688,6 +4763,14 @@ snapshots: '@smithy/types': 4.15.0 tslib: 2.8.1 + '@aws-sdk/credential-provider-env@3.972.49': + dependencies: + '@aws-sdk/core': 3.974.23 + '@aws-sdk/types': 3.973.13 + '@smithy/core': 3.25.1 + '@smithy/types': 4.15.0 + tslib: 2.8.1 + '@aws-sdk/credential-provider-http@3.972.50': dependencies: '@aws-sdk/core': 3.974.22 @@ -4698,6 +4781,16 @@ snapshots: '@smithy/types': 4.15.0 tslib: 2.8.1 + '@aws-sdk/credential-provider-http@3.972.51': + dependencies: + '@aws-sdk/core': 3.974.23 + '@aws-sdk/types': 3.973.13 + '@smithy/core': 3.25.1 + '@smithy/fetch-http-handler': 5.5.1 + '@smithy/node-http-handler': 4.8.1 + '@smithy/types': 4.15.0 + tslib: 2.8.1 + '@aws-sdk/credential-provider-ini@3.972.55': dependencies: '@aws-sdk/core': 3.974.22 @@ -4714,6 +4807,22 @@ snapshots: '@smithy/types': 4.15.0 tslib: 2.8.1 + '@aws-sdk/credential-provider-ini@3.972.56': + dependencies: + '@aws-sdk/core': 3.974.23 + '@aws-sdk/credential-provider-env': 3.972.49 + '@aws-sdk/credential-provider-http': 3.972.51 + '@aws-sdk/credential-provider-login': 3.972.55 + '@aws-sdk/credential-provider-process': 3.972.49 + '@aws-sdk/credential-provider-sso': 3.972.55 + '@aws-sdk/credential-provider-web-identity': 3.972.55 + '@aws-sdk/nested-clients': 3.997.23 + '@aws-sdk/types': 3.973.13 + '@smithy/core': 3.25.1 + '@smithy/credential-provider-imds': 4.4.1 + '@smithy/types': 4.15.0 + tslib: 2.8.1 + '@aws-sdk/credential-provider-login@3.972.54': dependencies: '@aws-sdk/core': 3.974.22 @@ -4723,6 +4832,15 @@ snapshots: '@smithy/types': 4.15.0 tslib: 2.8.1 + '@aws-sdk/credential-provider-login@3.972.55': + dependencies: + '@aws-sdk/core': 3.974.23 + '@aws-sdk/nested-clients': 3.997.23 + '@aws-sdk/types': 3.973.13 + '@smithy/core': 3.25.1 + '@smithy/types': 4.15.0 + tslib: 2.8.1 + '@aws-sdk/credential-provider-node@3.972.57': dependencies: '@aws-sdk/credential-provider-env': 3.972.48 @@ -4737,6 +4855,20 @@ snapshots: '@smithy/types': 4.15.0 tslib: 2.8.1 + '@aws-sdk/credential-provider-node@3.972.58': + dependencies: + '@aws-sdk/credential-provider-env': 3.972.49 + '@aws-sdk/credential-provider-http': 3.972.51 + '@aws-sdk/credential-provider-ini': 3.972.56 + '@aws-sdk/credential-provider-process': 3.972.49 + '@aws-sdk/credential-provider-sso': 3.972.55 + '@aws-sdk/credential-provider-web-identity': 3.972.55 + '@aws-sdk/types': 3.973.13 + '@smithy/core': 3.25.1 + '@smithy/credential-provider-imds': 4.4.1 + '@smithy/types': 4.15.0 + tslib: 2.8.1 + '@aws-sdk/credential-provider-process@3.972.48': dependencies: '@aws-sdk/core': 3.974.22 @@ -4745,6 +4877,14 @@ snapshots: '@smithy/types': 4.15.0 tslib: 2.8.1 + '@aws-sdk/credential-provider-process@3.972.49': + dependencies: + '@aws-sdk/core': 3.974.23 + '@aws-sdk/types': 3.973.13 + '@smithy/core': 3.25.1 + '@smithy/types': 4.15.0 + tslib: 2.8.1 + '@aws-sdk/credential-provider-sso@3.972.54': dependencies: '@aws-sdk/core': 3.974.22 @@ -4755,6 +4895,16 @@ snapshots: '@smithy/types': 4.15.0 tslib: 2.8.1 + '@aws-sdk/credential-provider-sso@3.972.55': + dependencies: + '@aws-sdk/core': 3.974.23 + '@aws-sdk/nested-clients': 3.997.23 + '@aws-sdk/token-providers': 3.1074.0 + '@aws-sdk/types': 3.973.13 + '@smithy/core': 3.25.1 + '@smithy/types': 4.15.0 + tslib: 2.8.1 + '@aws-sdk/credential-provider-web-identity@3.972.54': dependencies: '@aws-sdk/core': 3.974.22 @@ -4764,6 +4914,15 @@ snapshots: '@smithy/types': 4.15.0 tslib: 2.8.1 + '@aws-sdk/credential-provider-web-identity@3.972.55': + dependencies: + '@aws-sdk/core': 3.974.23 + '@aws-sdk/nested-clients': 3.997.23 + '@aws-sdk/types': 3.973.13 + '@smithy/core': 3.25.1 + '@smithy/types': 4.15.0 + tslib: 2.8.1 + '@aws-sdk/dynamodb-codec@3.973.22': dependencies: '@aws-sdk/core': 3.974.22 @@ -4873,6 +5032,19 @@ snapshots: '@smithy/types': 4.15.0 tslib: 2.8.1 + '@aws-sdk/nested-clients@3.997.23': + dependencies: + '@aws-crypto/sha256-browser': 5.2.0 + '@aws-crypto/sha256-js': 5.2.0 + '@aws-sdk/core': 3.974.23 + '@aws-sdk/signature-v4-multi-region': 3.996.35 + '@aws-sdk/types': 3.973.13 + '@smithy/core': 3.25.1 + '@smithy/fetch-http-handler': 5.5.1 + '@smithy/node-http-handler': 4.8.1 + '@smithy/types': 4.15.0 + tslib: 2.8.1 + '@aws-sdk/signature-v4-multi-region@3.996.35': dependencies: '@aws-sdk/types': 3.973.13 @@ -4898,6 +5070,15 @@ snapshots: '@smithy/types': 4.15.0 tslib: 2.8.1 + '@aws-sdk/token-providers@3.1074.0': + dependencies: + '@aws-sdk/core': 3.974.23 + '@aws-sdk/nested-clients': 3.997.23 + '@aws-sdk/types': 3.973.13 + '@smithy/core': 3.25.1 + '@smithy/types': 4.15.0 + tslib: 2.8.1 + '@aws-sdk/types@3.973.13': dependencies: '@smithy/types': 4.15.0 @@ -4905,7 +5086,7 @@ snapshots: '@aws-sdk/types@3.973.8': dependencies: - '@smithy/types': 4.14.2 + '@smithy/types': 4.15.0 tslib: 2.8.1 '@aws-sdk/util-locate-window@3.965.5': @@ -4918,6 +5099,11 @@ snapshots: fast-xml-parser: 5.7.3 tslib: 2.8.1 + '@aws-sdk/xml-builder@3.972.31': + dependencies: + '@smithy/types': 4.15.0 + tslib: 2.8.1 + '@aws/lambda-invoke-store@0.2.4': {} '@babel/code-frame@7.29.0': @@ -5856,10 +6042,6 @@ snapshots: '@smithy/types': 4.15.0 tslib: 2.8.1 - '@smithy/types@4.14.2': - dependencies: - tslib: 2.8.1 - '@smithy/types@4.15.0': dependencies: tslib: 2.8.1 diff --git a/src/app/page.tsx b/src/app/page.tsx index 12393aa..a02043e 100644 --- a/src/app/page.tsx +++ b/src/app/page.tsx @@ -26,6 +26,7 @@ import KeyboardShortcutsDialog from "@/components/layout/KeyboardShortcutsDialog import TemplatesDialog from "@/components/diagram/TemplatesDialog"; import TerraformPanel from "@/components/diagram/TerraformPanel"; import MiniStackPanel from "@/components/ministack/MiniStackPanel"; +import GlueStudioPanel from "@/components/ministack/GlueStudioPanel"; import { OnboardingTour } from "@/components/layout/OnboardingTour"; // Service @@ -93,6 +94,15 @@ function HomeContent() { return () => window.removeEventListener("openShortcutsDialog", handler); }, []); + useEffect(() => { + const handler = (e: Event) => { + const panel = (e as CustomEvent).detail; + if (panel) setActivePanel(panel); + }; + window.addEventListener("openarchflow:open-panel", handler); + return () => window.removeEventListener("openarchflow:open-panel", handler); + }, []); + // Initialize Local AI model if needed useEffect(() => { if (aiProvider === "local") { @@ -410,6 +420,12 @@ function HomeContent() { nodes={currentNodes} /> + setActivePanel(null)} + nodes={currentNodes} + /> + {/* Sidebar (Left Drawer) */} diff --git a/src/components/diagram/PropertiesPanel.tsx b/src/components/diagram/PropertiesPanel.tsx index 652344f..107b18f 100644 --- a/src/components/diagram/PropertiesPanel.tsx +++ b/src/components/diagram/PropertiesPanel.tsx @@ -694,8 +694,13 @@ function MiniStackSection({ nodeId, node }: { nodeId: string; node: any }) { const isDeployed = status === "deployed"; const isError = status === "error"; const service = (node.data.service ?? "").toLowerCase(); + const isGlue = service === "glue"; const resourceId = ms?.resourceId ?? ""; + const openGlueStudio = React.useCallback(() => { + window.dispatchEvent(new CustomEvent("openarchflow:open-panel", { detail: "glue" })); + }, []); + const handleQuickInvoke = React.useCallback(async () => { let body: unknown; try { body = JSON.parse(invokePayload); } catch { toast.error("Invalid JSON payload"); return; } @@ -872,7 +877,13 @@ function MiniStackSection({ nodeId, node }: { nodeId: string; node: any }) { {INVOKE_LABEL[service] ?? "Invoke"} )} - {isDeployed && ( + {isGlue && ( + + )} + {isDeployed && !isGlue && ( + + + {/* Node selector (when multiple Glue nodes) */} + {glueNodes.length > 1 && ( +
+ Glue node + +
+ )} + + {/* Connection warning — only when the endpoint is actually unreachable */} + {reachable === false && ( +
+ + + MiniStack is not reachable at {ministackConfig.endpoint}. Start it with{" "} + docker run -p 4566:4566 -v /var/run/docker.sock:/var/run/docker.sock ministackorg/ministack + {" "}(the socket mount is required for Spark job execution). + +
+ )} + + {/* Tabs */} +
+ {tabs.map((tab) => ( + + ))} +
+ + {/* Body */} +
+ {glueNodes.length === 0 ? ( +
+ +

No Glue node on the canvas

+

Add an AWS Glue service to your diagram to manage its Data Catalog and PySpark jobs here.

+
+ ) : ( + <> + {activeTab === "catalog" && ( + + )} + {activeTab === "tables" && ( + + )} + {activeTab === "jobs" && ( + { setActiveJob(job); setActiveTab("runs"); }} + /> + )} + {activeTab === "runs" && ( + + )} + + )} +
+ + )} + + ); + + if (!mounted) return null; + return createPortal(panel, document.body); +} diff --git a/src/components/ministack/MiniConsoleDialog.tsx b/src/components/ministack/MiniConsoleDialog.tsx index 1da7f98..18e52b4 100644 --- a/src/components/ministack/MiniConsoleDialog.tsx +++ b/src/components/ministack/MiniConsoleDialog.tsx @@ -35,6 +35,7 @@ const SERVICE_ICON: Record = { eventbridge: "⚡", apigateway: "🌐", "api-gateway": "🌐", + glue: "🧪", }; const SERVICE_LABEL: Record = { @@ -47,6 +48,7 @@ const SERVICE_LABEL: Record = { eventbridge: "EventBridge Bus", apigateway: "API Gateway", "api-gateway": "API Gateway", + glue: "Glue Catalog", }; function ConsoleBody({ diff --git a/src/components/ministack/glue/CatalogTab.tsx b/src/components/ministack/glue/CatalogTab.tsx new file mode 100644 index 0000000..c89355a --- /dev/null +++ b/src/components/ministack/glue/CatalogTab.tsx @@ -0,0 +1,133 @@ +"use client"; + +import React, { useState, useEffect, useCallback } from "react"; +import { Database, Plus, RefreshCw, Loader2, Trash2, Check } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { ScrollArea } from "@/components/ui/scroll-area"; +import { cn } from "@/lib/utils"; +import { toast } from "sonner"; +import type { MiniStackConfig } from "@/lib/ministack/types"; +import { + glueListDatabases, glueCreateDatabase, glueDeleteDatabase, sanitizeGlueName, + type GlueDatabaseInfo, +} from "@/lib/ministack/glue-actions"; + +interface CatalogTabProps { + config: MiniStackConfig; + activeDatabase: string; + onSelectDatabase: (name: string) => void; +} + +export function CatalogTab({ config, activeDatabase, onSelectDatabase }: CatalogTabProps) { + const [databases, setDatabases] = useState([]); + const [loading, setLoading] = useState(false); + const [newName, setNewName] = useState(""); + const [creating, setCreating] = useState(false); + + const load = useCallback(async () => { + setLoading(true); + try { + setDatabases(await glueListDatabases(config)); + } catch (e) { + toast.error(e instanceof Error ? e.message : "Failed to list databases"); + } finally { + setLoading(false); + } + }, [config]); + + useEffect(() => { load(); }, [load]); + + const handleCreate = async () => { + const name = sanitizeGlueName(newName.trim()); + if (!newName.trim()) { toast.error("Enter a database name"); return; } + setCreating(true); + try { + await glueCreateDatabase(config, name, "Created from OpenArchFlow Glue Studio"); + toast.success(`Database "${name}" created`); + setNewName(""); + onSelectDatabase(name); + await load(); + } catch (e) { + toast.error(e instanceof Error ? e.message : "Create failed"); + } finally { + setCreating(false); + } + }; + + const handleDelete = async (name: string) => { + try { + await glueDeleteDatabase(config, name); + toast.success(`Database "${name}" deleted`); + await load(); + } catch (e) { + toast.error(e instanceof Error ? e.message : "Delete failed"); + } + }; + + return ( +
+
+

+ Data Catalog databases +

+ +
+ +
+ setNewName(e.target.value)} + onKeyDown={(e) => { if (e.key === "Enter") handleCreate(); }} + placeholder="new_database_name" + className="h-8 flex-1 text-xs rounded border border-border bg-background px-2 font-mono" + /> + +
+ + +
+ {databases.length === 0 && ( +

+ {loading ? "Loading…" : "No databases yet — create one above."} +

+ )} + {databases.map((db) => { + const active = db.name === activeDatabase; + return ( +
onSelectDatabase(db.name)} + > +
+

+ {active && } + {db.name} +

+ {db.description && ( +

{db.description}

+ )} +
+ +
+ ); + })} +
+ +
+ ); +} diff --git a/src/components/ministack/glue/JobsTab.tsx b/src/components/ministack/glue/JobsTab.tsx new file mode 100644 index 0000000..cf9ca9b --- /dev/null +++ b/src/components/ministack/glue/JobsTab.tsx @@ -0,0 +1,212 @@ +"use client"; + +import React, { useState, useEffect, useCallback, useMemo } from "react"; +import { Cog, Plus, RefreshCw, Loader2, Trash2, Rocket, Play, ChevronLeft } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { ScrollArea } from "@/components/ui/scroll-area"; +import { cn } from "@/lib/utils"; +import { toast } from "sonner"; +import type { MiniStackConfig } from "@/lib/ministack/types"; +import type { AppNode, GlueJobConfig } from "@/lib/store"; +import { + glueListJobs, glueUpsertJob, glueDeleteJob, glueUploadScript, glueStartJobRun, sanitizeGlueName, +} from "@/lib/ministack/glue-actions"; +import PySparkEditor, { DEFAULT_PYSPARK_SCRIPT } from "./PySparkEditor"; + +const GLUE_VERSIONS = ["4.0", "3.0"]; +const WORKER_TYPES = ["G.1X", "G.2X"]; + +interface JobsTabProps { + config: MiniStackConfig; + node: AppNode | null; + setNodeGlueConfig: (nodeId: string, partial: { jobs?: GlueJobConfig[] }) => void; + onRunStarted: (jobName: string) => void; +} + +function newJob(): GlueJobConfig { + return { name: "", pysparkCode: DEFAULT_PYSPARK_SCRIPT, glueVersion: "4.0", workerType: "G.1X", numberOfWorkers: 2 }; +} + +export function JobsTab({ config, node, setNodeGlueConfig, onRunStarted }: JobsTabProps) { + const localJobs = useMemo(() => node?.data.glueConfig?.jobs ?? [], [node]); + const [remoteNames, setRemoteNames] = useState([]); + const [loading, setLoading] = useState(false); + const [editing, setEditing] = useState(null); + const [deploying, setDeploying] = useState(false); + const [running, setRunning] = useState(false); + + const load = useCallback(async () => { + setLoading(true); + try { + const jobs = await glueListJobs(config); + setRemoteNames(jobs.map((j) => j.name)); + } catch (e) { + toast.error(e instanceof Error ? e.message : "Failed to list jobs"); + } finally { + setLoading(false); + } + }, [config]); + + useEffect(() => { load(); }, [load]); + + // Merge locally-designed jobs with names that only exist on MiniStack. + const jobList = useMemo(() => { + const byName = new Map(); + for (const j of localJobs) byName.set(j.name, j); + for (const name of remoteNames) if (!byName.has(name)) byName.set(name, { ...newJob(), name }); + return [...byName.values()]; + }, [localJobs, remoteNames]); + + const persist = (job: GlueJobConfig) => { + if (!node) return; + const others = localJobs.filter((j) => j.name !== job.name); + setNodeGlueConfig(node.id, { jobs: [...others, job] }); + }; + + const handleDeploy = async () => { + if (!editing) return; + const name = sanitizeGlueName(editing.name); + if (!editing.name.trim()) { toast.error("Enter a job name"); return; } + setDeploying(true); + try { + const scriptLocation = await glueUploadScript(config, name, editing.pysparkCode); + await glueUpsertJob(config, { + name, + scriptLocation, + glueVersion: editing.glueVersion, + workerType: editing.workerType, + numberOfWorkers: editing.numberOfWorkers, + }); + const saved = { ...editing, name }; + persist(saved); + setEditing(saved); + toast.success(`Job "${name}" deployed (script uploaded to S3)`); + await load(); + } catch (e) { + toast.error(e instanceof Error ? e.message : "Deploy failed"); + } finally { + setDeploying(false); + } + }; + + const handleRun = async () => { + if (!editing) return; + const name = sanitizeGlueName(editing.name); + setRunning(true); + try { + const runId = await glueStartJobRun(config, name); + toast.success(`Run started: ${runId || name}`); + onRunStarted(name); + } catch (e) { + toast.error(e instanceof Error ? e.message : "Run failed"); + } finally { + setRunning(false); + } + }; + + const handleDelete = async (name: string) => { + try { + await glueDeleteJob(config, name); + if (node) setNodeGlueConfig(node.id, { jobs: localJobs.filter((j) => j.name !== name) }); + toast.success(`Job "${name}" deleted`); + await load(); + } catch (e) { + toast.error(e instanceof Error ? e.message : "Delete failed"); + } + }; + + if (editing) { + return ( +
+
+ +
+ + +
+
+ +
+
+ + setEditing({ ...editing, name: e.target.value })} + placeholder="my_etl_job" + className="h-7 w-full text-xs rounded border border-border bg-background px-2 font-mono" + /> +
+
+ + +
+
+ + +
+
+
+ + setEditing({ ...editing, numberOfWorkers: Math.max(2, Number(e.target.value)) })} + className="h-7 w-20 text-xs rounded border border-border bg-background px-2" + /> + Spark runs on the amazon/aws-glue-libs image (Docker required). Iceberg: add --datalake-formats iceberg. +
+ +
+ setEditing({ ...editing, pysparkCode: v })} /> +
+
+ ); + } + + return ( +
+
+

ETL Jobs (PySpark)

+
+ + +
+
+ +
+ {jobList.length === 0 && ( +

{loading ? "Loading…" : "No jobs yet — create one."}

+ )} + {jobList.map((j) => ( +
+ + +
+ ))} +
+ +
+ ); +} diff --git a/src/components/ministack/glue/PySparkEditor.tsx b/src/components/ministack/glue/PySparkEditor.tsx new file mode 100644 index 0000000..3a0eef0 --- /dev/null +++ b/src/components/ministack/glue/PySparkEditor.tsx @@ -0,0 +1,143 @@ +"use client"; + +import React, { useEffect } from "react"; +import Editor, { useMonaco } from "@monaco-editor/react"; +import { useTheme } from "next-themes"; + +// Default AWS Glue PySpark boilerplate (GlueContext + Job bookmarks). +export const DEFAULT_PYSPARK_SCRIPT = `import sys +from awsglue.transforms import * +from awsglue.utils import getResolvedOptions +from pyspark.context import SparkContext +from awsglue.context import GlueContext +from awsglue.job import Job + +args = getResolvedOptions(sys.argv, ["JOB_NAME"]) +sc = SparkContext() +glueContext = GlueContext(sc) +spark = glueContext.spark_session +job = Job(glueContext) +job.init(args["JOB_NAME"], args) + +# Read a table from the Glue Data Catalog +# source = glueContext.create_dynamic_frame.from_catalog( +# database="my_database", +# table_name="my_table", +# ) + +# Transform with Spark, then write back to S3 / the catalog +# source.toDF().show() +# glueContext.write_dynamic_frame.from_options( +# frame=source, +# connection_type="s3", +# connection_options={"path": "s3://my-bucket/output/"}, +# format="parquet", +# ) + +job.commit() +`; + +let snippetsRegistered = false; + +function registerPySparkSnippets(monaco: NonNullable>) { + if (snippetsRegistered) return; + snippetsRegistered = true; + + monaco.languages.registerCompletionItemProvider("python", { + provideCompletionItems: (model, position) => { + const word = model.getWordUntilPosition(position); + const range = { + startLineNumber: position.lineNumber, + endLineNumber: position.lineNumber, + startColumn: word.startColumn, + endColumn: word.endColumn, + }; + const items: { label: string; insertText: string; detail: string }[] = [ + { + label: "glue_boilerplate", + insertText: DEFAULT_PYSPARK_SCRIPT, + detail: "Full AWS Glue PySpark job skeleton", + }, + { + label: "from_catalog", + insertText: + 'glueContext.create_dynamic_frame.from_catalog(\n database="${1:database}",\n table_name="${2:table}",\n)', + detail: "Read a DynamicFrame from the Glue Data Catalog", + }, + { + label: "write_from_options", + insertText: + 'glueContext.write_dynamic_frame.from_options(\n frame=${1:frame},\n connection_type="s3",\n connection_options={"path": "s3://${2:bucket}/${3:prefix}/"},\n format="${4:parquet}",\n)', + detail: "Write a DynamicFrame to S3", + }, + { + label: "spark_read", + insertText: 'spark.read.${1:parquet}("s3://${2:bucket}/${3:path}/")', + detail: "Read a Spark DataFrame from S3", + }, + { + label: "job_commit", + insertText: "job.commit()", + detail: "Commit the Glue job (finalizes bookmarks)", + }, + ]; + return { + suggestions: items.map((s) => ({ + label: s.label, + kind: monaco.languages.CompletionItemKind.Snippet, + insertText: s.insertText, + insertTextRules: monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet, + detail: s.detail, + range, + })), + }; + }, + }); +} + +interface PySparkEditorProps { + value: string; + onChange?: (value: string) => void; + readOnly?: boolean; + height?: string; +} + +export default function PySparkEditor({ + value, + onChange, + readOnly = false, + height = "100%", +}: PySparkEditorProps) { + const { resolvedTheme } = useTheme(); + const monaco = useMonaco(); + + useEffect(() => { + if (!monaco) return; + registerPySparkSnippets(monaco); + }, [monaco]); + + return ( + onChange?.(val ?? "")} + options={{ + readOnly, + fontSize: 12, + fontFamily: "'JetBrains Mono', 'Fira Code', 'Cascadia Code', monospace", + minimap: { enabled: false }, + lineNumbers: "on", + wordWrap: "on", + scrollBeyondLastLine: false, + padding: { top: 8, bottom: 8 }, + overviewRulerLanes: 0, + renderLineHighlight: "gutter", + tabSize: 4, + suggestOnTriggerCharacters: true, + quickSuggestions: { other: true, comments: false, strings: false }, + }} + /> + ); +} diff --git a/src/components/ministack/glue/RunsTab.tsx b/src/components/ministack/glue/RunsTab.tsx new file mode 100644 index 0000000..db7cf65 --- /dev/null +++ b/src/components/ministack/glue/RunsTab.tsx @@ -0,0 +1,238 @@ +"use client"; + +import React, { useState, useEffect, useCallback, useRef } from "react"; +import { Activity, RefreshCw, Loader2, Terminal, Copy } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { ScrollArea } from "@/components/ui/scroll-area"; +import { cn } from "@/lib/utils"; +import { toast } from "sonner"; +import type { MiniStackConfig } from "@/lib/ministack/types"; +import { + glueListJobs, glueListJobRuns, glueGetJobRun, isTerminalRunState, + type GlueJobRunInfo, type GlueJobRunState, +} from "@/lib/ministack/glue-actions"; +import { cwlStreamEvents, cwlListGroups, type CwlLogEvent } from "@/lib/ministack/browser-actions"; + +const STATE_STYLES: Record = { + SUCCEEDED: "bg-green-500/15 text-green-600 border-green-500/30", + RUNNING: "bg-blue-500/15 text-blue-600 border-blue-500/30", + STARTING: "bg-blue-500/15 text-blue-600 border-blue-500/30", + WAITING: "bg-yellow-500/15 text-yellow-600 border-yellow-500/30", + STOPPING: "bg-yellow-500/15 text-yellow-600 border-yellow-500/30", + STOPPED: "bg-muted text-muted-foreground border-border", + FAILED: "bg-red-500/15 text-red-600 border-red-500/30", + TIMEOUT: "bg-red-500/15 text-red-600 border-red-500/30", + ERROR: "bg-red-500/15 text-red-600 border-red-500/30", +}; + +interface RunsTabProps { + config: MiniStackConfig; + activeJob: string; + onSelectJob: (job: string) => void; +} + +export function RunsTab({ config, activeJob, onSelectJob }: RunsTabProps) { + const [jobNames, setJobNames] = useState([]); + const [runs, setRuns] = useState([]); + const [loading, setLoading] = useState(false); + const [logs, setLogs] = useState([]); + const [streaming, setStreaming] = useState(false); + const [allGroups, setAllGroups] = useState(null); + const [selectedGroup, setSelectedGroup] = useState(""); // "" = all groups + + // List every CloudWatch log group so the user can see/pick where MiniStack + // writes Glue Spark logs (the naming is not guaranteed to be /aws-glue/*). + const loadGroups = useCallback(async () => { + try { + const res = await cwlListGroups(config); + const names = res.groups.map((g) => g.name).filter(Boolean); + setAllGroups(names); + // Default to a glue/spark group when present, otherwise stream all. + const preferred = names.find((n) => /glue|spark/i.test(n)); + setSelectedGroup((cur) => cur || preferred || ""); + } catch { + setAllGroups([]); + } + }, [config]); + + const loadJobs = useCallback(async () => { + try { + const jobs = await glueListJobs(config); + const names = jobs.map((j) => j.name); + setJobNames(names); + if (!activeJob && names.length > 0) onSelectJob(names[0]); + } catch { /* surfaced by run load */ } + }, [config, activeJob, onSelectJob]); + + const loadRuns = useCallback(async () => { + if (!activeJob) { setRuns([]); return; } + setLoading(true); + try { + setRuns(await glueListJobRuns(config, activeJob)); + } catch (e) { + toast.error(e instanceof Error ? e.message : "Failed to list runs"); + } finally { + setLoading(false); + } + }, [config, activeJob]); + + useEffect(() => { loadJobs(); }, [loadJobs]); + useEffect(() => { loadRuns(); }, [loadRuns]); + useEffect(() => { loadGroups(); }, [loadGroups]); + + // Poll while any run is non-terminal. + const hasActiveRun = runs.some((r) => !isTerminalRunState(r.state)); + + // Auto-start log streaming once a run is in flight. + useEffect(() => { + if (hasActiveRun) setStreaming(true); + }, [hasActiveRun]); + const pollRef = useRef(null); + useEffect(() => { + if (!activeJob || !hasActiveRun) return; + const tick = async () => { + try { + const updated = await Promise.all( + runs.map(async (r) => + isTerminalRunState(r.state) ? r : (await glueGetJobRun(config, activeJob, r.id)) ?? r), + ); + setRuns(updated); + } catch { /* keep last known */ } + }; + pollRef.current = window.setInterval(tick, 2500); + return () => { if (pollRef.current) window.clearInterval(pollRef.current); }; + }, [activeJob, hasActiveRun, runs, config]); + + // Tail Spark logs from the selected CloudWatch group (or all discovered + // groups). Uses a 30-minute lookback so a just-finished run is still caught. + useEffect(() => { + if (!streaming) return; + setLogs([]); + const append = (batch: CwlLogEvent[]) => + setLogs((prev) => [...prev, ...batch].sort((a, b) => (a.timestamp ?? 0) - (b.timestamp ?? 0)).slice(-1500)); + + const targets = selectedGroup + ? [selectedGroup] + : (allGroups && allGroups.length > 0 ? allGroups : ["/aws-glue/jobs/output", "/aws-glue/jobs/error"]); + const stops = targets.map((g) => cwlStreamEvents(config, g, undefined, append, 30 * 60 * 1000)); + return () => stops.forEach((s) => s()); + }, [streaming, config, selectedGroup, allGroups]); + + const fmtTime = (sec?: number) => (sec === undefined ? "—" : `${sec}s`); + + return ( +
+
+ + + +
+ +
+ {runs.length === 0 && ( +

{loading ? "Loading…" : "No runs yet — start one from the Jobs tab."}

+ )} + {runs.map((r) => ( +
+
+ {r.id} + + {!isTerminalRunState(r.state) && } + {r.state} + +
+
+ Exec: {fmtTime(r.executionTimeSeconds)} + {r.startedOn && Started: {new Date(r.startedOn).toLocaleTimeString()}} +
+ {r.errorMessage && ( +
+ 
{r.errorMessage}
+ +
+ )} +
+ ))} +
+ +
+
+ + + +
+
+ + +
+
+ {allGroups?.length === 0 && ( +

+ MiniStack doesn't expose Glue Spark logs via CloudWatch. Read raw output with docker logs <container>, or write results to S3 / a catalog table to view them in the consoles. Run status and errors still appear on the run cards above. +

+ )} + +
+ {logs.length === 0 && ( +
+

{streaming ? "Waiting for Spark logs…" : "Click Start to stream Spark logs"}

+ {allGroups !== null && ( + allGroups.length === 0 ? ( +

MiniStack has no CloudWatch log groups at all — it is not shipping Spark stdout to CloudWatch Logs. Read them directly with docker logs <container>.

+ ) : ( +

{allGroups.length} log group{allGroups.length !== 1 ? "s" : ""} available — pick one above if "All" shows nothing.

+ ) + )} +
+ )} + {logs.map((e, i) => ( +
+ {e.timestamp ? new Date(e.timestamp).toISOString().slice(11, 23) : "—"} + {e.message} +
+ ))} +
+ +
+ ); +} + +export type { GlueJobRunState }; diff --git a/src/components/ministack/glue/TablesTab.tsx b/src/components/ministack/glue/TablesTab.tsx new file mode 100644 index 0000000..b084cd6 --- /dev/null +++ b/src/components/ministack/glue/TablesTab.tsx @@ -0,0 +1,218 @@ +"use client"; + +import React, { useState, useEffect, useCallback } from "react"; +import { Table2, Plus, RefreshCw, Loader2, Trash2, Save, ChevronLeft } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { ScrollArea } from "@/components/ui/scroll-area"; +import { cn } from "@/lib/utils"; +import { toast } from "sonner"; +import type { MiniStackConfig } from "@/lib/ministack/types"; +import { + glueGetTables, glueCreateTable, glueUpdateTable, glueDeleteTable, sanitizeGlueName, + type GlueTableInfo, type GlueColumn, type GlueClassification, +} from "@/lib/ministack/glue-actions"; + +const COLUMN_TYPES = ["string", "int", "bigint", "double", "float", "boolean", "timestamp", "date", "array", "struct<>"]; +const CLASSIFICATIONS: GlueClassification[] = ["parquet", "json", "csv"]; + +interface TablesTabProps { + config: MiniStackConfig; + databaseName: string; +} + +function emptyTable(): GlueTableInfo { + return { name: "", columns: [{ name: "id", type: "string" }], partitionKeys: [], classification: "parquet", location: "" }; +} + +function ColumnRows({ + columns, onChange, label, +}: { + columns: GlueColumn[]; + onChange: (cols: GlueColumn[]) => void; + label: string; +}) { + const update = (i: number, patch: Partial) => + onChange(columns.map((c, j) => (j === i ? { ...c, ...patch } : c))); + return ( +
+
+ + +
+ {columns.length === 0 &&

None.

} + {columns.map((col, i) => ( +
+ update(i, { name: e.target.value })} + placeholder="column_name" + className="h-7 flex-1 text-xs rounded border border-border bg-background px-2 font-mono" + /> + + update(i, { comment: e.target.value })} + placeholder="comment" + className="h-7 w-24 text-xs rounded border border-border bg-background px-2" + /> + +
+ ))} +
+ ); +} + +export function TablesTab({ config, databaseName }: TablesTabProps) { + const [tables, setTables] = useState([]); + const [loading, setLoading] = useState(false); + const [editing, setEditing] = useState(null); + const [isNew, setIsNew] = useState(false); + const [saving, setSaving] = useState(false); + + const load = useCallback(async () => { + if (!databaseName) return; + setLoading(true); + try { + setTables(await glueGetTables(config, databaseName)); + } catch (e) { + toast.error(e instanceof Error ? e.message : "Failed to list tables"); + } finally { + setLoading(false); + } + }, [config, databaseName]); + + useEffect(() => { load(); }, [load]); + + const handleSave = async () => { + if (!editing) return; + const table: GlueTableInfo = { ...editing, name: sanitizeGlueName(editing.name) }; + if (!editing.name.trim()) { toast.error("Enter a table name"); return; } + setSaving(true); + try { + if (isNew) await glueCreateTable(config, databaseName, table); + else await glueUpdateTable(config, databaseName, table); + toast.success(`Table "${table.name}" saved`); + setEditing(null); + await load(); + } catch (e) { + toast.error(e instanceof Error ? e.message : "Save failed"); + } finally { + setSaving(false); + } + }; + + const handleDelete = async (name: string) => { + try { + await glueDeleteTable(config, databaseName, name); + toast.success(`Table "${name}" deleted`); + await load(); + } catch (e) { + toast.error(e instanceof Error ? e.message : "Delete failed"); + } + }; + + if (!databaseName) { + return
Select a database in the Catalog tab first.
; + } + + if (editing) { + return ( +
+
+ + +
+ +
+
+
+ + setEditing({ ...editing, name: e.target.value })} + placeholder="my_table" + className="h-7 w-full text-xs rounded border border-border bg-background px-2 font-mono disabled:opacity-60" + /> +
+
+ + +
+
+
+ + setEditing({ ...editing, location: e.target.value })} + placeholder="s3://bucket/prefix/" + className="h-7 w-full text-xs rounded border border-border bg-background px-2 font-mono" + /> +
+ setEditing({ ...editing, columns: cols })} /> + setEditing({ ...editing, partitionKeys: cols })} /> +
+ +
+ ); + } + + return ( +
+
+

+ Tables in {databaseName} +

+
+ + +
+
+ +
+ {tables.length === 0 && ( +

+ {loading ? "Loading…" : "No tables yet — create one."} +

+ )} + {tables.map((t) => ( +
+ + +
+ ))} +
+ +
+ ); +} diff --git a/src/lib/ministack/browser-actions.ts b/src/lib/ministack/browser-actions.ts index 9b2b874..f30bf33 100644 --- a/src/lib/ministack/browser-actions.ts +++ b/src/lib/ministack/browser-actions.ts @@ -8,6 +8,7 @@ import { getSSMClient, getKMSClient, getCloudWatchLogsClient, } from "./client"; import { getDeployDef, sanitizeResourceName } from "./service-map"; +import { deployGlue, teardownGlue } from "./glue-actions"; import type { MiniStackConfig, MiniStackDeployResult, MiniStackNodeState } from "./types"; // S3 @@ -359,6 +360,9 @@ async function deployOne( return { nodeId, status: "deployed", resourceId: apiId, resourceArn: `arn:aws:apigateway:${config.region}::/restapis/${apiId}`, endpoint: `${config.endpoint}/restapis/${apiId}` }; } + case "glue": + return await deployGlue(node, config); + default: return { nodeId, status: "not_supported" }; } @@ -466,6 +470,9 @@ async function teardownOne(node: TeardownNodeInput, config: MiniStackConfig): Pr await getAPIGatewayClient(config).send(new DeleteRestApiCommand({ restApiId: resourceId })); break; } + case "glue": { + return await teardownGlue(node, config); + } } return { nodeId, resourceId, ok: true }; } catch (err) { @@ -791,9 +798,10 @@ export function cwlStreamEvents( logGroupName: string, logStreamName: string | undefined, onBatch: (events: CwlLogEvent[]) => void, + lookbackMs = 2 * 60 * 1000, ): () => void { const cwl = getCloudWatchLogsClient(config); - let lastEventTime = Date.now() - 2 * 60 * 1000; + let lastEventTime = Date.now() - lookbackMs; const poll = async () => { let pageToken: string | undefined; diff --git a/src/lib/ministack/client.ts b/src/lib/ministack/client.ts index db579a7..d76c817 100644 --- a/src/lib/ministack/client.ts +++ b/src/lib/ministack/client.ts @@ -11,6 +11,7 @@ import { SecretsManagerClient } from "@aws-sdk/client-secrets-manager"; import { SSMClient } from "@aws-sdk/client-ssm"; import { KinesisClient } from "@aws-sdk/client-kinesis"; import { KMSClient } from "@aws-sdk/client-kms"; +import { GlueClient } from "@aws-sdk/client-glue"; import type { MiniStackConfig } from "./types"; /** @@ -77,3 +78,6 @@ export const getKinesisClient = (config: MiniStackConfig) => export const getKMSClient = (config: MiniStackConfig) => new KMSClient(baseConfig(config)); + +export const getGlueClient = (config: MiniStackConfig) => + new GlueClient(baseConfig(config)); diff --git a/src/lib/ministack/glue-actions.ts b/src/lib/ministack/glue-actions.ts new file mode 100644 index 0000000..d5bfc4d --- /dev/null +++ b/src/lib/ministack/glue-actions.ts @@ -0,0 +1,508 @@ +// AWS Glue operations for MiniStack — browser-compatible (no Node.js APIs). +// Covers the Data Catalog (databases, tables, partitions) and ETL jobs +// (Spark / PySpark) with script storage in the local S3 emulator. +// MiniStack supports CORS; import freely from any "use client" component. + +import { getGlueClient, getIAMClient, getS3Client } from "./client"; +import type { MiniStackConfig, MiniStackDeployResult } from "./types"; +import type { DeployNodeInput, TeardownNodeInput, TeardownResult } from "./browser-actions"; + +import { + GetDatabasesCommand, GetDatabaseCommand, CreateDatabaseCommand, DeleteDatabaseCommand, + GetTablesCommand, GetTableCommand, CreateTableCommand, UpdateTableCommand, DeleteTableCommand, + GetPartitionsCommand, CreatePartitionCommand, DeletePartitionCommand, + GetJobsCommand, CreateJobCommand, UpdateJobCommand, DeleteJobCommand, + StartJobRunCommand, GetJobRunCommand, GetJobRunsCommand, BatchStopJobRunCommand, + type Column as GlueSdkColumn, type WorkerType, +} from "@aws-sdk/client-glue"; +import { GetRoleCommand, CreateRoleCommand } from "@aws-sdk/client-iam"; +import { HeadBucketCommand, CreateBucketCommand, PutObjectCommand } from "@aws-sdk/client-s3"; + +// ── Constants ─────────────────────────────────────────────────────────────── + +export const GLUE_ROLE_NAME = "openarchflow-glue-role"; +export const GLUE_SCRIPTS_BUCKET = "openarchflow-glue-scripts"; + +// ── Public types ────────────────────────────────────────────────────────────── + +export interface GlueDatabaseInfo { + name: string; + description?: string; +} + +export interface GlueColumn { + name: string; + type: string; + comment?: string; +} + +export type GlueClassification = "parquet" | "json" | "csv"; + +export interface GlueTableInfo { + name: string; + columns: GlueColumn[]; + partitionKeys: GlueColumn[]; + location?: string; + classification?: GlueClassification; +} + +export interface GlueJobInfo { + name: string; + role?: string; + glueVersion?: string; + workerType?: string; + numberOfWorkers?: number; + scriptLocation?: string; + commandName?: string; +} + +export type GlueJobRunState = + | "STARTING" | "RUNNING" | "STOPPING" | "STOPPED" + | "SUCCEEDED" | "FAILED" | "TIMEOUT" | "ERROR" | "WAITING"; + +export interface GlueJobRunInfo { + id: string; + state: GlueJobRunState; + startedOn?: number; + completedOn?: number; + executionTimeSeconds?: number; + errorMessage?: string; +} + +// ── Name sanitization ───────────────────────────────────────────────────────── + +// Glue database / table names allow lowercase letters, digits and underscores +// only — hyphens are rejected, so we cannot reuse sanitizeResourceName here. +export function sanitizeGlueName(label: string, suffix?: string): string { + const base = label + .toLowerCase() + .replace(/[^a-z0-9_]/g, "_") + .replace(/_+/g, "_") + .replace(/^_|_$/g, "") + .slice(0, 50) || "catalog"; + const safeBase = /^[a-z_]/.test(base) ? base : `db_${base}`; + return suffix ? `${safeBase}_${suffix.slice(0, 8)}` : safeBase; +} + +// ── Storage descriptor formats ────────────────────────────────────────────── + +interface SerdeFormat { + inputFormat: string; + outputFormat: string; + serializationLibrary: string; +} + +const SERDE_FORMATS: Record = { + parquet: { + inputFormat: "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat", + outputFormat: "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat", + serializationLibrary: "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe", + }, + json: { + inputFormat: "org.apache.hadoop.mapred.TextInputFormat", + outputFormat: "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", + serializationLibrary: "org.openx.data.jsonserde.JsonSerDe", + }, + csv: { + inputFormat: "org.apache.hadoop.mapred.TextInputFormat", + outputFormat: "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", + serializationLibrary: "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", + }, +}; + +function toSdkColumns(columns: GlueColumn[]): GlueSdkColumn[] { + return columns + .filter((c) => c.name.trim()) + .map((c) => ({ Name: c.name.trim(), Type: c.type || "string", Comment: c.comment || undefined })); +} + +function fromSdkColumns(columns?: GlueSdkColumn[]): GlueColumn[] { + return (columns ?? []).map((c) => ({ name: c.Name ?? "", type: c.Type ?? "string", comment: c.Comment })); +} + +// ── IAM role & scripts bucket ─────────────────────────────────────────────── + +export async function ensureGlueRole(config: MiniStackConfig): Promise { + const iam = getIAMClient(config); + const arn = `arn:aws:iam::${config.accountId}:role/${GLUE_ROLE_NAME}`; + try { + await iam.send(new GetRoleCommand({ RoleName: GLUE_ROLE_NAME })); + return arn; + } catch { /* create below */ } + await iam.send(new CreateRoleCommand({ + RoleName: GLUE_ROLE_NAME, + AssumeRolePolicyDocument: JSON.stringify({ + Version: "2012-10-17", + Statement: [{ Effect: "Allow", Principal: { Service: "glue.amazonaws.com" }, Action: "sts:AssumeRole" }], + }), + })); + return arn; +} + +export async function ensureScriptsBucket(config: MiniStackConfig): Promise { + const s3 = getS3Client(config); + try { + await s3.send(new HeadBucketCommand({ Bucket: GLUE_SCRIPTS_BUCKET })); + } catch { + await s3.send(new CreateBucketCommand({ Bucket: GLUE_SCRIPTS_BUCKET })); + } + return GLUE_SCRIPTS_BUCKET; +} + +// Uploads a PySpark script to the local S3 emulator and returns its s3:// URI. +export async function glueUploadScript( + config: MiniStackConfig, + jobName: string, + code: string, +): Promise { + await ensureScriptsBucket(config); + const key = `${jobName}.py`; + await getS3Client(config).send(new PutObjectCommand({ + Bucket: GLUE_SCRIPTS_BUCKET, + Key: key, + Body: code, + ContentType: "text/x-python", + })); + return `s3://${GLUE_SCRIPTS_BUCKET}/${key}`; +} + +// ── Databases ───────────────────────────────────────────────────────────────── + +export async function glueListDatabases(config: MiniStackConfig): Promise { + const res = await getGlueClient(config).send(new GetDatabasesCommand({})); + return (res.DatabaseList ?? []).map((d) => ({ name: d.Name ?? "", description: d.Description })); +} + +export async function glueCreateDatabase( + config: MiniStackConfig, + name: string, + description?: string, +): Promise { + await getGlueClient(config).send(new CreateDatabaseCommand({ + DatabaseInput: { Name: name, Description: description }, + })); +} + +export async function glueDeleteDatabase(config: MiniStackConfig, name: string): Promise { + await getGlueClient(config).send(new DeleteDatabaseCommand({ Name: name })); +} + +// ── Tables ────────────────────────────────────────────────────────────────── + +export async function glueGetTables( + config: MiniStackConfig, + databaseName: string, +): Promise { + const res = await getGlueClient(config).send(new GetTablesCommand({ DatabaseName: databaseName })); + return (res.TableList ?? []).map((t) => ({ + name: t.Name ?? "", + columns: fromSdkColumns(t.StorageDescriptor?.Columns), + partitionKeys: fromSdkColumns(t.PartitionKeys), + location: t.StorageDescriptor?.Location, + classification: (t.Parameters?.classification as GlueClassification) ?? undefined, + })); +} + +export async function glueGetTable( + config: MiniStackConfig, + databaseName: string, + name: string, +): Promise { + const res = await getGlueClient(config).send(new GetTableCommand({ DatabaseName: databaseName, Name: name })); + const t = res.Table; + if (!t) return null; + return { + name: t.Name ?? "", + columns: fromSdkColumns(t.StorageDescriptor?.Columns), + partitionKeys: fromSdkColumns(t.PartitionKeys), + location: t.StorageDescriptor?.Location, + classification: (t.Parameters?.classification as GlueClassification) ?? undefined, + }; +} + +function buildTableInput(table: GlueTableInfo) { + const classification: GlueClassification = table.classification ?? "parquet"; + const fmt = SERDE_FORMATS[classification]; + return { + Name: table.name, + TableType: "EXTERNAL_TABLE", + Parameters: { classification, EXTERNAL: "TRUE" }, + PartitionKeys: toSdkColumns(table.partitionKeys), + StorageDescriptor: { + Columns: toSdkColumns(table.columns), + Location: table.location || `s3://${GLUE_SCRIPTS_BUCKET}/tables/${table.name}/`, + InputFormat: fmt.inputFormat, + OutputFormat: fmt.outputFormat, + SerdeInfo: { SerializationLibrary: fmt.serializationLibrary }, + }, + }; +} + +export async function glueCreateTable( + config: MiniStackConfig, + databaseName: string, + table: GlueTableInfo, +): Promise { + await getGlueClient(config).send(new CreateTableCommand({ + DatabaseName: databaseName, + TableInput: buildTableInput(table), + })); +} + +export async function glueUpdateTable( + config: MiniStackConfig, + databaseName: string, + table: GlueTableInfo, +): Promise { + await getGlueClient(config).send(new UpdateTableCommand({ + DatabaseName: databaseName, + TableInput: buildTableInput(table), + })); +} + +export async function glueDeleteTable( + config: MiniStackConfig, + databaseName: string, + name: string, +): Promise { + await getGlueClient(config).send(new DeleteTableCommand({ DatabaseName: databaseName, Name: name })); +} + +// ── Partitions ──────────────────────────────────────────────────────────────── + +export async function glueGetPartitions( + config: MiniStackConfig, + databaseName: string, + tableName: string, +): Promise<{ values: string[] }[]> { + const res = await getGlueClient(config).send(new GetPartitionsCommand({ + DatabaseName: databaseName, + TableName: tableName, + })); + return (res.Partitions ?? []).map((p) => ({ values: p.Values ?? [] })); +} + +export async function glueCreatePartition( + config: MiniStackConfig, + databaseName: string, + table: GlueTableInfo, + values: string[], +): Promise { + const classification: GlueClassification = table.classification ?? "parquet"; + const fmt = SERDE_FORMATS[classification]; + const suffix = values.join("/"); + await getGlueClient(config).send(new CreatePartitionCommand({ + DatabaseName: databaseName, + TableName: table.name, + PartitionInput: { + Values: values, + StorageDescriptor: { + Columns: toSdkColumns(table.columns), + Location: `${(table.location || `s3://${GLUE_SCRIPTS_BUCKET}/tables/${table.name}/`).replace(/\/$/, "")}/${suffix}/`, + InputFormat: fmt.inputFormat, + OutputFormat: fmt.outputFormat, + SerdeInfo: { SerializationLibrary: fmt.serializationLibrary }, + }, + }, + })); +} + +export async function glueDeletePartition( + config: MiniStackConfig, + databaseName: string, + tableName: string, + values: string[], +): Promise { + await getGlueClient(config).send(new DeletePartitionCommand({ + DatabaseName: databaseName, + TableName: tableName, + PartitionValues: values, + })); +} + +// ── Jobs ──────────────────────────────────────────────────────────────────── + +export interface GlueJobSpec { + name: string; + scriptLocation: string; + glueVersion: string; // "4.0" | "3.0" + workerType: string; // "G.1X" | "G.2X" + numberOfWorkers: number; + arguments?: Record; +} + +export async function glueListJobs(config: MiniStackConfig): Promise { + const res = await getGlueClient(config).send(new GetJobsCommand({})); + return (res.Jobs ?? []).map((j) => ({ + name: j.Name ?? "", + role: j.Role, + glueVersion: j.GlueVersion, + workerType: j.WorkerType, + numberOfWorkers: j.NumberOfWorkers, + scriptLocation: j.Command?.ScriptLocation, + commandName: j.Command?.Name, + })); +} + +function defaultJobArguments(extra?: Record): Record { + return { + "--job-language": "python", + "--enable-glue-datacatalog": "true", + "--enable-metrics": "true", + // Bookmark persistence is not reliably implemented locally; leaving it + // enabled makes job.commit() throw a py4j Java exception on MiniStack. + "--job-bookmark-option": "job-bookmark-disable", + "--TempDir": `s3://${GLUE_SCRIPTS_BUCKET}/tmp/`, + ...extra, + }; +} + +// Creates the job if it does not exist, otherwise updates it. Returns the role ARN used. +export async function glueUpsertJob( + config: MiniStackConfig, + spec: GlueJobSpec, +): Promise { + const glue = getGlueClient(config); + const roleArn = await ensureGlueRole(config); + const command = { + Name: "glueetl", + ScriptLocation: spec.scriptLocation, + PythonVersion: "3", + }; + const common = { + Role: roleArn, + Command: command, + GlueVersion: spec.glueVersion, + WorkerType: spec.workerType as WorkerType, + NumberOfWorkers: spec.numberOfWorkers, + DefaultArguments: defaultJobArguments(spec.arguments), + MaxRetries: 0, + Timeout: 60, + }; + let exists = false; + try { + const jobs = await glue.send(new GetJobsCommand({})); + exists = (jobs.Jobs ?? []).some((j) => j.Name === spec.name); + } catch { /* assume not present */ } + + if (exists) { + await glue.send(new UpdateJobCommand({ JobName: spec.name, JobUpdate: common })); + } else { + await glue.send(new CreateJobCommand({ Name: spec.name, ...common })); + } + return roleArn; +} + +export async function glueDeleteJob(config: MiniStackConfig, jobName: string): Promise { + await getGlueClient(config).send(new DeleteJobCommand({ JobName: jobName })); +} + +// ── Spark execution ────────────────────────────────────────────────────────── + +export async function glueStartJobRun( + config: MiniStackConfig, + jobName: string, + args?: Record, +): Promise { + // Real Glue auto-injects --JOB_NAME; MiniStack does not, so the script's + // getResolvedOptions(["JOB_NAME"]) fails unless we pass it explicitly. + const res = await getGlueClient(config).send(new StartJobRunCommand({ + JobName: jobName, + Arguments: { "--JOB_NAME": jobName, ...args }, + })); + return res.JobRunId ?? ""; +} + +function toRunInfo(run: { + Id?: string; + JobRunState?: string; + StartedOn?: Date; + CompletedOn?: Date; + ExecutionTime?: number; + ErrorMessage?: string; +}): GlueJobRunInfo { + return { + id: run.Id ?? "", + state: (run.JobRunState as GlueJobRunState) ?? "STARTING", + startedOn: run.StartedOn ? run.StartedOn.getTime() : undefined, + completedOn: run.CompletedOn ? run.CompletedOn.getTime() : undefined, + executionTimeSeconds: run.ExecutionTime, + errorMessage: run.ErrorMessage, + }; +} + +export async function glueGetJobRun( + config: MiniStackConfig, + jobName: string, + runId: string, +): Promise { + const res = await getGlueClient(config).send(new GetJobRunCommand({ JobName: jobName, RunId: runId })); + return res.JobRun ? toRunInfo(res.JobRun) : null; +} + +export async function glueListJobRuns( + config: MiniStackConfig, + jobName: string, +): Promise { + const res = await getGlueClient(config).send(new GetJobRunsCommand({ JobName: jobName })); + return (res.JobRuns ?? []).map(toRunInfo); +} + +export async function glueStopJobRun( + config: MiniStackConfig, + jobName: string, + runId: string, +): Promise { + await getGlueClient(config).send(new BatchStopJobRunCommand({ JobName: jobName, JobRunIds: [runId] })); +} + +const TERMINAL_STATES: GlueJobRunState[] = ["SUCCEEDED", "FAILED", "STOPPED", "TIMEOUT", "ERROR"]; + +export function isTerminalRunState(state: GlueJobRunState): boolean { + return TERMINAL_STATES.includes(state); +} + +// ── Deploy / teardown entry points (called from browser-actions) ───────────── + +export async function deployGlue( + node: DeployNodeInput, + config: MiniStackConfig, +): Promise { + const override = node.nodeConfig?.resourceNameOverride as string | undefined; + const dbName = sanitizeGlueName(override || node.label, node.nodeId); + + // Prepare the Spark prerequisites so the Studio is ready to author jobs. + try { await ensureGlueRole(config); } catch { /* role optional until first job */ } + try { await ensureScriptsBucket(config); } catch { /* bucket created on first upload */ } + + const glue = getGlueClient(config); + try { + await glue.send(new GetDatabaseCommand({ Name: dbName })); + } catch { + await glue.send(new CreateDatabaseCommand({ + DatabaseInput: { Name: dbName, Description: `OpenArchFlow catalog database for ${node.label}` }, + })); + } + return { + nodeId: node.nodeId, + status: "deployed", + resourceId: dbName, + resourceArn: `arn:aws:glue:${config.region}:${config.accountId}:database/${dbName}`, + }; +} + +// Deletes the catalog database (and its tables). ETL jobs are global Glue +// resources and are managed/removed from the Jobs tab, so they are left intact. +export async function teardownGlue( + node: TeardownNodeInput, + config: MiniStackConfig, +): Promise { + const resourceId = node.ministack.resourceId ?? ""; + if (!resourceId) return { nodeId: node.nodeId, resourceId: "", ok: true }; + try { + await getGlueClient(config).send(new DeleteDatabaseCommand({ Name: resourceId })); + return { nodeId: node.nodeId, resourceId, ok: true }; + } catch (err) { + return { nodeId: node.nodeId, resourceId, ok: false, error: err instanceof Error ? err.message : String(err) }; + } +} diff --git a/src/lib/ministack/service-map.ts b/src/lib/ministack/service-map.ts index 164a180..fe75e34 100644 --- a/src/lib/ministack/service-map.ts +++ b/src/lib/ministack/service-map.ts @@ -11,7 +11,8 @@ export type SdkService = | "secretsmanager" | "ssm" | "kms" - | "cloudwatchlogs"; + | "cloudwatchlogs" + | "glue"; export interface ServiceDeployDef { sdkService: SdkService; @@ -119,7 +120,7 @@ export const SERVICE_DEPLOY_MAP: Record = { "global-accelerator": unsupported(), // Analytics - glue: unsupported(), + glue: supported("glue", { requiresIAMRole: true, consoleLabel: "Glue Catalog" }), athena: unsupported(), emr: unsupported(), redshift: unsupported(), diff --git a/src/lib/store.ts b/src/lib/store.ts index 503ca5b..68f1694 100644 --- a/src/lib/store.ts +++ b/src/lib/store.ts @@ -153,6 +153,36 @@ export interface TerraformNodeConfig { customArgs?: Record; } +/** Locally-designed AWS Glue catalog + ETL jobs, persisted with the diagram. */ +export interface GlueColumnConfig { + name: string; + type: string; + comment?: string; +} + +export interface GlueTableConfig { + name: string; + columns: GlueColumnConfig[]; + partitionKeys?: GlueColumnConfig[]; + location?: string; + classification?: "parquet" | "json" | "csv"; +} + +export interface GlueJobConfig { + name: string; + pysparkCode: string; + glueVersion: string; + workerType: string; + numberOfWorkers: number; + arguments?: Record; +} + +export interface GlueNodeConfig { + databaseName?: string; + tables?: GlueTableConfig[]; + jobs?: GlueJobConfig[]; +} + export interface AppNodeData extends Record { label: string; service: string; @@ -166,6 +196,7 @@ export interface AppNodeData extends Record { terraform?: TerraformNodeConfig; }; ministack?: MiniStackNodeState; + glueConfig?: GlueNodeConfig; } export type { MiniStackNodeState, MiniStackConfig }; @@ -342,6 +373,7 @@ interface DiagramState { ministackConfig: MiniStackConfig; setMinistackConfig: (config: Partial) => void; setNodeMinistackState: (nodeId: string, state: Partial) => void; + setNodeGlueConfig: (nodeId: string, glueConfig: Partial) => void; resetNodeMinistackState: (nodeId: string) => void; resetAllMinistackStates: () => void; @@ -1613,6 +1645,34 @@ export const useDiagramStore = create()( }); }, + setNodeGlueConfig: (nodeId, glueConfig) => { + const { activeDiagramId } = get(); + if (!activeDiagramId) return; + set((state) => { + const activeDiagram = state.diagrams[activeDiagramId]; + const newNodes = activeDiagram.nodes.map((node) => { + if (node.id === nodeId) { + const merged = { ...(node.data.glueConfig ?? {}), ...glueConfig } as GlueNodeConfig; + return { + ...node, + data: { ...node.data, glueConfig: merged }, + }; + } + return node; + }); + return { + diagrams: { + ...state.diagrams, + [activeDiagramId]: { + ...activeDiagram, + nodes: newNodes as AppNode[], + lastModified: Date.now(), + }, + }, + }; + }); + }, + resetNodeMinistackState: (nodeId) => { const { activeDiagramId } = get(); if (!activeDiagramId) return; From 68dcdc095b685f33f4682f49e96a523847f4d639 Mon Sep 17 00:00:00 2001 From: Rafael Sales Date: Wed, 24 Jun 2026 06:39:16 -0300 Subject: [PATCH 2/2] chore: bump version to 0.12.1 --- CHANGELOG.md | 7 +++++++ README.md | 2 +- package.json | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 955518e..f8251eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 --- +## [0.12.1] - 2026-06-24 + +### Added + +- AWS Glue studio components and GlueStudioPanel interface. +- PySpark editor and action logs for MiniStack AWS Glue services. + ## [0.12.0] - 2026-06-20 ### Changed diff --git a/README.md b/README.md index 4e32e7b..9696d55 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@

- Version 0.12.0 + Version 0.12.1 MIT License Next.js 16+ AI Providers diff --git a/package.json b/package.json index a78811e..76099fc 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "open-arch-flow", - "version": "0.12.0", + "version": "0.12.1", "private": true, "packageManager": "pnpm@11.1.2", "author": "Rafael Sales ",