- Works e2e no verification.

Author: general-kroll-4-life

cicd/keys/.gitignore

@@ -1,4 +1,5 @@
 *
 !.gitignore
 !integration/
-!work/
+!work/
+!testing/

cicd/keys/testing/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

docs/walkthroughs/get-google-vms.md

@@ -0,0 +1,58 @@
+
+## Setup
+
+First, create a google service account key using the GCP Console, per [the GCP documentation](https://cloud.google.com/iam/docs/keys-create-delete).  Grant the service account at least `Viewer` role equivalent privileges, per [the GCP dumentation](https://cloud.google.com/iam/docs/create-service-agents#grant-roles).
+
+Then, do this in bash:
+
+```bash setup stackql-shell
+
+export GOOGLE_CREDENTIALS="$(cat cicd/keys/testing/google-credentials.json)";
+
+stackql shell --approot=./test/tmp/.get-google-vms.stackql
+```
+
+## Method
+
+Do this in the `stackql` shell, replacing `<project>` with your GCP project name:
+
+```sql stackql-shell input required project=ryuki-it-sandbox-01
+
+registry pull google;
+
+select 
+  name, 
+  id 
+FROM google.compute.instances 
+WHERE 
+  project = 'ryuki-it-sandbox-01' 
+  AND zone = 'australia-southeast1-a'
+;
+
+```
+
+## Result
+
+
+You will see something very much like this included in the output, presuming you have one VM (if you have zero, only the headers should appper, more VMs means more rows):
+
+```sql stackql stdout table-contains-data
+|--------------------------------------------------|---------------------|
+|                       name                       |         id          |
+|--------------------------------------------------|---------------------|
+| any-compute-cluster-1-default-abcd-00000001-0001 | 1000000000000000001 |
+|--------------------------------------------------|---------------------|
+```
+
+<!---  STDERR_REGEX_EXACT
+google\ provider,\ version\ 'v24.11.00274'\ successfully\ installed
+goodbye
+-->
+
+## Cleanup
+
+```bash teardown best-effort
+
+rm -rf ./test/tmp/.get-google-vms.stackql
+
+```

test/python/markdown_testing/markdown_testing.py

@@ -2,10 +2,26 @@
 
 from typing import List, Tuple
 
-import subprocess, os, sys
+import subprocess, os, sys, shutil, io
+
+import json
+
+_REPOSITORY_ROOT_PATH = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', '..'))
+
+"""
+Intentions:
+
+  - Support markdown parsing.
+  - Support sequential markdown code block execution, leveraging [info strings](https://spec.commonmark.org/0.30/#info-string).
+"""
 
 class ASTNode(object):
 
+    _STACKQL_SHELL_INVOCATION: str = 'stackql-shell'
+    _BASH: str = 'bash'
+    _SETUP: str = 'setup'
+    _TEARDOWN: str = 'teardown'
+
     def __init__(self, node: dict):
         self.node = node
         self.children = []
@@ -17,38 +33,44 @@ def get_type(self) -> str:
         return self.node.get('type', '')
 
     def get_text(self) -> str:
-        return self.node.get('text', '')
+        return self.node.get('raw', '').strip()
 
     def is_executable(self) -> bool:
-        return self.get_type() == 'code_block'
+        return self.get_type() == 'block_code'
+    
+    def _get_annotations(self) -> List[str]:
+        return self.node.get('attrs').get('info', '').split(' ')
+    
+    def is_stackql_shell_invocation(self) -> bool:
+        return self._STACKQL_SHELL_INVOCATION in self._get_annotations()
+    
+    def is_bash(self) -> bool:
+        return self._BASH in self._get_annotations()
+    
+    def is_setup(self) -> bool:
+        return self._SETUP in self._get_annotations()
+    
+    def is_teardown(self) -> bool:
+        return self._TEARDOWN in self._get_annotations()
 
     def get_execution_language(self) -> str:
         return self.node.get('lang', '')
 
+    def __str__(self):
+        return json.dumps(self.node, indent=2)
+    
+    def __repr__(self):
+        return self.__str__()
 
 class MdParser(object):
 
     def parse_markdown_file(self, file_path: str, lang=None) -> List[ASTNode]:
         markdown: mistune.Markdown = mistune.create_markdown(renderer='ast')
         with open(file_path, 'r') as f:
             txt = f.read()
-        return markdown(txt)
-
-
-class MdExecutor(object):
+        raw_list: List[dict] = markdown(txt)
+        return [ASTNode(node) for node in raw_list]
 
-    def __init__(self, renderer: MdParser):
-        self.renderer = renderer
-
-    def execute(self, file_path: str) -> None:
-        ast = self.renderer.parse_markdown_file(file_path)
-        for node in ast:
-            if node.is_executable():
-                lang = node.get_execution_language()
-                if lang == 'python':
-                    exec(node.get_text())
-                else:
-                    print(f'Unsupported language: {lang}')
 
 class WorkloadDTO(object):
 
@@ -65,20 +87,75 @@ def get_in_session(self) -> List[str]:
 
     def get_teardown(self) -> List[str]:
         return self._teardown
+    
+    def __str__(self):
+        return f'Setup: {self._setup}\nIn Session: {self._in_session}\nTeardown: {self._teardown}'
+    
+    def __repr__(self):
+        return self.__str__()
+
+class MdOrchestrator(object):
+
+    def __init__(
+        self,
+        parser: MdParser, 
+        max_setup_blocks: int = 1,
+        max_invocations_blocks: int = 1,
+        max_teardown_blocks: int = 1,
+        setup_contains_shell_invocation: bool = True
+    ):
+        self._parser = parser
+        self._max_setup_blocks = max_setup_blocks
+        self._max_invocations_blocks = max_invocations_blocks
+        self._max_teardown_blocks = max_teardown_blocks
+        self._setup_contains_shell_invocation = setup_contains_shell_invocation
+
+    def orchestrate(self, file_path: str) -> WorkloadDTO:
+        setup_count: int = 0
+        teardown_count: int = 0
+        invocation_count: int = 0
+        ast = self._parser.parse_markdown_file(file_path)
+        print(f'AST: {ast}')
+        setup_str: str = f'cd {_REPOSITORY_ROOT_PATH};\n'
+        in_session_commands: List[str] = []
+        teardown_str: str = f'cd {_REPOSITORY_ROOT_PATH};\n'
+        for node in ast:
+            if node.is_executable():
+                if node.is_setup():
+                    if setup_count < self._max_setup_blocks:
+                        setup_str += f'{node.get_text()}'
+                        setup_count += 1
+                    else:
+                        raise KeyError(f'Maximum setup blocks exceeded: {self._max_setup_blocks}')
+                elif node.is_teardown():
+                    if teardown_count < self._max_teardown_blocks:
+                        teardown_str += f'{node.get_text()}'
+                        teardown_count += 1
+                    else:
+                        raise KeyError(f'Maximum teardown blocks exceeded: {self._max_teardown_blocks}')
+                elif node.is_stackql_shell_invocation():
+                    if invocation_count < self._max_invocations_blocks:
+                        all_commands: str = node.get_text().split('\n\n')
+                        in_session_commands += all_commands
+                        invocation_count += 1
+                    else:
+                        raise KeyError(f'Maximum invocation blocks exceeded: {self._max_invocations_blocks}')
+        return WorkloadDTO(setup_str, in_session_commands, teardown_str)
 
 class SimpleE2E(object):
 
     def __init__(self, workload: WorkloadDTO):
         self._workload = workload
 
     def run(self) -> Tuple[bytes, bytes]:
+        bash_path = shutil.which('bash')
         pr: subprocess.Popen = subprocess.Popen(
             self._workload.get_setup(),
             stdin=subprocess.PIPE,
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
             shell=True,
-            executable='/bin/bash'
+            executable=bash_path
         )
         for cmd in self._workload.get_in_session():
             pr.stdin.write(f"{cmd}\n".encode(sys.getdefaultencoding()))
@@ -87,16 +164,11 @@ def run(self) -> Tuple[bytes, bytes]:
         return (stdoout_bytes, stderr_bytes,)
 
 if __name__ == '__main__':
-    workload_dto: WorkloadDTO = WorkloadDTO(
-        setup="""
-            export GOOGLE_CREDENTIALS="$(cat /Users/admin/stackql/secrets/concerted-testing/google-credentials.json)" && 
-            stackql shell""",
-        in_session=[
-            'registry pull google;',
-            'select name, id FROM google.compute.instances WHERE project = \'ryuki-it-sandbox-01\' AND zone = \'australia-southeast1-a\';'
-        ],
-        teardown='echo "Goodbye, World!"'
-    )
+    md_parser = MdParser()
+    orchestrator: MdOrchestrator = MdOrchestrator(md_parser)
+    workload_dto: WorkloadDTO = orchestrator.orchestrate(os.path.join(_REPOSITORY_ROOT_PATH, 'docs', 'walkthroughs', 'get-google-vms.md'))
+    print(f'Workload DTO: {workload_dto}')
+    # print(json.dumps(parsed_file, indent=2))
     e2e: SimpleE2E = SimpleE2E(workload_dto)
     stdout_bytes, stderr_bytes = e2e.run()
     print(stdout_bytes.decode(sys.getdefaultencoding()))

test/tmp/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore