Ported go_grader&grades into faasten

Makefile

@@ -29,10 +29,36 @@ prepdb: output/example_grader.tgz output/example_submission.tgz
 	sfdb -b cos316/example/grading_script - < output/example_grader.tgz
 	sfdb -b submission.tgz - < output/example_submission.tgz
 
-run/%: output/%.img payloads/%.jsonl
-	@singlevm --mem_size 1024 --kernel vmlinux-4.20.0 --rootfs python3.ext4 --appfs output/$*.img < payloads/$*.jsonl
+.PHONY: prepfs
+prepfs: output/example_grader.tgz output/example_submission.tgz example_grader/grader_config.json
+	sffs mkdir /gh_repo	--endorse false --secrecy true --integrity gh_repo
+	sffs mkdir /go_grader --endorse false --secrecy true --integrity go_grader
+	sffs mkdir /grades --endorse false --secrecy true --integrity grades
+	sffs mkdir /gh_repo/user --endorse false --secrecy user --integrity gh_repo
+	sffs mkdir /go_grader/user --endorse false --secrecy user --integrity go_grader
+	sffs mkdir /grades/user --endorse false --secrecy user --integrity grades
+	# write /gh_repo/user/submission.tgz
+	sffs mkfile /gh_repo/user/submission.tgz --endorse gh_repo --secrecy user --integrity gh_repo
+	sffs write /gh_repo/user/submission.tgz --endorse gh_repo --file output/example_submission.tgz
+	# write /cos316/example/grading_script & /cos316/example/grader_config.json
+	sffs mkdir /cos316 --endorse false --secrecy true --integrity cos316
+	sffs mkdir /cos316/example --endorse cos316 --secrecy true --integrity cos316
+	sffs mkfile /cos316/example/grading_script --endorse cos316 --secrecy cos316,go_grader --integrity cos316
+	sffs write /cos316/example/grading_script --endorse cos316 --file output/example_grader.tgz
+	sffs mkfile /cos316/example/grader_config.json --endorse cos316 --secrecy true --integrity cos316
+	sffs write /cos316/example/grader_config.json --endorse cos316 --file example_grader/grader_config.json
+
+
+run/%: output/%.img payloads/%.jsonl python3.ext4
+	@singlevm --mem_size 1024 --kernel vmlinux-4.20.0 --rootfs python3.ext4 --appfs output/$*.img --function $* --end_users user < payloads/$*.jsonl
 	@touch $@
 
+.PHONY: debug/%
+debug/%: export RUST_LOG=debug
+debug/%: output/%.img payloads/%.jsonl python3.ext4
+	@singlevm --mem_size 1024 --kernel vmlinux-4.20.0 --rootfs python3.ext4 --appfs output/$*.img --function $* --secrecy user --integrity user --kernel_args "console=ttyS0" \
+		--data test_results:1e38cb34f3a86d9cfadcb735cb92e6869a2ee3f3b952ca30d1d78c5d6ac4e1c0 < payloads/$*.jsonl
+
 .PHONY: clean
 clean:
-	rm -f $(OUTPUTS) $(RUNS)
+	rm -f $(OUTPUTS) $(RUNS) storage/*.mdb

example_grader/grader_config.json

@@ -0,0 +1 @@
+{"tests": {"TestNegate": {"points": 10}}}

functions/go_grader/Makefile

@@ -1,15 +1,16 @@
 SOURCES=workload.py
+GO_VERSION=1.17.10-r0
 
 
 .PHONY: all
 all: out
 
-go-1.17.9-r0.apk:
-	wget https://dl-cdn.alpinelinux.org/alpine/v3.15/community/x86_64/go-1.17.9-r0.apk
+go-$(GO_VERSION).apk:
+	wget https://dl-cdn.alpinelinux.org/alpine/v3.15/community/x86_64/go-$(GO_VERSION).apk
 
-out/usr/lib/go/bin/go: go-1.17.9-r0.apk
+out/usr/lib/go/bin/go: go-$(GO_VERSION).apk
 	mkdir -p out
-	tar -C out -xzf go-1.17.9-r0.apk usr/lib 2> /dev/null
+	tar -C out -xzf go-$(GO_VERSION).apk usr/lib 2> /dev/null
 
 out: ${SOURCES} out/usr/lib/go/bin/go
 	mkdir -p out

functions/go_grader/workload.py

@@ -3,43 +3,44 @@
 import os
 import subprocess
 
-def handle(req, syscall):
+def handle(req, data_handles, syscall):
     args = req["args"]
     workflow = req["workflow"]
     context = req["context"]
-    result = app_handle(args, context, syscall)
+    result, data_handles_out = app_handle(args, context, syscall)
     if len(workflow) > 0:
         next_function = workflow.pop(0)
         syscall.invoke(next_function, json.dumps({
             "args": result,
             "workflow": workflow,
             "context": context
-        }))
+        }), data_handles_out)
     return result
 
-def app_handle(args, state, syscall):
+def app_handle(args, context, syscall):
+    data_handles = dict()
+    secrecy = syscall.get_current_label().secrecy
     os.system("ifconfig lo up")
     # Fetch and untar submission tarball
-    assignment = state["metadata"]["assignment"]
+    assignment = context["metadata"]["assignment"]
     with tempfile.NamedTemporaryFile(suffix=".tar.gz") as submission_tar:
-        submission_tar_data = syscall.read_key(bytes(args["submission"], "utf-8"))
+        submission_tar_data = syscall.fs_read(args['submission'])
         submission_tar.write(submission_tar_data)
         submission_tar.flush()
         with tempfile.TemporaryDirectory() as submission_dir:
-            os.system("mkdir %s" % submission_dir)
             os.system("tar -C %s -xzf %s --strip-components=1" % (submission_dir, submission_tar.name))
 
             # Fetch and untar grading script tarball
             with tempfile.NamedTemporaryFile(suffix=".tar.gz") as script_tar:
-                script_tar_data = syscall.read_key(bytes("cos316/%s/grading_script" % assignment, "utf-8"))
+                script_tar_data = syscall.fs_read('/cos316/%s/grading_script' % assignment)
                 script_tar.write(script_tar_data)
                 script_tar.flush()
                 with tempfile.TemporaryDirectory() as script_dir:
                     os.system("tar -C %s -xzf %s" % (script_dir, script_tar.name))
 
                     # OK, run tests
                     os.putenv("GOCACHE", "%s/.cache" % script_dir)
-                    os.putenv("GOROOT", "/srv/usr/lib/go") 
+                    os.putenv("GOROOT", "/srv/usr/lib/go")
                     os.putenv("SOLUTION_DIR", submission_dir)
                     os.putenv("PATH", "%s:%s" % ("/srv/usr/lib/go/bin", os.getenv("PATH")))
                     os.chdir(script_dir)
@@ -58,12 +59,14 @@ def app_handle(args, state, syscall):
                         if tr["Action"] in ["pass", "fail", "run"]:
                             tr = dict((name.lower(), val) for name, val in tr.items())
                             final_results.append(json.dumps(tr))
-                    key = os.path.join(os.path.splitext(args["submission"])[0], "test_results.jsonl")
-                    syscall.write_key(bytes(key, "utf-8"), bytes('\n'.join(final_results), "utf-8"))
+                    data = bytes('\n'.join(final_results), "utf-8")
+                    with syscall.create_unnamed(len(data)) as handle:
+                        data_handles['test_results'] = handle.finalize(data)
                     testrun.wait()
+                    syscall.declassify(secrecy)
                     if testrun.returncode >= 0:
-                        return { "test_results": key }
+                        return {}, data_handles
                     else:
                         _, errlog = testrun.communicate()
-                        return { "error": { "testrun": str(errlog), "returncode": testrun.returncode } }
-    return {}
+                        return { "error": { "testrun": str(errlog), "returncode": testrun.returncode } }, data_handles
+    return {}, data_handles

functions/grades/workload.py

@@ -1,26 +1,36 @@
 import os
 import json
 
-def handle(req, syscall):
+def handle(req, data_handles, syscall):
     args = req["args"]
     workflow = req["workflow"]
     context = req["context"]
-    result = app_handle(args, context, syscall)
+    result, data_handles_out = app_handle(args, context, data_handles, syscall)
     if len(workflow) > 0:
         next_function = workflow.pop(0)
         syscall.invoke(next_function, json.dumps({
             "args": result,
             "workflow": workflow,
             "context": context
-        }))
+        }), data_handles_out)
     return result
 
-def app_handle(args, context, syscall):
-    test_lines = [ json.loads(line) for line in syscall.read_key(bytes(args["test_results"], "utf-8")).split(b'\n') ]
+def read_all(opened_blob):
+    buf = bytearray()
+    while True:
+        data = opened_blob.read()
+        if len(data) == 0:
+            return buf
+        buf.extend(data)
+    return buf
+
+def app_handle(args, context, data_handles, syscall):
+    with syscall.open_unnamed(data_handles['test_results']) as blob:
+        test_lines = [ json.loads(line) for line in read_all(blob).split(b'\n') ]
     test_runs = dict((line['test'], line) for line in test_lines if 'test' in line)
 
-    grader_config = "cos316/%s/grader_config" % context["metadata"]["assignment"]
-    config = json.loads(syscall.read_key(bytes(grader_config, "utf-8")))
+    grader_config = "/cos316/%s/grader_config.json" % context["metadata"]["assignment"]
+    config = json.loads(syscall.fs_read(grader_config))
 
     total_points = sum([ test["points"] for test in config["tests"].values() if "extraCredit" not in test or not test["extraCredit"]])
 
@@ -45,10 +55,8 @@ def app_handle(args, context, syscall):
         "push_date": context["push_date"]
     }
 
-    key = os.path.join(os.path.dirname(args["test_results"]),"grade.json")
-    syscall.write_key(bytes(key, "utf-8"), bytes(json.dumps(output), "utf-8"))
+    assignment = context['metadata']['assignment']
+    with syscall.create_unnamed() as blob:
+        handle = blob.finalize(bytes(json.dumps(output), "utf-8"))
 
-    return {
-        "grade": points / total_points,
-        "grade_report": key
-        }
+    return {"grade": points / total_points}, {"grade_report": handle}

payloads/go_grader.jsonl

@@ -1 +1 @@
-{"args": {"submission": "submission.tgz"}, "workflow": [ "grades", "generate_report" ], "context": { "metadata": {"assignment": "example"}}}
+{"args": {"submission": "/gh_repo/user/submission.tgz"}, "workflow": ["grades"], "context": { "metadata": {"assignment": "example"}}}

payloads/grades.jsonl

@@ -1 +1 @@
-{"args": {"test_results": "github/cos316/assignment2-starter-code/b541c851d79edad1d05fc64c1bcca88800703a30/test_results.jsonl"}, "workflow": [ "generate_report" ], "context": {"commit": "b541c851d79edad1d05fc64c1bcca88800703a30", "metadata": {"assignment": "assignment2"}, "push_date": 1642798607, "repository": "cos316/assignment2-starter-code"}}
+{"args": {"test_results": "/go_grader/user/submission/test_results.jsonl"}, "workflow": [], "context": {"commit": "b541c851d79edad1d05fc64c1bcca88800703a30", "metadata": {"assignment": "example"}, "push_date": 1642798607, "repository": "cos316/assignment2-starter-code"}}

test_ported_functions.sh

@@ -0,0 +1,46 @@
+SNAPFAAS=https://api.github.com/repos/princeton-sns/snapfaas/tarball/faasten
+IMAGES=https://api.github.com/repos/princeton-sns/snapfaas-images/tarball/faasten
+WD=$(dirname $(realpath $0))
+cd $WD
+
+echo '***********************************'
+echo '*    building faasten binaries    *'
+echo '***********************************'
+mkdir -p snapfaas
+curl -sL -H "Accept: application/vnd.github.v3+json" $SNAPFAAS --output snapfaas.tgz
+tar xzf snapfaas.tgz --strip-components=1 -C snapfaas
+cd snapfaas/snapfaas; cargo build --bins; cd $WD
+export PATH=snapfaas/target/debug:$PATH
+if [ ! command -v sffs 2>/dev/null ] || [ ! command -v singlevm 2>/dev/null ] || [ ! command -v firerunner 2>/dev/null ]; then
+    exit 1
+fi
+
+echo '************************'
+echo '*    getting images    *'
+echo '************************'
+# download the uncompressed kernel
+cp snapfaas/resources/images/vmlinux-4.20.0 .
+# build the python3.ext4
+mkdir -p rootfs
+curl -sL -H "Accept: application/vnd.github.v3+json" $IMAGES | \
+tar xzf - -C rootfs --strip-components=2 --wildcards '*/rootfs/common' '*/rootfs/faasten'
+cd rootfs/faasten; ./mk_rtimage.sh --local-snapfaas $(realpath ../../snapfaas) python3 ../../python3.ext4; cd $WD
+if [ ! -f python3.ext4 ] || [ ! -f vmlinux-4.20.0 ]; then
+    exit 1
+fi
+
+echo '*****************'
+echo '*    testing    *'
+echo '*****************'
+make clean
+make prepfs
+make run/go_grader
+make run/grades
+
+#echo '*********************'
+#echo '*    cleaning up    *'
+#echo '*********************'
+#rm -r snapfaas
+#rm -rf rootfs
+#rm vmlinux-4.20.0
+#rm python3.ext4