From 19014efebf897d0828219e05cb91f6429dd7cc4b Mon Sep 17 00:00:00 2001 From: Olmo Kramer Date: Tue, 7 Nov 2017 00:54:53 +0100 Subject: [PATCH 1/6] Use Directory subclasses instead of DirTypes enum --- cgfs.py | 261 +++++++++++++++++++++++++++++--------------------------- 1 file changed, 135 insertions(+), 126 deletions(-) diff --git a/cgfs.py b/cgfs.py index be504b2b..fb9751dd 100755 --- a/cgfs.py +++ b/cgfs.py @@ -69,12 +69,8 @@ def wrap_string(string, prefix, max_len): return ('\n').join(res), len(res) - 1 -class DirTypes(IntEnum): - FSROOT = 0 - COURSE = 1 - ASSIGNMENT = 2 - SUBMISSION = 3 - REGDIR = 4 +def split_path(path): + return [x for x in path.split('/') if x] class BaseFile(): @@ -109,10 +105,9 @@ def setattr(self, key, value): class Directory(BaseFile): - def __init__(self, data, name=None, type=DirTypes.REGDIR, writable=False): + def __init__(self, data, name=None, writable=False): super(Directory, self).__init__(data, name) - self.type = type self.writable = writable self.children = {} self.children_loaded = False @@ -149,6 +144,105 @@ def read(self): return res +class RootDirectory(Directory): + def load_courses(self): + for course in cgapi.get_courses(): + course_dir = CourseDirectory(course) + course_dir.getattr() + course_dir.insert_assignments(course['assignments']) + self.insert(course_dir) + self.children_loaded = True + + +class CourseDirectory(Directory): + def insert_assignments(self, assignments): + for assig in assignments: + assig_dir = AssignmentDirectory(assig) + assig_dir.getattr() + assig_dir.insert(AssignmentSettingsFile(cgapi, assig['id'])) + assig_dir.insert( + RubricEditorFile( + cgapi, assig['id'], self.rubric_append_only + ) + ) + assig_dir.insert(HelpFile(RubricEditorFile)) + assig_dir.insert( + SpecialFile( + '.cg-assignment-id', + data=str(assig['id']).encode() + b'\n' + ) + ) + self.insert(assig_dir) + self.children_loaded = True + + +class AssignmentDirectory(Directory): + def load_submissions(self, latest_only): + try: + submissions = cgapi.get_submissions(self.id) + except CGAPIException as e: # pragma: no cover + handle_cgapi_exception(e) + + seen = set() + + for sub in submissions: + if sub['user']['id'] in seen: + continue + + sub_dir = SubmissionDirectory( + sub, + name=sub['user']['name'] + ' - ' + sub['created_at'], + writable=True + ) + + if latest_only: + seen.add(sub['user']['id']) + + sub_dir.getattr() + sub_dir.insert(RubricSelectFile(cgapi, sub['id'], sub['user'])) + sub_dir.insert(GradeFile(cgapi, sub['id'])) + sub_dir.insert(FeedbackFile(cgapi, sub['id'])) + self.insert(sub_dir) + + self.children_loaded = True + + +class SubmissionDirectory(Directory): + def get_full_path(self, path): + parts = split_path(path) if isinstance(path, str) else path + return self.tld + '/' + '/'.join(parts) + + def load_files(self, submission): + try: + files = cgapi.get_submission_files(self.id) + except CGAPIException as e: + handle_cgapi_exception(e) + + def insert_tree(dir, tree): + for item in tree['entries']: + if 'entries' in item: + new_dir = RegularDirectory(item, writable=self.writable) + new_dir.getattr() + dir.insert(new_dir) + insert_tree(new_dir, item) + else: + dir.insert(File(item)) + dir.children_loaded = True + + insert_tree(self, files) + self.insert( + SpecialFile( + '.cg-submission-id', data=str(submission.id).encode() + b'\n' + ) + ) + self.tld = files['name'] + self.children_loaded = True + + +class RegularDirectory(Directory): + pass + + class TempDirectory(Directory): def __init__(self, *args, **kwargs): super(TempDirectory, self).__init__(*args, **kwargs) @@ -1143,11 +1237,11 @@ def __init__( self.rubric_append_only = rubric_append_only - self.files = Directory( + self.files = RootDirectory( { 'id': None, 'name': 'root' - }, type=DirTypes.FSROOT + } ) with self._lock: @@ -1166,119 +1260,31 @@ def strippath(self, path): path = os.path.abspath(path) return path[len(self.mountpoint):] - def load_courses(self): - for course in cgapi.get_courses(): - assignments = course['assignments'] - - course_dir = Directory(course, type=DirTypes.COURSE) - course_dir.getattr() - self.files.insert(course_dir) - - for assig in assignments: - assig_dir = Directory(assig, type=DirTypes.ASSIGNMENT) - assig_dir.getattr() - course_dir.insert(assig_dir) - assig_dir.insert(AssignmentSettingsFile(cgapi, assig['id'])) - assig_dir.insert( - RubricEditorFile( - cgapi, assig['id'], self.rubric_append_only - ) - ) - assig_dir.insert(HelpFile(RubricEditorFile)) - assig_dir.insert( - SpecialFile( - '.cg-assignment-id', - data=str(assig['id']).encode() + b'\n' - ) - ) - course_dir.children_loaded = True - self.files.children_loaded = True - - def load_submissions(self, assignment): - try: - submissions = cgapi.get_submissions(assignment.id) - except CGAPIException as e: # pragma: no cover - handle_cgapi_exception(e) - - seen = set() - - for sub in submissions: - if sub['user']['id'] in seen: - continue - - sub_dir = Directory( - sub, - name=sub['user']['name'] + ' - ' + sub['created_at'], - type=DirTypes.SUBMISSION, - writable=True - ) - - if self.latest_only: - seen.add(sub['user']['id']) - - sub_dir.getattr() - sub_dir.insert(RubricSelectFile(cgapi, sub['id'], sub['user'])) - sub_dir.insert(GradeFile(cgapi, sub['id'])) - sub_dir.insert(FeedbackFile(cgapi, sub['id'])) - assignment.insert(sub_dir) - - assignment.children_loaded = True - - def insert_tree(self, dir, tree): - for item in tree['entries']: - if 'entries' in item: - new_dir = Directory(item, writable=True) - new_dir.getattr() - dir.insert(new_dir) - self.insert_tree(new_dir, item) - else: - dir.insert(File(item)) - dir.children_loaded = True - - def load_submission_files(self, submission): - try: - files = cgapi.get_submission_files(submission.id) - except CGAPIException as e: - handle_cgapi_exception(e) - self.insert_tree(submission, files) - submission.insert( - SpecialFile( - '.cg-submission-id', data=str(submission.id).encode() + b'\n' - ) - ) - submission.tld = files['name'] - submission.children_loaded = True - - def split_path(self, path): - return [x for x in path.split('/') if x] - def get_submission(self, path): - parts = self.split_path(path) + parts = split_path(path) submission = self.get_file(parts[:3]) - try: - submission.tld - except AttributeError: - self.load_submission_files(submission) + + if not isinstance(submission, SubmissionDirectory): + raise FuseOSError(ENOENT) + if not submission.children_loaded: + submission.load_submission_files() return submission def get_file(self, path, start=None, expect_type=None): file = start if start is not None else self.files - parts = self.split_path(path) if isinstance(path, str) else path + parts = split_path(path) if isinstance(path, str) else path for part in parts: if part == '': # pragma: no cover continue try: - if not any( - not isinstance(f, SpecialFile) - for f in file.children.values() - ): - if file.type == DirTypes.ASSIGNMENT: - self.load_submissions(file) - elif file.type == DirTypes.SUBMISSION: - self.load_submission_files(file) + if not file.children_loaded: + if isinstance(file, AssignmentDirectory): + file.load_submissions() + elif isinstance(file, SubmissionDirectory): + file.load_files() except AttributeError: # pragma: no cover if not isinstance(file, Directory): raise FuseOSError(ENOTDIR) @@ -1290,7 +1296,10 @@ def get_file(self, path, start=None, expect_type=None): if expect_type is not None: if not isinstance(file, expect_type): - raise FuseOSError(EISDIR) + if issubclass(expect_type, Directory): + raise FuseOSError(ENOTDIR) + elif issubclass(expect_type, SingleFile): + raise FuseOSError(EISDIR) return file @@ -1308,7 +1317,7 @@ def create(self, path, mode): return self._create(path, mode) def _create(self, path, mode): - parts = self.split_path(path) + parts = split_path(path) if len(parts) <= 3: raise FuseOSError(EPERM) @@ -1318,7 +1327,7 @@ def _create(self, path, mode): assert fname not in parent.children submission = self.get_submission(path) - query_path = submission.tld + '/' + '/'.join(parts[3:]) + query_path = submission.get_full_path(parts[3:]) if self.fixed: file = TempFile(fname, self._tmpdir) @@ -1363,7 +1372,7 @@ def getattr(self, path, fh=None): def _getattr(self, path, fh): if fh is None: - parts = self.split_path(path) + parts = split_path(path) file = self.get_file(parts) else: file = self._open_files[fh] @@ -1377,7 +1386,7 @@ def _getattr(self, path, fh): except CGAPIException as e: handle_cgapi_exception(e) - query_path = submission.tld + '/' + '/'.join(parts[3:]) + query_path = submission.get_full_path(parts[3:]) if isinstance(file, Directory): query_path += '/' @@ -1403,7 +1412,7 @@ def mkdir(self, path, mode): return self._mkdir(path, mode) def _mkdir(self, path, mode): - parts = self.split_path(path) + parts = split_path(path) parent = self.get_dir(parts[:-1]) dname = parts[-1] @@ -1415,7 +1424,7 @@ def _mkdir(self, path, mode): parent.insert(TempDirectory({}, name=dname, writable=True)) else: submission = self.get_submission(path) - query_path = submission.tld + '/' + '/'.join(parts[3:]) + '/' + query_path = submission.get_full_path(parts[3:]) + '/' ddata = cgapi.create_file(submission.id, query_path) parent.insert(Directory(ddata, name=dname, writable=True)) @@ -1425,7 +1434,7 @@ def open(self, path, flags): return self._open(path, flags) def _open(self, path, flags): - parts = self.split_path(path) + parts = split_path(path) parent = self.get_dir(parts[:-1]) file = self.get_file(parts[-1], start=parent, expect_type=SingleFile) @@ -1453,9 +1462,9 @@ def readdir(self, path, fh): dir = self.get_dir(path) if not dir.children_loaded: - if dir.type == DirTypes.ASSIGNMENT: + if isinstance(dir, AssignmentDirectory): self.load_submissions(dir) - elif dir.type == DirTypes.SUBMISSION: + elif isinstance(dir, SubmissionDirectory): self.load_submission_files(dir) return dir.read() @@ -1478,14 +1487,14 @@ def rename(self, old, new): self._rename(old, new) def _rename(self, old, new): - old_parts = self.split_path(old) + old_parts = split_path(old) old_parent = self.get_dir(old_parts[:-1]) file = self.get_file(old_parts[-1], start=old_parent) if isinstance(file, SpecialFile): raise FuseOSError(EPERM) - new_parts = self.split_path(new) + new_parts = split_path(new) new_parent = self.get_dir(new_parts[:-1]) if new_parts[-1] in new_parent.children: @@ -1498,7 +1507,7 @@ def _rename(self, old, new): if submission.id != self.get_submission(new).id: raise FuseOSError(EPERM) - new_query_path = submission.tld + '/' + '/'.join(new_parts[3:]) + '/' + new_query_path = submission.full_path(new_parts[3:]) + '/' if not isinstance(file, (TempDirectory, TempFile)): if self.fixed: @@ -1520,11 +1529,11 @@ def rmdir(self, path): self._rmdir(path) def _rmdir(self, path): - parts = self.split_path(path) + parts = split_path(path) parent = self.get_dir(parts[:-1]) dir = self.get_file(parts[-1], start=parent) - if dir.type != DirTypes.REGDIR: + if not isinstance(dir, RegularDirectory): raise FuseOSError(EPERM) if dir.children: raise FuseOSError(ENOTEMPTY) @@ -1573,7 +1582,7 @@ def truncate(self, path, length, fh=None): def unlink(self, path): with self._lock: - parts = self.split_path(path) + parts = split_path(path) parent = self.get_dir(parts[:-1]) fname = parts[-1] file = self.get_file(fname, start=parent, expect_type=SingleFile) From 34a8a524068efed40cf9296634286c80e51c9d21 Mon Sep 17 00:00:00 2001 From: Olmo Kramer Date: Tue, 7 Nov 2017 01:25:23 +0100 Subject: [PATCH 2/6] Add is_dir argument to SubmissionDirectory.get_full_path --- cgfs.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/cgfs.py b/cgfs.py index fb9751dd..7b612841 100755 --- a/cgfs.py +++ b/cgfs.py @@ -208,9 +208,12 @@ def load_submissions(self, latest_only): class SubmissionDirectory(Directory): - def get_full_path(self, path): parts = split_path(path) if isinstance(path, str) else path - return self.tld + '/' + '/'.join(parts) + def get_full_path(self, path, is_dir=False): + full_path = self.tld + '/' + '/'.join(parts) + if (is_dir): + full_path += '/' + return full_path def load_files(self, submission): try: @@ -1424,7 +1427,7 @@ def _mkdir(self, path, mode): parent.insert(TempDirectory({}, name=dname, writable=True)) else: submission = self.get_submission(path) - query_path = submission.get_full_path(parts[3:]) + '/' + query_path = submission.get_full_path(parts[3:], is_dir=True) ddata = cgapi.create_file(submission.id, query_path) parent.insert(Directory(ddata, name=dname, writable=True)) From dd26feb60023262ff9ce32b841f4c1075fae1a7d Mon Sep 17 00:00:00 2001 From: Olmo Kramer Date: Tue, 7 Nov 2017 01:27:16 +0100 Subject: [PATCH 3/6] Move parts of get_file function to Directory class --- cgfs.py | 53 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/cgfs.py b/cgfs.py index 7b612841..8939bf2d 100755 --- a/cgfs.py +++ b/cgfs.py @@ -70,6 +70,8 @@ def wrap_string(string, prefix, max_len): def split_path(path): + if isinstance(path, list): + return path return [x for x in path.split('/') if x] @@ -138,6 +140,29 @@ def pop(self, filename): return file + def get_file(self, path): + parts = split_path(path) + name = parts[0] + parts = parts[1:] + + if not self.children_loaded: + if isinstance(self, AssignmentDirectory): + self.load_submissions() + elif isinstance(self, SubmissionDirectory): + self.load_files() + + if not name in self.children: + raise FuseOSError(ENOENT) + + child = self.children[name] + + if parts: + if not isinstance(child, Directory): + raise FuseOSError(ENOTDIR) + return child.get_file(parts) + else: + return child + def read(self): res = list(self.children) res.extend(['.', '..']) @@ -208,8 +233,8 @@ def load_submissions(self, latest_only): class SubmissionDirectory(Directory): - parts = split_path(path) if isinstance(path, str) else path def get_full_path(self, path, is_dir=False): + parts = split_path(path) full_path = self.tld + '/' + '/'.join(parts) if (is_dir): full_path += '/' @@ -1275,27 +1300,13 @@ def get_submission(self, path): return submission def get_file(self, path, start=None, expect_type=None): - file = start if start is not None else self.files - parts = split_path(path) if isinstance(path, str) else path - - for part in parts: - if part == '': # pragma: no cover - continue + dir = start if start is not None else self.files - try: - if not file.children_loaded: - if isinstance(file, AssignmentDirectory): - file.load_submissions() - elif isinstance(file, SubmissionDirectory): - file.load_files() - except AttributeError: # pragma: no cover - if not isinstance(file, Directory): - raise FuseOSError(ENOTDIR) - raise + if not isinstance(dir, Directory): + raise FuseOSError(ENOTDIR) - if part not in file.children or file.children[part] is None: - raise FuseOSError(ENOENT) - file = file.children[part] + parts = split_path(path) + file = dir.get_file(parts, expect_type) if expect_type is not None: if not isinstance(file, expect_type): @@ -1304,8 +1315,6 @@ def get_file(self, path, start=None, expect_type=None): elif issubclass(expect_type, SingleFile): raise FuseOSError(EISDIR) - return file - def get_dir(self, path, start=None): return self.get_file(path, start=start, expect_type=Directory) From 9be5a2cc2166fbe2490a39a1bf50c806a24e1963 Mon Sep 17 00:00:00 2001 From: Olmo Kramer Date: Tue, 7 Nov 2017 21:15:06 +0100 Subject: [PATCH 4/6] Rename global cgapi to __cgapi__ --- cgfs.py | 75 +++++++++++++++++++++++++++------------------------------ 1 file changed, 35 insertions(+), 40 deletions(-) diff --git a/cgfs.py b/cgfs.py index 8939bf2d..52bb4398 100755 --- a/cgfs.py +++ b/cgfs.py @@ -93,7 +93,7 @@ def getattr(self, submission=None, path=None): } if submission is not None and path is not None: - stat = cgapi.get_file_meta(submission.id, path) + stat = __cgapi__.get_file_meta(submission.id, path) self.stat['st_size'] = stat['size'] self.stat['st_mtime'] = stat['modification_date'] @@ -171,7 +171,7 @@ def read(self): class RootDirectory(Directory): def load_courses(self): - for course in cgapi.get_courses(): + for course in __cgapi__.get_courses(): course_dir = CourseDirectory(course) course_dir.getattr() course_dir.insert_assignments(course['assignments']) @@ -184,10 +184,10 @@ def insert_assignments(self, assignments): for assig in assignments: assig_dir = AssignmentDirectory(assig) assig_dir.getattr() - assig_dir.insert(AssignmentSettingsFile(cgapi, assig['id'])) + assig_dir.insert(AssignmentSettingsFile(assig['id'])) assig_dir.insert( RubricEditorFile( - cgapi, assig['id'], self.rubric_append_only + assig['id'], self.rubric_append_only ) ) assig_dir.insert(HelpFile(RubricEditorFile)) @@ -204,7 +204,7 @@ def insert_assignments(self, assignments): class AssignmentDirectory(Directory): def load_submissions(self, latest_only): try: - submissions = cgapi.get_submissions(self.id) + submissions = __cgapi__.get_submissions(self.id) except CGAPIException as e: # pragma: no cover handle_cgapi_exception(e) @@ -224,9 +224,9 @@ def load_submissions(self, latest_only): seen.add(sub['user']['id']) sub_dir.getattr() - sub_dir.insert(RubricSelectFile(cgapi, sub['id'], sub['user'])) - sub_dir.insert(GradeFile(cgapi, sub['id'])) - sub_dir.insert(FeedbackFile(cgapi, sub['id'])) + sub_dir.insert(RubricSelectFile(sub['id'], sub['user'])) + sub_dir.insert(GradeFile(sub['id'])) + sub_dir.insert(FeedbackFile(sub['id'])) self.insert(sub_dir) self.children_loaded = True @@ -242,7 +242,7 @@ def get_full_path(self, path, is_dir=False): def load_files(self, submission): try: - files = cgapi.get_submission_files(self.id) + files = __cgapi__.get_submission_files(self.id) except CGAPIException as e: handle_cgapi_exception(e) @@ -474,13 +474,12 @@ def truncate(self, length): class FeedbackFile(CachedSpecialFile): NAME = '.cg-feedback' - def __init__(self, api, submission_id): - self.api = api + def __init__(self, submission_id): super(FeedbackFile, self).__init__(name=self.NAME) self.submission_id = submission_id def get_online_data(self): - feedback = self.api.get_submission(self.submission_id)['comment'] + feedback = __cgapi__.get_submission(self.submission_id)['comment'] if not feedback: return b'' @@ -504,20 +503,19 @@ def parse(self, data): return ''.join(res).strip() def send_back(self, feedback): - self.api.set_submission(self.submission_id, feedback=feedback) + __cgapi__.set_submission(self.submission_id, feedback=feedback) class GradeFile(CachedSpecialFile): NAME = '.cg-grade' - def __init__(self, api, submission_id): - self.api = api + def __init__(self, submission_id): self.grade = None super(GradeFile, self).__init__(name=self.NAME) self.submission_id = submission_id def get_online_data(self): - grade = self.api.get_submission(self.submission_id)['grade'] + grade = __cgapi__.get_submission(self.submission_id)['grade'] if grade is None: return b'' @@ -544,23 +542,22 @@ def send_back(self, grade): if grade < 0 or grade > 10: raise FuseOSError(EPERM) - self.api.set_submission(self.submission_id, grade=grade) + __cgapi__.set_submission(self.submission_id, grade=grade) class RubricSelectFile(CachedSpecialFile): NAME = '.cg-rubric.md' - def __init__(self, api, submission_id, user): + def __init__(self, submission_id, user): super(RubricSelectFile, self).__init__(name=self.NAME) self.submission_id = submission_id self.user = user self.lookup = {} - self.api = api def get_online_data(self): res = [] self.lookup = {} - d = self.api.get_submission_rubric(self.submission_id) + d = __cgapi__.get_submission_rubric(self.submission_id) sel = set(i['id'] for i in d['selected']) l_num = 0 if d['rubrics']: @@ -618,7 +615,7 @@ def parse(self, data): return sel def send_back(self, sel): - self.api.select_rubricitems(self.submission_id, sel) + __cgapi__.select_rubricitems(self.submission_id, sel) class RubricEditorFile(CachedSpecialFile): @@ -677,9 +674,8 @@ class RubricEditorFile(CachedSpecialFile): """ NAME = '.cg-edit-rubric.md' - def __init__(self, api, assignment_id, append_only=True): + def __init__(self, assignment_id, append_only=True): super(RubricEditorFile, self).__init__(name=self.NAME) - self.api = api self.assignment_id = assignment_id self.append_only = append_only self.lookup = {} @@ -693,7 +689,7 @@ def get_online_data(self): res = [] self.lookup = {} - for rub in self.api.get_assignment_rubric(self.assignment_id): + for rub in __cgapi__.get_assignment_rubric(self.assignment_id): res.append('# ') res.append('[{}] '.format(self.hash_id(rub['id']))) res.append(rub['header']) @@ -880,24 +876,23 @@ def get_from_lookup(h): raise FuseOSError(EPERM) self.lookup = new_lookup - self.api.set_assignment_rubric(self.assignment_id, {'rows': res}) + __cgapi__.set_assignment_rubric(self.assignment_id, {'rows': res}) class AssignmentSettingsFile(CachedSpecialFile): TO_USE = {'state', 'deadline', 'name'} - def __init__(self, api, assignment_id): + def __init__(self, assignment_id): super(AssignmentSettingsFile, self).__init__(name='.cg-assignment-settings.ini') self.assignment_id = assignment_id - self.api = api def send_back(self, data): - self.api.set_assignment(self.assignment_id, data) + __cgapi__.set_assignment(self.assignment_id, data) def get_online_data(self): lines = [] - for k, v in self.api.get_assignment(self.assignment_id).items(): + for k, v in __cgapi__.get_assignment(self.assignment_id).items(): if k not in self.TO_USE: continue @@ -1022,7 +1017,7 @@ def __init__(self, data, name=None): @property def data(self): if self._data is None: - self._data = cgapi.get_file(self.id) + self._data = __cgapi__.get_file(self.id) self.stat['st_size'] = len(self._data) return self._data @@ -1060,7 +1055,7 @@ def flush(self): assert self._data is not None try: - res = cgapi.patch_file(self.id, self._data) + res = __cgapi__.patch_file(self.id, self._data) except CGAPIException as e: self.data = None self.dirty = False @@ -1169,7 +1164,7 @@ def delete_feedback(self, payload): return {'ok': False, 'error': 'File not found'} try: - res = cgapi.delete_feedback(f.id, line) + res = __cgapi__.delete_feedback(f.id, line) except: return {'ok': False, 'error': 'The server returned an error'} @@ -1199,7 +1194,7 @@ def get_feedback(self, payload): return {'ok': False, 'error': 'File not a sever file'} try: - res = cgapi.get_feedback(f.id) + res = __cgapi__.get_feedback(f.id) except: return {'ok': False, 'error': 'The server returned an error'} @@ -1220,7 +1215,7 @@ def set_feedback(self, payload): return {'ok': False, 'error': 'File not a sever file'} try: - cgapi.add_feedback(f.id, line, message) + __cgapi__.add_feedback(f.id, line, message) except: return {'ok': False, 'error': 'The server returned an error'} @@ -1345,7 +1340,7 @@ def _create(self, path, mode): file = TempFile(fname, self._tmpdir) else: try: - fdata = cgapi.create_file(submission.id, query_path) + fdata = __cgapi__.create_file(submission.id, query_path) except CGAPIException as e: handle_cgapi_exception(e) @@ -1437,7 +1432,7 @@ def _mkdir(self, path, mode): else: submission = self.get_submission(path) query_path = submission.get_full_path(parts[3:], is_dir=True) - ddata = cgapi.create_file(submission.id, query_path) + ddata = __cgapi__.create_file(submission.id, query_path) parent.insert(Directory(ddata, name=dname, writable=True)) @@ -1526,7 +1521,7 @@ def _rename(self, old, new): raise FuseOSError(EPERM) try: - res = cgapi.rename_file(file.id, new_query_path) + res = __cgapi__.rename_file(file.id, new_query_path) except CGAPIException as e: handle_cgapi_exception(e) @@ -1555,7 +1550,7 @@ def _rmdir(self, path): raise FuseOSError(EPERM) try: - cgapi.delete_file(dir.id) + __cgapi__.delete_file(dir.id) except CGAPIException as e: handle_cgapi_exception(e) @@ -1609,7 +1604,7 @@ def unlink(self, path): raise FuseOSError(EPERM) try: - cgapi.delete_file(file.id) + __cgapi__.delete_file(file.id) except CGAPIException as e: handle_cgapi_exception(e) @@ -1728,7 +1723,7 @@ def write(self, path, data, offset, fh): if args.debug: logging.basicConfig(level=logging.DEBUG) - cgapi = CGAPI( + __cgapi__ = CGAPI( username, password, args.url or getenv('CGAPI_BASE_URL', None) ) From 69ef44062d45d4341fad5d97fb18cfed17f4da33 Mon Sep 17 00:00:00 2001 From: Olmo Kramer Date: Tue, 7 Nov 2017 22:01:14 +0100 Subject: [PATCH 5/6] Move checks in Directory.get_file to appropriate subclasses --- cgfs.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/cgfs.py b/cgfs.py index 52bb4398..766982fd 100755 --- a/cgfs.py +++ b/cgfs.py @@ -145,12 +145,6 @@ def get_file(self, path): name = parts[0] parts = parts[1:] - if not self.children_loaded: - if isinstance(self, AssignmentDirectory): - self.load_submissions() - elif isinstance(self, SubmissionDirectory): - self.load_files() - if not name in self.children: raise FuseOSError(ENOENT) @@ -202,6 +196,11 @@ def insert_assignments(self, assignments): class AssignmentDirectory(Directory): + def get_file(self, path): + if not self.children_loaded: + self.load_submissions() + return super(AssignmentDirectory, self).get_file(self, path) + def load_submissions(self, latest_only): try: submissions = __cgapi__.get_submissions(self.id) @@ -233,6 +232,11 @@ def load_submissions(self, latest_only): class SubmissionDirectory(Directory): + def get_file(self, path): + if not self.children_loaded: + self.load_files() + return super(SubmissionDirectory, self).get_file(self, path) + def get_full_path(self, path, is_dir=False): parts = split_path(path) full_path = self.tld + '/' + '/'.join(parts) From 78bc7a37b318693f4eea372e12b9fc1b4989a698 Mon Sep 17 00:00:00 2001 From: Olmo Kramer Date: Tue, 7 Nov 2017 23:35:51 +0100 Subject: [PATCH 6/6] Refucktor --- cgfs.py | 265 +++++++++++++++++++++++++++++++------------------------- 1 file changed, 145 insertions(+), 120 deletions(-) diff --git a/cgfs.py b/cgfs.py index 766982fd..96f776b0 100755 --- a/cgfs.py +++ b/cgfs.py @@ -81,7 +81,7 @@ def __init__(self, data, name=None): self.name = name if name is not None else data['name'] self.stat = None - def getattr(self, submission=None, path=None): + def getattr(self): if self.stat is None: self.stat = { 'st_size': 0, @@ -92,11 +92,6 @@ def getattr(self, submission=None, path=None): 'st_gid': os.getegid(), } - if submission is not None and path is not None: - stat = __cgapi__.get_file_meta(submission.id, path) - self.stat['st_size'] = stat['size'] - self.stat['st_mtime'] = stat['modification_date'] - return self.stat def setattr(self, key, value): @@ -114,9 +109,9 @@ def __init__(self, data, name=None, writable=False): self.children = {} self.children_loaded = False - def getattr(self, submission=None, path=None): + def getattr(self): if self.stat is None: - super(Directory, self).getattr(submission, path) + super(Directory, self).getattr() mode = 0o770 if self.writable else 0o550 self.stat['st_mode'] = S_IFDIR | mode self.stat['st_nlink'] = 2 @@ -163,25 +158,37 @@ def read(self): return res -class RootDirectory(Directory): - def load_courses(self): - for course in __cgapi__.get_courses(): - course_dir = CourseDirectory(course) - course_dir.getattr() - course_dir.insert_assignments(course['assignments']) +class ServerDirectory(Directory): + def __init__(self, cgapi, *args, **kwargs): + super(ServerDirectory, self).__init__(*args, **kwargs) + self.cgapi = cgapi + + +class RootDirectory(ServerDirectory): + def __init__(self, cgapi): + super(RootDirectory, self).__init__(cgapi, { + 'id': None, + 'name': None, + }) + self.getattr() + + for course in self.cgapi.get_courses(): + course_dir = CourseDirectory(self.cgapi, course) self.insert(course_dir) self.children_loaded = True -class CourseDirectory(Directory): - def insert_assignments(self, assignments): - for assig in assignments: - assig_dir = AssignmentDirectory(assig) - assig_dir.getattr() - assig_dir.insert(AssignmentSettingsFile(assig['id'])) +class CourseDirectory(ServerDirectory): + def __init__(self, cgapi, course): + super(CourseDirectory, self).__init__(cgapi, course) + self.getattr() + + for assig in course['assignments']: + assig_dir = AssignmentDirectory(self.cgapi, assig) + assig_dir.insert(AssignmentSettingsFile(self.cgapi, assig['id'])) assig_dir.insert( RubricEditorFile( - assig['id'], self.rubric_append_only + self.cgapi, assig['id'], rubric_append_only ) ) assig_dir.insert(HelpFile(RubricEditorFile)) @@ -195,15 +202,14 @@ def insert_assignments(self, assignments): self.children_loaded = True -class AssignmentDirectory(Directory): - def get_file(self, path): - if not self.children_loaded: - self.load_submissions() - return super(AssignmentDirectory, self).get_file(self, path) +class AssignmentDirectory(ServerDirectory): + def __init__(self, cgapi, assig): + super(AssignmentDirectory, self).__init__(cgapi, assig) + self.getattr() - def load_submissions(self, latest_only): + def load_submissions(self, latest_only=True): try: - submissions = __cgapi__.get_submissions(self.id) + submissions = self.cgapi.get_submissions(self.id) except CGAPIException as e: # pragma: no cover handle_cgapi_exception(e) @@ -213,52 +219,54 @@ def load_submissions(self, latest_only): if sub['user']['id'] in seen: continue - sub_dir = SubmissionDirectory( - sub, - name=sub['user']['name'] + ' - ' + sub['created_at'], - writable=True - ) + sub_dir = SubmissionDirectory(self.cgapi, sub) if latest_only: seen.add(sub['user']['id']) - sub_dir.getattr() - sub_dir.insert(RubricSelectFile(sub['id'], sub['user'])) - sub_dir.insert(GradeFile(sub['id'])) - sub_dir.insert(FeedbackFile(sub['id'])) + sub_dir.insert(RubricSelectFile(self.cgapi, sub['id'], sub['user'])) + sub_dir.insert(GradeFile(self.cgapi, sub['id'])) + sub_dir.insert(FeedbackFile(self.cgapi, sub['id'])) self.insert(sub_dir) self.children_loaded = True - -class SubmissionDirectory(Directory): def get_file(self, path): if not self.children_loaded: - self.load_files() - return super(SubmissionDirectory, self).get_file(self, path) + self.load_submissions() + return super(AssignmentDirectory, self).get_file(path) - def get_full_path(self, path, is_dir=False): - parts = split_path(path) - full_path = self.tld + '/' + '/'.join(parts) - if (is_dir): - full_path += '/' - return full_path + def read(self): + if not self.children_loaded: + self.load_submissions() + return super(AssignmentDirectory, self).read() + + +class SubmissionDirectory(ServerDirectory): + def __init__(self, cgapi, sub): + super(SubmissionDirectory, self).__init__( + cgapi, + sub, + name=sub['user']['name'] + ' - ' + sub['created_at'], + writable=True + ) + self.getattr() def load_files(self, submission): try: - files = __cgapi__.get_submission_files(self.id) + files = self.cgapi.get_submission_files(self.id) except CGAPIException as e: handle_cgapi_exception(e) def insert_tree(dir, tree): for item in tree['entries']: if 'entries' in item: - new_dir = RegularDirectory(item, writable=self.writable) + new_dir = RegularDirectory(item) new_dir.getattr() dir.insert(new_dir) insert_tree(new_dir, item) else: - dir.insert(File(item)) + dir.insert(ServerFile(self.cgapi, item)) dir.children_loaded = True insert_tree(self, files) @@ -270,9 +278,27 @@ def insert_tree(dir, tree): self.tld = files['name'] self.children_loaded = True + def get_file(self, path): + if not self.children_loaded: + self.load_files() + return super(SubmissionDirectory, self).get_file(path) + + def read(self): + if not self.children_loaded: + self.load_submission_files(dir) + return super(SubmissionDirectory, self).read() + + def get_full_path(self, path, is_dir=False): + parts = split_path(path) + full_path = self.tld + '/' + '/'.join(parts) + if (is_dir): + full_path += '/' + return full_path + class RegularDirectory(Directory): - pass + def __init__(self, data): + super(RegularDirectory, self).__init__(data, writable=True) class TempDirectory(Directory): @@ -384,13 +410,14 @@ def flush(self): class CachedSpecialFile(SpecialFile): DELTA = datetime.timedelta(seconds=60) - def __init__(self, name): + def __init__(self, cgapi, name): super(CachedSpecialFile, self).__init__(name=name) self.data = None self.time = None self.mtime = time() self.mode = 0o770 self.overwrite = False + self.cgapi = cgapi def get_st_mtime(self): return self.mtime @@ -478,12 +505,12 @@ def truncate(self, length): class FeedbackFile(CachedSpecialFile): NAME = '.cg-feedback' - def __init__(self, submission_id): - super(FeedbackFile, self).__init__(name=self.NAME) + def __init__(self, cgapi, submission_id): + super(FeedbackFile, self).__init__(cgapi, name=self.NAME) self.submission_id = submission_id def get_online_data(self): - feedback = __cgapi__.get_submission(self.submission_id)['comment'] + feedback = self.cgapi.get_submission(self.submission_id)['comment'] if not feedback: return b'' @@ -507,19 +534,19 @@ def parse(self, data): return ''.join(res).strip() def send_back(self, feedback): - __cgapi__.set_submission(self.submission_id, feedback=feedback) + self.cgapi.set_submission(self.submission_id, feedback=feedback) class GradeFile(CachedSpecialFile): NAME = '.cg-grade' - def __init__(self, submission_id): + def __init__(self, cgapi, submission_id): self.grade = None - super(GradeFile, self).__init__(name=self.NAME) + super(GradeFile, self).__init__(cgapi, name=self.NAME) self.submission_id = submission_id def get_online_data(self): - grade = __cgapi__.get_submission(self.submission_id)['grade'] + grade = self.cgapi.get_submission(self.submission_id)['grade'] if grade is None: return b'' @@ -546,14 +573,14 @@ def send_back(self, grade): if grade < 0 or grade > 10: raise FuseOSError(EPERM) - __cgapi__.set_submission(self.submission_id, grade=grade) + self.cgapi.set_submission(self.submission_id, grade=grade) class RubricSelectFile(CachedSpecialFile): NAME = '.cg-rubric.md' - def __init__(self, submission_id, user): - super(RubricSelectFile, self).__init__(name=self.NAME) + def __init__(self, cgapi, submission_id, user): + super(RubricSelectFile, self).__init__(cgapi, name=self.NAME) self.submission_id = submission_id self.user = user self.lookup = {} @@ -561,7 +588,7 @@ def __init__(self, submission_id, user): def get_online_data(self): res = [] self.lookup = {} - d = __cgapi__.get_submission_rubric(self.submission_id) + d = self.cgapi.get_submission_rubric(self.submission_id) sel = set(i['id'] for i in d['selected']) l_num = 0 if d['rubrics']: @@ -619,7 +646,7 @@ def parse(self, data): return sel def send_back(self, sel): - __cgapi__.select_rubricitems(self.submission_id, sel) + self.cgapi.select_rubricitems(self.submission_id, sel) class RubricEditorFile(CachedSpecialFile): @@ -678,8 +705,8 @@ class RubricEditorFile(CachedSpecialFile): """ NAME = '.cg-edit-rubric.md' - def __init__(self, assignment_id, append_only=True): - super(RubricEditorFile, self).__init__(name=self.NAME) + def __init__(self, cgapi, assignment_id, append_only=True): + super(RubricEditorFile, self).__init__(cgapi, name=self.NAME) self.assignment_id = assignment_id self.append_only = append_only self.lookup = {} @@ -693,7 +720,7 @@ def get_online_data(self): res = [] self.lookup = {} - for rub in __cgapi__.get_assignment_rubric(self.assignment_id): + for rub in self.cgapi.get_assignment_rubric(self.assignment_id): res.append('# ') res.append('[{}] '.format(self.hash_id(rub['id']))) res.append(rub['header']) @@ -880,23 +907,23 @@ def get_from_lookup(h): raise FuseOSError(EPERM) self.lookup = new_lookup - __cgapi__.set_assignment_rubric(self.assignment_id, {'rows': res}) + self.cgapi.set_assignment_rubric(self.assignment_id, {'rows': res}) class AssignmentSettingsFile(CachedSpecialFile): TO_USE = {'state', 'deadline', 'name'} - def __init__(self, assignment_id): + def __init__(self, cgapi, assignment_id): super(AssignmentSettingsFile, - self).__init__(name='.cg-assignment-settings.ini') + self).__init__(cgapi, name='.cg-assignment-settings.ini') self.assignment_id = assignment_id def send_back(self, data): - __cgapi__.set_assignment(self.assignment_id, data) + self.cgapi.set_assignment(self.assignment_id, data) def get_online_data(self): lines = [] - for k, v in __cgapi__.get_assignment(self.assignment_id).items(): + for k, v in self.cgapi.get_assignment(self.assignment_id).items(): if k not in self.TO_USE: continue @@ -1011,17 +1038,18 @@ def truncate(self, length): self._handle.flush() -class File(BaseFile, SingleFile): - def __init__(self, data, name=None): - super(File, self).__init__(data, name) +class ServerFile(BaseFile, SingleFile): + def __init__(self, cgapi, data, name=None): + super(ServerFile, self).__init__(data, name) + self.cgapi = cgapi self._data = None self.dirty = False @property def data(self): if self._data is None: - self._data = __cgapi__.get_file(self.id) + self._data = self.cgapi.get_file(self.id) self.stat['st_size'] = len(self._data) return self._data @@ -1033,10 +1061,15 @@ def data(self, data): def getattr(self, submission=None, path=None): if self.stat is None: - super(File, self).getattr(submission, path) + super(ServerFile, self).getattr(submission, path) self.stat['st_mode'] = S_IFREG | 0o770 self.stat['st_nlink'] = 1 + if submission is not None and path is not None: + stat = __cgapi__.get_file_meta(submission.id, path) + self.stat['st_size'] = stat['size'] + self.stat['st_mtime'] = stat['modification_date'] + if self.stat['st_size'] is None: self.stat['st_size'] = len(self.data) return self.stat @@ -1059,7 +1092,7 @@ def flush(self): assert self._data is not None try: - res = __cgapi__.patch_file(self.id, self._data) + res = self.cgapi.patch_file(self.id, self._data) except CGAPIException as e: self.data = None self.dirty = False @@ -1113,7 +1146,8 @@ def fsync(self): class APIHandler: OPS = {'set_feedback', 'get_feedback', 'delete_feedback', 'is_file'} - def __init__(self, cgfs): + def __init__(self, cgapi, cgfs): + self.cgapi = cgapi self.cgfs = cgfs self.stop = False @@ -1168,7 +1202,7 @@ def delete_feedback(self, payload): return {'ok': False, 'error': 'File not found'} try: - res = __cgapi__.delete_feedback(f.id, line) + res = self.cgapi.delete_feedback(f.id, line) except: return {'ok': False, 'error': 'The server returned an error'} @@ -1183,7 +1217,7 @@ def is_file(self, payload): except: return {'ok': False, 'error': 'File not found'} - return {'ok': isinstance(f, File)} + return {'ok': isinstance(f, ServerFile)} def get_feedback(self, payload): f_name = self.cgfs.strippath(payload['file']) @@ -1198,7 +1232,7 @@ def get_feedback(self, payload): return {'ok': False, 'error': 'File not a sever file'} try: - res = __cgapi__.get_feedback(f.id) + res = self.cgapi.get_feedback(f.id) except: return {'ok': False, 'error': 'The server returned an error'} @@ -1219,7 +1253,7 @@ def set_feedback(self, payload): return {'ok': False, 'error': 'File not a sever file'} try: - __cgapi__.add_feedback(f.id, line, message) + self.cgapi.add_feedback(f.id, line, message) except: return {'ok': False, 'error': 'The server returned an error'} @@ -1231,6 +1265,7 @@ class CGFS(LoggingMixIn, Operations): def __init__( self, + cgapi, latest_only, socketfile, mountpoint, @@ -1239,6 +1274,7 @@ def __init__( rubric_append_only=True, quiet=False, ): + self.cgapi = cgapi self.latest_only = latest_only self.fixed = fixed self.files = {} @@ -1254,7 +1290,7 @@ def __init__( self.socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) self.socket.bind(self._socketfile) self.socket.listen() - self.api_handler = APIHandler(self) + self.api_handler = APIHandler(self.cgapi, self) threading.Thread( target=self.api_handler.run, args=(self.socket, ) ).start() @@ -1264,22 +1300,16 @@ def __init__( self.rubric_append_only = rubric_append_only - self.files = RootDirectory( - { - 'id': None, - 'name': 'root' - } - ) + self.files = RootDirectory(cgapi) with self._lock: - self.files.getattr() self.files.insert(self.special_socketfile) self.files.insert( SpecialFile( '.cg-mode', b'FIXED\n' if self.fixed else b'NOT_FIXED\n' ) ) - self.load_courses() + if not self.quiet: print('Mounted') @@ -1299,13 +1329,14 @@ def get_submission(self, path): return submission def get_file(self, path, start=None, expect_type=None): - dir = start if start is not None else self.files + file = start if start is not None else self.files - if not isinstance(dir, Directory): + if not isinstance(file, Directory): raise FuseOSError(ENOTDIR) parts = split_path(path) - file = dir.get_file(parts, expect_type) + if parts: + file = file.get_file(parts) if expect_type is not None: if not isinstance(file, expect_type): @@ -1314,6 +1345,8 @@ def get_file(self, path, start=None, expect_type=None): elif issubclass(expect_type, SingleFile): raise FuseOSError(EISDIR) + return file + def get_dir(self, path, start=None): return self.get_file(path, start=start, expect_type=Directory) @@ -1344,11 +1377,11 @@ def _create(self, path, mode): file = TempFile(fname, self._tmpdir) else: try: - fdata = __cgapi__.create_file(submission.id, query_path) + fdata = self.cgapi.create_file(submission.id, query_path) except CGAPIException as e: handle_cgapi_exception(e) - file = File(fdata, name=fname) + file = ServerFile(fdata, name=fname) file.setattr('st_size', fdata['size']) file.setattr('st_mtime', fdata['modification_date']) @@ -1388,10 +1421,12 @@ def _getattr(self, path, fh): else: file = self._open_files[fh] + print('file', file) + if isinstance(file, (TempFile, SpecialFile)): return file.getattr() - if file.stat is None and len(parts) > 3: + if file.stat is None and isinstance(file, ServerFile): try: submission = self.get_submission(path) except CGAPIException as e: @@ -1399,16 +1434,12 @@ def _getattr(self, path, fh): query_path = submission.get_full_path(parts[3:]) - if isinstance(file, Directory): - query_path += '/' - else: - submission = None - query_path = None + attrs = file.getattr(submission, query_path) + if self.fixed: + attrs['st_mode'] &= ~0o222 + return attrs - attrs = file.getattr(submission, query_path) - if self.fixed and isinstance(file, File): - attrs['st_mode'] &= ~0o222 - return attrs + return file.getattr() # TODO?: Add xattr support def getxattr(self, path, name, position=0): @@ -1436,7 +1467,7 @@ def _mkdir(self, path, mode): else: submission = self.get_submission(path) query_path = submission.get_full_path(parts[3:], is_dir=True) - ddata = __cgapi__.create_file(submission.id, query_path) + ddata = self.cgapi.create_file(submission.id, query_path) parent.insert(Directory(ddata, name=dname, writable=True)) @@ -1471,13 +1502,6 @@ def read(self, path, size, offset, fh): def readdir(self, path, fh): with self._lock: dir = self.get_dir(path) - - if not dir.children_loaded: - if isinstance(dir, AssignmentDirectory): - self.load_submissions(dir) - elif isinstance(dir, SubmissionDirectory): - self.load_submission_files(dir) - return dir.read() def readlink(self, path): @@ -1525,7 +1549,7 @@ def _rename(self, old, new): raise FuseOSError(EPERM) try: - res = __cgapi__.rename_file(file.id, new_query_path) + res = self.cgapi.rename_file(file.id, new_query_path) except CGAPIException as e: handle_cgapi_exception(e) @@ -1554,7 +1578,7 @@ def _rmdir(self, path): raise FuseOSError(EPERM) try: - __cgapi__.delete_file(dir.id) + self.cgapi.delete_file(dir.id) except CGAPIException as e: handle_cgapi_exception(e) @@ -1608,7 +1632,7 @@ def unlink(self, path): raise FuseOSError(EPERM) try: - __cgapi__.delete_file(file.id) + self.cgapi.delete_file(file.id) except CGAPIException as e: handle_cgapi_exception(e) @@ -1620,7 +1644,7 @@ def utimens(self, path, times=None): assert file is not None atime, mtime = times or (time(), time()) - if isinstance(file, File) and self.fixed: + if isinstance(file, ServerFile) and self.fixed: raise FuseOSError(EPERM) file.utimens(atime, mtime) @@ -1727,7 +1751,7 @@ def write(self, path, data, offset, fh): if args.debug: logging.basicConfig(level=logging.DEBUG) - __cgapi__ = CGAPI( + cgapi = CGAPI( username, password, args.url or getenv('CGAPI_BASE_URL', None) ) @@ -1735,6 +1759,7 @@ def write(self, path, data, offset, fh): sockfile = tempfile.NamedTemporaryFile().name try: fs = CGFS( + cgapi, latest_only, socketfile=sockfile, fixed=fixed,