From a81faa6806ad85fd2ec817fb8f2a8e93850529b6 Mon Sep 17 00:00:00 2001 From: Andy Date: Fri, 7 Mar 2014 19:05:48 -0500 Subject: [PATCH 1/3] test --- test.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 test.txt diff --git a/test.txt b/test.txt new file mode 100644 index 0000000..3bee695 --- /dev/null +++ b/test.txt @@ -0,0 +1 @@ +etst From cd321eeba53205ed66aba8a4fc4c08cecccac9ca Mon Sep 17 00:00:00 2001 From: Andy Date: Sat, 5 Apr 2014 17:39:12 -0400 Subject: [PATCH 2/3] Added files to framework importer scripts and command-line interface --- data/scripts/courses_importer.py | 9 +++++++++ data/scripts/importer.py | 6 ++++-- data/scripts/master_importer.py | 23 +++++++++++++++++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 data/scripts/courses_importer.py create mode 100644 data/scripts/master_importer.py diff --git a/data/scripts/courses_importer.py b/data/scripts/courses_importer.py new file mode 100644 index 0000000..74e5c8f --- /dev/null +++ b/data/scripts/courses_importer.py @@ -0,0 +1,9 @@ +from importer import Importer + +class Courses(Importer): + + def __init__(self, dump_file): + self.table_name = "courses_t" + self.dump_file = dump_file + + def import_data(self): diff --git a/data/scripts/importer.py b/data/scripts/importer.py index dad43e0..c087ae9 100644 --- a/data/scripts/importer.py +++ b/data/scripts/importer.py @@ -27,9 +27,9 @@ def _init_pg(self): def create_table(self): assert len(self.schema) > 0, "Schema is not defined." - print "Creating table `%s` with schema." $ self.table_name + print "Creating table `%s` with schema." % self.table_name cursor = self.conn.cursor() - db_query = 'CREATE TABLE IF NOT EXISTS housing_t (%s);' % (", ".join( + db_query = 'CREATE TABLE IF NOT EXISTS %s (%s);' % (self.table_name, ", ".join( ['%s %s' % column for column in schema])) cursor.execute(db_query) self.conn.commit() @@ -38,3 +38,5 @@ def drop_table(): print "Dropping table `%s`." % self.table_name self.cursor.execute("DROP TABLE %s;", self.table_name) self.conn.commit() + + def import_data(self, dump_file, self.filetype): diff --git a/data/scripts/master_importer.py b/data/scripts/master_importer.py new file mode 100644 index 0000000..d8a5bf4 --- /dev/null +++ b/data/scripts/master_importer.py @@ -0,0 +1,23 @@ +from courses_importer import Courses +import argparse + + +def main(): + parser = argparse.ArgumentParser(description="""Reads a dump file and imports + it to specified table""") + parser.add_argument('table_name', help="""Table name -- courses, housing, + housing_amenities""") + parser.add_argument('dump_file', help="""Name of dump file""") + + args = parser.parse_args() + # create object for corresponding table based on args + if (args.table_name == "courses") + table_object = Courses(args.dump_file) + elif (args.table_name =="housing") + table_object == Housing(args.dump_file) + + + load_data(args.dump_file) + +if __name__ == "__main__": + main() From fb73d30140a7304d87e6f8d5ec886e36633cb794 Mon Sep 17 00:00:00 2001 From: Andy Date: Mon, 14 Apr 2014 11:54:51 -0400 Subject: [PATCH 3/3] Small changes to interface --- data/scripts/courses_importer.py | 30 ++++++++++++++++++++++++++++++ data/scripts/importer.py | 7 +++++-- data/scripts/master_importer.py | 18 +++++++++++++----- test.txt | 1 - 4 files changed, 48 insertions(+), 8 deletions(-) delete mode 100644 test.txt diff --git a/data/scripts/courses_importer.py b/data/scripts/courses_importer.py index 74e5c8f..c752605 100644 --- a/data/scripts/courses_importer.py +++ b/data/scripts/courses_importer.py @@ -1,9 +1,39 @@ from importer import Importer +import json class Courses(Importer): def __init__(self, dump_file): self.table_name = "courses_t" self.dump_file = dump_file + self.schema = [ + ("Course", "varchar(32) primary key"), + ("CourseFull", "varchar(32)"), + ("PrefixName", "varchar(32)"), + ("DivisionCode", "varchar(32)"), + ("DivisionName", "varchar(64)"), + ("SchoolCode", "varchar(32)"), + ("SchoolName", "varchar(64)"), + ("DepartmentCode", "varchar(32)"), + ("DepartmentName", "varchar(64)"), + ("SubtermCode", "varchar(32)"), + ("SubtermName", "varchar(64)"), + ("EnrollmentStatus", "varchar(32)"), + ("NumFixedUnits", "int"), + ("MinUnits", "int"), + ("MaxUnits", "int"), + ("CourseTitle", "varchar(64)"), + ("CourseSubtitle", "varchar(64)"), + ("Approval", "varchar(32)"), + ("BulletinFlags", "varchar(32)"), + ("ClassNotes", "varchar(64)"), + ("PrefixLongname", "varchar(32)"), + ("Description", "text") + ] def import_data(self): + query = create_query(self) + send_query(query) + + def create_query(self): + return 0 diff --git a/data/scripts/importer.py b/data/scripts/importer.py index c087ae9..adeefe8 100644 --- a/data/scripts/importer.py +++ b/data/scripts/importer.py @@ -11,7 +11,7 @@ class Importer(object): def __init__(self, schema): self.table_name = None - self.schema = schema + self.schema = None self._init_pg() def _init_pg(self): @@ -39,4 +39,7 @@ def drop_table(): self.cursor.execute("DROP TABLE %s;", self.table_name) self.conn.commit() - def import_data(self, dump_file, self.filetype): + def send_query(query): + return 0 + + def diff --git a/data/scripts/master_importer.py b/data/scripts/master_importer.py index d8a5bf4..16f2686 100644 --- a/data/scripts/master_importer.py +++ b/data/scripts/master_importer.py @@ -5,19 +5,27 @@ def main(): parser = argparse.ArgumentParser(description="""Reads a dump file and imports it to specified table""") - parser.add_argument('table_name', help="""Table name -- courses, housing, + parser.add_argument('table_name', help="""Table name - e.g. courses, housing, housing_amenities""") + parser.add_argument('--create', action='store_true', help="""create the + courses_v2_t table if it doesn't already exist""") + parser.add_argument('--drop', action='store_true', help="""drop the + courses_v2_t table""") parser.add_argument('dump_file', help="""Name of dump file""") args = parser.parse_args() # create object for corresponding table based on args - if (args.table_name == "courses") + if (args.table_name == "courses"): table_object = Courses(args.dump_file) - elif (args.table_name =="housing") + elif (args.table_name =="housing"): table_object == Housing(args.dump_file) - - load_data(args.dump_file) + if (args.create): + table_object.create_table() + elif(args.drop): + table_object.create_table() + else: + table_object.import_data() if __name__ == "__main__": main() diff --git a/test.txt b/test.txt deleted file mode 100644 index 3bee695..0000000 --- a/test.txt +++ /dev/null @@ -1 +0,0 @@ -etst