Add backoff, add login with password, and add cookie handling.

.gitignore

@@ -1,5 +1,6 @@
 # Don't version user oauth credentials
 # These are generated per user, and will be generated on your system
+cookie.txt
 storage.json
 
 # Don't version cached or compiled python files

Pipfile

@@ -8,6 +8,7 @@ pylint = "*"
 twine = "*"
 coverage = "*"
 coveralls = "*"
+rope = "*"
 
 [packages]
 google-api-python-client = "*"
@@ -16,3 +17,4 @@ lxml = "*"
 python-dateutil = "*"
 pytz = "*"
 oauth2client = "*"
+backoff = "*"

README.md

@@ -14,7 +14,7 @@ So you want to work with the code? Awesome!
 This project uses [Pipenv](https://pipenv.readthedocs.io/en/latest/), after cloning the repo, do the following:
 
 1. Make sure you have python 3 installed.
-2. Create a pipenv in your working directory with `pipenv --python 3`.
+2. Create a pipenv in your working directory with `pipenv --three`.
 3. Install both the default and development packages from the Pipfile with `pipenv install --dev`.
 
 You should now be ready to work.

lectocal/gcalendar.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import backoff
 import datetime
 from httplib2 import Http
 import dateutil.parser
@@ -150,7 +151,7 @@ def _parse_event_to_lesson(event):
     if "description" in event:
         description = event["description"]
     else:
-        description = None
+        description = ""
     if "source" in event and "url" in event["source"]:
         link = event["source"]["url"]
     else:
@@ -173,28 +174,22 @@ def get_schedule(google_credentials, calendar_name, n_weeks):
     events = _get_events_in_date_range(service, calendar_id, start, end)
     return _parse_events_to_schedule(events)
 
-
+@backoff.on_exception(backoff.expo, HttpError,  max_tries=8)
 def _delete_lesson(service, calendar_id, lesson_id):
     service \
         .events() \
         .delete(calendarId=calendar_id, eventId=lesson_id) \
         .execute()
 
-
+@backoff.on_exception(backoff.expo, HttpError,  max_tries=4)
 def _add_lesson(service, calendar_id, lesson):
-    try:
-        service \
-            .events() \
-            .insert(calendarId=calendar_id, body=lesson.to_gcalendar_format()) \
-            .execute()
-    except HttpError as err:
-        #Status code 409 is conflict. In this case, it means the id already exists.
-        if err.resp.status == 409:
-            _update_lesson(service, calendar_id, lesson)
-        else:
-            raise err
-
+    # try:
+    service \
+        .events() \
+        .insert(calendarId=calendar_id, body=lesson.to_gcalendar_format()) \
+        .execute()
 
+@backoff.on_exception(backoff.expo, HttpError,  max_tries=8)
 def _update_lesson(service, calendar_id, lesson):
     service \
         .events() \
@@ -204,18 +199,29 @@ def _update_lesson(service, calendar_id, lesson):
         .execute()
 
 
+def _add_lesson_or_update_lesson(service, calendar_id, new_lesson):
+    try:
+        _add_lesson(service, calendar_id, new_lesson)
+    except HttpError as err:
+        #Status code 409 is conflict. In this case, it means the id already exists.
+        if err.resp.status == 409:
+            _update_lesson(service, calendar_id, new_lesson)
+        else:
+            raise err
+
+
 def _delete_removed_lessons(service, calendar_id, old_schedule, new_schedule):
     for old_lesson in old_schedule:
         if not any(new_lesson.id == old_lesson.id
-                   for new_lesson in new_schedule):
-            _delete_lesson(service, calendar_id, old_lesson.id)
+            for new_lesson in new_schedule):
+                _delete_lesson(service, calendar_id, old_lesson.id)
 
 
 def _add_new_lessons(service, calendar_id, old_schedule, new_schedule):
     for new_lesson in new_schedule:
         if not any(old_lesson.id == new_lesson.id
-                   for old_lesson in old_schedule):
-            _add_lesson(service, calendar_id, new_lesson)
+            for old_lesson in old_schedule):
+                _add_lesson_or_update_lesson(service, calendar_id, new_lesson)
 
 
 def _update_current_lessons(service, calendar_id, old_schedule, new_schedule):

lectocal/lectio.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import datetime
+import pickle
 import re
 import requests
 from lxml import html
@@ -23,8 +24,8 @@
 LESSON_STATUS = {None: "normal", "Ændret!": "changed", "Aflyst!": "cancelled"}
 
 
-class UserDoesNotExistError(Exception):
-    """ Attempted to get a non-existing user from Lectio. """
+class CannotLoginToLectioError(Exception):
+    """ Could not login to Lectio (using cookie or login provided). """
 
 
 class IdNotFoundInLinkError(Exception):
@@ -43,15 +44,53 @@ class InvalidLocationError(Exception):
     """ The line doesn't include any location. """
 
 
-def _get_user_page(school_id, user_type, user_id, week=""):
+def _get_user_page(school_id, user_type, user_id, week = "", login = "", password = ""):
     URL_TEMPLATE = "https://www.lectio.dk/lectio/{0}/" \
                    "SkemaNy.aspx?type={1}&{1}id={2}&week={3}"
-
-    r = requests.get(URL_TEMPLATE.format(school_id,
-                                         USER_TYPE[user_type],
-                                         user_id,
-                                         week),
+
+    LOGIN_URL = "https://www.lectio.dk/lectio/{0}/login.aspx".format(school_id)
+
+    # Start requests session
+    s = requests.Session()
+
+    if(login != ""):
+        # Get eventvalidation key
+        result = s.get(LOGIN_URL)
+        tree = html.fromstring(result.text)
+        authenticity_token = list(set(tree.xpath("//input[@name='__EVENTVALIDATION']/@value")))[0]
+
+        # Create payload
+        payload = {
+            "m$Content$username2": login,
+            "m$Content$password2": password,
+            "m$Content$passwordHidden": password,
+            "__EVENTVALIDATION": authenticity_token,
+            "__EVENTTARGET": "m$Content$submitbtn2",
+            "__EVENTARGUMENT": "",
+            "LectioPostbackId": ""
+        }
+
+        # Perform login
+        result = s.post(LOGIN_URL, data = payload, headers = dict(referer = LOGIN_URL))
+
+        # Save cookies to file
+        with open('cookie.txt', 'wb') as f:
+            pickle.dump(s.cookies, f)
+
+    else:
+        # Load cookies from file
+        with open('cookie.txt', 'rb') as f:
+            s.cookies.update(pickle.load(f))
+
+    # Scrape url and save cookies to file
+    r = s.get(URL_TEMPLATE.format(school_id,
+                                  USER_TYPE[user_type],
+                                  user_id,
+                                  week),
                      allow_redirects=False)
+    with open('cookie.txt', 'wb') as f:
+        pickle.dump(s.cookies, f)
+
     return r
 
 
@@ -65,10 +104,9 @@ def _get_lectio_weekformat_with_offset(offset):
 
 
 def _get_id_from_link(link):
-    match = re.search("(?:absid|ProeveholdId|outboundCensorID)=(\d+)", link)
+    match = re.search(r"(?:absid|ProeveholdId|outboundCensorID|aftaleid)=(\d+)", link)
     if match is None:
-        raise IdNotFoundInLinkError("Couldn't find id in link: {}".format(
-                                    link))
+        return None
     return match.group(1)
 
 def _get_complete_link(link):
@@ -92,8 +130,8 @@ def _is_time_line(line):
     # 8/4-2016 17:30 til 9/4-2016 01:00
     # 7/12-2015 10:00 til 11:30
     # 17/12-2015 10:00 til 11:30
-    match = re.search("\d{1,2}/\d{1,2}-\d{4} (?:Hele dagen|\d{2}:\d{2} til "
-                      "(?:\d{1,2}/\d{1,2}-\d{4} )?\d{2}:\d{2})", line)
+    match = re.search(r"\d{1,2}/\d{1,2}-\d{4} (?:Hele dagen|\d{2}:\d{2} til "
+                      r"(?:\d{1,2}/\d{1,2}-\d{4} )?\d{2}:\d{2})", line)
     return match is not None
 
 
@@ -132,8 +170,8 @@ def _get_time_from_line(line):
     # 2 - start time
     # 3 - end date
     # 4 - end time
-    match = re.search("(\d{1,2}/\d{1,2}-\d{4})(?: (\d{2}:\d{2}) til "
-                      "(\d{1,2}/\d{1,2}-\d{4})? ?(\d{2}:\d{2}))?", line)
+    match = re.search(r"(\d{1,2}/\d{1,2}-\d{4})(?: (\d{2}:\d{2}) til "
+                      r"(\d{1,2}/\d{1,2}-\d{4})? ?(\d{2}:\d{2}))?", line)
     if match is None:
         raise InvalidTimeLineError("No time found in line: '{}'".format(line))
 
@@ -218,8 +256,8 @@ def _parse_page_to_lessons(page):
     return lessons
 
 
-def _retreive_week_schedule(school_id, user_type, user_id, week):
-    r = _get_user_page(school_id, user_type, user_id, week)
+def _retreive_week_schedule(school_id, user_type, user_id, week, login = "", password = ""):
+    r = _get_user_page(school_id, user_type, user_id, week, login = "", password = "")
     schedule = _parse_page_to_lessons(r.content)
     return schedule
 
@@ -232,27 +270,29 @@ def _filter_for_duplicates(schedule):
     return filtered_schedule
 
 
-def _retreive_user_schedule(school_id, user_type, user_id, n_weeks):
+def _retreive_user_schedule(school_id, user_type, user_id, n_weeks, login = "", password = ""):
     schedule = []
     for week_offset in range(n_weeks + 1):
         week = _get_lectio_weekformat_with_offset(week_offset)
         week_schedule = _retreive_week_schedule(school_id,
                                                 user_type,
                                                 user_id,
-                                                week)
+                                                week, 
+                                                login = "", 
+                                                password = "")
         schedule += week_schedule
     filtered_schedule = _filter_for_duplicates(schedule)
     return filtered_schedule
 
 
-def _user_exists(school_id, user_type, user_id):
-    r = _get_user_page(school_id, user_type, user_id)
+def _can_login(school_id, user_type, user_id, login = "", password = ""):
+    r = _get_user_page(school_id, user_type, user_id, "", login, password)
     return r.status_code == requests.codes.ok
 
 
-def get_schedule(school_id, user_type, user_id, n_weeks):
-    if not _user_exists(school_id, user_type, user_id):
-        raise UserDoesNotExistError("Couldn't find user - school: {}, "
-                                    "type: {}, id: {} - in Lectio.".format(
-                                        school_id, user_type, user_id))
-    return _retreive_user_schedule(school_id, user_type, user_id, n_weeks)
+def get_schedule(school_id, user_type, user_id, n_weeks, login = "", password = ""):
+    if not _can_login(school_id, user_type, user_id, login, password):
+        raise CannotLoginToLectioError(
+            "Couldn't login user - school: {}, type: {}, id: {}, login: {} "
+            "- in Lectio.".format(school_id, user_type, user_id, login))
+    return _retreive_user_schedule(school_id, user_type, user_id, n_weeks, login = "", password = "")

lectocal/run.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import argparse
+import getpass
 from . import gauth
 from . import lectio
 from . import lesson
@@ -40,6 +41,14 @@ def _get_arguments():
                         default="Lectio",
                         help="Name to use for the calendar inside "
                         "Google Calendar. (default: Lectio)")
+    parser.add_argument("--login",
+                        default="",
+                        type=str,
+                        help="The username from a Lectio login.")
+    parser.add_argument('--keepalive',
+                        default=False,
+                        dest='keepalive', 
+                        action='store_true')
     parser.add_argument("--weeks",
                         type=int,
                         default=4,
@@ -51,21 +60,46 @@ def _get_arguments():
 
 def main():
     arguments = _get_arguments()
+
+    if arguments.keepalive:
+        _keepalive(arguments)
+        exit()
+
+    password = _get_password(arguments.login)
+
     google_credentials = gauth.get_credentials(arguments.credentials)
     if not gcalendar.has_calendar(google_credentials, arguments.calendar):
         gcalendar.create_calendar(google_credentials, arguments.calendar)
+
     lectio_schedule = lectio.get_schedule(arguments.school_id,
-                                          arguments.user_type,
-                                          arguments.user_id,
-                                          arguments.weeks)
+                                            arguments.user_type,
+                                            arguments.user_id,
+                                            arguments.weeks,
+                                            arguments.login,
+                                            password)
     google_schedule = gcalendar.get_schedule(google_credentials,
-                                             arguments.calendar,
-                                             arguments.weeks)
+                                                arguments.calendar,
+                                                arguments.weeks)
     if not lesson.schedules_are_identical(lectio_schedule, google_schedule):
         gcalendar.update_calendar_with_schedule(google_credentials,
                                                 arguments.calendar,
                                                 google_schedule,
                                                 lectio_schedule)
 
+
+def _keepalive(arguments):
+    lectio.get_schedule(arguments.school_id,
+                        arguments.user_type,
+                        arguments.user_id,
+                        1)
+
+
+def _get_password(login):
+    if(login != ""):
+        return getpass.getpass()
+    else:
+        return ""
+
+
 if __name__ == "__main__":
     main()

setup.py

@@ -49,7 +49,9 @@ def readme():
         "lxml",
         "pytz",
         "python-dateutil",
-        "oauth2client"
+        "oauth2client",
+        "backoff",
+        "pickle"
     ],
     package_data={
         "lectocal": [