Merge branch 'sync_url_authentication' into keyring

PFERD/ipd.py

@@ -86,11 +86,12 @@ class IpdCrawler:
             href: str = link.attrs.get("href")
             name = href.split("/")[-1]
 
-            modification_date: Optional[datetime.datetime]
+            modification_date: Optional[datetime.datetime] = None
             try:
                 enclosing_row: bs4.Tag = link.findParent(name="tr")
-                date_text = enclosing_row.find(name="td").text
+                if enclosing_row:
-                modification_date = datetime.datetime.strptime(date_text, "%d.%m.%Y")
+                    date_text = enclosing_row.find(name="td").text
+                    modification_date = datetime.datetime.strptime(date_text, "%d.%m.%Y")
             except ValueError:
                 modification_date = None
 

PFERD/pferd.py

@@ -343,6 +343,11 @@ class Pferd(Location):
         downloader = IpdDownloader(tmp_dir=tmp_dir, organizer=organizer, strategy=download_strategy)
         downloader.download_all(transformed)
 
+        if clean:
+            organizer.cleanup()
+
+        self._download_summary.merge(organizer.download_summary)
+
         return organizer
 
     @swallow_and_print_errors
@@ -403,4 +408,6 @@ class Pferd(Location):
         if clean:
             organizer.cleanup()
 
+        self._download_summary.merge(organizer.download_summary)
+
         return organizer

sync_url.py

@@ -5,7 +5,10 @@ A simple script to download a course by name from ILIAS.
 """
 
 import argparse
+import logging
+import sys
 from pathlib import Path
+from typing import Optional, Tuple
 from urllib.parse import urlparse
 
 from PFERD import Pferd
@@ -13,25 +16,57 @@ from PFERD.cookie_jar import CookieJar
 from PFERD.ilias import (IliasCrawler, IliasElementType,
                          KitShibbolethAuthenticator,
                          KeyringKitShibbolethAuthenticator)
+from PFERD.logging import PrettyLogger, enable_logging
 from PFERD.utils import to_path
 
+_LOGGER = logging.getLogger("sync_url")
+_PRETTY = PrettyLogger(_LOGGER)
+
+
+def _extract_credentials(file_path: Optional[str]) -> Tuple[Optional[str], Optional[str]]:
+    if not file_path:
+        return (None, None)
+
+    if not Path(file_path).exists():
+        _PRETTY.error("Credential file does not exist")
+        sys.exit(1)
+
+    with open(file_path, "r") as file:
+        first_line = file.readline()
+        read_name, *read_password = first_line.split(":", 1)
+
+        name = read_name if read_name else None
+        password = read_password[0] if read_password else None
+        return (name, password)
+
 
 def main() -> None:
+    enable_logging(name="sync_url")
+
     parser = argparse.ArgumentParser()
     parser.add_argument("--test-run", action="store_true")
     parser.add_argument('-c', '--cookies', nargs='?', default=None, help="File to store cookies in")
+    parser.add_argument('--credential-file', nargs='?', default=None,
+                        help="Path to a file containing credentials for Ilias. The file must have "
+                        "one line in the following format: '<user>:<password>'")
     parser.add_argument('--no-videos', nargs='?', default=None, help="Don't download videos")
     parser.add_argument("-k", "--keyring", action="store_true", help="Use the system keyring service for authentication")
     parser.add_argument('url', help="URL to the course page")
     parser.add_argument('folder', nargs='?', default=None, help="Folder to put stuff into")
     args = parser.parse_args()
 
-    url = urlparse(args.url)
-
     cookie_jar = CookieJar(to_path(args.cookies) if args.cookies else None)
     session = cookie_jar.create_session()
-    authenticator = (KeyringKitShibbolethAuthenticator() if args.keyring
+
-                     else KitShibbolethAuthenticator())
+    username, password = _extract_credentials(args.credential_file)
+    if args.keyring:
+        authenticator = KeyringKitShibbolethAuthenticator(
+            username=username, password=password)
+    else:
+        authenticator = KitShibbolethAuthenticator(
+            username=username, password=password)
+
+    url = urlparse(args.url)
     crawler = IliasCrawler(url.scheme + '://' + url.netloc, session,
                            authenticator, lambda x, y: True)