diff --git a/PFERD/ipd.py b/PFERD/ipd.py index 33aaff1..d602e0e 100644 --- a/PFERD/ipd.py +++ b/PFERD/ipd.py @@ -86,11 +86,12 @@ class IpdCrawler: href: str = link.attrs.get("href") name = href.split("/")[-1] - modification_date: Optional[datetime.datetime] + modification_date: Optional[datetime.datetime] = None try: enclosing_row: bs4.Tag = link.findParent(name="tr") - date_text = enclosing_row.find(name="td").text - modification_date = datetime.datetime.strptime(date_text, "%d.%m.%Y") + if enclosing_row: + date_text = enclosing_row.find(name="td").text + modification_date = datetime.datetime.strptime(date_text, "%d.%m.%Y") except ValueError: modification_date = None diff --git a/PFERD/pferd.py b/PFERD/pferd.py index f57f078..c01b5fd 100644 --- a/PFERD/pferd.py +++ b/PFERD/pferd.py @@ -343,6 +343,11 @@ class Pferd(Location): downloader = IpdDownloader(tmp_dir=tmp_dir, organizer=organizer, strategy=download_strategy) downloader.download_all(transformed) + if clean: + organizer.cleanup() + + self._download_summary.merge(organizer.download_summary) + return organizer @swallow_and_print_errors @@ -403,4 +408,6 @@ class Pferd(Location): if clean: organizer.cleanup() + self._download_summary.merge(organizer.download_summary) + return organizer diff --git a/sync_url.py b/sync_url.py index c3be0b9..ca2e5ac 100755 --- a/sync_url.py +++ b/sync_url.py @@ -5,7 +5,10 @@ A simple script to download a course by name from ILIAS. """ import argparse +import logging +import sys from pathlib import Path +from typing import Optional, Tuple from urllib.parse import urlparse from PFERD import Pferd @@ -13,25 +16,57 @@ from PFERD.cookie_jar import CookieJar from PFERD.ilias import (IliasCrawler, IliasElementType, KitShibbolethAuthenticator, KeyringKitShibbolethAuthenticator) +from PFERD.logging import PrettyLogger, enable_logging from PFERD.utils import to_path +_LOGGER = logging.getLogger("sync_url") +_PRETTY = PrettyLogger(_LOGGER) + + +def _extract_credentials(file_path: Optional[str]) -> Tuple[Optional[str], Optional[str]]: + if not file_path: + return (None, None) + + if not Path(file_path).exists(): + _PRETTY.error("Credential file does not exist") + sys.exit(1) + + with open(file_path, "r") as file: + first_line = file.readline() + read_name, *read_password = first_line.split(":", 1) + + name = read_name if read_name else None + password = read_password[0] if read_password else None + return (name, password) + def main() -> None: + enable_logging(name="sync_url") + parser = argparse.ArgumentParser() parser.add_argument("--test-run", action="store_true") parser.add_argument('-c', '--cookies', nargs='?', default=None, help="File to store cookies in") + parser.add_argument('--credential-file', nargs='?', default=None, + help="Path to a file containing credentials for Ilias. The file must have " + "one line in the following format: ':'") parser.add_argument('--no-videos', nargs='?', default=None, help="Don't download videos") parser.add_argument("-k", "--keyring", action="store_true", help="Use the system keyring service for authentication") parser.add_argument('url', help="URL to the course page") parser.add_argument('folder', nargs='?', default=None, help="Folder to put stuff into") args = parser.parse_args() - url = urlparse(args.url) - cookie_jar = CookieJar(to_path(args.cookies) if args.cookies else None) session = cookie_jar.create_session() - authenticator = (KeyringKitShibbolethAuthenticator() if args.keyring - else KitShibbolethAuthenticator()) + + username, password = _extract_credentials(args.credential_file) + if args.keyring: + authenticator = KeyringKitShibbolethAuthenticator( + username=username, password=password) + else: + authenticator = KitShibbolethAuthenticator( + username=username, password=password) + + url = urlparse(args.url) crawler = IliasCrawler(url.scheme + '://' + url.netloc, session, authenticator, lambda x, y: True)