From dcea7b22547c4db01ecbc36bae9743de52e4bd89 Mon Sep 17 00:00:00 2001 From: Lucas <24826124+Luro02@users.noreply.github.com> Date: Tue, 3 Nov 2020 13:38:33 +0100 Subject: [PATCH 1/4] sync_url: Add username and password args --- sync_url.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/sync_url.py b/sync_url.py index d2dce94..293333c 100755 --- a/sync_url.py +++ b/sync_url.py @@ -19,6 +19,8 @@ def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--test-run", action="store_true") parser.add_argument('-c', '--cookies', nargs='?', default=None, help="File to store cookies in") + parser.add_argument('-u', '--username', nargs='?', default=None, help="Username for Ilias") + parser.add_argument('-p', '--password', nargs='?', default=None, help="Password for Ilias") parser.add_argument('--no-videos', nargs='?', default=None, help="Don't download videos") parser.add_argument('url', help="URL to the course page") parser.add_argument('folder', nargs='?', default=None, help="Folder to put stuff into") @@ -28,7 +30,7 @@ def main() -> None: cookie_jar = CookieJar(to_path(args.cookies) if args.cookies else None) session = cookie_jar.create_session() - authenticator = KitShibbolethAuthenticator() + authenticator = KitShibbolethAuthenticator(username=args.username, password=args.password) crawler = IliasCrawler(url.scheme + '://' + url.netloc, session, authenticator, lambda x, y: True) @@ -59,7 +61,9 @@ def main() -> None: target=folder, full_url=args.url, cookies=args.cookies, - dir_filter=dir_filter + dir_filter=dir_filter, + username=args.username, + password=args.password ) From 0da2fafcd8b7dbe258775cb5ff4f84fa671b846f Mon Sep 17 00:00:00 2001 From: I-Al-Istannen Date: Wed, 4 Nov 2020 14:38:15 +0100 Subject: [PATCH 2/4] Fix links outside tables --- PFERD/ipd.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/PFERD/ipd.py b/PFERD/ipd.py index 33aaff1..d602e0e 100644 --- a/PFERD/ipd.py +++ b/PFERD/ipd.py @@ -86,11 +86,12 @@ class IpdCrawler: href: str = link.attrs.get("href") name = href.split("/")[-1] - modification_date: Optional[datetime.datetime] + modification_date: Optional[datetime.datetime] = None try: enclosing_row: bs4.Tag = link.findParent(name="tr") - date_text = enclosing_row.find(name="td").text - modification_date = datetime.datetime.strptime(date_text, "%d.%m.%Y") + if enclosing_row: + date_text = enclosing_row.find(name="td").text + modification_date = datetime.datetime.strptime(date_text, "%d.%m.%Y") except ValueError: modification_date = None From ef343dec7c9ad7554ffbaf0b7301ce99666caaa9 Mon Sep 17 00:00:00 2001 From: I-Al-Istannen Date: Wed, 4 Nov 2020 15:06:58 +0100 Subject: [PATCH 3/4] Merge organizer download summaries --- PFERD/pferd.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/PFERD/pferd.py b/PFERD/pferd.py index f57f078..c01b5fd 100644 --- a/PFERD/pferd.py +++ b/PFERD/pferd.py @@ -343,6 +343,11 @@ class Pferd(Location): downloader = IpdDownloader(tmp_dir=tmp_dir, organizer=organizer, strategy=download_strategy) downloader.download_all(transformed) + if clean: + organizer.cleanup() + + self._download_summary.merge(organizer.download_summary) + return organizer @swallow_and_print_errors @@ -403,4 +408,6 @@ class Pferd(Location): if clean: organizer.cleanup() + self._download_summary.merge(organizer.download_summary) + return organizer From add5cc59155deb952f6ea5ef6b3ffd34ffc73dac Mon Sep 17 00:00:00 2001 From: I-Al-Istannen Date: Wed, 4 Nov 2020 21:18:48 +0100 Subject: [PATCH 4/4] Use credential file --- sync_url.py | 43 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/sync_url.py b/sync_url.py index 293333c..43a96e1 100755 --- a/sync_url.py +++ b/sync_url.py @@ -5,32 +5,61 @@ A simple script to download a course by name from ILIAS. """ import argparse +import logging +import sys from pathlib import Path +from typing import Optional, Tuple from urllib.parse import urlparse from PFERD import Pferd from PFERD.cookie_jar import CookieJar from PFERD.ilias import (IliasCrawler, IliasElementType, KitShibbolethAuthenticator) +from PFERD.logging import PrettyLogger, enable_logging from PFERD.utils import to_path +_LOGGER = logging.getLogger("sync_url") +_PRETTY = PrettyLogger(_LOGGER) + + +def _extract_credentials(file_path: Optional[str]) -> Tuple[Optional[str], Optional[str]]: + if not file_path: + return (None, None) + + if not Path(file_path).exists(): + _PRETTY.error("Credential file does not exist") + sys.exit(1) + + with open(file_path, "r") as file: + first_line = file.readline() + read_name, *read_password = first_line.split(":", 1) + + name = read_name if read_name else None + password = read_password[0] if read_password else None + return (name, password) + def main() -> None: + enable_logging(name="sync_url") + parser = argparse.ArgumentParser() parser.add_argument("--test-run", action="store_true") parser.add_argument('-c', '--cookies', nargs='?', default=None, help="File to store cookies in") - parser.add_argument('-u', '--username', nargs='?', default=None, help="Username for Ilias") - parser.add_argument('-p', '--password', nargs='?', default=None, help="Password for Ilias") + parser.add_argument('--credential-file', nargs='?', default=None, + help="Path to a file containing credentials for Ilias. The file must have " + "one line in the following format: ':'") parser.add_argument('--no-videos', nargs='?', default=None, help="Don't download videos") parser.add_argument('url', help="URL to the course page") parser.add_argument('folder', nargs='?', default=None, help="Folder to put stuff into") args = parser.parse_args() - url = urlparse(args.url) - cookie_jar = CookieJar(to_path(args.cookies) if args.cookies else None) session = cookie_jar.create_session() - authenticator = KitShibbolethAuthenticator(username=args.username, password=args.password) + + username, password = _extract_credentials(args.credential_file) + authenticator = KitShibbolethAuthenticator(username=username, password=password) + + url = urlparse(args.url) crawler = IliasCrawler(url.scheme + '://' + url.netloc, session, authenticator, lambda x, y: True) @@ -62,8 +91,8 @@ def main() -> None: full_url=args.url, cookies=args.cookies, dir_filter=dir_filter, - username=args.username, - password=args.password + username=username, + password=password )