Garmelon/PFERD
1
0
Fork 0
mirror of https://github.com/Garmelon/PFERD.git synced 2026-04-12 15:35:05 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fix wrong base URL for multi-stage pages

Browse source
This commit is contained in:
I-Al-Istannen 2021-06-13 15:39:22 +02:00
parent 70b33ecfd9
commit 70ec64a48b
3 changed files with 5 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

2
PFERD/crawl/ilias/kit_ilias_html.py
View file

@ -480,7 +480,7 @@ class IliasPage:
return None
if "opencast" in str(img_tag["alt"]).lower():
return IliasElementType.VIDEO_FOLDER
return IliasElementType.VIDEO_FOLDER_MAYBE_PAGINATED
if str(img_tag["src"]).endswith("icon_exc.svg"):
return IliasElementType.EXERCISE

2
PFERD/crawl/ilias/kit_ilias_web_crawler.py
View file

@ -253,7 +253,7 @@ instance's greatest bottleneck.
soup = await self._get_page(next_stage_url)
log.explain_topic(f"Parsing HTML page for {fmt_path(path)}")
log.explain(f"URL: {next_stage_url}")
page = IliasPage(soup, url, parent)
page = IliasPage(soup, next_stage_url, parent)
next_stage_url = page.get_next_stage_url()
elements.extend(page.get_child_elements())