Garmelon/PFERD
1
0
Fork 0
mirror of https://github.com/Garmelon/PFERD.git synced 2026-04-12 15:35:05 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Collapse multi-line f-string that gives an error on version <= 3.11

Browse source
This commit is contained in:
PinieP 2024-11-02 17:55:21 +01:00 • committed by I-Al-Istannen
parent 7797301288
commit 1d3880f840
1 changed files with 9 additions and 18 deletions
Download patch file Download diff file Expand all files Collapse all files

27
PFERD/crawl/ilias/ilias_web_crawler.py
View file

@ -269,8 +269,7 @@ instance's greatest bottleneck.
while next_stage_url: while next_stage_url:
soup = await self._get_page(next_stage_url) soup = await self._get_page(next_stage_url)
log.explain_topic(f"Parsing HTML page for { log.explain_topic(f"Parsing HTML page for {fmt_path(cl.path)}")
fmt_path(cl.path)}")
log.explain(f"URL: {next_stage_url}") log.explain(f"URL: {next_stage_url}")
# If we expect to find a root course, enforce it # If we expect to find a root course, enforce it
@ -447,8 +446,7 @@ instance's greatest bottleneck.
element: IliasPageElement, element: IliasPageElement,
element_path: PurePath, element_path: PurePath,
) -> Optional[Coroutine[Any, Any, None]]: ) -> Optional[Coroutine[Any, Any, None]]:
log.explain_topic(f"Decision: Crawl Booking Link { log.explain_topic(f"Decision: Crawl Booking Link {fmt_path(element_path)}")
fmt_path(element_path)}")
log.explain(f"Links type is {self._links}") log.explain(f"Links type is {self._links}")
link_template_maybe = self._links.template() link_template_maybe = self._links.template()
@ -574,25 +572,21 @@ instance's greatest bottleneck.
return [] return []
cached_value = cast(dict[str, Any], custom_value) cached_value = cast(dict[str, Any], custom_value)
if "known_paths" not in cached_value or "own_path" not in cached_value: if "known_paths" not in cached_value or "own_path" not in cached_value:
log.explain(f"'known_paths' or 'own_path' missing from cached value: { log.explain(f"'known_paths' or 'own_path' missing from cached value: {cached_value}")
cached_value}")
return [] return []
transformed_own_path = self._transformer.transform(element_path) transformed_own_path = self._transformer.transform(element_path)
if cached_value["own_path"] != str(transformed_own_path): if cached_value["own_path"] != str(transformed_own_path):
log.explain( log.explain(
f"own_path '{transformed_own_path}' does not match cached value: '{ f"own_path '{transformed_own_path}' does not match cached value: '{cached_value['own_path']}"
cached_value['own_path']}"
) )
return [] return []
return [PurePath(name) for name in cached_value["known_paths"]] return [PurePath(name) for name in cached_value["known_paths"]]
def _all_opencast_videos_locally_present(self, element: IliasPageElement, element_path: PurePath) -> bool: def _all_opencast_videos_locally_present(self, element: IliasPageElement, element_path: PurePath) -> bool:
log.explain_topic(f"Checking local cache for video { log.explain_topic(f"Checking local cache for video {fmt_path(element_path)}")
fmt_path(element_path)}")
if contained_videos := self._previous_contained_opencast_videos(element, element_path): if contained_videos := self._previous_contained_opencast_videos(element, element_path):
log.explain( log.explain(
f"The following contained videos are known: { f"The following contained videos are known: {','.join(map(fmt_path, contained_videos))}"
','.join(map(fmt_path, contained_videos))}"
) )
if all(self._output_dir.resolve(path).exists() for path in contained_videos): if all(self._output_dir.resolve(path).exists() for path in contained_videos):
log.explain( log.explain(
@ -638,8 +632,7 @@ instance's greatest bottleneck.
if not maybe_dl: if not maybe_dl:
continue continue
async with maybe_dl as (bar, sink): async with maybe_dl as (bar, sink):
log.explain(f"Streaming video from real url { log.explain(f"Streaming video from real url {stream_element.url}")
stream_element.url}")
contained_video_paths.append( contained_video_paths.append(
str(self._transformer.transform(maybe_dl.path))) str(self._transformer.transform(maybe_dl.path)))
await self._stream_from_url(stream_element.url, sink, bar, is_video=True) await self._stream_from_url(stream_element.url, sink, bar, is_video=True)
@ -803,8 +796,7 @@ instance's greatest bottleneck.
elements: List[IliasLearningModulePage] = [] elements: List[IliasLearningModulePage] = []
async with cl: async with cl:
log.explain_topic(f"Parsing initial HTML page for { log.explain_topic(f"Parsing initial HTML page for {fmt_path(cl.path)}")
fmt_path(cl.path)}")
log.explain(f"URL: {element.url}") log.explain(f"URL: {element.url}")
soup = await self._get_page(element.url) soup = await self._get_page(element.url)
page = IliasPage(soup, element.url, element) page = IliasPage(soup, element.url, element)
@ -849,8 +841,7 @@ instance's greatest bottleneck.
next_element_url: Optional[str] = start_url next_element_url: Optional[str] = start_url
counter = 0 counter = 0
while next_element_url: while next_element_url:
log.explain_topic(f"Parsing HTML page for { log.explain_topic(f"Parsing HTML page for {fmt_path(path)} ({dir}-{counter})")
fmt_path(path)} ({dir}-{counter})")
log.explain(f"URL: {next_element_url}") log.explain(f"URL: {next_element_url}")
soup = await self._get_page(next_element_url) soup = await self._get_page(next_element_url)
page = IliasPage(soup, next_element_url, parent_element) page = IliasPage(soup, next_element_url, parent_element)