Garmelon/PFERD
1
0
Fork 0
mirror of https://github.com/Garmelon/PFERD.git synced 2026-04-12 15:35:05 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

More general search pattern, only focusing on .pdf links

Browse source
This commit is contained in:
Toorero 2021-10-26 03:23:49 +02:00
parent 0b2bcae44c
commit ac35dd9a47
1 changed files with 5 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

7
PFERD/crawl/kit_ipd_crawler.py
View file

@ -1,4 +1,5 @@
import os
import re
from dataclasses import dataclass
from pathlib import PurePath
from typing import List, Set, Union
@ -89,7 +90,9 @@ class KitIpdCrawler(HttpCrawler):
for element in elements:
enclosing_table: Tag = element.findParent(name="table")
label: Tag = enclosing_table.findPreviousSibling(name="h3")
if (enclosing_table) is None:
continue
label: Tag = enclosing_table.findPreviousSibling(name=re.compile('^h[1-6]$'))
folder_tags.add(label)
print(folder_tags)
@ -115,7 +118,7 @@ class KitIpdCrawler(HttpCrawler):
return KitIpdFile(name, url)
def _find_file_links(self, tag: Union[Tag, BeautifulSoup]) -> List[Tag]:
return tag.findAll(name="a", attrs={"href": lambda x: x and "intern" in x})
return tag.findAll(name="a", attrs={"href": lambda x: x and ".pdf" in x})
def _abs_url_from_link(self, link_tag: Tag) -> str:
return urljoin(self._url, link_tag.get("href"))