Garmelon/PFERD
1
0
Fork 0
mirror of https://github.com/Garmelon/PFERD.git synced 2026-04-12 23:45:05 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Catch errors when requesting resource version

Browse source
This commit is contained in:
Scriptim 2024-10-27 17:04:58 +01:00
parent eb80c67535
commit 2193adadb4
No known key found for this signature in database
GPG key ID: 1ABB18EA42CCAAF6
1 changed files with 17 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

3
PFERD/crawl/http_crawler.py
View file

@ -198,6 +198,7 @@ class HttpCrawler(Crawler):
Requests the ETag and Last-Modified headers of a resource via a HEAD request. Requests the ETag and Last-Modified headers of a resource via a HEAD request.
If no entity tag / modification date can be obtained, the according value will be None. If no entity tag / modification date can be obtained, the according value will be None.
""" """
try:
async with self.session.head(resource_url) as resp: async with self.session.head(resource_url) as resp:
if resp.status != 200: if resp.status != 200:
return None, None return None, None
@ -215,6 +216,8 @@ class HttpCrawler(Crawler):
pass pass
return etag_header, last_modified return etag_header, last_modified
except aiohttp.ClientError:
return None, None
async def run(self) -> None: async def run(self) -> None:
self._request_count = 0 self._request_count = 0