Garmelon/PFERD
1
0
Fork 0
mirror of https://github.com/Garmelon/PFERD.git synced 2026-04-12 15:35:05 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Make local crawler synchronous

Browse source
This commit is contained in:
Joscha 2022-10-26 19:41:34 +02:00
parent 0294ceb7d5
commit a2ffce4702
1 changed files with 1 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

6
PFERD/crawl/local_crawler.py
View file

@ -71,8 +71,6 @@ class LocalCrawler(Crawler):
if not cl: if not cl:
return return
tasks = []
async with cl: async with cl:
await asyncio.sleep(random.uniform( await asyncio.sleep(random.uniform(
0.5 * self._crawl_delay, 0.5 * self._crawl_delay,
@ -81,9 +79,7 @@ class LocalCrawler(Crawler):
for child in path.iterdir(): for child in path.iterdir():
pure_child = cl.path / child.name pure_child = cl.path / child.name
tasks.append(self._crawl_path(child, pure_child)) await self._crawl_path(child, pure_child)
await self.gather(tasks)
async def _crawl_file(self, path: Path, pure: PurePath) -> None: async def _crawl_file(self, path: Path, pure: PurePath) -> None:
stat = path.stat() stat = path.stat()