Garmelon/PFERD
1
0
Fork 0
mirror of https://github.com/Garmelon/PFERD.git synced 2026-04-12 23:45:05 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Make local crawler synchronous

Browse source
This commit is contained in:
Joscha 2022-10-26 19:41:34 +02:00
parent 0294ceb7d5
commit a2ffce4702
1 changed files with 1 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

6
PFERD/crawl/local_crawler.py
View file

@ -71,8 +71,6 @@ class LocalCrawler(Crawler):
if not cl:
return
tasks = []
async with cl:
await asyncio.sleep(random.uniform(
0.5 * self._crawl_delay,
@ -81,9 +79,7 @@ class LocalCrawler(Crawler):
for child in path.iterdir():
pure_child = cl.path / child.name
tasks.append(self._crawl_path(child, pure_child))
await self.gather(tasks)
await self._crawl_path(child, pure_child)
async def _crawl_file(self, path: Path, pure: PurePath) -> None:
stat = path.stat()