Garmelon/wikilyze
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix redirect importing

Browse source
This commit is contained in:
Joscha 2022-10-22 19:38:35 +02:00
parent e91a2db1b1
commit 2e6539cbc5
1 changed files with 10 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

8
brood/src/commands/ingest.rs
View file

@ -79,6 +79,13 @@ fn first_stage() -> io::Result<(AdjacencyList<PageInfo, LinkInfo>, Titles)> {
}, },
}); });
if let Some(to) = json_page.redirect {
let to = titles.insert(util::normalize_link(&to));
result.links.push(Link {
to,
data: LinkInfo { start: 0, end: 0 },
});
} else {
for (to, start, end) in json_page.links { for (to, start, end) in json_page.links {
let to = titles.insert(util::normalize_link(&to)); let to = titles.insert(util::normalize_link(&to));
result.links.push(Link { result.links.push(Link {
@ -86,6 +93,7 @@ fn first_stage() -> io::Result<(AdjacencyList<PageInfo, LinkInfo>, Titles)> {
data: LinkInfo { start, end }, data: LinkInfo { start, end },
}); });
} }
}
if (i + 1) % 100_000 == 0 { if (i + 1) % 100_000 == 0 {
eprintln!("{} pages imported", i + 1) eprintln!("{} pages imported", i + 1)