Garmelon/wikilyze
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Remove PageIdx and LinkIdx again

Browse source 
I don't think the type safety is worth the effort right now.
This commit is contained in:
Joscha 2024-08-25 22:45:20 +02:00
parent 17b118693f
commit 76abf5ea6e
7 changed files with 113 additions and 163 deletions
Download patch file Download diff file Expand all files Collapse all files

60
brood/src/commands/ingest.rs
View file

@ -7,9 +7,7 @@ use std::u32;
use rustc_hash::FxHashMap;
use serde::Deserialize;
use crate::data::adjacency_list::{
AdjacencyList, Link, LinkIdx, Page, PageIdx, SENTINEL_PAGE_MARKER,
};
use crate::data::adjacency_list::{AdjacencyList, Page};
use crate::data::info::{LinkInfo, PageInfo};
use crate::data::store;
use crate::util;
@ -74,29 +72,20 @@ fn first_stage() -> io::Result<(AdjacencyList<PageInfo, LinkInfo>, Titles)> {
for (i, line) in stdin.lines().enumerate() {
let json_page = serde_json::from_str::<JsonPage>(&line?).unwrap();
result.pages.push(Page {
start: LinkIdx(result.links.len() as u32),
data: PageInfo {
id: json_page.id,
length: json_page.length,
redirect: json_page.redirect.is_some(),
title: json_page.title,
},
result.push_page(PageInfo {
id: json_page.id,
length: json_page.length,
redirect: json_page.redirect.is_some(),
title: json_page.title,
});
if let Some(to) = json_page.redirect {
let to = titles.insert(util::normalize_link(&to));
result.links.push(Link {
to: PageIdx(to),
data: LinkInfo::default(),
});
result.push_link(to, LinkInfo::default());
} else {
for (to, start, len, flags) in json_page.links {
let to = titles.insert(util::normalize_link(&to));
result.links.push(Link {
to: PageIdx(to),
data: LinkInfo { start, len, flags },
});
result.push_link(to, LinkInfo { start, len, flags });
}
}
@ -110,16 +99,6 @@ fn first_stage() -> io::Result<(AdjacencyList<PageInfo, LinkInfo>, Titles)> {
eprintln!("Titles: {}", titles.titles.len());
eprintln!("Title map entries: {}", titles.map.len());
result.pages.push(Page {
start: LinkIdx(result.links.len() as u32),
data: PageInfo {
id: u32::MAX,
length: 0,
redirect: false,
title: SENTINEL_PAGE_MARKER.to_string(),
},
});
Ok((result, titles))
}
@ -151,26 +130,19 @@ fn second_stage(
let pages_map = initialize_pages_map(&first_stage.pages);
let mut result = AdjacencyList::default();
for page_idx in 0..first_stage.pages.len() - 1 {
let mut page = first_stage.pages[page_idx].clone();
let start_link_idx = page.start;
let end_link_idx = first_stage.pages[page_idx + 1].start;
for (page_idx, page) in first_stage.pages() {
result.push_page(page.data.clone());
page.start.0 = result.links.len() as u32;
result.pages.push(page);
for link_idx in start_link_idx.0..end_link_idx.0 {
let mut link = first_stage.links[link_idx as usize];
let title = util::normalize_link(titles.get(link.to.0));
for (_, link) in first_stage.links(page_idx) {
let title = util::normalize_link(titles.get(link.to));
if let Some(to) = pages_map.get(&title) {
// The link points to an existing article, we should keep it
link.to.0 = *to;
result.links.push(link);
result.push_link(*to, link.data);
}
}
if (page_idx + 1) % 100_000 == 0 {
eprintln!("{} pages processed", page_idx + 1)
eprintln!("{} pages imported", page_idx + 1)
}
}
@ -178,10 +150,6 @@ fn second_stage(
eprintln!("Links: {}", result.links.len());
eprintln!("Page map entries: {}", pages_map.len());
let mut sentinel = first_stage.pages.last().unwrap().clone();
sentinel.start.0 = result.links.len() as u32;
result.pages.push(sentinel);
result
}

4
brood/src/commands/list_pages.rs
View file

@ -2,15 +2,13 @@ use std::fs::File;
use std::io::{self, BufReader};
use std::path::Path;
use crate::data::adjacency_list::PageIdx;
use crate::data::store;
pub fn run(datafile: &Path) -> io::Result<()> {
let mut databuf = BufReader::new(File::open(datafile)?);
let data = store::read_adjacency_list(&mut databuf)?;
for (page_idx, page) in data.pages.iter().enumerate() {
let page_idx = PageIdx(page_idx as u32);
for (page_idx, page) in data.pages() {
if page.data.redirect {
for link_idx in data.link_range(page_idx) {
let target_page = data.page(data.link(link_idx).to);

52
brood/src/commands/longest_shortest_path.rs
View file

@ -3,14 +3,15 @@ use std::fs::File;
use std::io::{self, BufReader};
use std::path::Path;
use crate::data::adjacency_list::{AdjacencyList, PageIdx};
use crate::data::adjacency_list::AdjacencyList;
use crate::data::info::{LinkInfo, PageInfo};
use crate::data::store;
use crate::util;
struct DijkstraPageInfo {
cost: u32,
prev: PageIdx,
/// Index of the previous page.
prev: u32,
redirect: bool,
}
@ -18,7 +19,7 @@ impl DijkstraPageInfo {
fn from_page_info(info: PageInfo) -> Self {
Self {
cost: u32::MAX,
prev: PageIdx::MAX,
prev: u32::MAX,
redirect: info.redirect,
}
}
@ -42,12 +43,12 @@ impl DijkstraLinkInfo {
#[derive(Clone, Copy, PartialEq, Eq)]
struct Entry {
cost: u32,
idx: PageIdx,
page_idx: u32,
}
impl Entry {
pub fn new(cost: u32, idx: PageIdx) -> Self {
Self { cost, idx }
pub fn new(cost: u32, page_idx: u32) -> Self {
Self { cost, page_idx }
}
}
@ -57,7 +58,7 @@ impl Ord for Entry {
other
.cost
.cmp(&self.cost)
.then_with(|| self.idx.cmp(&other.idx))
.then_with(|| self.page_idx.cmp(&other.page_idx))
}
}
@ -70,22 +71,18 @@ impl PartialOrd for Entry {
/// Closely matches the dijkstra example in [std::collections::binary_heap].
fn full_dijkstra(
data: AdjacencyList<PageInfo, LinkInfo>,
from_idx: PageIdx,
from: u32,
) -> AdjacencyList<DijkstraPageInfo, DijkstraLinkInfo> {
println!("> Prepare state");
let mut data = data
.change_page_data(DijkstraPageInfo::from_page_info)
.change_link_data(DijkstraLinkInfo::from_link_info);
let mut queue = BinaryHeap::new();
data.page_mut(from_idx).data.cost = 0;
queue.push(Entry::new(0, from_idx));
data.page_mut(from).data.cost = 0;
queue.push(Entry::new(0, from));
println!("> Run dijkstra");
while let Some(Entry {
cost,
idx: page_idx,
}) = queue.pop()
{
while let Some(Entry { cost, page_idx }) = queue.pop() {
let page = data.page(page_idx);
if cost > page.data.cost {
// This queue entry is outdated
@ -98,7 +95,7 @@ fn full_dijkstra(
let next = Entry {
cost: cost + if redirect { 0 } else { link.data.cost },
idx: link.to,
page_idx: link.to,
};
let target_page = data.page_mut(link.to);
@ -115,23 +112,22 @@ fn full_dijkstra(
fn find_longest_shortest_path(
data: AdjacencyList<DijkstraPageInfo, DijkstraLinkInfo>,
from: PageIdx,
) -> Option<Vec<PageIdx>> {
let to = PageIdx(
data.pages
.iter()
.enumerate()
.filter(|(_, p)| p.data.cost != u32::MAX)
.max_by_key(|(_, p)| p.data.cost)?
.0 as u32,
);
from: u32,
) -> Option<Vec<u32>> {
let to = data
.pages
.iter()
.enumerate()
.filter(|(_, p)| p.data.cost != u32::MAX)
.max_by_key(|(_, p)| p.data.cost)?
.0 as u32;
let mut steps = vec![];
let mut at = to;
loop {
steps.push(at);
at = data.page(at).data.prev;
if at == PageIdx::MAX {
if at == u32::MAX {
break;
};
}
@ -162,7 +158,7 @@ pub fn run(datafile: &Path, from: &str) -> io::Result<()> {
if let Some(path) = path {
println!("Path found:");
for page_idx in path {
let page = &pages[page_idx.0 as usize];
let page = &pages[page_idx as usize];
if page.data.redirect {
println!(" v {:?}", page.data.title);
} else {

32
brood/src/commands/path.rs
View file

@ -3,14 +3,14 @@ use std::fs::File;
use std::io::{self, BufReader};
use std::path::Path;
use crate::data::adjacency_list::{AdjacencyList, PageIdx};
use crate::data::adjacency_list::AdjacencyList;
use crate::data::info::{LinkInfo, PageInfo};
use crate::data::store;
use crate::util;
struct DijkstraPageInfo {
cost: u32,
prev: PageIdx,
prev: u32,
redirect: bool,
}
@ -18,7 +18,7 @@ impl DijkstraPageInfo {
fn from_page_info(info: PageInfo) -> Self {
Self {
cost: u32::MAX,
prev: PageIdx::MAX,
prev: u32::MAX,
redirect: info.redirect,
}
}
@ -42,12 +42,12 @@ impl DijkstraLinkInfo {
#[derive(Clone, Copy, PartialEq, Eq)]
struct Entry {
cost: u32,
idx: PageIdx,
page_idx: u32,
}
impl Entry {
pub fn new(cost: u32, idx: PageIdx) -> Self {
Self { cost, idx }
pub fn new(cost: u32, page_idx: u32) -> Self {
Self { cost, page_idx }
}
}
@ -57,7 +57,7 @@ impl Ord for Entry {
other
.cost
.cmp(&self.cost)
.then_with(|| self.idx.cmp(&other.idx))
.then_with(|| self.page_idx.cmp(&other.page_idx))
}
}
@ -68,11 +68,7 @@ impl PartialOrd for Entry {
}
/// Closely matches the dijkstra example in [std::collections::binary_heap].
fn dijkstra(
data: AdjacencyList<PageInfo, LinkInfo>,
from: PageIdx,
to: PageIdx,
) -> Option<Vec<PageIdx>> {
fn dijkstra(data: AdjacencyList<PageInfo, LinkInfo>, from: u32, to: u32) -> Option<Vec<u32>> {
println!("> Prepare state");
let mut data = data
.change_page_data(DijkstraPageInfo::from_page_info)
@ -82,11 +78,7 @@ fn dijkstra(
queue.push(Entry::new(0, from));
println!("> Run dijkstra");
while let Some(Entry {
cost,
idx: page_idx,
}) = queue.pop()
{
while let Some(Entry { cost, page_idx }) = queue.pop() {
if page_idx == to {
// We've found the shortest path to our target
break;
@ -104,7 +96,7 @@ fn dijkstra(
let next = Entry {
cost: cost + if redirect { 0 } else { link.data.cost },
idx: link.to,
page_idx: link.to,
};
let target_page = data.page_mut(link.to);
@ -122,7 +114,7 @@ fn dijkstra(
loop {
steps.push(at);
at = data.page(at).data.prev;
if at == PageIdx::MAX {
if at == u32::MAX {
break;
};
}
@ -152,7 +144,7 @@ pub fn path(datafile: &Path, from: &str, to: &str) -> io::Result<()> {
if let Some(path) = path {
println!("Path found:");
for page_idx in path {
let page = &pages[page_idx.0 as usize];
let page = &pages[page_idx as usize];
if page.data.redirect {
println!(" v {:?}", page.data.title);
} else {