From 3842aa9940ce29e52b3905aed8f322d359d6deca Mon Sep 17 00:00:00 2001 From: Michael Chen <920901835@qq.com> Date: Mon, 21 Oct 2019 18:38:46 +0800 Subject: [PATCH 01/25] feat: Edge partition by edge or node label has been supported in StaticGraph --- src/graph_impl/graph_map/graph.rs | 2 +- src/graph_impl/graph_vec/mod.rs | 4 +- src/graph_impl/static_graph/edge_vec.rs | 2 + src/graph_impl/static_graph/graph.rs | 313 ++++++++++++++++-- src/graph_impl/static_graph/mod.rs | 1 + src/graph_impl/static_graph/sorted_adj_vec.rs | 61 ++++ src/lib.rs | 2 + tests/static_graph.rs | 75 ++++- 8 files changed, 432 insertions(+), 28 deletions(-) create mode 100644 src/graph_impl/static_graph/sorted_adj_vec.rs diff --git a/src/graph_impl/graph_map/graph.rs b/src/graph_impl/graph_map/graph.rs index 3b714eef..82f0b670 100644 --- a/src/graph_impl/graph_map/graph.rs +++ b/src/graph_impl/graph_map/graph.rs @@ -680,7 +680,7 @@ impl GeneralGraph +impl TypedGraphMap { pub fn reorder_id( diff --git a/src/graph_impl/graph_vec/mod.rs b/src/graph_impl/graph_vec/mod.rs index 397a42b9..51a5e135 100644 --- a/src/graph_impl/graph_vec/mod.rs +++ b/src/graph_impl/graph_vec/mod.rs @@ -45,7 +45,9 @@ pub struct TypedGraphVec TypedGraphVec { +impl + TypedGraphVec +{ pub fn new() -> Self { TypedGraphVec { nodes: HashMap::new(), diff --git a/src/graph_impl/static_graph/edge_vec.rs b/src/graph_impl/static_graph/edge_vec.rs index b635fd15..42d7ff0d 100644 --- a/src/graph_impl/static_graph/edge_vec.rs +++ b/src/graph_impl/static_graph/edge_vec.rs @@ -238,6 +238,7 @@ pub trait EdgeVecTrait { } impl EdgeVec { + // no label pub fn new(offsets: Vec, edges: Vec) -> Self { EdgeVec { offsets: offsets.into(), @@ -246,6 +247,7 @@ impl EdgeVec { } } + // with label pub fn with_labels(offsets: Vec, edges: Vec, labels: Vec) -> Self { if edges.len() != labels.len() { panic!( diff --git a/src/graph_impl/static_graph/graph.rs b/src/graph_impl/static_graph/graph.rs index 394a369e..eea6fba4 100644 --- a/src/graph_impl/static_graph/graph.rs +++ b/src/graph_impl/static_graph/graph.rs @@ -18,6 +18,8 @@ * specific language governing permissions and limitations * under the License. */ +#![feature(test)] + use std::borrow::Cow; use std::hash::{Hash, Hasher}; use std::marker::PhantomData; @@ -26,16 +28,21 @@ use itertools::Itertools; use serde; use generic::{ - DefaultId, DefaultTy, DiGraphTrait, Directed, EdgeType, GeneralGraph, GraphLabelTrait, - GraphTrait, GraphType, IdType, Iter, MapTrait, MutMapTrait, NodeType, UnGraphTrait, Undirected, + DefaultId, DefaultTy, DiGraphTrait, Directed, EdgeTrait, EdgeType, GeneralGraph, + GraphLabelTrait, GraphTrait, GraphType, IdType, Iter, MapTrait, MutMapTrait, NodeTrait, + NodeType, UnGraphTrait, Undirected, }; use graph_impl::static_graph::node::StaticNode; +use graph_impl::static_graph::sorted_adj_vec::SortedAdjVec; use graph_impl::static_graph::static_edge_iter::StaticEdgeIndexIter; use graph_impl::static_graph::{EdgeVec, EdgeVecTrait}; use graph_impl::{Edge, GraphImpl}; +use hashbrown::HashMap; use io::serde::{Deserialize, Serialize}; use map::SetMap; +use std::cmp; use std::ops::Add; +use test::Options; pub type TypedUnStaticGraph = TypedStaticGraph; pub type TypedDiStaticGraph = TypedStaticGraph; @@ -47,11 +54,28 @@ pub type DiStaticGraph = StaticGraph -{ +pub struct TypedStaticGraph< + Id: IdType, + NL: Hash + Eq + Clone, + EL: Hash + Eq + Clone, + Ty: GraphType, + L: IdType = Id, +> { num_nodes: usize, num_edges: usize, + + // node Ids indexed by type and random access to node types. + node_ids: Vec, + // node_types[node_id] = node_label_id + node_types: Vec, + node_type_offsets: Vec, + + fwd_adj_lists: Vec>>, + bwd_adj_lists: Vec>>, + + // bwd_adj_vec edge_vec: EdgeVec, + // fwd_adj_vec in_edge_vec: Option>, // Maintain the node's labels, whose index is aligned with `offsets`. labels: Option>, @@ -63,7 +87,7 @@ pub struct TypedStaticGraph, } -impl PartialEq +impl PartialEq for TypedStaticGraph { fn eq(&self, other: &TypedStaticGraph) -> bool { @@ -87,12 +111,12 @@ impl Partial } } -impl Eq +impl Eq for TypedStaticGraph { } -impl Hash +impl Hash for TypedStaticGraph { fn hash(&self, state: &mut H) { @@ -118,7 +142,7 @@ impl Hash } } -impl Serialize +impl Serialize for TypedStaticGraph where Id: serde::Serialize, @@ -128,7 +152,7 @@ where { } -impl Deserialize +impl Deserialize for TypedStaticGraph where Id: for<'de> serde::Deserialize<'de>, @@ -138,13 +162,14 @@ where { } -impl +impl TypedStaticGraph { pub fn empty() -> Self { Self::new(EdgeVec::default(), None, None, None) } + //without node label and edge label pub fn new( edges: EdgeVec, in_edges: Option>, @@ -187,16 +212,24 @@ impl edges.num_edges() >> 1 }; - TypedStaticGraph { + let mut g = TypedStaticGraph { num_nodes, num_edges, + node_ids: vec![], + node_types: vec![], + node_type_offsets: vec![], + fwd_adj_lists: vec![], + bwd_adj_lists: vec![], edge_vec: edges, in_edge_vec: in_edges, labels: None, node_label_map: SetMap::::new(), edge_label_map: SetMap::::new(), graph_type: PhantomData, - } + }; + g.load_vertices(); + g.load_edges(); + g } pub fn with_labels( @@ -248,16 +281,26 @@ impl debug!("{} nodes, but {} labels", num_nodes, labels.len()); } - TypedStaticGraph { + if edge_label_map.len() != 0 {} + + let mut g = TypedStaticGraph { num_nodes, num_edges, + node_ids: vec![], + node_types: vec![], + node_type_offsets: vec![], + fwd_adj_lists: vec![], + bwd_adj_lists: vec![], edge_vec: edges, in_edge_vec: in_edges, labels: Some(labels), node_label_map, edge_label_map, graph_type: PhantomData, - } + }; + g.load_vertices(); + g.load_edges(); + g } pub fn from_raw( @@ -318,16 +361,24 @@ impl } } - TypedStaticGraph { + let mut g = TypedStaticGraph { num_nodes, num_edges, + node_ids: vec![], + node_types: vec![], + node_type_offsets: vec![], + fwd_adj_lists: vec![], + bwd_adj_lists: vec![], edge_vec, in_edge_vec, labels, node_label_map, edge_label_map, graph_type: PhantomData, - } + }; + g.load_vertices(); + g.load_edges(); + g } #[inline] @@ -415,10 +466,217 @@ impl pub fn find_edge_index(&self, start: Id, target: Id) -> Option { self.edge_vec.find_edge_index(start, target) } + + fn load_vertices(&mut self) { + if 0 == self.num_of_node_labels() { + let mut node_ids = vec![0; self.num_nodes]; + for i in 0..self.num_nodes { + node_ids[i] = i; + } + self.node_ids = node_ids; + self.node_types = vec![0; self.num_nodes]; + self.node_type_offsets = vec![0, self.num_nodes]; + return; + } + let offsets = self.get_node_offsets(); + let num_vertices = offsets[offsets.len() - 1] as usize; + + let mut node_ids = vec![0; num_vertices]; + let mut node_types = vec![0; num_vertices]; + let mut curr_idx_by_type = vec![0; offsets.len() - 1]; + self.node_indices().for_each(|id| { + let node_id = id.id(); + let node_label_id = self + .get_node(id) + .get_label_id() + .unwrap_or(IdType::new(0)) + .id(); + node_ids[offsets[node_label_id] + curr_idx_by_type[node_label_id]] = node_id; + curr_idx_by_type[node_label_id] += 1; + node_types[node_id] = node_label_id; + }); + self.node_ids = node_ids; + self.node_types = node_types; + self.node_type_offsets = offsets; + } + + fn load_edges(&mut self) { + let highest_node_id = self.num_nodes - 1; + let sort_by_node = self.num_of_edge_labels() == 0 && self.num_of_node_labels() > 0; + let (fwd_adj_meta_data, bwd_adj_meta_data) = self.get_adj_meta_data(sort_by_node); + let num_vertices = highest_node_id + 1; + let mut fwd_adj_lists: Vec>> = vec![Option::None; num_vertices]; + let mut bwd_adj_lists: Vec>> = vec![Option::None; num_vertices]; + let mut fwd_adj_list_curr_idx = HashMap::new(); + let mut bwd_adj_list_curr_idx = HashMap::new(); + let offset_size = { + if sort_by_node { + self.num_of_node_labels() + } else { + self.num_of_edge_labels() + } + }; + for node_id in 0..num_vertices { + fwd_adj_lists[node_id] = Some(SortedAdjVec::new( + fwd_adj_meta_data.get(&node_id).unwrap().to_owned(), + )); + fwd_adj_list_curr_idx.insert(node_id, vec![0; offset_size + 1 as usize]); + bwd_adj_lists[node_id] = Some(SortedAdjVec::new( + bwd_adj_meta_data.get(&node_id).unwrap().to_owned(), + )); + bwd_adj_list_curr_idx.insert(node_id, vec![0; offset_size + 1 as usize]); + } + self.edge_indices().for_each(|(from, to)| { + let label_id = self + .get_edge(from, to) + .get_label_id() + .unwrap_or(IdType::new(0)) + .id(); + let from_type_or_label = { + if sort_by_node { + self.node_types[from.id()] + } else { + label_id + } + }; + let to_type_or_label = { + if sort_by_node { + self.node_types[to.id()] + } else { + label_id + } + }; + let mut idx = fwd_adj_list_curr_idx.get(&from.id()).unwrap()[to_type_or_label]; + let mut offset = fwd_adj_meta_data.get(&from.id()).unwrap()[to_type_or_label]; + fwd_adj_list_curr_idx.get_mut(&from.id()).unwrap()[to_type_or_label] += 1; + fwd_adj_lists[from.id()] + .as_mut() + .unwrap() + .set_neighbor_id(to, idx + offset); + idx = bwd_adj_list_curr_idx.get(&to.id()).unwrap()[from_type_or_label]; + offset = bwd_adj_meta_data.get(&to.id()).unwrap()[from_type_or_label]; + bwd_adj_list_curr_idx.get_mut(&to.id()).unwrap()[from_type_or_label] += 1; + bwd_adj_lists[to.id()] + .as_mut() + .unwrap() + .set_neighbor_id(from, idx + offset); + }); + + for node_id in 0..num_vertices { + fwd_adj_lists[node_id].as_mut().unwrap().sort(); + bwd_adj_lists[node_id].as_mut().unwrap().sort(); + } + + self.fwd_adj_lists = fwd_adj_lists; + self.bwd_adj_lists = bwd_adj_lists; + } + + fn get_node_offsets(&mut self) -> Vec { + let mut type_to_count_map: HashMap = HashMap::new(); + self.node_indices().for_each(|x| { + let label_id = self + .get_node(x) + .get_label_id() + .unwrap_or(IdType::new(0)) + .id(); + let default_v = 0; + let v = type_to_count_map.get(&label_id).unwrap_or(&default_v); + type_to_count_map.insert(label_id, v + 1); + }); + + let next_node_label_key = self.num_of_node_labels(); + let mut offsets = vec![0; next_node_label_key + 1]; + type_to_count_map.iter().for_each(|(label_id, cnt)| { + let label_id = label_id.to_owned(); + let label_cnt = cnt.to_owned(); + + if label_id < next_node_label_key - 1 { + offsets[label_id + 1] = label_cnt.to_owned(); + } + offsets[next_node_label_key] += label_cnt; + }); + for i in 1..offsets.len() - 1 { + offsets[i] += offsets[i - 1]; + } + offsets + } + + fn get_adj_meta_data( + &self, + sort_by_node: bool, + ) -> (HashMap>, HashMap>) { + let mut fwd_adj_list_metadata = HashMap::new(); + let mut bwd_adj_list_metadata = HashMap::new(); + let next_label_or_type = { + if sort_by_node { + cmp::max(self.num_of_node_labels(), 1) + } else { + cmp::max(self.num_of_edge_labels(), 1) + } + }; + for i in 0..self.node_count() { + fwd_adj_list_metadata.insert(i, vec![0; next_label_or_type + 1]); + bwd_adj_list_metadata.insert(i, vec![0; next_label_or_type + 1]); + } + self.edge_indices().for_each(|(from, to)| { + if sort_by_node { + let from_type = self.node_types[from.id()]; + let to_type = self.node_types[to.id()]; + fwd_adj_list_metadata.get_mut(&from.id()).unwrap()[to_type + 1] += 1; + bwd_adj_list_metadata.get_mut(&to.id()).unwrap()[from_type + 1] += 1; + } else { + let from_label_id = self + .get_edge(from, to) + .get_label_id() + .unwrap_or(IdType::new(0)) + .id(); + let to_label_id = self + .get_edge(to, from) + .get_label_id() + .unwrap_or(IdType::new(0)) + .id(); + fwd_adj_list_metadata.get_mut(&from.id()).unwrap()[from_label_id + 1] += 1; + bwd_adj_list_metadata.get_mut(&to.id()).unwrap()[to_label_id + 1] += 1; + } + }); + fwd_adj_list_metadata.iter_mut().for_each(|(_id, offsets)| { + for i in 1..offsets.len() - 1 { + offsets[next_label_or_type] += offsets[i]; + offsets[i] += offsets[i - 1]; + } + }); + bwd_adj_list_metadata.iter_mut().for_each(|(_id, offsets)| { + for i in 1..offsets.len() - 1 { + offsets[next_label_or_type] += offsets[i]; + offsets[i] += offsets[i - 1]; + } + }); + (fwd_adj_list_metadata, bwd_adj_list_metadata) + } + + pub fn get_node_ids(&self) -> &Vec { + self.node_ids.as_ref() + } + + pub fn get_node_types(&self) -> &Vec { + self.node_types.as_ref() + } + + pub fn get_node_type_offsets(&self) -> &Vec { + self.node_type_offsets.as_ref() + } + + pub fn get_fwd_adj_list(&self) -> &Vec>> { + self.fwd_adj_lists.as_ref() + } + + pub fn get_bwd_adj_list(&self) -> &Vec>> { + self.bwd_adj_lists.as_ref() + } } -impl GraphTrait - for TypedStaticGraph +impl + GraphTrait for TypedStaticGraph { #[inline] fn get_node(&self, id: Id) -> NodeType { @@ -557,7 +815,7 @@ impl GraphTr } } -impl +impl GraphLabelTrait for TypedStaticGraph { #[inline(always)] @@ -571,12 +829,12 @@ impl } } -impl UnGraphTrait +impl UnGraphTrait for TypedUnStaticGraph { } -impl DiGraphTrait +impl DiGraphTrait for TypedDiStaticGraph { #[inline] @@ -597,8 +855,8 @@ impl DiGraphTrait } } -impl GeneralGraph - for TypedUnStaticGraph +impl + GeneralGraph for TypedUnStaticGraph { #[inline(always)] fn as_graph(&self) -> &GraphTrait { @@ -616,8 +874,8 @@ impl GeneralGraph GeneralGraph - for TypedDiStaticGraph +impl + GeneralGraph for TypedDiStaticGraph { #[inline(always)] fn as_graph(&self) -> &GraphTrait { @@ -681,6 +939,11 @@ impl None, diff --git a/src/graph_impl/static_graph/mod.rs b/src/graph_impl/static_graph/mod.rs index 39bee9b2..e1d30e68 100644 --- a/src/graph_impl/static_graph/mod.rs +++ b/src/graph_impl/static_graph/mod.rs @@ -21,6 +21,7 @@ pub mod edge_vec; pub mod graph; pub mod node; +pub mod sorted_adj_vec; pub mod static_edge_iter; pub use graph_impl::static_graph::edge_vec::{EdgeVec, EdgeVecTrait}; diff --git a/src/graph_impl/static_graph/sorted_adj_vec.rs b/src/graph_impl/static_graph/sorted_adj_vec.rs new file mode 100644 index 00000000..43f17e5f --- /dev/null +++ b/src/graph_impl/static_graph/sorted_adj_vec.rs @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2018 UNSW Sydney, Data and Knowledge Group. + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +use generic::IdType; + +#[derive(Debug, Hash, PartialEq, Eq, Clone, Serialize, Deserialize)] +pub struct SortedAdjVec { + label_or_type_offsets: Vec, + neighbour_ids: Vec, +} + +impl SortedAdjVec { + pub fn new(offset: Vec) -> Self { + let len = offset[offset.len() - 1]; + Self { + label_or_type_offsets: offset, + neighbour_ids: vec![IdType::new(0); len], + } + } + + pub fn get_neighbor_id(&self, idx: Id) -> Id { + self.neighbour_ids[idx.id()] + } + + pub fn set_neighbor_id(&mut self, neighbor_id: Id, idx: usize) { + self.neighbour_ids[idx] = neighbor_id + } + + pub fn get_offsets(&self) -> &Vec { + self.label_or_type_offsets.as_ref() + } + + pub fn get_neighbour_ids(&self) -> &Vec { + self.neighbour_ids.as_ref() + } + + pub fn sort(&mut self) { + for i in 0..self.label_or_type_offsets.len() - 1 { + let block = &mut self.neighbour_ids + [self.label_or_type_offsets[i]..self.label_or_type_offsets[i + 1]]; + block.sort(); + } + } +} diff --git a/src/lib.rs b/src/lib.rs index 13186999..67d37225 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -18,6 +18,7 @@ * specific language governing permissions and limitations * under the License. */ +#![feature(test)] extern crate bincode; extern crate counter; extern crate csv; @@ -39,6 +40,7 @@ extern crate serde_derive; #[cfg(feature = "hdfs")] extern crate hdfs; +extern crate test; pub mod algorithm; pub mod generic; diff --git a/tests/static_graph.rs b/tests/static_graph.rs index d092d2e9..409c2339 100644 --- a/tests/static_graph.rs +++ b/tests/static_graph.rs @@ -36,6 +36,30 @@ fn test_directed() { let in_edge_vec = EdgeVec::new(vec![0, 2, 3, 4], vec![1, 2, 0, 0]); let g = DiStaticGraph::::new(edge_vec, Some(in_edge_vec), None, None); + assert_eq!(g.get_node_ids(), &vec![0, 1, 2]); + assert_eq!(g.get_node_types(), &vec![0, 0, 0]); + assert_eq!(g.get_node_type_offsets(), &vec![0, 3]); + + let fwd_adj_list = g.get_fwd_adj_list()[0].as_ref().unwrap(); + assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 2]); + assert_eq!(fwd_adj_list.get_neighbour_ids(), &vec![1, 2]); + let fwd_adj_list = g.get_fwd_adj_list()[1].as_ref().unwrap(); + assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 1]); + assert_eq!(fwd_adj_list.get_neighbour_ids(), &vec![0]); + let fwd_adj_list = g.get_fwd_adj_list()[2].as_ref().unwrap(); + assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 1]); + assert_eq!(fwd_adj_list.get_neighbour_ids(), &vec![0]); + + let bwd_adj_list = g.get_bwd_adj_list()[0].as_ref().unwrap(); + assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 2]); + assert_eq!(bwd_adj_list.get_neighbour_ids(), &vec![1, 2]); + let bwd_adj_list = g.get_bwd_adj_list()[1].as_ref().unwrap(); + assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 1]); + assert_eq!(bwd_adj_list.get_neighbour_ids(), &vec![0]); + let bwd_adj_list = g.get_bwd_adj_list()[2].as_ref().unwrap(); + assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 1]); + assert_eq!(bwd_adj_list.get_neighbour_ids(), &vec![0]); + assert_eq!(g.node_count(), 3); assert_eq!(g.edge_count(), 4); @@ -95,7 +119,32 @@ fn test_undirected() { let edge_vec = EdgeVec::new(vec![0, 2, 4, 6], vec![1, 2, 0, 2, 0, 1]); let g = UnStaticGraph::::new(edge_vec, None, None, None); let edges: Vec<_> = g.edge_indices().collect(); - assert_eq!(edges, vec![(0, 1), (0, 2), (1, 2)]) + assert_eq!(edges, vec![(0, 1), (0, 2), (1, 2)]); + + assert_eq!(g.get_node_ids(), &vec![0, 1, 2]); + // Without node labels + assert_eq!(g.get_node_types(), &vec![0, 0, 0]); + assert_eq!(g.get_node_type_offsets(), &vec![0, 3]); + + let fwd_adj_list = g.get_fwd_adj_list()[0].as_ref().unwrap(); + assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 2]); + assert_eq!(fwd_adj_list.get_neighbour_ids(), &vec![1, 2]); + let fwd_adj_list = g.get_fwd_adj_list()[1].as_ref().unwrap(); + assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 1]); + assert_eq!(fwd_adj_list.get_neighbour_ids(), &vec![2]); + let fwd_adj_list = g.get_fwd_adj_list()[2].as_ref().unwrap(); + assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 0]); + assert_eq!(fwd_adj_list.get_neighbour_ids().len(), 0); + + let bwd_adj_list = g.get_bwd_adj_list()[0].as_ref().unwrap(); + assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 0]); + assert_eq!(bwd_adj_list.get_neighbour_ids().len(), 0); + let bwd_adj_list = g.get_bwd_adj_list()[1].as_ref().unwrap(); + assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 1]); + assert_eq!(bwd_adj_list.get_neighbour_ids(), &vec![0]); + let bwd_adj_list = g.get_bwd_adj_list()[2].as_ref().unwrap(); + assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 2]); + assert_eq!(bwd_adj_list.get_neighbour_ids(), &vec![0, 1]); } #[test] @@ -156,6 +205,30 @@ fn test_labeled() { assert!(edges.contains(&g.get_edge(0, 2))); assert!(edges.contains(&g.get_edge(1, 0))); assert!(edges.contains(&g.get_edge(2, 0))); + + assert_eq!(g.get_node_ids(), &vec![1, 0, 2]); + assert_eq!(g.get_node_types(), &vec![1, 0, 1]); + assert_eq!(g.get_node_type_offsets(), &vec![0, 1, 3]); + + let fwd_adj_list = g.get_fwd_adj_list()[0].as_ref().unwrap(); + assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 1, 2]); + assert_eq!(fwd_adj_list.get_neighbour_ids(), &vec![1, 2]); + let fwd_adj_list = g.get_fwd_adj_list()[1].as_ref().unwrap(); + assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 1, 1]); + assert_eq!(fwd_adj_list.get_neighbour_ids(), &vec![0]); + let fwd_adj_list = g.get_fwd_adj_list()[2].as_ref().unwrap(); + assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 0, 1]); + assert_eq!(fwd_adj_list.get_neighbour_ids(), &vec![0]); + + let bwd_adj_list = g.get_bwd_adj_list()[0].as_ref().unwrap(); + assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 1, 2]); + assert_eq!(bwd_adj_list.get_neighbour_ids(), &vec![1, 2]); + let bwd_adj_list = g.get_bwd_adj_list()[1].as_ref().unwrap(); + assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 1, 1]); + assert_eq!(bwd_adj_list.get_neighbour_ids(), &vec![0]); + let bwd_adj_list = g.get_bwd_adj_list()[2].as_ref().unwrap(); + assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 0, 1]); + assert_eq!(bwd_adj_list.get_neighbour_ids(), &vec![0]); } #[test] From 9ecdbb158b055115e9ac0aff8e805fac53184b78 Mon Sep 17 00:00:00 2001 From: Michael Chen <920901835@qq.com> Date: Mon, 21 Oct 2019 18:48:12 +0800 Subject: [PATCH 02/25] fix: remove useless derive(Clone) for TypedStaticGraph --- src/graph_impl/graph_map/graph.rs | 2 +- src/graph_impl/graph_vec/mod.rs | 4 +-- src/graph_impl/static_graph/graph.rs | 41 ++++++++++++---------------- 3 files changed, 19 insertions(+), 28 deletions(-) diff --git a/src/graph_impl/graph_map/graph.rs b/src/graph_impl/graph_map/graph.rs index 82f0b670..3b714eef 100644 --- a/src/graph_impl/graph_map/graph.rs +++ b/src/graph_impl/graph_map/graph.rs @@ -680,7 +680,7 @@ impl GeneralGraph +impl TypedGraphMap { pub fn reorder_id( diff --git a/src/graph_impl/graph_vec/mod.rs b/src/graph_impl/graph_vec/mod.rs index 51a5e135..397a42b9 100644 --- a/src/graph_impl/graph_vec/mod.rs +++ b/src/graph_impl/graph_vec/mod.rs @@ -45,9 +45,7 @@ pub struct TypedGraphVec - TypedGraphVec -{ +impl TypedGraphVec { pub fn new() -> Self { TypedGraphVec { nodes: HashMap::new(), diff --git a/src/graph_impl/static_graph/graph.rs b/src/graph_impl/static_graph/graph.rs index eea6fba4..34ac868b 100644 --- a/src/graph_impl/static_graph/graph.rs +++ b/src/graph_impl/static_graph/graph.rs @@ -54,13 +54,8 @@ pub type DiStaticGraph = StaticGraph { +pub struct TypedStaticGraph +{ num_nodes: usize, num_edges: usize, @@ -73,9 +68,7 @@ pub struct TypedStaticGraph< fwd_adj_lists: Vec>>, bwd_adj_lists: Vec>>, - // bwd_adj_vec edge_vec: EdgeVec, - // fwd_adj_vec in_edge_vec: Option>, // Maintain the node's labels, whose index is aligned with `offsets`. labels: Option>, @@ -87,7 +80,7 @@ pub struct TypedStaticGraph< edge_label_map: SetMap, } -impl PartialEq +impl PartialEq for TypedStaticGraph { fn eq(&self, other: &TypedStaticGraph) -> bool { @@ -111,12 +104,12 @@ impl Eq +impl Eq for TypedStaticGraph { } -impl Hash +impl Hash for TypedStaticGraph { fn hash(&self, state: &mut H) { @@ -142,7 +135,7 @@ impl Serialize +impl Serialize for TypedStaticGraph where Id: serde::Serialize, @@ -152,7 +145,7 @@ where { } -impl Deserialize +impl Deserialize for TypedStaticGraph where Id: for<'de> serde::Deserialize<'de>, @@ -162,7 +155,7 @@ where { } -impl +impl TypedStaticGraph { pub fn empty() -> Self { @@ -675,8 +668,8 @@ impl - GraphTrait for TypedStaticGraph +impl GraphTrait + for TypedStaticGraph { #[inline] fn get_node(&self, id: Id) -> NodeType { @@ -815,7 +808,7 @@ impl +impl GraphLabelTrait for TypedStaticGraph { #[inline(always)] @@ -829,12 +822,12 @@ impl UnGraphTrait +impl UnGraphTrait for TypedUnStaticGraph { } -impl DiGraphTrait +impl DiGraphTrait for TypedDiStaticGraph { #[inline] @@ -855,8 +848,8 @@ impl DiGrap } } -impl - GeneralGraph for TypedUnStaticGraph +impl GeneralGraph + for TypedUnStaticGraph { #[inline(always)] fn as_graph(&self) -> &GraphTrait { @@ -874,8 +867,8 @@ impl } } -impl - GeneralGraph for TypedDiStaticGraph +impl GeneralGraph + for TypedDiStaticGraph { #[inline(always)] fn as_graph(&self) -> &GraphTrait { From 590d2a8694c0bb9133dd5dc9dd7efd3ac406a9e8 Mon Sep 17 00:00:00 2001 From: Michael Chen <920901835@qq.com> Date: Tue, 22 Oct 2019 16:07:27 +0800 Subject: [PATCH 03/25] fix: edge partition for undirect graph --- src/graph_impl/static_graph/graph.rs | 142 ++++++++++-------- src/graph_impl/static_graph/sorted_adj_vec.rs | 5 +- tests/static_graph.rs | 16 +- 3 files changed, 90 insertions(+), 73 deletions(-) diff --git a/src/graph_impl/static_graph/graph.rs b/src/graph_impl/static_graph/graph.rs index 34ac868b..1f9b017b 100644 --- a/src/graph_impl/static_graph/graph.rs +++ b/src/graph_impl/static_graph/graph.rs @@ -220,8 +220,8 @@ impl edge_label_map: SetMap::::new(), graph_type: PhantomData, }; - g.load_vertices(); - g.load_edges(); + g.partition_nodes(); + g.partition_edges(); g } @@ -291,8 +291,8 @@ impl edge_label_map, graph_type: PhantomData, }; - g.load_vertices(); - g.load_edges(); + g.partition_nodes(); + g.partition_edges(); g } @@ -369,8 +369,8 @@ impl edge_label_map, graph_type: PhantomData, }; - g.load_vertices(); - g.load_edges(); + g.partition_nodes(); + g.partition_edges(); g } @@ -460,7 +460,8 @@ impl self.edge_vec.find_edge_index(start, target) } - fn load_vertices(&mut self) { + // Partition nodes by type and generating node_ids && offsets for retrieving. + fn partition_nodes(&mut self) { if 0 == self.num_of_node_labels() { let mut node_ids = vec![0; self.num_nodes]; for i in 0..self.num_nodes { @@ -493,7 +494,8 @@ impl self.node_type_offsets = offsets; } - fn load_edges(&mut self) { + // Partition edges by edge label or node label(if there did not exist edge labels in graph) + fn partition_edges(&mut self) { let highest_node_id = self.num_nodes - 1; let sort_by_node = self.num_of_edge_labels() == 0 && self.num_of_node_labels() > 0; let (fwd_adj_meta_data, bwd_adj_meta_data) = self.get_adj_meta_data(sort_by_node); @@ -519,41 +521,48 @@ impl )); bwd_adj_list_curr_idx.insert(node_id, vec![0; offset_size + 1 as usize]); } - self.edge_indices().for_each(|(from, to)| { - let label_id = self - .get_edge(from, to) - .get_label_id() - .unwrap_or(IdType::new(0)) - .id(); - let from_type_or_label = { - if sort_by_node { - self.node_types[from.id()] - } else { - label_id - } - }; - let to_type_or_label = { - if sort_by_node { - self.node_types[to.id()] - } else { - label_id + self.edge_indices() + .flat_map(|(from, to)| { + if !Ty::is_directed() { + return vec![(from, to), (to, from)]; } - }; - let mut idx = fwd_adj_list_curr_idx.get(&from.id()).unwrap()[to_type_or_label]; - let mut offset = fwd_adj_meta_data.get(&from.id()).unwrap()[to_type_or_label]; - fwd_adj_list_curr_idx.get_mut(&from.id()).unwrap()[to_type_or_label] += 1; - fwd_adj_lists[from.id()] - .as_mut() - .unwrap() - .set_neighbor_id(to, idx + offset); - idx = bwd_adj_list_curr_idx.get(&to.id()).unwrap()[from_type_or_label]; - offset = bwd_adj_meta_data.get(&to.id()).unwrap()[from_type_or_label]; - bwd_adj_list_curr_idx.get_mut(&to.id()).unwrap()[from_type_or_label] += 1; - bwd_adj_lists[to.id()] - .as_mut() - .unwrap() - .set_neighbor_id(from, idx + offset); - }); + vec![(from, to)] + }) + .for_each(|(from, to)| { + let label_id = self + .get_edge(from, to) + .get_label_id() + .unwrap_or(IdType::new(0)) + .id(); + let from_type_or_label = { + if sort_by_node { + self.node_types[from.id()] + } else { + label_id + } + }; + let to_type_or_label = { + if sort_by_node { + self.node_types[to.id()] + } else { + label_id + } + }; + let mut idx = fwd_adj_list_curr_idx.get(&from.id()).unwrap()[to_type_or_label]; + let mut offset = fwd_adj_meta_data.get(&from.id()).unwrap()[to_type_or_label]; + fwd_adj_list_curr_idx.get_mut(&from.id()).unwrap()[to_type_or_label] += 1; + fwd_adj_lists[from.id()] + .as_mut() + .unwrap() + .set_neighbor_id(to, idx + offset); + idx = bwd_adj_list_curr_idx.get(&to.id()).unwrap()[from_type_or_label]; + offset = bwd_adj_meta_data.get(&to.id()).unwrap()[from_type_or_label]; + bwd_adj_list_curr_idx.get_mut(&to.id()).unwrap()[from_type_or_label] += 1; + bwd_adj_lists[to.id()] + .as_mut() + .unwrap() + .set_neighbor_id(from, idx + offset); + }); for node_id in 0..num_vertices { fwd_adj_lists[node_id].as_mut().unwrap().sort(); @@ -611,27 +620,34 @@ impl fwd_adj_list_metadata.insert(i, vec![0; next_label_or_type + 1]); bwd_adj_list_metadata.insert(i, vec![0; next_label_or_type + 1]); } - self.edge_indices().for_each(|(from, to)| { - if sort_by_node { - let from_type = self.node_types[from.id()]; - let to_type = self.node_types[to.id()]; - fwd_adj_list_metadata.get_mut(&from.id()).unwrap()[to_type + 1] += 1; - bwd_adj_list_metadata.get_mut(&to.id()).unwrap()[from_type + 1] += 1; - } else { - let from_label_id = self - .get_edge(from, to) - .get_label_id() - .unwrap_or(IdType::new(0)) - .id(); - let to_label_id = self - .get_edge(to, from) - .get_label_id() - .unwrap_or(IdType::new(0)) - .id(); - fwd_adj_list_metadata.get_mut(&from.id()).unwrap()[from_label_id + 1] += 1; - bwd_adj_list_metadata.get_mut(&to.id()).unwrap()[to_label_id + 1] += 1; - } - }); + self.edge_indices() + .flat_map(|(from, to)| { + if !Ty::is_directed() { + return vec![(from, to), (to, from)]; + } + vec![(from, to)] + }) + .for_each(|(from, to)| { + if sort_by_node { + let from_type = self.node_types[from.id()]; + let to_type = self.node_types[to.id()]; + fwd_adj_list_metadata.get_mut(&from.id()).unwrap()[to_type + 1] += 1; + bwd_adj_list_metadata.get_mut(&to.id()).unwrap()[from_type + 1] += 1; + } else { + let from_label_id = self + .get_edge(from, to) + .get_label_id() + .unwrap_or(IdType::new(0)) + .id(); + let to_label_id = self + .get_edge(to, from) + .get_label_id() + .unwrap_or(IdType::new(0)) + .id(); + fwd_adj_list_metadata.get_mut(&from.id()).unwrap()[from_label_id + 1] += 1; + bwd_adj_list_metadata.get_mut(&to.id()).unwrap()[to_label_id + 1] += 1; + } + }); fwd_adj_list_metadata.iter_mut().for_each(|(_id, offsets)| { for i in 1..offsets.len() - 1 { offsets[next_label_or_type] += offsets[i]; diff --git a/src/graph_impl/static_graph/sorted_adj_vec.rs b/src/graph_impl/static_graph/sorted_adj_vec.rs index 43f17e5f..b6c39dda 100644 --- a/src/graph_impl/static_graph/sorted_adj_vec.rs +++ b/src/graph_impl/static_graph/sorted_adj_vec.rs @@ -53,8 +53,9 @@ impl SortedAdjVec { pub fn sort(&mut self) { for i in 0..self.label_or_type_offsets.len() - 1 { - let block = &mut self.neighbour_ids - [self.label_or_type_offsets[i]..self.label_or_type_offsets[i + 1]]; + let block = self.neighbour_ids + [self.label_or_type_offsets[i]..self.label_or_type_offsets[i + 1]] + .as_mut(); block.sort(); } } diff --git a/tests/static_graph.rs b/tests/static_graph.rs index 409c2339..b6075345 100644 --- a/tests/static_graph.rs +++ b/tests/static_graph.rs @@ -130,18 +130,18 @@ fn test_undirected() { assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 2]); assert_eq!(fwd_adj_list.get_neighbour_ids(), &vec![1, 2]); let fwd_adj_list = g.get_fwd_adj_list()[1].as_ref().unwrap(); - assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 1]); - assert_eq!(fwd_adj_list.get_neighbour_ids(), &vec![2]); + assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 2]); + assert_eq!(fwd_adj_list.get_neighbour_ids(), &vec![0, 2]); let fwd_adj_list = g.get_fwd_adj_list()[2].as_ref().unwrap(); - assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 0]); - assert_eq!(fwd_adj_list.get_neighbour_ids().len(), 0); + assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 2]); + assert_eq!(fwd_adj_list.get_neighbour_ids(), &vec![0, 1]); let bwd_adj_list = g.get_bwd_adj_list()[0].as_ref().unwrap(); - assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 0]); - assert_eq!(bwd_adj_list.get_neighbour_ids().len(), 0); + assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 2]); + assert_eq!(bwd_adj_list.get_neighbour_ids(), &vec![1, 2]); let bwd_adj_list = g.get_bwd_adj_list()[1].as_ref().unwrap(); - assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 1]); - assert_eq!(bwd_adj_list.get_neighbour_ids(), &vec![0]); + assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 2]); + assert_eq!(bwd_adj_list.get_neighbour_ids(), &vec![0, 2]); let bwd_adj_list = g.get_bwd_adj_list()[2].as_ref().unwrap(); assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 2]); assert_eq!(bwd_adj_list.get_neighbour_ids(), &vec![0, 1]); From 200bbba0730332a17309248051a04f90059885b2 Mon Sep 17 00:00:00 2001 From: Michael Chen <920901835@qq.com> Date: Sun, 27 Oct 2019 18:08:58 +0800 Subject: [PATCH 04/25] fix:bugs --- src/graph_impl/graph_vec/mod.rs | 11 ++- src/graph_impl/static_graph/graph.rs | 139 +++++++++++++-------------- tests/static_graph.rs | 44 ++++----- 3 files changed, 93 insertions(+), 101 deletions(-) diff --git a/src/graph_impl/graph_vec/mod.rs b/src/graph_impl/graph_vec/mod.rs index 397a42b9..b611512b 100644 --- a/src/graph_impl/graph_vec/mod.rs +++ b/src/graph_impl/graph_vec/mod.rs @@ -157,7 +157,6 @@ impl TypedGraphVec::new(); g.add_node(0, Some("node0")); g.add_node(2, Some("node2")); - g.add_node(2, Some("node2")); g.add_edge(0, 1, Some("(0,1)")); g.add_edge(1, 0, Some("(0,1)")); g.add_edge(0, 3, Some("(0,3)")); + g.add_edge(3, 0, Some("(0,3)")); let un_graph = g.clone().into_static::(); let un_graph_true = UnStaticGraph::<&str, &str, u16>::from_raw( 4, - 1, - EdgeVec::with_labels(vec![0, 2, 3, 3, 3], vec![1, 3, 0], vec![0, 1, 0]), + 2, + EdgeVec::with_labels(vec![0, 2, 3, 3, 4], vec![1, 3, 0, 0], vec![0, 1, 0, 1]), None, Some(vec![0, u16::max_value(), 1, u16::max_value()]), vec!["node0", "node2"].into(), @@ -366,17 +365,19 @@ mod tests { g.add_edge(0, 1, Some("(0,1)")); g.add_in_edge(1, 0); g.add_edge(0, 3, Some("(0,3)")); + g.add_in_edge(3, 0); assert_eq!(g.node_count(), 2); assert_eq!(g.edge_count(), 2); let di_graph = g.clone().into_static::(); + println!("My turn..."); let di_graph_true = DiStaticGraph::<&str>::from_raw( 4, 2, EdgeVec::with_labels(vec![0, 2, 2, 2, 2], vec![1, 3], vec![0, 1]), - Some(EdgeVec::new(vec![0, 0, 1, 1, 1], vec![0])), + Some(EdgeVec::new(vec![0, 0, 1, 1, 2], vec![0,0])), Some(vec![0, u32::max_value(), 1, u32::max_value()]), vec!["node0", "node2"].into(), vec!["(0,1)", "(0,3)"].into(), diff --git a/src/graph_impl/static_graph/graph.rs b/src/graph_impl/static_graph/graph.rs index 1f9b017b..aaafa48a 100644 --- a/src/graph_impl/static_graph/graph.rs +++ b/src/graph_impl/static_graph/graph.rs @@ -47,7 +47,7 @@ use test::Options; pub type TypedUnStaticGraph = TypedStaticGraph; pub type TypedDiStaticGraph = TypedStaticGraph; pub type StaticGraph = - TypedStaticGraph; +TypedStaticGraph; pub type UnStaticGraph = StaticGraph; pub type DiStaticGraph = StaticGraph; @@ -62,6 +62,7 @@ pub struct TypedStaticGraph, // node_types[node_id] = node_label_id + // the node_label_id has been shifted right and id 0 is prepared for no label item. node_types: Vec, node_type_offsets: Vec, @@ -81,7 +82,7 @@ pub struct TypedStaticGraph PartialEq - for TypedStaticGraph +for TypedStaticGraph { fn eq(&self, other: &TypedStaticGraph) -> bool { if !self.node_count() == other.node_count() || !self.edge_count() == other.edge_count() { @@ -105,12 +106,11 @@ impl Partial } impl Eq - for TypedStaticGraph -{ -} +for TypedStaticGraph +{} impl Hash - for TypedStaticGraph +for TypedStaticGraph { fn hash(&self, state: &mut H) { { @@ -136,27 +136,25 @@ impl Hash } impl Serialize - for TypedStaticGraph -where - Id: serde::Serialize, - NL: serde::Serialize, - EL: serde::Serialize, - L: serde::Serialize, -{ -} +for TypedStaticGraph + where + Id: serde::Serialize, + NL: serde::Serialize, + EL: serde::Serialize, + L: serde::Serialize, +{} impl Deserialize - for TypedStaticGraph -where - Id: for<'de> serde::Deserialize<'de>, - NL: for<'de> serde::Deserialize<'de>, - EL: for<'de> serde::Deserialize<'de>, - L: for<'de> serde::Deserialize<'de>, -{ -} +for TypedStaticGraph + where + Id: for<'de> serde::Deserialize<'de>, + NL: for<'de> serde::Deserialize<'de>, + EL: for<'de> serde::Deserialize<'de>, + L: for<'de> serde::Deserialize<'de>, +{} impl - TypedStaticGraph +TypedStaticGraph { pub fn empty() -> Self { Self::new(EdgeVec::default(), None, None, None) @@ -343,7 +341,6 @@ impl edge_vec.num_edges() ); } - if labels.is_some() { let num_of_labels = labels.as_ref().unwrap().len(); if num_nodes != num_of_labels { @@ -469,22 +466,22 @@ impl } self.node_ids = node_ids; self.node_types = vec![0; self.num_nodes]; - self.node_type_offsets = vec![0, self.num_nodes]; + self.node_type_offsets = vec![0, self.num_nodes + 1]; return; } let offsets = self.get_node_offsets(); - let num_vertices = offsets[offsets.len() - 1] as usize; + let num_nodes = offsets[offsets.len() - 1]; - let mut node_ids = vec![0; num_vertices]; - let mut node_types = vec![0; num_vertices]; - let mut curr_idx_by_type = vec![0; offsets.len() - 1]; + let mut node_ids = vec![0; num_nodes]; + let mut node_types = vec![0; num_nodes]; + let mut curr_idx_by_type = vec![0; offsets.len()]; self.node_indices().for_each(|id| { let node_id = id.id(); let node_label_id = self .get_node(id) .get_label_id() - .unwrap_or(IdType::new(0)) - .id(); + .map(|op| op.id() + 1) + .unwrap_or(0); node_ids[offsets[node_label_id] + curr_idx_by_type[node_label_id]] = node_id; curr_idx_by_type[node_label_id] += 1; node_types[node_id] = node_label_id; @@ -496,10 +493,9 @@ impl // Partition edges by edge label or node label(if there did not exist edge labels in graph) fn partition_edges(&mut self) { - let highest_node_id = self.num_nodes - 1; let sort_by_node = self.num_of_edge_labels() == 0 && self.num_of_node_labels() > 0; let (fwd_adj_meta_data, bwd_adj_meta_data) = self.get_adj_meta_data(sort_by_node); - let num_vertices = highest_node_id + 1; + let num_vertices = self.num_nodes; let mut fwd_adj_lists: Vec>> = vec![Option::None; num_vertices]; let mut bwd_adj_lists: Vec>> = vec![Option::None; num_vertices]; let mut fwd_adj_list_curr_idx = HashMap::new(); @@ -515,11 +511,11 @@ impl fwd_adj_lists[node_id] = Some(SortedAdjVec::new( fwd_adj_meta_data.get(&node_id).unwrap().to_owned(), )); - fwd_adj_list_curr_idx.insert(node_id, vec![0; offset_size + 1 as usize]); + fwd_adj_list_curr_idx.insert(node_id, vec![0; offset_size + 1]); bwd_adj_lists[node_id] = Some(SortedAdjVec::new( bwd_adj_meta_data.get(&node_id).unwrap().to_owned(), )); - bwd_adj_list_curr_idx.insert(node_id, vec![0; offset_size + 1 as usize]); + bwd_adj_list_curr_idx.insert(node_id, vec![0; offset_size + 1]); } self.edge_indices() .flat_map(|(from, to)| { @@ -532,8 +528,8 @@ impl let label_id = self .get_edge(from, to) .get_label_id() - .unwrap_or(IdType::new(0)) - .id(); + .map(|op| op.id() + 1) + .unwrap_or(0); let from_type_or_label = { if sort_by_node { self.node_types[from.id()] @@ -554,14 +550,14 @@ impl fwd_adj_lists[from.id()] .as_mut() .unwrap() - .set_neighbor_id(to, idx + offset); + .set_neighbor_id(to, offset + idx); idx = bwd_adj_list_curr_idx.get(&to.id()).unwrap()[from_type_or_label]; offset = bwd_adj_meta_data.get(&to.id()).unwrap()[from_type_or_label]; bwd_adj_list_curr_idx.get_mut(&to.id()).unwrap()[from_type_or_label] += 1; bwd_adj_lists[to.id()] .as_mut() .unwrap() - .set_neighbor_id(from, idx + offset); + .set_neighbor_id(from, offset + idx); }); for node_id in 0..num_vertices { @@ -579,23 +575,23 @@ impl let label_id = self .get_node(x) .get_label_id() - .unwrap_or(IdType::new(0)) - .id(); + .map(|op| op.id() + 1) + .unwrap_or(0); let default_v = 0; let v = type_to_count_map.get(&label_id).unwrap_or(&default_v); type_to_count_map.insert(label_id, v + 1); }); let next_node_label_key = self.num_of_node_labels(); - let mut offsets = vec![0; next_node_label_key + 1]; + let mut offsets = vec![0; next_node_label_key + 3]; type_to_count_map.iter().for_each(|(label_id, cnt)| { let label_id = label_id.to_owned(); let label_cnt = cnt.to_owned(); - if label_id < next_node_label_key - 1 { - offsets[label_id + 1] = label_cnt.to_owned(); + if label_id < next_node_label_key + 1 { + offsets[label_id + 1] = label_cnt; } - offsets[next_node_label_key] += label_cnt; + offsets[next_node_label_key + 2] += label_cnt; }); for i in 1..offsets.len() - 1 { offsets[i] += offsets[i - 1]; @@ -609,7 +605,7 @@ impl ) -> (HashMap>, HashMap>) { let mut fwd_adj_list_metadata = HashMap::new(); let mut bwd_adj_list_metadata = HashMap::new(); - let next_label_or_type = { + let next_node_or_edge = { if sort_by_node { cmp::max(self.num_of_node_labels(), 1) } else { @@ -617,15 +613,15 @@ impl } }; for i in 0..self.node_count() { - fwd_adj_list_metadata.insert(i, vec![0; next_label_or_type + 1]); - bwd_adj_list_metadata.insert(i, vec![0; next_label_or_type + 1]); + fwd_adj_list_metadata.insert(i, vec![0; next_node_or_edge + 3]); + bwd_adj_list_metadata.insert(i, vec![0; next_node_or_edge + 3]); } self.edge_indices() .flat_map(|(from, to)| { - if !Ty::is_directed() { - return vec![(from, to), (to, from)]; + if Ty::is_directed() { + return vec![(from, to)]; } - vec![(from, to)] + return vec![(from, to), (to, from)]; }) .for_each(|(from, to)| { if sort_by_node { @@ -634,32 +630,28 @@ impl fwd_adj_list_metadata.get_mut(&from.id()).unwrap()[to_type + 1] += 1; bwd_adj_list_metadata.get_mut(&to.id()).unwrap()[from_type + 1] += 1; } else { - let from_label_id = self + let label_id = self .get_edge(from, to) .get_label_id() - .unwrap_or(IdType::new(0)) - .id(); - let to_label_id = self - .get_edge(to, from) - .get_label_id() - .unwrap_or(IdType::new(0)) - .id(); - fwd_adj_list_metadata.get_mut(&from.id()).unwrap()[from_label_id + 1] += 1; - bwd_adj_list_metadata.get_mut(&to.id()).unwrap()[to_label_id + 1] += 1; + .map(|op| op.id() + 1) + .unwrap_or(0); + fwd_adj_list_metadata.get_mut(&from.id()).unwrap()[label_id + 1] += 1; + bwd_adj_list_metadata.get_mut(&to.id()).unwrap()[label_id + 1] += 1; } }); fwd_adj_list_metadata.iter_mut().for_each(|(_id, offsets)| { - for i in 1..offsets.len() - 1 { - offsets[next_label_or_type] += offsets[i]; + for i in 1..next_node_or_edge + 2 { + offsets[next_node_or_edge + 2] += offsets[i]; offsets[i] += offsets[i - 1]; } }); bwd_adj_list_metadata.iter_mut().for_each(|(_id, offsets)| { - for i in 1..offsets.len() - 1 { - offsets[next_label_or_type] += offsets[i]; + for i in 1..next_node_or_edge + 2 { + offsets[next_node_or_edge + 2] += offsets[i]; offsets[i] += offsets[i - 1]; } }); + (fwd_adj_list_metadata, bwd_adj_list_metadata) } @@ -685,7 +677,7 @@ impl } impl GraphTrait - for TypedStaticGraph +for TypedStaticGraph { #[inline] fn get_node(&self, id: Id) -> NodeType { @@ -825,7 +817,7 @@ impl GraphTr } impl - GraphLabelTrait for TypedStaticGraph +GraphLabelTrait for TypedStaticGraph { #[inline(always)] fn get_node_label_map(&self) -> &SetMap { @@ -839,12 +831,11 @@ impl } impl UnGraphTrait - for TypedUnStaticGraph -{ -} +for TypedUnStaticGraph +{} impl DiGraphTrait - for TypedDiStaticGraph +for TypedDiStaticGraph { #[inline] fn in_degree(&self, id: Id) -> usize { @@ -865,7 +856,7 @@ impl DiGraphTrait } impl GeneralGraph - for TypedUnStaticGraph +for TypedUnStaticGraph { #[inline(always)] fn as_graph(&self) -> &GraphTrait { @@ -884,7 +875,7 @@ impl GeneralGraph GeneralGraph - for TypedDiStaticGraph +for TypedDiStaticGraph { #[inline(always)] fn as_graph(&self) -> &GraphTrait { @@ -930,7 +921,7 @@ fn _merge_labels(_labels1: Option>, _labels2: Option>) -> Op } impl Add - for TypedStaticGraph +for TypedStaticGraph { type Output = TypedStaticGraph; diff --git a/tests/static_graph.rs b/tests/static_graph.rs index b6075345..86b029f3 100644 --- a/tests/static_graph.rs +++ b/tests/static_graph.rs @@ -38,26 +38,26 @@ fn test_directed() { assert_eq!(g.get_node_ids(), &vec![0, 1, 2]); assert_eq!(g.get_node_types(), &vec![0, 0, 0]); - assert_eq!(g.get_node_type_offsets(), &vec![0, 3]); + assert_eq!(g.get_node_type_offsets(), &vec![0, 4]); let fwd_adj_list = g.get_fwd_adj_list()[0].as_ref().unwrap(); - assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 2]); + assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 2, 2, 2]); assert_eq!(fwd_adj_list.get_neighbour_ids(), &vec![1, 2]); let fwd_adj_list = g.get_fwd_adj_list()[1].as_ref().unwrap(); - assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 1]); + assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 1, 1, 1]); assert_eq!(fwd_adj_list.get_neighbour_ids(), &vec![0]); let fwd_adj_list = g.get_fwd_adj_list()[2].as_ref().unwrap(); - assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 1]); + assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 1, 1, 1]); assert_eq!(fwd_adj_list.get_neighbour_ids(), &vec![0]); let bwd_adj_list = g.get_bwd_adj_list()[0].as_ref().unwrap(); - assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 2]); + assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 2, 2, 2]); assert_eq!(bwd_adj_list.get_neighbour_ids(), &vec![1, 2]); let bwd_adj_list = g.get_bwd_adj_list()[1].as_ref().unwrap(); - assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 1]); + assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 1, 1, 1]); assert_eq!(bwd_adj_list.get_neighbour_ids(), &vec![0]); let bwd_adj_list = g.get_bwd_adj_list()[2].as_ref().unwrap(); - assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 1]); + assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 1, 1, 1]); assert_eq!(bwd_adj_list.get_neighbour_ids(), &vec![0]); assert_eq!(g.node_count(), 3); @@ -124,26 +124,26 @@ fn test_undirected() { assert_eq!(g.get_node_ids(), &vec![0, 1, 2]); // Without node labels assert_eq!(g.get_node_types(), &vec![0, 0, 0]); - assert_eq!(g.get_node_type_offsets(), &vec![0, 3]); + assert_eq!(g.get_node_type_offsets(), &vec![0, 4]); let fwd_adj_list = g.get_fwd_adj_list()[0].as_ref().unwrap(); - assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 2]); + assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 2, 2, 2]); assert_eq!(fwd_adj_list.get_neighbour_ids(), &vec![1, 2]); let fwd_adj_list = g.get_fwd_adj_list()[1].as_ref().unwrap(); - assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 2]); + assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 2, 2, 2]); assert_eq!(fwd_adj_list.get_neighbour_ids(), &vec![0, 2]); let fwd_adj_list = g.get_fwd_adj_list()[2].as_ref().unwrap(); - assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 2]); + assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 2, 2, 2]); assert_eq!(fwd_adj_list.get_neighbour_ids(), &vec![0, 1]); let bwd_adj_list = g.get_bwd_adj_list()[0].as_ref().unwrap(); - assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 2]); + assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 2, 2, 2]); assert_eq!(bwd_adj_list.get_neighbour_ids(), &vec![1, 2]); let bwd_adj_list = g.get_bwd_adj_list()[1].as_ref().unwrap(); - assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 2]); + assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 2, 2, 2]); assert_eq!(bwd_adj_list.get_neighbour_ids(), &vec![0, 2]); let bwd_adj_list = g.get_bwd_adj_list()[2].as_ref().unwrap(); - assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 2]); + assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 2, 2, 2]); assert_eq!(bwd_adj_list.get_neighbour_ids(), &vec![0, 1]); } @@ -207,27 +207,27 @@ fn test_labeled() { assert!(edges.contains(&g.get_edge(2, 0))); assert_eq!(g.get_node_ids(), &vec![1, 0, 2]); - assert_eq!(g.get_node_types(), &vec![1, 0, 1]); - assert_eq!(g.get_node_type_offsets(), &vec![0, 1, 3]); + assert_eq!(g.get_node_types(), &vec![2, 1, 2]); + assert_eq!(g.get_node_type_offsets(), &vec![0, 0, 1, 3, 3]); let fwd_adj_list = g.get_fwd_adj_list()[0].as_ref().unwrap(); - assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 1, 2]); + assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 0, 1, 2, 2]); assert_eq!(fwd_adj_list.get_neighbour_ids(), &vec![1, 2]); let fwd_adj_list = g.get_fwd_adj_list()[1].as_ref().unwrap(); - assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 1, 1]); + assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 0, 1, 1, 1]); assert_eq!(fwd_adj_list.get_neighbour_ids(), &vec![0]); let fwd_adj_list = g.get_fwd_adj_list()[2].as_ref().unwrap(); - assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 0, 1]); + assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 0, 0, 1, 1]); assert_eq!(fwd_adj_list.get_neighbour_ids(), &vec![0]); let bwd_adj_list = g.get_bwd_adj_list()[0].as_ref().unwrap(); - assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 1, 2]); + assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 0, 1, 2, 2]); assert_eq!(bwd_adj_list.get_neighbour_ids(), &vec![1, 2]); let bwd_adj_list = g.get_bwd_adj_list()[1].as_ref().unwrap(); - assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 1, 1]); + assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 0, 1, 1, 1]); assert_eq!(bwd_adj_list.get_neighbour_ids(), &vec![0]); let bwd_adj_list = g.get_bwd_adj_list()[2].as_ref().unwrap(); - assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 0, 1]); + assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 0, 0, 1, 1]); assert_eq!(bwd_adj_list.get_neighbour_ids(), &vec![0]); } From 2cc0847635897be9c2e2924a9ab78ea1b1f0ccaf Mon Sep 17 00:00:00 2001 From: Michael Chen <920901835@qq.com> Date: Tue, 19 Nov 2019 10:53:21 +0800 Subject: [PATCH 05/25] feat:func get_neighbours_by_label && fix some code style --- src/generic/graph.rs | 29 +- src/generic/node.rs | 6 + src/graph_impl/graph_vec/mod.rs | 2 +- src/graph_impl/mod.rs | 3 + .../catalog/adj_list_descriptor.rs | 40 ++ src/graph_impl/multi_graph/catalog/catalog.rs | 373 ++++++++++++++++++ .../multi_graph/catalog/catalog_plans.rs | 94 +++++ src/graph_impl/multi_graph/catalog/mod.rs | 7 + .../multi_graph/catalog/query_edge.rs | 35 ++ .../multi_graph/catalog/query_graph.rs | 214 ++++++++++ .../catalog/subgraph_mapping_iterator.rs | 176 +++++++++ src/graph_impl/multi_graph/graph.rs | 157 ++++++++ src/graph_impl/multi_graph/mod.rs | 5 + src/graph_impl/multi_graph/node.rs | 58 +++ src/graph_impl/multi_graph/plan/mod.rs | 2 + .../multi_graph/plan/operator/mod.rs | 3 + .../multi_graph/plan/operator/operator.rs | 196 +++++++++ .../multi_graph/plan/operator/scan/mod.rs | 2 + .../multi_graph/plan/operator/scan/scan.rs | 38 ++ .../plan/operator/scan/scan_sampling.rs | 87 ++++ .../multi_graph/plan/operator/sink/mod.rs | 1 + .../multi_graph/plan/operator/sink/sink.rs | 117 ++++++ src/graph_impl/multi_graph/plan/query_plan.rs | 81 ++++ src/graph_impl/multi_graph/query/mod.rs | 1 + .../multi_graph/query/query_graph_set.rs | 41 ++ src/graph_impl/static_graph/graph.rs | 123 ++++-- src/lib.rs | 1 + tests/static_graph.rs | 38 ++ 28 files changed, 1889 insertions(+), 41 deletions(-) create mode 100644 src/graph_impl/multi_graph/catalog/adj_list_descriptor.rs create mode 100644 src/graph_impl/multi_graph/catalog/catalog.rs create mode 100644 src/graph_impl/multi_graph/catalog/catalog_plans.rs create mode 100644 src/graph_impl/multi_graph/catalog/mod.rs create mode 100644 src/graph_impl/multi_graph/catalog/query_edge.rs create mode 100644 src/graph_impl/multi_graph/catalog/query_graph.rs create mode 100644 src/graph_impl/multi_graph/catalog/subgraph_mapping_iterator.rs create mode 100644 src/graph_impl/multi_graph/graph.rs create mode 100644 src/graph_impl/multi_graph/mod.rs create mode 100644 src/graph_impl/multi_graph/node.rs create mode 100644 src/graph_impl/multi_graph/plan/mod.rs create mode 100644 src/graph_impl/multi_graph/plan/operator/mod.rs create mode 100644 src/graph_impl/multi_graph/plan/operator/operator.rs create mode 100644 src/graph_impl/multi_graph/plan/operator/scan/mod.rs create mode 100644 src/graph_impl/multi_graph/plan/operator/scan/scan.rs create mode 100644 src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs create mode 100644 src/graph_impl/multi_graph/plan/operator/sink/mod.rs create mode 100644 src/graph_impl/multi_graph/plan/operator/sink/sink.rs create mode 100644 src/graph_impl/multi_graph/plan/query_plan.rs create mode 100644 src/graph_impl/multi_graph/query/mod.rs create mode 100644 src/graph_impl/multi_graph/query/query_graph_set.rs diff --git a/src/generic/graph.rs b/src/generic/graph.rs index 9ed35be2..8f356241 100644 --- a/src/generic/graph.rs +++ b/src/generic/graph.rs @@ -34,7 +34,7 @@ use graph_impl::GraphImpl; use map::SetMap; pub trait GeneralGraph: - GraphTrait + GraphLabelTrait +GraphTrait + GraphLabelTrait { fn as_graph(&self) -> &GraphTrait; @@ -54,7 +54,7 @@ pub trait GeneralGraph Clone - for Box> +for Box> { fn clone(&self) -> Self { let g = if self.as_digraph().is_some() { @@ -169,7 +169,7 @@ pub trait MutGraphTrait: - GraphTrait +GraphTrait { /// Return the node label - id mapping. fn get_node_label_map(&self) -> &SetMap; @@ -243,7 +243,7 @@ pub trait GraphLabelTrait: } pub trait MutGraphLabelTrait: - MutGraphTrait + GraphLabelTrait +MutGraphTrait + GraphLabelTrait { /// Update the node label. fn update_node_label(&mut self, node_id: Id, label: Option) -> bool; @@ -252,6 +252,13 @@ pub trait MutGraphLabelTrait) -> bool; } +/// Trait for labelled graphs. +pub trait LabelledGraphTrait: GraphLabelTrait +{ + fn neighbours_of_node(&self, id: Id, label: Option) -> Iter; + fn neighbours_of_edge(&self, id: Id, label: Option) -> Iter; +} + /// Trait for undirected graphs. pub trait UnGraphTrait: GraphTrait {} @@ -294,7 +301,7 @@ pub fn equal( } impl PartialEq - for Box> +for Box> { fn eq(&self, other: &Box>) -> bool { equal(self.as_ref(), other.as_ref()) @@ -304,7 +311,7 @@ impl PartialEq impl Eq for Box> {} impl Hash - for Box> +for Box> { fn hash(&self, state: &mut H) { { @@ -332,7 +339,7 @@ impl Hash } impl PartialOrd - for Box> +for Box> { fn partial_cmp(&self, other: &Self) -> Option { if self.node_count() != other.node_count() { @@ -350,11 +357,11 @@ impl PartialOrd } else { for (nbr1, nbr2) in self.neighbors_iter(node1).zip(other.neighbors_iter(node2)) - { - if nbr1 != nbr2 { - return Some(nbr1.cmp(&nbr2)); + { + if nbr1 != nbr2 { + return Some(nbr1.cmp(&nbr2)); + } } - } } } } diff --git a/src/generic/node.rs b/src/generic/node.rs index 460c4936..da45cf21 100644 --- a/src/generic/node.rs +++ b/src/generic/node.rs @@ -20,6 +20,7 @@ */ use generic::IdType; pub use graph_impl::graph_map::NodeMap; +use graph_impl::multi_graph::node::MultiNode; pub use graph_impl::static_graph::StaticNode; pub trait NodeTrait { @@ -55,6 +56,7 @@ pub enum OwnedNodeType { pub enum NodeType<'a, Id: 'a + IdType, L: 'a + IdType = Id> { NodeMap(&'a NodeMap), StaticNode(StaticNode), + MultiNode(MultiNode), None, } @@ -133,6 +135,7 @@ impl<'a, Id: IdType, L: IdType> NodeType<'a, Id, L> { match self { NodeType::NodeMap(node) => node, NodeType::StaticNode(_) => panic!("`unwrap_nodemap()` on `StaticNode`"), + NodeType::MultiNode(_) => panic!("`unwrap_nodemap()` on `MultiNode`"), NodeType::None => panic!("`unwrap_nodemap()` on `None`"), } } @@ -142,6 +145,7 @@ impl<'a, Id: IdType, L: IdType> NodeType<'a, Id, L> { match self { NodeType::NodeMap(_) => panic!("`unwrap_staticnode()` on `NodeMap`"), NodeType::StaticNode(node) => node, + NodeType::MultiNode(node) => panic!("`unwrap_staticnode()` on `MultiNode`"), NodeType::None => panic!("`unwrap_staticnode()` on `None`"), } } @@ -161,6 +165,7 @@ impl<'a, Id: IdType, L: IdType> NodeTrait for NodeType<'a, Id, L> { match self { NodeType::NodeMap(node) => node.get_id(), NodeType::StaticNode(ref node) => node.get_id(), + NodeType::MultiNode(ref node) => node.get_id(), NodeType::None => panic!("`get_id()` on `None`"), } } @@ -170,6 +175,7 @@ impl<'a, Id: IdType, L: IdType> NodeTrait for NodeType<'a, Id, L> { match self { NodeType::NodeMap(node) => node.get_label_id(), NodeType::StaticNode(ref node) => node.get_label_id(), + NodeType::MultiNode(ref node) => node.get_label_id(), NodeType::None => None, } } diff --git a/src/graph_impl/graph_vec/mod.rs b/src/graph_impl/graph_vec/mod.rs index b611512b..413f3d8d 100644 --- a/src/graph_impl/graph_vec/mod.rs +++ b/src/graph_impl/graph_vec/mod.rs @@ -377,7 +377,7 @@ mod tests { 4, 2, EdgeVec::with_labels(vec![0, 2, 2, 2, 2], vec![1, 3], vec![0, 1]), - Some(EdgeVec::new(vec![0, 0, 1, 1, 2], vec![0,0])), + Some(EdgeVec::new(vec![0, 0, 1, 1, 2], vec![0, 0])), Some(vec![0, u32::max_value(), 1, u32::max_value()]), vec!["node0", "node2"].into(), vec!["(0,1)", "(0,3)"].into(), diff --git a/src/graph_impl/mod.rs b/src/graph_impl/mod.rs index 653feb10..de0661bb 100644 --- a/src/graph_impl/mod.rs +++ b/src/graph_impl/mod.rs @@ -20,6 +20,7 @@ */ pub mod graph_map; pub mod graph_vec; +pub mod multi_graph; pub mod static_graph; pub use graph_impl::graph_map::{ @@ -36,6 +37,7 @@ pub use graph_impl::static_graph::{ pub enum GraphImpl { GraphMap, StaticGraph, + MultiGraph, } impl ::std::str::FromStr for GraphImpl { @@ -45,6 +47,7 @@ impl ::std::str::FromStr for GraphImpl { match s.as_ref() { "graphmap" => Ok(GraphImpl::GraphMap), "staticgraph" => Ok(GraphImpl::StaticGraph), + "multigraph" => Ok(GraphImpl::MultiGraph), _other => Err(format!("Unsupported implementation {:?}", _other)), } } diff --git a/src/graph_impl/multi_graph/catalog/adj_list_descriptor.rs b/src/graph_impl/multi_graph/catalog/adj_list_descriptor.rs new file mode 100644 index 00000000..2458b2f7 --- /dev/null +++ b/src/graph_impl/multi_graph/catalog/adj_list_descriptor.rs @@ -0,0 +1,40 @@ +use serde::export::fmt::Error; +use serde::export::Formatter; +use std::fmt::Display; +#[derive(Clone)] +pub enum Direction { + Fwd, + Bwd, +} + +pub struct AdjListDescriptor { + pub from_query_vertex: String, + vertex_idx: usize, + pub direction: Direction, + pub label: usize, +} + +impl AdjListDescriptor { + pub fn new( + from_query_vertex: String, + vertex_idx: usize, + direction: Direction, + label: usize, + ) -> Self { + Self { + from_query_vertex, + vertex_idx, + direction, + label, + } + } +} + +impl Display for Direction { + fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error> { + match self { + Direction::Fwd => write!(f, "Fwd"), + Direction::Bwd => write!(f, "Bwd"), + } + } +} diff --git a/src/graph_impl/multi_graph/catalog/catalog.rs b/src/graph_impl/multi_graph/catalog/catalog.rs new file mode 100644 index 00000000..f1828f1a --- /dev/null +++ b/src/graph_impl/multi_graph/catalog/catalog.rs @@ -0,0 +1,373 @@ +use generic::{GraphTrait, GraphType, IdType}; +use graph_impl::multi_graph::catalog::adj_list_descriptor::{AdjListDescriptor, Direction}; +use graph_impl::multi_graph::catalog::catalog_plans::{ + CatalogPlans, DEF_MAX_INPUT_NUM_VERTICES, DEF_NUM_EDGES_TO_SAMPLE, +}; +use graph_impl::multi_graph::catalog::query_graph::QueryGraph; +use graph_impl::multi_graph::plan::operator::operator::Operator; +use graph_impl::multi_graph::plan::query_plan::QueryPlan; +use graph_impl::TypedStaticGraph; +use hashbrown::{HashMap, HashSet}; +use indexmap::Equivalent; +use itertools::Itertools; +use std::hash::Hash; +use std::iter::FromIterator; +use std::panic::catch_unwind; +use std::ptr::null; + +pub struct Catalog { + in_subgraphs: Vec, + sampled_icost: HashMap>, + sampled_selectivity: HashMap>, + is_sorted_by_edge: bool, + num_sampled_edge: usize, + max_input_num_vertices: usize, + elapsed_time: f32, +} + +impl Catalog { + pub fn new(num_sampled_edge: usize, max_input_num_vertices: usize) -> Self { + Self { + in_subgraphs: vec![], + sampled_icost: Default::default(), + sampled_selectivity: Default::default(), + is_sorted_by_edge: false, + num_sampled_edge, + max_input_num_vertices, + elapsed_time: 0.0, + } + } + + // Icost is the intersection costs sampled. + // Cardinality is the sampledSelectivity sampled. + // InSubgraphs are the set of input subgraphs sampled. + pub fn with_cost( + i_cost: HashMap>, + cardinality: HashMap>, + in_subgraphs: Vec, + ) -> Self { + Self { + in_subgraphs, + sampled_icost: i_cost, + sampled_selectivity: cardinality, + is_sorted_by_edge: false, + num_sampled_edge: 0, + max_input_num_vertices: 0, + elapsed_time: 0.0, + } + } + + pub fn get_icost( + &self, + query_graph: &mut QueryGraph, + alds: Vec, + to_type: usize, + ) -> f64 { + let mut approx_icost = 0.0; + let mut min_icost = std::f64::MAX; + alds.iter().for_each(|ald| { + // Get each ALD icost by finding the largest subgraph (num vertices then num edges) + // of queryGraph used in stats collection and also minimizing sampledIcost. + for num_vertices in (DEF_NUM_EDGES_TO_SAMPLE - 1)..=2 { + min_icost = std::f64::MAX; + let mut num_edges_matched = 0; + for (i, sub_graph) in self.in_subgraphs.iter().enumerate() { + if sub_graph.get_num_qvertices() != num_vertices { + continue; + } + let new_num_edges_matched = query_graph.get_query_edges().len(); + let it = query_graph + .get_subgraph_mapping_iterator(self.in_subgraphs.get(i).unwrap()); + if new_num_edges_matched < num_edges_matched { + continue; + } + while it.has_next() { + let new_vertex_mapping = it.next().unwrap(); + if new_vertex_mapping.get(&ald.from_query_vertex).is_none() { + continue; + } + let sampled_icost; + let aldas_str = "(".to_string() + + new_vertex_mapping.get(&ald.from_query_vertex).unwrap() + + ") " + + &ald.direction.to_string() + + "[" + + &ald.label.to_string() + + "]"; + if self.is_sorted_by_edge { + sampled_icost = self + .sampled_selectivity + .get(&i) + .unwrap() + .get(&(aldas_str + "~" + &to_type.to_string())) + .unwrap() + .clone(); + } else { + sampled_icost = self + .sampled_icost + .get(&i) + .unwrap() + .get(&aldas_str) + .unwrap() + .clone(); + } + if new_num_edges_matched > num_edges_matched || min_icost > sampled_icost { + min_icost = sampled_icost; + num_edges_matched = new_num_edges_matched; + } + } + } + if min_icost < std::f64::MAX { + break; + } + } + approx_icost += min_icost; + }); + return approx_icost; + } + + pub fn get_selectivity( + &self, + in_subgraph: &mut QueryGraph, + alds: &Vec, + to_type: usize, + ) -> f64 { + let mut approx_selectivity = std::f64::MAX; + let mut num_vertices = DEF_MAX_INPUT_NUM_VERTICES - 1; + while num_vertices >= 2 { + let mut num_alds_matched = 0; + for (i, sub_graph) in self.in_subgraphs.iter().enumerate() { + if sub_graph.get_num_qvertices() != num_vertices { + continue; + } + let it = in_subgraph.get_subgraph_mapping_iterator(sub_graph); + while it.has_next() { + let vertex_mapping = it.next().unwrap(); + let new_num_alds_matched = self.get_num_alds_matched(&alds, vertex_mapping); + if new_num_alds_matched == 0 || new_num_alds_matched < num_alds_matched { + continue; + } + let selectivity_map = self.sampled_selectivity.get(&i).unwrap(); + let sampled_selectivity = selectivity_map + .get(&self.get_alds_as_str(&alds, Some(vertex_mapping), Some(to_type))) + .unwrap() + .clone(); + if new_num_alds_matched > num_alds_matched + || sampled_selectivity < approx_selectivity + { + num_alds_matched = new_num_alds_matched; + approx_selectivity = sampled_selectivity; + } + } + } + num_vertices -= 1; + } + approx_selectivity + } + + fn get_alds_as_str( + &self, + alds: &Vec, + vertex_mapping: Option<&HashMap>, + to_type: Option, + ) -> String { + let mut from_qvertices_and_dirs = alds + .iter() + .map(|ald| { + let tail = (") ".to_owned() + + &ald.direction.to_string() + + "[" + + &ald.label.to_string() + + "]"); + if vertex_mapping.is_none() { + return Some("(".to_owned() + &ald.from_query_vertex + &tail); + } else { + let vertex_mapping = vertex_mapping.unwrap(); + if let Some(from) = vertex_mapping.get(&ald.from_query_vertex) { + return Some("(".to_owned() + from + &tail); + } + } + None + }) + .skip_while(|x| x.is_none()) + .map(|x| x.unwrap()) + .sorted() + .join(","); + if to_type.is_some() { + from_qvertices_and_dirs += &("~".to_owned() + &to_type.unwrap().to_string()); + } + from_qvertices_and_dirs + } + + fn get_num_alds_matched( + &self, + alds: &Vec, + vertex_mapping: &HashMap, + ) -> usize { + let mut from_vertices_in_alds = HashSet::new(); + for ald in alds.iter() { + from_vertices_in_alds.insert(ald.from_query_vertex.clone()); + } + let num_alds_matched = 0; + vertex_mapping + .keys() + .filter(|&vertex| { + from_vertices_in_alds.contains(vertex) && vertex_mapping.get(vertex).unwrap() != "" + }) + .count() + } + + pub fn populate( + &mut self, + graph: TypedStaticGraph, + num_threads: usize, + filename: String, + ) { + self.is_sorted_by_edge = graph.is_sorted_by_node(); + self.sampled_icost = HashMap::new(); + self.sampled_selectivity = HashMap::new(); + let mut plans = CatalogPlans::new( + &graph, + num_threads, + self.num_sampled_edge, + self.max_input_num_vertices, + ); + self.set_input_subgraphs(plans.query_graphs_to_extend().get_query_graph_set()); + self.add_zero_selectivities(&graph, &mut plans); + + let query_plans = plans.get_query_plan_arrs(); + for (idx, query_plan_arr) in query_plans.iter_mut().enumerate() { + self.init(&graph, query_plan_arr); + // execute(query_plan_arr); + // logOutput(graph, query_plan_arr); + // query_plans.set(i, null); + } + } + + fn init( + &self, + graph: &TypedStaticGraph, + query_plan_arr: &mut Vec, + ) { + for query_plan in query_plan_arr { + let probe_tuple = vec![0; self.max_input_num_vertices + 1]; + if let Some(scan) = query_plan.get_scan_sampling() { + scan.init(probe_tuple, graph); + } + } + } + + fn set_input_subgraphs(&mut self, in_subgraphs: Vec) { + self.in_subgraphs = vec![]; + for mut in_subgraph in in_subgraphs { + let mut is_unique = true; + for subgraph in self.in_subgraphs.iter_mut() { + if subgraph.is_isomorphic_to(&mut in_subgraph) { + is_unique = false; + break; + } + } + if is_unique { + self.in_subgraphs.push(in_subgraph); + } + } + } + + fn get_subgraph_idx(&mut self, in_subgraph: &mut QueryGraph) -> usize { + for (idx, sub_graph) in self.in_subgraphs.iter_mut().enumerate() { + if in_subgraph.is_isomorphic_to(sub_graph) { + return idx; + } + } + //TODO:Fix the case when the given subgraph not found + 0 + } + + fn generate_direction_patterns(&self, size: usize, is_directed: bool) -> Vec> { + let mut direction_patterns = vec![]; + let mut directions = vec![Direction::Bwd; size]; + self.sub_generate_direction_patterns( + &mut directions, + size, + &mut direction_patterns, + is_directed, + ); + direction_patterns + } + + fn sub_generate_direction_patterns( + &self, + directions: &mut Vec, + size: usize, + direction_pattern: &mut Vec>, + is_directed: bool, + ) { + if size <= 0 { + direction_pattern.push(directions.to_vec()); + } else { + directions[size - 1] = Direction::Bwd; + self.sub_generate_direction_patterns( + directions, + size - 1, + direction_pattern, + is_directed, + ); + if is_directed { + directions[size - 1] = Direction::Fwd; + self.sub_generate_direction_patterns( + directions, + size - 1, + direction_pattern, + is_directed, + ); + } + } + } + + fn add_zero_selectivities< + Id: IdType, + NL: Hash + Eq, + EL: Hash + Eq, + Ty: GraphType, + L: IdType, + >( + &mut self, + graph: &TypedStaticGraph, + plans: &mut CatalogPlans, + ) { + let selectivity_zero = plans.get_selectivity_zero(); + for select in selectivity_zero { + let subgraph_idx = self.get_subgraph_idx(&mut select.0); + if self.sampled_selectivity.get(&subgraph_idx).is_none() { + self.sampled_selectivity + .insert(subgraph_idx, HashMap::new()); + } + let mut alds_as_str_list = vec![]; + let alds_str = self.get_alds_as_str(&select.1, None, None); + if !graph.is_directed() { + let splits: Vec<&str> = alds_str.split(',').collect(); + let direction_patterns = + self.generate_direction_patterns(splits.len(), !graph.is_directed()); + for pattern in direction_patterns { + let mut alds_str_with_pattern = "".to_string(); + for i in 0..pattern.len() { + let ok: Vec<&str> = splits[i].split("Bwd").collect(); + alds_str_with_pattern = + alds_str_with_pattern + ok[0] + &pattern[i].to_string() + &ok[1]; + if i != pattern.len() - 1 { + alds_str_with_pattern.push_str(","); + } + } + alds_as_str_list.push(alds_str_with_pattern); + } + } else { + alds_as_str_list.push(alds_str); + } + for alds_as_str in alds_as_str_list { + if let Some(selectivity) = self.sampled_selectivity.get_mut(&subgraph_idx) { + selectivity.insert(alds_as_str + "~" + &select.2.to_string(), 0.00); + } + } + } + } +} diff --git a/src/graph_impl/multi_graph/catalog/catalog_plans.rs b/src/graph_impl/multi_graph/catalog/catalog_plans.rs new file mode 100644 index 00000000..8fddf7af --- /dev/null +++ b/src/graph_impl/multi_graph/catalog/catalog_plans.rs @@ -0,0 +1,94 @@ +use generic::{GraphLabelTrait, GraphTrait, GraphType, IdType}; +use graph_impl::multi_graph::catalog::adj_list_descriptor::AdjListDescriptor; +use graph_impl::multi_graph::catalog::query_graph::QueryGraph; +use graph_impl::multi_graph::plan::query_plan::QueryPlan; +use graph_impl::multi_graph::query::query_graph_set::QueryGraphSet; +use graph_impl::TypedStaticGraph; +use hashbrown::HashMap; +use std::hash::Hash; + +pub static DEF_NUM_EDGES_TO_SAMPLE: usize = 1000; +pub static DEF_MAX_INPUT_NUM_VERTICES: usize = 3; +static QUERY_VERTICES: [&str; 7] = ["a", "b", "c", "d", "e", "f", "g"]; + +pub struct CatalogPlans { + num_sampled_edges: usize, + max_input_num_vertices: usize, + num_types: usize, + num_labels: usize, + sorted_by_node: bool, + query_graphs_to_extend: QueryGraphSet, + query_plans_arrs: Vec>, + is_directed: bool, + selectivity_zero: Vec<(QueryGraph, Vec, usize)>, + query_vertices: Vec, + query_vertex_to_idx_map: HashMap, +} + +impl CatalogPlans { + pub fn new( + graph: &TypedStaticGraph, + num_thread: usize, + num_sampled_edges: usize, + max_input_num_vertices: usize, + ) -> Self { + let mut plans = CatalogPlans { + num_sampled_edges, + max_input_num_vertices, + num_types: graph.num_of_edge_labels(), + num_labels: graph.num_of_node_labels(), + sorted_by_node: graph.is_sorted_by_node(), + query_graphs_to_extend: QueryGraphSet::new(), + query_plans_arrs: vec![], + is_directed: graph.is_directed(), + selectivity_zero: vec![], + query_vertices: vec![ + "a".to_string(), + "b".to_string(), + "c".to_string(), + "d".to_string(), + "e".to_string(), + "f".to_string(), + "g".to_string(), + ], + query_vertex_to_idx_map: HashMap::new(), + }; + for (i, v) in plans.query_vertices.iter().enumerate() { + plans.query_vertex_to_idx_map.insert(v.clone(), i); + } + // let scans = Vec::new(); + // TODO: Implement scan operator + + plans + } + + pub fn set_next_operators< + Id: IdType, + NL: Hash + Eq, + EL: Hash + Eq, + Ty: GraphType, + L: IdType, + >( + graph: TypedStaticGraph, + ) { + } + + pub fn query_graphs_to_extend(&self) -> &QueryGraphSet { + &self.query_graphs_to_extend + } + + pub fn get_selectivity_zero( + &mut self, + ) -> &mut Vec<(QueryGraph, Vec, usize)> { + &mut self.selectivity_zero + } + + pub fn get_query_plan_arrs(&mut self) -> &mut Vec> { + self.query_plans_arrs.as_mut() + } +} + +pub struct Descriptor { + out_subgraph: QueryGraph, + alds: Vec, +} diff --git a/src/graph_impl/multi_graph/catalog/mod.rs b/src/graph_impl/multi_graph/catalog/mod.rs new file mode 100644 index 00000000..c1a4a663 --- /dev/null +++ b/src/graph_impl/multi_graph/catalog/mod.rs @@ -0,0 +1,7 @@ +// Code in this module have not been finished. +pub mod adj_list_descriptor; +pub mod catalog; +pub mod catalog_plans; +pub mod query_edge; +pub mod query_graph; +pub mod subgraph_mapping_iterator; diff --git a/src/graph_impl/multi_graph/catalog/query_edge.rs b/src/graph_impl/multi_graph/catalog/query_edge.rs new file mode 100644 index 00000000..c5f4b918 --- /dev/null +++ b/src/graph_impl/multi_graph/catalog/query_edge.rs @@ -0,0 +1,35 @@ +#[derive(Clone)] +pub struct QueryEdge { + pub from_query_vertex: String, + pub to_query_vertex: String, + pub from_type: usize, + pub to_type: usize, + pub label: usize, +} + +impl QueryEdge { + pub fn new( + from_qvertex: String, + to_qvertex: String, + from_type: usize, + to_type: usize, + label: usize, + ) -> Self { + Self { + from_query_vertex: from_qvertex, + to_query_vertex: to_qvertex, + from_type, + to_type, + label, + } + } + pub fn default(from_qvertex: String, to_qvertex: String) -> Self { + Self { + from_query_vertex: from_qvertex, + to_query_vertex: to_qvertex, + from_type: 0, + to_type: 0, + label: 0, + } + } +} diff --git a/src/graph_impl/multi_graph/catalog/query_graph.rs b/src/graph_impl/multi_graph/catalog/query_graph.rs new file mode 100644 index 00000000..e96b9942 --- /dev/null +++ b/src/graph_impl/multi_graph/catalog/query_graph.rs @@ -0,0 +1,214 @@ +use graph_impl::multi_graph::catalog::query_edge::QueryEdge; +use graph_impl::multi_graph::catalog::subgraph_mapping_iterator::SubgraphMappingIterator; +use hashbrown::HashMap; +use itertools::Itertools; +use std::iter::FromIterator; + +#[derive(Clone)] +pub struct QueryGraph { + qvertex_to_qedges_map: HashMap>>, + qvertex_to_type_map: HashMap, + qvertex_to_deg_map: HashMap>, + q_edges: Vec, + // Using `Box` here to enable clone. + it: Option>, + encoding: Option, +} + +impl QueryGraph { + pub fn get_num_qvertices(&self) -> usize { + self.qvertex_to_qedges_map.len() + } + + pub fn get_subgraph_mapping_iterator(&mut self, query_graph: &QueryGraph) -> &mut Self { + if self.it.is_none() { + self.it.replace(Box::new(SubgraphMappingIterator::new( + self.qvertex_to_qedges_map + .keys() + .map(|x| x.clone()) + .collect(), + ))); + } + let mut it = self.it.take().unwrap(); + self.init_subgraph_iterator(&mut it, query_graph); + self.it.replace(it); + self + } + + pub fn get_vertex_to_deg_map(&self) -> &HashMap> { + &self.qvertex_to_deg_map + } + + pub fn get_vertex_to_qedges_map(&self) -> &HashMap>> { + &self.qvertex_to_qedges_map + } + + pub fn get_vertex_to_type_map(&self) -> &HashMap { + &self.qvertex_to_type_map + } + + pub fn get_query_vertices(&self) -> Vec { + let mut vec = vec![]; + for key in self.qvertex_to_qedges_map.keys() { + vec.push(key.clone()); + } + vec + } + + pub fn get_query_vertex_type(&self, query_vertex: &str) -> usize { + let vertex_type = self.qvertex_to_type_map.get(query_vertex); + if let Some(vertex_type) = vertex_type { + return vertex_type.clone(); + } + 0 + } + + pub fn set_query_vertex_type(&mut self, query_vertex: String, to_type: usize) { + self.qvertex_to_type_map + .insert(query_vertex.clone(), to_type); + for edge in self.q_edges.iter_mut() { + if edge.from_query_vertex == query_vertex { + edge.from_type = to_type; + } else if edge.to_query_vertex == query_vertex { + edge.to_type = to_type; + } + } + } + + pub fn contains_query_edge(&self, v1: &String, v2: &String) -> bool { + if let Some(map) = self.qvertex_to_qedges_map.get(v1) { + return map.contains_key(v2); + } + false + } + + pub fn get_query_edges_by_neighbor( + &self, + variable: &String, + neighbor_variable: &String, + ) -> Option<&Vec> { + if let Some(edges) = self.get_vertex_to_qedges_map().get(variable) { + return edges.get(neighbor_variable); + } + None + } + + pub fn get_query_edges(&self) -> &Vec { + &self.q_edges + } + + pub fn has_next(&mut self) -> bool { + let mut iterator = self.it.take().unwrap(); + let res = iterator.has_next(&self); + self.it.replace(iterator); + res + } + + pub fn next(&mut self) -> Option<&HashMap> { + if let Some(mut iterator) = self.it.take() { + iterator.next(&self); + self.it.replace(iterator); + return Some(&self.it.as_ref().unwrap().next); + } + None + } + + fn init_subgraph_iterator( + &mut self, + it: &mut SubgraphMappingIterator, + o_query_graph: &QueryGraph, + ) { + it.o_qvertices = o_query_graph.get_query_vertices(); + it.current_idx = 0; + it.vertex_indices = vec![0; it.o_qvertices.len()]; + it.curr_mapping.clear(); + for i in 0..it.o_qvertices.len() { + if it.vertices_for_idx.len() <= i { + it.vertices_for_idx.push(vec![]); + } else { + it.vertices_for_idx.get_mut(i).unwrap().clear(); + } + let o_qvertex = it.o_qvertices.get(i).unwrap(); + let o_qvertex_deg = o_query_graph + .get_vertex_to_deg_map() + .get(o_qvertex) + .unwrap(); + for j in 0..it.query_vertices.len() { + let q_vertex = it.query_vertices.get(j).unwrap(); + let vertex_type = self.get_vertex_to_type_map().get(q_vertex).unwrap(); + let q_vertex_deg = self.get_vertex_to_deg_map().get(q_vertex).unwrap(); + if o_query_graph + .get_vertex_to_type_map() + .get(o_qvertex) + .unwrap() + == vertex_type + && o_qvertex_deg.eq(q_vertex_deg) + || (it.o_qvertices.len() < it.query_vertices.len() + && q_vertex_deg[0] >= o_qvertex_deg[0] + && q_vertex_deg[1] >= o_qvertex_deg[1]) + { + it.vertices_for_idx + .get_mut(i) + .unwrap() + .push(q_vertex.clone()); + } + } + if 0 == it.vertices_for_idx.get(i).unwrap().len() { + it.is_next_computed = true; + return; + } + } + it.is_next_computed = false; + it.has_next(self); + } + + pub fn get_encoding(&mut self) -> String { + if self.encoding.is_some() { + return self.encoding.as_ref().unwrap().clone(); + } + let mut query_vertices_encoded = vec![String::from(""); self.qvertex_to_qedges_map.len()]; + let mut vertex_idx = 0; + for from_vertex in self.qvertex_to_qedges_map.keys() { + let from_vertex = from_vertex.clone(); + let mut encoding_str = "".to_string(); + if let Some(edge_map) = self.qvertex_to_qedges_map.get(&from_vertex) { + edge_map.keys().for_each(|to_vertex| { + if let Some(query_edges) = edge_map.get(to_vertex) { + for query_edge in query_edges { + if from_vertex == query_edge.from_query_vertex { + encoding_str += "F"; + } else { + encoding_str += "B"; + } + } + } + }); + } + let encoding_to_sort = String::from_iter(encoding_str.chars().into_iter().sorted()); + query_vertices_encoded[vertex_idx] = encoding_to_sort; + vertex_idx += 1; + } + query_vertices_encoded.sort(); + self.encoding = Some(query_vertices_encoded.join(".")); + self.encoding.as_ref().unwrap().clone() + } + + fn get_subgraph_mapping_if_any( + &mut self, + other_query_graph: &QueryGraph, + ) -> Option<&HashMap> { + let it = self.get_subgraph_mapping_iterator(other_query_graph); + if it.has_next() { + return it.next(); + } + None + } + + pub fn is_isomorphic_to(&mut self, other_query_graph: &mut QueryGraph) -> bool { + other_query_graph.get_encoding() == self.get_encoding() + && ((self.q_edges.len() == 0 && other_query_graph.q_edges.len() == 0) + || self + .get_subgraph_mapping_if_any(other_query_graph) + .is_some()) + } +} diff --git a/src/graph_impl/multi_graph/catalog/subgraph_mapping_iterator.rs b/src/graph_impl/multi_graph/catalog/subgraph_mapping_iterator.rs new file mode 100644 index 00000000..5a85f6cc --- /dev/null +++ b/src/graph_impl/multi_graph/catalog/subgraph_mapping_iterator.rs @@ -0,0 +1,176 @@ +use graph_impl::multi_graph::catalog::query_graph::QueryGraph; +use hashbrown::HashMap; + +// An iterator over a set of possible mappings between two query graphs. +#[derive(Clone)] +pub struct SubgraphMappingIterator { + pub query_vertices: Vec, + pub o_qvertices: Vec, + o_qgraph: Option, + pub next: HashMap, + pub is_next_computed: bool, + pub curr_mapping: Vec, + pub current_idx: usize, + pub vertex_indices: Vec, + pub vertices_for_idx: Vec>, +} + +impl SubgraphMappingIterator { + pub fn new(query_vertices: Vec) -> Self { + let mut next = HashMap::new(); + query_vertices.iter().for_each(|v| { + next.insert(v.clone(), String::from("")); + }); + Self { + query_vertices, + o_qvertices: vec![], + o_qgraph: None, + next, + is_next_computed: false, + curr_mapping: vec![], + current_idx: 0, + vertex_indices: vec![], + vertices_for_idx: vec![], + } + } + + pub fn has_next(&mut self, parent_graph: &QueryGraph) -> bool { + if !self.is_next_computed { + if self.curr_mapping.len() == self.o_qvertices.len() { + self.curr_mapping.pop(); + } + loop { + let next_idx = self.curr_mapping.len(); + if next_idx == 0 + && self.vertex_indices[0] < self.vertices_for_idx.get(0).unwrap().len() + { + self.curr_mapping.push( + self.vertices_for_idx + .get(0) + .unwrap() + .get(self.vertex_indices[0]) + .unwrap() + .clone(), + ); + self.vertex_indices[0] += 1; + } else if self.vertex_indices[next_idx] + < self.vertices_for_idx.get(next_idx).unwrap().len() + { + let vertices = self.vertices_for_idx.get(next_idx).unwrap(); + let new_var = vertices.get(self.vertex_indices[next_idx]).unwrap(); + self.vertex_indices[next_idx] += 1; + let other_for_new = self.o_qvertices.get(next_idx).unwrap(); + let mut invalid_map = false; + for i in 0..self.curr_mapping.len() { + let prev_var = self.curr_mapping.get(i).unwrap(); + if prev_var.eq(new_var) { + invalid_map = true; + break; + } + let other_for_prev = self.o_qvertices.get(i).unwrap(); + let q_edges = parent_graph + .get_vertex_to_qedges_map() + .get(new_var) + .unwrap() + .get(prev_var); + let o_qgraph = self.o_qgraph.as_ref().unwrap(); + let o_qedges = o_qgraph + .get_vertex_to_qedges_map() + .get(other_for_new) + .unwrap() + .get(other_for_prev); + if q_edges.is_none() && o_qedges.is_none() { + continue; + } + if q_edges.is_none() + || o_qedges.is_none() + || q_edges.unwrap().len() != o_qedges.unwrap().len() + { + invalid_map = true; + break; + } + if q_edges.unwrap().len() == 0 { + continue; + } + let q_edge = q_edges.unwrap().get(0).unwrap(); + let o_qedge = o_qedges.unwrap().get(0).unwrap(); + if q_edge.label != o_qedge.label { + continue; + } + if !q_edge.from_query_vertex.eq(prev_var) + && o_qedge.from_query_vertex.eq(other_for_prev) + || (q_edge.from_query_vertex.eq(new_var) + && o_qedge.from_query_vertex.eq(other_for_new)) + { + invalid_map = true; + break; + } + } + if invalid_map { + break; + } + self.curr_mapping.push(new_var.clone()); + } else if self.vertex_indices[next_idx] + >= self.vertices_for_idx.get(next_idx).unwrap().len() + { + self.curr_mapping.pop(); + self.vertex_indices[next_idx] = 0; + } + if self.curr_mapping.len() == self.o_qvertices.len() + || self.vertex_indices[0] < self.vertices_for_idx.get(0).unwrap().len() + || self.curr_mapping.is_empty() + { + break; + } + } + self.is_next_computed = true; + } + if self.curr_mapping.is_empty() { + return !self.curr_mapping.is_empty(); + } + let mut same_edge_labels = true; + for i in 0..self.curr_mapping.len() { + for j in (i + 1)..self.curr_mapping.len() { + let q_vertex = self.curr_mapping.get(i).unwrap(); + let o_qvertex = self.curr_mapping.get(j).unwrap(); + if !parent_graph.contains_query_edge(q_vertex, o_qvertex) { + continue; + } + let q_edge = parent_graph.get_query_edges_by_neighbor(q_vertex, o_qvertex); + let o_graph = self.o_qgraph.as_ref().unwrap(); + let o_qedge = o_graph.get_query_edges_by_neighbor( + self.o_qvertices.get(i).unwrap(), + self.o_qvertices.get(j).unwrap(), + ); + if q_edge.is_none() + || o_qedge.is_none() + || q_edge.unwrap().get(0).unwrap().label + != o_qedge.unwrap().get(0).unwrap().label + { + same_edge_labels = false; + break; + } + } + } + if !same_edge_labels { + self.is_next_computed = false; + return self.has_next(parent_graph); + } + !self.curr_mapping.is_empty() + } + + pub fn next(&mut self, parent_graph: &QueryGraph) -> Option<&HashMap> { + if !self.has_next(parent_graph) { + return None; + } + self.is_next_computed = false; + self.next.clear(); + for i in 0..self.o_qvertices.len() { + self.next.insert( + self.curr_mapping.get(i).unwrap().clone(), + self.query_vertices.get(i).unwrap().clone(), + ); + } + return Some(&self.next); + } +} diff --git a/src/graph_impl/multi_graph/graph.rs b/src/graph_impl/multi_graph/graph.rs new file mode 100644 index 00000000..5ddb29a7 --- /dev/null +++ b/src/graph_impl/multi_graph/graph.rs @@ -0,0 +1,157 @@ +use generic::{EdgeType, GraphTrait, GraphType, IdType, Iter, NodeType}; +use graph_impl::multi_graph::node::MultiNode; +use graph_impl::static_graph::sorted_adj_vec::SortedAdjVec; +use graph_impl::static_graph::static_edge_iter::StaticEdgeIndexIter; +use graph_impl::static_graph::EdgeVecTrait; +use graph_impl::{Edge, EdgeVec, GraphImpl}; +use map::SetMap; +use std::borrow::Cow; +use std::hash::Hash; +use std::marker::PhantomData; + +// TODO: Implement a multi_graph support multi_edge and multi_label +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct TypedMultiGraph +{ + num_nodes: usize, + num_edges: usize, + + // node Ids indexed by type and random access to node types. + node_ids: Vec, + // node_types[node_id] = node_label_id + // the node_label_id has been shifted right and id 0 is prepared for no label item. + node_types: Vec, + node_type_offsets: Vec, + + fwd_adj_lists: Vec>>, + bwd_adj_lists: Vec>>, + + edge_vec: EdgeVec, + in_edge_vec: Option>, + // Maintain the node's labels, whose index is aligned with `offsets`. + labels: Option>, + // A marker of thr graph type, namely, directed or undirected. + graph_type: PhantomData, + // A map of node labels. + node_label_map: SetMap, + // A map of edge labels. + edge_label_map: SetMap, +} + +impl PartialEq + for TypedMultiGraph +{ + fn eq(&self, other: &Self) -> bool { + if !self.node_count() == other.node_count() || !self.edge_count() == other.edge_count() { + return false; + } + for n in self.node_indices() { + // if !other.has_node(n) || self.get_node_label(n) != other.get_node_label(n) { + // return false; + // } + } + for (s, d) in self.edge_indices() { + // if !other.has_edge(s, d) || self.get_edge_label(s, d) != other.get_edge_label(s, d) { + // return false; + // } + } + true + } +} + +impl GraphTrait + for TypedMultiGraph +{ + fn get_node(&self, id: Id) -> NodeType { + if !self.has_node(id) { + return NodeType::None; + } + match self.labels { + Some(ref labels) => NodeType::MultiNode(MultiNode::new_static(id, labels[id.id()])), + None => NodeType::MultiNode(MultiNode::new(id, None)), + } + } + + fn get_edge(&self, start: Id, target: Id) -> EdgeType { + unimplemented!() + } + + fn has_node(&self, id: Id) -> bool { + unimplemented!() + } + + fn has_edge(&self, start: Id, target: Id) -> bool { + unimplemented!() + } + + fn node_count(&self) -> usize { + unimplemented!() + } + + fn edge_count(&self) -> usize { + unimplemented!() + } + + fn is_directed(&self) -> bool { + unimplemented!() + } + + fn node_indices(&self) -> Iter { + unimplemented!() + } + + fn edge_indices(&self) -> Iter<(Id, Id)> { + Iter::new(Box::new(StaticEdgeIndexIter::new( + Box::new(&self.edge_vec), + self.is_directed(), + ))) + } + + fn nodes(&self) -> Iter> { + unimplemented!() + } + + fn edges(&self) -> Iter> { + let labels = self.edge_vec.get_labels(); + if labels.is_empty() { + Iter::new(Box::new( + self.edge_indices() + .map(|i| EdgeType::Edge(Edge::new(i.0, i.1, None))), + )) + } else { + Iter::new(Box::new(self.edge_indices().zip(labels.iter()).map(|e| { + EdgeType::Edge(Edge::new_static((e.0).0, (e.0).1, *e.1)) + }))) + } + } + + fn degree(&self, id: Id) -> usize { + self.edge_vec.degree(id) + } + + fn total_degree(&self, id: Id) -> usize { + let mut degree = self.degree(id); + if self.is_directed() { + degree += self.in_edge_vec.as_ref().unwrap().neighbors(id).len() + } + degree + } + + fn neighbors_iter(&self, id: Id) -> Iter { + let neighbors = self.edge_vec.neighbors(id); + Iter::new(Box::new(neighbors.iter().map(|x| *x))) + } + + // if exist multi edges, the neighbours will repeat. + fn neighbors(&self, id: Id) -> Cow<[Id]> { + self.edge_vec.neighbors(id).into() + } + + fn max_seen_id(&self) -> Option { + Some(Id::new(self.node_count() - 1)) + } + + fn implementation(&self) -> GraphImpl { + GraphImpl::MultiGraph + } +} diff --git a/src/graph_impl/multi_graph/mod.rs b/src/graph_impl/multi_graph/mod.rs new file mode 100644 index 00000000..037e156d --- /dev/null +++ b/src/graph_impl/multi_graph/mod.rs @@ -0,0 +1,5 @@ +pub mod catalog; +pub mod graph; +pub mod node; +pub mod plan; +pub mod query; diff --git a/src/graph_impl/multi_graph/node.rs b/src/graph_impl/multi_graph/node.rs new file mode 100644 index 00000000..710e3c8d --- /dev/null +++ b/src/graph_impl/multi_graph/node.rs @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2018 UNSW Sydney, Data and Knowledge Group. + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +use generic::{IdType, NodeTrait}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MultiNode { + id: Id, + label: Option, +} + +impl MultiNode { + #[inline(always)] + pub fn new(id: Id, label: Option) -> Self { + MultiNode { id, label } + } + + #[inline(always)] + pub fn new_static(id: Id, label: L) -> Self { + MultiNode { + id, + label: if label == L::max_value() { + None + } else { + Some(label) + }, + } + } +} + +impl NodeTrait for MultiNode { + #[inline(always)] + fn get_id(&self) -> Id { + self.id + } + + #[inline(always)] + fn get_label_id(&self) -> Option { + self.label + } +} diff --git a/src/graph_impl/multi_graph/plan/mod.rs b/src/graph_impl/multi_graph/plan/mod.rs new file mode 100644 index 00000000..ec19d264 --- /dev/null +++ b/src/graph_impl/multi_graph/plan/mod.rs @@ -0,0 +1,2 @@ +pub mod operator; +pub mod query_plan; diff --git a/src/graph_impl/multi_graph/plan/operator/mod.rs b/src/graph_impl/multi_graph/plan/operator/mod.rs new file mode 100644 index 00000000..2850ad05 --- /dev/null +++ b/src/graph_impl/multi_graph/plan/operator/mod.rs @@ -0,0 +1,3 @@ +pub mod operator; +pub mod scan; +pub mod sink; diff --git a/src/graph_impl/multi_graph/plan/operator/operator.rs b/src/graph_impl/multi_graph/plan/operator/operator.rs new file mode 100644 index 00000000..c95e7276 --- /dev/null +++ b/src/graph_impl/multi_graph/plan/operator/operator.rs @@ -0,0 +1,196 @@ +use generic::{GraphType, IdType}; +use graph_impl::multi_graph::catalog::query_graph::QueryGraph; +use graph_impl::TypedStaticGraph; +use hashbrown::{HashMap, HashSet}; +use std::hash::{BuildHasherDefault, Hash}; +use std::iter::FromIterator; +use std::rc::Rc; + +#[derive(Clone)] +pub enum OpType { + Base, + Sink, +} + +#[derive(Clone)] +pub struct BaseOperator { + pub name: String, + pub op_type: OpType, + pub next: Option>, + pub prev: Option>, + pub probe_tuple: Vec, + pub out_tuple_len: usize, + pub in_subgraph: Option>, + pub out_subgraph: Rc, + pub out_qvertex_to_idx_map: HashMap, + pub last_repeated_vertex_idx: usize, + pub num_out_tuples: usize, + pub icost: usize, +} + +impl BaseOperator { + pub fn new(out_subgraph: Rc, in_subgraph: Option>) -> BaseOperator { + BaseOperator { + name: "".to_string(), + op_type: OpType::Base, + next: None, + prev: None, + probe_tuple: vec![], + out_tuple_len: out_subgraph.get_num_qvertices(), + in_subgraph, + out_subgraph, + out_qvertex_to_idx_map: HashMap::new(), + last_repeated_vertex_idx: 0, + num_out_tuples: 0, + icost: 0, + } + } + + pub fn copy(&self, is_thread_safe: bool) -> Option> { + unimplemented!(); + } + pub fn copy_default(&self) -> Option> { + self.copy(false) + } +} + +pub trait Operator { + fn get_op(&self) -> BaseOperator; + fn get_name(&self) -> String; + fn get_icost(&self) -> usize; + fn get_type(&self) -> OpType; + fn get_num_out_tuples(&self) -> usize; + fn get_query_vertex_id_map(&self) -> &HashMap; + fn get_out_query_vertices(&self) -> Vec { + self.get_query_vertex_id_map() + .iter() + .map(|(key, _val)| key.clone()) + .collect() + } + fn init( + &mut self, + probe_tuple: Vec, + graph: &TypedStaticGraph, + ); + fn is_same_as(&self, op: &BaseOperator) -> bool; + fn get_next_vec(&self) -> &Option>; + fn get_next_vec_as_mut(&mut self) -> &mut Option>; + fn get_prev(&self) -> Option<&Rc>; + fn get_next(&self, index: usize) -> &BaseOperator { + let next = self.get_next_vec().as_ref().unwrap(); + &next[index] + } + fn set_next(&mut self, op: BaseOperator) { + self.get_next_vec_as_mut().replace(vec![op]); + } + fn set_next_vec(&mut self, op: Vec) { + self.get_next_vec_as_mut().replace(op); + } + fn set_next_batch(&mut self, ops: Vec); + fn process_new_tuple(); + fn execute(&self) { + if let Some(prev) = self.get_prev() { + prev.execute(); + } + } + fn get_alds_as_string() -> String { + String::from("") + } + fn update_operator_name(query_vertex_to_index_map: HashMap); + fn get_operator_metrics_next_operators( + &self, + operator_metrics: &mut Vec<(String, usize, usize)>, + ) { + operator_metrics.push((self.get_name(), self.get_icost(), self.get_num_out_tuples())); + if let Some(next) = self.get_next_vec() { + next.iter() + .filter(|op| { + if let OpType::Sink = op.get_type() { + return true; + } + false + }) + .for_each(|op| { + op.get_operator_metrics_next_operators(operator_metrics); + }); + } + } + + fn get_last_operators(&self, last_operators: &mut Vec) { + if let Some(next) = self.get_next_vec() { + for op in next { + op.get_last_operators(last_operators); + } + } else { + last_operators.push(self.get_op()); + } + } + fn has_multi_edge_extends(&self) -> bool { + if let Some(prev) = self.get_prev() { + return prev.has_multi_edge_extends(); + } + return false; + } +} + +impl Operator for BaseOperator { + fn get_op(&self) -> BaseOperator { + self.clone() + } + + fn get_name(&self) -> String { + self.name.clone() + } + + fn get_icost(&self) -> usize { + self.icost + } + + fn get_type(&self) -> OpType { + self.op_type.clone() + } + + fn get_num_out_tuples(&self) -> usize { + self.num_out_tuples + } + + fn get_query_vertex_id_map(&self) -> &HashMap { + &self.out_qvertex_to_idx_map + } + + fn init( + &mut self, + probe_tuple: Vec, + graph: &TypedStaticGraph, + ) { + panic!("`init()` on `BaseOperator`") + } + + fn is_same_as(&self, op: &BaseOperator) -> bool { + panic!("`is_same_as()` on `BaseOperator`") + } + + fn get_next_vec(&self) -> &Option> { + &self.next + } + + fn get_next_vec_as_mut(&mut self) -> &mut Option> { + &mut self.next + } + + fn get_prev(&self) -> Option<&Rc> { + self.prev.as_ref() + } + + fn set_next_batch(&mut self, ops: Vec) { + self.next = Some(ops); + } + + fn process_new_tuple() { + unimplemented!() + } + + fn update_operator_name(query_vertex_to_index_map: HashMap) { + panic!("`update_operator_name()` on `BaseOperator`") + } +} diff --git a/src/graph_impl/multi_graph/plan/operator/scan/mod.rs b/src/graph_impl/multi_graph/plan/operator/scan/mod.rs new file mode 100644 index 00000000..5d33f337 --- /dev/null +++ b/src/graph_impl/multi_graph/plan/operator/scan/mod.rs @@ -0,0 +1,2 @@ +pub mod scan; +pub mod scan_sampling; diff --git a/src/graph_impl/multi_graph/plan/operator/scan/scan.rs b/src/graph_impl/multi_graph/plan/operator/scan/scan.rs new file mode 100644 index 00000000..84b2da56 --- /dev/null +++ b/src/graph_impl/multi_graph/plan/operator/scan/scan.rs @@ -0,0 +1,38 @@ +use graph_impl::multi_graph::catalog::query_graph::QueryGraph; +use graph_impl::multi_graph::plan::operator::operator::BaseOperator; +use graph_impl::static_graph::sorted_adj_vec::SortedAdjVec; +use std::rc::Rc; + +//TODO: Fixing Scan Operator +pub struct Scan { + pub base_op: BaseOperator, + from_query_vertex: String, + to_query_vertex: String, + from_type: usize, + to_type: usize, + label_or_to_type: usize, + fwd_adj_list: Vec>, + vertex_ids: Vec, + vertex_types: Vec, + from_vertex_start_idx: usize, + from_vertex_end_idx: usize, +} + +impl Scan { + //TODO: fixing functions + pub fn new(out_subgraph: Rc) -> Scan { + Scan { + base_op: BaseOperator::new(out_subgraph, None), + from_query_vertex: "".to_string(), + to_query_vertex: "".to_string(), + from_type: 0, + to_type: 0, + label_or_to_type: 0, + fwd_adj_list: vec![], + vertex_ids: vec![], + vertex_types: vec![], + from_vertex_start_idx: 0, + from_vertex_end_idx: 0, + } + } +} diff --git a/src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs b/src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs new file mode 100644 index 00000000..045010a6 --- /dev/null +++ b/src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs @@ -0,0 +1,87 @@ +use generic::{GraphType, IdType}; +use graph_impl::multi_graph::catalog::query_graph::QueryGraph; +use graph_impl::multi_graph::plan::operator::operator::{BaseOperator, OpType, Operator}; +use graph_impl::multi_graph::plan::operator::scan::scan::Scan; +use graph_impl::TypedStaticGraph; +use hashbrown::HashMap; +use std::hash::{BuildHasherDefault, Hash}; +use std::rc::Rc; + +//TODO: Fixing ScanSampling +pub struct ScanSampling { + pub scan: Scan, + pub edges_queue: Vec>, +} + +impl ScanSampling { + pub fn new(out_subgraph: Rc) -> Self { + Self { + scan: Scan::new(out_subgraph), + edges_queue: vec![], + } + } + + pub fn get_last_operator() {} +} + +impl Operator for ScanSampling { + fn get_op(&self) -> BaseOperator { + unimplemented!() + } + + fn get_name(&self) -> String { + unimplemented!() + } + + fn get_icost(&self) -> usize { + unimplemented!() + } + + fn get_type(&self) -> OpType { + unimplemented!() + } + + fn get_num_out_tuples(&self) -> usize { + unimplemented!() + } + + fn get_query_vertex_id_map(&self) -> &HashMap { + unimplemented!() + } + + fn init( + &mut self, + probe_tuple: Vec, + graph: &TypedStaticGraph, + ) { + unimplemented!() + } + + fn is_same_as(&self, op: &BaseOperator) -> bool { + unimplemented!() + } + + fn get_next_vec(&self) -> &Option> { + unimplemented!() + } + + fn get_next_vec_as_mut(&mut self) -> &mut Option> { + unimplemented!() + } + + fn get_prev(&self) -> Option<&Rc> { + unimplemented!() + } + + fn set_next_batch(&mut self, ops: Vec) { + unimplemented!() + } + + fn process_new_tuple() { + unimplemented!() + } + + fn update_operator_name(query_vertex_to_index_map: HashMap) { + unimplemented!() + } +} diff --git a/src/graph_impl/multi_graph/plan/operator/sink/mod.rs b/src/graph_impl/multi_graph/plan/operator/sink/mod.rs new file mode 100644 index 00000000..0ecbfb92 --- /dev/null +++ b/src/graph_impl/multi_graph/plan/operator/sink/mod.rs @@ -0,0 +1 @@ +pub mod sink; diff --git a/src/graph_impl/multi_graph/plan/operator/sink/sink.rs b/src/graph_impl/multi_graph/plan/operator/sink/sink.rs new file mode 100644 index 00000000..bc5adb72 --- /dev/null +++ b/src/graph_impl/multi_graph/plan/operator/sink/sink.rs @@ -0,0 +1,117 @@ +use generic::{GraphType, IdType}; +use graph_impl::multi_graph::catalog::query_graph::QueryGraph; +use graph_impl::multi_graph::plan::operator::operator::{BaseOperator, OpType, Operator}; +use graph_impl::TypedStaticGraph; +use hashbrown::HashMap; +use std::hash::{BuildHasherDefault, Hash}; +use std::ops::Deref; +use std::rc::Rc; + +pub enum SinkType { + Copy, + Print, + Limit, + Counter, +} + +pub struct Sink { + pub base_op: BaseOperator, + pub previous: Option>, +} + +impl Sink { + pub fn new(query_graph: Rc) -> Self { + Self { + base_op: BaseOperator::new(query_graph.clone(), Some(query_graph)), + previous: None, + } + } + pub fn copy(&self, is_thread_safe: bool) -> Sink { + let mut sink = Sink::new(self.base_op.out_subgraph.clone()); + if let Some(prev) = &self.base_op.prev { + sink.base_op.prev = prev.copy(is_thread_safe); + } else { + sink.base_op.prev = None; + } + sink + } +} + +impl Operator for Sink { + fn get_op(&self) -> BaseOperator { + self.base_op.clone() + } + + fn get_name(&self) -> String { + unimplemented!() + } + + fn get_icost(&self) -> usize { + unimplemented!() + } + + fn get_type(&self) -> OpType { + unimplemented!() + } + + fn get_num_out_tuples(&self) -> usize { + if let Some(prev) = &self.previous { + prev.iter().map(|op| op.get_num_out_tuples()).sum() + } else { + self.base_op.get_num_out_tuples() + } + } + + fn get_query_vertex_id_map(&self) -> &HashMap { + unimplemented!() + } + + fn init( + &mut self, + probe_tuple: Vec, + graph: &TypedStaticGraph, + ) { + self.base_op.probe_tuple = probe_tuple; + } + + fn is_same_as(&self, op: &BaseOperator) -> bool { + if let OpType::Sink = op.get_type() { + if let Some(prev) = self.get_prev() { + return prev.is_same_as(op.get_prev().unwrap()); + } + } + false + } + + fn get_next_vec(&self) -> &Option> { + unimplemented!() + } + + fn get_next_vec_as_mut(&mut self) -> &mut Option> { + unimplemented!() + } + + fn get_prev(&self) -> Option<&Rc> { + unimplemented!() + } + + fn set_next_batch(&mut self, ops: Vec) { + unimplemented!() + } + + fn process_new_tuple() {} + + fn execute(&self) { + if let Some(prev) = &self.previous { + prev[0].execute(); + } else { + if let Some(prev) = self.base_op.get_prev() { + prev.execute(); + } + } + } + + fn update_operator_name(query_vertex_to_index_map: HashMap) { + unimplemented!() + } +} diff --git a/src/graph_impl/multi_graph/plan/query_plan.rs b/src/graph_impl/multi_graph/plan/query_plan.rs new file mode 100644 index 00000000..ebbe8cad --- /dev/null +++ b/src/graph_impl/multi_graph/plan/query_plan.rs @@ -0,0 +1,81 @@ +use graph_impl::multi_graph::plan::operator::operator::{BaseOperator, Operator}; +use graph_impl::multi_graph::plan::operator::scan::scan_sampling::ScanSampling; +use graph_impl::multi_graph::plan::operator::sink::sink::{Sink, SinkType}; +use hashbrown::HashMap; +use std::rc::Rc; + +pub struct QueryPlan { + sink: Option, + sink_type: SinkType, + scan_sampling: Option, + last_operator: Option>, + out_tuples_limit: usize, + elapsed_time: f64, + icost: usize, + num_intermediate_tuples: usize, + num_out_tuples: usize, + operator_metrics: Vec<(String, usize, usize)>, + executed: bool, + adaptive_enabled: bool, + subplans: Vec>, + estimated_icost: f64, + estimated_num_out_tuples: f64, + q_vertex_to_num_out_tuples: HashMap, +} + +impl QueryPlan { + pub fn new(scan_sampling: ScanSampling) -> Self { + let mut last_operators = Vec::new(); + scan_sampling + .scan + .base_op + .get_last_operators(&mut last_operators); + let out_subgraph = last_operators.get(0).unwrap().out_subgraph.clone(); + let mut sink = Sink::new(out_subgraph); + for op in last_operators.iter_mut() { + op.set_next(sink.base_op.clone()) + } + sink.previous = Some(last_operators); + Self { + sink: Some(sink), + sink_type: SinkType::Counter, + scan_sampling: Some(scan_sampling), + last_operator: None, + out_tuples_limit: 0, + elapsed_time: 0.0, + icost: 0, + num_intermediate_tuples: 0, + num_out_tuples: 0, + operator_metrics: vec![], + executed: false, + adaptive_enabled: false, + subplans: vec![], + estimated_icost: 0.0, + estimated_num_out_tuples: 0.0, + q_vertex_to_num_out_tuples: HashMap::new(), + } + } + pub fn new_from_operator(last_operator: Rc) -> Self { + Self { + sink: None, + sink_type: SinkType::Counter, + scan_sampling: None, + last_operator: Some(last_operator.clone()), + out_tuples_limit: 0, + elapsed_time: 0.0, + icost: 0, + num_intermediate_tuples: 0, + num_out_tuples: 0, + operator_metrics: vec![], + executed: false, + adaptive_enabled: false, + subplans: vec![last_operator], + estimated_icost: 0.0, + estimated_num_out_tuples: 0.0, + q_vertex_to_num_out_tuples: HashMap::new(), + } + } + pub fn get_scan_sampling(&mut self) -> Option<&mut ScanSampling> { + self.scan_sampling.as_mut() + } +} diff --git a/src/graph_impl/multi_graph/query/mod.rs b/src/graph_impl/multi_graph/query/mod.rs new file mode 100644 index 00000000..28b54a80 --- /dev/null +++ b/src/graph_impl/multi_graph/query/mod.rs @@ -0,0 +1 @@ +pub mod query_graph_set; diff --git a/src/graph_impl/multi_graph/query/query_graph_set.rs b/src/graph_impl/multi_graph/query/query_graph_set.rs new file mode 100644 index 00000000..a8482972 --- /dev/null +++ b/src/graph_impl/multi_graph/query/query_graph_set.rs @@ -0,0 +1,41 @@ +use graph_impl::multi_graph::catalog::query_graph::QueryGraph; +use hashbrown::{HashMap, HashSet}; + +pub struct QueryGraphSet { + query_graphs: HashMap>, +} + +impl QueryGraphSet { + pub fn new() -> Self { + Self { + query_graphs: HashMap::new(), + } + } + + pub fn add(&mut self, mut query_graph: QueryGraph) { + let encoding = query_graph.get_encoding(); + let graph = self.query_graphs.get_mut(&encoding); + if graph.is_none() { + self.query_graphs.insert(encoding, vec![query_graph]); + } + } + + pub fn contains(&mut self, query_graph: &mut QueryGraph) -> bool { + if let Some(query_graphs) = self.query_graphs.get_mut(&query_graph.get_encoding()) { + for other_query_graph in query_graphs { + if query_graph.is_isomorphic_to(other_query_graph) { + return true; + } + } + } + false + } + + pub fn get_query_graph_set(&self) -> Vec { + self.query_graphs + .values() + .map(|g| g.clone()) + .flatten() + .collect() + } +} diff --git a/src/graph_impl/static_graph/graph.rs b/src/graph_impl/static_graph/graph.rs index aaafa48a..1e590cb7 100644 --- a/src/graph_impl/static_graph/graph.rs +++ b/src/graph_impl/static_graph/graph.rs @@ -27,6 +27,7 @@ use std::marker::PhantomData; use itertools::Itertools; use serde; +use generic::graph::LabelledGraphTrait; use generic::{ DefaultId, DefaultTy, DiGraphTrait, Directed, EdgeTrait, EdgeType, GeneralGraph, GraphLabelTrait, GraphTrait, GraphType, IdType, Iter, MapTrait, MutMapTrait, NodeTrait, @@ -43,6 +44,8 @@ use map::SetMap; use std::cmp; use std::ops::Add; use test::Options; +use test::bench::iter; +use std::any::Any; pub type TypedUnStaticGraph = TypedStaticGraph; pub type TypedDiStaticGraph = TypedStaticGraph; @@ -58,9 +61,10 @@ pub struct TypedStaticGraph, + node_ids: Vec, // node_types[node_id] = node_label_id // the node_label_id has been shifted right and id 0 is prepared for no label item. node_types: Vec, @@ -206,6 +210,7 @@ TypedStaticGraph let mut g = TypedStaticGraph { num_nodes, num_edges, + sort_by_node: false, node_ids: vec![], node_types: vec![], node_type_offsets: vec![], @@ -277,6 +282,7 @@ TypedStaticGraph let mut g = TypedStaticGraph { num_nodes, num_edges, + sort_by_node: false, node_ids: vec![], node_types: vec![], node_type_offsets: vec![], @@ -354,6 +360,7 @@ TypedStaticGraph let mut g = TypedStaticGraph { num_nodes, num_edges, + sort_by_node: false, node_ids: vec![], node_types: vec![], node_type_offsets: vec![], @@ -371,6 +378,10 @@ TypedStaticGraph g } + pub fn is_sorted_by_node(&self) -> bool { + self.sort_by_node + } + #[inline] pub fn get_edge_vec(&self) -> &EdgeVec { &self.edge_vec @@ -460,9 +471,9 @@ TypedStaticGraph // Partition nodes by type and generating node_ids && offsets for retrieving. fn partition_nodes(&mut self) { if 0 == self.num_of_node_labels() { - let mut node_ids = vec![0; self.num_nodes]; + let mut node_ids = vec![Id::new(0); self.num_nodes]; for i in 0..self.num_nodes { - node_ids[i] = i; + node_ids[i] = Id::new(i); } self.node_ids = node_ids; self.node_types = vec![0; self.num_nodes]; @@ -472,7 +483,7 @@ TypedStaticGraph let offsets = self.get_node_offsets(); let num_nodes = offsets[offsets.len() - 1]; - let mut node_ids = vec![0; num_nodes]; + let mut node_ids = vec![Id::new(0); num_nodes]; let mut node_types = vec![0; num_nodes]; let mut curr_idx_by_type = vec![0; offsets.len()]; self.node_indices().for_each(|id| { @@ -482,7 +493,7 @@ TypedStaticGraph .get_label_id() .map(|op| op.id() + 1) .unwrap_or(0); - node_ids[offsets[node_label_id] + curr_idx_by_type[node_label_id]] = node_id; + node_ids[offsets[node_label_id] + curr_idx_by_type[node_label_id]] = id; curr_idx_by_type[node_label_id] += 1; node_types[node_id] = node_label_id; }); @@ -493,15 +504,15 @@ TypedStaticGraph // Partition edges by edge label or node label(if there did not exist edge labels in graph) fn partition_edges(&mut self) { - let sort_by_node = self.num_of_edge_labels() == 0 && self.num_of_node_labels() > 0; - let (fwd_adj_meta_data, bwd_adj_meta_data) = self.get_adj_meta_data(sort_by_node); + self.sort_by_node = self.num_of_edge_labels() == 0 && self.num_of_node_labels() > 0; + let (fwd_adj_meta_data, bwd_adj_meta_data) = self.get_adj_meta_data(); let num_vertices = self.num_nodes; let mut fwd_adj_lists: Vec>> = vec![Option::None; num_vertices]; let mut bwd_adj_lists: Vec>> = vec![Option::None; num_vertices]; let mut fwd_adj_list_curr_idx = HashMap::new(); let mut bwd_adj_list_curr_idx = HashMap::new(); let offset_size = { - if sort_by_node { + if self.sort_by_node { self.num_of_node_labels() } else { self.num_of_edge_labels() @@ -530,20 +541,12 @@ TypedStaticGraph .get_label_id() .map(|op| op.id() + 1) .unwrap_or(0); - let from_type_or_label = { - if sort_by_node { - self.node_types[from.id()] - } else { - label_id - } - }; - let to_type_or_label = { - if sort_by_node { - self.node_types[to.id()] + let (from_type_or_label, to_type_or_label) = + if self.sort_by_node { + (self.node_types[from.id()], self.node_types[to.id()]) } else { - label_id - } - }; + (label_id, label_id) + }; let mut idx = fwd_adj_list_curr_idx.get(&from.id()).unwrap()[to_type_or_label]; let mut offset = fwd_adj_meta_data.get(&from.id()).unwrap()[to_type_or_label]; fwd_adj_list_curr_idx.get_mut(&from.id()).unwrap()[to_type_or_label] += 1; @@ -599,14 +602,11 @@ TypedStaticGraph offsets } - fn get_adj_meta_data( - &self, - sort_by_node: bool, - ) -> (HashMap>, HashMap>) { + fn get_adj_meta_data(&self) -> (HashMap>, HashMap>) { let mut fwd_adj_list_metadata = HashMap::new(); let mut bwd_adj_list_metadata = HashMap::new(); let next_node_or_edge = { - if sort_by_node { + if self.sort_by_node { cmp::max(self.num_of_node_labels(), 1) } else { cmp::max(self.num_of_edge_labels(), 1) @@ -624,7 +624,7 @@ TypedStaticGraph return vec![(from, to), (to, from)]; }) .for_each(|(from, to)| { - if sort_by_node { + if self.sort_by_node { let from_type = self.node_types[from.id()]; let to_type = self.node_types[to.id()]; fwd_adj_list_metadata.get_mut(&from.id()).unwrap()[to_type + 1] += 1; @@ -655,8 +655,8 @@ TypedStaticGraph (fwd_adj_list_metadata, bwd_adj_list_metadata) } - pub fn get_node_ids(&self) -> &Vec { - self.node_ids.as_ref() + pub fn get_node_ids(&self) -> &Vec { + &self.node_ids } pub fn get_node_types(&self) -> &Vec { @@ -830,6 +830,70 @@ GraphLabelTrait for TypedStaticGraph } } +impl +LabelledGraphTrait for TypedStaticGraph +{ + fn neighbours_of_node(&self, id: Id, label: Option) -> Iter { + if !self.is_sorted_by_node() { + panic!("Call `neighbours_of_edge` on a graph partition by node"); + } + if let Some(label) = label { + let mut iters = vec![]; + // we need to search backward(bwd) list if undirected. + if !Ty::is_directed() { + if let Some(bwd_list) = &self.bwd_adj_lists[id.id()] { + let offset = bwd_list.get_offsets(); + let node_label_id = self.node_label_map.find_index(&label).map_or(0, |id| id + 1); + let start = offset[node_label_id]; + let end = offset[node_label_id + 1]; + iters.push(Iter::new(Box::new(bwd_list.get_neighbour_ids()[start..end].iter()))); + } + } + if let Some(fwd_list) = &self.fwd_adj_lists[id.id()] { + let offset = fwd_list.get_offsets(); + let node_label_id = self.node_label_map.find_index(&label).map_or(0, |id| id + 1); + let start = offset[node_label_id]; + let end = offset[node_label_id + 1]; + println!("s:{},e:{}", start, end); + println!("offset：{:?}", offset); + println!("neighbour_ids:{:?}", &fwd_list.get_neighbour_ids()); + println!("label_neighbour_ids:{:?}", &fwd_list.get_neighbour_ids()[start..end]); + iters.push(Iter::new(Box::new(fwd_list.get_neighbour_ids()[start..end].iter()))); + } + return Iter::new(Box::new(iters.into_iter().flat_map(|it| it.map(|x| *x)))); + } + self.neighbors_iter(id) + } + + fn neighbours_of_edge(&self, id: Id, label: Option) -> Iter { + if self.is_sorted_by_node() { + panic!("Call `neighbours_of_edge` on a graph partition by node"); + } + if let Some(label) = label { + let mut iters = vec![]; + // we need to search backward(bwd) list if undirected. + if !Ty::is_directed() { + if let Some(bwd_list) = &self.bwd_adj_lists[id.id()] { + let offset = bwd_list.get_offsets(); + let edge_label_id = self.edge_label_map.find_index(&label).map_or(0, |id| id + 1); + let start = offset[edge_label_id]; + let end = offset[edge_label_id + 1]; + iters.push(Iter::new(Box::new(bwd_list.get_neighbour_ids()[start..end].iter()))); + } + } + if let Some(fwd_list) = &self.fwd_adj_lists[id.id()] { + let offset = fwd_list.get_offsets(); + let edge_label_id = self.edge_label_map.find_index(&label).map_or(0, |id| id + 1); + let start = offset[edge_label_id]; + let end = offset[edge_label_id + 1]; + iters.push(Iter::new(Box::new(fwd_list.get_neighbour_ids()[start..end].iter()))); + } + return Iter::new(Box::new(iters.into_iter().flat_map(|it| it.map(|x| *x)))); + } + self.neighbors_iter(id) + } +} + impl UnGraphTrait for TypedUnStaticGraph {} @@ -939,6 +1003,7 @@ for TypedStaticGraph let mut graph = TypedStaticGraph { num_nodes: 0, num_edges: 0, + sort_by_node: false, node_ids: vec![], node_types: vec![], node_type_offsets: vec![], diff --git a/src/lib.rs b/src/lib.rs index 67d37225..574a2378 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -38,6 +38,7 @@ extern crate log; #[macro_use] extern crate serde_derive; +extern crate core; #[cfg(feature = "hdfs")] extern crate hdfs; extern crate test; diff --git a/tests/static_graph.rs b/tests/static_graph.rs index 86b029f3..183fb9fe 100644 --- a/tests/static_graph.rs +++ b/tests/static_graph.rs @@ -29,6 +29,7 @@ use rust_graph::graph_impl::EdgeVec; use rust_graph::map::SetMap; use rust_graph::prelude::*; use rust_graph::{DiStaticGraph, UnStaticGraph}; +use rust_graph::generic::graph::LabelledGraphTrait; #[test] fn test_directed() { @@ -229,6 +230,43 @@ fn test_labeled() { let bwd_adj_list = g.get_bwd_adj_list()[2].as_ref().unwrap(); assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 0, 0, 1, 1]); assert_eq!(bwd_adj_list.get_neighbour_ids(), &vec![0]); + + let neighbour_edge_no: Vec = g.neighbours_of_edge(0, None).collect(); + let neighbour_edge_0_a: Vec = g.neighbours_of_edge(0, Some("a")).collect(); + let neighbour_edge_0_b: Vec = g.neighbours_of_edge(0, Some("b")).collect(); + let neighbour_edge_1_a: Vec = g.neighbours_of_edge(1, Some("a")).collect(); + let neighbour_edge_2_b: Vec = g.neighbours_of_edge(2, Some("b")).collect(); + assert_eq!(&neighbour_edge_no, &vec![1, 2]); + assert_eq!(&neighbour_edge_0_a, &vec![1]); + assert_eq!(&neighbour_edge_0_b, &vec![2]); + assert_eq!(&neighbour_edge_1_a, &vec![0]); + assert_eq!(&neighbour_edge_2_b, &vec![0]); +} + +#[test] +fn test_get_neighbours_by_label(){ + let edge_vec = EdgeVec::new(vec![0, 2, 3, 4], vec![1, 2, 0, 0]); + let in_edge_vec = EdgeVec::new(vec![0, 2, 3, 4], vec![1, 2, 0, 0]); + let labels = vec![1, 0, 1]; + let g = DiStaticGraph::<&str>::with_labels( + edge_vec, + Some(in_edge_vec), + labels, + setmap!["a", "b"], + setmap![], + None, + None, + ); + let neighbour_edge_no: Vec = g.neighbours_of_node(0, None).collect(); + let neighbour_edge_0_a: Vec = g.neighbours_of_node(0, Some("a")).collect(); + let neighbour_edge_0_b: Vec = g.neighbours_of_node(0, Some("b")).collect(); + let neighbour_edge_1_a: Vec = g.neighbours_of_node(1, Some("a")).collect(); + let neighbour_edge_2_b: Vec = g.neighbours_of_node(2, Some("b")).collect(); + assert_eq!(&neighbour_edge_no, &vec![1, 2]); + assert_eq!(&neighbour_edge_0_a, &vec![1]); + assert_eq!(&neighbour_edge_0_b, &vec![2]); + assert_eq!(&neighbour_edge_1_a, &(Vec::::new())); + assert_eq!(&neighbour_edge_2_b, &vec![0]); } #[test] From db7f81cdcc72ad5e5de633806c637f12e9c3c362 Mon Sep 17 00:00:00 2001 From: Michael Chen <920901835@qq.com> Date: Thu, 21 Nov 2019 10:27:07 +0800 Subject: [PATCH 06/25] feat:created Cow API for partitioned graph --- src/generic/graph.rs | 6 +- src/graph_impl/static_graph/graph.rs | 93 +++++++++++++--------------- tests/static_graph.rs | 42 ++++++++----- 3 files changed, 73 insertions(+), 68 deletions(-) diff --git a/src/generic/graph.rs b/src/generic/graph.rs index 8f356241..886f13e2 100644 --- a/src/generic/graph.rs +++ b/src/generic/graph.rs @@ -255,8 +255,10 @@ MutGraphTrait + GraphLabelTrait /// Trait for labelled graphs. pub trait LabelledGraphTrait: GraphLabelTrait { - fn neighbours_of_node(&self, id: Id, label: Option) -> Iter; - fn neighbours_of_edge(&self, id: Id, label: Option) -> Iter; + fn neighbors_of_node_iter(&self, id: Id, label: Option) -> Iter; + fn neighbors_of_edge_iter(&self, id: Id, label: Option) -> Iter; + fn neighbors_of_node(&self, id: Id, label: Option) -> Cow<[Id]>; + fn neighbors_of_edge(&self, id: Id, label: Option) -> Cow<[Id]>; } /// Trait for undirected graphs. diff --git a/src/graph_impl/static_graph/graph.rs b/src/graph_impl/static_graph/graph.rs index 1e590cb7..2c7f9b4f 100644 --- a/src/graph_impl/static_graph/graph.rs +++ b/src/graph_impl/static_graph/graph.rs @@ -655,6 +655,29 @@ TypedStaticGraph (fwd_adj_list_metadata, bwd_adj_list_metadata) } + fn get_neighbors_slice_by_node(&self, id: Id, label: Option) -> &[Id] { + if let Some(label) = label { + if let Some(fwd_list) = &self.fwd_adj_lists[id.id()] { + let offset = fwd_list.get_offsets(); + let label_id = self.node_label_map.find_index(&label).map_or(0, |id| id + 1); + + return &fwd_list.get_neighbour_ids()[offset[label_id]..offset[label_id + 1]]; + } + } + self.edge_vec.neighbors(id) + } + + fn get_neighbors_slice_by_edge(&self, id: Id, label: Option) -> &[Id] { + if let Some(label) = label { + if let Some(fwd_list) = &self.fwd_adj_lists[id.id()] { + let offset = fwd_list.get_offsets(); + let label_id = self.edge_label_map.find_index(&label).map_or(0, |id| id + 1); + return &fwd_list.get_neighbour_ids()[offset[label_id]..offset[label_id + 1]]; + } + } + self.edge_vec.neighbors(id) + } + pub fn get_node_ids(&self) -> &Vec { &self.node_ids } @@ -833,64 +856,32 @@ GraphLabelTrait for TypedStaticGraph impl LabelledGraphTrait for TypedStaticGraph { - fn neighbours_of_node(&self, id: Id, label: Option) -> Iter { + fn neighbors_of_node_iter(&self, id: Id, label: Option) -> Iter { if !self.is_sorted_by_node() { - panic!("Call `neighbours_of_edge` on a graph partition by node"); + panic!("Call `neighbors_of_node` on a graph partition by edge"); } - if let Some(label) = label { - let mut iters = vec![]; - // we need to search backward(bwd) list if undirected. - if !Ty::is_directed() { - if let Some(bwd_list) = &self.bwd_adj_lists[id.id()] { - let offset = bwd_list.get_offsets(); - let node_label_id = self.node_label_map.find_index(&label).map_or(0, |id| id + 1); - let start = offset[node_label_id]; - let end = offset[node_label_id + 1]; - iters.push(Iter::new(Box::new(bwd_list.get_neighbour_ids()[start..end].iter()))); - } - } - if let Some(fwd_list) = &self.fwd_adj_lists[id.id()] { - let offset = fwd_list.get_offsets(); - let node_label_id = self.node_label_map.find_index(&label).map_or(0, |id| id + 1); - let start = offset[node_label_id]; - let end = offset[node_label_id + 1]; - println!("s:{},e:{}", start, end); - println!("offset：{:?}", offset); - println!("neighbour_ids:{:?}", &fwd_list.get_neighbour_ids()); - println!("label_neighbour_ids:{:?}", &fwd_list.get_neighbour_ids()[start..end]); - iters.push(Iter::new(Box::new(fwd_list.get_neighbour_ids()[start..end].iter()))); - } - return Iter::new(Box::new(iters.into_iter().flat_map(|it| it.map(|x| *x)))); - } - self.neighbors_iter(id) + Iter::new(Box::new(self.get_neighbors_slice_by_node(id, label).iter().map(|x| *x))) } - fn neighbours_of_edge(&self, id: Id, label: Option) -> Iter { + fn neighbors_of_edge_iter(&self, id: Id, label: Option) -> Iter { if self.is_sorted_by_node() { - panic!("Call `neighbours_of_edge` on a graph partition by node"); + panic!("Call `neighbors_of_edge` on a graph partition by node"); } - if let Some(label) = label { - let mut iters = vec![]; - // we need to search backward(bwd) list if undirected. - if !Ty::is_directed() { - if let Some(bwd_list) = &self.bwd_adj_lists[id.id()] { - let offset = bwd_list.get_offsets(); - let edge_label_id = self.edge_label_map.find_index(&label).map_or(0, |id| id + 1); - let start = offset[edge_label_id]; - let end = offset[edge_label_id + 1]; - iters.push(Iter::new(Box::new(bwd_list.get_neighbour_ids()[start..end].iter()))); - } - } - if let Some(fwd_list) = &self.fwd_adj_lists[id.id()] { - let offset = fwd_list.get_offsets(); - let edge_label_id = self.edge_label_map.find_index(&label).map_or(0, |id| id + 1); - let start = offset[edge_label_id]; - let end = offset[edge_label_id + 1]; - iters.push(Iter::new(Box::new(fwd_list.get_neighbour_ids()[start..end].iter()))); - } - return Iter::new(Box::new(iters.into_iter().flat_map(|it| it.map(|x| *x)))); + Iter::new(Box::new(self.get_neighbors_slice_by_edge(id, label).iter().map(|x| *x))) + } + + fn neighbors_of_node(&self, id: Id, label: Option) -> Cow<[Id]> { + if !self.is_sorted_by_node() { + panic!("Call `neighbors_of_node` on a graph partition by edge"); + } + self.get_neighbors_slice_by_node(id, label).into() + } + + fn neighbors_of_edge(&self, id: Id, label: Option) -> Cow<[Id]> { + if self.is_sorted_by_node() { + panic!("Call `neighbors_of_edge` on a graph partition by node"); } - self.neighbors_iter(id) + self.get_neighbors_slice_by_edge(id, label).into() } } diff --git a/tests/static_graph.rs b/tests/static_graph.rs index 183fb9fe..78e1d3d0 100644 --- a/tests/static_graph.rs +++ b/tests/static_graph.rs @@ -21,6 +21,7 @@ #[macro_use] extern crate rust_graph; extern crate tempfile; +extern crate itertools; use rust_graph::generic::DefaultId; use rust_graph::graph_impl::static_graph::StaticNode; @@ -30,6 +31,7 @@ use rust_graph::map::SetMap; use rust_graph::prelude::*; use rust_graph::{DiStaticGraph, UnStaticGraph}; use rust_graph::generic::graph::LabelledGraphTrait; +use itertools::Itertools; #[test] fn test_directed() { @@ -231,11 +233,11 @@ fn test_labeled() { assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 0, 0, 1, 1]); assert_eq!(bwd_adj_list.get_neighbour_ids(), &vec![0]); - let neighbour_edge_no: Vec = g.neighbours_of_edge(0, None).collect(); - let neighbour_edge_0_a: Vec = g.neighbours_of_edge(0, Some("a")).collect(); - let neighbour_edge_0_b: Vec = g.neighbours_of_edge(0, Some("b")).collect(); - let neighbour_edge_1_a: Vec = g.neighbours_of_edge(1, Some("a")).collect(); - let neighbour_edge_2_b: Vec = g.neighbours_of_edge(2, Some("b")).collect(); + let neighbour_edge_no: Vec = g.neighbors_of_edge_iter(0, None).collect(); + let neighbour_edge_0_a: Vec = g.neighbors_of_edge_iter(0, Some("a")).collect(); + let neighbour_edge_0_b: Vec = g.neighbors_of_edge_iter(0, Some("b")).collect(); + let neighbour_edge_1_a: Vec = g.neighbors_of_edge_iter(1, Some("a")).collect(); + let neighbour_edge_2_b: Vec = g.neighbors_of_edge_iter(2, Some("b")).collect(); assert_eq!(&neighbour_edge_no, &vec![1, 2]); assert_eq!(&neighbour_edge_0_a, &vec![1]); assert_eq!(&neighbour_edge_0_b, &vec![2]); @@ -257,16 +259,26 @@ fn test_get_neighbours_by_label(){ None, None, ); - let neighbour_edge_no: Vec = g.neighbours_of_node(0, None).collect(); - let neighbour_edge_0_a: Vec = g.neighbours_of_node(0, Some("a")).collect(); - let neighbour_edge_0_b: Vec = g.neighbours_of_node(0, Some("b")).collect(); - let neighbour_edge_1_a: Vec = g.neighbours_of_node(1, Some("a")).collect(); - let neighbour_edge_2_b: Vec = g.neighbours_of_node(2, Some("b")).collect(); - assert_eq!(&neighbour_edge_no, &vec![1, 2]); - assert_eq!(&neighbour_edge_0_a, &vec![1]); - assert_eq!(&neighbour_edge_0_b, &vec![2]); - assert_eq!(&neighbour_edge_1_a, &(Vec::::new())); - assert_eq!(&neighbour_edge_2_b, &vec![0]); + let neighbour_edge_no_iter: Vec = g.neighbors_of_node_iter(0, None).collect(); + let neighbour_edge_0_a_iter: Vec = g.neighbors_of_node_iter(0, Some("a")).collect(); + let neighbour_edge_0_b_iter: Vec = g.neighbors_of_node_iter(0, Some("b")).collect(); + let neighbour_edge_1_a_iter: Vec = g.neighbors_of_node_iter(1, Some("a")).collect(); + let neighbour_edge_2_b_iter: Vec = g.neighbors_of_node_iter(2, Some("b")).collect(); + assert_eq!(&neighbour_edge_no_iter, &vec![1, 2]); + assert_eq!(&neighbour_edge_0_a_iter, &vec![1]); + assert_eq!(&neighbour_edge_0_b_iter, &vec![2]); + assert_eq!(&neighbour_edge_1_a_iter, &(Vec::::new())); + assert_eq!(&neighbour_edge_2_b_iter, &vec![0]); + let neighbour_edge_no = g.neighbors_of_node(0, None); + let neighbour_edge_0_a = g.neighbors_of_node(0, Some("a")); + let neighbour_edge_0_b = g.neighbors_of_node(0, Some("b")); + let neighbour_edge_1_a = g.neighbors_of_node(1, Some("a")); + let neighbour_edge_2_b = g.neighbors_of_node(2, Some("b")); + assert_eq!(&neighbour_edge_no.iter().collect_vec(), &vec![&1, &2]); + assert_eq!(&neighbour_edge_0_a.iter().collect_vec(), &vec![&1]); + assert_eq!(&neighbour_edge_0_b.iter().collect_vec(), &vec![&2]); + assert_eq!(&neighbour_edge_1_a.iter().collect_vec(), &(Vec::<&u32>::new())); + assert_eq!(&neighbour_edge_2_b.iter().collect_vec(), &vec![&0]); } #[test] From d60ba7c3a7cc4cd0cac811806215265b399f6a22 Mon Sep 17 00:00:00 2001 From: Michael Chen <920901835@qq.com> Date: Fri, 22 Nov 2019 18:03:22 +0800 Subject: [PATCH 07/25] feat: nodes_with_labels && move LabelledTrait into GraphLabelTrait --- src/generic/graph.rs | 39 ++++++----- src/graph_impl/graph_map/graph.rs | 98 +++++++++++++++++----------- src/graph_impl/static_graph/graph.rs | 89 +++++++++++++++++++------ tests/static_graph.rs | 32 +++++++-- 4 files changed, 174 insertions(+), 84 deletions(-) diff --git a/src/generic/graph.rs b/src/generic/graph.rs index 886f13e2..03ee0252 100644 --- a/src/generic/graph.rs +++ b/src/generic/graph.rs @@ -34,7 +34,7 @@ use graph_impl::GraphImpl; use map::SetMap; pub trait GeneralGraph: -GraphTrait + GraphLabelTrait + GraphTrait + GraphLabelTrait { fn as_graph(&self) -> &GraphTrait; @@ -54,7 +54,7 @@ GraphTrait + GraphLabelTrait } impl Clone -for Box> + for Box> { fn clone(&self) -> Self { let g = if self.as_digraph().is_some() { @@ -169,7 +169,7 @@ pub trait MutGraphTrait: -GraphTrait + GraphTrait { /// Return the node label - id mapping. fn get_node_label_map(&self) -> &SetMap; @@ -240,10 +240,18 @@ GraphTrait .filter_map(|(s, d)| self.get_edge_label(s, d)) .collect() } + + /// Trait for labelled graphs. + fn neighbors_of_node_iter(&self, id: Id, label: Option) -> Iter; + fn neighbors_of_edge_iter(&self, id: Id, label: Option) -> Iter; + fn neighbors_of_node(&self, id: Id, label: Option) -> Cow<[Id]>; + fn neighbors_of_edge(&self, id: Id, label: Option) -> Cow<[Id]>; + fn nodes_with_label(&self, label: Option) -> Iter; + fn edges_with_label(&self, label: Option) -> Iter<(Id, Id)>; } pub trait MutGraphLabelTrait: -MutGraphTrait + GraphLabelTrait + MutGraphTrait + GraphLabelTrait { /// Update the node label. fn update_node_label(&mut self, node_id: Id, label: Option) -> bool; @@ -252,15 +260,6 @@ MutGraphTrait + GraphLabelTrait fn update_edge_label(&mut self, start: Id, target: Id, label: Option) -> bool; } -/// Trait for labelled graphs. -pub trait LabelledGraphTrait: GraphLabelTrait -{ - fn neighbors_of_node_iter(&self, id: Id, label: Option) -> Iter; - fn neighbors_of_edge_iter(&self, id: Id, label: Option) -> Iter; - fn neighbors_of_node(&self, id: Id, label: Option) -> Cow<[Id]>; - fn neighbors_of_edge(&self, id: Id, label: Option) -> Cow<[Id]>; -} - /// Trait for undirected graphs. pub trait UnGraphTrait: GraphTrait {} @@ -303,7 +302,7 @@ pub fn equal( } impl PartialEq -for Box> + for Box> { fn eq(&self, other: &Box>) -> bool { equal(self.as_ref(), other.as_ref()) @@ -313,7 +312,7 @@ for Box> impl Eq for Box> {} impl Hash -for Box> + for Box> { fn hash(&self, state: &mut H) { { @@ -341,7 +340,7 @@ for Box> } impl PartialOrd -for Box> + for Box> { fn partial_cmp(&self, other: &Self) -> Option { if self.node_count() != other.node_count() { @@ -359,11 +358,11 @@ for Box> } else { for (nbr1, nbr2) in self.neighbors_iter(node1).zip(other.neighbors_iter(node2)) - { - if nbr1 != nbr2 { - return Some(nbr1.cmp(&nbr2)); - } + { + if nbr1 != nbr2 { + return Some(nbr1.cmp(&nbr2)); } + } } } } diff --git a/src/graph_impl/graph_map/graph.rs b/src/graph_impl/graph_map/graph.rs index 3b714eef..b9b33dc9 100644 --- a/src/graph_impl/graph_map/graph.rs +++ b/src/graph_impl/graph_map/graph.rs @@ -87,7 +87,7 @@ pub struct TypedGraphMap PartialEq - for TypedGraphMap +for TypedGraphMap { fn eq(&self, other: &TypedGraphMap) -> bool { if !self.node_count() == other.node_count() || !self.edge_count() == other.edge_count() { @@ -111,12 +111,11 @@ impl Partial } impl Eq - for TypedGraphMap -{ -} +for TypedGraphMap +{} impl Hash - for TypedGraphMap +for TypedGraphMap { fn hash(&self, state: &mut H) { { @@ -142,27 +141,25 @@ impl Hash } impl Serialize - for TypedGraphMap -where - Id: serde::Serialize, - NL: serde::Serialize, - EL: serde::Serialize, - L: serde::Serialize, -{ -} +for TypedGraphMap + where + Id: serde::Serialize, + NL: serde::Serialize, + EL: serde::Serialize, + L: serde::Serialize, +{} impl Deserialize - for TypedGraphMap -where - Id: for<'de> serde::Deserialize<'de>, - NL: for<'de> serde::Deserialize<'de>, - EL: for<'de> serde::Deserialize<'de>, - L: for<'de> serde::Deserialize<'de>, -{ -} +for TypedGraphMap + where + Id: for<'de> serde::Deserialize<'de>, + NL: for<'de> serde::Deserialize<'de>, + EL: for<'de> serde::Deserialize<'de>, + L: for<'de> serde::Deserialize<'de>, +{} impl - TypedGraphMap +TypedGraphMap { /// Constructs a new graph. pub fn new() -> Self { @@ -239,7 +236,7 @@ impl } impl Default - for TypedGraphMap +for TypedGraphMap { fn default() -> Self { Self::new() @@ -247,7 +244,7 @@ impl Default } impl - MutGraphTrait for TypedGraphMap +MutGraphTrait for TypedGraphMap { /// Add a node with `id` and `label`. If the node of the `id` already presents, /// replace the node's label with the new `label` and return `false`. @@ -414,7 +411,7 @@ impl } impl GraphTrait - for TypedGraphMap +for TypedGraphMap { #[inline] fn get_node(&self, id: Id) -> NodeType { @@ -552,7 +549,7 @@ impl GraphTr } impl - GraphLabelTrait for TypedGraphMap +GraphLabelTrait for TypedGraphMap { #[inline(always)] fn get_node_label_map(&self) -> &SetMap { @@ -563,10 +560,34 @@ impl fn get_edge_label_map(&self) -> &SetMap { &self.edge_label_map } + + fn neighbors_of_node_iter(&self, id: Id, label: Option) -> Iter { + unimplemented!() + } + + fn neighbors_of_edge_iter(&self, id: Id, label: Option) -> Iter { + unimplemented!() + } + + fn neighbors_of_node(&self, id: Id, label: Option) -> Cow<[Id]> { + unimplemented!() + } + + fn neighbors_of_edge(&self, id: Id, label: Option) -> Cow<[Id]> { + unimplemented!() + } + + fn nodes_with_label(&self, label: Option) -> Iter { + unimplemented!() + } + + fn edges_with_label(&self, label: Option) -> Iter<(Id, Id)> { + unimplemented!() + } } impl - MutGraphLabelTrait for TypedGraphMap +MutGraphLabelTrait for TypedGraphMap { #[inline] fn update_node_label(&mut self, node_id: Id, label: Option) -> bool { @@ -592,12 +613,11 @@ impl } impl UnGraphTrait - for TypedUnGraphMap -{ -} +for TypedUnGraphMap +{} impl DiGraphTrait - for TypedDiGraphMap +for TypedDiGraphMap { #[inline] fn in_degree(&self, id: Id) -> usize { @@ -628,7 +648,7 @@ impl DiGraphTrait } impl GeneralGraph - for TypedUnGraphMap +for TypedUnGraphMap { #[inline(always)] fn as_graph(&self) -> &GraphTrait { @@ -652,7 +672,7 @@ impl GeneralGraph GeneralGraph - for TypedDiGraphMap +for TypedDiGraphMap { #[inline(always)] fn as_graph(&self) -> &GraphTrait { @@ -681,7 +701,7 @@ impl GeneralGraph - TypedGraphMap +TypedGraphMap { pub fn reorder_id( self, @@ -886,7 +906,7 @@ impl } if let (Some(_in_offset), Some(_in_edge_vec)) = - (in_offset.as_mut(), in_edge_vec.as_mut()) + (in_offset.as_mut(), in_edge_vec.as_mut()) { let in_neighbors = mem::replace(&mut node.in_neighbors, BTreeSet::new()); @@ -948,9 +968,9 @@ impl } fn reorder_label_map(new_map: &impl MapTrait, old_map: impl MapTrait) -> SetMap -where - Id: IdType, - L: Hash + Eq, + where + Id: IdType, + L: Hash + Eq, { let mut old_map_vec: Vec<_> = old_map.items_vec().into_iter().map(|i| Some(i)).collect(); let mut result = SetMap::new(); @@ -970,7 +990,7 @@ pub struct ReorderResult - ReorderResult +ReorderResult { #[inline] pub fn take_graph(&mut self) -> Option> { diff --git a/src/graph_impl/static_graph/graph.rs b/src/graph_impl/static_graph/graph.rs index 2c7f9b4f..370aa245 100644 --- a/src/graph_impl/static_graph/graph.rs +++ b/src/graph_impl/static_graph/graph.rs @@ -24,10 +24,6 @@ use std::borrow::Cow; use std::hash::{Hash, Hasher}; use std::marker::PhantomData; -use itertools::Itertools; -use serde; - -use generic::graph::LabelledGraphTrait; use generic::{ DefaultId, DefaultTy, DiGraphTrait, Directed, EdgeTrait, EdgeType, GeneralGraph, GraphLabelTrait, GraphTrait, GraphType, IdType, Iter, MapTrait, MutMapTrait, NodeTrait, @@ -40,12 +36,14 @@ use graph_impl::static_graph::{EdgeVec, EdgeVecTrait}; use graph_impl::{Edge, GraphImpl}; use hashbrown::HashMap; use io::serde::{Deserialize, Serialize}; +use itertools::Itertools; use map::SetMap; +use serde; +use std::any::Any; use std::cmp; use std::ops::Add; -use test::Options; use test::bench::iter; -use std::any::Any; +use test::Options; pub type TypedUnStaticGraph = TypedStaticGraph; pub type TypedDiStaticGraph = TypedStaticGraph; @@ -541,12 +539,11 @@ TypedStaticGraph .get_label_id() .map(|op| op.id() + 1) .unwrap_or(0); - let (from_type_or_label, to_type_or_label) = - if self.sort_by_node { - (self.node_types[from.id()], self.node_types[to.id()]) - } else { - (label_id, label_id) - }; + let (from_type_or_label, to_type_or_label) = if self.sort_by_node { + (self.node_types[from.id()], self.node_types[to.id()]) + } else { + (label_id, label_id) + }; let mut idx = fwd_adj_list_curr_idx.get(&from.id()).unwrap()[to_type_or_label]; let mut offset = fwd_adj_meta_data.get(&from.id()).unwrap()[to_type_or_label]; fwd_adj_list_curr_idx.get_mut(&from.id()).unwrap()[to_type_or_label] += 1; @@ -659,7 +656,10 @@ TypedStaticGraph if let Some(label) = label { if let Some(fwd_list) = &self.fwd_adj_lists[id.id()] { let offset = fwd_list.get_offsets(); - let label_id = self.node_label_map.find_index(&label).map_or(0, |id| id + 1); + let label_id = self + .node_label_map + .find_index(&label) + .map_or(0, |id| id + 1); return &fwd_list.get_neighbour_ids()[offset[label_id]..offset[label_id + 1]]; } @@ -671,7 +671,10 @@ TypedStaticGraph if let Some(label) = label { if let Some(fwd_list) = &self.fwd_adj_lists[id.id()] { let offset = fwd_list.get_offsets(); - let label_id = self.edge_label_map.find_index(&label).map_or(0, |id| id + 1); + let label_id = self + .edge_label_map + .find_index(&label) + .map_or(0, |id| id + 1); return &fwd_list.get_neighbour_ids()[offset[label_id]..offset[label_id + 1]]; } } @@ -851,23 +854,27 @@ GraphLabelTrait for TypedStaticGraph fn get_edge_label_map(&self) -> &SetMap { &self.edge_label_map } -} -impl -LabelledGraphTrait for TypedStaticGraph -{ fn neighbors_of_node_iter(&self, id: Id, label: Option) -> Iter { if !self.is_sorted_by_node() { panic!("Call `neighbors_of_node` on a graph partition by edge"); } - Iter::new(Box::new(self.get_neighbors_slice_by_node(id, label).iter().map(|x| *x))) + Iter::new(Box::new( + self.get_neighbors_slice_by_node(id, label) + .iter() + .map(|x| *x), + )) } fn neighbors_of_edge_iter(&self, id: Id, label: Option) -> Iter { if self.is_sorted_by_node() { panic!("Call `neighbors_of_edge` on a graph partition by node"); } - Iter::new(Box::new(self.get_neighbors_slice_by_edge(id, label).iter().map(|x| *x))) + Iter::new(Box::new( + self.get_neighbors_slice_by_edge(id, label) + .iter() + .map(|x| *x), + )) } fn neighbors_of_node(&self, id: Id, label: Option) -> Cow<[Id]> { @@ -883,6 +890,48 @@ LabelledGraphTrait for TypedStaticGraph } self.get_neighbors_slice_by_edge(id, label).into() } + + fn nodes_with_label(&self, label: Option) -> Iter { + if let Some(label) = label { + let label_id = self + .node_label_map + .find_index(&label) + .map_or(0, |id| id + 1); + return Iter::new(Box::new(self.fwd_adj_lists + .iter() + .skip_while(|&x| x.is_none()) + .enumerate() + .flat_map(move|(sid, list_op)| { + let list = list_op.as_ref().unwrap(); + let offset = list.get_offsets(); + let label = label_id.clone(); + let neighbors = &list.get_neighbour_ids()[offset[label]..offset[label + 1]]; + neighbors.iter().map(move|id| *id) + }).unique())); + } + self.node_indices() + } + + fn edges_with_label(&self, label: Option) -> Iter<(Id, Id)> { + if let Some(label) = label { + let label_id = self + .edge_label_map + .find_index(&label) + .map_or(0, |id| id + 1); + return Iter::new(Box::new(self.fwd_adj_lists + .iter() + .skip_while(|&x| x.is_none()) + .enumerate() + .flat_map(move|(sid, list_op)| { + let list = list_op.as_ref().unwrap(); + let offset = list.get_offsets(); + let label = label_id.clone(); + let neighbors = &list.get_neighbour_ids()[offset[label]..offset[label + 1]]; + neighbors.iter().map(move|id| (Id::new(sid), *id)) + }))); + } + self.edge_indices() + } } impl UnGraphTrait diff --git a/tests/static_graph.rs b/tests/static_graph.rs index 78e1d3d0..7ca187d8 100644 --- a/tests/static_graph.rs +++ b/tests/static_graph.rs @@ -20,9 +20,10 @@ */ #[macro_use] extern crate rust_graph; -extern crate tempfile; extern crate itertools; +extern crate tempfile; +use itertools::Itertools; use rust_graph::generic::DefaultId; use rust_graph::graph_impl::static_graph::StaticNode; use rust_graph::graph_impl::Edge; @@ -30,8 +31,6 @@ use rust_graph::graph_impl::EdgeVec; use rust_graph::map::SetMap; use rust_graph::prelude::*; use rust_graph::{DiStaticGraph, UnStaticGraph}; -use rust_graph::generic::graph::LabelledGraphTrait; -use itertools::Itertools; #[test] fn test_directed() { @@ -243,10 +242,20 @@ fn test_labeled() { assert_eq!(&neighbour_edge_0_b, &vec![2]); assert_eq!(&neighbour_edge_1_a, &vec![0]); assert_eq!(&neighbour_edge_2_b, &vec![0]); + + let nodes_a = g.nodes_with_label(Some("a")); + let nodes_b = g.nodes_with_label(Some("b")); + assert_eq!(nodes_a.collect_vec(), vec![1, 0]); + assert_eq!(nodes_b.collect_vec(), vec![2, 0]); + + let edges_a = g.edges_with_label(Some("a")); + let edges_b = g.edges_with_label(Some("b")); + assert_eq!(edges_a.collect_vec(), vec![(0, 1), (1, 0)]); + assert_eq!(edges_b.collect_vec(), vec![(0, 2), (2, 0)]); } #[test] -fn test_get_neighbours_by_label(){ +fn test_get_neighbours_by_label() { let edge_vec = EdgeVec::new(vec![0, 2, 3, 4], vec![1, 2, 0, 0]); let in_edge_vec = EdgeVec::new(vec![0, 2, 3, 4], vec![1, 2, 0, 0]); let labels = vec![1, 0, 1]; @@ -277,8 +286,21 @@ fn test_get_neighbours_by_label(){ assert_eq!(&neighbour_edge_no.iter().collect_vec(), &vec![&1, &2]); assert_eq!(&neighbour_edge_0_a.iter().collect_vec(), &vec![&1]); assert_eq!(&neighbour_edge_0_b.iter().collect_vec(), &vec![&2]); - assert_eq!(&neighbour_edge_1_a.iter().collect_vec(), &(Vec::<&u32>::new())); + assert_eq!( + &neighbour_edge_1_a.iter().collect_vec(), + &(Vec::<&u32>::new()) + ); assert_eq!(&neighbour_edge_2_b.iter().collect_vec(), &vec![&0]); + + let nodes_a = g.nodes_with_label(Some("a")); + let nodes_b = g.nodes_with_label(Some("b")); + assert_eq!(nodes_a.collect_vec(), vec![1]); + assert_eq!(nodes_b.collect_vec(), vec![2, 0]); + + let edges_a = g.edges_with_label(Some("a")); + let edges_b = g.edges_with_label(Some("b")); + assert_eq!(edges_a.collect_vec(), Vec::<(u32, u32)>::new()); + assert_eq!(edges_b.collect_vec(), Vec::<(u32, u32)>::new()); } #[test] From 0a82da6bad02743c94bf7c5d02c13bee90b05af6 Mon Sep 17 00:00:00 2001 From: Michael Chen <920901835@qq.com> Date: Mon, 16 Dec 2019 20:14:37 +0800 Subject: [PATCH 08/25] fix:operator->extend --- Cargo.toml | 1 + src/graph_impl/graph_map/graph.rs | 74 ++-- .../catalog/adj_list_descriptor.rs | 13 +- src/graph_impl/multi_graph/catalog/catalog.rs | 118 +++++- .../multi_graph/catalog/catalog_plans.rs | 22 +- src/graph_impl/multi_graph/graph.rs | 157 -------- src/graph_impl/multi_graph/mod.rs | 5 +- src/graph_impl/multi_graph/plan/mod.rs | 1 + .../multi_graph/plan/operator/extend/EI.rs | 353 ++++++++++++++++++ .../plan/operator/extend/Extend.rs | 101 +++++ .../plan/operator/extend/Intersect.rs | 135 +++++++ .../plan/operator/extend/extend.rs | 101 +++++ .../plan/operator/extend/intersect.rs | 135 +++++++ .../multi_graph/plan/operator/extend/mod.rs | 3 + .../multi_graph/plan/operator/mod.rs | 77 ++++ .../multi_graph/plan/operator/operator.rs | 257 +++++++------ .../multi_graph/plan/operator/scan/scan.rs | 47 ++- .../plan/operator/scan/scan_sampling.rs | 79 ++-- .../multi_graph/plan/operator/sink/sink.rs | 113 ++---- src/graph_impl/multi_graph/plan/query_plan.rs | 41 +- src/graph_impl/multi_graph/utils/io_utils.rs | 10 + src/graph_impl/multi_graph/utils/mod.rs | 1 + src/graph_impl/static_graph/graph.rs | 72 ++-- src/graph_impl/static_graph/sorted_adj_vec.rs | 46 ++- tests/static_graph.rs | 36 +- 25 files changed, 1454 insertions(+), 544 deletions(-) delete mode 100644 src/graph_impl/multi_graph/graph.rs create mode 100644 src/graph_impl/multi_graph/plan/operator/extend/EI.rs create mode 100644 src/graph_impl/multi_graph/plan/operator/extend/Extend.rs create mode 100644 src/graph_impl/multi_graph/plan/operator/extend/Intersect.rs create mode 100644 src/graph_impl/multi_graph/plan/operator/extend/extend.rs create mode 100644 src/graph_impl/multi_graph/plan/operator/extend/intersect.rs create mode 100644 src/graph_impl/multi_graph/plan/operator/extend/mod.rs create mode 100644 src/graph_impl/multi_graph/utils/io_utils.rs create mode 100644 src/graph_impl/multi_graph/utils/mod.rs diff --git a/Cargo.toml b/Cargo.toml index 7a350716..bef9ffa2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,7 @@ rocksdb = "0.12.2" lru = "0.1.15" scoped_threadpool = "0.1.9" walkdir = "2.0.0" +time = "0.1" [dependencies.hdfs] git="https://github.com/UNSW-database/hdfs-rs.git" diff --git a/src/graph_impl/graph_map/graph.rs b/src/graph_impl/graph_map/graph.rs index b9b33dc9..baae10f2 100644 --- a/src/graph_impl/graph_map/graph.rs +++ b/src/graph_impl/graph_map/graph.rs @@ -87,7 +87,7 @@ pub struct TypedGraphMap PartialEq -for TypedGraphMap + for TypedGraphMap { fn eq(&self, other: &TypedGraphMap) -> bool { if !self.node_count() == other.node_count() || !self.edge_count() == other.edge_count() { @@ -111,11 +111,12 @@ for TypedGraphMap } impl Eq -for TypedGraphMap -{} + for TypedGraphMap +{ +} impl Hash -for TypedGraphMap + for TypedGraphMap { fn hash(&self, state: &mut H) { { @@ -141,25 +142,27 @@ for TypedGraphMap } impl Serialize -for TypedGraphMap - where - Id: serde::Serialize, - NL: serde::Serialize, - EL: serde::Serialize, - L: serde::Serialize, -{} + for TypedGraphMap +where + Id: serde::Serialize, + NL: serde::Serialize, + EL: serde::Serialize, + L: serde::Serialize, +{ +} impl Deserialize -for TypedGraphMap - where - Id: for<'de> serde::Deserialize<'de>, - NL: for<'de> serde::Deserialize<'de>, - EL: for<'de> serde::Deserialize<'de>, - L: for<'de> serde::Deserialize<'de>, -{} + for TypedGraphMap +where + Id: for<'de> serde::Deserialize<'de>, + NL: for<'de> serde::Deserialize<'de>, + EL: for<'de> serde::Deserialize<'de>, + L: for<'de> serde::Deserialize<'de>, +{ +} impl -TypedGraphMap + TypedGraphMap { /// Constructs a new graph. pub fn new() -> Self { @@ -236,7 +239,7 @@ TypedGraphMap } impl Default -for TypedGraphMap + for TypedGraphMap { fn default() -> Self { Self::new() @@ -244,7 +247,7 @@ for TypedGraphMap } impl -MutGraphTrait for TypedGraphMap + MutGraphTrait for TypedGraphMap { /// Add a node with `id` and `label`. If the node of the `id` already presents, /// replace the node's label with the new `label` and return `false`. @@ -411,7 +414,7 @@ MutGraphTrait for TypedGraphMap } impl GraphTrait -for TypedGraphMap + for TypedGraphMap { #[inline] fn get_node(&self, id: Id) -> NodeType { @@ -549,7 +552,7 @@ for TypedGraphMap } impl -GraphLabelTrait for TypedGraphMap + GraphLabelTrait for TypedGraphMap { #[inline(always)] fn get_node_label_map(&self) -> &SetMap { @@ -587,7 +590,7 @@ GraphLabelTrait for TypedGraphMap } impl -MutGraphLabelTrait for TypedGraphMap + MutGraphLabelTrait for TypedGraphMap { #[inline] fn update_node_label(&mut self, node_id: Id, label: Option) -> bool { @@ -613,11 +616,12 @@ MutGraphLabelTrait for TypedGraphMap } impl UnGraphTrait -for TypedUnGraphMap -{} + for TypedUnGraphMap +{ +} impl DiGraphTrait -for TypedDiGraphMap + for TypedDiGraphMap { #[inline] fn in_degree(&self, id: Id) -> usize { @@ -648,7 +652,7 @@ for TypedDiGraphMap } impl GeneralGraph -for TypedUnGraphMap + for TypedUnGraphMap { #[inline(always)] fn as_graph(&self) -> &GraphTrait { @@ -672,7 +676,7 @@ for TypedUnGraphMap } impl GeneralGraph -for TypedDiGraphMap + for TypedDiGraphMap { #[inline(always)] fn as_graph(&self) -> &GraphTrait { @@ -701,7 +705,7 @@ for TypedDiGraphMap } impl -TypedGraphMap + TypedGraphMap { pub fn reorder_id( self, @@ -906,7 +910,7 @@ TypedGraphMap } if let (Some(_in_offset), Some(_in_edge_vec)) = - (in_offset.as_mut(), in_edge_vec.as_mut()) + (in_offset.as_mut(), in_edge_vec.as_mut()) { let in_neighbors = mem::replace(&mut node.in_neighbors, BTreeSet::new()); @@ -968,9 +972,9 @@ TypedGraphMap } fn reorder_label_map(new_map: &impl MapTrait, old_map: impl MapTrait) -> SetMap - where - Id: IdType, - L: Hash + Eq, +where + Id: IdType, + L: Hash + Eq, { let mut old_map_vec: Vec<_> = old_map.items_vec().into_iter().map(|i| Some(i)).collect(); let mut result = SetMap::new(); @@ -990,7 +994,7 @@ pub struct ReorderResult -ReorderResult + ReorderResult { #[inline] pub fn take_graph(&mut self) -> Option> { diff --git a/src/graph_impl/multi_graph/catalog/adj_list_descriptor.rs b/src/graph_impl/multi_graph/catalog/adj_list_descriptor.rs index 2458b2f7..ff7dd6b4 100644 --- a/src/graph_impl/multi_graph/catalog/adj_list_descriptor.rs +++ b/src/graph_impl/multi_graph/catalog/adj_list_descriptor.rs @@ -1,15 +1,17 @@ use serde::export::fmt::Error; use serde::export::Formatter; use std::fmt::Display; + #[derive(Clone)] pub enum Direction { Fwd, Bwd, } +#[derive(Clone)] pub struct AdjListDescriptor { pub from_query_vertex: String, - vertex_idx: usize, + pub vertex_idx: usize, pub direction: Direction, pub label: usize, } @@ -38,3 +40,12 @@ impl Display for Direction { } } } + +impl PartialEq for Direction { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Direction::Fwd, Direction::Fwd, ) | (Direction::Bwd, Direction::Bwd) => true, + _ => false + } + } +} diff --git a/src/graph_impl/multi_graph/catalog/catalog.rs b/src/graph_impl/multi_graph/catalog/catalog.rs index f1828f1a..f6d31e01 100644 --- a/src/graph_impl/multi_graph/catalog/catalog.rs +++ b/src/graph_impl/multi_graph/catalog/catalog.rs @@ -4,22 +4,30 @@ use graph_impl::multi_graph::catalog::catalog_plans::{ CatalogPlans, DEF_MAX_INPUT_NUM_VERTICES, DEF_NUM_EDGES_TO_SAMPLE, }; use graph_impl::multi_graph::catalog::query_graph::QueryGraph; -use graph_impl::multi_graph::plan::operator::operator::Operator; +use graph_impl::multi_graph::plan::operator::operator::{ + BaseOperator, CommonOperatorTrait, Operator, +}; use graph_impl::multi_graph::plan::query_plan::QueryPlan; +use graph_impl::multi_graph::plan::operator::extend::EI::EI; +use graph_impl::multi_graph::utils::io_utils; use graph_impl::TypedStaticGraph; use hashbrown::{HashMap, HashSet}; use indexmap::Equivalent; use itertools::Itertools; +use std::cell::RefCell; use std::hash::Hash; use std::iter::FromIterator; +use std::mem::replace; +use std::ops::Deref; use std::panic::catch_unwind; use std::ptr::null; +use std::thread; pub struct Catalog { in_subgraphs: Vec, sampled_icost: HashMap>, sampled_selectivity: HashMap>, - is_sorted_by_edge: bool, + is_sorted_by_node: bool, num_sampled_edge: usize, max_input_num_vertices: usize, elapsed_time: f32, @@ -31,7 +39,7 @@ impl Catalog { in_subgraphs: vec![], sampled_icost: Default::default(), sampled_selectivity: Default::default(), - is_sorted_by_edge: false, + is_sorted_by_node: false, num_sampled_edge, max_input_num_vertices, elapsed_time: 0.0, @@ -50,13 +58,14 @@ impl Catalog { in_subgraphs, sampled_icost: i_cost, sampled_selectivity: cardinality, - is_sorted_by_edge: false, + is_sorted_by_node: false, num_sampled_edge: 0, max_input_num_vertices: 0, elapsed_time: 0.0, } } + /// Returns the i-cost of a particular extension from an input. pub fn get_icost( &self, query_graph: &mut QueryGraph, @@ -94,7 +103,7 @@ impl Catalog { + "[" + &ald.label.to_string() + "]"; - if self.is_sorted_by_edge { + if self.is_sorted_by_node { sampled_icost = self .sampled_selectivity .get(&i) @@ -126,6 +135,7 @@ impl Catalog { return approx_icost; } + /// Returns the sampledSelectivity of a particular extension from an input. pub fn get_selectivity( &self, in_subgraph: &mut QueryGraph, @@ -217,13 +227,14 @@ impl Catalog { .count() } - pub fn populate( + pub fn populate( &mut self, graph: TypedStaticGraph, num_threads: usize, filename: String, ) { - self.is_sorted_by_edge = graph.is_sorted_by_node(); + let start_time = io_utils::current_time(); + self.is_sorted_by_node = graph.is_sorted_by_node(); self.sampled_icost = HashMap::new(); self.sampled_selectivity = HashMap::new(); let mut plans = CatalogPlans::new( @@ -235,28 +246,105 @@ impl Catalog { self.set_input_subgraphs(plans.query_graphs_to_extend().get_query_graph_set()); self.add_zero_selectivities(&graph, &mut plans); - let query_plans = plans.get_query_plan_arrs(); - for (idx, query_plan_arr) in query_plans.iter_mut().enumerate() { + for query_plan_arr in plans.get_query_plan_arrs() { self.init(&graph, query_plan_arr); - // execute(query_plan_arr); - // logOutput(graph, query_plan_arr); - // query_plans.set(i, null); + self.execute(query_plan_arr); + self.log_output(&graph, query_plan_arr); + query_plan_arr.clear(); } + self.elapsed_time = io_utils::get_elapsed_time_in_millis(start_time); + //log here } fn init( &self, graph: &TypedStaticGraph, - query_plan_arr: &mut Vec, + query_plan_arr: &mut Vec>, ) { for query_plan in query_plan_arr { - let probe_tuple = vec![0; self.max_input_num_vertices + 1]; + let probe_tuple = vec![Id::new(0); self.max_input_num_vertices + 1]; if let Some(scan) = query_plan.get_scan_sampling() { scan.init(probe_tuple, graph); } } } + fn execute(&self, query_plan_arr: &mut Vec>) { + if query_plan_arr.len() > 1 { + let mut handlers = vec![]; + for i in 0..query_plan_arr.len() { + let mut sink = query_plan_arr[i].get_sink().clone(); + handlers.push(thread::spawn(move || { + sink.execute(); + })); + } + for handler in handlers { + handler.join(); + } + } else { + let sink = query_plan_arr[0].get_sink(); + sink.execute(); + } + } + + fn log_output( + &mut self, + graph: &TypedStaticGraph, + query_plan_arr: &mut Vec>, + ) { + let mut other: Vec<&mut Operator> = query_plan_arr + .iter_mut() + .map(|query_plan| { + let mut op = &mut query_plan.get_sink().previous.as_mut().unwrap()[0]; + while if let Operator::ScanSampling(sp) = op.deref() { + false + } else { + true + } { + let prev_op = get_op_attr_as_mut!(op, prev).as_mut().unwrap(); + op = prev_op.as_mut(); + } + &mut get_op_attr_as_mut!(op, next).as_mut().unwrap()[0] + }) + .collect(); + + let op = other.remove(0); + if self.is_sorted_by_node { + self.add_icost_and_selectivity_sorted_by_node(op, other, graph.is_directed()); + } else { + self.add_icost_and_selectivity(op, other, graph.is_directed()); + } + } + + fn add_icost_and_selectivity_sorted_by_node( + &self, + operator: &mut Operator, + other: Vec<&mut Operator>, + is_directed: bool, + ) {} + + fn add_icost_and_selectivity( + &mut self, + operator: &mut Operator, + other: Vec<&mut Operator>, + is_directed: bool, + ) { + if let Operator::Sink(sink) = &get_op_attr_as_ref!(operator, next).as_ref().unwrap()[0] { + return; + } + let mut num_input_tuples = get_op_attr!(operator, num_out_tuples); + for other_op in other { + num_input_tuples += get_op_attr!(other_op, num_out_tuples); + } + let in_subgraph = get_op_attr_as_mut!(operator, out_subgraph); + let subgraph_idx = self.get_subgraph_idx(in_subgraph); + let next_list = get_op_attr_as_ref!(operator, next).as_ref().unwrap(); + for (i, next) in next_list.iter().enumerate() { + let intersect = next; + //TODO: Implement following code. + } + } + fn set_input_subgraphs(&mut self, in_subgraphs: Vec) { self.in_subgraphs = vec![]; for mut in_subgraph in in_subgraphs { @@ -333,7 +421,7 @@ impl Catalog { >( &mut self, graph: &TypedStaticGraph, - plans: &mut CatalogPlans, + plans: &mut CatalogPlans, ) { let selectivity_zero = plans.get_selectivity_zero(); for select in selectivity_zero { diff --git a/src/graph_impl/multi_graph/catalog/catalog_plans.rs b/src/graph_impl/multi_graph/catalog/catalog_plans.rs index 8fddf7af..edecfb97 100644 --- a/src/graph_impl/multi_graph/catalog/catalog_plans.rs +++ b/src/graph_impl/multi_graph/catalog/catalog_plans.rs @@ -11,22 +11,22 @@ pub static DEF_NUM_EDGES_TO_SAMPLE: usize = 1000; pub static DEF_MAX_INPUT_NUM_VERTICES: usize = 3; static QUERY_VERTICES: [&str; 7] = ["a", "b", "c", "d", "e", "f", "g"]; -pub struct CatalogPlans { +pub struct CatalogPlans { num_sampled_edges: usize, max_input_num_vertices: usize, num_types: usize, num_labels: usize, sorted_by_node: bool, query_graphs_to_extend: QueryGraphSet, - query_plans_arrs: Vec>, + query_plans_arrs: Vec>>, is_directed: bool, selectivity_zero: Vec<(QueryGraph, Vec, usize)>, query_vertices: Vec, query_vertex_to_idx_map: HashMap, } -impl CatalogPlans { - pub fn new( +impl CatalogPlans { + pub fn new( graph: &TypedStaticGraph, num_thread: usize, num_sampled_edges: usize, @@ -62,16 +62,8 @@ impl CatalogPlans { plans } - pub fn set_next_operators< - Id: IdType, - NL: Hash + Eq, - EL: Hash + Eq, - Ty: GraphType, - L: IdType, - >( - graph: TypedStaticGraph, - ) { - } + pub fn set_next_operators( + graph: TypedStaticGraph, ) {} pub fn query_graphs_to_extend(&self) -> &QueryGraphSet { &self.query_graphs_to_extend @@ -83,7 +75,7 @@ impl CatalogPlans { &mut self.selectivity_zero } - pub fn get_query_plan_arrs(&mut self) -> &mut Vec> { + pub fn get_query_plan_arrs(&mut self) -> &mut Vec>> { self.query_plans_arrs.as_mut() } } diff --git a/src/graph_impl/multi_graph/graph.rs b/src/graph_impl/multi_graph/graph.rs deleted file mode 100644 index 5ddb29a7..00000000 --- a/src/graph_impl/multi_graph/graph.rs +++ /dev/null @@ -1,157 +0,0 @@ -use generic::{EdgeType, GraphTrait, GraphType, IdType, Iter, NodeType}; -use graph_impl::multi_graph::node::MultiNode; -use graph_impl::static_graph::sorted_adj_vec::SortedAdjVec; -use graph_impl::static_graph::static_edge_iter::StaticEdgeIndexIter; -use graph_impl::static_graph::EdgeVecTrait; -use graph_impl::{Edge, EdgeVec, GraphImpl}; -use map::SetMap; -use std::borrow::Cow; -use std::hash::Hash; -use std::marker::PhantomData; - -// TODO: Implement a multi_graph support multi_edge and multi_label -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct TypedMultiGraph -{ - num_nodes: usize, - num_edges: usize, - - // node Ids indexed by type and random access to node types. - node_ids: Vec, - // node_types[node_id] = node_label_id - // the node_label_id has been shifted right and id 0 is prepared for no label item. - node_types: Vec, - node_type_offsets: Vec, - - fwd_adj_lists: Vec>>, - bwd_adj_lists: Vec>>, - - edge_vec: EdgeVec, - in_edge_vec: Option>, - // Maintain the node's labels, whose index is aligned with `offsets`. - labels: Option>, - // A marker of thr graph type, namely, directed or undirected. - graph_type: PhantomData, - // A map of node labels. - node_label_map: SetMap, - // A map of edge labels. - edge_label_map: SetMap, -} - -impl PartialEq - for TypedMultiGraph -{ - fn eq(&self, other: &Self) -> bool { - if !self.node_count() == other.node_count() || !self.edge_count() == other.edge_count() { - return false; - } - for n in self.node_indices() { - // if !other.has_node(n) || self.get_node_label(n) != other.get_node_label(n) { - // return false; - // } - } - for (s, d) in self.edge_indices() { - // if !other.has_edge(s, d) || self.get_edge_label(s, d) != other.get_edge_label(s, d) { - // return false; - // } - } - true - } -} - -impl GraphTrait - for TypedMultiGraph -{ - fn get_node(&self, id: Id) -> NodeType { - if !self.has_node(id) { - return NodeType::None; - } - match self.labels { - Some(ref labels) => NodeType::MultiNode(MultiNode::new_static(id, labels[id.id()])), - None => NodeType::MultiNode(MultiNode::new(id, None)), - } - } - - fn get_edge(&self, start: Id, target: Id) -> EdgeType { - unimplemented!() - } - - fn has_node(&self, id: Id) -> bool { - unimplemented!() - } - - fn has_edge(&self, start: Id, target: Id) -> bool { - unimplemented!() - } - - fn node_count(&self) -> usize { - unimplemented!() - } - - fn edge_count(&self) -> usize { - unimplemented!() - } - - fn is_directed(&self) -> bool { - unimplemented!() - } - - fn node_indices(&self) -> Iter { - unimplemented!() - } - - fn edge_indices(&self) -> Iter<(Id, Id)> { - Iter::new(Box::new(StaticEdgeIndexIter::new( - Box::new(&self.edge_vec), - self.is_directed(), - ))) - } - - fn nodes(&self) -> Iter> { - unimplemented!() - } - - fn edges(&self) -> Iter> { - let labels = self.edge_vec.get_labels(); - if labels.is_empty() { - Iter::new(Box::new( - self.edge_indices() - .map(|i| EdgeType::Edge(Edge::new(i.0, i.1, None))), - )) - } else { - Iter::new(Box::new(self.edge_indices().zip(labels.iter()).map(|e| { - EdgeType::Edge(Edge::new_static((e.0).0, (e.0).1, *e.1)) - }))) - } - } - - fn degree(&self, id: Id) -> usize { - self.edge_vec.degree(id) - } - - fn total_degree(&self, id: Id) -> usize { - let mut degree = self.degree(id); - if self.is_directed() { - degree += self.in_edge_vec.as_ref().unwrap().neighbors(id).len() - } - degree - } - - fn neighbors_iter(&self, id: Id) -> Iter { - let neighbors = self.edge_vec.neighbors(id); - Iter::new(Box::new(neighbors.iter().map(|x| *x))) - } - - // if exist multi edges, the neighbours will repeat. - fn neighbors(&self, id: Id) -> Cow<[Id]> { - self.edge_vec.neighbors(id).into() - } - - fn max_seen_id(&self) -> Option { - Some(Id::new(self.node_count() - 1)) - } - - fn implementation(&self) -> GraphImpl { - GraphImpl::MultiGraph - } -} diff --git a/src/graph_impl/multi_graph/mod.rs b/src/graph_impl/multi_graph/mod.rs index 037e156d..21fb3639 100644 --- a/src/graph_impl/multi_graph/mod.rs +++ b/src/graph_impl/multi_graph/mod.rs @@ -1,5 +1,6 @@ +#[macro_use] +pub mod plan; pub mod catalog; -pub mod graph; pub mod node; -pub mod plan; pub mod query; +pub mod utils; diff --git a/src/graph_impl/multi_graph/plan/mod.rs b/src/graph_impl/multi_graph/plan/mod.rs index ec19d264..f3b1fffe 100644 --- a/src/graph_impl/multi_graph/plan/mod.rs +++ b/src/graph_impl/multi_graph/plan/mod.rs @@ -1,2 +1,3 @@ +#[macro_use] pub mod operator; pub mod query_plan; diff --git a/src/graph_impl/multi_graph/plan/operator/extend/EI.rs b/src/graph_impl/multi_graph/plan/operator/extend/EI.rs new file mode 100644 index 00000000..4b3a4692 --- /dev/null +++ b/src/graph_impl/multi_graph/plan/operator/extend/EI.rs @@ -0,0 +1,353 @@ +use graph_impl::multi_graph::catalog::adj_list_descriptor::{AdjListDescriptor, Direction}; +use graph_impl::multi_graph::catalog::query_graph::QueryGraph; +use graph_impl::multi_graph::plan::operator::operator::{BaseOperator, CommonOperatorTrait, Operator}; +use graph_impl::multi_graph::plan::operator::extend::extend::Extend; +use graph_impl::multi_graph::plan::operator::extend::intersect::{Intersect, IntersectType}; +use graph_impl::static_graph::sorted_adj_vec::SortedAdjVec; +use itertools::Itertools; +use hashbrown::HashMap; +use generic::{IdType, GraphType}; +use graph_impl::TypedStaticGraph; +use std::hash::{Hash, BuildHasherDefault}; +use indexmap::Equivalent; + +pub static DIFFERENTIATE_FWD_BWD_SINGLE_ALD: bool = false; + +#[derive(Clone)] +pub enum CachingType { + None, + FullCaching, + PartialCaching, +} + + +impl PartialEq for CachingType { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (CachingType::None, CachingType::None) | + (CachingType::FullCaching, CachingType::FullCaching) | + (CachingType::PartialCaching, CachingType::PartialCaching) => true, + _ => false + } + } +} + +#[derive(Clone)] +pub enum EI { + Base(BaseEI), + Extend(Extend), + Intersect(Intersect), +} + +#[derive(Clone)] +pub struct BaseEI { + pub base_op: BaseOperator, + pub vertex_types: Vec, + pub to_type: usize, + pub to_query_vertex: String, + pub alds: Vec, + pub out_idx: usize, + pub vertex_idx: Vec, + pub vertex_idx_to_cache: Vec, + pub labels_or_to_types: Vec, + pub labels_or_to_types_to_cache: Vec, + pub adj_lists: Vec>>>, + pub adj_lists_to_cache: Vec>>>, + pub caching_type: CachingType, + is_intersection_cached: bool, + last_vertex_ids_intersected: Option>, + pub out_neighbours: Neighbours, + pub init_neighbours: Neighbours, + pub temp_neighbours: Neighbours, + pub cached_neighbours: Neighbours, +} + +impl BaseEI { + pub fn new( + to_query_vertex: String, + to_type: usize, + alds: Vec, + out_subgraph: Box, + in_subgraph: Option>, + ) -> BaseEI { + let mut ei = BaseEI { + base_op: BaseOperator::new(out_subgraph, in_subgraph), + vertex_types: vec![], + to_type, + to_query_vertex, + alds, + out_idx: 0, + vertex_idx: vec![], + vertex_idx_to_cache: vec![], + labels_or_to_types: vec![], + labels_or_to_types_to_cache: vec![], + adj_lists: vec![], + adj_lists_to_cache: vec![], + caching_type: CachingType::None, + is_intersection_cached: false, + last_vertex_ids_intersected: None, + out_neighbours: Neighbours::new(), + init_neighbours: Neighbours::new(), + temp_neighbours: Neighbours::new(), + cached_neighbours: Neighbours::new(), + }; + ei.set_operator_name(); + ei + } + + fn set_operator_name(&mut self) { + let variables = self.alds.iter() + .map(|ald| { + ald.from_query_vertex.clone() + "[" + if let Direction::Fwd = ald.direction { "Fwd" } else { "Bwd" } + "]" + }).sorted().join("-"); + if 1 == self.alds.len() { + self.base_op.name = "Single-Edge-extend".to_string(); + } else { + self.base_op.name = "Multi-Edge-extend".to_string(); + } + self.base_op.name += &(" TO (".to_owned() + &self.to_query_vertex + ") From (" + &variables + ")"); + } + + pub fn make(to_qvertex: String, to_type: usize, alds: Vec, + out_subgraph: QueryGraph, in_subgraph: QueryGraph, + out_qvertex_to_idx_map: HashMap) -> EI { + if 1 == alds.len() { + return EI::Extend(Extend::new(to_qvertex, to_type, alds, Box::new(out_subgraph), + Some(Box::new(in_subgraph)), out_qvertex_to_idx_map)); + } + EI::Intersect(Intersect::new(to_qvertex, to_type, alds, Box::new(out_subgraph), + Some(Box::new(in_subgraph)), out_qvertex_to_idx_map)) + } + + pub fn is_intersection_cached(&mut self) -> bool { + self.is_intersection_cached = true; + for i in 0..self.last_vertex_ids_intersected.as_ref().unwrap().len() { + if self.last_vertex_ids_intersected.as_ref().unwrap()[i] != self.base_op.probe_tuple[self.vertex_idx_to_cache[i]] { + self.is_intersection_cached = false; + self.last_vertex_ids_intersected.as_mut().unwrap()[i] = self.base_op.probe_tuple[self.vertex_idx_to_cache[i]]; + } + } + self.is_intersection_cached + } + pub fn init_caching(&mut self, last_repeated_vertex_idx: usize) { + if self.alds.len() == 1 { + return; + } + let num_cached_alds = self.alds.iter().filter(|&ald| ald.vertex_idx <= last_repeated_vertex_idx).count(); + if num_cached_alds <= 1 { + return; + } + if num_cached_alds == self.alds.len() { + self.caching_type = CachingType::FullCaching; + } else { + self.caching_type = CachingType::PartialCaching; + } + self.last_vertex_ids_intersected = Some(vec![Id::new(0); num_cached_alds]); + } + + fn init_extensions(&mut self, graph: &TypedStaticGraph) { + match self.caching_type { + CachingType::None | CachingType::FullCaching => { + *&mut self.out_neighbours = Neighbours::new(); + if 1 == self.alds.len() { + return; + } + } + _ => {} + } + let mut largest_adj_list_size = 0; + for ald in &self.alds { + let label = if graph.is_sorted_by_node() { self.to_type } else { ald.label }; + let adj_list_size = graph.get_largest_adj_list_size(label, ald.direction.clone()); + if adj_list_size > largest_adj_list_size { + largest_adj_list_size = adj_list_size; + } + } + if let CachingType::PartialCaching = self.caching_type { + self.out_neighbours = Neighbours::with_capacity(largest_adj_list_size); + } + self.init_neighbours = Neighbours::new(); + self.cached_neighbours = Neighbours::with_capacity(largest_adj_list_size); + if self.alds.len() > 2 { + self.temp_neighbours = Neighbours::with_capacity(largest_adj_list_size); + } + } + + fn set_alds_and_adj_lists(&mut self, graph: &TypedStaticGraph, last_repeated_vertex_idx: usize) { + let num_cached_alds = if let Some(ids) = &mut self.last_vertex_ids_intersected { + ids.len() + } else { + self.alds.len() + }; + self.vertex_idx_to_cache = vec![0; num_cached_alds]; + self.labels_or_to_types_to_cache = vec![0; num_cached_alds]; + self.adj_lists_to_cache = vec![vec![]; num_cached_alds]; + if let CachingType::PartialCaching = self.caching_type { + self.vertex_idx = vec![0; self.alds.len() - num_cached_alds]; + self.labels_or_to_types = vec![0; self.alds.len() - num_cached_alds]; + self.adj_lists = vec![vec![]; self.alds.len() - num_cached_alds]; + } + let mut idx = 0; + let mut idx_to_cache = 0; + for ald in &self.alds { + if let CachingType::PartialCaching = self.caching_type { + if ald.vertex_idx > last_repeated_vertex_idx { + self.vertex_idx[idx] = ald.vertex_idx; + self.labels_or_to_types[idx] = if graph.is_sorted_by_node() { + self.to_type + } else { + ald.label + }; + self.adj_lists[idx] = if let Direction::Fwd = ald.direction { + graph.get_fwd_adj_list().clone() + } else { + graph.get_bwd_adj_list().clone() + }; + idx += 1; + continue; + } + } + self.vertex_idx_to_cache[idx_to_cache] = ald.vertex_idx; + self.labels_or_to_types_to_cache[idx_to_cache] = if graph.is_sorted_by_node() { + self.to_type + } else { + ald.label + }; + self.adj_lists_to_cache[idx_to_cache] = if let Direction::Fwd = ald.direction { + graph.get_fwd_adj_list().clone() + } else { + graph.get_bwd_adj_list().clone() + }; + idx_to_cache += 1; + } + } + pub fn execute_intersect(&mut self, idx: usize, intersect_type: IntersectType) -> usize { + let (adj_vec, label_or_type) = match intersect_type { + IntersectType::CachedOut| IntersectType::TempOut => ( + self.adj_lists_to_cache[idx][self.base_op.probe_tuple[self.vertex_idx[idx]].id()].as_ref(), + self.labels_or_to_types[idx] + ), + _ => ( + self.adj_lists_to_cache[idx][self.base_op.probe_tuple[self.vertex_idx_to_cache[idx]].id()].as_ref(), + self.labels_or_to_types_to_cache[idx] + ) + }; + let init = &mut self.init_neighbours; + let cached = &mut self.cached_neighbours; + let temp = &mut self.temp_neighbours; + let out = &mut self.out_neighbours; + adj_vec.map_or(0, |adj| { + match intersect_type { + IntersectType::InitCached => adj.intersect(label_or_type, init, cached), + IntersectType::TempCached => adj.intersect(label_or_type, temp, cached), + IntersectType::CachedOut => adj.intersect(label_or_type, cached, out), + IntersectType::TempOut => adj.intersect(label_or_type, temp, out), + } + }) + } +} + +impl CommonOperatorTrait for EI { + fn init(&mut self, probe_tuple: Vec, graph: &TypedStaticGraph) { + let base_ei = get_ei_as_mut!(self); + let base_op = &mut base_ei.base_op; + *&mut base_op.probe_tuple = probe_tuple.clone(); + *&mut base_ei.caching_type = CachingType::None; + *&mut base_ei.vertex_types = graph.get_node_types().clone(); + let prev = base_op.prev.as_mut().unwrap().as_mut(); + let last_repeated_vertex_idx = get_op_attr_as_mut!(prev,last_repeated_vertex_idx).clone(); + base_ei.init_caching(last_repeated_vertex_idx); + base_ei.init_extensions(graph); + base_ei.set_alds_and_adj_lists(graph, last_repeated_vertex_idx); +// let base_ei = get_ei_as_mut!(self); + base_ei.base_op.next.as_mut().unwrap().iter_mut().foreach(|next_op| { + next_op.init(probe_tuple.clone(), graph); + }); + } + + fn process_new_tuple(&mut self) { + unimplemented!() + } + + fn execute(&mut self) { + unimplemented!() + } + + fn get_alds_as_string(&self) -> String { + let base_ei = get_ei_as_ref!(self); + if !DIFFERENTIATE_FWD_BWD_SINGLE_ALD && 1 == base_ei.alds.len() { + return "E".to_owned() + &base_ei.alds[0].label.to_string(); + } + let mut directions = vec!["".to_owned(); base_ei.alds.len()]; + for ald in &base_ei.alds { + let dir = if let Direction::Fwd = ald.direction { "F".to_owned() } else { "B".to_owned() }; + directions.push(dir + &ald.label.to_string()); + } + directions.sort(); + directions.join("-") + } + + fn update_operator_name(&mut self, mut query_vertex_to_index_map: HashMap) { + let mut prev_to_query_vertices = vec!["".to_owned(); query_vertex_to_index_map.len()]; + for (query_vertex, &index) in &query_vertex_to_index_map { + prev_to_query_vertices[index] = query_vertex.clone(); + } + let base_ei = get_ei_as_mut!(self); + *&mut base_ei.base_op.name = serde_json::to_string(&prev_to_query_vertices).unwrap() + " - " + &base_ei.base_op.name; + query_vertex_to_index_map.insert(base_ei.to_query_vertex.clone(), query_vertex_to_index_map.len()); + if let Some(next) = &mut base_ei.base_op.next { + next.iter_mut().foreach(|op| op.update_operator_name(query_vertex_to_index_map.clone())) + } + } + + fn copy(&self, is_thread_safe: bool) -> Option> { + unimplemented!() + } + + fn is_same_as(&mut self, op: &mut Operator) -> bool { + unimplemented!() + } +} + +impl EI { + pub fn has_multi_edge_extends(&self) -> bool { + let base_ei = get_ei_as_ref!(self); + if base_ei.alds.len() > 1 { + return true; + } + if let Some(prev) = &base_ei.base_op.prev { + return prev.has_multi_edge_extends(); + } + false + } +} + +#[derive(Clone)] +pub struct Neighbours { + pub ids: Vec, + pub start_idx: usize, + pub end_idx: usize, +} + +impl Neighbours { + pub fn new() -> Neighbours { + Neighbours { + ids: vec![], + start_idx: 0, + end_idx: 0, + } + } + + pub fn with_capacity(capacity: usize) -> Neighbours { + Neighbours { + ids: vec![Id::new(0); capacity], + start_idx: 0, + end_idx: 0, + } + } + + pub fn reset(&mut self) { + self.start_idx = 0; + self.end_idx = 0; + } +} diff --git a/src/graph_impl/multi_graph/plan/operator/extend/Extend.rs b/src/graph_impl/multi_graph/plan/operator/extend/Extend.rs new file mode 100644 index 00000000..bf14873c --- /dev/null +++ b/src/graph_impl/multi_graph/plan/operator/extend/Extend.rs @@ -0,0 +1,101 @@ +use graph_impl::multi_graph::plan::operator::extend::EI::{BaseEI, Neighbours, EI, DIFFERENTIATE_FWD_BWD_SINGLE_ALD}; +use graph_impl::multi_graph::catalog::adj_list_descriptor::{Direction, AdjListDescriptor}; +use graph_impl::static_graph::sorted_adj_vec::SortedAdjVec; +use graph_impl::multi_graph::catalog::query_graph::QueryGraph; +use hashbrown::HashMap; +use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; +use graph_impl::TypedStaticGraph; +use generic::{IdType, GraphType}; +use std::hash::Hash; + +#[derive(Clone)] +pub struct Extend { + pub base_ei: BaseEI, + vertex_index: usize, + label_or_to_type: usize, + pub dir: Direction, + adj_list: Vec>>, +} + +impl Extend { + pub fn new(to_qvertex: String, to_type: usize, alds: Vec, + out_subgraph: Box, in_subgraph: Option>, + out_qvertex_to_idx_map: HashMap) -> Extend { + let ald = alds.get(0).unwrap().clone(); + let mut extend = Extend { + base_ei: BaseEI::new(to_qvertex.clone(), to_type, alds, out_subgraph, in_subgraph), + vertex_index: ald.vertex_idx, + label_or_to_type: ald.label, + dir: ald.direction.clone(), + adj_list: vec![], + }; + extend.base_ei.base_op.last_repeated_vertex_idx = extend.base_ei.base_op.out_tuple_len - 2; + extend.base_ei.out_idx = out_qvertex_to_idx_map.get(&to_qvertex).unwrap().clone(); + extend.base_ei.base_op.out_qvertex_to_idx_map = out_qvertex_to_idx_map; + extend + } +} + +impl CommonOperatorTrait for Extend { + fn init(&mut self, probe_tuple: Vec, graph: &TypedStaticGraph) { + self.base_ei.out_neighbours = Neighbours::new(); + self.base_ei.base_op.probe_tuple = probe_tuple.clone(); + self.base_ei.vertex_types = graph.get_node_types().clone(); + self.adj_list = if let Direction::Fwd = self.dir { graph.get_fwd_adj_list() } else { graph.get_bwd_adj_list() }.clone(); + if graph.is_sorted_by_node() { + self.label_or_to_type = self.base_ei.to_type; + self.base_ei.to_type = 0; + } + for next_operator in self.base_ei.base_op.next.as_mut().unwrap() { + next_operator.init(probe_tuple.clone(), graph); + } + } + + fn process_new_tuple(&mut self) { + let adj_vec = self.adj_list[self.base_ei.base_op.probe_tuple[self.vertex_index].id()].as_mut().unwrap(); + let out_neighbour = &mut self.base_ei.out_neighbours; + adj_vec.set_neighbor_ids(self.label_or_to_type, out_neighbour); + self.base_ei.base_op.icost += out_neighbour.end_idx - out_neighbour.start_idx; + for idx in out_neighbour.start_idx..out_neighbour.end_idx { + if self.base_ei.to_type == 0 || self.base_ei.to_type == self.base_ei.vertex_types[out_neighbour.ids[idx].id()] { + self.base_ei.base_op.num_out_tuples += 1; + self.base_ei.base_op.probe_tuple[self.base_ei.out_idx] = out_neighbour.ids[idx]; + self.base_ei.base_op.next.as_mut().unwrap()[0].process_new_tuple(); + } + } + } + + fn execute(&mut self) { + unimplemented!() + } + + fn copy(&self, is_thread_safe: bool) -> Option> { + let base_ei = &self.base_ei; + let base_op = &base_ei.base_op; + let mut extend = Extend::new( + base_ei.to_query_vertex.clone(), base_ei.to_type, + base_ei.alds.clone(), + base_op.out_subgraph.clone(), base_op.in_subgraph.clone(), + base_op.out_qvertex_to_idx_map.clone(), + ); + let extend_copy = extend.clone(); + extend.base_ei.base_op.prev = base_op.prev.as_ref().unwrap().copy(is_thread_safe).map(|op| Box::new(op)); + let prev = extend.base_ei.base_op.prev.as_mut().unwrap().as_mut(); + *get_op_attr_as_mut!(prev,next) = Some(vec![Operator::EI(EI::Extend(extend_copy))]); + let last_repeated_vertex_idx = get_op_attr!(prev,last_repeated_vertex_idx); + extend.base_ei.init_caching(last_repeated_vertex_idx); + Some(Operator::EI(EI::Extend(extend))) + } + + fn is_same_as(&mut self, op: &mut Operator) -> bool { + if let Operator::EI(EI::Extend(extend)) = op { + return (!DIFFERENTIATE_FWD_BWD_SINGLE_ALD || self.dir == extend.dir) && + self.label_or_to_type == extend.label_or_to_type && + self.base_ei.to_type == extend.base_ei.to_type && + self.base_ei.base_op.in_subgraph.as_mut().unwrap().is_isomorphic_to(get_op_attr_as_mut!(op,in_subgraph).as_mut().unwrap().as_mut()) && + self.base_ei.base_op.out_subgraph.is_isomorphic_to(get_op_attr_as_mut!(op,out_subgraph).as_mut()) && + self.base_ei.base_op.prev.as_mut().unwrap().is_same_as(get_op_attr_as_mut!(op,prev).as_mut().unwrap()); + } + false + } +} \ No newline at end of file diff --git a/src/graph_impl/multi_graph/plan/operator/extend/Intersect.rs b/src/graph_impl/multi_graph/plan/operator/extend/Intersect.rs new file mode 100644 index 00000000..1db6d15f --- /dev/null +++ b/src/graph_impl/multi_graph/plan/operator/extend/Intersect.rs @@ -0,0 +1,135 @@ +use graph_impl::multi_graph::plan::operator::extend::EI::{BaseEI, Neighbours, CachingType, EI}; +use graph_impl::multi_graph::plan::operator::extend::EI::EI::Base; +use graph_impl::multi_graph::catalog::query_graph::QueryGraph; +use hashbrown::HashMap; +use graph_impl::multi_graph::catalog::adj_list_descriptor::AdjListDescriptor; +use generic::{IdType, GraphType}; +use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; +use graph_impl::TypedStaticGraph; +use std::hash::Hash; + +pub enum IntersectType { + InitCached, + TempCached, + CachedOut, + TempOut, +} + +#[derive(Clone)] +pub struct Intersect { + pub base_ei: BaseEI, +} + +impl Intersect { + pub fn new(to_qvertex: String, to_type: usize, alds: Vec, + out_subgraph: Box, in_subgraph: Option>, + out_qvertex_to_idx_map: HashMap) -> Intersect { + let mut intersect = Intersect { + base_ei: BaseEI::new(to_qvertex.clone(), to_type, alds, out_subgraph, in_subgraph), + }; + let base_op = &mut intersect.base_ei.base_op; + base_op.last_repeated_vertex_idx = base_op.out_tuple_len - 2; + base_op.out_qvertex_to_idx_map = out_qvertex_to_idx_map; + intersect.base_ei.out_idx = base_op.out_qvertex_to_idx_map.get(&to_qvertex).unwrap().clone(); + intersect + } +} + +impl CommonOperatorTrait for Intersect { + fn init(&mut self, probe_tuple: Vec, graph: &TypedStaticGraph) { + unimplemented!() + } + + fn process_new_tuple(&mut self) { + let mut temp = Neighbours::new(); + if CachingType::None == self.base_ei.caching_type || !self.base_ei.is_intersection_cached() { + let base_ei = &mut self.base_ei; + let mut cache_id = base_ei.vertex_idx_to_cache[0]; + let to_id = base_ei.base_op.probe_tuple[cache_id].id(); + let mut adj_vec = base_ei.adj_lists_to_cache[0][to_id].as_ref(); + cache_id = base_ei.labels_or_to_types_to_cache[0]; + let neighbours = &mut base_ei.init_neighbours; + adj_vec.map(|adj| adj.set_neighbor_ids(cache_id, neighbours)); + base_ei.base_op.icost += (base_ei.init_neighbours.end_idx - base_ei.init_neighbours.start_idx); + base_ei.base_op.icost += base_ei.execute_intersect(1, IntersectType::InitCached); + + if base_ei.to_type != 0 { + let mut curr_end_idx = 0; + let cached_neighbours = &mut base_ei.cached_neighbours; + for i in cached_neighbours.start_idx..cached_neighbours.end_idx { + if base_ei.vertex_types[cached_neighbours.ids[i].id()] == base_ei.to_type { + cached_neighbours.ids[curr_end_idx] = cached_neighbours.ids[i]; + curr_end_idx += 1; + } + } + cached_neighbours.end_idx = curr_end_idx; + } + for i in 2..base_ei.adj_lists_to_cache.len() { + temp = base_ei.cached_neighbours.clone(); + base_ei.cached_neighbours = base_ei.temp_neighbours.clone(); + base_ei.temp_neighbours = temp.clone(); + base_ei.base_op.icost += base_ei.execute_intersect(i, IntersectType::TempCached); + } + } + + let base_ei = &mut self.base_ei; + match base_ei.caching_type { + CachingType::None | CachingType::FullCaching => base_ei.out_neighbours = base_ei.cached_neighbours.clone(), + CachingType::PartialCaching => { + let cost = base_ei.execute_intersect(0, IntersectType::CachedOut); + base_ei.base_op.icost += cost; + for i in 1..base_ei.adj_lists.len() { + temp = base_ei.out_neighbours.clone(); + base_ei.out_neighbours = base_ei.temp_neighbours.clone(); + base_ei.temp_neighbours = temp.clone(); + base_ei.base_op.icost += base_ei.execute_intersect(i, IntersectType::TempOut); + } + } + } + + let base_op = &mut base_ei.base_op; + // setAdjListSortOrder the initNeighbours ids in the output tuple. + let out_neighbours = &mut base_ei.out_neighbours; + base_op.num_out_tuples += (out_neighbours.end_idx - out_neighbours.start_idx); + for idx in out_neighbours.start_idx..out_neighbours.end_idx { + base_op.probe_tuple[base_ei.out_idx] = out_neighbours.ids[idx]; + base_op.next.as_mut().map(|op_vec| { + op_vec.get_mut(0).map(|op| op.process_new_tuple()); + }); + } + } + + fn execute(&mut self) { + unimplemented!() + } + + fn copy(&self, is_thread_safe: bool) -> Option> { + let base_ei = &self.base_ei; + let base_op = &base_ei.base_op; + let mut intersect = Intersect::new( + base_ei.to_query_vertex.clone(), base_ei.to_type, + base_ei.alds.clone(), + base_op.out_subgraph.clone(), base_op.in_subgraph.clone(), + base_op.out_qvertex_to_idx_map.clone(), + ); + let intersect_copy = intersect.clone(); + intersect.base_ei.base_op.prev = base_op.prev.as_ref().unwrap().copy(is_thread_safe).map(|op| Box::new(op)); + let prev = intersect.base_ei.base_op.prev.as_mut().unwrap().as_mut(); + *get_op_attr_as_mut!(prev,next) = Some(vec![Operator::EI(EI::Intersect(intersect_copy))]); + let last_repeated_vertex_idx = get_op_attr!(prev,last_repeated_vertex_idx); + intersect.base_ei.init_caching(last_repeated_vertex_idx); + Some(Operator::EI(EI::Intersect(intersect))) + } + + fn is_same_as(&mut self, op: &mut Operator) -> bool { + if let Operator::EI(EI::Intersect(intersect)) = op { + return + self.base_ei.caching_type == intersect.base_ei.caching_type && + self.get_alds_as_string() == intersect.get_alds_as_string() && + self.base_ei.base_op.in_subgraph.as_mut().unwrap().is_isomorphic_to(get_op_attr_as_mut!(op,in_subgraph).as_mut().unwrap().as_mut()) && + self.base_ei.base_op.out_subgraph.is_isomorphic_to(get_op_attr_as_mut!(op,out_subgraph).as_mut()) && + self.base_ei.base_op.prev.as_mut().unwrap().is_same_as(get_op_attr_as_mut!(op,prev).as_mut().unwrap()); + } + false + } +} \ No newline at end of file diff --git a/src/graph_impl/multi_graph/plan/operator/extend/extend.rs b/src/graph_impl/multi_graph/plan/operator/extend/extend.rs new file mode 100644 index 00000000..bf14873c --- /dev/null +++ b/src/graph_impl/multi_graph/plan/operator/extend/extend.rs @@ -0,0 +1,101 @@ +use graph_impl::multi_graph::plan::operator::extend::EI::{BaseEI, Neighbours, EI, DIFFERENTIATE_FWD_BWD_SINGLE_ALD}; +use graph_impl::multi_graph::catalog::adj_list_descriptor::{Direction, AdjListDescriptor}; +use graph_impl::static_graph::sorted_adj_vec::SortedAdjVec; +use graph_impl::multi_graph::catalog::query_graph::QueryGraph; +use hashbrown::HashMap; +use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; +use graph_impl::TypedStaticGraph; +use generic::{IdType, GraphType}; +use std::hash::Hash; + +#[derive(Clone)] +pub struct Extend { + pub base_ei: BaseEI, + vertex_index: usize, + label_or_to_type: usize, + pub dir: Direction, + adj_list: Vec>>, +} + +impl Extend { + pub fn new(to_qvertex: String, to_type: usize, alds: Vec, + out_subgraph: Box, in_subgraph: Option>, + out_qvertex_to_idx_map: HashMap) -> Extend { + let ald = alds.get(0).unwrap().clone(); + let mut extend = Extend { + base_ei: BaseEI::new(to_qvertex.clone(), to_type, alds, out_subgraph, in_subgraph), + vertex_index: ald.vertex_idx, + label_or_to_type: ald.label, + dir: ald.direction.clone(), + adj_list: vec![], + }; + extend.base_ei.base_op.last_repeated_vertex_idx = extend.base_ei.base_op.out_tuple_len - 2; + extend.base_ei.out_idx = out_qvertex_to_idx_map.get(&to_qvertex).unwrap().clone(); + extend.base_ei.base_op.out_qvertex_to_idx_map = out_qvertex_to_idx_map; + extend + } +} + +impl CommonOperatorTrait for Extend { + fn init(&mut self, probe_tuple: Vec, graph: &TypedStaticGraph) { + self.base_ei.out_neighbours = Neighbours::new(); + self.base_ei.base_op.probe_tuple = probe_tuple.clone(); + self.base_ei.vertex_types = graph.get_node_types().clone(); + self.adj_list = if let Direction::Fwd = self.dir { graph.get_fwd_adj_list() } else { graph.get_bwd_adj_list() }.clone(); + if graph.is_sorted_by_node() { + self.label_or_to_type = self.base_ei.to_type; + self.base_ei.to_type = 0; + } + for next_operator in self.base_ei.base_op.next.as_mut().unwrap() { + next_operator.init(probe_tuple.clone(), graph); + } + } + + fn process_new_tuple(&mut self) { + let adj_vec = self.adj_list[self.base_ei.base_op.probe_tuple[self.vertex_index].id()].as_mut().unwrap(); + let out_neighbour = &mut self.base_ei.out_neighbours; + adj_vec.set_neighbor_ids(self.label_or_to_type, out_neighbour); + self.base_ei.base_op.icost += out_neighbour.end_idx - out_neighbour.start_idx; + for idx in out_neighbour.start_idx..out_neighbour.end_idx { + if self.base_ei.to_type == 0 || self.base_ei.to_type == self.base_ei.vertex_types[out_neighbour.ids[idx].id()] { + self.base_ei.base_op.num_out_tuples += 1; + self.base_ei.base_op.probe_tuple[self.base_ei.out_idx] = out_neighbour.ids[idx]; + self.base_ei.base_op.next.as_mut().unwrap()[0].process_new_tuple(); + } + } + } + + fn execute(&mut self) { + unimplemented!() + } + + fn copy(&self, is_thread_safe: bool) -> Option> { + let base_ei = &self.base_ei; + let base_op = &base_ei.base_op; + let mut extend = Extend::new( + base_ei.to_query_vertex.clone(), base_ei.to_type, + base_ei.alds.clone(), + base_op.out_subgraph.clone(), base_op.in_subgraph.clone(), + base_op.out_qvertex_to_idx_map.clone(), + ); + let extend_copy = extend.clone(); + extend.base_ei.base_op.prev = base_op.prev.as_ref().unwrap().copy(is_thread_safe).map(|op| Box::new(op)); + let prev = extend.base_ei.base_op.prev.as_mut().unwrap().as_mut(); + *get_op_attr_as_mut!(prev,next) = Some(vec![Operator::EI(EI::Extend(extend_copy))]); + let last_repeated_vertex_idx = get_op_attr!(prev,last_repeated_vertex_idx); + extend.base_ei.init_caching(last_repeated_vertex_idx); + Some(Operator::EI(EI::Extend(extend))) + } + + fn is_same_as(&mut self, op: &mut Operator) -> bool { + if let Operator::EI(EI::Extend(extend)) = op { + return (!DIFFERENTIATE_FWD_BWD_SINGLE_ALD || self.dir == extend.dir) && + self.label_or_to_type == extend.label_or_to_type && + self.base_ei.to_type == extend.base_ei.to_type && + self.base_ei.base_op.in_subgraph.as_mut().unwrap().is_isomorphic_to(get_op_attr_as_mut!(op,in_subgraph).as_mut().unwrap().as_mut()) && + self.base_ei.base_op.out_subgraph.is_isomorphic_to(get_op_attr_as_mut!(op,out_subgraph).as_mut()) && + self.base_ei.base_op.prev.as_mut().unwrap().is_same_as(get_op_attr_as_mut!(op,prev).as_mut().unwrap()); + } + false + } +} \ No newline at end of file diff --git a/src/graph_impl/multi_graph/plan/operator/extend/intersect.rs b/src/graph_impl/multi_graph/plan/operator/extend/intersect.rs new file mode 100644 index 00000000..1db6d15f --- /dev/null +++ b/src/graph_impl/multi_graph/plan/operator/extend/intersect.rs @@ -0,0 +1,135 @@ +use graph_impl::multi_graph::plan::operator::extend::EI::{BaseEI, Neighbours, CachingType, EI}; +use graph_impl::multi_graph::plan::operator::extend::EI::EI::Base; +use graph_impl::multi_graph::catalog::query_graph::QueryGraph; +use hashbrown::HashMap; +use graph_impl::multi_graph::catalog::adj_list_descriptor::AdjListDescriptor; +use generic::{IdType, GraphType}; +use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; +use graph_impl::TypedStaticGraph; +use std::hash::Hash; + +pub enum IntersectType { + InitCached, + TempCached, + CachedOut, + TempOut, +} + +#[derive(Clone)] +pub struct Intersect { + pub base_ei: BaseEI, +} + +impl Intersect { + pub fn new(to_qvertex: String, to_type: usize, alds: Vec, + out_subgraph: Box, in_subgraph: Option>, + out_qvertex_to_idx_map: HashMap) -> Intersect { + let mut intersect = Intersect { + base_ei: BaseEI::new(to_qvertex.clone(), to_type, alds, out_subgraph, in_subgraph), + }; + let base_op = &mut intersect.base_ei.base_op; + base_op.last_repeated_vertex_idx = base_op.out_tuple_len - 2; + base_op.out_qvertex_to_idx_map = out_qvertex_to_idx_map; + intersect.base_ei.out_idx = base_op.out_qvertex_to_idx_map.get(&to_qvertex).unwrap().clone(); + intersect + } +} + +impl CommonOperatorTrait for Intersect { + fn init(&mut self, probe_tuple: Vec, graph: &TypedStaticGraph) { + unimplemented!() + } + + fn process_new_tuple(&mut self) { + let mut temp = Neighbours::new(); + if CachingType::None == self.base_ei.caching_type || !self.base_ei.is_intersection_cached() { + let base_ei = &mut self.base_ei; + let mut cache_id = base_ei.vertex_idx_to_cache[0]; + let to_id = base_ei.base_op.probe_tuple[cache_id].id(); + let mut adj_vec = base_ei.adj_lists_to_cache[0][to_id].as_ref(); + cache_id = base_ei.labels_or_to_types_to_cache[0]; + let neighbours = &mut base_ei.init_neighbours; + adj_vec.map(|adj| adj.set_neighbor_ids(cache_id, neighbours)); + base_ei.base_op.icost += (base_ei.init_neighbours.end_idx - base_ei.init_neighbours.start_idx); + base_ei.base_op.icost += base_ei.execute_intersect(1, IntersectType::InitCached); + + if base_ei.to_type != 0 { + let mut curr_end_idx = 0; + let cached_neighbours = &mut base_ei.cached_neighbours; + for i in cached_neighbours.start_idx..cached_neighbours.end_idx { + if base_ei.vertex_types[cached_neighbours.ids[i].id()] == base_ei.to_type { + cached_neighbours.ids[curr_end_idx] = cached_neighbours.ids[i]; + curr_end_idx += 1; + } + } + cached_neighbours.end_idx = curr_end_idx; + } + for i in 2..base_ei.adj_lists_to_cache.len() { + temp = base_ei.cached_neighbours.clone(); + base_ei.cached_neighbours = base_ei.temp_neighbours.clone(); + base_ei.temp_neighbours = temp.clone(); + base_ei.base_op.icost += base_ei.execute_intersect(i, IntersectType::TempCached); + } + } + + let base_ei = &mut self.base_ei; + match base_ei.caching_type { + CachingType::None | CachingType::FullCaching => base_ei.out_neighbours = base_ei.cached_neighbours.clone(), + CachingType::PartialCaching => { + let cost = base_ei.execute_intersect(0, IntersectType::CachedOut); + base_ei.base_op.icost += cost; + for i in 1..base_ei.adj_lists.len() { + temp = base_ei.out_neighbours.clone(); + base_ei.out_neighbours = base_ei.temp_neighbours.clone(); + base_ei.temp_neighbours = temp.clone(); + base_ei.base_op.icost += base_ei.execute_intersect(i, IntersectType::TempOut); + } + } + } + + let base_op = &mut base_ei.base_op; + // setAdjListSortOrder the initNeighbours ids in the output tuple. + let out_neighbours = &mut base_ei.out_neighbours; + base_op.num_out_tuples += (out_neighbours.end_idx - out_neighbours.start_idx); + for idx in out_neighbours.start_idx..out_neighbours.end_idx { + base_op.probe_tuple[base_ei.out_idx] = out_neighbours.ids[idx]; + base_op.next.as_mut().map(|op_vec| { + op_vec.get_mut(0).map(|op| op.process_new_tuple()); + }); + } + } + + fn execute(&mut self) { + unimplemented!() + } + + fn copy(&self, is_thread_safe: bool) -> Option> { + let base_ei = &self.base_ei; + let base_op = &base_ei.base_op; + let mut intersect = Intersect::new( + base_ei.to_query_vertex.clone(), base_ei.to_type, + base_ei.alds.clone(), + base_op.out_subgraph.clone(), base_op.in_subgraph.clone(), + base_op.out_qvertex_to_idx_map.clone(), + ); + let intersect_copy = intersect.clone(); + intersect.base_ei.base_op.prev = base_op.prev.as_ref().unwrap().copy(is_thread_safe).map(|op| Box::new(op)); + let prev = intersect.base_ei.base_op.prev.as_mut().unwrap().as_mut(); + *get_op_attr_as_mut!(prev,next) = Some(vec![Operator::EI(EI::Intersect(intersect_copy))]); + let last_repeated_vertex_idx = get_op_attr!(prev,last_repeated_vertex_idx); + intersect.base_ei.init_caching(last_repeated_vertex_idx); + Some(Operator::EI(EI::Intersect(intersect))) + } + + fn is_same_as(&mut self, op: &mut Operator) -> bool { + if let Operator::EI(EI::Intersect(intersect)) = op { + return + self.base_ei.caching_type == intersect.base_ei.caching_type && + self.get_alds_as_string() == intersect.get_alds_as_string() && + self.base_ei.base_op.in_subgraph.as_mut().unwrap().is_isomorphic_to(get_op_attr_as_mut!(op,in_subgraph).as_mut().unwrap().as_mut()) && + self.base_ei.base_op.out_subgraph.is_isomorphic_to(get_op_attr_as_mut!(op,out_subgraph).as_mut()) && + self.base_ei.base_op.prev.as_mut().unwrap().is_same_as(get_op_attr_as_mut!(op,prev).as_mut().unwrap()); + } + false + } +} \ No newline at end of file diff --git a/src/graph_impl/multi_graph/plan/operator/extend/mod.rs b/src/graph_impl/multi_graph/plan/operator/extend/mod.rs new file mode 100644 index 00000000..eacb3cf0 --- /dev/null +++ b/src/graph_impl/multi_graph/plan/operator/extend/mod.rs @@ -0,0 +1,3 @@ +pub mod EI; +pub mod extend; +pub mod intersect; \ No newline at end of file diff --git a/src/graph_impl/multi_graph/plan/operator/mod.rs b/src/graph_impl/multi_graph/plan/operator/mod.rs index 2850ad05..9ce3b569 100644 --- a/src/graph_impl/multi_graph/plan/operator/mod.rs +++ b/src/graph_impl/multi_graph/plan/operator/mod.rs @@ -1,3 +1,80 @@ +/// Get common attributes(mutable) from EI Operator +#[macro_export] +macro_rules! get_ei_as_mut { + ($item:expr) => { + match $item { + EI::Base(base)=>base, + EI::Extend(base)=> &mut base.base_ei, + EI::Intersect(base)=> &mut base.base_ei, + } + }; +} + +#[macro_export] +macro_rules! get_ei_as_ref { + ($item:expr) => { + match $item { + EI::Base(base)=>base, + EI::Extend(base)=> &base.base_ei, + EI::Intersect(base)=> &base.base_ei, + } + }; +} + +/// Get common attributes(Origin) from Operator +#[macro_export] +macro_rules! get_op_attr { + ($item:expr,$name:ident) => { + match $item { + Operator::Base(base) => base.$name, + Operator::Sink(sink) => sink.base_op.$name, + Operator::Scan(scan) => scan.base_op.$name, + Operator::ScanSampling(sp) => sp.scan.base_op.$name, + Operator::EI(ei) => match ei { + EI::Base(base)=>base.base_op.$name, + EI::Extend(base)=>base.base_ei.base_op.$name, + EI::Intersect(base)=>base.base_ei.base_op.$name, + }, + } + }; +} + +/// Get common attributes(reference) from Operator +#[macro_export] +macro_rules! get_op_attr_as_ref { + ($item:expr,$name:ident) => { + match $item { + Operator::Base(base) => &base.$name, + Operator::Sink(sink) => &sink.base_op.$name, + Operator::Scan(scan) => &scan.base_op.$name, + Operator::ScanSampling(sp) => &sp.scan.base_op.$name, + Operator::EI(ei) => match ei { + EI::Base(base)=>&base.base_op.$name, + EI::Extend(base)=>&base.base_ei.base_op.$name, + EI::Intersect(base)=>&base.base_ei.base_op.$name, + }, + } + }; +} + +/// Get common attributes(mutable) from Operator +#[macro_export] +macro_rules! get_op_attr_as_mut { + ($item:expr,$name:ident) => { + match $item { + Operator::Base(base) => &mut base.$name, + Operator::Sink(sink) => &mut sink.base_op.$name, + Operator::Scan(scan) => &mut scan.base_op.$name, + Operator::ScanSampling(sp) => &mut sp.scan.base_op.$name, + Operator::EI(ei) => match ei { + EI::Base(base)=>&mut base.base_op.$name, + EI::Extend(base)=>&mut base.base_ei.base_op.$name, + EI::Intersect(base)=>&mut base.base_ei.base_op.$name, + }, + } + }; +} pub mod operator; pub mod scan; pub mod sink; +pub mod extend; \ No newline at end of file diff --git a/src/graph_impl/multi_graph/plan/operator/operator.rs b/src/graph_impl/multi_graph/plan/operator/operator.rs index c95e7276..5d721ebc 100644 --- a/src/graph_impl/multi_graph/plan/operator/operator.rs +++ b/src/graph_impl/multi_graph/plan/operator/operator.rs @@ -1,38 +1,48 @@ use generic::{GraphType, IdType}; use graph_impl::multi_graph::catalog::query_graph::QueryGraph; +use graph_impl::multi_graph::plan::operator::extend::EI::EI; +use graph_impl::multi_graph::plan::operator::scan::scan::Scan; +use graph_impl::multi_graph::plan::operator::scan::scan_sampling::ScanSampling; +use graph_impl::multi_graph::plan::operator::sink::sink::Sink; use graph_impl::TypedStaticGraph; use hashbrown::{HashMap, HashSet}; use std::hash::{BuildHasherDefault, Hash}; use std::iter::FromIterator; use std::rc::Rc; +/// Operator types #[derive(Clone)] -pub enum OpType { - Base, - Sink, +pub enum Operator { + Base(BaseOperator), + Sink(Sink), + Scan(Scan), + ScanSampling(ScanSampling), + EI(EI), } +/// Basic operator #[derive(Clone)] -pub struct BaseOperator { +pub struct BaseOperator { pub name: String, - pub op_type: OpType, - pub next: Option>, - pub prev: Option>, - pub probe_tuple: Vec, + pub next: Option>>, + pub prev: Option>>, + pub probe_tuple: Vec, pub out_tuple_len: usize, - pub in_subgraph: Option>, - pub out_subgraph: Rc, + pub in_subgraph: Option>, + pub out_subgraph: Box, pub out_qvertex_to_idx_map: HashMap, pub last_repeated_vertex_idx: usize, pub num_out_tuples: usize, pub icost: usize, } -impl BaseOperator { - pub fn new(out_subgraph: Rc, in_subgraph: Option>) -> BaseOperator { +impl BaseOperator { + pub fn new( + out_subgraph: Box, + in_subgraph: Option>, + ) -> BaseOperator { BaseOperator { name: "".to_string(), - op_type: OpType::Base, next: None, prev: None, probe_tuple: vec![], @@ -46,151 +56,136 @@ impl BaseOperator { } } - pub fn copy(&self, is_thread_safe: bool) -> Option> { - unimplemented!(); + fn get_out_query_vertices(&self) -> Vec<&String> { + let idx_map = &self.out_qvertex_to_idx_map; + idx_map.iter().map(|(key, _val)| key).collect() } - pub fn copy_default(&self) -> Option> { - self.copy(false) - } -} -pub trait Operator { - fn get_op(&self) -> BaseOperator; - fn get_name(&self) -> String; - fn get_icost(&self) -> usize; - fn get_type(&self) -> OpType; - fn get_num_out_tuples(&self) -> usize; - fn get_query_vertex_id_map(&self) -> &HashMap; - fn get_out_query_vertices(&self) -> Vec { - self.get_query_vertex_id_map() - .iter() - .map(|(key, _val)| key.clone()) - .collect() - } - fn init( - &mut self, - probe_tuple: Vec, - graph: &TypedStaticGraph, - ); - fn is_same_as(&self, op: &BaseOperator) -> bool; - fn get_next_vec(&self) -> &Option>; - fn get_next_vec_as_mut(&mut self) -> &mut Option>; - fn get_prev(&self) -> Option<&Rc>; - fn get_next(&self, index: usize) -> &BaseOperator { - let next = self.get_next_vec().as_ref().unwrap(); + fn get_next(&self, index: usize) -> &Operator { + let next = self.next.as_ref().unwrap(); &next[index] } - fn set_next(&mut self, op: BaseOperator) { - self.get_next_vec_as_mut().replace(vec![op]); - } - fn set_next_vec(&mut self, op: Vec) { - self.get_next_vec_as_mut().replace(op); - } - fn set_next_batch(&mut self, ops: Vec); - fn process_new_tuple(); - fn execute(&self) { - if let Some(prev) = self.get_prev() { - prev.execute(); - } - } - fn get_alds_as_string() -> String { - String::from("") - } - fn update_operator_name(query_vertex_to_index_map: HashMap); - fn get_operator_metrics_next_operators( - &self, - operator_metrics: &mut Vec<(String, usize, usize)>, - ) { - operator_metrics.push((self.get_name(), self.get_icost(), self.get_num_out_tuples())); - if let Some(next) = self.get_next_vec() { - next.iter() - .filter(|op| { - if let OpType::Sink = op.get_type() { - return true; - } - false - }) - .for_each(|op| { - op.get_operator_metrics_next_operators(operator_metrics); - }); - } + fn set_next_vec(&mut self, op: Vec>) { + self.next.replace(op); } - fn get_last_operators(&self, last_operators: &mut Vec) { - if let Some(next) = self.get_next_vec() { - for op in next { - op.get_last_operators(last_operators); - } - } else { - last_operators.push(self.get_op()); - } - } - fn has_multi_edge_extends(&self) -> bool { - if let Some(prev) = self.get_prev() { - return prev.has_multi_edge_extends(); + fn execute(&mut self) { + if let Some(prev) = self.prev.as_mut() { + prev.execute(); } - return false; } } -impl Operator for BaseOperator { - fn get_op(&self) -> BaseOperator { - self.clone() - } - - fn get_name(&self) -> String { - self.name.clone() - } - - fn get_icost(&self) -> usize { - self.icost - } - - fn get_type(&self) -> OpType { - self.op_type.clone() - } - - fn get_num_out_tuples(&self) -> usize { - self.num_out_tuples +/// Common operations for every kind of operator +pub trait CommonOperatorTrait { + fn init( + &mut self, + probe_tuple: Vec, + graph: &TypedStaticGraph, + ); + fn process_new_tuple(&mut self); + fn execute(&mut self); + fn get_alds_as_string(&self) -> String { + "".to_string() } - - fn get_query_vertex_id_map(&self) -> &HashMap { - &self.out_qvertex_to_idx_map + fn update_operator_name(&mut self, query_vertex_to_index_map: HashMap) { + panic!("`update_operator_name()` on `BaseOperator`") } + fn copy(&self, is_thread_safe: bool) -> Option>; + fn is_same_as(&mut self, op: &mut Operator) -> bool; +} - fn init( +/// Abstract methods +impl CommonOperatorTrait for Operator { + fn init( &mut self, - probe_tuple: Vec, + probe_tuple: Vec, graph: &TypedStaticGraph, ) { - panic!("`init()` on `BaseOperator`") + match self { + Operator::Base(base) => {} + Operator::Sink(sink) => sink.init(probe_tuple, graph), + Operator::Scan(scan) => scan.init(probe_tuple, graph), + Operator::ScanSampling(sp) => sp.init(probe_tuple, graph), + _ => {} + } } - fn is_same_as(&self, op: &BaseOperator) -> bool { - panic!("`is_same_as()` on `BaseOperator`") + fn process_new_tuple(&mut self) { + match self { + Operator::Base(base) => {} + Operator::Sink(sink) => sink.process_new_tuple(), + Operator::Scan(scan) => scan.process_new_tuple(), + Operator::ScanSampling(sp) => sp.process_new_tuple(), + _ => {} + } } - fn get_next_vec(&self) -> &Option> { - &self.next + fn execute(&mut self) { + match self { + Operator::Base(base) => base.execute(), + Operator::Sink(sink) => sink.execute(), + Operator::Scan(scan) => scan.execute(), + Operator::ScanSampling(sp) => sp.execute(), + _ => {} + } } - - fn get_next_vec_as_mut(&mut self) -> &mut Option> { - &mut self.next + fn copy(&self, is_thread_safe: bool) -> Option> { + match self { + Operator::Base(base) => None, + Operator::Sink(sink) => sink.copy(is_thread_safe), + Operator::Scan(scan) => scan.copy(is_thread_safe), + Operator::ScanSampling(sp) => sp.copy(is_thread_safe), + _ => None + } } - - fn get_prev(&self) -> Option<&Rc> { - self.prev.as_ref() + fn is_same_as(&mut self, op: &mut Operator) -> bool { + match self { + Operator::Base(base) => false, + Operator::Sink(sink) => sink.is_same_as(op), + Operator::Scan(scan) => scan.is_same_as(op), + Operator::ScanSampling(sp) => sp.is_same_as(op), + _ => false + } } +} - fn set_next_batch(&mut self, ops: Vec) { - self.next = Some(ops); +impl Operator { + pub fn get_last_operators(&self, last_operators: &mut Vec>) { + if let Some(next) = get_op_attr_as_ref!(self, next) { + for op in next { + op.get_last_operators(last_operators); + } + } else { + last_operators.push(self.clone()); + } } - fn process_new_tuple() { - unimplemented!() + pub fn get_operator_metrics_next_operators( + &self, + operator_metrics: &mut Vec<(String, usize, usize)>, + ) { + let name: &String = get_op_attr_as_ref!(self, name); + let icost = get_op_attr!(self, icost); + let num_out_tuples = get_op_attr!(self, num_out_tuples); + operator_metrics.push((name.clone(), icost, num_out_tuples)); + if let Some(next) = get_op_attr_as_ref!(self, next) { + next.iter().for_each(|op| match op { + Operator::Sink(_) => {} + _ => op.get_operator_metrics_next_operators(operator_metrics), + }); + } } - fn update_operator_name(query_vertex_to_index_map: HashMap) { - panic!("`update_operator_name()` on `BaseOperator`") + pub fn has_multi_edge_extends(&self) -> bool { + match self { + Operator::EI(ei) => ei.has_multi_edge_extends(), + _ => { + if let Some(prev) = get_op_attr_as_ref!(self, prev) { + return prev.has_multi_edge_extends(); + } + false + } + } } } diff --git a/src/graph_impl/multi_graph/plan/operator/scan/scan.rs b/src/graph_impl/multi_graph/plan/operator/scan/scan.rs index 84b2da56..0050597e 100644 --- a/src/graph_impl/multi_graph/plan/operator/scan/scan.rs +++ b/src/graph_impl/multi_graph/plan/operator/scan/scan.rs @@ -1,11 +1,18 @@ +use generic::{GraphType, IdType}; use graph_impl::multi_graph::catalog::query_graph::QueryGraph; -use graph_impl::multi_graph::plan::operator::operator::BaseOperator; +use graph_impl::multi_graph::plan::operator::operator::{ + BaseOperator, CommonOperatorTrait, Operator, +}; +use graph_impl::multi_graph::plan::operator::sink::sink::Sink; use graph_impl::static_graph::sorted_adj_vec::SortedAdjVec; +use graph_impl::TypedStaticGraph; +use std::hash::Hash; use std::rc::Rc; //TODO: Fixing Scan Operator -pub struct Scan { - pub base_op: BaseOperator, +#[derive(Clone)] +pub struct Scan { + pub base_op: BaseOperator, from_query_vertex: String, to_query_vertex: String, from_type: usize, @@ -18,9 +25,9 @@ pub struct Scan { from_vertex_end_idx: usize, } -impl Scan { +impl Scan { //TODO: fixing functions - pub fn new(out_subgraph: Rc) -> Scan { + pub fn new(out_subgraph: Box) -> Scan { Scan { base_op: BaseOperator::new(out_subgraph, None), from_query_vertex: "".to_string(), @@ -36,3 +43,33 @@ impl Scan { } } } + +impl CommonOperatorTrait for Scan { + fn init( + &mut self, + probe_tuple: Vec, + graph: &TypedStaticGraph, + ) { + unimplemented!() + } + + fn process_new_tuple(&mut self) { + unimplemented!() + } + + fn execute(&mut self) { + unimplemented!() + } + + fn copy(&self, is_thread_safe: bool) -> Option> { + if is_thread_safe { + //TODO: Fixing Scanblocking + // return ScanBlocking::new(outSubgraph); + } + Some(Operator::Scan(Scan::new(self.base_op.out_subgraph.clone()))) + } + + fn is_same_as(&mut self, op: &mut Operator) -> bool { + unimplemented!() + } +} diff --git a/src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs b/src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs index 045010a6..c14e40c7 100644 --- a/src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs +++ b/src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs @@ -1,6 +1,8 @@ use generic::{GraphType, IdType}; use graph_impl::multi_graph::catalog::query_graph::QueryGraph; -use graph_impl::multi_graph::plan::operator::operator::{BaseOperator, OpType, Operator}; +use graph_impl::multi_graph::plan::operator::operator::{ + BaseOperator, CommonOperatorTrait, Operator, +}; use graph_impl::multi_graph::plan::operator::scan::scan::Scan; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; @@ -8,13 +10,14 @@ use std::hash::{BuildHasherDefault, Hash}; use std::rc::Rc; //TODO: Fixing ScanSampling -pub struct ScanSampling { - pub scan: Scan, - pub edges_queue: Vec>, +#[derive(Clone)] +pub struct ScanSampling { + pub scan: Scan, + pub edges_queue: Vec>, } -impl ScanSampling { - pub fn new(out_subgraph: Rc) -> Self { +impl ScanSampling { + pub fn new(out_subgraph: Box) -> Self { Self { scan: Scan::new(out_subgraph), edges_queue: vec![], @@ -24,64 +27,38 @@ impl ScanSampling { pub fn get_last_operator() {} } -impl Operator for ScanSampling { - fn get_op(&self) -> BaseOperator { - unimplemented!() - } - - fn get_name(&self) -> String { - unimplemented!() - } - - fn get_icost(&self) -> usize { - unimplemented!() - } - - fn get_type(&self) -> OpType { - unimplemented!() - } - - fn get_num_out_tuples(&self) -> usize { - unimplemented!() - } - - fn get_query_vertex_id_map(&self) -> &HashMap { - unimplemented!() - } - - fn init( +impl CommonOperatorTrait for ScanSampling { + fn init( &mut self, - probe_tuple: Vec, + probe_tuple: Vec, graph: &TypedStaticGraph, ) { unimplemented!() } - fn is_same_as(&self, op: &BaseOperator) -> bool { - unimplemented!() - } - - fn get_next_vec(&self) -> &Option> { + fn process_new_tuple(&mut self) { unimplemented!() } - fn get_next_vec_as_mut(&mut self) -> &mut Option> { - unimplemented!() - } - - fn get_prev(&self) -> Option<&Rc> { - unimplemented!() - } - - fn set_next_batch(&mut self, ops: Vec) { - unimplemented!() + fn execute(&mut self) { + while !self.edges_queue.is_empty() { + let edge = self.edges_queue.pop().unwrap(); + self.scan.base_op.probe_tuple[0] = edge[0]; + self.scan.base_op.probe_tuple[1] = edge[0]; + self.scan.base_op.num_out_tuples += 1; + for next_op in self.scan.base_op.next.as_mut().unwrap() { + next_op.process_new_tuple(); + } + } } - fn process_new_tuple() { - unimplemented!() + fn copy(&self, is_thread_safe: bool) -> Option> { + let mut scan_sampling = ScanSampling::new(self.scan.base_op.out_subgraph.clone()); + scan_sampling.edges_queue = self.edges_queue.clone(); + Some(Operator::ScanSampling(scan_sampling)) } - fn update_operator_name(query_vertex_to_index_map: HashMap) { + fn is_same_as(&mut self, op: &mut Operator) -> bool { unimplemented!() } } diff --git a/src/graph_impl/multi_graph/plan/operator/sink/sink.rs b/src/graph_impl/multi_graph/plan/operator/sink/sink.rs index bc5adb72..95bd8979 100644 --- a/src/graph_impl/multi_graph/plan/operator/sink/sink.rs +++ b/src/graph_impl/multi_graph/plan/operator/sink/sink.rs @@ -1,10 +1,12 @@ use generic::{GraphType, IdType}; use graph_impl::multi_graph::catalog::query_graph::QueryGraph; -use graph_impl::multi_graph::plan::operator::operator::{BaseOperator, OpType, Operator}; +use graph_impl::multi_graph::plan::operator::operator::{ + BaseOperator, CommonOperatorTrait, Operator, +}; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; +use graph_impl::multi_graph::plan::operator::extend::EI::EI; use std::hash::{BuildHasherDefault, Hash}; -use std::ops::Deref; use std::rc::Rc; pub enum SinkType { @@ -14,104 +16,67 @@ pub enum SinkType { Counter, } -pub struct Sink { - pub base_op: BaseOperator, - pub previous: Option>, +#[derive(Clone)] +pub struct Sink { + pub base_op: BaseOperator, + pub previous: Option>>, } -impl Sink { - pub fn new(query_graph: Rc) -> Self { +impl Sink { + pub fn new(query_graph: Box) -> Self { Self { base_op: BaseOperator::new(query_graph.clone(), Some(query_graph)), previous: None, } } - pub fn copy(&self, is_thread_safe: bool) -> Sink { - let mut sink = Sink::new(self.base_op.out_subgraph.clone()); - if let Some(prev) = &self.base_op.prev { - sink.base_op.prev = prev.copy(is_thread_safe); - } else { - sink.base_op.prev = None; - } - sink - } -} - -impl Operator for Sink { - fn get_op(&self) -> BaseOperator { - self.base_op.clone() - } - - fn get_name(&self) -> String { - unimplemented!() - } - - fn get_icost(&self) -> usize { - unimplemented!() - } - - fn get_type(&self) -> OpType { - unimplemented!() - } fn get_num_out_tuples(&self) -> usize { if let Some(prev) = &self.previous { - prev.iter().map(|op| op.get_num_out_tuples()).sum() + prev.iter() + .map(|op| get_op_attr_as_ref!(op, num_out_tuples)) + .sum() } else { - self.base_op.get_num_out_tuples() + self.base_op.num_out_tuples } } +} - fn get_query_vertex_id_map(&self) -> &HashMap { - unimplemented!() - } - - fn init( +impl CommonOperatorTrait for Sink { + fn init( &mut self, - probe_tuple: Vec, + probe_tuple: Vec, graph: &TypedStaticGraph, ) { self.base_op.probe_tuple = probe_tuple; } - fn is_same_as(&self, op: &BaseOperator) -> bool { - if let OpType::Sink = op.get_type() { - if let Some(prev) = self.get_prev() { - return prev.is_same_as(op.get_prev().unwrap()); - } - } - false - } - - fn get_next_vec(&self) -> &Option> { - unimplemented!() - } - - fn get_next_vec_as_mut(&mut self) -> &mut Option> { - unimplemented!() - } + fn process_new_tuple(&mut self) {} - fn get_prev(&self) -> Option<&Rc> { - unimplemented!() - } - - fn set_next_batch(&mut self, ops: Vec) { - unimplemented!() + fn execute(&mut self) { + if let Some(prev) = self.previous.as_mut() { + prev[0].execute(); + } else { + self.base_op.prev.as_mut().unwrap().execute(); + } } - fn process_new_tuple() {} - - fn execute(&self) { - if let Some(prev) = &self.previous { - prev[0].execute(); + fn copy(&self, is_thread_safe: bool) -> Option> { + let mut sink = Sink::new(self.base_op.out_subgraph.clone()); + if let Some(prev) = &self.base_op.prev { + sink.base_op.prev = Some(Box::new(prev.copy(is_thread_safe).unwrap())); } else { - if let Some(prev) = self.base_op.get_prev() { - prev.execute(); - } + sink.base_op.prev = None; } + Some(Operator::Sink(sink)) } - fn update_operator_name(query_vertex_to_index_map: HashMap) { - unimplemented!() + fn is_same_as(&mut self, op: &mut Operator) -> bool { + if let Operator::Sink(sink) = op { + if let Some(prev) = &mut self.base_op.prev { + let op_prev = get_op_attr_as_mut!(op, prev).as_mut().unwrap(); + return prev.is_same_as(op_prev.as_mut()); + } + } + false } } diff --git a/src/graph_impl/multi_graph/plan/query_plan.rs b/src/graph_impl/multi_graph/plan/query_plan.rs index ebbe8cad..e05c7e75 100644 --- a/src/graph_impl/multi_graph/plan/query_plan.rs +++ b/src/graph_impl/multi_graph/plan/query_plan.rs @@ -1,14 +1,17 @@ use graph_impl::multi_graph::plan::operator::operator::{BaseOperator, Operator}; use graph_impl::multi_graph::plan::operator::scan::scan_sampling::ScanSampling; use graph_impl::multi_graph::plan::operator::sink::sink::{Sink, SinkType}; +use graph_impl::multi_graph::plan::operator::extend::EI::EI; use hashbrown::HashMap; +use std::ops::Deref; use std::rc::Rc; +use generic::IdType; -pub struct QueryPlan { - sink: Option, +pub struct QueryPlan { + sink: Option>, sink_type: SinkType, - scan_sampling: Option, - last_operator: Option>, + scan_sampling: Option>, + last_operator: Option>>, out_tuples_limit: usize, elapsed_time: f64, icost: usize, @@ -17,23 +20,23 @@ pub struct QueryPlan { operator_metrics: Vec<(String, usize, usize)>, executed: bool, adaptive_enabled: bool, - subplans: Vec>, + subplans: Vec>>, estimated_icost: f64, estimated_num_out_tuples: f64, q_vertex_to_num_out_tuples: HashMap, } -impl QueryPlan { - pub fn new(scan_sampling: ScanSampling) -> Self { +impl QueryPlan { + pub fn new(scan_sampling: ScanSampling) -> Self { let mut last_operators = Vec::new(); - scan_sampling - .scan - .base_op - .get_last_operators(&mut last_operators); - let out_subgraph = last_operators.get(0).unwrap().out_subgraph.clone(); + let scan_sampling_op = Operator::ScanSampling(scan_sampling.clone()); + scan_sampling_op.get_last_operators(&mut last_operators); + let op = &last_operators[0]; + let out_subgraph = Box::new(get_op_attr_as_ref!(op, out_subgraph).as_ref().clone()); let mut sink = Sink::new(out_subgraph); for op in last_operators.iter_mut() { - op.set_next(sink.base_op.clone()) + let next = get_op_attr_as_mut!(op, next); + next.replace(vec![Operator::Sink(sink.clone())]); } sink.previous = Some(last_operators); Self { @@ -55,7 +58,7 @@ impl QueryPlan { q_vertex_to_num_out_tuples: HashMap::new(), } } - pub fn new_from_operator(last_operator: Rc) -> Self { + pub fn new_from_operator(last_operator: Rc>) -> Self { Self { sink: None, sink_type: SinkType::Counter, @@ -75,7 +78,15 @@ impl QueryPlan { q_vertex_to_num_out_tuples: HashMap::new(), } } - pub fn get_scan_sampling(&mut self) -> Option<&mut ScanSampling> { + pub fn get_scan_sampling(&mut self) -> Option<&mut ScanSampling> { self.scan_sampling.as_mut() } + + pub fn get_sink(&mut self) -> &mut Sink { + self.sink.as_mut().unwrap() + } + + pub fn get_sink_as_ref(&self) -> &Sink { + self.sink.as_ref().unwrap() + } } diff --git a/src/graph_impl/multi_graph/utils/io_utils.rs b/src/graph_impl/multi_graph/utils/io_utils.rs new file mode 100644 index 00000000..df851f2d --- /dev/null +++ b/src/graph_impl/multi_graph/utils/io_utils.rs @@ -0,0 +1,10 @@ +extern crate time; + +pub fn current_time() -> i64 { + let timespec = time::get_time(); + timespec.sec * 1000 + (timespec.nsec as f64 / 1000.0 / 1000.0) as i64 +} + +pub fn get_elapsed_time_in_millis(begin_time: i64) -> f32 { + (current_time() - begin_time / 1000000.0 as i64) as f32 +} diff --git a/src/graph_impl/multi_graph/utils/mod.rs b/src/graph_impl/multi_graph/utils/mod.rs new file mode 100644 index 00000000..53126b15 --- /dev/null +++ b/src/graph_impl/multi_graph/utils/mod.rs @@ -0,0 +1 @@ +pub mod io_utils; diff --git a/src/graph_impl/static_graph/graph.rs b/src/graph_impl/static_graph/graph.rs index 370aa245..268a9cdb 100644 --- a/src/graph_impl/static_graph/graph.rs +++ b/src/graph_impl/static_graph/graph.rs @@ -44,6 +44,7 @@ use std::cmp; use std::ops::Add; use test::bench::iter; use test::Options; +use graph_impl::multi_graph::catalog::adj_list_descriptor::Direction; pub type TypedUnStaticGraph = TypedStaticGraph; pub type TypedDiStaticGraph = TypedStaticGraph; @@ -71,6 +72,9 @@ pub struct TypedStaticGraph>>, bwd_adj_lists: Vec>>, + label_to_largest_fwd_adj_list_size: Vec, + label_to_largest_bwd_adj_list_size: Vec, + edge_vec: EdgeVec, in_edge_vec: Option>, // Maintain the node's labels, whose index is aligned with `offsets`. @@ -214,6 +218,8 @@ TypedStaticGraph node_type_offsets: vec![], fwd_adj_lists: vec![], bwd_adj_lists: vec![], + label_to_largest_fwd_adj_list_size: vec![], + label_to_largest_bwd_adj_list_size: vec![], edge_vec: edges, in_edge_vec: in_edges, labels: None, @@ -286,6 +292,8 @@ TypedStaticGraph node_type_offsets: vec![], fwd_adj_lists: vec![], bwd_adj_lists: vec![], + label_to_largest_fwd_adj_list_size: vec![], + label_to_largest_bwd_adj_list_size: vec![], edge_vec: edges, in_edge_vec: in_edges, labels: Some(labels), @@ -364,6 +372,8 @@ TypedStaticGraph node_type_offsets: vec![], fwd_adj_lists: vec![], bwd_adj_lists: vec![], + label_to_largest_fwd_adj_list_size: vec![], + label_to_largest_bwd_adj_list_size: vec![], edge_vec, in_edge_vec, labels, @@ -661,7 +671,7 @@ TypedStaticGraph .find_index(&label) .map_or(0, |id| id + 1); - return &fwd_list.get_neighbour_ids()[offset[label_id]..offset[label_id + 1]]; + return &fwd_list.get_neighbor_ids()[offset[label_id]..offset[label_id + 1]]; } } self.edge_vec.neighbors(id) @@ -675,7 +685,7 @@ TypedStaticGraph .edge_label_map .find_index(&label) .map_or(0, |id| id + 1); - return &fwd_list.get_neighbour_ids()[offset[label_id]..offset[label_id + 1]]; + return &fwd_list.get_neighbor_ids()[offset[label_id]..offset[label_id + 1]]; } } self.edge_vec.neighbors(id) @@ -700,6 +710,13 @@ TypedStaticGraph pub fn get_bwd_adj_list(&self) -> &Vec>> { self.bwd_adj_lists.as_ref() } + + pub fn get_largest_adj_list_size(&self, node_or_edge_label: usize, direction: Direction) -> usize { + if let Direction::Fwd = direction { + return self.label_to_largest_fwd_adj_list_size[node_or_edge_label]; + } + self.label_to_largest_bwd_adj_list_size[node_or_edge_label] + } } impl GraphTrait @@ -897,17 +914,20 @@ GraphLabelTrait for TypedStaticGraph .node_label_map .find_index(&label) .map_or(0, |id| id + 1); - return Iter::new(Box::new(self.fwd_adj_lists - .iter() - .skip_while(|&x| x.is_none()) - .enumerate() - .flat_map(move|(sid, list_op)| { - let list = list_op.as_ref().unwrap(); - let offset = list.get_offsets(); - let label = label_id.clone(); - let neighbors = &list.get_neighbour_ids()[offset[label]..offset[label + 1]]; - neighbors.iter().map(move|id| *id) - }).unique())); + return Iter::new(Box::new( + self.fwd_adj_lists + .iter() + .skip_while(|&x| x.is_none()) + .enumerate() + .flat_map(move |(_sid, list_op)| { + let list = list_op.as_ref().unwrap(); + let offset = list.get_offsets(); + let label = label_id.clone(); + let neighbors = &list.get_neighbor_ids()[offset[label]..offset[label + 1]]; + neighbors.iter().map(move |id| *id) + }) + .unique(), + )); } self.node_indices() } @@ -918,17 +938,19 @@ GraphLabelTrait for TypedStaticGraph .edge_label_map .find_index(&label) .map_or(0, |id| id + 1); - return Iter::new(Box::new(self.fwd_adj_lists - .iter() - .skip_while(|&x| x.is_none()) - .enumerate() - .flat_map(move|(sid, list_op)| { - let list = list_op.as_ref().unwrap(); - let offset = list.get_offsets(); - let label = label_id.clone(); - let neighbors = &list.get_neighbour_ids()[offset[label]..offset[label + 1]]; - neighbors.iter().map(move|id| (Id::new(sid), *id)) - }))); + return Iter::new(Box::new( + self.fwd_adj_lists + .iter() + .skip_while(|&x| x.is_none()) + .enumerate() + .flat_map(move |(sid, list_op)| { + let list = list_op.as_ref().unwrap(); + let offset = list.get_offsets(); + let label = label_id.clone(); + let neighbors = &list.get_neighbor_ids()[offset[label]..offset[label + 1]]; + neighbors.iter().map(move |id| (Id::new(sid), *id)) + }), + )); } self.edge_indices() } @@ -1049,6 +1071,8 @@ for TypedStaticGraph node_type_offsets: vec![], fwd_adj_lists: vec![], bwd_adj_lists: vec![], + label_to_largest_fwd_adj_list_size: vec![], + label_to_largest_bwd_adj_list_size: vec![], edge_vec: self.edge_vec + other.edge_vec, in_edge_vec: match (self.in_edge_vec, other.in_edge_vec) { (None, None) => None, diff --git a/src/graph_impl/static_graph/sorted_adj_vec.rs b/src/graph_impl/static_graph/sorted_adj_vec.rs index b6c39dda..c1c248a8 100644 --- a/src/graph_impl/static_graph/sorted_adj_vec.rs +++ b/src/graph_impl/static_graph/sorted_adj_vec.rs @@ -19,6 +19,7 @@ * under the License. */ use generic::IdType; +use graph_impl::multi_graph::plan::operator::extend::EI::Neighbours; #[derive(Debug, Hash, PartialEq, Eq, Clone, Serialize, Deserialize)] pub struct SortedAdjVec { @@ -43,11 +44,17 @@ impl SortedAdjVec { self.neighbour_ids[idx] = neighbor_id } + pub fn set_neighbor_ids(&self, label_or_type: usize, neighbours: &mut Neighbours) { + neighbours.ids = self.neighbour_ids.clone(); + neighbours.start_idx = self.label_or_type_offsets[label_or_type]; + neighbours.end_idx = self.label_or_type_offsets[label_or_type + 1]; + } + pub fn get_offsets(&self) -> &Vec { self.label_or_type_offsets.as_ref() } - pub fn get_neighbour_ids(&self) -> &Vec { + pub fn get_neighbor_ids(&self) -> &Vec { self.neighbour_ids.as_ref() } @@ -59,4 +66,41 @@ impl SortedAdjVec { block.sort(); } } + + pub fn intersect(&self, label_or_type: usize, some_neighbours: &mut Neighbours, neighbours: &mut Neighbours) -> usize { + self.inner_intersect( + some_neighbours, neighbours, &self.neighbour_ids, + self.label_or_type_offsets[label_or_type], + self.label_or_type_offsets[label_or_type + 1], + ); + self.label_or_type_offsets[label_or_type + 1] - self.label_or_type_offsets[label_or_type] + } + + fn inner_intersect(&self, some_neighbours: &mut Neighbours, neighbours: &mut Neighbours, + neighbour_ids: &Vec, mut this_idx: usize, this_idx_end: usize, + ) { + neighbours.reset(); + let some_neighbour_ids = some_neighbours.ids.clone(); + let mut some_idx = some_neighbours.start_idx; + let some_end_idx = some_neighbours.end_idx; + while this_idx < this_idx_end && some_idx < some_end_idx { + if neighbour_ids[this_idx] < some_neighbour_ids[some_idx] { + this_idx += 1; + while this_idx < this_idx_end && neighbour_ids[this_idx] < some_neighbour_ids[some_idx] { + this_idx += 1; + } + } else if neighbour_ids[this_idx] > some_neighbour_ids[some_idx] { + some_idx += 1; + ; + while some_idx < some_end_idx && neighbour_ids[this_idx] > some_neighbour_ids[some_idx] { + some_idx += 1; + } + } else { + neighbours.ids[neighbours.end_idx] = neighbour_ids[this_idx]; + neighbours.end_idx += 1; + this_idx += 1; + some_idx += 1; + } + } + } } diff --git a/tests/static_graph.rs b/tests/static_graph.rs index 7ca187d8..79ea093b 100644 --- a/tests/static_graph.rs +++ b/tests/static_graph.rs @@ -44,23 +44,23 @@ fn test_directed() { let fwd_adj_list = g.get_fwd_adj_list()[0].as_ref().unwrap(); assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 2, 2, 2]); - assert_eq!(fwd_adj_list.get_neighbour_ids(), &vec![1, 2]); + assert_eq!(fwd_adj_list.get_neighbor_ids(), &vec![1, 2]); let fwd_adj_list = g.get_fwd_adj_list()[1].as_ref().unwrap(); assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 1, 1, 1]); - assert_eq!(fwd_adj_list.get_neighbour_ids(), &vec![0]); + assert_eq!(fwd_adj_list.get_neighbor_ids(), &vec![0]); let fwd_adj_list = g.get_fwd_adj_list()[2].as_ref().unwrap(); assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 1, 1, 1]); - assert_eq!(fwd_adj_list.get_neighbour_ids(), &vec![0]); + assert_eq!(fwd_adj_list.get_neighbor_ids(), &vec![0]); let bwd_adj_list = g.get_bwd_adj_list()[0].as_ref().unwrap(); assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 2, 2, 2]); - assert_eq!(bwd_adj_list.get_neighbour_ids(), &vec![1, 2]); + assert_eq!(bwd_adj_list.get_neighbor_ids(), &vec![1, 2]); let bwd_adj_list = g.get_bwd_adj_list()[1].as_ref().unwrap(); assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 1, 1, 1]); - assert_eq!(bwd_adj_list.get_neighbour_ids(), &vec![0]); + assert_eq!(bwd_adj_list.get_neighbor_ids(), &vec![0]); let bwd_adj_list = g.get_bwd_adj_list()[2].as_ref().unwrap(); assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 1, 1, 1]); - assert_eq!(bwd_adj_list.get_neighbour_ids(), &vec![0]); + assert_eq!(bwd_adj_list.get_neighbor_ids(), &vec![0]); assert_eq!(g.node_count(), 3); assert_eq!(g.edge_count(), 4); @@ -130,23 +130,23 @@ fn test_undirected() { let fwd_adj_list = g.get_fwd_adj_list()[0].as_ref().unwrap(); assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 2, 2, 2]); - assert_eq!(fwd_adj_list.get_neighbour_ids(), &vec![1, 2]); + assert_eq!(fwd_adj_list.get_neighbor_ids(), &vec![1, 2]); let fwd_adj_list = g.get_fwd_adj_list()[1].as_ref().unwrap(); assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 2, 2, 2]); - assert_eq!(fwd_adj_list.get_neighbour_ids(), &vec![0, 2]); + assert_eq!(fwd_adj_list.get_neighbor_ids(), &vec![0, 2]); let fwd_adj_list = g.get_fwd_adj_list()[2].as_ref().unwrap(); assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 2, 2, 2]); - assert_eq!(fwd_adj_list.get_neighbour_ids(), &vec![0, 1]); + assert_eq!(fwd_adj_list.get_neighbor_ids(), &vec![0, 1]); let bwd_adj_list = g.get_bwd_adj_list()[0].as_ref().unwrap(); assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 2, 2, 2]); - assert_eq!(bwd_adj_list.get_neighbour_ids(), &vec![1, 2]); + assert_eq!(bwd_adj_list.get_neighbor_ids(), &vec![1, 2]); let bwd_adj_list = g.get_bwd_adj_list()[1].as_ref().unwrap(); assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 2, 2, 2]); - assert_eq!(bwd_adj_list.get_neighbour_ids(), &vec![0, 2]); + assert_eq!(bwd_adj_list.get_neighbor_ids(), &vec![0, 2]); let bwd_adj_list = g.get_bwd_adj_list()[2].as_ref().unwrap(); assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 2, 2, 2]); - assert_eq!(bwd_adj_list.get_neighbour_ids(), &vec![0, 1]); + assert_eq!(bwd_adj_list.get_neighbor_ids(), &vec![0, 1]); } #[test] @@ -214,23 +214,23 @@ fn test_labeled() { let fwd_adj_list = g.get_fwd_adj_list()[0].as_ref().unwrap(); assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 0, 1, 2, 2]); - assert_eq!(fwd_adj_list.get_neighbour_ids(), &vec![1, 2]); + assert_eq!(fwd_adj_list.get_neighbor_ids(), &vec![1, 2]); let fwd_adj_list = g.get_fwd_adj_list()[1].as_ref().unwrap(); assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 0, 1, 1, 1]); - assert_eq!(fwd_adj_list.get_neighbour_ids(), &vec![0]); + assert_eq!(fwd_adj_list.get_neighbor_ids(), &vec![0]); let fwd_adj_list = g.get_fwd_adj_list()[2].as_ref().unwrap(); assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 0, 0, 1, 1]); - assert_eq!(fwd_adj_list.get_neighbour_ids(), &vec![0]); + assert_eq!(fwd_adj_list.get_neighbor_ids(), &vec![0]); let bwd_adj_list = g.get_bwd_adj_list()[0].as_ref().unwrap(); assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 0, 1, 2, 2]); - assert_eq!(bwd_adj_list.get_neighbour_ids(), &vec![1, 2]); + assert_eq!(bwd_adj_list.get_neighbor_ids(), &vec![1, 2]); let bwd_adj_list = g.get_bwd_adj_list()[1].as_ref().unwrap(); assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 0, 1, 1, 1]); - assert_eq!(bwd_adj_list.get_neighbour_ids(), &vec![0]); + assert_eq!(bwd_adj_list.get_neighbor_ids(), &vec![0]); let bwd_adj_list = g.get_bwd_adj_list()[2].as_ref().unwrap(); assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 0, 0, 1, 1]); - assert_eq!(bwd_adj_list.get_neighbour_ids(), &vec![0]); + assert_eq!(bwd_adj_list.get_neighbor_ids(), &vec![0]); let neighbour_edge_no: Vec = g.neighbors_of_edge_iter(0, None).collect(); let neighbour_edge_0_a: Vec = g.neighbors_of_edge_iter(0, Some("a")).collect(); From cfefd16e0b628f2e8d5ef9a43b95c3976728ffd4 Mon Sep 17 00:00:00 2001 From: Michael Chen <920901835@qq.com> Date: Tue, 17 Dec 2019 23:17:53 +0800 Subject: [PATCH 09/25] fix:Scan Operator --- src/graph_impl/multi_graph/catalog/catalog.rs | 7 +- .../multi_graph/plan/operator/extend/EI.rs | 1 + .../plan/operator/extend/Extend.rs | 1 + .../plan/operator/extend/Intersect.rs | 1 + .../plan/operator/extend/extend.rs | 1 + .../plan/operator/extend/intersect.rs | 1 + .../multi_graph/plan/operator/mod.rs | 43 ++++- .../multi_graph/plan/operator/operator.rs | 38 ++-- .../multi_graph/plan/operator/scan/mod.rs | 1 + .../multi_graph/plan/operator/scan/scan.rs | 111 +++++++++--- .../plan/operator/scan/scan_blocking.rs | 169 ++++++++++++++++++ .../plan/operator/scan/scan_sampling.rs | 47 +++-- .../multi_graph/plan/operator/sink/sink.rs | 1 + src/graph_impl/multi_graph/plan/query_plan.rs | 3 +- 14 files changed, 367 insertions(+), 58 deletions(-) create mode 100644 src/graph_impl/multi_graph/plan/operator/scan/scan_blocking.rs diff --git a/src/graph_impl/multi_graph/catalog/catalog.rs b/src/graph_impl/multi_graph/catalog/catalog.rs index f6d31e01..2e4ba8a0 100644 --- a/src/graph_impl/multi_graph/catalog/catalog.rs +++ b/src/graph_impl/multi_graph/catalog/catalog.rs @@ -4,6 +4,7 @@ use graph_impl::multi_graph::catalog::catalog_plans::{ CatalogPlans, DEF_MAX_INPUT_NUM_VERTICES, DEF_NUM_EDGES_TO_SAMPLE, }; use graph_impl::multi_graph::catalog::query_graph::QueryGraph; +use graph_impl::multi_graph::plan::operator::scan::scan::Scan; use graph_impl::multi_graph::plan::operator::operator::{ BaseOperator, CommonOperatorTrait, Operator, }; @@ -296,11 +297,7 @@ impl Catalog { .iter_mut() .map(|query_plan| { let mut op = &mut query_plan.get_sink().previous.as_mut().unwrap()[0]; - while if let Operator::ScanSampling(sp) = op.deref() { - false - } else { - true - } { + while if let Operator::Scan(Scan::ScanSampling(sp)) = op.deref() { false } else { true } { let prev_op = get_op_attr_as_mut!(op, prev).as_mut().unwrap(); op = prev_op.as_mut(); } diff --git a/src/graph_impl/multi_graph/plan/operator/extend/EI.rs b/src/graph_impl/multi_graph/plan/operator/extend/EI.rs index 4b3a4692..d25c975b 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/EI.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/EI.rs @@ -4,6 +4,7 @@ use graph_impl::multi_graph::plan::operator::operator::{BaseOperator, CommonOper use graph_impl::multi_graph::plan::operator::extend::extend::Extend; use graph_impl::multi_graph::plan::operator::extend::intersect::{Intersect, IntersectType}; use graph_impl::static_graph::sorted_adj_vec::SortedAdjVec; +use::graph_impl::multi_graph::plan::operator::scan::scan::Scan; use itertools::Itertools; use hashbrown::HashMap; use generic::{IdType, GraphType}; diff --git a/src/graph_impl/multi_graph/plan/operator/extend/Extend.rs b/src/graph_impl/multi_graph/plan/operator/extend/Extend.rs index bf14873c..9ae987d2 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/Extend.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/Extend.rs @@ -2,6 +2,7 @@ use graph_impl::multi_graph::plan::operator::extend::EI::{BaseEI, Neighbours, EI use graph_impl::multi_graph::catalog::adj_list_descriptor::{Direction, AdjListDescriptor}; use graph_impl::static_graph::sorted_adj_vec::SortedAdjVec; use graph_impl::multi_graph::catalog::query_graph::QueryGraph; +use graph_impl::multi_graph::plan::operator::scan::scan::Scan; use hashbrown::HashMap; use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; use graph_impl::TypedStaticGraph; diff --git a/src/graph_impl/multi_graph/plan/operator/extend/Intersect.rs b/src/graph_impl/multi_graph/plan/operator/extend/Intersect.rs index 1db6d15f..87e924a7 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/Intersect.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/Intersect.rs @@ -1,5 +1,6 @@ use graph_impl::multi_graph::plan::operator::extend::EI::{BaseEI, Neighbours, CachingType, EI}; use graph_impl::multi_graph::plan::operator::extend::EI::EI::Base; +use graph_impl::multi_graph::plan::operator::scan::scan::Scan; use graph_impl::multi_graph::catalog::query_graph::QueryGraph; use hashbrown::HashMap; use graph_impl::multi_graph::catalog::adj_list_descriptor::AdjListDescriptor; diff --git a/src/graph_impl/multi_graph/plan/operator/extend/extend.rs b/src/graph_impl/multi_graph/plan/operator/extend/extend.rs index bf14873c..9ae987d2 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/extend.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/extend.rs @@ -2,6 +2,7 @@ use graph_impl::multi_graph::plan::operator::extend::EI::{BaseEI, Neighbours, EI use graph_impl::multi_graph::catalog::adj_list_descriptor::{Direction, AdjListDescriptor}; use graph_impl::static_graph::sorted_adj_vec::SortedAdjVec; use graph_impl::multi_graph::catalog::query_graph::QueryGraph; +use graph_impl::multi_graph::plan::operator::scan::scan::Scan; use hashbrown::HashMap; use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; use graph_impl::TypedStaticGraph; diff --git a/src/graph_impl/multi_graph/plan/operator/extend/intersect.rs b/src/graph_impl/multi_graph/plan/operator/extend/intersect.rs index 1db6d15f..87e924a7 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/intersect.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/intersect.rs @@ -1,5 +1,6 @@ use graph_impl::multi_graph::plan::operator::extend::EI::{BaseEI, Neighbours, CachingType, EI}; use graph_impl::multi_graph::plan::operator::extend::EI::EI::Base; +use graph_impl::multi_graph::plan::operator::scan::scan::Scan; use graph_impl::multi_graph::catalog::query_graph::QueryGraph; use hashbrown::HashMap; use graph_impl::multi_graph::catalog::adj_list_descriptor::AdjListDescriptor; diff --git a/src/graph_impl/multi_graph/plan/operator/mod.rs b/src/graph_impl/multi_graph/plan/operator/mod.rs index 9ce3b569..e83a5c66 100644 --- a/src/graph_impl/multi_graph/plan/operator/mod.rs +++ b/src/graph_impl/multi_graph/plan/operator/mod.rs @@ -21,6 +21,28 @@ macro_rules! get_ei_as_ref { }; } +#[macro_export] +macro_rules! get_scan_as_mut { + ($item:expr) => { + match $item { + Scan::Base(base)=>base, + Scan::ScanSampling(base)=> &mut base.base_scan, + Scan::ScanBlocking(base)=> &mut base.base_scan, + } + }; +} + +#[macro_export] +macro_rules! get_scan_as_ref { + ($item:expr) => { + match $item { + Scan::Base(base)=>base, + Scan::ScanSampling(base)=> &base.base_scan, + Scan::ScanBlocking(base)=> &base.base_scan, + } + }; +} + /// Get common attributes(Origin) from Operator #[macro_export] macro_rules! get_op_attr { @@ -28,8 +50,11 @@ macro_rules! get_op_attr { match $item { Operator::Base(base) => base.$name, Operator::Sink(sink) => sink.base_op.$name, - Operator::Scan(scan) => scan.base_op.$name, - Operator::ScanSampling(sp) => sp.scan.base_op.$name, + Operator::Scan(scan) => match scan{ + Scan::Base(base)=>base.base_op.$name, + Scan::ScanSampling(base)=>base.base_scan.base_op.$name, + Scan::ScanBlocking(base)=>base.base_scan.base_op.$name, + }, Operator::EI(ei) => match ei { EI::Base(base)=>base.base_op.$name, EI::Extend(base)=>base.base_ei.base_op.$name, @@ -46,8 +71,11 @@ macro_rules! get_op_attr_as_ref { match $item { Operator::Base(base) => &base.$name, Operator::Sink(sink) => &sink.base_op.$name, - Operator::Scan(scan) => &scan.base_op.$name, - Operator::ScanSampling(sp) => &sp.scan.base_op.$name, + Operator::Scan(scan) => match scan{ + Scan::Base(base)=>&base.base_op.$name, + Scan::ScanSampling(base)=>&base.base_scan.base_op.$name, + Scan::ScanBlocking(base)=>&base.base_scan.base_op.$name, + }, Operator::EI(ei) => match ei { EI::Base(base)=>&base.base_op.$name, EI::Extend(base)=>&base.base_ei.base_op.$name, @@ -64,8 +92,11 @@ macro_rules! get_op_attr_as_mut { match $item { Operator::Base(base) => &mut base.$name, Operator::Sink(sink) => &mut sink.base_op.$name, - Operator::Scan(scan) => &mut scan.base_op.$name, - Operator::ScanSampling(sp) => &mut sp.scan.base_op.$name, + Operator::Scan(scan) => match scan{ + Scan::Base(base)=>&mut base.base_op.$name, + Scan::ScanSampling(base)=>&mut base.base_scan.base_op.$name, + Scan::ScanBlocking(base)=>&mut base.base_scan.base_op.$name, + }, Operator::EI(ei) => match ei { EI::Base(base)=>&mut base.base_op.$name, EI::Extend(base)=>&mut base.base_ei.base_op.$name, diff --git a/src/graph_impl/multi_graph/plan/operator/operator.rs b/src/graph_impl/multi_graph/plan/operator/operator.rs index 5d721ebc..1e749fc0 100644 --- a/src/graph_impl/multi_graph/plan/operator/operator.rs +++ b/src/graph_impl/multi_graph/plan/operator/operator.rs @@ -1,7 +1,7 @@ use generic::{GraphType, IdType}; use graph_impl::multi_graph::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::plan::operator::extend::EI::EI; -use graph_impl::multi_graph::plan::operator::scan::scan::Scan; +use graph_impl::multi_graph::plan::operator::scan::scan::{BaseScan, Scan}; use graph_impl::multi_graph::plan::operator::scan::scan_sampling::ScanSampling; use graph_impl::multi_graph::plan::operator::sink::sink::Sink; use graph_impl::TypedStaticGraph; @@ -16,7 +16,6 @@ pub enum Operator { Base(BaseOperator), Sink(Sink), Scan(Scan), - ScanSampling(ScanSampling), EI(EI), } @@ -105,8 +104,11 @@ impl CommonOperatorTrait for Operator { match self { Operator::Base(base) => {} Operator::Sink(sink) => sink.init(probe_tuple, graph), - Operator::Scan(scan) => scan.init(probe_tuple, graph), - Operator::ScanSampling(sp) => sp.init(probe_tuple, graph), + Operator::Scan(scan) => match scan{ + Scan::Base(base)=>base.init(probe_tuple, graph), + Scan::ScanBlocking(sb)=>sb.init(probe_tuple, graph), + Scan::ScanSampling(ss)=>ss.init(probe_tuple, graph), + }, _ => {} } } @@ -115,8 +117,11 @@ impl CommonOperatorTrait for Operator { match self { Operator::Base(base) => {} Operator::Sink(sink) => sink.process_new_tuple(), - Operator::Scan(scan) => scan.process_new_tuple(), - Operator::ScanSampling(sp) => sp.process_new_tuple(), + Operator::Scan(scan) => match scan{ + Scan::Base(base)=>base.process_new_tuple(), + Scan::ScanBlocking(sb)=>sb.process_new_tuple(), + Scan::ScanSampling(ss)=>ss.process_new_tuple(), + }, _ => {} } } @@ -125,8 +130,11 @@ impl CommonOperatorTrait for Operator { match self { Operator::Base(base) => base.execute(), Operator::Sink(sink) => sink.execute(), - Operator::Scan(scan) => scan.execute(), - Operator::ScanSampling(sp) => sp.execute(), + Operator::Scan(scan) => match scan{ + Scan::Base(base)=>base.execute(), + Scan::ScanBlocking(sb)=>sb.execute(), + Scan::ScanSampling(ss)=>ss.execute(), + }, _ => {} } } @@ -134,8 +142,11 @@ impl CommonOperatorTrait for Operator { match self { Operator::Base(base) => None, Operator::Sink(sink) => sink.copy(is_thread_safe), - Operator::Scan(scan) => scan.copy(is_thread_safe), - Operator::ScanSampling(sp) => sp.copy(is_thread_safe), + Operator::Scan(scan) => match scan{ + Scan::Base(base)=>base.copy(is_thread_safe), + Scan::ScanBlocking(sb)=>sb.copy(is_thread_safe), + Scan::ScanSampling(ss)=>ss.copy(is_thread_safe), + }, _ => None } } @@ -143,8 +154,11 @@ impl CommonOperatorTrait for Operator { match self { Operator::Base(base) => false, Operator::Sink(sink) => sink.is_same_as(op), - Operator::Scan(scan) => scan.is_same_as(op), - Operator::ScanSampling(sp) => sp.is_same_as(op), + Operator::Scan(scan) => match scan{ + Scan::Base(base)=>base.is_same_as(op), + Scan::ScanBlocking(sb)=>sb.is_same_as(op), + Scan::ScanSampling(ss)=>ss.is_same_as(op), + }, _ => false } } diff --git a/src/graph_impl/multi_graph/plan/operator/scan/mod.rs b/src/graph_impl/multi_graph/plan/operator/scan/mod.rs index 5d33f337..d0c2d704 100644 --- a/src/graph_impl/multi_graph/plan/operator/scan/mod.rs +++ b/src/graph_impl/multi_graph/plan/operator/scan/mod.rs @@ -1,2 +1,3 @@ pub mod scan; pub mod scan_sampling; +pub mod scan_blocking; diff --git a/src/graph_impl/multi_graph/plan/operator/scan/scan.rs b/src/graph_impl/multi_graph/plan/operator/scan/scan.rs index 0050597e..47e43e62 100644 --- a/src/graph_impl/multi_graph/plan/operator/scan/scan.rs +++ b/src/graph_impl/multi_graph/plan/operator/scan/scan.rs @@ -1,4 +1,4 @@ -use generic::{GraphType, IdType}; +use generic::{GraphType, IdType, GraphTrait}; use graph_impl::multi_graph::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::plan::operator::operator::{ BaseOperator, CommonOperatorTrait, Operator, @@ -6,29 +6,37 @@ use graph_impl::multi_graph::plan::operator::operator::{ use graph_impl::multi_graph::plan::operator::sink::sink::Sink; use graph_impl::static_graph::sorted_adj_vec::SortedAdjVec; use graph_impl::TypedStaticGraph; -use std::hash::Hash; +use std::hash::{Hash, BuildHasherDefault}; use std::rc::Rc; +use graph_impl::multi_graph::plan::operator::scan::scan_sampling::ScanSampling; +use graph_impl::multi_graph::plan::operator::scan::scan_blocking::ScanBlocking; +use hashbrown::HashMap; -//TODO: Fixing Scan Operator #[derive(Clone)] -pub struct Scan { +pub enum Scan { + Base(BaseScan), + ScanSampling(ScanSampling), + ScanBlocking(ScanBlocking), +} + +#[derive(Clone)] +pub struct BaseScan { pub base_op: BaseOperator, from_query_vertex: String, to_query_vertex: String, - from_type: usize, - to_type: usize, - label_or_to_type: usize, - fwd_adj_list: Vec>, - vertex_ids: Vec, - vertex_types: Vec, + pub from_type: usize, + pub to_type: usize, + pub label_or_to_type: usize, + pub fwd_adj_list: Vec>>, + pub vertex_ids: Vec, + pub vertex_types: Vec, from_vertex_start_idx: usize, from_vertex_end_idx: usize, } -impl Scan { - //TODO: fixing functions - pub fn new(out_subgraph: Box) -> Scan { - Scan { +impl BaseScan { + pub fn new(out_subgraph: Box) -> BaseScan { + let mut scan = BaseScan { base_op: BaseOperator::new(out_subgraph, None), from_query_vertex: "".to_string(), to_query_vertex: "".to_string(), @@ -40,36 +48,95 @@ impl Scan { vertex_types: vec![], from_vertex_start_idx: 0, from_vertex_end_idx: 0, + }; + let out_subgraph = &scan.base_op.out_subgraph; + if out_subgraph.get_query_edges().len() > 1 { + panic!("IllegalArgumentException"); } + let query_edge = &out_subgraph.get_query_edges()[0]; + scan.from_type = query_edge.from_type; + scan.to_type = query_edge.to_type; + scan.label_or_to_type = query_edge.label; + scan.base_op.last_repeated_vertex_idx = 0; + scan.from_query_vertex = query_edge.from_query_vertex.clone(); + scan.to_query_vertex = query_edge.to_query_vertex.clone(); + scan.base_op.out_qvertex_to_idx_map.insert(scan.from_query_vertex.clone(), 0); + scan.base_op.out_qvertex_to_idx_map.insert(scan.to_query_vertex.clone(), 1); + scan.base_op.name = "SCAN (".to_owned() + &scan.from_query_vertex + ")->(" + &scan.to_query_vertex + ")"; + scan } } -impl CommonOperatorTrait for Scan { +impl CommonOperatorTrait for BaseScan { fn init( &mut self, probe_tuple: Vec, graph: &TypedStaticGraph, ) { - unimplemented!() + self.base_op.probe_tuple = probe_tuple.clone(); + self.vertex_ids = graph.get_node_ids().clone(); + self.vertex_types = graph.get_node_types().clone(); + if 0 != self.from_type { + self.from_vertex_start_idx = graph.get_node_type_offsets()[self.from_type]; + self.from_vertex_end_idx = graph.get_node_type_offsets()[self.from_type + 1]; + } else { + self.from_vertex_start_idx = 0; + self.from_vertex_end_idx = graph.node_count(); + } + self.fwd_adj_list = graph.get_fwd_adj_list().clone(); + if graph.is_sorted_by_node() { + self.label_or_to_type = self.to_type; + self.to_type = 0; + } + for next_op in self.base_op.next.as_mut().unwrap() { + next_op.init(probe_tuple.clone(), graph); + } } fn process_new_tuple(&mut self) { - unimplemented!() + panic!("Operator `scan` does not support execute().") } fn execute(&mut self) { - unimplemented!() + for from_idx in self.from_vertex_start_idx..self.from_vertex_end_idx { + let from_vertex = self.vertex_ids[from_idx]; + self.base_op.probe_tuple[0] = from_vertex; + let to_vertex_start_idx = self.fwd_adj_list[from_idx].as_ref().unwrap().get_offsets()[self.label_or_to_type]; + let to_vertex_end_idx = self.fwd_adj_list[from_idx].as_ref().unwrap().get_offsets()[self.label_or_to_type + 1]; + for to_idx in to_vertex_start_idx..to_vertex_end_idx { + self.base_op.probe_tuple[1] = self.fwd_adj_list[from_idx].as_ref().unwrap().get_neighbor_id(Id::new(to_idx)); + if self.to_type == 0 || self.vertex_types[self.base_op.probe_tuple[1].id()] == self.to_type { + self.base_op.num_out_tuples += 1; + self.base_op.next.as_mut().map(|next| (&mut next[0]).process_new_tuple()); + } + } + } + } + + fn update_operator_name(&mut self, mut query_vertex_to_index_map: HashMap) { + query_vertex_to_index_map = HashMap::new(); + query_vertex_to_index_map.insert(self.from_query_vertex.clone(), 0); + query_vertex_to_index_map.insert(self.to_query_vertex.clone(), 1); + self.base_op.next.as_mut().map(|next| { + for next_op in next { + next_op.update_operator_name(query_vertex_to_index_map.clone()); + } + }); } fn copy(&self, is_thread_safe: bool) -> Option> { if is_thread_safe { - //TODO: Fixing Scanblocking - // return ScanBlocking::new(outSubgraph); + return Some(Operator::Scan(Scan::ScanBlocking(ScanBlocking::new(self.base_op.out_subgraph.clone())))); } - Some(Operator::Scan(Scan::new(self.base_op.out_subgraph.clone()))) + Some(Operator::Scan(Scan::Base(BaseScan::new(self.base_op.out_subgraph.clone())))) } fn is_same_as(&mut self, op: &mut Operator) -> bool { - unimplemented!() + if let Operator::Scan(scan) = op { + return self.from_type == get_scan_as_ref!(scan).from_type && + self.to_type == get_scan_as_ref!(scan).to_type && + self.label_or_to_type == get_scan_as_ref!(scan).label_or_to_type; + } + false } } diff --git a/src/graph_impl/multi_graph/plan/operator/scan/scan_blocking.rs b/src/graph_impl/multi_graph/plan/operator/scan/scan_blocking.rs new file mode 100644 index 00000000..b0f31d3f --- /dev/null +++ b/src/graph_impl/multi_graph/plan/operator/scan/scan_blocking.rs @@ -0,0 +1,169 @@ +use generic::{IdType, GraphType, GraphTrait}; +use graph_impl::multi_graph::plan::operator::scan::scan::BaseScan; +use graph_impl::multi_graph::catalog::query_graph::QueryGraph; +use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; +use graph_impl::TypedStaticGraph; +use std::hash::Hash; + +static PARTITION_SIZE: usize = 100; + +#[derive(Clone)] +struct VertexIdxLimits { + from_variable_index_limit: usize, + to_variable_index_limit: usize, +} + +///TODO:ReentrantLock +#[derive(Clone)] +pub struct ScanBlocking { + pub base_scan: BaseScan, + curr_from_idx: usize, + curr_to_idx: usize, + from_idx_limit: usize, + to_idx_limit: usize, + highest_from_idx: usize, + highest_to_idx: usize, + global_vertices_idx_limits: VertexIdxLimits, +} + +impl ScanBlocking { + pub fn new(out_subgraph: Box) -> ScanBlocking { + ScanBlocking { + base_scan: BaseScan::new(out_subgraph), + curr_from_idx: 0, + curr_to_idx: 0, + from_idx_limit: 0, + to_idx_limit: 0, + highest_from_idx: 0, + highest_to_idx: 0, + global_vertices_idx_limits: VertexIdxLimits { from_variable_index_limit: 0, to_variable_index_limit: 0 }, + } + } + + fn update_indices_limits(&mut self) { + //ReentrantLock lock here. + self.curr_from_idx = self.global_vertices_idx_limits.from_variable_index_limit; + self.curr_to_idx = self.global_vertices_idx_limits.to_variable_index_limit; + self.from_idx_limit = self.curr_from_idx; + self.to_idx_limit = self.curr_to_idx; + let mut num_edges_left = PARTITION_SIZE; + while num_edges_left > 0 { + let flag = self.from_idx_limit == self.highest_from_idx - 1 && + self.to_idx_limit < self.highest_to_idx - 1 || + self.from_idx_limit < self.highest_from_idx - 1; + if !flag { break; } + let mut label = self.base_scan.label_or_to_type; + let to_limit = self.base_scan.fwd_adj_list[self.from_idx_limit].as_mut().map_or(0, |adj| { + adj.get_offsets()[label + 1] + }); + if self.to_idx_limit + num_edges_left <= to_limit - 1 { + self.to_idx_limit += (num_edges_left - 1); + num_edges_left = 0; + } else { + num_edges_left -= (to_limit - 1 - self.to_idx_limit); + self.to_idx_limit = to_limit; + if self.from_idx_limit == self.highest_from_idx - 1 { + break; + } + self.from_idx_limit += 1; + label = self.base_scan.label_or_to_type; + self.to_idx_limit = self.base_scan.fwd_adj_list[self.from_idx_limit].as_mut().map_or(0, |adj| { + adj.get_offsets()[label] + }); + } + } + self.global_vertices_idx_limits.from_variable_index_limit = self.from_idx_limit; + self.global_vertices_idx_limits.to_variable_index_limit = self.to_idx_limit; + } + + fn produce_new_edges(&mut self, from_idx: usize, start_to_idx: usize, end_to_idx: usize) { + let base_op = &mut self.base_scan.base_op; + for to_idx in start_to_idx..end_to_idx { + base_op.probe_tuple[0] = self.base_scan.vertex_ids[from_idx]; + base_op.probe_tuple[1] = self.base_scan.fwd_adj_list[from_idx].as_mut().unwrap().get_neighbor_id(Id::new(to_idx)); + base_op.num_out_tuples += 1; + base_op.next.as_mut().map(|next| next.get_mut(0).map(|next_op| next_op.process_new_tuple())); + } + } + + fn produce_new_edges_default(&mut self) { + for from_idx in self.curr_from_idx + 1..self.from_idx_limit { + let mut label = self.base_scan.label_or_to_type; + self.base_scan.base_op.probe_tuple[0] = self.base_scan.vertex_ids[from_idx]; + let to_vertex_idx_start = self.base_scan.fwd_adj_list[from_idx].as_mut().map_or(0, |adj| { + adj.get_offsets()[label] + }); + let to_vertex_idx_limit = self.base_scan.fwd_adj_list[from_idx].as_mut().map_or(0, |adj| { + adj.get_offsets()[label + 1] + }); + for to_idx in to_vertex_idx_start..to_vertex_idx_limit { + self.base_scan.base_op.probe_tuple[1] = self.base_scan.fwd_adj_list[from_idx].as_mut().unwrap().get_neighbor_id(Id::new(to_idx)); + if self.base_scan.to_type == 0 || self.base_scan.vertex_types[self.base_scan.base_op.probe_tuple[1].id()] == self.base_scan.to_type { + self.base_scan.base_op.num_out_tuples += 1; + self.base_scan.base_op.next.as_mut().map(|next| next.get_mut(0).map(|next_op| next_op.process_new_tuple())); + } + } + } + } +} + +impl CommonOperatorTrait for ScanBlocking { + fn init(&mut self, probe_tuple: Vec, graph: &TypedStaticGraph) { + self.base_scan.init(probe_tuple.clone(), graph); + if self.base_scan.from_type != 0 { + self.curr_from_idx = graph.get_node_type_offsets()[self.base_scan.from_type]; + self.highest_from_idx = graph.get_node_type_offsets()[self.base_scan.from_type + 1]; + } else { + self.curr_from_idx = 0; + self.highest_from_idx = graph.node_count() + 1; + } + let mut label = self.base_scan.label_or_to_type; + self.curr_to_idx = self.base_scan.fwd_adj_list[self.base_scan.vertex_ids[self.curr_from_idx].id()].as_mut().map_or(0, |adj| { + adj.get_offsets()[label] + }); + self.highest_to_idx = self.base_scan.fwd_adj_list[self.base_scan.vertex_ids[self.highest_from_idx - 1].id()].as_mut().map_or(0, |adj| { + adj.get_offsets()[label + 1] + }); + self.from_idx_limit = self.curr_from_idx; + self.to_idx_limit = self.curr_to_idx; + self.base_scan.base_op.next.as_mut().map(|next| + next.iter_mut().for_each(|next_op| next_op.init(probe_tuple.clone(), graph)) + ); + } + + fn process_new_tuple(&mut self) { + unimplemented!() + } + + fn execute(&mut self) { + self.update_indices_limits(); + while self.curr_from_idx == self.highest_from_idx - 1 && + self.curr_to_idx < self.highest_to_idx - 1 || + self.curr_from_idx < self.highest_from_idx - 1 + { + if self.curr_from_idx == self.from_idx_limit { + self.produce_new_edges(self.curr_from_idx, self.curr_to_idx, self.to_idx_limit); + } else if self.curr_from_idx < self.from_idx_limit { + let label = self.base_scan.label_or_to_type; + let to_vertex_idx_limit = self.base_scan.fwd_adj_list[self.base_scan.vertex_ids[self.curr_from_idx].id()].as_mut().map_or(0, |adj| { + adj.get_offsets()[label + 1] + }); + self.produce_new_edges(self.curr_from_idx, self.curr_to_idx, to_vertex_idx_limit); + self.produce_new_edges_default(/* startFromIdx: currFromIdx + 1, endFromIdx: fromIdxLimit */); + let start_idx = self.base_scan.fwd_adj_list[self.base_scan.vertex_ids[self.from_idx_limit].id()].as_mut().map_or(0, |adj| { + adj.get_offsets()[label] + }); + self.produce_new_edges(self.from_idx_limit,start_idx , self.to_idx_limit); + } + self.update_indices_limits(); + } + } + + fn copy(&self, is_thread_safe: bool) -> Option> { + unimplemented!() + } + + fn is_same_as(&mut self, op: &mut Operator) -> bool { + unimplemented!() + } +} \ No newline at end of file diff --git a/src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs b/src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs index c14e40c7..bfe490bb 100644 --- a/src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs +++ b/src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs @@ -3,28 +3,44 @@ use graph_impl::multi_graph::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::plan::operator::operator::{ BaseOperator, CommonOperatorTrait, Operator, }; -use graph_impl::multi_graph::plan::operator::scan::scan::Scan; +use graph_impl::multi_graph::plan::operator::scan::scan::{BaseScan, Scan}; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; use std::hash::{BuildHasherDefault, Hash}; use std::rc::Rc; +use rand::{thread_rng, Rng}; -//TODO: Fixing ScanSampling #[derive(Clone)] pub struct ScanSampling { - pub scan: Scan, + pub base_scan: BaseScan, pub edges_queue: Vec>, } impl ScanSampling { pub fn new(out_subgraph: Box) -> Self { Self { - scan: Scan::new(out_subgraph), + base_scan: BaseScan::new(out_subgraph), edges_queue: vec![], } } - pub fn get_last_operator() {} + pub fn set_edge_indices_to_sample(&mut self, edges: Vec, num_edges_to_sample: usize) { + let mut rng = thread_rng(); + let num_edges = edges.len() / 2; + while self.edges_queue.len() < num_edges_to_sample { + let edge_idx = rng.gen_range(0, num_edges); + self.edges_queue.push(vec![edges[edge_idx], edges[edge_idx + 1]]); + } + } + + pub fn set_edge_indices_to_sample_by_edges(&mut self, edges: Vec>, num_edges_to_sample: usize) { + let mut rng = thread_rng(); + self.edges_queue = vec![vec![]; num_edges_to_sample]; + while self.edges_queue.len() < num_edges_to_sample { + let edge_idx = rng.gen_range(0, edges.len()); + self.edges_queue.push(edges[edge_idx].clone()); + } + } } impl CommonOperatorTrait for ScanSampling { @@ -33,7 +49,14 @@ impl CommonOperatorTrait for ScanSampling { probe_tuple: Vec, graph: &TypedStaticGraph, ) { - unimplemented!() + if self.base_scan.base_op.probe_tuple.is_empty() { + self.base_scan.base_op.probe_tuple = probe_tuple.clone(); + self.base_scan.base_op.next.as_mut().map(|next| { + next.iter_mut().for_each(|next_op| { + next_op.init(probe_tuple.clone(), graph); + }) + }); + } } fn process_new_tuple(&mut self) { @@ -43,19 +66,19 @@ impl CommonOperatorTrait for ScanSampling { fn execute(&mut self) { while !self.edges_queue.is_empty() { let edge = self.edges_queue.pop().unwrap(); - self.scan.base_op.probe_tuple[0] = edge[0]; - self.scan.base_op.probe_tuple[1] = edge[0]; - self.scan.base_op.num_out_tuples += 1; - for next_op in self.scan.base_op.next.as_mut().unwrap() { + self.base_scan.base_op.probe_tuple[0] = edge[0]; + self.base_scan.base_op.probe_tuple[1] = edge[0]; + self.base_scan.base_op.num_out_tuples += 1; + for next_op in self.base_scan.base_op.next.as_mut().unwrap() { next_op.process_new_tuple(); } } } fn copy(&self, is_thread_safe: bool) -> Option> { - let mut scan_sampling = ScanSampling::new(self.scan.base_op.out_subgraph.clone()); + let mut scan_sampling = ScanSampling::new(self.base_scan.base_op.out_subgraph.clone()); scan_sampling.edges_queue = self.edges_queue.clone(); - Some(Operator::ScanSampling(scan_sampling)) + Some(Operator::Scan(Scan::ScanSampling(scan_sampling))) } fn is_same_as(&mut self, op: &mut Operator) -> bool { diff --git a/src/graph_impl/multi_graph/plan/operator/sink/sink.rs b/src/graph_impl/multi_graph/plan/operator/sink/sink.rs index 95bd8979..efe515d5 100644 --- a/src/graph_impl/multi_graph/plan/operator/sink/sink.rs +++ b/src/graph_impl/multi_graph/plan/operator/sink/sink.rs @@ -3,6 +3,7 @@ use graph_impl::multi_graph::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::plan::operator::operator::{ BaseOperator, CommonOperatorTrait, Operator, }; +use::graph_impl::multi_graph::plan::operator::scan::scan::Scan; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; use graph_impl::multi_graph::plan::operator::extend::EI::EI; diff --git a/src/graph_impl/multi_graph/plan/query_plan.rs b/src/graph_impl/multi_graph/plan/query_plan.rs index e05c7e75..da3b5735 100644 --- a/src/graph_impl/multi_graph/plan/query_plan.rs +++ b/src/graph_impl/multi_graph/plan/query_plan.rs @@ -1,6 +1,7 @@ use graph_impl::multi_graph::plan::operator::operator::{BaseOperator, Operator}; use graph_impl::multi_graph::plan::operator::scan::scan_sampling::ScanSampling; use graph_impl::multi_graph::plan::operator::sink::sink::{Sink, SinkType}; +use graph_impl::multi_graph::plan::operator::scan::scan::Scan; use graph_impl::multi_graph::plan::operator::extend::EI::EI; use hashbrown::HashMap; use std::ops::Deref; @@ -29,7 +30,7 @@ pub struct QueryPlan { impl QueryPlan { pub fn new(scan_sampling: ScanSampling) -> Self { let mut last_operators = Vec::new(); - let scan_sampling_op = Operator::ScanSampling(scan_sampling.clone()); + let scan_sampling_op = Operator::Scan(Scan::ScanSampling(scan_sampling.clone())); scan_sampling_op.get_last_operators(&mut last_operators); let op = &last_operators[0]; let out_subgraph = Box::new(get_op_attr_as_ref!(op, out_subgraph).as_ref().clone()); From 6b31059416d41e1fdef354f08d85e2cda0e7aa41 Mon Sep 17 00:00:00 2001 From: Michael Chen <920901835@qq.com> Date: Wed, 18 Dec 2019 12:39:13 +0800 Subject: [PATCH 10/25] fix:operator --- src/graph_impl/multi_graph/catalog/catalog.rs | 4 +- .../multi_graph/plan/operator/extend/EI.rs | 121 ++++++++--- .../plan/operator/extend/Extend.rs | 17 +- .../plan/operator/extend/Intersect.rs | 19 +- .../plan/operator/extend/extend.rs | 17 +- .../plan/operator/extend/intersect.rs | 19 +- .../multi_graph/plan/operator/mod.rs | 46 +++- .../multi_graph/plan/operator/operator.rs | 196 +++++++++++------- .../multi_graph/plan/operator/scan/scan.rs | 75 +++++++ .../plan/operator/scan/scan_blocking.rs | 21 +- .../plan/operator/scan/scan_sampling.rs | 16 +- .../multi_graph/plan/operator/sink/mod.rs | 3 + .../multi_graph/plan/operator/sink/sink.rs | 125 +++++++++-- .../plan/operator/sink/sink_copy.rs | 68 ++++++ .../plan/operator/sink/sink_limit.rs | 66 ++++++ .../plan/operator/sink/sink_print.rs | 58 ++++++ src/graph_impl/multi_graph/plan/query_plan.rs | 8 +- 17 files changed, 731 insertions(+), 148 deletions(-) create mode 100644 src/graph_impl/multi_graph/plan/operator/sink/sink_copy.rs create mode 100644 src/graph_impl/multi_graph/plan/operator/sink/sink_limit.rs create mode 100644 src/graph_impl/multi_graph/plan/operator/sink/sink_print.rs diff --git a/src/graph_impl/multi_graph/catalog/catalog.rs b/src/graph_impl/multi_graph/catalog/catalog.rs index 2e4ba8a0..8d10484b 100644 --- a/src/graph_impl/multi_graph/catalog/catalog.rs +++ b/src/graph_impl/multi_graph/catalog/catalog.rs @@ -5,6 +5,7 @@ use graph_impl::multi_graph::catalog::catalog_plans::{ }; use graph_impl::multi_graph::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::plan::operator::scan::scan::Scan; +use graph_impl::multi_graph::plan::operator::sink::sink::Sink; use graph_impl::multi_graph::plan::operator::operator::{ BaseOperator, CommonOperatorTrait, Operator, }; @@ -296,7 +297,8 @@ impl Catalog { let mut other: Vec<&mut Operator> = query_plan_arr .iter_mut() .map(|query_plan| { - let mut op = &mut query_plan.get_sink().previous.as_mut().unwrap()[0]; + let base_sink = get_sink_as_mut!(query_plan.get_sink()); + let mut op = &mut base_sink.previous.as_mut().unwrap()[0]; while if let Operator::Scan(Scan::ScanSampling(sp)) = op.deref() { false } else { true } { let prev_op = get_op_attr_as_mut!(op, prev).as_mut().unwrap(); op = prev_op.as_mut(); diff --git a/src/graph_impl/multi_graph/plan/operator/extend/EI.rs b/src/graph_impl/multi_graph/plan/operator/extend/EI.rs index d25c975b..ceef7729 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/EI.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/EI.rs @@ -4,7 +4,8 @@ use graph_impl::multi_graph::plan::operator::operator::{BaseOperator, CommonOper use graph_impl::multi_graph::plan::operator::extend::extend::Extend; use graph_impl::multi_graph::plan::operator::extend::intersect::{Intersect, IntersectType}; use graph_impl::static_graph::sorted_adj_vec::SortedAdjVec; -use::graph_impl::multi_graph::plan::operator::scan::scan::Scan; +use graph_impl::multi_graph::plan::operator::sink::sink::Sink; +use graph_impl::multi_graph::plan::operator::scan::scan::Scan; use itertools::Itertools; use hashbrown::HashMap; use generic::{IdType, GraphType}; @@ -222,9 +223,10 @@ impl BaseEI { idx_to_cache += 1; } } + pub fn execute_intersect(&mut self, idx: usize, intersect_type: IntersectType) -> usize { let (adj_vec, label_or_type) = match intersect_type { - IntersectType::CachedOut| IntersectType::TempOut => ( + IntersectType::CachedOut | IntersectType::TempOut => ( self.adj_lists_to_cache[idx][self.base_op.probe_tuple[self.vertex_idx[idx]].id()].as_ref(), self.labels_or_to_types[idx] ), @@ -248,39 +250,35 @@ impl BaseEI { } } -impl CommonOperatorTrait for EI { +impl CommonOperatorTrait for BaseEI { fn init(&mut self, probe_tuple: Vec, graph: &TypedStaticGraph) { - let base_ei = get_ei_as_mut!(self); - let base_op = &mut base_ei.base_op; - *&mut base_op.probe_tuple = probe_tuple.clone(); - *&mut base_ei.caching_type = CachingType::None; - *&mut base_ei.vertex_types = graph.get_node_types().clone(); - let prev = base_op.prev.as_mut().unwrap().as_mut(); + self.base_op.probe_tuple = probe_tuple.clone(); + self.caching_type = CachingType::None; + self.vertex_types = graph.get_node_types().clone(); + let prev = self.base_op.prev.as_mut().unwrap().as_mut(); let last_repeated_vertex_idx = get_op_attr_as_mut!(prev,last_repeated_vertex_idx).clone(); - base_ei.init_caching(last_repeated_vertex_idx); - base_ei.init_extensions(graph); - base_ei.set_alds_and_adj_lists(graph, last_repeated_vertex_idx); -// let base_ei = get_ei_as_mut!(self); - base_ei.base_op.next.as_mut().unwrap().iter_mut().foreach(|next_op| { + self.init_caching(last_repeated_vertex_idx); + self.init_extensions(graph); + self.set_alds_and_adj_lists(graph, last_repeated_vertex_idx); + self.base_op.next.as_mut().unwrap().iter_mut().foreach(|next_op| { next_op.init(probe_tuple.clone(), graph); }); } fn process_new_tuple(&mut self) { - unimplemented!() + panic!("unsupported operation exception") } fn execute(&mut self) { - unimplemented!() + self.base_op.execute() } fn get_alds_as_string(&self) -> String { - let base_ei = get_ei_as_ref!(self); - if !DIFFERENTIATE_FWD_BWD_SINGLE_ALD && 1 == base_ei.alds.len() { - return "E".to_owned() + &base_ei.alds[0].label.to_string(); + if !DIFFERENTIATE_FWD_BWD_SINGLE_ALD && 1 == self.alds.len() { + return "E".to_owned() + &self.alds[0].label.to_string(); } - let mut directions = vec!["".to_owned(); base_ei.alds.len()]; - for ald in &base_ei.alds { + let mut directions = vec!["".to_owned(); self.alds.len()]; + for ald in &self.alds { let dir = if let Direction::Fwd = ald.direction { "F".to_owned() } else { "B".to_owned() }; directions.push(dir + &ald.label.to_string()); } @@ -293,20 +291,23 @@ impl CommonOperatorTrait for EI { for (query_vertex, &index) in &query_vertex_to_index_map { prev_to_query_vertices[index] = query_vertex.clone(); } - let base_ei = get_ei_as_mut!(self); - *&mut base_ei.base_op.name = serde_json::to_string(&prev_to_query_vertices).unwrap() + " - " + &base_ei.base_op.name; - query_vertex_to_index_map.insert(base_ei.to_query_vertex.clone(), query_vertex_to_index_map.len()); - if let Some(next) = &mut base_ei.base_op.next { + self.base_op.name = serde_json::to_string(&prev_to_query_vertices).unwrap() + " - " + &self.base_op.name; + query_vertex_to_index_map.insert(self.to_query_vertex.clone(), query_vertex_to_index_map.len()); + if let Some(next) = &mut self.base_op.next { next.iter_mut().foreach(|op| op.update_operator_name(query_vertex_to_index_map.clone())) } } fn copy(&self, is_thread_safe: bool) -> Option> { - unimplemented!() + panic!("unsupported operation exception") } fn is_same_as(&mut self, op: &mut Operator) -> bool { - unimplemented!() + panic!("unsupported operation exception") + } + + fn get_num_out_tuples(&self) -> usize { + self.base_op.num_out_tuples } } @@ -323,6 +324,72 @@ impl EI { } } +impl CommonOperatorTrait for EI { + fn init(&mut self, probe_tuple: Vec, graph: &TypedStaticGraph) { + match self { + EI::Base(base) => base.init(probe_tuple, graph), + EI::Intersect(intersect) => intersect.init(probe_tuple, graph), + EI::Extend(extend) => extend.init(probe_tuple, graph), + } + } + + fn process_new_tuple(&mut self) { + match self { + EI::Base(base) => base.process_new_tuple(), + EI::Intersect(intersect) => intersect.process_new_tuple(), + EI::Extend(extend) => extend.process_new_tuple(), + } + } + + fn execute(&mut self) { + match self { + EI::Base(base) => base.execute(), + EI::Intersect(intersect) => intersect.execute(), + EI::Extend(extend) => extend.execute(), + } + } + + fn get_alds_as_string(&self) -> String { + match self { + EI::Base(base) => base.get_alds_as_string(), + EI::Intersect(intersect) => intersect.get_alds_as_string(), + EI::Extend(extend) => extend.get_alds_as_string(), + } + } + + fn update_operator_name(&mut self, mut query_vertex_to_index_map: HashMap) { + match self { + EI::Base(base) => base.update_operator_name(query_vertex_to_index_map), + EI::Intersect(intersect) => intersect.update_operator_name(query_vertex_to_index_map), + EI::Extend(extend) => extend.update_operator_name(query_vertex_to_index_map), + } + } + + fn copy(&self, is_thread_safe: bool) -> Option> { + match self { + EI::Base(base) => base.copy(is_thread_safe), + EI::Intersect(intersect) => intersect.copy(is_thread_safe), + EI::Extend(extend) => extend.copy(is_thread_safe), + } + } + + fn is_same_as(&mut self, op: &mut Operator) -> bool { + match self { + EI::Base(base) => base.is_same_as(op), + EI::Intersect(intersect) => intersect.is_same_as(op), + EI::Extend(extend) => extend.is_same_as(op), + } + } + + fn get_num_out_tuples(&self) -> usize { + match self { + EI::Base(base) => base.get_num_out_tuples(), + EI::Intersect(intersect) => intersect.get_num_out_tuples(), + EI::Extend(extend) => extend.get_num_out_tuples(), + } + } +} + #[derive(Clone)] pub struct Neighbours { pub ids: Vec, diff --git a/src/graph_impl/multi_graph/plan/operator/extend/Extend.rs b/src/graph_impl/multi_graph/plan/operator/extend/Extend.rs index 9ae987d2..82fdcffa 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/Extend.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/Extend.rs @@ -3,11 +3,12 @@ use graph_impl::multi_graph::catalog::adj_list_descriptor::{Direction, AdjListDe use graph_impl::static_graph::sorted_adj_vec::SortedAdjVec; use graph_impl::multi_graph::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::plan::operator::scan::scan::Scan; +use graph_impl::multi_graph::plan::operator::sink::sink::Sink; use hashbrown::HashMap; use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; use graph_impl::TypedStaticGraph; use generic::{IdType, GraphType}; -use std::hash::Hash; +use std::hash::{Hash, BuildHasherDefault}; #[derive(Clone)] pub struct Extend { @@ -67,7 +68,15 @@ impl CommonOperatorTrait for Extend { } fn execute(&mut self) { - unimplemented!() + self.base_ei.execute() + } + + fn get_alds_as_string(&self) -> String { + self.base_ei.get_alds_as_string() + } + + fn update_operator_name(&mut self, query_vertex_to_index_map: HashMap) { + self.base_ei.update_operator_name(query_vertex_to_index_map) } fn copy(&self, is_thread_safe: bool) -> Option> { @@ -99,4 +108,8 @@ impl CommonOperatorTrait for Extend { } false } + + fn get_num_out_tuples(&self) -> usize { + self.base_ei.get_num_out_tuples() + } } \ No newline at end of file diff --git a/src/graph_impl/multi_graph/plan/operator/extend/Intersect.rs b/src/graph_impl/multi_graph/plan/operator/extend/Intersect.rs index 87e924a7..d3f57e80 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/Intersect.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/Intersect.rs @@ -1,13 +1,14 @@ use graph_impl::multi_graph::plan::operator::extend::EI::{BaseEI, Neighbours, CachingType, EI}; use graph_impl::multi_graph::plan::operator::extend::EI::EI::Base; use graph_impl::multi_graph::plan::operator::scan::scan::Scan; +use graph_impl::multi_graph::plan::operator::sink::sink::Sink; use graph_impl::multi_graph::catalog::query_graph::QueryGraph; use hashbrown::HashMap; use graph_impl::multi_graph::catalog::adj_list_descriptor::AdjListDescriptor; use generic::{IdType, GraphType}; use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; use graph_impl::TypedStaticGraph; -use std::hash::Hash; +use std::hash::{Hash, BuildHasherDefault}; pub enum IntersectType { InitCached, @@ -38,7 +39,7 @@ impl Intersect { impl CommonOperatorTrait for Intersect { fn init(&mut self, probe_tuple: Vec, graph: &TypedStaticGraph) { - unimplemented!() + self.base_ei.init(probe_tuple, graph) } fn process_new_tuple(&mut self) { @@ -101,7 +102,15 @@ impl CommonOperatorTrait for Intersect { } fn execute(&mut self) { - unimplemented!() + self.base_ei.execute() + } + + fn get_alds_as_string(&self) -> String { + self.base_ei.get_alds_as_string() + } + + fn update_operator_name(&mut self, query_vertex_to_index_map: HashMap) { + self.base_ei.update_operator_name(query_vertex_to_index_map) } fn copy(&self, is_thread_safe: bool) -> Option> { @@ -133,4 +142,8 @@ impl CommonOperatorTrait for Intersect { } false } + + fn get_num_out_tuples(&self) -> usize { + self.base_ei.get_num_out_tuples() + } } \ No newline at end of file diff --git a/src/graph_impl/multi_graph/plan/operator/extend/extend.rs b/src/graph_impl/multi_graph/plan/operator/extend/extend.rs index 9ae987d2..82fdcffa 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/extend.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/extend.rs @@ -3,11 +3,12 @@ use graph_impl::multi_graph::catalog::adj_list_descriptor::{Direction, AdjListDe use graph_impl::static_graph::sorted_adj_vec::SortedAdjVec; use graph_impl::multi_graph::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::plan::operator::scan::scan::Scan; +use graph_impl::multi_graph::plan::operator::sink::sink::Sink; use hashbrown::HashMap; use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; use graph_impl::TypedStaticGraph; use generic::{IdType, GraphType}; -use std::hash::Hash; +use std::hash::{Hash, BuildHasherDefault}; #[derive(Clone)] pub struct Extend { @@ -67,7 +68,15 @@ impl CommonOperatorTrait for Extend { } fn execute(&mut self) { - unimplemented!() + self.base_ei.execute() + } + + fn get_alds_as_string(&self) -> String { + self.base_ei.get_alds_as_string() + } + + fn update_operator_name(&mut self, query_vertex_to_index_map: HashMap) { + self.base_ei.update_operator_name(query_vertex_to_index_map) } fn copy(&self, is_thread_safe: bool) -> Option> { @@ -99,4 +108,8 @@ impl CommonOperatorTrait for Extend { } false } + + fn get_num_out_tuples(&self) -> usize { + self.base_ei.get_num_out_tuples() + } } \ No newline at end of file diff --git a/src/graph_impl/multi_graph/plan/operator/extend/intersect.rs b/src/graph_impl/multi_graph/plan/operator/extend/intersect.rs index 87e924a7..d3f57e80 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/intersect.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/intersect.rs @@ -1,13 +1,14 @@ use graph_impl::multi_graph::plan::operator::extend::EI::{BaseEI, Neighbours, CachingType, EI}; use graph_impl::multi_graph::plan::operator::extend::EI::EI::Base; use graph_impl::multi_graph::plan::operator::scan::scan::Scan; +use graph_impl::multi_graph::plan::operator::sink::sink::Sink; use graph_impl::multi_graph::catalog::query_graph::QueryGraph; use hashbrown::HashMap; use graph_impl::multi_graph::catalog::adj_list_descriptor::AdjListDescriptor; use generic::{IdType, GraphType}; use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; use graph_impl::TypedStaticGraph; -use std::hash::Hash; +use std::hash::{Hash, BuildHasherDefault}; pub enum IntersectType { InitCached, @@ -38,7 +39,7 @@ impl Intersect { impl CommonOperatorTrait for Intersect { fn init(&mut self, probe_tuple: Vec, graph: &TypedStaticGraph) { - unimplemented!() + self.base_ei.init(probe_tuple, graph) } fn process_new_tuple(&mut self) { @@ -101,7 +102,15 @@ impl CommonOperatorTrait for Intersect { } fn execute(&mut self) { - unimplemented!() + self.base_ei.execute() + } + + fn get_alds_as_string(&self) -> String { + self.base_ei.get_alds_as_string() + } + + fn update_operator_name(&mut self, query_vertex_to_index_map: HashMap) { + self.base_ei.update_operator_name(query_vertex_to_index_map) } fn copy(&self, is_thread_safe: bool) -> Option> { @@ -133,4 +142,8 @@ impl CommonOperatorTrait for Intersect { } false } + + fn get_num_out_tuples(&self) -> usize { + self.base_ei.get_num_out_tuples() + } } \ No newline at end of file diff --git a/src/graph_impl/multi_graph/plan/operator/mod.rs b/src/graph_impl/multi_graph/plan/operator/mod.rs index e83a5c66..7d876e50 100644 --- a/src/graph_impl/multi_graph/plan/operator/mod.rs +++ b/src/graph_impl/multi_graph/plan/operator/mod.rs @@ -1,4 +1,27 @@ -/// Get common attributes(mutable) from EI Operator +#[macro_export] +macro_rules! get_sink_as_mut { + ($item:expr) => { + match $item { + Sink::BaseSink(base)=>base, + Sink::SinkCopy(base)=> &mut base.base_sink, + Sink::SinkPrint(base)=> &mut base.base_sink, + Sink::SinkLimit(base)=> &mut base.base_sink, + } + }; +} + +#[macro_export] +macro_rules! get_sink_as_ref { + ($item:expr) => { + match $item { + Sink::BaseSink(base)=>base, + Sink::SinkCopy(base)=> &base.base_sink, + Sink::SinkPrint(base)=> &base.base_sink, + Sink::SinkLimit(base)=> &base.base_sink, + } + }; +} + #[macro_export] macro_rules! get_ei_as_mut { ($item:expr) => { @@ -49,7 +72,12 @@ macro_rules! get_op_attr { ($item:expr,$name:ident) => { match $item { Operator::Base(base) => base.$name, - Operator::Sink(sink) => sink.base_op.$name, + Operator::Sink(sink) => match sink{ + Sink::BaseSink(base)=>base.base_op.$name, + Sink::SinkCopy(sc)=>sc.base_sink.base_op.$name, + Sink::SinkPrint(sp)=>sp.base_sink.base_op.$name, + Sink::SinkLimit(sl)=>sl.base_sink.base_op.$name, + }, Operator::Scan(scan) => match scan{ Scan::Base(base)=>base.base_op.$name, Scan::ScanSampling(base)=>base.base_scan.base_op.$name, @@ -70,7 +98,12 @@ macro_rules! get_op_attr_as_ref { ($item:expr,$name:ident) => { match $item { Operator::Base(base) => &base.$name, - Operator::Sink(sink) => &sink.base_op.$name, + Operator::Sink(sink) => match sink{ + Sink::BaseSink(base)=>&base.base_op.$name, + Sink::SinkCopy(sc)=>&sc.base_sink.base_op.$name, + Sink::SinkPrint(sp)=>&sp.base_sink.base_op.$name, + Sink::SinkLimit(sl)=>&sl.base_sink.base_op.$name, + }, Operator::Scan(scan) => match scan{ Scan::Base(base)=>&base.base_op.$name, Scan::ScanSampling(base)=>&base.base_scan.base_op.$name, @@ -91,7 +124,12 @@ macro_rules! get_op_attr_as_mut { ($item:expr,$name:ident) => { match $item { Operator::Base(base) => &mut base.$name, - Operator::Sink(sink) => &mut sink.base_op.$name, + Operator::Sink(sink) => match sink{ + Sink::BaseSink(base)=>&mut base.base_op.$name, + Sink::SinkCopy(sc)=>&mut sc.base_sink.base_op.$name, + Sink::SinkPrint(sp)=>&mut sp.base_sink.base_op.$name, + Sink::SinkLimit(sl)=>&mut sl.base_sink.base_op.$name, + }, Operator::Scan(scan) => match scan{ Scan::Base(base)=>&mut base.base_op.$name, Scan::ScanSampling(base)=>&mut base.base_scan.base_op.$name, diff --git a/src/graph_impl/multi_graph/plan/operator/operator.rs b/src/graph_impl/multi_graph/plan/operator/operator.rs index 1e749fc0..29530242 100644 --- a/src/graph_impl/multi_graph/plan/operator/operator.rs +++ b/src/graph_impl/multi_graph/plan/operator/operator.rs @@ -67,12 +67,6 @@ impl BaseOperator { fn set_next_vec(&mut self, op: Vec>) { self.next.replace(op); } - - fn execute(&mut self) { - if let Some(prev) = self.prev.as_mut() { - prev.execute(); - } - } } /// Common operations for every kind of operator @@ -84,14 +78,88 @@ pub trait CommonOperatorTrait { ); fn process_new_tuple(&mut self); fn execute(&mut self); + fn get_alds_as_string(&self) -> String; + fn update_operator_name(&mut self, query_vertex_to_index_map: HashMap); + fn copy(&self, is_thread_safe: bool) -> Option>; + fn is_same_as(&mut self, op: &mut Operator) -> bool; + fn get_num_out_tuples(&self) -> usize; +} + +impl Operator { + pub fn get_last_operators(&self, last_operators: &mut Vec>) { + if let Some(next) = get_op_attr_as_ref!(self, next) { + for op in next { + op.get_last_operators(last_operators); + } + } else { + last_operators.push(self.clone()); + } + } + + pub fn get_operator_metrics_next_operators( + &self, + operator_metrics: &mut Vec<(String, usize, usize)>, + ) { + let name: &String = get_op_attr_as_ref!(self, name); + let icost = get_op_attr!(self, icost); + let num_out_tuples = get_op_attr!(self, num_out_tuples); + operator_metrics.push((name.clone(), icost, num_out_tuples)); + if let Some(next) = get_op_attr_as_ref!(self, next) { + next.iter().for_each(|op| match op { + Operator::Sink(_) => {} + _ => op.get_operator_metrics_next_operators(operator_metrics), + }); + } + } + + pub fn has_multi_edge_extends(&self) -> bool { + match self { + Operator::EI(ei) => ei.has_multi_edge_extends(), + _ => { + if let Some(prev) = get_op_attr_as_ref!(self, prev) { + return prev.has_multi_edge_extends(); + } + false + } + } + } +} + + +impl CommonOperatorTrait for BaseOperator { + fn init(&mut self, probe_tuple: Vec, graph: &TypedStaticGraph) { + panic!("unsupported operation exception") + } + + fn process_new_tuple(&mut self) { + panic!("unsupported operation exception") + } + + fn execute(&mut self) { + if let Some(prev) = self.prev.as_mut() { + prev.execute(); + } + } + fn get_alds_as_string(&self) -> String { - "".to_string() + String::from("") } + fn update_operator_name(&mut self, query_vertex_to_index_map: HashMap) { - panic!("`update_operator_name()` on `BaseOperator`") + panic!("`update_operator_name()` on neither `EI` or `Scan`") + } + + fn copy(&self, is_thread_safe: bool) -> Option> { + panic!("unsupported operation exception") + } + + fn is_same_as(&mut self, op: &mut Operator) -> bool { + panic!("unsupported operation exception") + } + + fn get_num_out_tuples(&self) -> usize { + self.num_out_tuples } - fn copy(&self, is_thread_safe: bool) -> Option>; - fn is_same_as(&mut self, op: &mut Operator) -> bool; } /// Abstract methods @@ -102,27 +170,19 @@ impl CommonOperatorTrait for Operator { graph: &TypedStaticGraph, ) { match self { - Operator::Base(base) => {} + Operator::Base(base) => base.init(probe_tuple, graph), Operator::Sink(sink) => sink.init(probe_tuple, graph), - Operator::Scan(scan) => match scan{ - Scan::Base(base)=>base.init(probe_tuple, graph), - Scan::ScanBlocking(sb)=>sb.init(probe_tuple, graph), - Scan::ScanSampling(ss)=>ss.init(probe_tuple, graph), - }, - _ => {} + Operator::Scan(scan) => scan.init(probe_tuple, graph), + Operator::EI(ei) => ei.init(probe_tuple, graph), } } fn process_new_tuple(&mut self) { match self { - Operator::Base(base) => {} + Operator::Base(base) => base.process_new_tuple(), Operator::Sink(sink) => sink.process_new_tuple(), - Operator::Scan(scan) => match scan{ - Scan::Base(base)=>base.process_new_tuple(), - Scan::ScanBlocking(sb)=>sb.process_new_tuple(), - Scan::ScanSampling(ss)=>ss.process_new_tuple(), - }, - _ => {} + Operator::Scan(scan) => scan.process_new_tuple(), + Operator::EI(ei) => ei.process_new_tuple(), } } @@ -130,76 +190,52 @@ impl CommonOperatorTrait for Operator { match self { Operator::Base(base) => base.execute(), Operator::Sink(sink) => sink.execute(), - Operator::Scan(scan) => match scan{ - Scan::Base(base)=>base.execute(), - Scan::ScanBlocking(sb)=>sb.execute(), - Scan::ScanSampling(ss)=>ss.execute(), - }, - _ => {} + Operator::Scan(scan) => scan.execute(), + Operator::EI(ei) => ei.execute(), } } - fn copy(&self, is_thread_safe: bool) -> Option> { + + fn get_alds_as_string(&self) -> String { match self { - Operator::Base(base) => None, - Operator::Sink(sink) => sink.copy(is_thread_safe), - Operator::Scan(scan) => match scan{ - Scan::Base(base)=>base.copy(is_thread_safe), - Scan::ScanBlocking(sb)=>sb.copy(is_thread_safe), - Scan::ScanSampling(ss)=>ss.copy(is_thread_safe), - }, - _ => None + Operator::Base(base) => base.get_alds_as_string(), + Operator::Sink(sink) => sink.get_alds_as_string(), + Operator::Scan(scan) => scan.get_alds_as_string(), + Operator::EI(ei) => ei.get_alds_as_string(), } } - fn is_same_as(&mut self, op: &mut Operator) -> bool { + + fn update_operator_name(&mut self, query_vertex_to_index_map: HashMap) { match self { - Operator::Base(base) => false, - Operator::Sink(sink) => sink.is_same_as(op), - Operator::Scan(scan) => match scan{ - Scan::Base(base)=>base.is_same_as(op), - Scan::ScanBlocking(sb)=>sb.is_same_as(op), - Scan::ScanSampling(ss)=>ss.is_same_as(op), - }, - _ => false + Operator::Base(base) => base.update_operator_name(query_vertex_to_index_map), + Operator::Sink(sink) => sink.update_operator_name(query_vertex_to_index_map), + Operator::Scan(scan) => scan.update_operator_name(query_vertex_to_index_map), + Operator::EI(ei) => ei.update_operator_name(query_vertex_to_index_map), } } -} -impl Operator { - pub fn get_last_operators(&self, last_operators: &mut Vec>) { - if let Some(next) = get_op_attr_as_ref!(self, next) { - for op in next { - op.get_last_operators(last_operators); - } - } else { - last_operators.push(self.clone()); + fn copy(&self, is_thread_safe: bool) -> Option> { + match self { + Operator::Base(base) => base.copy(is_thread_safe), + Operator::Sink(sink) => sink.copy(is_thread_safe), + Operator::Scan(scan) => scan.copy(is_thread_safe), + Operator::EI(ei) => ei.copy(is_thread_safe), } } - - pub fn get_operator_metrics_next_operators( - &self, - operator_metrics: &mut Vec<(String, usize, usize)>, - ) { - let name: &String = get_op_attr_as_ref!(self, name); - let icost = get_op_attr!(self, icost); - let num_out_tuples = get_op_attr!(self, num_out_tuples); - operator_metrics.push((name.clone(), icost, num_out_tuples)); - if let Some(next) = get_op_attr_as_ref!(self, next) { - next.iter().for_each(|op| match op { - Operator::Sink(_) => {} - _ => op.get_operator_metrics_next_operators(operator_metrics), - }); + fn is_same_as(&mut self, op: &mut Operator) -> bool { + match self { + Operator::Base(base) => base.is_same_as(op), + Operator::Sink(sink) => sink.is_same_as(op), + Operator::Scan(scan) => scan.is_same_as(op), + Operator::EI(ei) => ei.is_same_as(op), } } - pub fn has_multi_edge_extends(&self) -> bool { + fn get_num_out_tuples(&self) -> usize { match self { - Operator::EI(ei) => ei.has_multi_edge_extends(), - _ => { - if let Some(prev) = get_op_attr_as_ref!(self, prev) { - return prev.has_multi_edge_extends(); - } - false - } + Operator::Base(base) => base.get_num_out_tuples(), + Operator::Sink(sink) => sink.get_num_out_tuples(), + Operator::Scan(scan) => scan.get_num_out_tuples(), + Operator::EI(ei) => ei.get_num_out_tuples(), } } -} +} \ No newline at end of file diff --git a/src/graph_impl/multi_graph/plan/operator/scan/scan.rs b/src/graph_impl/multi_graph/plan/operator/scan/scan.rs index 47e43e62..dc5086ca 100644 --- a/src/graph_impl/multi_graph/plan/operator/scan/scan.rs +++ b/src/graph_impl/multi_graph/plan/operator/scan/scan.rs @@ -113,6 +113,10 @@ impl CommonOperatorTrait for BaseScan { } } + fn get_alds_as_string(&self) -> String { + self.base_op.get_alds_as_string() + } + fn update_operator_name(&mut self, mut query_vertex_to_index_map: HashMap) { query_vertex_to_index_map = HashMap::new(); query_vertex_to_index_map.insert(self.from_query_vertex.clone(), 0); @@ -139,4 +143,75 @@ impl CommonOperatorTrait for BaseScan { } false } + + fn get_num_out_tuples(&self) -> usize { + self.base_op.get_num_out_tuples() + } +} + +impl CommonOperatorTrait for Scan { + fn init(&mut self, probe_tuple: Vec, graph: &TypedStaticGraph) { + match self { + Scan::Base(base) => base.init(probe_tuple, graph), + Scan::ScanSampling(ss) => ss.init(probe_tuple, graph), + Scan::ScanBlocking(sb) => sb.init(probe_tuple, graph), + } + } + + fn process_new_tuple(&mut self) { + match self { + Scan::Base(base) => base.process_new_tuple(), + Scan::ScanSampling(ss) => ss.process_new_tuple(), + Scan::ScanBlocking(sb) => sb.process_new_tuple(), + } + } + + fn execute(&mut self) { + match self { + Scan::Base(base) => base.execute(), + Scan::ScanSampling(ss) => ss.execute(), + Scan::ScanBlocking(sb) => sb.execute(), + } + } + + fn get_alds_as_string(&self) -> String { + match self { + Scan::Base(base) => base.get_alds_as_string(), + Scan::ScanSampling(ss) => ss.get_alds_as_string(), + Scan::ScanBlocking(sb) => sb.get_alds_as_string(), + } + } + + fn update_operator_name(&mut self, query_vertex_to_index_map: HashMap) { + match self { + Scan::Base(base) => base.update_operator_name(query_vertex_to_index_map), + Scan::ScanSampling(ss) => ss.update_operator_name(query_vertex_to_index_map), + Scan::ScanBlocking(sb) => sb.update_operator_name(query_vertex_to_index_map), + } + } + + fn copy(&self, is_thread_safe: bool) -> Option> { + match self { + Scan::Base(base) => base.copy(is_thread_safe), + Scan::ScanSampling(ss) => ss.copy(is_thread_safe), + Scan::ScanBlocking(sb) => sb.copy(is_thread_safe), + } + } + + fn is_same_as(&mut self, op: &mut Operator) -> bool { + match self { + Scan::Base(base) => base.is_same_as(op), + Scan::ScanSampling(ss) => ss.is_same_as(op), + Scan::ScanBlocking(sb) => sb.is_same_as(op), + } + } + + fn get_num_out_tuples(&self) -> usize { + match self { + Scan::Base(base) => base.get_num_out_tuples(), + Scan::ScanSampling(ss) => ss.get_num_out_tuples(), + Scan::ScanBlocking(sb) => sb.get_num_out_tuples(), + } + } } + diff --git a/src/graph_impl/multi_graph/plan/operator/scan/scan_blocking.rs b/src/graph_impl/multi_graph/plan/operator/scan/scan_blocking.rs index b0f31d3f..12de3e41 100644 --- a/src/graph_impl/multi_graph/plan/operator/scan/scan_blocking.rs +++ b/src/graph_impl/multi_graph/plan/operator/scan/scan_blocking.rs @@ -3,7 +3,8 @@ use graph_impl::multi_graph::plan::operator::scan::scan::BaseScan; use graph_impl::multi_graph::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; use graph_impl::TypedStaticGraph; -use std::hash::Hash; +use std::hash::{Hash, BuildHasherDefault}; +use hashbrown::HashMap; static PARTITION_SIZE: usize = 100; @@ -132,7 +133,7 @@ impl CommonOperatorTrait for ScanBlocking { } fn process_new_tuple(&mut self) { - unimplemented!() + self.base_scan.process_new_tuple() } fn execute(&mut self) { @@ -159,11 +160,23 @@ impl CommonOperatorTrait for ScanBlocking { } } + fn get_alds_as_string(&self) -> String { + self.base_scan.get_alds_as_string() + } + + fn update_operator_name(&mut self, query_vertex_to_index_map: HashMap) { + self.base_scan.update_operator_name(query_vertex_to_index_map) + } + fn copy(&self, is_thread_safe: bool) -> Option> { - unimplemented!() + self.base_scan.copy(is_thread_safe) } fn is_same_as(&mut self, op: &mut Operator) -> bool { - unimplemented!() + self.base_scan.is_same_as(op) + } + + fn get_num_out_tuples(&self) -> usize { + self.base_scan.get_num_out_tuples() } } \ No newline at end of file diff --git a/src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs b/src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs index bfe490bb..4dee571c 100644 --- a/src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs +++ b/src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs @@ -60,7 +60,7 @@ impl CommonOperatorTrait for ScanSampling { } fn process_new_tuple(&mut self) { - unimplemented!() + self.base_scan.process_new_tuple() } fn execute(&mut self) { @@ -75,6 +75,14 @@ impl CommonOperatorTrait for ScanSampling { } } + fn get_alds_as_string(&self) -> String { + self.base_scan.get_alds_as_string() +} + + fn update_operator_name(&mut self, query_vertex_to_index_map: HashMap) { + self.base_scan.update_operator_name(query_vertex_to_index_map) + } + fn copy(&self, is_thread_safe: bool) -> Option> { let mut scan_sampling = ScanSampling::new(self.base_scan.base_op.out_subgraph.clone()); scan_sampling.edges_queue = self.edges_queue.clone(); @@ -82,6 +90,10 @@ impl CommonOperatorTrait for ScanSampling { } fn is_same_as(&mut self, op: &mut Operator) -> bool { - unimplemented!() + self.base_scan.is_same_as(op) + } + + fn get_num_out_tuples(&self) -> usize { + self.base_scan.get_num_out_tuples() } } diff --git a/src/graph_impl/multi_graph/plan/operator/sink/mod.rs b/src/graph_impl/multi_graph/plan/operator/sink/mod.rs index 0ecbfb92..a0c364bf 100644 --- a/src/graph_impl/multi_graph/plan/operator/sink/mod.rs +++ b/src/graph_impl/multi_graph/plan/operator/sink/mod.rs @@ -1 +1,4 @@ pub mod sink; +pub mod sink_copy; +pub mod sink_limit; +pub mod sink_print; diff --git a/src/graph_impl/multi_graph/plan/operator/sink/sink.rs b/src/graph_impl/multi_graph/plan/operator/sink/sink.rs index efe515d5..92c5c50d 100644 --- a/src/graph_impl/multi_graph/plan/operator/sink/sink.rs +++ b/src/graph_impl/multi_graph/plan/operator/sink/sink.rs @@ -3,12 +3,15 @@ use graph_impl::multi_graph::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::plan::operator::operator::{ BaseOperator, CommonOperatorTrait, Operator, }; -use::graph_impl::multi_graph::plan::operator::scan::scan::Scan; +use ::graph_impl::multi_graph::plan::operator::scan::scan::Scan; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; use graph_impl::multi_graph::plan::operator::extend::EI::EI; use std::hash::{BuildHasherDefault, Hash}; use std::rc::Rc; +use graph_impl::multi_graph::plan::operator::sink::sink_copy::SinkCopy; +use graph_impl::multi_graph::plan::operator::sink::sink_print::SinkPrint; +use graph_impl::multi_graph::plan::operator::sink::sink_limit::SinkLimit; pub enum SinkType { Copy, @@ -18,31 +21,29 @@ pub enum SinkType { } #[derive(Clone)] -pub struct Sink { +pub enum Sink { + BaseSink(BaseSink), + SinkCopy(SinkCopy), + SinkPrint(SinkPrint), + SinkLimit(SinkLimit), +} + +#[derive(Clone)] +pub struct BaseSink { pub base_op: BaseOperator, pub previous: Option>>, } -impl Sink { +impl BaseSink { pub fn new(query_graph: Box) -> Self { Self { base_op: BaseOperator::new(query_graph.clone(), Some(query_graph)), previous: None, } } - - fn get_num_out_tuples(&self) -> usize { - if let Some(prev) = &self.previous { - prev.iter() - .map(|op| get_op_attr_as_ref!(op, num_out_tuples)) - .sum() - } else { - self.base_op.num_out_tuples - } - } } -impl CommonOperatorTrait for Sink { +impl CommonOperatorTrait for BaseSink { fn init( &mut self, probe_tuple: Vec, @@ -61,14 +62,22 @@ impl CommonOperatorTrait for Sink { } } + fn get_alds_as_string(&self) -> String { + self.base_op.get_alds_as_string() + } + + fn update_operator_name(&mut self, query_vertex_to_index_map: HashMap) { + self.base_op.update_operator_name(query_vertex_to_index_map) + } + fn copy(&self, is_thread_safe: bool) -> Option> { - let mut sink = Sink::new(self.base_op.out_subgraph.clone()); + let mut sink = BaseSink::new(self.base_op.out_subgraph.clone()); if let Some(prev) = &self.base_op.prev { sink.base_op.prev = Some(Box::new(prev.copy(is_thread_safe).unwrap())); } else { sink.base_op.prev = None; } - Some(Operator::Sink(sink)) + Some(Operator::Sink(Sink::BaseSink(sink))) } fn is_same_as(&mut self, op: &mut Operator) -> bool { @@ -80,4 +89,88 @@ impl CommonOperatorTrait for Sink { } false } + + fn get_num_out_tuples(&self) -> usize { + if let Some(prev) = &self.previous { + prev.iter() + .map(|op| get_op_attr_as_ref!(op, num_out_tuples)) + .sum() + } else { + self.base_op.num_out_tuples + } + } } + +impl CommonOperatorTrait for Sink { + fn init(&mut self, probe_tuple: Vec, graph: &TypedStaticGraph) { + match self { + Sink::BaseSink(base) => base.init(probe_tuple, graph), + Sink::SinkCopy(sc) => sc.init(probe_tuple, graph), + Sink::SinkPrint(sp) => sp.init(probe_tuple, graph), + Sink::SinkLimit(sl) => sl.init(probe_tuple, graph), + } + } + + fn process_new_tuple(&mut self) { + match self { + Sink::BaseSink(base) => base.process_new_tuple(), + Sink::SinkCopy(sc) => sc.process_new_tuple(), + Sink::SinkPrint(sp) => sp.process_new_tuple(), + Sink::SinkLimit(sl) => sl.process_new_tuple(), + } + } + + fn execute(&mut self) { + match self { + Sink::BaseSink(base) => base.execute(), + Sink::SinkCopy(sc) => sc.execute(), + Sink::SinkPrint(sp) => sp.execute(), + Sink::SinkLimit(sl) => sl.execute(), + } + } + + fn get_alds_as_string(&self) -> String { + match self { + Sink::BaseSink(base) => base.get_alds_as_string(), + Sink::SinkCopy(sc) => sc.get_alds_as_string(), + Sink::SinkPrint(sp) => sp.get_alds_as_string(), + Sink::SinkLimit(sl) => sl.get_alds_as_string(), + } + } + + fn update_operator_name(&mut self, query_vertex_to_index_map: HashMap) { + match self { + Sink::BaseSink(base) => base.update_operator_name(query_vertex_to_index_map), + Sink::SinkCopy(sc) => sc.update_operator_name(query_vertex_to_index_map), + Sink::SinkPrint(sp) => sp.update_operator_name(query_vertex_to_index_map), + Sink::SinkLimit(sl) => sl.update_operator_name(query_vertex_to_index_map), + } + } + + fn copy(&self, is_thread_safe: bool) -> Option> { + match self { + Sink::BaseSink(base) => base.copy(is_thread_safe), + Sink::SinkCopy(sc) => sc.copy(is_thread_safe), + Sink::SinkPrint(sp) => sp.copy(is_thread_safe), + Sink::SinkLimit(sl) => sl.copy(is_thread_safe), + } + } + + fn is_same_as(&mut self, op: &mut Operator) -> bool { + match self { + Sink::BaseSink(base) => base.is_same_as(op), + Sink::SinkCopy(sc) => sc.is_same_as(op), + Sink::SinkPrint(sp) => sp.is_same_as(op), + Sink::SinkLimit(sl) => sl.is_same_as(op), + } + } + + fn get_num_out_tuples(&self) -> usize { + match self { + Sink::BaseSink(base) => base.get_num_out_tuples(), + Sink::SinkCopy(sc) => sc.get_num_out_tuples(), + Sink::SinkPrint(sp) => sp.get_num_out_tuples(), + Sink::SinkLimit(sl) => sl.get_num_out_tuples(), + } + } +} \ No newline at end of file diff --git a/src/graph_impl/multi_graph/plan/operator/sink/sink_copy.rs b/src/graph_impl/multi_graph/plan/operator/sink/sink_copy.rs new file mode 100644 index 00000000..3d257aa0 --- /dev/null +++ b/src/graph_impl/multi_graph/plan/operator/sink/sink_copy.rs @@ -0,0 +1,68 @@ +use graph_impl::multi_graph::plan::operator::sink::sink::{Sink, BaseSink}; +use generic::{IdType, GraphType}; +use std::io::sink; +use graph_impl::multi_graph::catalog::query_graph::QueryGraph; +use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; +use graph_impl::TypedStaticGraph; +use std::hash::{Hash, BuildHasherDefault}; +use hashbrown::HashMap; + +#[derive(Clone)] +pub struct SinkCopy { + pub base_sink: BaseSink, + output_tuple: Vec, +} + +impl SinkCopy { + pub fn new(query_graph: Box, out_tuple_length: usize) -> SinkCopy { + let mut sink_copy = SinkCopy { + base_sink: BaseSink::new(query_graph), + output_tuple: vec![], + }; + sink_copy.base_sink.base_op.out_tuple_len = out_tuple_length; + sink_copy.output_tuple = vec![Id::new(0); out_tuple_length]; + sink_copy + } +} + +impl CommonOperatorTrait for SinkCopy { + fn init(&mut self, probe_tuple: Vec, graph: &TypedStaticGraph) { + self.base_sink.init(probe_tuple, graph); + } + + fn process_new_tuple(&mut self) { + let len = self.output_tuple.len(); + self.output_tuple.clone_from_slice(&self.base_sink.base_op.probe_tuple[0..len]); + } + + fn execute(&mut self) { + self.base_sink.execute(); + } + + fn get_alds_as_string(&self) -> String { + self.base_sink.get_alds_as_string() + } + + fn update_operator_name(&mut self, query_vertex_to_index_map: HashMap) { + self.base_sink.update_operator_name(query_vertex_to_index_map) + } + + fn copy(&self, is_thread_safe: bool) -> Option> { + let base_op = &self.base_sink.base_op; + let mut sink = SinkCopy::new( + base_op.out_subgraph.clone(), + base_op.out_tuple_len, + ); + let origin_prev = base_op.prev.as_ref().unwrap(); + sink.base_sink.base_op.prev = origin_prev.copy(is_thread_safe).map(|x| Box::new(x)); + Some(Operator::Sink(Sink::SinkCopy(sink))) + } + + fn is_same_as(&mut self, op: &mut Operator) -> bool { + self.base_sink.is_same_as(op) + } + + fn get_num_out_tuples(&self) -> usize { + self.base_sink.get_num_out_tuples() + } +} \ No newline at end of file diff --git a/src/graph_impl/multi_graph/plan/operator/sink/sink_limit.rs b/src/graph_impl/multi_graph/plan/operator/sink/sink_limit.rs new file mode 100644 index 00000000..f636e413 --- /dev/null +++ b/src/graph_impl/multi_graph/plan/operator/sink/sink_limit.rs @@ -0,0 +1,66 @@ +use generic::{IdType, GraphType}; +use graph_impl::multi_graph::plan::operator::sink::sink::{BaseSink, Sink}; +use graph_impl::multi_graph::plan::operator::scan::scan::Scan; +use graph_impl::multi_graph::plan::operator::extend::EI::EI; +use graph_impl::multi_graph::catalog::query_graph::QueryGraph; +use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; +use graph_impl::TypedStaticGraph; +use std::hash::{Hash, BuildHasherDefault}; +use graph_impl::multi_graph::utils::io_utils::{current_time, get_elapsed_time_in_millis}; +use hashbrown::HashMap; + +#[derive(Clone)] +pub struct SinkLimit { + pub base_sink: BaseSink, + start_time: i64, + elapsed_time: f32, + out_tuples_limit: usize, +} + +impl SinkLimit { + pub fn new(query_graph: Box, out_tuple_limit: usize) -> SinkLimit { + SinkLimit { + base_sink: BaseSink::new(query_graph), + start_time: 0, + elapsed_time: 0.0, + out_tuples_limit: out_tuple_limit, + } + } +} + +impl CommonOperatorTrait for SinkLimit { + fn init(&mut self, probe_tuple: Vec, graph: &TypedStaticGraph) { + self.base_sink.init(probe_tuple,graph) + } + + fn process_new_tuple(&mut self) { + let prev = self.base_sink.base_op.prev.as_ref().unwrap().as_ref(); + if get_op_attr!(prev,num_out_tuples) >= self.out_tuples_limit { + self.elapsed_time = get_elapsed_time_in_millis(self.start_time); + } + } + + fn execute(&mut self) { + self.base_sink.execute(); + } + + fn get_alds_as_string(&self) -> String { + self.base_sink.get_alds_as_string() + } + + fn update_operator_name(&mut self, query_vertex_to_index_map: HashMap) { + self.base_sink.update_operator_name(query_vertex_to_index_map) + } + + fn copy(&self, is_thread_safe: bool) -> Option> { + self.base_sink.copy(is_thread_safe) + } + + fn is_same_as(&mut self, op: &mut Operator) -> bool { + self.base_sink.is_same_as(op) + } + + fn get_num_out_tuples(&self) -> usize { + self.base_sink.get_num_out_tuples() + } +} \ No newline at end of file diff --git a/src/graph_impl/multi_graph/plan/operator/sink/sink_print.rs b/src/graph_impl/multi_graph/plan/operator/sink/sink_print.rs new file mode 100644 index 00000000..d804b3ac --- /dev/null +++ b/src/graph_impl/multi_graph/plan/operator/sink/sink_print.rs @@ -0,0 +1,58 @@ +use generic::{IdType, GraphType}; +use graph_impl::multi_graph::plan::operator::sink::sink::{BaseSink, Sink}; +use graph_impl::multi_graph::catalog::query_graph::QueryGraph; +use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; +use graph_impl::TypedStaticGraph; +use std::hash::{Hash, BuildHasherDefault}; +use hashbrown::HashMap; + +#[derive(Clone)] +pub struct SinkPrint { + pub base_sink: BaseSink, +} + +impl SinkPrint { + pub fn new(query_graph: Box) -> SinkPrint { + SinkPrint { + base_sink: BaseSink::new(query_graph) + } + } +} + +impl CommonOperatorTrait for SinkPrint { + fn init(&mut self, probe_tuple: Vec, graph: &TypedStaticGraph) { + self.base_sink.init(probe_tuple,graph) + } + + fn process_new_tuple(&mut self) { + println!("{:?}", self.base_sink.base_op.probe_tuple); + } + + fn execute(&mut self) { + self.base_sink.execute() + } + + fn get_alds_as_string(&self) -> String { + self.base_sink.get_alds_as_string() + } + + fn update_operator_name(&mut self, query_vertex_to_index_map: HashMap) { + self.base_sink.update_operator_name(query_vertex_to_index_map) + } + + fn copy(&self, is_thread_safe: bool) -> Option> { + let base_op = &self.base_sink.base_op; + let mut sink = SinkPrint::new(base_op.out_subgraph.clone()); + let origin_prev = base_op.prev.as_ref().unwrap(); + sink.base_sink.base_op.prev = origin_prev.copy(is_thread_safe).map(|x| Box::new(x)); + Some(Operator::Sink(Sink::SinkPrint(sink))) + } + + fn is_same_as(&mut self, op: &mut Operator) -> bool { + self.base_sink.is_same_as(op) + } + + fn get_num_out_tuples(&self) -> usize { + self.base_sink.get_num_out_tuples() + } +} \ No newline at end of file diff --git a/src/graph_impl/multi_graph/plan/query_plan.rs b/src/graph_impl/multi_graph/plan/query_plan.rs index da3b5735..f4608673 100644 --- a/src/graph_impl/multi_graph/plan/query_plan.rs +++ b/src/graph_impl/multi_graph/plan/query_plan.rs @@ -1,6 +1,6 @@ use graph_impl::multi_graph::plan::operator::operator::{BaseOperator, Operator}; use graph_impl::multi_graph::plan::operator::scan::scan_sampling::ScanSampling; -use graph_impl::multi_graph::plan::operator::sink::sink::{Sink, SinkType}; +use graph_impl::multi_graph::plan::operator::sink::sink::{Sink, SinkType, BaseSink}; use graph_impl::multi_graph::plan::operator::scan::scan::Scan; use graph_impl::multi_graph::plan::operator::extend::EI::EI; use hashbrown::HashMap; @@ -34,14 +34,14 @@ impl QueryPlan { scan_sampling_op.get_last_operators(&mut last_operators); let op = &last_operators[0]; let out_subgraph = Box::new(get_op_attr_as_ref!(op, out_subgraph).as_ref().clone()); - let mut sink = Sink::new(out_subgraph); + let mut sink = BaseSink::new(out_subgraph); for op in last_operators.iter_mut() { let next = get_op_attr_as_mut!(op, next); - next.replace(vec![Operator::Sink(sink.clone())]); + next.replace(vec![Operator::Sink(Sink::BaseSink(sink.clone()))]); } sink.previous = Some(last_operators); Self { - sink: Some(sink), + sink: Some(Sink::BaseSink(sink)), sink_type: SinkType::Counter, scan_sampling: Some(scan_sampling), last_operator: None, From 1d39233cae571ab17561bf86820fdb05b56802ff Mon Sep 17 00:00:00 2001 From: Michael Chen <920901835@qq.com> Date: Fri, 20 Dec 2019 23:06:14 +0800 Subject: [PATCH 11/25] fix:implemented all operators --- .../catalog/adj_list_descriptor.rs | 4 +- src/graph_impl/multi_graph/catalog/catalog.rs | 25 +- .../multi_graph/catalog/catalog_plans.rs | 6 +- .../multi_graph/catalog/query_graph.rs | 21 ++ .../multi_graph/plan/operator/extend/EI.rs | 181 ++++++++---- .../plan/operator/extend/Extend.rs | 103 +++++-- .../plan/operator/extend/Intersect.rs | 103 +++++-- .../plan/operator/extend/extend.rs | 103 +++++-- .../plan/operator/extend/intersect.rs | 103 +++++-- .../multi_graph/plan/operator/extend/mod.rs | 2 +- .../plan/operator/hashjoin/build.rs | 105 +++++++ .../plan/operator/hashjoin/hash_join.rs | 188 +++++++++++++ .../plan/operator/hashjoin/hash_table.rs | 147 ++++++++++ .../multi_graph/plan/operator/hashjoin/mod.rs | 7 + .../plan/operator/hashjoin/probe.rs | 236 ++++++++++++++++ .../plan/operator/hashjoin/probe_cartesian.rs | 143 ++++++++++ .../operator/hashjoin/probe_multi_vertices.rs | 258 ++++++++++++++++++ .../probe_multi_vertices_cartesian.rs | 152 +++++++++++ .../multi_graph/plan/operator/mod.rs | 168 ++++++++---- .../multi_graph/plan/operator/operator.rs | 59 +++- .../multi_graph/plan/operator/scan/mod.rs | 2 +- .../multi_graph/plan/operator/scan/scan.rs | 66 +++-- .../plan/operator/scan/scan_blocking.rs | 149 ++++++---- .../plan/operator/scan/scan_sampling.rs | 20 +- .../multi_graph/plan/operator/sink/sink.rs | 28 +- .../plan/operator/sink/sink_copy.rs | 33 ++- .../plan/operator/sink/sink_limit.rs | 31 ++- .../plan/operator/sink/sink_print.rs | 27 +- src/graph_impl/multi_graph/plan/query_plan.rs | 43 ++- src/graph_impl/static_graph/graph.rs | 68 +++-- src/graph_impl/static_graph/sorted_adj_vec.rs | 29 +- 31 files changed, 2176 insertions(+), 434 deletions(-) create mode 100644 src/graph_impl/multi_graph/plan/operator/hashjoin/build.rs create mode 100644 src/graph_impl/multi_graph/plan/operator/hashjoin/hash_join.rs create mode 100644 src/graph_impl/multi_graph/plan/operator/hashjoin/hash_table.rs create mode 100644 src/graph_impl/multi_graph/plan/operator/hashjoin/mod.rs create mode 100644 src/graph_impl/multi_graph/plan/operator/hashjoin/probe.rs create mode 100644 src/graph_impl/multi_graph/plan/operator/hashjoin/probe_cartesian.rs create mode 100644 src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices.rs create mode 100644 src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices_cartesian.rs diff --git a/src/graph_impl/multi_graph/catalog/adj_list_descriptor.rs b/src/graph_impl/multi_graph/catalog/adj_list_descriptor.rs index ff7dd6b4..e8188c45 100644 --- a/src/graph_impl/multi_graph/catalog/adj_list_descriptor.rs +++ b/src/graph_impl/multi_graph/catalog/adj_list_descriptor.rs @@ -44,8 +44,8 @@ impl Display for Direction { impl PartialEq for Direction { fn eq(&self, other: &Self) -> bool { match (self, other) { - (Direction::Fwd, Direction::Fwd, ) | (Direction::Bwd, Direction::Bwd) => true, - _ => false + (Direction::Fwd, Direction::Fwd) | (Direction::Bwd, Direction::Bwd) => true, + _ => false, } } } diff --git a/src/graph_impl/multi_graph/catalog/catalog.rs b/src/graph_impl/multi_graph/catalog/catalog.rs index 8d10484b..331389f3 100644 --- a/src/graph_impl/multi_graph/catalog/catalog.rs +++ b/src/graph_impl/multi_graph/catalog/catalog.rs @@ -4,13 +4,15 @@ use graph_impl::multi_graph::catalog::catalog_plans::{ CatalogPlans, DEF_MAX_INPUT_NUM_VERTICES, DEF_NUM_EDGES_TO_SAMPLE, }; use graph_impl::multi_graph::catalog::query_graph::QueryGraph; -use graph_impl::multi_graph::plan::operator::scan::scan::Scan; -use graph_impl::multi_graph::plan::operator::sink::sink::Sink; +use graph_impl::multi_graph::plan::operator::extend::EI::EI; +use graph_impl::multi_graph::plan::operator::hashjoin::probe::Probe; +use graph_impl::multi_graph::plan::operator::hashjoin::probe_multi_vertices::PMV; use graph_impl::multi_graph::plan::operator::operator::{ BaseOperator, CommonOperatorTrait, Operator, }; +use graph_impl::multi_graph::plan::operator::scan::scan::Scan; +use graph_impl::multi_graph::plan::operator::sink::sink::Sink; use graph_impl::multi_graph::plan::query_plan::QueryPlan; -use graph_impl::multi_graph::plan::operator::extend::EI::EI; use graph_impl::multi_graph::utils::io_utils; use graph_impl::TypedStaticGraph; use hashbrown::{HashMap, HashSet}; @@ -229,7 +231,7 @@ impl Catalog { .count() } - pub fn populate( + pub fn populate( &mut self, graph: TypedStaticGraph, num_threads: usize, @@ -271,7 +273,7 @@ impl Catalog { } } - fn execute(&self, query_plan_arr: &mut Vec>) { + fn execute(&self, query_plan_arr: &mut Vec>) { if query_plan_arr.len() > 1 { let mut handlers = vec![]; for i in 0..query_plan_arr.len() { @@ -297,9 +299,13 @@ impl Catalog { let mut other: Vec<&mut Operator> = query_plan_arr .iter_mut() .map(|query_plan| { - let base_sink = get_sink_as_mut!(query_plan.get_sink()); + let mut base_sink = get_sink_as_mut!(query_plan.get_sink()); let mut op = &mut base_sink.previous.as_mut().unwrap()[0]; - while if let Operator::Scan(Scan::ScanSampling(sp)) = op.deref() { false } else { true } { + while if let Operator::Scan(Scan::ScanSampling(sp)) = op.deref() { + false + } else { + true + } { let prev_op = get_op_attr_as_mut!(op, prev).as_mut().unwrap(); op = prev_op.as_mut(); } @@ -320,7 +326,8 @@ impl Catalog { operator: &mut Operator, other: Vec<&mut Operator>, is_directed: bool, - ) {} + ) { + } fn add_icost_and_selectivity( &mut self, @@ -336,7 +343,7 @@ impl Catalog { num_input_tuples += get_op_attr!(other_op, num_out_tuples); } let in_subgraph = get_op_attr_as_mut!(operator, out_subgraph); - let subgraph_idx = self.get_subgraph_idx(in_subgraph); + let subgraph_idx = self.get_subgraph_idx(in_subgraph.as_mut()); let next_list = get_op_attr_as_ref!(operator, next).as_ref().unwrap(); for (i, next) in next_list.iter().enumerate() { let intersect = next; diff --git a/src/graph_impl/multi_graph/catalog/catalog_plans.rs b/src/graph_impl/multi_graph/catalog/catalog_plans.rs index edecfb97..50e7f18c 100644 --- a/src/graph_impl/multi_graph/catalog/catalog_plans.rs +++ b/src/graph_impl/multi_graph/catalog/catalog_plans.rs @@ -62,8 +62,10 @@ impl CatalogPlans { plans } - pub fn set_next_operators( - graph: TypedStaticGraph, ) {} + pub fn set_next_operators( + graph: TypedStaticGraph, + ) { + } pub fn query_graphs_to_extend(&self) -> &QueryGraphSet { &self.query_graphs_to_extend diff --git a/src/graph_impl/multi_graph/catalog/query_graph.rs b/src/graph_impl/multi_graph/catalog/query_graph.rs index e96b9942..5d9e97b2 100644 --- a/src/graph_impl/multi_graph/catalog/query_graph.rs +++ b/src/graph_impl/multi_graph/catalog/query_graph.rs @@ -16,6 +16,17 @@ pub struct QueryGraph { } impl QueryGraph { + pub fn empty() -> QueryGraph { + QueryGraph { + qvertex_to_qedges_map: HashMap::new(), + qvertex_to_type_map: HashMap::new(), + qvertex_to_deg_map: HashMap::new(), + q_edges: vec![], + it: None, + encoding: None, + } + } + pub fn get_num_qvertices(&self) -> usize { self.qvertex_to_qedges_map.len() } @@ -211,4 +222,14 @@ impl QueryGraph { .get_subgraph_mapping_if_any(other_query_graph) .is_some()) } + + pub fn get_isomorphic_mapping_if_any( + &mut self, + other_query_graph: &mut QueryGraph, + ) -> Option<&HashMap> { + if self.is_isomorphic_to(other_query_graph) { + return self.get_subgraph_mapping_if_any(other_query_graph); + } + None + } } diff --git a/src/graph_impl/multi_graph/plan/operator/extend/EI.rs b/src/graph_impl/multi_graph/plan/operator/extend/EI.rs index ceef7729..0af832aa 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/EI.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/EI.rs @@ -1,17 +1,21 @@ +use generic::{GraphType, IdType}; use graph_impl::multi_graph::catalog::adj_list_descriptor::{AdjListDescriptor, Direction}; use graph_impl::multi_graph::catalog::query_graph::QueryGraph; -use graph_impl::multi_graph::plan::operator::operator::{BaseOperator, CommonOperatorTrait, Operator}; use graph_impl::multi_graph::plan::operator::extend::extend::Extend; use graph_impl::multi_graph::plan::operator::extend::intersect::{Intersect, IntersectType}; -use graph_impl::static_graph::sorted_adj_vec::SortedAdjVec; -use graph_impl::multi_graph::plan::operator::sink::sink::Sink; +use graph_impl::multi_graph::plan::operator::hashjoin::probe::Probe; +use graph_impl::multi_graph::plan::operator::hashjoin::probe_multi_vertices::PMV; +use graph_impl::multi_graph::plan::operator::operator::{ + BaseOperator, CommonOperatorTrait, Operator, +}; use graph_impl::multi_graph::plan::operator::scan::scan::Scan; -use itertools::Itertools; -use hashbrown::HashMap; -use generic::{IdType, GraphType}; +use graph_impl::multi_graph::plan::operator::sink::sink::Sink; +use graph_impl::static_graph::sorted_adj_vec::SortedAdjVec; use graph_impl::TypedStaticGraph; -use std::hash::{Hash, BuildHasherDefault}; +use hashbrown::HashMap; use indexmap::Equivalent; +use itertools::Itertools; +use std::hash::{BuildHasherDefault, Hash}; pub static DIFFERENTIATE_FWD_BWD_SINGLE_ALD: bool = false; @@ -22,14 +26,13 @@ pub enum CachingType { PartialCaching, } - impl PartialEq for CachingType { fn eq(&self, other: &Self) -> bool { match (self, other) { - (CachingType::None, CachingType::None) | - (CachingType::FullCaching, CachingType::FullCaching) | - (CachingType::PartialCaching, CachingType::PartialCaching) => true, - _ => false + (CachingType::None, CachingType::None) + | (CachingType::FullCaching, CachingType::FullCaching) + | (CachingType::PartialCaching, CachingType::PartialCaching) => true, + _ => false, } } } @@ -98,35 +101,67 @@ impl BaseEI { } fn set_operator_name(&mut self) { - let variables = self.alds.iter() + let variables = self + .alds + .iter() .map(|ald| { - ald.from_query_vertex.clone() + "[" + if let Direction::Fwd = ald.direction { "Fwd" } else { "Bwd" } + "]" - }).sorted().join("-"); + ald.from_query_vertex.clone() + + "[" + + if let Direction::Fwd = ald.direction { + "Fwd" + } else { + "Bwd" + } + + "]" + }) + .sorted() + .join("-"); if 1 == self.alds.len() { self.base_op.name = "Single-Edge-extend".to_string(); } else { self.base_op.name = "Multi-Edge-extend".to_string(); } - self.base_op.name += &(" TO (".to_owned() + &self.to_query_vertex + ") From (" + &variables + ")"); + self.base_op.name += + &(" TO (".to_owned() + &self.to_query_vertex + ") From (" + &variables + ")"); } - pub fn make(to_qvertex: String, to_type: usize, alds: Vec, - out_subgraph: QueryGraph, in_subgraph: QueryGraph, - out_qvertex_to_idx_map: HashMap) -> EI { + pub fn make( + to_qvertex: String, + to_type: usize, + alds: Vec, + out_subgraph: QueryGraph, + in_subgraph: QueryGraph, + out_qvertex_to_idx_map: HashMap, + ) -> EI { if 1 == alds.len() { - return EI::Extend(Extend::new(to_qvertex, to_type, alds, Box::new(out_subgraph), - Some(Box::new(in_subgraph)), out_qvertex_to_idx_map)); + return EI::Extend(Extend::new( + to_qvertex, + to_type, + alds, + Box::new(out_subgraph), + Some(Box::new(in_subgraph)), + out_qvertex_to_idx_map, + )); } - EI::Intersect(Intersect::new(to_qvertex, to_type, alds, Box::new(out_subgraph), - Some(Box::new(in_subgraph)), out_qvertex_to_idx_map)) + EI::Intersect(Intersect::new( + to_qvertex, + to_type, + alds, + Box::new(out_subgraph), + Some(Box::new(in_subgraph)), + out_qvertex_to_idx_map, + )) } pub fn is_intersection_cached(&mut self) -> bool { self.is_intersection_cached = true; for i in 0..self.last_vertex_ids_intersected.as_ref().unwrap().len() { - if self.last_vertex_ids_intersected.as_ref().unwrap()[i] != self.base_op.probe_tuple[self.vertex_idx_to_cache[i]] { + if self.last_vertex_ids_intersected.as_ref().unwrap()[i] + != self.base_op.probe_tuple[self.vertex_idx_to_cache[i]] + { self.is_intersection_cached = false; - self.last_vertex_ids_intersected.as_mut().unwrap()[i] = self.base_op.probe_tuple[self.vertex_idx_to_cache[i]]; + self.last_vertex_ids_intersected.as_mut().unwrap()[i] = + self.base_op.probe_tuple[self.vertex_idx_to_cache[i]]; } } self.is_intersection_cached @@ -135,7 +170,11 @@ impl BaseEI { if self.alds.len() == 1 { return; } - let num_cached_alds = self.alds.iter().filter(|&ald| ald.vertex_idx <= last_repeated_vertex_idx).count(); + let num_cached_alds = self + .alds + .iter() + .filter(|&ald| ald.vertex_idx <= last_repeated_vertex_idx) + .count(); if num_cached_alds <= 1 { return; } @@ -147,7 +186,10 @@ impl BaseEI { self.last_vertex_ids_intersected = Some(vec![Id::new(0); num_cached_alds]); } - fn init_extensions(&mut self, graph: &TypedStaticGraph) { + fn init_extensions( + &mut self, + graph: &TypedStaticGraph, + ) { match self.caching_type { CachingType::None | CachingType::FullCaching => { *&mut self.out_neighbours = Neighbours::new(); @@ -159,7 +201,11 @@ impl BaseEI { } let mut largest_adj_list_size = 0; for ald in &self.alds { - let label = if graph.is_sorted_by_node() { self.to_type } else { ald.label }; + let label = if graph.is_sorted_by_node() { + self.to_type + } else { + ald.label + }; let adj_list_size = graph.get_largest_adj_list_size(label, ald.direction.clone()); if adj_list_size > largest_adj_list_size { largest_adj_list_size = adj_list_size; @@ -175,7 +221,11 @@ impl BaseEI { } } - fn set_alds_and_adj_lists(&mut self, graph: &TypedStaticGraph, last_repeated_vertex_idx: usize) { + fn set_alds_and_adj_lists( + &mut self, + graph: &TypedStaticGraph, + last_repeated_vertex_idx: usize, + ) { let num_cached_alds = if let Some(ids) = &mut self.last_vertex_ids_intersected { ids.len() } else { @@ -227,42 +277,52 @@ impl BaseEI { pub fn execute_intersect(&mut self, idx: usize, intersect_type: IntersectType) -> usize { let (adj_vec, label_or_type) = match intersect_type { IntersectType::CachedOut | IntersectType::TempOut => ( - self.adj_lists_to_cache[idx][self.base_op.probe_tuple[self.vertex_idx[idx]].id()].as_ref(), - self.labels_or_to_types[idx] + self.adj_lists_to_cache[idx][self.base_op.probe_tuple[self.vertex_idx[idx]].id()] + .as_ref(), + self.labels_or_to_types[idx], ), _ => ( - self.adj_lists_to_cache[idx][self.base_op.probe_tuple[self.vertex_idx_to_cache[idx]].id()].as_ref(), - self.labels_or_to_types_to_cache[idx] - ) + self.adj_lists_to_cache[idx] + [self.base_op.probe_tuple[self.vertex_idx_to_cache[idx]].id()] + .as_ref(), + self.labels_or_to_types_to_cache[idx], + ), }; let init = &mut self.init_neighbours; let cached = &mut self.cached_neighbours; let temp = &mut self.temp_neighbours; let out = &mut self.out_neighbours; - adj_vec.map_or(0, |adj| { - match intersect_type { - IntersectType::InitCached => adj.intersect(label_or_type, init, cached), - IntersectType::TempCached => adj.intersect(label_or_type, temp, cached), - IntersectType::CachedOut => adj.intersect(label_or_type, cached, out), - IntersectType::TempOut => adj.intersect(label_or_type, temp, out), - } + adj_vec.map_or(0, |adj| match intersect_type { + IntersectType::InitCached => adj.intersect(label_or_type, init, cached), + IntersectType::TempCached => adj.intersect(label_or_type, temp, cached), + IntersectType::CachedOut => adj.intersect(label_or_type, cached, out), + IntersectType::TempOut => adj.intersect(label_or_type, temp, out), }) } } impl CommonOperatorTrait for BaseEI { - fn init(&mut self, probe_tuple: Vec, graph: &TypedStaticGraph) { + fn init( + &mut self, + probe_tuple: Vec, + graph: &TypedStaticGraph, + ) { self.base_op.probe_tuple = probe_tuple.clone(); self.caching_type = CachingType::None; self.vertex_types = graph.get_node_types().clone(); let prev = self.base_op.prev.as_mut().unwrap().as_mut(); - let last_repeated_vertex_idx = get_op_attr_as_mut!(prev,last_repeated_vertex_idx).clone(); + let last_repeated_vertex_idx = get_op_attr_as_mut!(prev, last_repeated_vertex_idx).clone(); self.init_caching(last_repeated_vertex_idx); self.init_extensions(graph); self.set_alds_and_adj_lists(graph, last_repeated_vertex_idx); - self.base_op.next.as_mut().unwrap().iter_mut().foreach(|next_op| { - next_op.init(probe_tuple.clone(), graph); - }); + self.base_op + .next + .as_mut() + .unwrap() + .iter_mut() + .foreach(|next_op| { + next_op.init(probe_tuple.clone(), graph); + }); } fn process_new_tuple(&mut self) { @@ -279,7 +339,11 @@ impl CommonOperatorTrait for BaseEI { } let mut directions = vec!["".to_owned(); self.alds.len()]; for ald in &self.alds { - let dir = if let Direction::Fwd = ald.direction { "F".to_owned() } else { "B".to_owned() }; + let dir = if let Direction::Fwd = ald.direction { + "F".to_owned() + } else { + "B".to_owned() + }; directions.push(dir + &ald.label.to_string()); } directions.sort(); @@ -291,14 +355,19 @@ impl CommonOperatorTrait for BaseEI { for (query_vertex, &index) in &query_vertex_to_index_map { prev_to_query_vertices[index] = query_vertex.clone(); } - self.base_op.name = serde_json::to_string(&prev_to_query_vertices).unwrap() + " - " + &self.base_op.name; - query_vertex_to_index_map.insert(self.to_query_vertex.clone(), query_vertex_to_index_map.len()); + self.base_op.name = + serde_json::to_string(&prev_to_query_vertices).unwrap() + " - " + &self.base_op.name; + query_vertex_to_index_map.insert( + self.to_query_vertex.clone(), + query_vertex_to_index_map.len(), + ); if let Some(next) = &mut self.base_op.next { - next.iter_mut().foreach(|op| op.update_operator_name(query_vertex_to_index_map.clone())) + next.iter_mut() + .foreach(|op| op.update_operator_name(query_vertex_to_index_map.clone())) } } - fn copy(&self, is_thread_safe: bool) -> Option> { + fn copy(&self, is_thread_safe: bool) -> Operator { panic!("unsupported operation exception") } @@ -325,7 +394,11 @@ impl EI { } impl CommonOperatorTrait for EI { - fn init(&mut self, probe_tuple: Vec, graph: &TypedStaticGraph) { + fn init( + &mut self, + probe_tuple: Vec, + graph: &TypedStaticGraph, + ) { match self { EI::Base(base) => base.init(probe_tuple, graph), EI::Intersect(intersect) => intersect.init(probe_tuple, graph), @@ -357,7 +430,7 @@ impl CommonOperatorTrait for EI { } } - fn update_operator_name(&mut self, mut query_vertex_to_index_map: HashMap) { + fn update_operator_name(&mut self, query_vertex_to_index_map: HashMap) { match self { EI::Base(base) => base.update_operator_name(query_vertex_to_index_map), EI::Intersect(intersect) => intersect.update_operator_name(query_vertex_to_index_map), @@ -365,7 +438,7 @@ impl CommonOperatorTrait for EI { } } - fn copy(&self, is_thread_safe: bool) -> Option> { + fn copy(&self, is_thread_safe: bool) -> Operator { match self { EI::Base(base) => base.copy(is_thread_safe), EI::Intersect(intersect) => intersect.copy(is_thread_safe), diff --git a/src/graph_impl/multi_graph/plan/operator/extend/Extend.rs b/src/graph_impl/multi_graph/plan/operator/extend/Extend.rs index 82fdcffa..f7a05b9b 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/Extend.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/Extend.rs @@ -1,14 +1,18 @@ -use graph_impl::multi_graph::plan::operator::extend::EI::{BaseEI, Neighbours, EI, DIFFERENTIATE_FWD_BWD_SINGLE_ALD}; -use graph_impl::multi_graph::catalog::adj_list_descriptor::{Direction, AdjListDescriptor}; -use graph_impl::static_graph::sorted_adj_vec::SortedAdjVec; +use generic::{GraphType, IdType}; +use graph_impl::multi_graph::catalog::adj_list_descriptor::{AdjListDescriptor, Direction}; use graph_impl::multi_graph::catalog::query_graph::QueryGraph; +use graph_impl::multi_graph::plan::operator::extend::EI::{ + BaseEI, Neighbours, DIFFERENTIATE_FWD_BWD_SINGLE_ALD, EI, +}; +use graph_impl::multi_graph::plan::operator::hashjoin::probe::Probe; +use graph_impl::multi_graph::plan::operator::hashjoin::probe_multi_vertices::PMV; +use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; use graph_impl::multi_graph::plan::operator::scan::scan::Scan; use graph_impl::multi_graph::plan::operator::sink::sink::Sink; -use hashbrown::HashMap; -use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; +use graph_impl::static_graph::sorted_adj_vec::SortedAdjVec; use graph_impl::TypedStaticGraph; -use generic::{IdType, GraphType}; -use std::hash::{Hash, BuildHasherDefault}; +use hashbrown::HashMap; +use std::hash::{BuildHasherDefault, Hash}; #[derive(Clone)] pub struct Extend { @@ -20,9 +24,14 @@ pub struct Extend { } impl Extend { - pub fn new(to_qvertex: String, to_type: usize, alds: Vec, - out_subgraph: Box, in_subgraph: Option>, - out_qvertex_to_idx_map: HashMap) -> Extend { + pub fn new( + to_qvertex: String, + to_type: usize, + alds: Vec, + out_subgraph: Box, + in_subgraph: Option>, + out_qvertex_to_idx_map: HashMap, + ) -> Extend { let ald = alds.get(0).unwrap().clone(); let mut extend = Extend { base_ei: BaseEI::new(to_qvertex.clone(), to_type, alds, out_subgraph, in_subgraph), @@ -39,11 +48,20 @@ impl Extend { } impl CommonOperatorTrait for Extend { - fn init(&mut self, probe_tuple: Vec, graph: &TypedStaticGraph) { + fn init( + &mut self, + probe_tuple: Vec, + graph: &TypedStaticGraph, + ) { self.base_ei.out_neighbours = Neighbours::new(); self.base_ei.base_op.probe_tuple = probe_tuple.clone(); self.base_ei.vertex_types = graph.get_node_types().clone(); - self.adj_list = if let Direction::Fwd = self.dir { graph.get_fwd_adj_list() } else { graph.get_bwd_adj_list() }.clone(); + self.adj_list = if let Direction::Fwd = self.dir { + graph.get_fwd_adj_list() + } else { + graph.get_bwd_adj_list() + } + .clone(); if graph.is_sorted_by_node() { self.label_or_to_type = self.base_ei.to_type; self.base_ei.to_type = 0; @@ -54,12 +72,16 @@ impl CommonOperatorTrait for Extend { } fn process_new_tuple(&mut self) { - let adj_vec = self.adj_list[self.base_ei.base_op.probe_tuple[self.vertex_index].id()].as_mut().unwrap(); + let adj_vec = self.adj_list[self.base_ei.base_op.probe_tuple[self.vertex_index].id()] + .as_mut() + .unwrap(); let out_neighbour = &mut self.base_ei.out_neighbours; adj_vec.set_neighbor_ids(self.label_or_to_type, out_neighbour); self.base_ei.base_op.icost += out_neighbour.end_idx - out_neighbour.start_idx; for idx in out_neighbour.start_idx..out_neighbour.end_idx { - if self.base_ei.to_type == 0 || self.base_ei.to_type == self.base_ei.vertex_types[out_neighbour.ids[idx].id()] { + if self.base_ei.to_type == 0 + || self.base_ei.to_type == self.base_ei.vertex_types[out_neighbour.ids[idx].id()] + { self.base_ei.base_op.num_out_tuples += 1; self.base_ei.base_op.probe_tuple[self.base_ei.out_idx] = out_neighbour.ids[idx]; self.base_ei.base_op.next.as_mut().unwrap()[0].process_new_tuple(); @@ -79,32 +101,57 @@ impl CommonOperatorTrait for Extend { self.base_ei.update_operator_name(query_vertex_to_index_map) } - fn copy(&self, is_thread_safe: bool) -> Option> { + fn copy(&self, is_thread_safe: bool) -> Operator { let base_ei = &self.base_ei; let base_op = &base_ei.base_op; let mut extend = Extend::new( - base_ei.to_query_vertex.clone(), base_ei.to_type, + base_ei.to_query_vertex.clone(), + base_ei.to_type, base_ei.alds.clone(), - base_op.out_subgraph.clone(), base_op.in_subgraph.clone(), + base_op.out_subgraph.clone(), + base_op.in_subgraph.clone(), base_op.out_qvertex_to_idx_map.clone(), ); let extend_copy = extend.clone(); - extend.base_ei.base_op.prev = base_op.prev.as_ref().unwrap().copy(is_thread_safe).map(|op| Box::new(op)); + extend.base_ei.base_op.prev = Some(Box::new( + base_op.prev.as_ref().unwrap().copy(is_thread_safe), + )); let prev = extend.base_ei.base_op.prev.as_mut().unwrap().as_mut(); - *get_op_attr_as_mut!(prev,next) = Some(vec![Operator::EI(EI::Extend(extend_copy))]); - let last_repeated_vertex_idx = get_op_attr!(prev,last_repeated_vertex_idx); + get_op_attr_as_mut!(prev, next).replace(vec![Operator::EI(EI::Extend(extend_copy))]); + let last_repeated_vertex_idx = get_op_attr!(prev, last_repeated_vertex_idx); extend.base_ei.init_caching(last_repeated_vertex_idx); - Some(Operator::EI(EI::Extend(extend))) + Operator::EI(EI::Extend(extend)) } fn is_same_as(&mut self, op: &mut Operator) -> bool { if let Operator::EI(EI::Extend(extend)) = op { - return (!DIFFERENTIATE_FWD_BWD_SINGLE_ALD || self.dir == extend.dir) && - self.label_or_to_type == extend.label_or_to_type && - self.base_ei.to_type == extend.base_ei.to_type && - self.base_ei.base_op.in_subgraph.as_mut().unwrap().is_isomorphic_to(get_op_attr_as_mut!(op,in_subgraph).as_mut().unwrap().as_mut()) && - self.base_ei.base_op.out_subgraph.is_isomorphic_to(get_op_attr_as_mut!(op,out_subgraph).as_mut()) && - self.base_ei.base_op.prev.as_mut().unwrap().is_same_as(get_op_attr_as_mut!(op,prev).as_mut().unwrap()); + return (!DIFFERENTIATE_FWD_BWD_SINGLE_ALD || self.dir == extend.dir) + && self.label_or_to_type == extend.label_or_to_type + && self.base_ei.to_type == extend.base_ei.to_type + && self + .base_ei + .base_op + .in_subgraph + .as_mut() + .unwrap() + .is_isomorphic_to( + get_op_attr_as_mut!(op, in_subgraph) + .as_mut() + .unwrap() + .as_mut(), + ) + && self + .base_ei + .base_op + .out_subgraph + .is_isomorphic_to(get_op_attr_as_mut!(op, out_subgraph).as_mut()) + && self + .base_ei + .base_op + .prev + .as_mut() + .unwrap() + .is_same_as(get_op_attr_as_mut!(op, prev).as_mut().unwrap()); } false } @@ -112,4 +159,4 @@ impl CommonOperatorTrait for Extend { fn get_num_out_tuples(&self) -> usize { self.base_ei.get_num_out_tuples() } -} \ No newline at end of file +} diff --git a/src/graph_impl/multi_graph/plan/operator/extend/Intersect.rs b/src/graph_impl/multi_graph/plan/operator/extend/Intersect.rs index d3f57e80..2aad84ca 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/Intersect.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/Intersect.rs @@ -1,14 +1,16 @@ -use graph_impl::multi_graph::plan::operator::extend::EI::{BaseEI, Neighbours, CachingType, EI}; +use generic::{GraphType, IdType}; +use graph_impl::multi_graph::catalog::adj_list_descriptor::AdjListDescriptor; +use graph_impl::multi_graph::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::plan::operator::extend::EI::EI::Base; +use graph_impl::multi_graph::plan::operator::extend::EI::{BaseEI, CachingType, Neighbours, EI}; +use graph_impl::multi_graph::plan::operator::hashjoin::probe::Probe; +use graph_impl::multi_graph::plan::operator::hashjoin::probe_multi_vertices::PMV; +use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; use graph_impl::multi_graph::plan::operator::scan::scan::Scan; use graph_impl::multi_graph::plan::operator::sink::sink::Sink; -use graph_impl::multi_graph::catalog::query_graph::QueryGraph; -use hashbrown::HashMap; -use graph_impl::multi_graph::catalog::adj_list_descriptor::AdjListDescriptor; -use generic::{IdType, GraphType}; -use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; use graph_impl::TypedStaticGraph; -use std::hash::{Hash, BuildHasherDefault}; +use hashbrown::HashMap; +use std::hash::{BuildHasherDefault, Hash}; pub enum IntersectType { InitCached, @@ -23,36 +25,51 @@ pub struct Intersect { } impl Intersect { - pub fn new(to_qvertex: String, to_type: usize, alds: Vec, - out_subgraph: Box, in_subgraph: Option>, - out_qvertex_to_idx_map: HashMap) -> Intersect { + pub fn new( + to_qvertex: String, + to_type: usize, + alds: Vec, + out_subgraph: Box, + in_subgraph: Option>, + out_qvertex_to_idx_map: HashMap, + ) -> Intersect { let mut intersect = Intersect { base_ei: BaseEI::new(to_qvertex.clone(), to_type, alds, out_subgraph, in_subgraph), }; let base_op = &mut intersect.base_ei.base_op; base_op.last_repeated_vertex_idx = base_op.out_tuple_len - 2; base_op.out_qvertex_to_idx_map = out_qvertex_to_idx_map; - intersect.base_ei.out_idx = base_op.out_qvertex_to_idx_map.get(&to_qvertex).unwrap().clone(); + intersect.base_ei.out_idx = base_op + .out_qvertex_to_idx_map + .get(&to_qvertex) + .unwrap() + .clone(); intersect } } impl CommonOperatorTrait for Intersect { - fn init(&mut self, probe_tuple: Vec, graph: &TypedStaticGraph) { + fn init( + &mut self, + probe_tuple: Vec, + graph: &TypedStaticGraph, + ) { self.base_ei.init(probe_tuple, graph) } fn process_new_tuple(&mut self) { let mut temp = Neighbours::new(); - if CachingType::None == self.base_ei.caching_type || !self.base_ei.is_intersection_cached() { + if CachingType::None == self.base_ei.caching_type || !self.base_ei.is_intersection_cached() + { let base_ei = &mut self.base_ei; let mut cache_id = base_ei.vertex_idx_to_cache[0]; let to_id = base_ei.base_op.probe_tuple[cache_id].id(); - let mut adj_vec = base_ei.adj_lists_to_cache[0][to_id].as_ref(); + let adj_vec = base_ei.adj_lists_to_cache[0][to_id].as_ref(); cache_id = base_ei.labels_or_to_types_to_cache[0]; let neighbours = &mut base_ei.init_neighbours; adj_vec.map(|adj| adj.set_neighbor_ids(cache_id, neighbours)); - base_ei.base_op.icost += (base_ei.init_neighbours.end_idx - base_ei.init_neighbours.start_idx); + base_ei.base_op.icost += + (base_ei.init_neighbours.end_idx - base_ei.init_neighbours.start_idx); base_ei.base_op.icost += base_ei.execute_intersect(1, IntersectType::InitCached); if base_ei.to_type != 0 { @@ -76,7 +93,9 @@ impl CommonOperatorTrait for Intersect { let base_ei = &mut self.base_ei; match base_ei.caching_type { - CachingType::None | CachingType::FullCaching => base_ei.out_neighbours = base_ei.cached_neighbours.clone(), + CachingType::None | CachingType::FullCaching => { + base_ei.out_neighbours = base_ei.cached_neighbours.clone() + } CachingType::PartialCaching => { let cost = base_ei.execute_intersect(0, IntersectType::CachedOut); base_ei.base_op.icost += cost; @@ -113,32 +132,56 @@ impl CommonOperatorTrait for Intersect { self.base_ei.update_operator_name(query_vertex_to_index_map) } - fn copy(&self, is_thread_safe: bool) -> Option> { + fn copy(&self, is_thread_safe: bool) -> Operator { let base_ei = &self.base_ei; let base_op = &base_ei.base_op; let mut intersect = Intersect::new( - base_ei.to_query_vertex.clone(), base_ei.to_type, + base_ei.to_query_vertex.clone(), + base_ei.to_type, base_ei.alds.clone(), - base_op.out_subgraph.clone(), base_op.in_subgraph.clone(), + base_op.out_subgraph.clone(), + base_op.in_subgraph.clone(), base_op.out_qvertex_to_idx_map.clone(), ); let intersect_copy = intersect.clone(); - intersect.base_ei.base_op.prev = base_op.prev.as_ref().unwrap().copy(is_thread_safe).map(|op| Box::new(op)); + intersect.base_ei.base_op.prev = Some(Box::new( + base_op.prev.as_ref().unwrap().copy(is_thread_safe), + )); let prev = intersect.base_ei.base_op.prev.as_mut().unwrap().as_mut(); - *get_op_attr_as_mut!(prev,next) = Some(vec![Operator::EI(EI::Intersect(intersect_copy))]); - let last_repeated_vertex_idx = get_op_attr!(prev,last_repeated_vertex_idx); + *get_op_attr_as_mut!(prev, next) = Some(vec![Operator::EI(EI::Intersect(intersect_copy))]); + let last_repeated_vertex_idx = get_op_attr!(prev, last_repeated_vertex_idx); intersect.base_ei.init_caching(last_repeated_vertex_idx); - Some(Operator::EI(EI::Intersect(intersect))) + Operator::EI(EI::Intersect(intersect)) } fn is_same_as(&mut self, op: &mut Operator) -> bool { if let Operator::EI(EI::Intersect(intersect)) = op { - return - self.base_ei.caching_type == intersect.base_ei.caching_type && - self.get_alds_as_string() == intersect.get_alds_as_string() && - self.base_ei.base_op.in_subgraph.as_mut().unwrap().is_isomorphic_to(get_op_attr_as_mut!(op,in_subgraph).as_mut().unwrap().as_mut()) && - self.base_ei.base_op.out_subgraph.is_isomorphic_to(get_op_attr_as_mut!(op,out_subgraph).as_mut()) && - self.base_ei.base_op.prev.as_mut().unwrap().is_same_as(get_op_attr_as_mut!(op,prev).as_mut().unwrap()); + return self.base_ei.caching_type == intersect.base_ei.caching_type + && self.get_alds_as_string() == intersect.get_alds_as_string() + && self + .base_ei + .base_op + .in_subgraph + .as_mut() + .unwrap() + .is_isomorphic_to( + get_op_attr_as_mut!(op, in_subgraph) + .as_mut() + .unwrap() + .as_mut(), + ) + && self + .base_ei + .base_op + .out_subgraph + .is_isomorphic_to(get_op_attr_as_mut!(op, out_subgraph).as_mut()) + && self + .base_ei + .base_op + .prev + .as_mut() + .unwrap() + .is_same_as(get_op_attr_as_mut!(op, prev).as_mut().unwrap()); } false } @@ -146,4 +189,4 @@ impl CommonOperatorTrait for Intersect { fn get_num_out_tuples(&self) -> usize { self.base_ei.get_num_out_tuples() } -} \ No newline at end of file +} diff --git a/src/graph_impl/multi_graph/plan/operator/extend/extend.rs b/src/graph_impl/multi_graph/plan/operator/extend/extend.rs index 82fdcffa..f7a05b9b 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/extend.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/extend.rs @@ -1,14 +1,18 @@ -use graph_impl::multi_graph::plan::operator::extend::EI::{BaseEI, Neighbours, EI, DIFFERENTIATE_FWD_BWD_SINGLE_ALD}; -use graph_impl::multi_graph::catalog::adj_list_descriptor::{Direction, AdjListDescriptor}; -use graph_impl::static_graph::sorted_adj_vec::SortedAdjVec; +use generic::{GraphType, IdType}; +use graph_impl::multi_graph::catalog::adj_list_descriptor::{AdjListDescriptor, Direction}; use graph_impl::multi_graph::catalog::query_graph::QueryGraph; +use graph_impl::multi_graph::plan::operator::extend::EI::{ + BaseEI, Neighbours, DIFFERENTIATE_FWD_BWD_SINGLE_ALD, EI, +}; +use graph_impl::multi_graph::plan::operator::hashjoin::probe::Probe; +use graph_impl::multi_graph::plan::operator::hashjoin::probe_multi_vertices::PMV; +use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; use graph_impl::multi_graph::plan::operator::scan::scan::Scan; use graph_impl::multi_graph::plan::operator::sink::sink::Sink; -use hashbrown::HashMap; -use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; +use graph_impl::static_graph::sorted_adj_vec::SortedAdjVec; use graph_impl::TypedStaticGraph; -use generic::{IdType, GraphType}; -use std::hash::{Hash, BuildHasherDefault}; +use hashbrown::HashMap; +use std::hash::{BuildHasherDefault, Hash}; #[derive(Clone)] pub struct Extend { @@ -20,9 +24,14 @@ pub struct Extend { } impl Extend { - pub fn new(to_qvertex: String, to_type: usize, alds: Vec, - out_subgraph: Box, in_subgraph: Option>, - out_qvertex_to_idx_map: HashMap) -> Extend { + pub fn new( + to_qvertex: String, + to_type: usize, + alds: Vec, + out_subgraph: Box, + in_subgraph: Option>, + out_qvertex_to_idx_map: HashMap, + ) -> Extend { let ald = alds.get(0).unwrap().clone(); let mut extend = Extend { base_ei: BaseEI::new(to_qvertex.clone(), to_type, alds, out_subgraph, in_subgraph), @@ -39,11 +48,20 @@ impl Extend { } impl CommonOperatorTrait for Extend { - fn init(&mut self, probe_tuple: Vec, graph: &TypedStaticGraph) { + fn init( + &mut self, + probe_tuple: Vec, + graph: &TypedStaticGraph, + ) { self.base_ei.out_neighbours = Neighbours::new(); self.base_ei.base_op.probe_tuple = probe_tuple.clone(); self.base_ei.vertex_types = graph.get_node_types().clone(); - self.adj_list = if let Direction::Fwd = self.dir { graph.get_fwd_adj_list() } else { graph.get_bwd_adj_list() }.clone(); + self.adj_list = if let Direction::Fwd = self.dir { + graph.get_fwd_adj_list() + } else { + graph.get_bwd_adj_list() + } + .clone(); if graph.is_sorted_by_node() { self.label_or_to_type = self.base_ei.to_type; self.base_ei.to_type = 0; @@ -54,12 +72,16 @@ impl CommonOperatorTrait for Extend { } fn process_new_tuple(&mut self) { - let adj_vec = self.adj_list[self.base_ei.base_op.probe_tuple[self.vertex_index].id()].as_mut().unwrap(); + let adj_vec = self.adj_list[self.base_ei.base_op.probe_tuple[self.vertex_index].id()] + .as_mut() + .unwrap(); let out_neighbour = &mut self.base_ei.out_neighbours; adj_vec.set_neighbor_ids(self.label_or_to_type, out_neighbour); self.base_ei.base_op.icost += out_neighbour.end_idx - out_neighbour.start_idx; for idx in out_neighbour.start_idx..out_neighbour.end_idx { - if self.base_ei.to_type == 0 || self.base_ei.to_type == self.base_ei.vertex_types[out_neighbour.ids[idx].id()] { + if self.base_ei.to_type == 0 + || self.base_ei.to_type == self.base_ei.vertex_types[out_neighbour.ids[idx].id()] + { self.base_ei.base_op.num_out_tuples += 1; self.base_ei.base_op.probe_tuple[self.base_ei.out_idx] = out_neighbour.ids[idx]; self.base_ei.base_op.next.as_mut().unwrap()[0].process_new_tuple(); @@ -79,32 +101,57 @@ impl CommonOperatorTrait for Extend { self.base_ei.update_operator_name(query_vertex_to_index_map) } - fn copy(&self, is_thread_safe: bool) -> Option> { + fn copy(&self, is_thread_safe: bool) -> Operator { let base_ei = &self.base_ei; let base_op = &base_ei.base_op; let mut extend = Extend::new( - base_ei.to_query_vertex.clone(), base_ei.to_type, + base_ei.to_query_vertex.clone(), + base_ei.to_type, base_ei.alds.clone(), - base_op.out_subgraph.clone(), base_op.in_subgraph.clone(), + base_op.out_subgraph.clone(), + base_op.in_subgraph.clone(), base_op.out_qvertex_to_idx_map.clone(), ); let extend_copy = extend.clone(); - extend.base_ei.base_op.prev = base_op.prev.as_ref().unwrap().copy(is_thread_safe).map(|op| Box::new(op)); + extend.base_ei.base_op.prev = Some(Box::new( + base_op.prev.as_ref().unwrap().copy(is_thread_safe), + )); let prev = extend.base_ei.base_op.prev.as_mut().unwrap().as_mut(); - *get_op_attr_as_mut!(prev,next) = Some(vec![Operator::EI(EI::Extend(extend_copy))]); - let last_repeated_vertex_idx = get_op_attr!(prev,last_repeated_vertex_idx); + get_op_attr_as_mut!(prev, next).replace(vec![Operator::EI(EI::Extend(extend_copy))]); + let last_repeated_vertex_idx = get_op_attr!(prev, last_repeated_vertex_idx); extend.base_ei.init_caching(last_repeated_vertex_idx); - Some(Operator::EI(EI::Extend(extend))) + Operator::EI(EI::Extend(extend)) } fn is_same_as(&mut self, op: &mut Operator) -> bool { if let Operator::EI(EI::Extend(extend)) = op { - return (!DIFFERENTIATE_FWD_BWD_SINGLE_ALD || self.dir == extend.dir) && - self.label_or_to_type == extend.label_or_to_type && - self.base_ei.to_type == extend.base_ei.to_type && - self.base_ei.base_op.in_subgraph.as_mut().unwrap().is_isomorphic_to(get_op_attr_as_mut!(op,in_subgraph).as_mut().unwrap().as_mut()) && - self.base_ei.base_op.out_subgraph.is_isomorphic_to(get_op_attr_as_mut!(op,out_subgraph).as_mut()) && - self.base_ei.base_op.prev.as_mut().unwrap().is_same_as(get_op_attr_as_mut!(op,prev).as_mut().unwrap()); + return (!DIFFERENTIATE_FWD_BWD_SINGLE_ALD || self.dir == extend.dir) + && self.label_or_to_type == extend.label_or_to_type + && self.base_ei.to_type == extend.base_ei.to_type + && self + .base_ei + .base_op + .in_subgraph + .as_mut() + .unwrap() + .is_isomorphic_to( + get_op_attr_as_mut!(op, in_subgraph) + .as_mut() + .unwrap() + .as_mut(), + ) + && self + .base_ei + .base_op + .out_subgraph + .is_isomorphic_to(get_op_attr_as_mut!(op, out_subgraph).as_mut()) + && self + .base_ei + .base_op + .prev + .as_mut() + .unwrap() + .is_same_as(get_op_attr_as_mut!(op, prev).as_mut().unwrap()); } false } @@ -112,4 +159,4 @@ impl CommonOperatorTrait for Extend { fn get_num_out_tuples(&self) -> usize { self.base_ei.get_num_out_tuples() } -} \ No newline at end of file +} diff --git a/src/graph_impl/multi_graph/plan/operator/extend/intersect.rs b/src/graph_impl/multi_graph/plan/operator/extend/intersect.rs index d3f57e80..2aad84ca 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/intersect.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/intersect.rs @@ -1,14 +1,16 @@ -use graph_impl::multi_graph::plan::operator::extend::EI::{BaseEI, Neighbours, CachingType, EI}; +use generic::{GraphType, IdType}; +use graph_impl::multi_graph::catalog::adj_list_descriptor::AdjListDescriptor; +use graph_impl::multi_graph::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::plan::operator::extend::EI::EI::Base; +use graph_impl::multi_graph::plan::operator::extend::EI::{BaseEI, CachingType, Neighbours, EI}; +use graph_impl::multi_graph::plan::operator::hashjoin::probe::Probe; +use graph_impl::multi_graph::plan::operator::hashjoin::probe_multi_vertices::PMV; +use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; use graph_impl::multi_graph::plan::operator::scan::scan::Scan; use graph_impl::multi_graph::plan::operator::sink::sink::Sink; -use graph_impl::multi_graph::catalog::query_graph::QueryGraph; -use hashbrown::HashMap; -use graph_impl::multi_graph::catalog::adj_list_descriptor::AdjListDescriptor; -use generic::{IdType, GraphType}; -use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; use graph_impl::TypedStaticGraph; -use std::hash::{Hash, BuildHasherDefault}; +use hashbrown::HashMap; +use std::hash::{BuildHasherDefault, Hash}; pub enum IntersectType { InitCached, @@ -23,36 +25,51 @@ pub struct Intersect { } impl Intersect { - pub fn new(to_qvertex: String, to_type: usize, alds: Vec, - out_subgraph: Box, in_subgraph: Option>, - out_qvertex_to_idx_map: HashMap) -> Intersect { + pub fn new( + to_qvertex: String, + to_type: usize, + alds: Vec, + out_subgraph: Box, + in_subgraph: Option>, + out_qvertex_to_idx_map: HashMap, + ) -> Intersect { let mut intersect = Intersect { base_ei: BaseEI::new(to_qvertex.clone(), to_type, alds, out_subgraph, in_subgraph), }; let base_op = &mut intersect.base_ei.base_op; base_op.last_repeated_vertex_idx = base_op.out_tuple_len - 2; base_op.out_qvertex_to_idx_map = out_qvertex_to_idx_map; - intersect.base_ei.out_idx = base_op.out_qvertex_to_idx_map.get(&to_qvertex).unwrap().clone(); + intersect.base_ei.out_idx = base_op + .out_qvertex_to_idx_map + .get(&to_qvertex) + .unwrap() + .clone(); intersect } } impl CommonOperatorTrait for Intersect { - fn init(&mut self, probe_tuple: Vec, graph: &TypedStaticGraph) { + fn init( + &mut self, + probe_tuple: Vec, + graph: &TypedStaticGraph, + ) { self.base_ei.init(probe_tuple, graph) } fn process_new_tuple(&mut self) { let mut temp = Neighbours::new(); - if CachingType::None == self.base_ei.caching_type || !self.base_ei.is_intersection_cached() { + if CachingType::None == self.base_ei.caching_type || !self.base_ei.is_intersection_cached() + { let base_ei = &mut self.base_ei; let mut cache_id = base_ei.vertex_idx_to_cache[0]; let to_id = base_ei.base_op.probe_tuple[cache_id].id(); - let mut adj_vec = base_ei.adj_lists_to_cache[0][to_id].as_ref(); + let adj_vec = base_ei.adj_lists_to_cache[0][to_id].as_ref(); cache_id = base_ei.labels_or_to_types_to_cache[0]; let neighbours = &mut base_ei.init_neighbours; adj_vec.map(|adj| adj.set_neighbor_ids(cache_id, neighbours)); - base_ei.base_op.icost += (base_ei.init_neighbours.end_idx - base_ei.init_neighbours.start_idx); + base_ei.base_op.icost += + (base_ei.init_neighbours.end_idx - base_ei.init_neighbours.start_idx); base_ei.base_op.icost += base_ei.execute_intersect(1, IntersectType::InitCached); if base_ei.to_type != 0 { @@ -76,7 +93,9 @@ impl CommonOperatorTrait for Intersect { let base_ei = &mut self.base_ei; match base_ei.caching_type { - CachingType::None | CachingType::FullCaching => base_ei.out_neighbours = base_ei.cached_neighbours.clone(), + CachingType::None | CachingType::FullCaching => { + base_ei.out_neighbours = base_ei.cached_neighbours.clone() + } CachingType::PartialCaching => { let cost = base_ei.execute_intersect(0, IntersectType::CachedOut); base_ei.base_op.icost += cost; @@ -113,32 +132,56 @@ impl CommonOperatorTrait for Intersect { self.base_ei.update_operator_name(query_vertex_to_index_map) } - fn copy(&self, is_thread_safe: bool) -> Option> { + fn copy(&self, is_thread_safe: bool) -> Operator { let base_ei = &self.base_ei; let base_op = &base_ei.base_op; let mut intersect = Intersect::new( - base_ei.to_query_vertex.clone(), base_ei.to_type, + base_ei.to_query_vertex.clone(), + base_ei.to_type, base_ei.alds.clone(), - base_op.out_subgraph.clone(), base_op.in_subgraph.clone(), + base_op.out_subgraph.clone(), + base_op.in_subgraph.clone(), base_op.out_qvertex_to_idx_map.clone(), ); let intersect_copy = intersect.clone(); - intersect.base_ei.base_op.prev = base_op.prev.as_ref().unwrap().copy(is_thread_safe).map(|op| Box::new(op)); + intersect.base_ei.base_op.prev = Some(Box::new( + base_op.prev.as_ref().unwrap().copy(is_thread_safe), + )); let prev = intersect.base_ei.base_op.prev.as_mut().unwrap().as_mut(); - *get_op_attr_as_mut!(prev,next) = Some(vec![Operator::EI(EI::Intersect(intersect_copy))]); - let last_repeated_vertex_idx = get_op_attr!(prev,last_repeated_vertex_idx); + *get_op_attr_as_mut!(prev, next) = Some(vec![Operator::EI(EI::Intersect(intersect_copy))]); + let last_repeated_vertex_idx = get_op_attr!(prev, last_repeated_vertex_idx); intersect.base_ei.init_caching(last_repeated_vertex_idx); - Some(Operator::EI(EI::Intersect(intersect))) + Operator::EI(EI::Intersect(intersect)) } fn is_same_as(&mut self, op: &mut Operator) -> bool { if let Operator::EI(EI::Intersect(intersect)) = op { - return - self.base_ei.caching_type == intersect.base_ei.caching_type && - self.get_alds_as_string() == intersect.get_alds_as_string() && - self.base_ei.base_op.in_subgraph.as_mut().unwrap().is_isomorphic_to(get_op_attr_as_mut!(op,in_subgraph).as_mut().unwrap().as_mut()) && - self.base_ei.base_op.out_subgraph.is_isomorphic_to(get_op_attr_as_mut!(op,out_subgraph).as_mut()) && - self.base_ei.base_op.prev.as_mut().unwrap().is_same_as(get_op_attr_as_mut!(op,prev).as_mut().unwrap()); + return self.base_ei.caching_type == intersect.base_ei.caching_type + && self.get_alds_as_string() == intersect.get_alds_as_string() + && self + .base_ei + .base_op + .in_subgraph + .as_mut() + .unwrap() + .is_isomorphic_to( + get_op_attr_as_mut!(op, in_subgraph) + .as_mut() + .unwrap() + .as_mut(), + ) + && self + .base_ei + .base_op + .out_subgraph + .is_isomorphic_to(get_op_attr_as_mut!(op, out_subgraph).as_mut()) + && self + .base_ei + .base_op + .prev + .as_mut() + .unwrap() + .is_same_as(get_op_attr_as_mut!(op, prev).as_mut().unwrap()); } false } @@ -146,4 +189,4 @@ impl CommonOperatorTrait for Intersect { fn get_num_out_tuples(&self) -> usize { self.base_ei.get_num_out_tuples() } -} \ No newline at end of file +} diff --git a/src/graph_impl/multi_graph/plan/operator/extend/mod.rs b/src/graph_impl/multi_graph/plan/operator/extend/mod.rs index eacb3cf0..f1c11a29 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/mod.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/mod.rs @@ -1,3 +1,3 @@ pub mod EI; pub mod extend; -pub mod intersect; \ No newline at end of file +pub mod intersect; diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/build.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/build.rs new file mode 100644 index 00000000..5dd3287e --- /dev/null +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/build.rs @@ -0,0 +1,105 @@ +use generic::{GraphTrait, GraphType, IdType}; +use graph_impl::multi_graph::catalog::query_graph::QueryGraph; +use graph_impl::multi_graph::plan::operator::hashjoin::hash_table::HashTable; +use graph_impl::multi_graph::plan::operator::operator::{ + BaseOperator, CommonOperatorTrait, Operator, +}; +use graph_impl::TypedStaticGraph; +use hashbrown::HashMap; +use std::hash::{BuildHasherDefault, Hash}; + +#[derive(Clone)] +pub struct Build { + pub base_op: BaseOperator, + hash_table: Option>, + pub probing_subgraph: Option>, + query_vertex_to_hash: String, + build_hash_idx: usize, + hashed_tuple_len: usize, +} + +impl Build { + pub fn new( + in_subgraph: Box, + query_vertex_to_hash: String, + build_hash_idx: usize, + ) -> Build { + let mut build = Build { + base_op: BaseOperator::empty(), + hash_table: None, + probing_subgraph: None, + query_vertex_to_hash: query_vertex_to_hash.clone(), + build_hash_idx, + hashed_tuple_len: 0, + }; + build.hashed_tuple_len = in_subgraph.get_num_qvertices() - 1; + build.base_op.out_tuple_len = in_subgraph.get_num_qvertices(); + build.base_op.in_subgraph = Some(in_subgraph); + build.base_op.name = "HASH ON (".to_owned() + &query_vertex_to_hash + ")"; + build + } +} + +impl CommonOperatorTrait for Build { + fn init( + &mut self, + probe_tuple: Vec, + graph: &TypedStaticGraph, + ) { + if self.base_op.probe_tuple.len() == 0 { + self.base_op.probe_tuple = probe_tuple; + self.hash_table.as_mut().map(|table| { + table.allocate_initial_memory(graph.node_count() + 1); + }); + } + } + + fn process_new_tuple(&mut self) { + let probe_tuple = self.base_op.probe_tuple.clone(); + self.hash_table.as_mut().map(|table| { + table.insert_tuple(probe_tuple); + }); + } + + fn execute(&mut self) { + self.base_op.execute() + } + + fn get_alds_as_string(&self) -> String { + self.base_op.get_alds_as_string() + } + + fn update_operator_name(&mut self, query_vertex_to_index_map: HashMap) { + self.base_op.update_operator_name(query_vertex_to_index_map) + } + + fn copy(&self, is_thread_safe: bool) -> Operator { + let mut build = Build::new( + self.base_op.in_subgraph.as_ref().unwrap().clone(), + self.query_vertex_to_hash.clone(), + self.build_hash_idx, + ); + build.base_op.prev = self.base_op.prev.as_ref().map(|prev| prev.clone()); + build.base_op.next = Some(vec![Operator::Build(build.clone())]); + build.probing_subgraph = self.probing_subgraph.clone(); + Operator::Build(build) + } + + fn is_same_as(&mut self, op: &mut Operator) -> bool { + if let Operator::Build(build) = op { + let base_op = &mut self.base_op; + let in_subgraph = base_op.in_subgraph.as_mut().map_or(false, |in_subgraph| { + in_subgraph.is_isomorphic_to(build.base_op.in_subgraph.as_mut().unwrap()) + }); + let prev = base_op.prev.as_mut().map_or(false, |prev| { + prev.is_same_as(build.base_op.prev.as_mut().unwrap().as_mut()) + }); + return in_subgraph && prev; + } + false + } + + fn get_num_out_tuples(&self) -> usize { + self.base_op.get_num_out_tuples() + } +} diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/hash_join.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/hash_join.rs new file mode 100644 index 00000000..b57cdb39 --- /dev/null +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/hash_join.rs @@ -0,0 +1,188 @@ +use generic::IdType; +use graph_impl::multi_graph::catalog::query_graph::QueryGraph; +use graph_impl::multi_graph::plan::operator::extend::EI::EI; +use graph_impl::multi_graph::plan::operator::hashjoin::build::Build; +use graph_impl::multi_graph::plan::operator::hashjoin::probe::{BaseProbe, Probe}; +use graph_impl::multi_graph::plan::operator::hashjoin::probe_cartesian::ProbeCartesian; +use graph_impl::multi_graph::plan::operator::hashjoin::probe_multi_vertices::{ + ProbeMultiVertices, PMV, +}; +use graph_impl::multi_graph::plan::operator::hashjoin::probe_multi_vertices_cartesian::ProbeMultiVerticesCartesian; +use graph_impl::multi_graph::plan::operator::operator::Operator; +use graph_impl::multi_graph::plan::operator::scan::scan::Scan; +use graph_impl::multi_graph::plan::operator::sink::sink::Sink; +use graph_impl::multi_graph::plan::query_plan::QueryPlan; +use hashbrown::HashMap; + +pub struct HashJoin {} + +impl HashJoin { + pub fn make( + out_subgraph: QueryGraph, + build_plan: QueryPlan, + probe_plan: QueryPlan, + ) -> QueryPlan { + QueryPlan::new_from_subplans(HashJoin::inner_make( + out_subgraph, + build_plan.shallow_copy().subplans, + probe_plan.shallow_copy().subplans, + )) + } + + fn inner_make( + out_subgraph: QueryGraph, + mut build_subplans: Vec>>, + mut probe_subplans: Vec>>, + ) -> Vec>> { + let mut pre_build = *build_subplans.pop().unwrap(); + let mut pre_probe = *probe_subplans.pop().unwrap(); + let join_qvertices: Vec = pre_build + .get_out_query_vertices() + .intersection(&pre_probe.get_out_query_vertices()) + .map(|x| x.clone()) + .collect(); + + let build_qvertex_to_idx_map = + get_op_attr_as_ref!(&pre_build, out_qvertex_to_idx_map).clone(); + let query_vertex_to_hash = &join_qvertices[0]; + let build_hash_idx = build_qvertex_to_idx_map + .get(query_vertex_to_hash) + .unwrap() + .clone(); + let mut build = Build::new( + get_op_attr_as_ref!(&pre_build, out_subgraph).clone(), + query_vertex_to_hash.clone(), + build_hash_idx.clone(), + ); + build.base_op.prev.replace(Box::new(pre_build.clone())); + get_op_attr_as_mut!(&mut pre_build, next).replace(vec![Operator::Build(build.clone())]); + build_subplans.push(Box::new(Operator::Build(build.clone()))); + + let out_subgraph_probe = get_op_attr_as_mut!(&mut pre_probe, out_subgraph).as_mut(); + let out_subgraph_build = get_op_attr_as_mut!(&mut pre_build, out_subgraph).as_mut(); + let mapping = out_subgraph_build.get_isomorphic_mapping_if_any(out_subgraph_probe); + let mut probe_qvertex_to_idx_map; + if let Some(map) = mapping { + probe_qvertex_to_idx_map = HashMap::new(); + build_qvertex_to_idx_map + .iter() + .for_each(|(query_vertex, &idx)| { + probe_qvertex_to_idx_map.insert( + query_vertex.clone(), + if idx < build_hash_idx { idx } else { idx - 1 }, + ); + }); + probe_qvertex_to_idx_map.insert( + map.get(&join_qvertices[0]).unwrap().clone(), + build_qvertex_to_idx_map.len() - 1, + ); + } else { + probe_qvertex_to_idx_map = + get_op_attr_as_ref!(&pre_probe, out_qvertex_to_idx_map).clone(); + } + let &probe_hash_idx = probe_qvertex_to_idx_map.get(query_vertex_to_hash).unwrap(); + let out_qvertex_to_idx_map = HashJoin::compute_out_qvertex_to_idx_map( + &join_qvertices, + &build_qvertex_to_idx_map, + &probe_qvertex_to_idx_map, + ); + let hashed_tuple_len = build_qvertex_to_idx_map.len() - 1; + let mut probe_indices = vec![0; join_qvertices.len() - 1]; + let mut build_indices = vec![0; join_qvertices.len() - 1]; + for (i, join_qvertex) in join_qvertices.iter().enumerate() { + probe_indices[i - 1] = probe_qvertex_to_idx_map.get(join_qvertex).unwrap().clone(); + let mut other_build_idx = build_qvertex_to_idx_map.get(join_qvertex).unwrap().clone(); + if build_hash_idx < other_build_idx { + other_build_idx -= 1; + } + build_indices[i - 1] = other_build_idx; + } + let mut probe; + let in_subgraph = get_op_attr_as_ref!(&pre_probe, out_subgraph) + .as_ref() + .clone(); + let out_tuple_len = get_op_attr!(&pre_probe, out_tuple_len); + if let Some(map) = mapping { + if probe_indices.len() == 0 { + probe = Operator::Probe(Probe::PC(ProbeCartesian::new( + out_subgraph, + in_subgraph, + join_qvertices, + probe_hash_idx, + hashed_tuple_len, + out_tuple_len, + out_qvertex_to_idx_map, + ))); + } else { + probe = Operator::Probe(Probe::PMV(PMV::PMVC(ProbeMultiVerticesCartesian::new( + out_subgraph, + in_subgraph, + join_qvertices, + probe_hash_idx, + probe_indices, + build_indices, + hashed_tuple_len, + out_tuple_len, + out_qvertex_to_idx_map, + )))); + } + } else { + if probe_indices.len() == 0 { + probe = Operator::Probe(Probe::BaseProbe(BaseProbe::new( + out_subgraph, + in_subgraph, + join_qvertices, + probe_hash_idx, + hashed_tuple_len, + out_tuple_len, + out_qvertex_to_idx_map, + ))); + } else { + probe = Operator::Probe(Probe::PMV(PMV::BasePMV(ProbeMultiVertices::new( + out_subgraph, + in_subgraph, + join_qvertices, + probe_hash_idx, + probe_indices, + build_indices, + hashed_tuple_len, + out_tuple_len, + out_qvertex_to_idx_map, + )))); + } + get_op_attr_as_mut!(&mut probe, prev).replace(Box::new(pre_probe.clone())); + let probe_clone = probe.clone(); + get_op_attr_as_mut!(&mut probe, next).replace(vec![probe_clone]); + let last_index = probe_subplans.len() - 1; + probe_subplans[last_index] = Box::new(probe.clone()); + } + build.probing_subgraph = get_op_attr_as_ref!(&probe, in_subgraph).clone(); + + let mut subplans = build_subplans.clone(); + if let None = mapping { + subplans.append(&mut probe_subplans); + } else { + subplans.push(Box::new(probe)); + } + subplans + } + + pub fn compute_out_qvertex_to_idx_map( + join_qvertices: &Vec, + build_qvertex_to_idx_map: &HashMap, + probe_qvertex_to_idx_map: &HashMap, + ) -> HashMap { + let mut out_qvertices_to_idx_map = probe_qvertex_to_idx_map.clone(); + let mut build_qvertices = vec![String::from(""); build_qvertex_to_idx_map.len()]; + for (build_qvertex, idx) in build_qvertex_to_idx_map { + build_qvertices[idx.clone()] = build_qvertex.clone(); + } + for build_qvertex in build_qvertices { + if join_qvertices.contains(&build_qvertex) { + continue; + } + out_qvertices_to_idx_map.insert(build_qvertex, out_qvertices_to_idx_map.len()); + } + out_qvertices_to_idx_map + } +} diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/hash_table.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/hash_table.rs new file mode 100644 index 00000000..cd70410c --- /dev/null +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/hash_table.rs @@ -0,0 +1,147 @@ +use fxhash::hash; +use generic::IdType; +use itertools::Itertools; + +#[derive(Clone)] +pub struct BlockInfo { + pub block: Vec, + pub start_offset: usize, + pub end_offset: usize, +} + +impl BlockInfo { + pub fn empty() -> BlockInfo { + BlockInfo { + block: vec![], + start_offset: 0, + end_offset: 0, + } + } +} + +#[derive(Clone)] +pub struct HashTable { + blocks: Vec>, + extra_blocks: Vec>, + block_ids_and_chunk_offsets: Vec>, + pub(crate) num_chunks: Vec, + initial_num_blocks: usize, + num_tuples_per_chunk: usize, + num_chunks_per_block: usize, + initial_num_chunks_per_vertex: usize, + block_sz: usize, + chunk_sz: usize, + next_global_block_id: usize, + next_global_chunk_offset: usize, + build_hash_idx: usize, + build_tuple_len: usize, + hashed_tuple_len: usize, +} + +impl HashTable { + pub fn new(build_hash_idx: usize, hashed_tuple_len: usize) -> HashTable { + let mut hash_table = HashTable { + blocks: vec![], + extra_blocks: vec![], + block_ids_and_chunk_offsets: vec![], + num_chunks: vec![], + initial_num_blocks: 1000, + num_tuples_per_chunk: 64, + num_chunks_per_block: 8000, + initial_num_chunks_per_vertex: 6, + block_sz: 0, + chunk_sz: 0, + next_global_block_id: 0, + next_global_chunk_offset: 0, + build_hash_idx, + build_tuple_len: hashed_tuple_len + 1, + hashed_tuple_len, + }; + hash_table.chunk_sz = hash_table.num_tuples_per_chunk * hash_table.hashed_tuple_len; + hash_table.block_sz = hash_table.chunk_sz * hash_table.num_chunks_per_block; + hash_table + } + + pub fn allocate_initial_memory(&mut self, highest_vertex_id: usize) { + self.blocks = vec![vec![Id::new(0); self.block_sz]; self.initial_num_blocks]; + self.extra_blocks = vec![vec![]; self.initial_num_blocks]; + self.block_ids_and_chunk_offsets = + vec![vec![0; self.initial_num_chunks_per_vertex * 3]; highest_vertex_id + 1]; + self.num_chunks = vec![0; highest_vertex_id + 1]; + } + pub fn insert_tuple(&mut self, build_tuple: Vec) { + let hash_vertex = build_tuple[self.build_hash_idx].id(); + let mut last_chunk_idx = self.num_chunks[hash_vertex]; + if 0 == last_chunk_idx { + self.num_chunks[hash_vertex] += 1; + self.update_block_ids_and_global_and_chunk_offset(hash_vertex); + } + last_chunk_idx = 3 * (self.num_chunks[hash_vertex] - 1); + let block_id = self.block_ids_and_chunk_offsets[hash_vertex][last_chunk_idx]; + let start_offset = self.block_ids_and_chunk_offsets[hash_vertex][last_chunk_idx + 1]; + let mut end_offset = self.block_ids_and_chunk_offsets[hash_vertex][last_chunk_idx + 2]; + let block = if block_id < self.initial_num_blocks { + &mut self.blocks[block_id] + } else { + &mut self.extra_blocks[block_id - self.initial_num_blocks] + }; + for i in 0..self.build_tuple_len { + if i != self.build_hash_idx { + block[end_offset] = build_tuple[i]; + end_offset += 1; + } + } + self.block_ids_and_chunk_offsets[hash_vertex][last_chunk_idx + 2] = end_offset; + if self.chunk_sz <= end_offset - start_offset + self.hashed_tuple_len { + self.num_chunks[hash_vertex] += 1; + self.resize_block_ids_and_global_and_chunk_offset(hash_vertex); + self.update_block_ids_and_global_and_chunk_offset(hash_vertex); + } + } + + pub fn get_block_and_offsets( + &self, + hash_vertex: usize, + chunk_idx: usize, + block_info: &mut BlockInfo, + ) { + let block_id = self.block_ids_and_chunk_offsets[hash_vertex][chunk_idx * 3]; + block_info.start_offset = self.block_ids_and_chunk_offsets[hash_vertex][chunk_idx * 3 + 1]; + block_info.end_offset = self.block_ids_and_chunk_offsets[hash_vertex][chunk_idx * 3 + 2]; + block_info.block = if block_id < self.initial_num_blocks { + self.blocks[block_id].clone() + } else { + self.extra_blocks[block_id - self.initial_num_blocks].clone() + }; + } + + pub fn resize_block_ids_and_global_and_chunk_offset(&mut self, hash_vertex: usize) { + if self.num_chunks[hash_vertex] + 1 + > (self.block_ids_and_chunk_offsets[hash_vertex].len() / 3) + { + let mut new_chunk_block_id_offset_array = + vec![0; (self.num_chunks[hash_vertex] + 2) * 3]; + self.block_ids_and_chunk_offsets[hash_vertex] + .iter() + .enumerate() + .foreach(|(i, x)| new_chunk_block_id_offset_array[i] = x.clone()); + self.block_ids_and_chunk_offsets[hash_vertex] = new_chunk_block_id_offset_array; + } + } + pub fn update_block_ids_and_global_and_chunk_offset(&mut self, hash_vertex: usize) { + let last_chunk_idx = (self.num_chunks[hash_vertex] - 1) * 3; + self.block_ids_and_chunk_offsets[hash_vertex][last_chunk_idx] = self.next_global_block_id; + self.block_ids_and_chunk_offsets[hash_vertex][last_chunk_idx + 1] = + self.next_global_chunk_offset; + self.block_ids_and_chunk_offsets[hash_vertex][last_chunk_idx + 2] = + self.next_global_chunk_offset; + self.next_global_chunk_offset += self.chunk_sz; + if self.next_global_chunk_offset == self.block_sz { + self.next_global_block_id += 1; + if self.next_global_block_id >= self.initial_num_blocks { + self.extra_blocks.push(vec![Id::new(0); self.block_sz]); + } + self.next_global_chunk_offset = 0; + } + } +} diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/mod.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/mod.rs new file mode 100644 index 00000000..20754c7a --- /dev/null +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/mod.rs @@ -0,0 +1,7 @@ +pub mod build; +pub mod hash_join; +pub mod hash_table; +pub mod probe; +pub mod probe_cartesian; +pub mod probe_multi_vertices; +pub mod probe_multi_vertices_cartesian; diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe.rs new file mode 100644 index 00000000..75b91567 --- /dev/null +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe.rs @@ -0,0 +1,236 @@ +use generic::{GraphType, IdType}; +use graph_impl::multi_graph::catalog::query_graph::QueryGraph; +use graph_impl::multi_graph::plan::operator::hashjoin::hash_table::{BlockInfo, HashTable}; +use graph_impl::multi_graph::plan::operator::operator::{ + BaseOperator, CommonOperatorTrait, Operator, +}; + +use graph_impl::multi_graph::plan::operator::hashjoin::probe_cartesian::ProbeCartesian; +use graph_impl::multi_graph::plan::operator::hashjoin::probe_multi_vertices::PMV; +use graph_impl::TypedStaticGraph; +use hashbrown::HashMap; +use std::hash::{BuildHasherDefault, Hash}; + +#[derive(Clone)] +pub enum Probe { + BaseProbe(BaseProbe), + PC(ProbeCartesian), + PMV(PMV), +} + +#[derive(Clone)] +pub struct BaseProbe { + pub base_op: BaseOperator, + pub hash_tables: Vec>, + pub join_qvertices: Vec, + pub probe_hash_idx: usize, + pub hashed_tuple_len: usize, + pub probe_tuple_len: usize, + pub block_info: BlockInfo, +} + +impl BaseProbe { + pub fn new( + out_subgraph: QueryGraph, + in_subgraph: QueryGraph, + join_qvertices: Vec, + probe_hash_idx: usize, + hashed_tuple_len: usize, + probe_tuple_len: usize, + out_qvertex_to_idx_map: HashMap, + ) -> BaseProbe { + let mut probe = BaseProbe { + base_op: BaseOperator::new(Box::new(out_subgraph), Some(Box::new(in_subgraph))), + hash_tables: vec![], + join_qvertices, + probe_hash_idx, + hashed_tuple_len, + probe_tuple_len, + block_info: BlockInfo::empty(), + }; + probe.base_op.out_tuple_len = out_qvertex_to_idx_map.len(); + probe.base_op.out_qvertex_to_idx_map = out_qvertex_to_idx_map; + probe.base_op.name = "PROBE ON (".to_owned() + &probe.join_qvertices[0] + ")"; + probe + } +} + +impl CommonOperatorTrait for BaseProbe { + fn init( + &mut self, + probe_tuple: Vec, + graph: &TypedStaticGraph, + ) { + if self.base_op.probe_tuple.len() == 0 { + self.base_op.probe_tuple = probe_tuple.clone(); + self.block_info = BlockInfo::empty(); + self.base_op.next.as_mut().map(|next| { + next.iter_mut() + .for_each(|next_op| next_op.init(probe_tuple.clone(), graph)) + }); + } + } + + fn process_new_tuple(&mut self) { + let hash_vertex = self.base_op.probe_tuple[self.probe_hash_idx].id(); + for mut hash_table in &mut self.hash_tables { + let last_chunk_idx = hash_table.num_chunks[hash_vertex]; + let mut prev_first_item = -1i32; + for chunk_idx in 0..last_chunk_idx { + hash_table.get_block_and_offsets(hash_vertex, chunk_idx, &mut self.block_info); + let mut offset = self.block_info.start_offset; + while offset < self.block_info.end_offset { + self.base_op.num_out_tuples += 1; + if self.hashed_tuple_len == 2 { + let first_item = self.block_info.block[offset]; + offset += 1; + if prev_first_item != first_item.id() as i32 { + self.base_op.probe_tuple[self.probe_tuple_len] = first_item; + prev_first_item = first_item.id() as i32; + } + self.base_op.probe_tuple[self.probe_tuple_len + 1] = + self.block_info.block[offset]; + offset += 1; + } else { + for k in 0..self.hashed_tuple_len { + self.base_op.probe_tuple[self.probe_tuple_len + k] = + self.block_info.block[offset]; + offset += 1; + } + } + self.base_op + .next + .as_mut() + .map(|next| next.get_mut(0).map(|next_op| next_op.process_new_tuple())); + } + } + } + } + + fn execute(&mut self) { + self.base_op.execute() + } + + fn get_alds_as_string(&self) -> String { + self.base_op.get_alds_as_string() + } + + fn update_operator_name(&mut self, query_vertex_to_index_map: HashMap) { + self.base_op.update_operator_name(query_vertex_to_index_map) + } + + fn copy(&self, is_thread_safe: bool) -> Operator { + let op = &self.base_op; + let mut probe = BaseProbe::new( + op.out_subgraph.as_ref().clone(), + op.in_subgraph.as_ref().unwrap().as_ref().clone(), + self.join_qvertices.clone(), + self.probe_hash_idx, + self.hashed_tuple_len, + self.probe_tuple_len, + op.out_qvertex_to_idx_map.clone(), + ); + probe.base_op.prev = op + .prev + .as_ref() + .map(|prev| Box::new(prev.copy(is_thread_safe))); + probe + .base_op + .next + .replace(vec![Operator::Probe(Probe::BaseProbe(probe.clone()))]); + Operator::Probe(Probe::BaseProbe(probe)) + } + + fn is_same_as(&mut self, op: &mut Operator) -> bool { + if let Operator::Probe(Probe::BaseProbe(probe)) = op { + let self_op = &mut self.base_op; + let other_op = &mut probe.base_op; + let in_subgraph = self_op.in_subgraph.as_mut().map_or(false, |in_subgraph| { + in_subgraph.is_isomorphic_to(other_op.in_subgraph.as_mut().unwrap()) + }); + let out_subgraph = self_op + .out_subgraph + .as_mut() + .is_isomorphic_to(other_op.out_subgraph.as_mut()); + let prev = self_op.prev.as_mut().map_or(false, |prev| { + prev.is_same_as(other_op.prev.as_mut().unwrap().as_mut()) + }); + return in_subgraph && out_subgraph && prev; + } + false + } + + fn get_num_out_tuples(&self) -> usize { + self.base_op.get_num_out_tuples() + } +} + +impl CommonOperatorTrait for Probe { + fn init( + &mut self, + probe_tuple: Vec, + graph: &TypedStaticGraph, + ) { + match self { + Probe::BaseProbe(base) => base.init(probe_tuple, graph), + Probe::PC(pc) => pc.init(probe_tuple, graph), + Probe::PMV(pmv) => pmv.init(probe_tuple, graph), + } + } + + fn process_new_tuple(&mut self) { + match self { + Probe::BaseProbe(base) => base.process_new_tuple(), + Probe::PC(pc) => pc.process_new_tuple(), + Probe::PMV(pmv) => pmv.process_new_tuple(), + } + } + + fn execute(&mut self) { + match self { + Probe::BaseProbe(base) => base.execute(), + Probe::PC(pc) => pc.execute(), + Probe::PMV(pmv) => pmv.execute(), + } + } + + fn get_alds_as_string(&self) -> String { + match self { + Probe::BaseProbe(base) => base.get_alds_as_string(), + Probe::PC(pc) => pc.get_alds_as_string(), + Probe::PMV(pmv) => pmv.get_alds_as_string(), + } + } + + fn update_operator_name(&mut self, query_vertex_to_index_map: HashMap) { + match self { + Probe::BaseProbe(base) => base.update_operator_name(query_vertex_to_index_map), + Probe::PC(pc) => pc.update_operator_name(query_vertex_to_index_map), + Probe::PMV(pmv) => pmv.update_operator_name(query_vertex_to_index_map), + } + } + + fn copy(&self, is_thread_safe: bool) -> Operator { + match self { + Probe::BaseProbe(base) => base.copy(is_thread_safe), + Probe::PC(pc) => pc.copy(is_thread_safe), + Probe::PMV(pmv) => pmv.copy(is_thread_safe), + } + } + + fn is_same_as(&mut self, op: &mut Operator) -> bool { + match self { + Probe::BaseProbe(base) => base.is_same_as(op), + Probe::PC(pc) => pc.is_same_as(op), + Probe::PMV(pmv) => pmv.is_same_as(op), + } + } + + fn get_num_out_tuples(&self) -> usize { + match self { + Probe::BaseProbe(base) => base.get_num_out_tuples(), + Probe::PC(pc) => pc.get_num_out_tuples(), + Probe::PMV(pmv) => pmv.get_num_out_tuples(), + } + } +} diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_cartesian.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_cartesian.rs new file mode 100644 index 00000000..12e844ee --- /dev/null +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_cartesian.rs @@ -0,0 +1,143 @@ +use generic::{GraphTrait, GraphType, IdType}; +use graph_impl::multi_graph::catalog::query_graph::QueryGraph; +use graph_impl::multi_graph::plan::operator::hashjoin::hash_table::BlockInfo; +use graph_impl::multi_graph::plan::operator::hashjoin::probe::{BaseProbe, Probe}; +use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; +use graph_impl::TypedStaticGraph; +use hashbrown::HashMap; +use itertools::join; +use std::hash::{BuildHasherDefault, Hash}; + +#[derive(Clone)] +pub struct ProbeCartesian { + pub base_probe: BaseProbe, + other_block_info: BlockInfo, + highest_vertex_id: usize, +} + +impl ProbeCartesian { + pub fn new( + out_subgraph: QueryGraph, + in_subgraph: QueryGraph, + join_qvertices: Vec, + probe_hash_idx: usize, + hashed_tuple_len: usize, + probe_tuple_len: usize, + out_qvertex_to_idx_map: HashMap, + ) -> ProbeCartesian { + let mut pc = ProbeCartesian { + base_probe: BaseProbe::new( + out_subgraph, + in_subgraph, + join_qvertices, + probe_hash_idx, + hashed_tuple_len, + probe_tuple_len, + out_qvertex_to_idx_map, + ), + other_block_info: BlockInfo::empty(), + highest_vertex_id: 0, + }; + pc.base_probe.base_op.name = "CARTESIAN ".to_owned() + &pc.base_probe.base_op.name; + pc + } +} + +impl CommonOperatorTrait for ProbeCartesian { + fn init( + &mut self, + probe_tuple: Vec, + graph: &TypedStaticGraph, + ) { + if self.base_probe.base_op.probe_tuple.len() == 0 { + self.highest_vertex_id = graph.node_count() + 1; + self.other_block_info = BlockInfo::empty(); + } + self.base_probe.init(probe_tuple, graph); + } + + fn process_new_tuple(&mut self) { + for a_hash_vertex in 0..self.highest_vertex_id { + let base_probe = &mut self.base_probe; + base_probe.base_op.probe_tuple[base_probe.hashed_tuple_len] = Id::new(a_hash_vertex); + for hash_table in base_probe.hash_tables.clone() { + let a_last_chunk_idx = hash_table.num_chunks[a_hash_vertex]; + let mut a_prev_first_vertex = -1i32; + for a_chunk_idx in 0..a_last_chunk_idx { + hash_table.get_block_and_offsets( + a_hash_vertex, + a_chunk_idx, + &mut self.other_block_info, + ); + let mut an_offset = self.other_block_info.start_offset; + while an_offset < self.other_block_info.end_offset { + if base_probe.hashed_tuple_len == 2 { + let first_vertex = self.other_block_info.block[an_offset]; + an_offset += 1; + if a_prev_first_vertex != first_vertex.id() as i32 { + base_probe.base_op.probe_tuple[0] = first_vertex; + a_prev_first_vertex = first_vertex.id() as i32; + } + base_probe.base_op.probe_tuple[1] = + self.other_block_info.block[an_offset]; + an_offset += 1; + } else { + for k in 0..base_probe.hashed_tuple_len { + base_probe.base_op.probe_tuple[k] = + self.other_block_info.block[an_offset]; + an_offset += 1; + } + } + base_probe.process_new_tuple(); + } + } + } + } + } + + fn execute(&mut self) { + self.base_probe.execute() + } + + fn get_alds_as_string(&self) -> String { + self.base_probe.get_alds_as_string() + } + + fn update_operator_name(&mut self, query_vertex_to_index_map: HashMap) { + self.base_probe + .update_operator_name(query_vertex_to_index_map) + } + + fn copy(&self, is_thread_safe: bool) -> Operator { + let op = &self.base_probe.base_op; + Operator::Probe(Probe::PC(ProbeCartesian::new( + op.out_subgraph.as_ref().clone(), + op.in_subgraph.as_ref().unwrap().as_ref().clone(), + self.base_probe.join_qvertices.clone(), + self.base_probe.probe_hash_idx, + self.base_probe.hashed_tuple_len, + self.base_probe.probe_tuple_len, + op.out_qvertex_to_idx_map.clone(), + ))) + } + + fn is_same_as(&mut self, op: &mut Operator) -> bool { + if let Operator::Probe(Probe::PC(pc)) = op { + let self_op = &mut self.base_probe.base_op; + let other_op = &mut pc.base_probe.base_op; + let in_subgraph = self_op.in_subgraph.as_mut().map_or(false, |in_subgraph| { + in_subgraph.is_isomorphic_to(other_op.in_subgraph.as_mut().unwrap()) + }); + let out_subgraph = self_op + .out_subgraph + .as_mut() + .is_isomorphic_to(other_op.out_subgraph.as_mut()); + return in_subgraph && out_subgraph; + } + false + } + + fn get_num_out_tuples(&self) -> usize { + self.base_probe.get_num_out_tuples() + } +} diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices.rs new file mode 100644 index 00000000..6055d3ec --- /dev/null +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices.rs @@ -0,0 +1,258 @@ +use generic::{GraphType, IdType}; +use graph_impl::multi_graph::catalog::query_graph::QueryGraph; +use graph_impl::multi_graph::plan::operator::hashjoin::probe::{BaseProbe, Probe}; +use graph_impl::multi_graph::plan::operator::hashjoin::probe_multi_vertices_cartesian::ProbeMultiVerticesCartesian; +use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; +use graph_impl::TypedStaticGraph; +use hashbrown::HashMap; +use itertools::Itertools; +use std::hash::{BuildHasherDefault, Hash}; + +#[derive(Clone)] +pub enum PMV { + BasePMV(ProbeMultiVertices), + PMVC(ProbeMultiVerticesCartesian), +} + +#[derive(Clone)] +pub struct ProbeMultiVertices { + pub base_probe: BaseProbe, + pub probe_indices: Vec, + pub build_indices: Vec, +} + +impl ProbeMultiVertices { + pub fn new( + out_subgraph: QueryGraph, + in_subgraph: QueryGraph, + join_qvertices: Vec, + probe_hash_idx: usize, + probe_indices: Vec, + build_indices: Vec, + hashed_tuple_len: usize, + probe_tuple_len: usize, + out_qvertex_to_idx_map: HashMap, + ) -> ProbeMultiVertices { + let mut name = "PROBE ON ".to_owned(); + if 1 == join_qvertices.len() { + name = name + "(" + join_qvertices.get(0).unwrap() + ")"; + } else { + for i in 0..join_qvertices.len() { + if i > 0 && i < join_qvertices.len() - 1 { + name += ", "; + } + if i == join_qvertices.len() - 1 { + name += " & " + } + name = name + "(" + &join_qvertices.get(i).unwrap() + ")"; + } + } + let mut pmv = ProbeMultiVertices { + base_probe: BaseProbe::new( + out_subgraph, + in_subgraph, + join_qvertices, + probe_hash_idx, + hashed_tuple_len, + probe_tuple_len, + out_qvertex_to_idx_map, + ), + probe_indices, + build_indices, + }; + pmv.base_probe.base_op.name = name; + pmv + } +} + +impl CommonOperatorTrait for ProbeMultiVertices { + fn init( + &mut self, + probe_tuple: Vec, + graph: &TypedStaticGraph, + ) { + self.base_probe.init(probe_tuple, graph); + } + + fn process_new_tuple(&mut self) { + let hash_vertex = self.base_probe.base_op.probe_tuple[self.base_probe.probe_hash_idx].id(); + for mut hash_table in self.base_probe.hash_tables.clone() { + let last_chunk_idx = hash_table.num_chunks[hash_vertex]; + for chunk_idx in 0..last_chunk_idx { + hash_table.get_block_and_offsets( + hash_vertex, + chunk_idx, + &mut self.base_probe.block_info, + ); + let offset = self.base_probe.block_info.start_offset; + (self.base_probe.block_info.start_offset..self.base_probe.block_info.end_offset) + .step(self.base_probe.hashed_tuple_len) + .map(|offset| { + let mut flag = true; + for i in 0..self.probe_indices.len() { + if self.base_probe.base_op.probe_tuple[self.probe_indices[i]] + != self.base_probe.block_info.block[offset + self.build_indices[i]] + { + flag = false; + break; + } + } + if flag { + self.base_probe.base_op.num_out_tuples += 1; + let mut out = 0; + for k in 0..self.base_probe.hashed_tuple_len { + let mut copy = true; + for build_idx in &self.build_indices { + if k == build_idx.clone() { + copy = false; + break; + } + } + if copy { + self.base_probe.base_op.probe_tuple + [self.base_probe.probe_tuple_len + out] = + self.base_probe.block_info.block[offset + k]; + out += 1; + } + } + self.base_probe.base_op.next.as_mut().map(|next| { + next.get_mut(0).map(|next_op| next_op.process_new_tuple()) + }); + } + }); + } + } + } + + fn execute(&mut self) { + self.base_probe.execute(); + } + + fn get_alds_as_string(&self) -> String { + self.base_probe.get_alds_as_string() + } + + fn update_operator_name(&mut self, query_vertex_to_index_map: HashMap) { + self.base_probe + .update_operator_name(query_vertex_to_index_map); + } + + fn copy(&self, is_thread_safe: bool) -> Operator { + let mut probe = ProbeMultiVertices::new( + self.base_probe.base_op.out_subgraph.as_ref().clone(), + self.base_probe + .base_op + .in_subgraph + .as_ref() + .unwrap() + .as_ref() + .clone(), + self.base_probe.join_qvertices.clone(), + self.base_probe.probe_hash_idx, + self.probe_indices.clone(), + self.build_indices.clone(), + self.base_probe.hashed_tuple_len, + self.base_probe.probe_tuple_len, + self.base_probe.base_op.out_qvertex_to_idx_map.clone(), + ); + probe.base_probe.base_op.prev = self + .base_probe + .base_op + .prev + .as_ref() + .map(|prev| Box::new(prev.copy(is_thread_safe))); + probe + .base_probe + .base_op + .next + .replace(vec![Operator::Probe(Probe::PMV(PMV::BasePMV( + probe.clone(), + )))]); + Operator::Probe(Probe::PMV(PMV::BasePMV(probe))) + } + + fn is_same_as(&mut self, op: &mut Operator) -> bool { + if let Operator::Probe(Probe::PMV(PMV::BasePMV(probe))) = op { + let self_op = &mut self.base_probe.base_op; + let other_op = &mut probe.base_probe.base_op; + let in_subgraph = self_op.in_subgraph.as_mut().map_or(false, |in_subgraph| { + in_subgraph.is_isomorphic_to(other_op.in_subgraph.as_mut().unwrap()) + }); + let out_subgraph = self_op + .out_subgraph + .as_mut() + .is_isomorphic_to(other_op.out_subgraph.as_mut()); + let prev = self_op.prev.as_mut().map_or(false, |prev| { + prev.is_same_as(other_op.prev.as_mut().unwrap().as_mut()) + }); + return in_subgraph && out_subgraph && prev; + } + false + } + + fn get_num_out_tuples(&self) -> usize { + self.base_probe.get_num_out_tuples() + } +} + +impl CommonOperatorTrait for PMV { + fn init( + &mut self, + probe_tuple: Vec, + graph: &TypedStaticGraph, + ) { + match self { + PMV::BasePMV(base) => base.init(probe_tuple, graph), + PMV::PMVC(pmvc) => pmvc.init(probe_tuple, graph), + } + } + + fn process_new_tuple(&mut self) { + match self { + PMV::BasePMV(base) => base.process_new_tuple(), + PMV::PMVC(pmvc) => pmvc.process_new_tuple(), + } + } + + fn execute(&mut self) { + match self { + PMV::BasePMV(base) => base.execute(), + PMV::PMVC(pmvc) => pmvc.execute(), + } + } + + fn get_alds_as_string(&self) -> String { + match self { + PMV::BasePMV(base) => base.get_alds_as_string(), + PMV::PMVC(pmvc) => pmvc.get_alds_as_string(), + } + } + + fn update_operator_name(&mut self, query_vertex_to_index_map: HashMap) { + match self { + PMV::BasePMV(base) => base.update_operator_name(query_vertex_to_index_map), + PMV::PMVC(pmvc) => pmvc.update_operator_name(query_vertex_to_index_map), + } + } + + fn copy(&self, is_thread_safe: bool) -> Operator { + match self { + PMV::BasePMV(base) => base.copy(is_thread_safe), + PMV::PMVC(pmvc) => pmvc.copy(is_thread_safe), + } + } + + fn is_same_as(&mut self, op: &mut Operator) -> bool { + match self { + PMV::BasePMV(base) => base.is_same_as(op), + PMV::PMVC(pmvc) => pmvc.is_same_as(op), + } + } + + fn get_num_out_tuples(&self) -> usize { + match self { + PMV::BasePMV(base) => base.get_num_out_tuples(), + PMV::PMVC(pmvc) => pmvc.get_num_out_tuples(), + } + } +} diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices_cartesian.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices_cartesian.rs new file mode 100644 index 00000000..66760b86 --- /dev/null +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices_cartesian.rs @@ -0,0 +1,152 @@ +use generic::{GraphTrait, GraphType, IdType}; +use graph_impl::multi_graph::catalog::query_graph::QueryGraph; +use graph_impl::multi_graph::plan::operator::hashjoin::hash_table::BlockInfo; +use graph_impl::multi_graph::plan::operator::hashjoin::probe::Probe; +use graph_impl::multi_graph::plan::operator::hashjoin::probe_multi_vertices::{ + ProbeMultiVertices, PMV, +}; +use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; +use graph_impl::TypedStaticGraph; +use hashbrown::HashMap; +use std::hash::{BuildHasherDefault, Hash}; + +#[derive(Clone)] +pub struct ProbeMultiVerticesCartesian { + pub base_pmv: ProbeMultiVertices, + other_block_info: BlockInfo, + highest_vertex_id: usize, +} + +impl ProbeMultiVerticesCartesian { + pub fn new( + out_subgraph: QueryGraph, + in_subgraph: QueryGraph, + join_qvertices: Vec, + probe_hash_idx: usize, + probe_indices: Vec, + build_indices: Vec, + hashed_tuple_len: usize, + probe_tuple_len: usize, + out_qvertex_to_idx_map: HashMap, + ) -> ProbeMultiVerticesCartesian { + let mut pmvc = ProbeMultiVerticesCartesian { + base_pmv: ProbeMultiVertices::new( + out_subgraph, + in_subgraph, + join_qvertices, + probe_hash_idx, + probe_indices, + build_indices, + hashed_tuple_len, + probe_tuple_len, + out_qvertex_to_idx_map, + ), + other_block_info: BlockInfo::empty(), + highest_vertex_id: 0, + }; + pmvc.base_pmv.base_probe.base_op.name = + "CARTESIAN ".to_owned() + &pmvc.base_pmv.base_probe.base_op.name; + pmvc + } +} + +impl CommonOperatorTrait for ProbeMultiVerticesCartesian { + fn init( + &mut self, + probe_tuple: Vec, + graph: &TypedStaticGraph, + ) { + if self.base_pmv.base_probe.base_op.probe_tuple.len() == 0 { + self.highest_vertex_id = graph.node_count() + 1; + self.other_block_info = BlockInfo::empty(); + } + self.base_pmv.init(probe_tuple, graph); + } + + fn process_new_tuple(&mut self) { + for a_hash_vertex in 0..self.highest_vertex_id { + self.base_pmv.base_probe.base_op.probe_tuple + [self.base_pmv.base_probe.hashed_tuple_len] = Id::new(a_hash_vertex); + for hash_table in self.base_pmv.base_probe.hash_tables.clone() { + let a_last_chunk_idx = hash_table.num_chunks[a_hash_vertex]; + let mut a_prev_first_vertex = -1; + for a_chunk_idx in 0..a_last_chunk_idx { + hash_table.get_block_and_offsets( + a_hash_vertex, + a_chunk_idx, + &mut self.other_block_info, + ); + let mut an_offset = self.other_block_info.start_offset; + while an_offset < self.other_block_info.end_offset { + if self.base_pmv.base_probe.hashed_tuple_len == 2 { + let first_vertex = self.other_block_info.block[an_offset]; + an_offset += 1; + if a_prev_first_vertex != first_vertex.id() as i32 { + self.base_pmv.base_probe.base_op.probe_tuple[0] = first_vertex; + a_prev_first_vertex = first_vertex.id() as i32; + } + self.base_pmv.base_probe.base_op.probe_tuple[1] = + self.other_block_info.block[an_offset]; + an_offset += 1; + } else { + for k in 0..self.base_pmv.base_probe.hashed_tuple_len { + self.base_pmv.base_probe.base_op.probe_tuple[k] = + self.other_block_info.block[an_offset]; + an_offset += 1; + } + } + self.base_pmv.process_new_tuple(); + } + } + } + } + } + + fn execute(&mut self) { + self.base_pmv.execute() + } + + fn get_alds_as_string(&self) -> String { + self.base_pmv.get_alds_as_string() + } + + fn update_operator_name(&mut self, query_vertex_to_index_map: HashMap) { + self.base_pmv + .update_operator_name(query_vertex_to_index_map) + } + + fn copy(&self, is_thread_safe: bool) -> Operator { + let op = &self.base_pmv.base_probe.base_op; + Operator::Probe(Probe::PMV(PMV::PMVC(ProbeMultiVerticesCartesian::new( + op.out_subgraph.as_ref().clone(), + op.in_subgraph.as_ref().unwrap().as_ref().clone(), + self.base_pmv.base_probe.join_qvertices.clone(), + self.base_pmv.base_probe.probe_hash_idx, + self.base_pmv.probe_indices.clone(), + self.base_pmv.build_indices.clone(), + self.base_pmv.base_probe.hashed_tuple_len, + self.base_pmv.base_probe.probe_tuple_len, + op.out_qvertex_to_idx_map.clone(), + )))) + } + + fn is_same_as(&mut self, op: &mut Operator) -> bool { + if let Operator::Probe(Probe::PMV(PMV::PMVC(pc))) = op { + let self_op = &mut self.base_pmv.base_probe.base_op; + let other_op = &mut pc.base_pmv.base_probe.base_op; + let in_subgraph = self_op.in_subgraph.as_mut().map_or(false, |in_subgraph| { + in_subgraph.is_isomorphic_to(other_op.in_subgraph.as_mut().unwrap()) + }); + let out_subgraph = self_op + .out_subgraph + .as_mut() + .is_isomorphic_to(other_op.out_subgraph.as_mut()); + return in_subgraph && out_subgraph; + } + false + } + + fn get_num_out_tuples(&self) -> usize { + self.base_pmv.get_num_out_tuples() + } +} diff --git a/src/graph_impl/multi_graph/plan/operator/mod.rs b/src/graph_impl/multi_graph/plan/operator/mod.rs index 7d876e50..dc9d3bb9 100644 --- a/src/graph_impl/multi_graph/plan/operator/mod.rs +++ b/src/graph_impl/multi_graph/plan/operator/mod.rs @@ -2,10 +2,10 @@ macro_rules! get_sink_as_mut { ($item:expr) => { match $item { - Sink::BaseSink(base)=>base, - Sink::SinkCopy(base)=> &mut base.base_sink, - Sink::SinkPrint(base)=> &mut base.base_sink, - Sink::SinkLimit(base)=> &mut base.base_sink, + Sink::BaseSink(base) => base, + Sink::SinkCopy(base) => &mut base.base_sink, + Sink::SinkPrint(base) => &mut base.base_sink, + Sink::SinkLimit(base) => &mut base.base_sink, } }; } @@ -14,10 +14,10 @@ macro_rules! get_sink_as_mut { macro_rules! get_sink_as_ref { ($item:expr) => { match $item { - Sink::BaseSink(base)=>base, - Sink::SinkCopy(base)=> &base.base_sink, - Sink::SinkPrint(base)=> &base.base_sink, - Sink::SinkLimit(base)=> &base.base_sink, + Sink::BaseSink(base) => base, + Sink::SinkCopy(base) => &base.base_sink, + Sink::SinkPrint(base) => &base.base_sink, + Sink::SinkLimit(base) => &base.base_sink, } }; } @@ -26,9 +26,9 @@ macro_rules! get_sink_as_ref { macro_rules! get_ei_as_mut { ($item:expr) => { match $item { - EI::Base(base)=>base, - EI::Extend(base)=> &mut base.base_ei, - EI::Intersect(base)=> &mut base.base_ei, + EI::Base(base) => base, + EI::Extend(base) => &mut base.base_ei, + EI::Intersect(base) => &mut base.base_ei, } }; } @@ -37,9 +37,9 @@ macro_rules! get_ei_as_mut { macro_rules! get_ei_as_ref { ($item:expr) => { match $item { - EI::Base(base)=>base, - EI::Extend(base)=> &base.base_ei, - EI::Intersect(base)=> &base.base_ei, + EI::Base(base) => base, + EI::Extend(base) => &base.base_ei, + EI::Intersect(base) => &base.base_ei, } }; } @@ -48,9 +48,9 @@ macro_rules! get_ei_as_ref { macro_rules! get_scan_as_mut { ($item:expr) => { match $item { - Scan::Base(base)=>base, - Scan::ScanSampling(base)=> &mut base.base_scan, - Scan::ScanBlocking(base)=> &mut base.base_scan, + Scan::Base(base) => base, + Scan::ScanSampling(base) => &mut base.base_scan, + Scan::ScanBlocking(base) => &mut base.base_scan, } }; } @@ -59,9 +59,41 @@ macro_rules! get_scan_as_mut { macro_rules! get_scan_as_ref { ($item:expr) => { match $item { - Scan::Base(base)=>base, - Scan::ScanSampling(base)=> &base.base_scan, - Scan::ScanBlocking(base)=> &base.base_scan, + Scan::Base(base) => base, + Scan::ScanSampling(base) => &base.base_scan, + Scan::ScanBlocking(base) => &base.base_scan, + } + }; +} + +#[macro_export] +macro_rules! get_base_op_as_mut { + ($item:expr) => { + match $item { + Operator::Base(base) => &mut base, + Operator::Sink(sink) => match sink { + Sink::BaseSink(base) => &mut base.base_op, + Sink::SinkCopy(sc) => &mut sc.base_sink.base_op, + Sink::SinkPrint(sp) => &mut sp.base_sink.base_op, + Sink::SinkLimit(sl) => &mut sl.base_sink.base_op, + }, + Operator::Scan(scan) => match scan { + Scan::Base(base) => &mut base.base_op, + Scan::ScanSampling(ss) => &mut ss.base_scan.base_op, + Scan::ScanBlocking(sb) => &mut sb.base_scan.base_op, + }, + Operator::EI(ei) => match ei { + EI::Base(base) => &mut base.base_op, + EI::Extend(base) => &mut base.base_ei.base_op, + EI::Intersect(base) => &mut base.base_ei.base_op, + }, + Operator::Build(build) => &mut build.base_op, + Operator::Probe(probe) => match probe { + Probe::BaseProbe(base) => &mut base.base_op, + Probe::PC(pc) => &mut pc.base_probe.base_op, + Probe::PMV(PMV::BasePMV(base)) => &mut base.base_probe.base_op, + Probe::PMV(PMV::PMVC(pmvc)) => &mut pmvc.base_pmv.base_probe.base_op, + }, } }; } @@ -72,21 +104,28 @@ macro_rules! get_op_attr { ($item:expr,$name:ident) => { match $item { Operator::Base(base) => base.$name, - Operator::Sink(sink) => match sink{ - Sink::BaseSink(base)=>base.base_op.$name, - Sink::SinkCopy(sc)=>sc.base_sink.base_op.$name, - Sink::SinkPrint(sp)=>sp.base_sink.base_op.$name, - Sink::SinkLimit(sl)=>sl.base_sink.base_op.$name, + Operator::Sink(sink) => match sink { + Sink::BaseSink(base) => base.base_op.$name, + Sink::SinkCopy(sc) => sc.base_sink.base_op.$name, + Sink::SinkPrint(sp) => sp.base_sink.base_op.$name, + Sink::SinkLimit(sl) => sl.base_sink.base_op.$name, }, - Operator::Scan(scan) => match scan{ - Scan::Base(base)=>base.base_op.$name, - Scan::ScanSampling(base)=>base.base_scan.base_op.$name, - Scan::ScanBlocking(base)=>base.base_scan.base_op.$name, + Operator::Scan(scan) => match scan { + Scan::Base(base) => base.base_op.$name, + Scan::ScanSampling(base) => base.base_scan.base_op.$name, + Scan::ScanBlocking(base) => base.base_scan.base_op.$name, }, Operator::EI(ei) => match ei { - EI::Base(base)=>base.base_op.$name, - EI::Extend(base)=>base.base_ei.base_op.$name, - EI::Intersect(base)=>base.base_ei.base_op.$name, + EI::Base(base) => base.base_op.$name, + EI::Extend(base) => base.base_ei.base_op.$name, + EI::Intersect(base) => base.base_ei.base_op.$name, + }, + Operator::Build(build) => build.base_op.$name, + Operator::Probe(probe) => match probe { + Probe::BaseProbe(base) => base.base_op.$name, + Probe::PC(pc) => pc.base_probe.base_op.$name, + Probe::PMV(PMV::BasePMV(base)) => base.base_probe.base_op.$name, + Probe::PMV(PMV::PMVC(pmvc)) => pmvc.base_pmv.base_probe.base_op.$name, }, } }; @@ -98,21 +137,28 @@ macro_rules! get_op_attr_as_ref { ($item:expr,$name:ident) => { match $item { Operator::Base(base) => &base.$name, - Operator::Sink(sink) => match sink{ - Sink::BaseSink(base)=>&base.base_op.$name, - Sink::SinkCopy(sc)=>&sc.base_sink.base_op.$name, - Sink::SinkPrint(sp)=>&sp.base_sink.base_op.$name, - Sink::SinkLimit(sl)=>&sl.base_sink.base_op.$name, + Operator::Sink(sink) => match sink { + Sink::BaseSink(base) => &base.base_op.$name, + Sink::SinkCopy(sc) => &sc.base_sink.base_op.$name, + Sink::SinkPrint(sp) => &sp.base_sink.base_op.$name, + Sink::SinkLimit(sl) => &sl.base_sink.base_op.$name, }, - Operator::Scan(scan) => match scan{ - Scan::Base(base)=>&base.base_op.$name, - Scan::ScanSampling(base)=>&base.base_scan.base_op.$name, - Scan::ScanBlocking(base)=>&base.base_scan.base_op.$name, + Operator::Scan(scan) => match scan { + Scan::Base(base) => &base.base_op.$name, + Scan::ScanSampling(base) => &base.base_scan.base_op.$name, + Scan::ScanBlocking(base) => &base.base_scan.base_op.$name, }, Operator::EI(ei) => match ei { - EI::Base(base)=>&base.base_op.$name, - EI::Extend(base)=>&base.base_ei.base_op.$name, - EI::Intersect(base)=>&base.base_ei.base_op.$name, + EI::Base(base) => &base.base_op.$name, + EI::Extend(base) => &base.base_ei.base_op.$name, + EI::Intersect(base) => &base.base_ei.base_op.$name, + }, + Operator::Build(build) => &build.base_op.$name, + Operator::Probe(probe) => match probe { + Probe::BaseProbe(base) => &base.base_op.$name, + Probe::PC(pc) => &pc.base_probe.base_op.$name, + Probe::PMV(PMV::BasePMV(base)) => &base.base_probe.base_op.$name, + Probe::PMV(PMV::PMVC(pmvc)) => &pmvc.base_pmv.base_probe.base_op.$name, }, } }; @@ -124,26 +170,34 @@ macro_rules! get_op_attr_as_mut { ($item:expr,$name:ident) => { match $item { Operator::Base(base) => &mut base.$name, - Operator::Sink(sink) => match sink{ - Sink::BaseSink(base)=>&mut base.base_op.$name, - Sink::SinkCopy(sc)=>&mut sc.base_sink.base_op.$name, - Sink::SinkPrint(sp)=>&mut sp.base_sink.base_op.$name, - Sink::SinkLimit(sl)=>&mut sl.base_sink.base_op.$name, + Operator::Sink(sink) => match sink { + Sink::BaseSink(base) => &mut base.base_op.$name, + Sink::SinkCopy(sc) => &mut sc.base_sink.base_op.$name, + Sink::SinkPrint(sp) => &mut sp.base_sink.base_op.$name, + Sink::SinkLimit(sl) => &mut sl.base_sink.base_op.$name, }, - Operator::Scan(scan) => match scan{ - Scan::Base(base)=>&mut base.base_op.$name, - Scan::ScanSampling(base)=>&mut base.base_scan.base_op.$name, - Scan::ScanBlocking(base)=>&mut base.base_scan.base_op.$name, + Operator::Scan(scan) => match scan { + Scan::Base(base) => &mut base.base_op.$name, + Scan::ScanSampling(ss) => &mut ss.base_scan.base_op.$name, + Scan::ScanBlocking(sb) => &mut sb.base_scan.base_op.$name, }, Operator::EI(ei) => match ei { - EI::Base(base)=>&mut base.base_op.$name, - EI::Extend(base)=>&mut base.base_ei.base_op.$name, - EI::Intersect(base)=>&mut base.base_ei.base_op.$name, + EI::Base(base) => &mut base.base_op.$name, + EI::Extend(base) => &mut base.base_ei.base_op.$name, + EI::Intersect(base) => &mut base.base_ei.base_op.$name, + }, + Operator::Build(build) => &mut build.base_op.$name, + Operator::Probe(probe) => match probe { + Probe::BaseProbe(base) => &mut base.base_op.$name, + Probe::PC(pc) => &mut pc.base_probe.base_op.$name, + Probe::PMV(PMV::BasePMV(base)) => &mut base.base_probe.base_op.$name, + Probe::PMV(PMV::PMVC(pmvc)) => &mut pmvc.base_pmv.base_probe.base_op.$name, }, } }; } +pub mod extend; +pub mod hashjoin; pub mod operator; pub mod scan; pub mod sink; -pub mod extend; \ No newline at end of file diff --git a/src/graph_impl/multi_graph/plan/operator/operator.rs b/src/graph_impl/multi_graph/plan/operator/operator.rs index 29530242..b4630bab 100644 --- a/src/graph_impl/multi_graph/plan/operator/operator.rs +++ b/src/graph_impl/multi_graph/plan/operator/operator.rs @@ -1,11 +1,15 @@ use generic::{GraphType, IdType}; use graph_impl::multi_graph::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::plan::operator::extend::EI::EI; +use graph_impl::multi_graph::plan::operator::hashjoin::build::Build; +use graph_impl::multi_graph::plan::operator::hashjoin::probe::Probe; +use graph_impl::multi_graph::plan::operator::hashjoin::probe_multi_vertices::PMV; use graph_impl::multi_graph::plan::operator::scan::scan::{BaseScan, Scan}; use graph_impl::multi_graph::plan::operator::scan::scan_sampling::ScanSampling; use graph_impl::multi_graph::plan::operator::sink::sink::Sink; use graph_impl::TypedStaticGraph; use hashbrown::{HashMap, HashSet}; +use map::SetMap; use std::hash::{BuildHasherDefault, Hash}; use std::iter::FromIterator; use std::rc::Rc; @@ -17,6 +21,8 @@ pub enum Operator { Sink(Sink), Scan(Scan), EI(EI), + Build(Build), + Probe(Probe), } /// Basic operator @@ -55,9 +61,20 @@ impl BaseOperator { } } - fn get_out_query_vertices(&self) -> Vec<&String> { - let idx_map = &self.out_qvertex_to_idx_map; - idx_map.iter().map(|(key, _val)| key).collect() + pub fn empty() -> BaseOperator { + BaseOperator { + name: "".to_string(), + next: None, + prev: None, + probe_tuple: vec![], + out_tuple_len: 0, + in_subgraph: None, + out_subgraph: Box::new(QueryGraph::empty()), + out_qvertex_to_idx_map: HashMap::new(), + last_repeated_vertex_idx: 0, + num_out_tuples: 0, + icost: 0, + } } fn get_next(&self, index: usize) -> &Operator { @@ -80,7 +97,7 @@ pub trait CommonOperatorTrait { fn execute(&mut self); fn get_alds_as_string(&self) -> String; fn update_operator_name(&mut self, query_vertex_to_index_map: HashMap); - fn copy(&self, is_thread_safe: bool) -> Option>; + fn copy(&self, is_thread_safe: bool) -> Operator; fn is_same_as(&mut self, op: &mut Operator) -> bool; fn get_num_out_tuples(&self) -> usize; } @@ -123,11 +140,19 @@ impl Operator { } } } -} + pub fn get_out_query_vertices(&self) -> HashSet { + let idx_map = get_op_attr_as_ref!(self, out_qvertex_to_idx_map); + idx_map.iter().map(|(key, _val)| key.clone()).collect() + } +} impl CommonOperatorTrait for BaseOperator { - fn init(&mut self, probe_tuple: Vec, graph: &TypedStaticGraph) { + fn init( + &mut self, + probe_tuple: Vec, + graph: &TypedStaticGraph, + ) { panic!("unsupported operation exception") } @@ -149,7 +174,7 @@ impl CommonOperatorTrait for BaseOperator { panic!("`update_operator_name()` on neither `EI` or `Scan`") } - fn copy(&self, is_thread_safe: bool) -> Option> { + fn copy(&self, is_thread_safe: bool) -> Operator { panic!("unsupported operation exception") } @@ -174,6 +199,8 @@ impl CommonOperatorTrait for Operator { Operator::Sink(sink) => sink.init(probe_tuple, graph), Operator::Scan(scan) => scan.init(probe_tuple, graph), Operator::EI(ei) => ei.init(probe_tuple, graph), + Operator::Build(build) => build.init(probe_tuple, graph), + Operator::Probe(probe) => probe.init(probe_tuple, graph), } } @@ -183,6 +210,8 @@ impl CommonOperatorTrait for Operator { Operator::Sink(sink) => sink.process_new_tuple(), Operator::Scan(scan) => scan.process_new_tuple(), Operator::EI(ei) => ei.process_new_tuple(), + Operator::Build(build) => build.process_new_tuple(), + Operator::Probe(probe) => probe.process_new_tuple(), } } @@ -192,6 +221,8 @@ impl CommonOperatorTrait for Operator { Operator::Sink(sink) => sink.execute(), Operator::Scan(scan) => scan.execute(), Operator::EI(ei) => ei.execute(), + Operator::Build(build) => build.execute(), + Operator::Probe(probe) => probe.execute(), } } @@ -201,6 +232,8 @@ impl CommonOperatorTrait for Operator { Operator::Sink(sink) => sink.get_alds_as_string(), Operator::Scan(scan) => scan.get_alds_as_string(), Operator::EI(ei) => ei.get_alds_as_string(), + Operator::Build(build) => build.get_alds_as_string(), + Operator::Probe(probe) => probe.get_alds_as_string(), } } @@ -210,15 +243,19 @@ impl CommonOperatorTrait for Operator { Operator::Sink(sink) => sink.update_operator_name(query_vertex_to_index_map), Operator::Scan(scan) => scan.update_operator_name(query_vertex_to_index_map), Operator::EI(ei) => ei.update_operator_name(query_vertex_to_index_map), + Operator::Build(build) => build.update_operator_name(query_vertex_to_index_map), + Operator::Probe(probe) => probe.update_operator_name(query_vertex_to_index_map), } } - fn copy(&self, is_thread_safe: bool) -> Option> { + fn copy(&self, is_thread_safe: bool) -> Operator { match self { Operator::Base(base) => base.copy(is_thread_safe), Operator::Sink(sink) => sink.copy(is_thread_safe), Operator::Scan(scan) => scan.copy(is_thread_safe), Operator::EI(ei) => ei.copy(is_thread_safe), + Operator::Build(build) => build.copy(is_thread_safe), + Operator::Probe(probe) => probe.copy(is_thread_safe), } } fn is_same_as(&mut self, op: &mut Operator) -> bool { @@ -227,6 +264,8 @@ impl CommonOperatorTrait for Operator { Operator::Sink(sink) => sink.is_same_as(op), Operator::Scan(scan) => scan.is_same_as(op), Operator::EI(ei) => ei.is_same_as(op), + Operator::Build(build) => build.is_same_as(op), + Operator::Probe(probe) => probe.is_same_as(op), } } @@ -236,6 +275,8 @@ impl CommonOperatorTrait for Operator { Operator::Sink(sink) => sink.get_num_out_tuples(), Operator::Scan(scan) => scan.get_num_out_tuples(), Operator::EI(ei) => ei.get_num_out_tuples(), + Operator::Build(build) => build.get_num_out_tuples(), + Operator::Probe(probe) => probe.get_num_out_tuples(), } } -} \ No newline at end of file +} diff --git a/src/graph_impl/multi_graph/plan/operator/scan/mod.rs b/src/graph_impl/multi_graph/plan/operator/scan/mod.rs index d0c2d704..6cc8b375 100644 --- a/src/graph_impl/multi_graph/plan/operator/scan/mod.rs +++ b/src/graph_impl/multi_graph/plan/operator/scan/mod.rs @@ -1,3 +1,3 @@ pub mod scan; -pub mod scan_sampling; pub mod scan_blocking; +pub mod scan_sampling; diff --git a/src/graph_impl/multi_graph/plan/operator/scan/scan.rs b/src/graph_impl/multi_graph/plan/operator/scan/scan.rs index dc5086ca..6cfbe01c 100644 --- a/src/graph_impl/multi_graph/plan/operator/scan/scan.rs +++ b/src/graph_impl/multi_graph/plan/operator/scan/scan.rs @@ -1,16 +1,16 @@ -use generic::{GraphType, IdType, GraphTrait}; +use generic::{GraphTrait, GraphType, IdType}; use graph_impl::multi_graph::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::plan::operator::operator::{ BaseOperator, CommonOperatorTrait, Operator, }; +use graph_impl::multi_graph::plan::operator::scan::scan_blocking::ScanBlocking; +use graph_impl::multi_graph::plan::operator::scan::scan_sampling::ScanSampling; use graph_impl::multi_graph::plan::operator::sink::sink::Sink; use graph_impl::static_graph::sorted_adj_vec::SortedAdjVec; use graph_impl::TypedStaticGraph; -use std::hash::{Hash, BuildHasherDefault}; -use std::rc::Rc; -use graph_impl::multi_graph::plan::operator::scan::scan_sampling::ScanSampling; -use graph_impl::multi_graph::plan::operator::scan::scan_blocking::ScanBlocking; use hashbrown::HashMap; +use std::hash::{BuildHasherDefault, Hash}; +use std::rc::Rc; #[derive(Clone)] pub enum Scan { @@ -49,7 +49,7 @@ impl BaseScan { from_vertex_start_idx: 0, from_vertex_end_idx: 0, }; - let out_subgraph = &scan.base_op.out_subgraph; + let out_subgraph = scan.base_op.out_subgraph.as_ref(); if out_subgraph.get_query_edges().len() > 1 { panic!("IllegalArgumentException"); } @@ -60,9 +60,14 @@ impl BaseScan { scan.base_op.last_repeated_vertex_idx = 0; scan.from_query_vertex = query_edge.from_query_vertex.clone(); scan.to_query_vertex = query_edge.to_query_vertex.clone(); - scan.base_op.out_qvertex_to_idx_map.insert(scan.from_query_vertex.clone(), 0); - scan.base_op.out_qvertex_to_idx_map.insert(scan.to_query_vertex.clone(), 1); - scan.base_op.name = "SCAN (".to_owned() + &scan.from_query_vertex + ")->(" + &scan.to_query_vertex + ")"; + scan.base_op + .out_qvertex_to_idx_map + .insert(scan.from_query_vertex.clone(), 0); + scan.base_op + .out_qvertex_to_idx_map + .insert(scan.to_query_vertex.clone(), 1); + scan.base_op.name = + "SCAN (".to_owned() + &scan.from_query_vertex + ")->(" + &scan.to_query_vertex + ")"; scan } } @@ -101,13 +106,23 @@ impl CommonOperatorTrait for BaseScan { for from_idx in self.from_vertex_start_idx..self.from_vertex_end_idx { let from_vertex = self.vertex_ids[from_idx]; self.base_op.probe_tuple[0] = from_vertex; - let to_vertex_start_idx = self.fwd_adj_list[from_idx].as_ref().unwrap().get_offsets()[self.label_or_to_type]; - let to_vertex_end_idx = self.fwd_adj_list[from_idx].as_ref().unwrap().get_offsets()[self.label_or_to_type + 1]; + let to_vertex_start_idx = + self.fwd_adj_list[from_idx].as_ref().unwrap().get_offsets()[self.label_or_to_type]; + let to_vertex_end_idx = self.fwd_adj_list[from_idx].as_ref().unwrap().get_offsets() + [self.label_or_to_type + 1]; for to_idx in to_vertex_start_idx..to_vertex_end_idx { - self.base_op.probe_tuple[1] = self.fwd_adj_list[from_idx].as_ref().unwrap().get_neighbor_id(Id::new(to_idx)); - if self.to_type == 0 || self.vertex_types[self.base_op.probe_tuple[1].id()] == self.to_type { + self.base_op.probe_tuple[1] = self.fwd_adj_list[from_idx] + .as_ref() + .unwrap() + .get_neighbor_id(Id::new(to_idx)); + if self.to_type == 0 + || self.vertex_types[self.base_op.probe_tuple[1].id()] == self.to_type + { self.base_op.num_out_tuples += 1; - self.base_op.next.as_mut().map(|next| (&mut next[0]).process_new_tuple()); + self.base_op + .next + .as_mut() + .map(|next| (&mut next[0]).process_new_tuple()); } } } @@ -128,18 +143,20 @@ impl CommonOperatorTrait for BaseScan { }); } - fn copy(&self, is_thread_safe: bool) -> Option> { + fn copy(&self, is_thread_safe: bool) -> Operator { if is_thread_safe { - return Some(Operator::Scan(Scan::ScanBlocking(ScanBlocking::new(self.base_op.out_subgraph.clone())))); + return Operator::Scan(Scan::ScanBlocking(ScanBlocking::new( + self.base_op.out_subgraph.clone(), + ))); } - Some(Operator::Scan(Scan::Base(BaseScan::new(self.base_op.out_subgraph.clone())))) + Operator::Scan(Scan::Base(BaseScan::new(self.base_op.out_subgraph.clone()))) } fn is_same_as(&mut self, op: &mut Operator) -> bool { if let Operator::Scan(scan) = op { - return self.from_type == get_scan_as_ref!(scan).from_type && - self.to_type == get_scan_as_ref!(scan).to_type && - self.label_or_to_type == get_scan_as_ref!(scan).label_or_to_type; + return self.from_type == get_scan_as_ref!(scan).from_type + && self.to_type == get_scan_as_ref!(scan).to_type + && self.label_or_to_type == get_scan_as_ref!(scan).label_or_to_type; } false } @@ -150,7 +167,11 @@ impl CommonOperatorTrait for BaseScan { } impl CommonOperatorTrait for Scan { - fn init(&mut self, probe_tuple: Vec, graph: &TypedStaticGraph) { + fn init( + &mut self, + probe_tuple: Vec, + graph: &TypedStaticGraph, + ) { match self { Scan::Base(base) => base.init(probe_tuple, graph), Scan::ScanSampling(ss) => ss.init(probe_tuple, graph), @@ -190,7 +211,7 @@ impl CommonOperatorTrait for Scan { } } - fn copy(&self, is_thread_safe: bool) -> Option> { + fn copy(&self, is_thread_safe: bool) -> Operator { match self { Scan::Base(base) => base.copy(is_thread_safe), Scan::ScanSampling(ss) => ss.copy(is_thread_safe), @@ -214,4 +235,3 @@ impl CommonOperatorTrait for Scan { } } } - diff --git a/src/graph_impl/multi_graph/plan/operator/scan/scan_blocking.rs b/src/graph_impl/multi_graph/plan/operator/scan/scan_blocking.rs index 12de3e41..2a53b6a8 100644 --- a/src/graph_impl/multi_graph/plan/operator/scan/scan_blocking.rs +++ b/src/graph_impl/multi_graph/plan/operator/scan/scan_blocking.rs @@ -1,10 +1,10 @@ -use generic::{IdType, GraphType, GraphTrait}; -use graph_impl::multi_graph::plan::operator::scan::scan::BaseScan; +use generic::{GraphTrait, GraphType, IdType}; use graph_impl::multi_graph::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; +use graph_impl::multi_graph::plan::operator::scan::scan::BaseScan; use graph_impl::TypedStaticGraph; -use std::hash::{Hash, BuildHasherDefault}; use hashbrown::HashMap; +use std::hash::{BuildHasherDefault, Hash}; static PARTITION_SIZE: usize = 100; @@ -37,7 +37,10 @@ impl ScanBlocking { to_idx_limit: 0, highest_from_idx: 0, highest_to_idx: 0, - global_vertices_idx_limits: VertexIdxLimits { from_variable_index_limit: 0, to_variable_index_limit: 0 }, + global_vertices_idx_limits: VertexIdxLimits { + from_variable_index_limit: 0, + to_variable_index_limit: 0, + }, } } @@ -49,14 +52,16 @@ impl ScanBlocking { self.to_idx_limit = self.curr_to_idx; let mut num_edges_left = PARTITION_SIZE; while num_edges_left > 0 { - let flag = self.from_idx_limit == self.highest_from_idx - 1 && - self.to_idx_limit < self.highest_to_idx - 1 || - self.from_idx_limit < self.highest_from_idx - 1; - if !flag { break; } + let flag = self.from_idx_limit == self.highest_from_idx - 1 + && self.to_idx_limit < self.highest_to_idx - 1 + || self.from_idx_limit < self.highest_from_idx - 1; + if !flag { + break; + } let mut label = self.base_scan.label_or_to_type; - let to_limit = self.base_scan.fwd_adj_list[self.from_idx_limit].as_mut().map_or(0, |adj| { - adj.get_offsets()[label + 1] - }); + let to_limit = self.base_scan.fwd_adj_list[self.from_idx_limit] + .as_mut() + .map_or(0, |adj| adj.get_offsets()[label + 1]); if self.to_idx_limit + num_edges_left <= to_limit - 1 { self.to_idx_limit += (num_edges_left - 1); num_edges_left = 0; @@ -68,9 +73,9 @@ impl ScanBlocking { } self.from_idx_limit += 1; label = self.base_scan.label_or_to_type; - self.to_idx_limit = self.base_scan.fwd_adj_list[self.from_idx_limit].as_mut().map_or(0, |adj| { - adj.get_offsets()[label] - }); + self.to_idx_limit = self.base_scan.fwd_adj_list[self.from_idx_limit] + .as_mut() + .map_or(0, |adj| adj.get_offsets()[label]); } } self.global_vertices_idx_limits.from_variable_index_limit = self.from_idx_limit; @@ -81,27 +86,43 @@ impl ScanBlocking { let base_op = &mut self.base_scan.base_op; for to_idx in start_to_idx..end_to_idx { base_op.probe_tuple[0] = self.base_scan.vertex_ids[from_idx]; - base_op.probe_tuple[1] = self.base_scan.fwd_adj_list[from_idx].as_mut().unwrap().get_neighbor_id(Id::new(to_idx)); + base_op.probe_tuple[1] = self.base_scan.fwd_adj_list[from_idx] + .as_mut() + .unwrap() + .get_neighbor_id(Id::new(to_idx)); base_op.num_out_tuples += 1; - base_op.next.as_mut().map(|next| next.get_mut(0).map(|next_op| next_op.process_new_tuple())); + base_op + .next + .as_mut() + .map(|next| next.get_mut(0).map(|next_op| next_op.process_new_tuple())); } } fn produce_new_edges_default(&mut self) { for from_idx in self.curr_from_idx + 1..self.from_idx_limit { - let mut label = self.base_scan.label_or_to_type; + let label = self.base_scan.label_or_to_type; self.base_scan.base_op.probe_tuple[0] = self.base_scan.vertex_ids[from_idx]; - let to_vertex_idx_start = self.base_scan.fwd_adj_list[from_idx].as_mut().map_or(0, |adj| { - adj.get_offsets()[label] - }); - let to_vertex_idx_limit = self.base_scan.fwd_adj_list[from_idx].as_mut().map_or(0, |adj| { - adj.get_offsets()[label + 1] - }); + let to_vertex_idx_start = self.base_scan.fwd_adj_list[from_idx] + .as_mut() + .map_or(0, |adj| adj.get_offsets()[label]); + let to_vertex_idx_limit = self.base_scan.fwd_adj_list[from_idx] + .as_mut() + .map_or(0, |adj| adj.get_offsets()[label + 1]); for to_idx in to_vertex_idx_start..to_vertex_idx_limit { - self.base_scan.base_op.probe_tuple[1] = self.base_scan.fwd_adj_list[from_idx].as_mut().unwrap().get_neighbor_id(Id::new(to_idx)); - if self.base_scan.to_type == 0 || self.base_scan.vertex_types[self.base_scan.base_op.probe_tuple[1].id()] == self.base_scan.to_type { + self.base_scan.base_op.probe_tuple[1] = self.base_scan.fwd_adj_list[from_idx] + .as_mut() + .unwrap() + .get_neighbor_id(Id::new(to_idx)); + if self.base_scan.to_type == 0 + || self.base_scan.vertex_types[self.base_scan.base_op.probe_tuple[1].id()] + == self.base_scan.to_type + { self.base_scan.base_op.num_out_tuples += 1; - self.base_scan.base_op.next.as_mut().map(|next| next.get_mut(0).map(|next_op| next_op.process_new_tuple())); + self.base_scan + .base_op + .next + .as_mut() + .map(|next| next.get_mut(0).map(|next_op| next_op.process_new_tuple())); } } } @@ -109,7 +130,11 @@ impl ScanBlocking { } impl CommonOperatorTrait for ScanBlocking { - fn init(&mut self, probe_tuple: Vec, graph: &TypedStaticGraph) { + fn init( + &mut self, + probe_tuple: Vec, + graph: &TypedStaticGraph, + ) { self.base_scan.init(probe_tuple.clone(), graph); if self.base_scan.from_type != 0 { self.curr_from_idx = graph.get_node_type_offsets()[self.base_scan.from_type]; @@ -118,18 +143,21 @@ impl CommonOperatorTrait for ScanBlocking { self.curr_from_idx = 0; self.highest_from_idx = graph.node_count() + 1; } - let mut label = self.base_scan.label_or_to_type; - self.curr_to_idx = self.base_scan.fwd_adj_list[self.base_scan.vertex_ids[self.curr_from_idx].id()].as_mut().map_or(0, |adj| { - adj.get_offsets()[label] - }); - self.highest_to_idx = self.base_scan.fwd_adj_list[self.base_scan.vertex_ids[self.highest_from_idx - 1].id()].as_mut().map_or(0, |adj| { - adj.get_offsets()[label + 1] - }); + let label = self.base_scan.label_or_to_type; + self.curr_to_idx = self.base_scan.fwd_adj_list + [self.base_scan.vertex_ids[self.curr_from_idx].id()] + .as_mut() + .map_or(0, |adj| adj.get_offsets()[label]); + self.highest_to_idx = self.base_scan.fwd_adj_list + [self.base_scan.vertex_ids[self.highest_from_idx - 1].id()] + .as_mut() + .map_or(0, |adj| adj.get_offsets()[label + 1]); self.from_idx_limit = self.curr_from_idx; self.to_idx_limit = self.curr_to_idx; - self.base_scan.base_op.next.as_mut().map(|next| - next.iter_mut().for_each(|next_op| next_op.init(probe_tuple.clone(), graph)) - ); + self.base_scan.base_op.next.as_mut().map(|next| { + next.iter_mut() + .for_each(|next_op| next_op.init(probe_tuple.clone(), graph)) + }); } fn process_new_tuple(&mut self) { @@ -138,26 +166,28 @@ impl CommonOperatorTrait for ScanBlocking { fn execute(&mut self) { self.update_indices_limits(); - while self.curr_from_idx == self.highest_from_idx - 1 && - self.curr_to_idx < self.highest_to_idx - 1 || - self.curr_from_idx < self.highest_from_idx - 1 - { - if self.curr_from_idx == self.from_idx_limit { - self.produce_new_edges(self.curr_from_idx, self.curr_to_idx, self.to_idx_limit); - } else if self.curr_from_idx < self.from_idx_limit { - let label = self.base_scan.label_or_to_type; - let to_vertex_idx_limit = self.base_scan.fwd_adj_list[self.base_scan.vertex_ids[self.curr_from_idx].id()].as_mut().map_or(0, |adj| { - adj.get_offsets()[label + 1] - }); - self.produce_new_edges(self.curr_from_idx, self.curr_to_idx, to_vertex_idx_limit); - self.produce_new_edges_default(/* startFromIdx: currFromIdx + 1, endFromIdx: fromIdxLimit */); - let start_idx = self.base_scan.fwd_adj_list[self.base_scan.vertex_ids[self.from_idx_limit].id()].as_mut().map_or(0, |adj| { - adj.get_offsets()[label] - }); - self.produce_new_edges(self.from_idx_limit,start_idx , self.to_idx_limit); - } - self.update_indices_limits(); + while self.curr_from_idx == self.highest_from_idx - 1 + && self.curr_to_idx < self.highest_to_idx - 1 + || self.curr_from_idx < self.highest_from_idx - 1 + { + if self.curr_from_idx == self.from_idx_limit { + self.produce_new_edges(self.curr_from_idx, self.curr_to_idx, self.to_idx_limit); + } else if self.curr_from_idx < self.from_idx_limit { + let label = self.base_scan.label_or_to_type; + let to_vertex_idx_limit = self.base_scan.fwd_adj_list + [self.base_scan.vertex_ids[self.curr_from_idx].id()] + .as_mut() + .map_or(0, |adj| adj.get_offsets()[label + 1]); + self.produce_new_edges(self.curr_from_idx, self.curr_to_idx, to_vertex_idx_limit); + self.produce_new_edges_default(/* startFromIdx: currFromIdx + 1, endFromIdx: fromIdxLimit */); + let start_idx = self.base_scan.fwd_adj_list + [self.base_scan.vertex_ids[self.from_idx_limit].id()] + .as_mut() + .map_or(0, |adj| adj.get_offsets()[label]); + self.produce_new_edges(self.from_idx_limit, start_idx, self.to_idx_limit); } + self.update_indices_limits(); + } } fn get_alds_as_string(&self) -> String { @@ -165,10 +195,11 @@ impl CommonOperatorTrait for ScanBlocking { } fn update_operator_name(&mut self, query_vertex_to_index_map: HashMap) { - self.base_scan.update_operator_name(query_vertex_to_index_map) + self.base_scan + .update_operator_name(query_vertex_to_index_map) } - fn copy(&self, is_thread_safe: bool) -> Option> { + fn copy(&self, is_thread_safe: bool) -> Operator { self.base_scan.copy(is_thread_safe) } @@ -179,4 +210,4 @@ impl CommonOperatorTrait for ScanBlocking { fn get_num_out_tuples(&self) -> usize { self.base_scan.get_num_out_tuples() } -} \ No newline at end of file +} diff --git a/src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs b/src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs index 4dee571c..f5426dc2 100644 --- a/src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs +++ b/src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs @@ -6,9 +6,9 @@ use graph_impl::multi_graph::plan::operator::operator::{ use graph_impl::multi_graph::plan::operator::scan::scan::{BaseScan, Scan}; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; +use rand::{thread_rng, Rng}; use std::hash::{BuildHasherDefault, Hash}; use std::rc::Rc; -use rand::{thread_rng, Rng}; #[derive(Clone)] pub struct ScanSampling { @@ -29,11 +29,16 @@ impl ScanSampling { let num_edges = edges.len() / 2; while self.edges_queue.len() < num_edges_to_sample { let edge_idx = rng.gen_range(0, num_edges); - self.edges_queue.push(vec![edges[edge_idx], edges[edge_idx + 1]]); + self.edges_queue + .push(vec![edges[edge_idx], edges[edge_idx + 1]]); } } - pub fn set_edge_indices_to_sample_by_edges(&mut self, edges: Vec>, num_edges_to_sample: usize) { + pub fn set_edge_indices_to_sample_by_edges( + &mut self, + edges: Vec>, + num_edges_to_sample: usize, + ) { let mut rng = thread_rng(); self.edges_queue = vec![vec![]; num_edges_to_sample]; while self.edges_queue.len() < num_edges_to_sample { @@ -77,16 +82,17 @@ impl CommonOperatorTrait for ScanSampling { fn get_alds_as_string(&self) -> String { self.base_scan.get_alds_as_string() -} + } fn update_operator_name(&mut self, query_vertex_to_index_map: HashMap) { - self.base_scan.update_operator_name(query_vertex_to_index_map) + self.base_scan + .update_operator_name(query_vertex_to_index_map) } - fn copy(&self, is_thread_safe: bool) -> Option> { + fn copy(&self, is_thread_safe: bool) -> Operator { let mut scan_sampling = ScanSampling::new(self.base_scan.base_op.out_subgraph.clone()); scan_sampling.edges_queue = self.edges_queue.clone(); - Some(Operator::Scan(Scan::ScanSampling(scan_sampling))) + Operator::Scan(Scan::ScanSampling(scan_sampling)) } fn is_same_as(&mut self, op: &mut Operator) -> bool { diff --git a/src/graph_impl/multi_graph/plan/operator/sink/sink.rs b/src/graph_impl/multi_graph/plan/operator/sink/sink.rs index 92c5c50d..82844b00 100644 --- a/src/graph_impl/multi_graph/plan/operator/sink/sink.rs +++ b/src/graph_impl/multi_graph/plan/operator/sink/sink.rs @@ -1,17 +1,19 @@ use generic::{GraphType, IdType}; use graph_impl::multi_graph::catalog::query_graph::QueryGraph; +use graph_impl::multi_graph::plan::operator::extend::EI::EI; +use graph_impl::multi_graph::plan::operator::hashjoin::probe::Probe; +use graph_impl::multi_graph::plan::operator::hashjoin::probe_multi_vertices::PMV; use graph_impl::multi_graph::plan::operator::operator::{ BaseOperator, CommonOperatorTrait, Operator, }; -use ::graph_impl::multi_graph::plan::operator::scan::scan::Scan; +use graph_impl::multi_graph::plan::operator::scan::scan::Scan; +use graph_impl::multi_graph::plan::operator::sink::sink_copy::SinkCopy; +use graph_impl::multi_graph::plan::operator::sink::sink_limit::SinkLimit; +use graph_impl::multi_graph::plan::operator::sink::sink_print::SinkPrint; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; -use graph_impl::multi_graph::plan::operator::extend::EI::EI; use std::hash::{BuildHasherDefault, Hash}; use std::rc::Rc; -use graph_impl::multi_graph::plan::operator::sink::sink_copy::SinkCopy; -use graph_impl::multi_graph::plan::operator::sink::sink_print::SinkPrint; -use graph_impl::multi_graph::plan::operator::sink::sink_limit::SinkLimit; pub enum SinkType { Copy, @@ -70,14 +72,14 @@ impl CommonOperatorTrait for BaseSink { self.base_op.update_operator_name(query_vertex_to_index_map) } - fn copy(&self, is_thread_safe: bool) -> Option> { + fn copy(&self, is_thread_safe: bool) -> Operator { let mut sink = BaseSink::new(self.base_op.out_subgraph.clone()); if let Some(prev) = &self.base_op.prev { - sink.base_op.prev = Some(Box::new(prev.copy(is_thread_safe).unwrap())); + sink.base_op.prev = Some(Box::new(prev.copy(is_thread_safe))); } else { sink.base_op.prev = None; } - Some(Operator::Sink(Sink::BaseSink(sink))) + Operator::Sink(Sink::BaseSink(sink)) } fn is_same_as(&mut self, op: &mut Operator) -> bool { @@ -102,7 +104,11 @@ impl CommonOperatorTrait for BaseSink { } impl CommonOperatorTrait for Sink { - fn init(&mut self, probe_tuple: Vec, graph: &TypedStaticGraph) { + fn init( + &mut self, + probe_tuple: Vec, + graph: &TypedStaticGraph, + ) { match self { Sink::BaseSink(base) => base.init(probe_tuple, graph), Sink::SinkCopy(sc) => sc.init(probe_tuple, graph), @@ -147,7 +153,7 @@ impl CommonOperatorTrait for Sink { } } - fn copy(&self, is_thread_safe: bool) -> Option> { + fn copy(&self, is_thread_safe: bool) -> Operator { match self { Sink::BaseSink(base) => base.copy(is_thread_safe), Sink::SinkCopy(sc) => sc.copy(is_thread_safe), @@ -173,4 +179,4 @@ impl CommonOperatorTrait for Sink { Sink::SinkLimit(sl) => sl.get_num_out_tuples(), } } -} \ No newline at end of file +} diff --git a/src/graph_impl/multi_graph/plan/operator/sink/sink_copy.rs b/src/graph_impl/multi_graph/plan/operator/sink/sink_copy.rs index 3d257aa0..fb8084a6 100644 --- a/src/graph_impl/multi_graph/plan/operator/sink/sink_copy.rs +++ b/src/graph_impl/multi_graph/plan/operator/sink/sink_copy.rs @@ -1,11 +1,11 @@ -use graph_impl::multi_graph::plan::operator::sink::sink::{Sink, BaseSink}; -use generic::{IdType, GraphType}; -use std::io::sink; +use generic::{GraphType, IdType}; use graph_impl::multi_graph::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; +use graph_impl::multi_graph::plan::operator::sink::sink::{BaseSink, Sink}; use graph_impl::TypedStaticGraph; -use std::hash::{Hash, BuildHasherDefault}; use hashbrown::HashMap; +use std::hash::{BuildHasherDefault, Hash}; +use std::io::sink; #[derive(Clone)] pub struct SinkCopy { @@ -26,13 +26,18 @@ impl SinkCopy { } impl CommonOperatorTrait for SinkCopy { - fn init(&mut self, probe_tuple: Vec, graph: &TypedStaticGraph) { + fn init( + &mut self, + probe_tuple: Vec, + graph: &TypedStaticGraph, + ) { self.base_sink.init(probe_tuple, graph); } fn process_new_tuple(&mut self) { let len = self.output_tuple.len(); - self.output_tuple.clone_from_slice(&self.base_sink.base_op.probe_tuple[0..len]); + self.output_tuple + .clone_from_slice(&self.base_sink.base_op.probe_tuple[0..len]); } fn execute(&mut self) { @@ -44,18 +49,16 @@ impl CommonOperatorTrait for SinkCopy { } fn update_operator_name(&mut self, query_vertex_to_index_map: HashMap) { - self.base_sink.update_operator_name(query_vertex_to_index_map) + self.base_sink + .update_operator_name(query_vertex_to_index_map) } - fn copy(&self, is_thread_safe: bool) -> Option> { + fn copy(&self, is_thread_safe: bool) -> Operator { let base_op = &self.base_sink.base_op; - let mut sink = SinkCopy::new( - base_op.out_subgraph.clone(), - base_op.out_tuple_len, - ); + let mut sink = SinkCopy::new(base_op.out_subgraph.clone(), base_op.out_tuple_len); let origin_prev = base_op.prev.as_ref().unwrap(); - sink.base_sink.base_op.prev = origin_prev.copy(is_thread_safe).map(|x| Box::new(x)); - Some(Operator::Sink(Sink::SinkCopy(sink))) + sink.base_sink.base_op.prev = Some(Box::new(origin_prev.copy(is_thread_safe))); + Operator::Sink(Sink::SinkCopy(sink)) } fn is_same_as(&mut self, op: &mut Operator) -> bool { @@ -65,4 +68,4 @@ impl CommonOperatorTrait for SinkCopy { fn get_num_out_tuples(&self) -> usize { self.base_sink.get_num_out_tuples() } -} \ No newline at end of file +} diff --git a/src/graph_impl/multi_graph/plan/operator/sink/sink_limit.rs b/src/graph_impl/multi_graph/plan/operator/sink/sink_limit.rs index f636e413..3fc829ce 100644 --- a/src/graph_impl/multi_graph/plan/operator/sink/sink_limit.rs +++ b/src/graph_impl/multi_graph/plan/operator/sink/sink_limit.rs @@ -1,13 +1,15 @@ -use generic::{IdType, GraphType}; -use graph_impl::multi_graph::plan::operator::sink::sink::{BaseSink, Sink}; -use graph_impl::multi_graph::plan::operator::scan::scan::Scan; -use graph_impl::multi_graph::plan::operator::extend::EI::EI; +use generic::{GraphType, IdType}; use graph_impl::multi_graph::catalog::query_graph::QueryGraph; +use graph_impl::multi_graph::plan::operator::extend::EI::EI; +use graph_impl::multi_graph::plan::operator::hashjoin::probe::Probe; +use graph_impl::multi_graph::plan::operator::hashjoin::probe_multi_vertices::PMV; use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; -use graph_impl::TypedStaticGraph; -use std::hash::{Hash, BuildHasherDefault}; +use graph_impl::multi_graph::plan::operator::scan::scan::Scan; +use graph_impl::multi_graph::plan::operator::sink::sink::{BaseSink, Sink}; use graph_impl::multi_graph::utils::io_utils::{current_time, get_elapsed_time_in_millis}; +use graph_impl::TypedStaticGraph; use hashbrown::HashMap; +use std::hash::{BuildHasherDefault, Hash}; #[derive(Clone)] pub struct SinkLimit { @@ -29,13 +31,17 @@ impl SinkLimit { } impl CommonOperatorTrait for SinkLimit { - fn init(&mut self, probe_tuple: Vec, graph: &TypedStaticGraph) { - self.base_sink.init(probe_tuple,graph) + fn init( + &mut self, + probe_tuple: Vec, + graph: &TypedStaticGraph, + ) { + self.base_sink.init(probe_tuple, graph) } fn process_new_tuple(&mut self) { let prev = self.base_sink.base_op.prev.as_ref().unwrap().as_ref(); - if get_op_attr!(prev,num_out_tuples) >= self.out_tuples_limit { + if get_op_attr!(prev, num_out_tuples) >= self.out_tuples_limit { self.elapsed_time = get_elapsed_time_in_millis(self.start_time); } } @@ -49,10 +55,11 @@ impl CommonOperatorTrait for SinkLimit { } fn update_operator_name(&mut self, query_vertex_to_index_map: HashMap) { - self.base_sink.update_operator_name(query_vertex_to_index_map) + self.base_sink + .update_operator_name(query_vertex_to_index_map) } - fn copy(&self, is_thread_safe: bool) -> Option> { + fn copy(&self, is_thread_safe: bool) -> Operator { self.base_sink.copy(is_thread_safe) } @@ -63,4 +70,4 @@ impl CommonOperatorTrait for SinkLimit { fn get_num_out_tuples(&self) -> usize { self.base_sink.get_num_out_tuples() } -} \ No newline at end of file +} diff --git a/src/graph_impl/multi_graph/plan/operator/sink/sink_print.rs b/src/graph_impl/multi_graph/plan/operator/sink/sink_print.rs index d804b3ac..1da5c345 100644 --- a/src/graph_impl/multi_graph/plan/operator/sink/sink_print.rs +++ b/src/graph_impl/multi_graph/plan/operator/sink/sink_print.rs @@ -1,10 +1,10 @@ -use generic::{IdType, GraphType}; -use graph_impl::multi_graph::plan::operator::sink::sink::{BaseSink, Sink}; +use generic::{GraphType, IdType}; use graph_impl::multi_graph::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; +use graph_impl::multi_graph::plan::operator::sink::sink::{BaseSink, Sink}; use graph_impl::TypedStaticGraph; -use std::hash::{Hash, BuildHasherDefault}; use hashbrown::HashMap; +use std::hash::{BuildHasherDefault, Hash}; #[derive(Clone)] pub struct SinkPrint { @@ -14,14 +14,18 @@ pub struct SinkPrint { impl SinkPrint { pub fn new(query_graph: Box) -> SinkPrint { SinkPrint { - base_sink: BaseSink::new(query_graph) + base_sink: BaseSink::new(query_graph), } } } impl CommonOperatorTrait for SinkPrint { - fn init(&mut self, probe_tuple: Vec, graph: &TypedStaticGraph) { - self.base_sink.init(probe_tuple,graph) + fn init( + &mut self, + probe_tuple: Vec, + graph: &TypedStaticGraph, + ) { + self.base_sink.init(probe_tuple, graph) } fn process_new_tuple(&mut self) { @@ -37,15 +41,16 @@ impl CommonOperatorTrait for SinkPrint { } fn update_operator_name(&mut self, query_vertex_to_index_map: HashMap) { - self.base_sink.update_operator_name(query_vertex_to_index_map) + self.base_sink + .update_operator_name(query_vertex_to_index_map) } - fn copy(&self, is_thread_safe: bool) -> Option> { + fn copy(&self, is_thread_safe: bool) -> Operator { let base_op = &self.base_sink.base_op; let mut sink = SinkPrint::new(base_op.out_subgraph.clone()); let origin_prev = base_op.prev.as_ref().unwrap(); - sink.base_sink.base_op.prev = origin_prev.copy(is_thread_safe).map(|x| Box::new(x)); - Some(Operator::Sink(Sink::SinkPrint(sink))) + sink.base_sink.base_op.prev = Some(Box::new(origin_prev.copy(is_thread_safe))); + Operator::Sink(Sink::SinkPrint(sink)) } fn is_same_as(&mut self, op: &mut Operator) -> bool { @@ -55,4 +60,4 @@ impl CommonOperatorTrait for SinkPrint { fn get_num_out_tuples(&self) -> usize { self.base_sink.get_num_out_tuples() } -} \ No newline at end of file +} diff --git a/src/graph_impl/multi_graph/plan/query_plan.rs b/src/graph_impl/multi_graph/plan/query_plan.rs index f4608673..476e29b3 100644 --- a/src/graph_impl/multi_graph/plan/query_plan.rs +++ b/src/graph_impl/multi_graph/plan/query_plan.rs @@ -1,18 +1,20 @@ +use generic::IdType; +use graph_impl::multi_graph::plan::operator::extend::EI::EI; +use graph_impl::multi_graph::plan::operator::hashjoin::probe::Probe; +use graph_impl::multi_graph::plan::operator::hashjoin::probe_multi_vertices::PMV; use graph_impl::multi_graph::plan::operator::operator::{BaseOperator, Operator}; -use graph_impl::multi_graph::plan::operator::scan::scan_sampling::ScanSampling; -use graph_impl::multi_graph::plan::operator::sink::sink::{Sink, SinkType, BaseSink}; use graph_impl::multi_graph::plan::operator::scan::scan::Scan; -use graph_impl::multi_graph::plan::operator::extend::EI::EI; +use graph_impl::multi_graph::plan::operator::scan::scan_sampling::ScanSampling; +use graph_impl::multi_graph::plan::operator::sink::sink::{BaseSink, Sink, SinkType}; use hashbrown::HashMap; use std::ops::Deref; use std::rc::Rc; -use generic::IdType; pub struct QueryPlan { sink: Option>, sink_type: SinkType, scan_sampling: Option>, - last_operator: Option>>, + last_operator: Option>>, out_tuples_limit: usize, elapsed_time: f64, icost: usize, @@ -21,13 +23,13 @@ pub struct QueryPlan { operator_metrics: Vec<(String, usize, usize)>, executed: bool, adaptive_enabled: bool, - subplans: Vec>>, + pub subplans: Vec>>, estimated_icost: f64, estimated_num_out_tuples: f64, q_vertex_to_num_out_tuples: HashMap, } -impl QueryPlan { +impl QueryPlan { pub fn new(scan_sampling: ScanSampling) -> Self { let mut last_operators = Vec::new(); let scan_sampling_op = Operator::Scan(Scan::ScanSampling(scan_sampling.clone())); @@ -59,7 +61,7 @@ impl QueryPlan { q_vertex_to_num_out_tuples: HashMap::new(), } } - pub fn new_from_operator(last_operator: Rc>) -> Self { + pub fn new_from_operator(last_operator: Box>) -> Self { Self { sink: None, sink_type: SinkType::Counter, @@ -79,6 +81,27 @@ impl QueryPlan { q_vertex_to_num_out_tuples: HashMap::new(), } } + pub fn new_from_subplans(subplans: Vec>>) -> Self { + Self { + sink: None, + sink_type: SinkType::Copy, + scan_sampling: None, + last_operator: subplans.get(subplans.len() - 1).map(|x| x.clone()), + out_tuples_limit: 0, + elapsed_time: 0.0, + icost: 0, + num_intermediate_tuples: 0, + num_out_tuples: 0, + operator_metrics: vec![], + executed: false, + adaptive_enabled: false, + subplans, + estimated_icost: 0.0, + estimated_num_out_tuples: 0.0, + q_vertex_to_num_out_tuples: HashMap::new(), + } + } + pub fn get_scan_sampling(&mut self) -> Option<&mut ScanSampling> { self.scan_sampling.as_mut() } @@ -90,4 +113,8 @@ impl QueryPlan { pub fn get_sink_as_ref(&self) -> &Sink { self.sink.as_ref().unwrap() } + + pub fn shallow_copy(&self) -> QueryPlan { + QueryPlan::new_from_subplans(self.subplans.clone()) + } } diff --git a/src/graph_impl/static_graph/graph.rs b/src/graph_impl/static_graph/graph.rs index 268a9cdb..0bbff03d 100644 --- a/src/graph_impl/static_graph/graph.rs +++ b/src/graph_impl/static_graph/graph.rs @@ -29,6 +29,7 @@ use generic::{ GraphLabelTrait, GraphTrait, GraphType, IdType, Iter, MapTrait, MutMapTrait, NodeTrait, NodeType, UnGraphTrait, Undirected, }; +use graph_impl::multi_graph::catalog::adj_list_descriptor::Direction; use graph_impl::static_graph::node::StaticNode; use graph_impl::static_graph::sorted_adj_vec::SortedAdjVec; use graph_impl::static_graph::static_edge_iter::StaticEdgeIndexIter; @@ -44,12 +45,11 @@ use std::cmp; use std::ops::Add; use test::bench::iter; use test::Options; -use graph_impl::multi_graph::catalog::adj_list_descriptor::Direction; pub type TypedUnStaticGraph = TypedStaticGraph; pub type TypedDiStaticGraph = TypedStaticGraph; pub type StaticGraph = -TypedStaticGraph; + TypedStaticGraph; pub type UnStaticGraph = StaticGraph; pub type DiStaticGraph = StaticGraph; @@ -88,7 +88,7 @@ pub struct TypedStaticGraph PartialEq -for TypedStaticGraph + for TypedStaticGraph { fn eq(&self, other: &TypedStaticGraph) -> bool { if !self.node_count() == other.node_count() || !self.edge_count() == other.edge_count() { @@ -112,11 +112,12 @@ for TypedStaticGraph } impl Eq -for TypedStaticGraph -{} + for TypedStaticGraph +{ +} impl Hash -for TypedStaticGraph + for TypedStaticGraph { fn hash(&self, state: &mut H) { { @@ -142,25 +143,27 @@ for TypedStaticGraph } impl Serialize -for TypedStaticGraph - where - Id: serde::Serialize, - NL: serde::Serialize, - EL: serde::Serialize, - L: serde::Serialize, -{} + for TypedStaticGraph +where + Id: serde::Serialize, + NL: serde::Serialize, + EL: serde::Serialize, + L: serde::Serialize, +{ +} impl Deserialize -for TypedStaticGraph - where - Id: for<'de> serde::Deserialize<'de>, - NL: for<'de> serde::Deserialize<'de>, - EL: for<'de> serde::Deserialize<'de>, - L: for<'de> serde::Deserialize<'de>, -{} + for TypedStaticGraph +where + Id: for<'de> serde::Deserialize<'de>, + NL: for<'de> serde::Deserialize<'de>, + EL: for<'de> serde::Deserialize<'de>, + L: for<'de> serde::Deserialize<'de>, +{ +} impl -TypedStaticGraph + TypedStaticGraph { pub fn empty() -> Self { Self::new(EdgeVec::default(), None, None, None) @@ -711,7 +714,11 @@ TypedStaticGraph self.bwd_adj_lists.as_ref() } - pub fn get_largest_adj_list_size(&self, node_or_edge_label: usize, direction: Direction) -> usize { + pub fn get_largest_adj_list_size( + &self, + node_or_edge_label: usize, + direction: Direction, + ) -> usize { if let Direction::Fwd = direction { return self.label_to_largest_fwd_adj_list_size[node_or_edge_label]; } @@ -720,7 +727,7 @@ TypedStaticGraph } impl GraphTrait -for TypedStaticGraph + for TypedStaticGraph { #[inline] fn get_node(&self, id: Id) -> NodeType { @@ -860,7 +867,7 @@ for TypedStaticGraph } impl -GraphLabelTrait for TypedStaticGraph + GraphLabelTrait for TypedStaticGraph { #[inline(always)] fn get_node_label_map(&self) -> &SetMap { @@ -957,11 +964,12 @@ GraphLabelTrait for TypedStaticGraph } impl UnGraphTrait -for TypedUnStaticGraph -{} + for TypedUnStaticGraph +{ +} impl DiGraphTrait -for TypedDiStaticGraph + for TypedDiStaticGraph { #[inline] fn in_degree(&self, id: Id) -> usize { @@ -982,7 +990,7 @@ for TypedDiStaticGraph } impl GeneralGraph -for TypedUnStaticGraph + for TypedUnStaticGraph { #[inline(always)] fn as_graph(&self) -> &GraphTrait { @@ -1001,7 +1009,7 @@ for TypedUnStaticGraph } impl GeneralGraph -for TypedDiStaticGraph + for TypedDiStaticGraph { #[inline(always)] fn as_graph(&self) -> &GraphTrait { @@ -1047,7 +1055,7 @@ fn _merge_labels(_labels1: Option>, _labels2: Option>) -> Op } impl Add -for TypedStaticGraph + for TypedStaticGraph { type Output = TypedStaticGraph; diff --git a/src/graph_impl/static_graph/sorted_adj_vec.rs b/src/graph_impl/static_graph/sorted_adj_vec.rs index c1c248a8..8bd939c8 100644 --- a/src/graph_impl/static_graph/sorted_adj_vec.rs +++ b/src/graph_impl/static_graph/sorted_adj_vec.rs @@ -67,17 +67,29 @@ impl SortedAdjVec { } } - pub fn intersect(&self, label_or_type: usize, some_neighbours: &mut Neighbours, neighbours: &mut Neighbours) -> usize { + pub fn intersect( + &self, + label_or_type: usize, + some_neighbours: &mut Neighbours, + neighbours: &mut Neighbours, + ) -> usize { self.inner_intersect( - some_neighbours, neighbours, &self.neighbour_ids, + some_neighbours, + neighbours, + &self.neighbour_ids, self.label_or_type_offsets[label_or_type], self.label_or_type_offsets[label_or_type + 1], ); self.label_or_type_offsets[label_or_type + 1] - self.label_or_type_offsets[label_or_type] } - fn inner_intersect(&self, some_neighbours: &mut Neighbours, neighbours: &mut Neighbours, - neighbour_ids: &Vec, mut this_idx: usize, this_idx_end: usize, + fn inner_intersect( + &self, + some_neighbours: &mut Neighbours, + neighbours: &mut Neighbours, + neighbour_ids: &Vec, + mut this_idx: usize, + this_idx_end: usize, ) { neighbours.reset(); let some_neighbour_ids = some_neighbours.ids.clone(); @@ -86,13 +98,16 @@ impl SortedAdjVec { while this_idx < this_idx_end && some_idx < some_end_idx { if neighbour_ids[this_idx] < some_neighbour_ids[some_idx] { this_idx += 1; - while this_idx < this_idx_end && neighbour_ids[this_idx] < some_neighbour_ids[some_idx] { + while this_idx < this_idx_end + && neighbour_ids[this_idx] < some_neighbour_ids[some_idx] + { this_idx += 1; } } else if neighbour_ids[this_idx] > some_neighbour_ids[some_idx] { some_idx += 1; - ; - while some_idx < some_end_idx && neighbour_ids[this_idx] > some_neighbour_ids[some_idx] { + while some_idx < some_end_idx + && neighbour_ids[this_idx] > some_neighbour_ids[some_idx] + { some_idx += 1; } } else { From 911dbc0f4effb1b4e0e51beb73aa1f90d6543793 Mon Sep 17 00:00:00 2001 From: Michael Chen <920901835@qq.com> Date: Fri, 27 Dec 2019 20:44:11 +0800 Subject: [PATCH 12/25] fix:planner --- src/graph_impl/multi_graph/mod.rs | 2 +- .../multi_graph/plan/operator/extend/EI.rs | 62 +- .../plan/operator/extend/Extend.rs | 6 +- .../plan/operator/extend/Intersect.rs | 4 +- .../plan/operator/extend/extend.rs | 6 +- .../plan/operator/extend/intersect.rs | 4 +- .../plan/operator/hashjoin/build.rs | 2 +- .../plan/operator/hashjoin/hash_join.rs | 2 +- .../plan/operator/hashjoin/probe.rs | 4 +- .../plan/operator/hashjoin/probe_cartesian.rs | 2 +- .../operator/hashjoin/probe_multi_vertices.rs | 6 +- .../probe_multi_vertices_cartesian.rs | 2 +- .../multi_graph/plan/operator/mod.rs | 34 +- .../multi_graph/plan/operator/operator.rs | 2 +- .../multi_graph/plan/operator/scan/scan.rs | 6 +- .../plan/operator/scan/scan_blocking.rs | 2 +- .../plan/operator/scan/scan_sampling.rs | 2 +- .../multi_graph/plan/operator/sink/sink.rs | 3 +- .../plan/operator/sink/sink_copy.rs | 2 +- .../plan/operator/sink/sink_limit.rs | 2 +- .../plan/operator/sink/sink_print.rs | 2 +- src/graph_impl/multi_graph/plan/query_plan.rs | 37 +- .../catalog/adj_list_descriptor.rs | 0 .../{ => planner}/catalog/catalog.rs | 19 +- .../{ => planner}/catalog/catalog_plans.rs | 4 +- .../multi_graph/{ => planner}/catalog/mod.rs | 1 + .../catalog/operator/intersect_catalog.rs | 193 ++++++ .../planner/catalog/operator/mod.rs | 2 + .../planner/catalog/operator/noop.rs | 67 ++ .../{ => planner}/catalog/query_edge.rs | 0 .../{ => planner}/catalog/query_graph.rs | 136 +++- .../catalog/subgraph_mapping_iterator.rs | 2 +- src/graph_impl/multi_graph/planner/mod.rs | 3 + .../multi_graph/planner/query_planner.rs | 602 ++++++++++++++++++ .../multi_graph/planner/query_planner_big.rs | 170 +++++ .../multi_graph/query/query_graph_set.rs | 2 +- src/graph_impl/static_graph/graph.rs | 81 ++- 37 files changed, 1375 insertions(+), 101 deletions(-) rename src/graph_impl/multi_graph/{ => planner}/catalog/adj_list_descriptor.rs (100%) rename src/graph_impl/multi_graph/{ => planner}/catalog/catalog.rs (97%) rename src/graph_impl/multi_graph/{ => planner}/catalog/catalog_plans.rs (94%) rename src/graph_impl/multi_graph/{ => planner}/catalog/mod.rs (91%) create mode 100644 src/graph_impl/multi_graph/planner/catalog/operator/intersect_catalog.rs create mode 100644 src/graph_impl/multi_graph/planner/catalog/operator/mod.rs create mode 100644 src/graph_impl/multi_graph/planner/catalog/operator/noop.rs rename src/graph_impl/multi_graph/{ => planner}/catalog/query_edge.rs (100%) rename src/graph_impl/multi_graph/{ => planner}/catalog/query_graph.rs (62%) rename src/graph_impl/multi_graph/{ => planner}/catalog/subgraph_mapping_iterator.rs (99%) create mode 100644 src/graph_impl/multi_graph/planner/mod.rs create mode 100644 src/graph_impl/multi_graph/planner/query_planner.rs create mode 100644 src/graph_impl/multi_graph/planner/query_planner_big.rs diff --git a/src/graph_impl/multi_graph/mod.rs b/src/graph_impl/multi_graph/mod.rs index 21fb3639..1cafe0be 100644 --- a/src/graph_impl/multi_graph/mod.rs +++ b/src/graph_impl/multi_graph/mod.rs @@ -1,6 +1,6 @@ #[macro_use] pub mod plan; -pub mod catalog; pub mod node; +pub mod planner; pub mod query; pub mod utils; diff --git a/src/graph_impl/multi_graph/plan/operator/extend/EI.rs b/src/graph_impl/multi_graph/plan/operator/extend/EI.rs index 0af832aa..29e89173 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/EI.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/EI.rs @@ -1,6 +1,4 @@ use generic::{GraphType, IdType}; -use graph_impl::multi_graph::catalog::adj_list_descriptor::{AdjListDescriptor, Direction}; -use graph_impl::multi_graph::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::plan::operator::extend::extend::Extend; use graph_impl::multi_graph::plan::operator::extend::intersect::{Intersect, IntersectType}; use graph_impl::multi_graph::plan::operator::hashjoin::probe::Probe; @@ -10,6 +8,10 @@ use graph_impl::multi_graph::plan::operator::operator::{ }; use graph_impl::multi_graph::plan::operator::scan::scan::Scan; use graph_impl::multi_graph::plan::operator::sink::sink::Sink; +use graph_impl::multi_graph::planner::catalog::adj_list_descriptor::{ + AdjListDescriptor, Direction, +}; +use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::static_graph::sorted_adj_vec::SortedAdjVec; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; @@ -125,34 +127,6 @@ impl BaseEI { &(" TO (".to_owned() + &self.to_query_vertex + ") From (" + &variables + ")"); } - pub fn make( - to_qvertex: String, - to_type: usize, - alds: Vec, - out_subgraph: QueryGraph, - in_subgraph: QueryGraph, - out_qvertex_to_idx_map: HashMap, - ) -> EI { - if 1 == alds.len() { - return EI::Extend(Extend::new( - to_qvertex, - to_type, - alds, - Box::new(out_subgraph), - Some(Box::new(in_subgraph)), - out_qvertex_to_idx_map, - )); - } - EI::Intersect(Intersect::new( - to_qvertex, - to_type, - alds, - Box::new(out_subgraph), - Some(Box::new(in_subgraph)), - out_qvertex_to_idx_map, - )) - } - pub fn is_intersection_cached(&mut self) -> bool { self.is_intersection_cached = true; for i in 0..self.last_vertex_ids_intersected.as_ref().unwrap().len() { @@ -391,6 +365,34 @@ impl EI { } false } + + pub fn make( + to_qvertex: String, + to_type: usize, + alds: Vec, + out_subgraph: QueryGraph, + in_subgraph: QueryGraph, + out_qvertex_to_idx_map: HashMap, + ) -> EI { + if 1 == alds.len() { + return EI::Extend(Extend::new( + to_qvertex, + to_type, + alds, + Box::new(out_subgraph), + Some(Box::new(in_subgraph)), + out_qvertex_to_idx_map, + )); + } + EI::Intersect(Intersect::new( + to_qvertex, + to_type, + alds, + Box::new(out_subgraph), + Some(Box::new(in_subgraph)), + out_qvertex_to_idx_map, + )) + } } impl CommonOperatorTrait for EI { diff --git a/src/graph_impl/multi_graph/plan/operator/extend/Extend.rs b/src/graph_impl/multi_graph/plan/operator/extend/Extend.rs index f7a05b9b..e5afee1b 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/Extend.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/Extend.rs @@ -1,6 +1,4 @@ use generic::{GraphType, IdType}; -use graph_impl::multi_graph::catalog::adj_list_descriptor::{AdjListDescriptor, Direction}; -use graph_impl::multi_graph::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::plan::operator::extend::EI::{ BaseEI, Neighbours, DIFFERENTIATE_FWD_BWD_SINGLE_ALD, EI, }; @@ -9,6 +7,10 @@ use graph_impl::multi_graph::plan::operator::hashjoin::probe_multi_vertices::PMV use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; use graph_impl::multi_graph::plan::operator::scan::scan::Scan; use graph_impl::multi_graph::plan::operator::sink::sink::Sink; +use graph_impl::multi_graph::planner::catalog::adj_list_descriptor::{ + AdjListDescriptor, Direction, +}; +use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::static_graph::sorted_adj_vec::SortedAdjVec; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; diff --git a/src/graph_impl/multi_graph/plan/operator/extend/Intersect.rs b/src/graph_impl/multi_graph/plan/operator/extend/Intersect.rs index 2aad84ca..347227e3 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/Intersect.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/Intersect.rs @@ -1,6 +1,4 @@ use generic::{GraphType, IdType}; -use graph_impl::multi_graph::catalog::adj_list_descriptor::AdjListDescriptor; -use graph_impl::multi_graph::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::plan::operator::extend::EI::EI::Base; use graph_impl::multi_graph::plan::operator::extend::EI::{BaseEI, CachingType, Neighbours, EI}; use graph_impl::multi_graph::plan::operator::hashjoin::probe::Probe; @@ -8,6 +6,8 @@ use graph_impl::multi_graph::plan::operator::hashjoin::probe_multi_vertices::PMV use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; use graph_impl::multi_graph::plan::operator::scan::scan::Scan; use graph_impl::multi_graph::plan::operator::sink::sink::Sink; +use graph_impl::multi_graph::planner::catalog::adj_list_descriptor::AdjListDescriptor; +use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; use std::hash::{BuildHasherDefault, Hash}; diff --git a/src/graph_impl/multi_graph/plan/operator/extend/extend.rs b/src/graph_impl/multi_graph/plan/operator/extend/extend.rs index f7a05b9b..e5afee1b 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/extend.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/extend.rs @@ -1,6 +1,4 @@ use generic::{GraphType, IdType}; -use graph_impl::multi_graph::catalog::adj_list_descriptor::{AdjListDescriptor, Direction}; -use graph_impl::multi_graph::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::plan::operator::extend::EI::{ BaseEI, Neighbours, DIFFERENTIATE_FWD_BWD_SINGLE_ALD, EI, }; @@ -9,6 +7,10 @@ use graph_impl::multi_graph::plan::operator::hashjoin::probe_multi_vertices::PMV use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; use graph_impl::multi_graph::plan::operator::scan::scan::Scan; use graph_impl::multi_graph::plan::operator::sink::sink::Sink; +use graph_impl::multi_graph::planner::catalog::adj_list_descriptor::{ + AdjListDescriptor, Direction, +}; +use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::static_graph::sorted_adj_vec::SortedAdjVec; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; diff --git a/src/graph_impl/multi_graph/plan/operator/extend/intersect.rs b/src/graph_impl/multi_graph/plan/operator/extend/intersect.rs index 2aad84ca..347227e3 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/intersect.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/intersect.rs @@ -1,6 +1,4 @@ use generic::{GraphType, IdType}; -use graph_impl::multi_graph::catalog::adj_list_descriptor::AdjListDescriptor; -use graph_impl::multi_graph::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::plan::operator::extend::EI::EI::Base; use graph_impl::multi_graph::plan::operator::extend::EI::{BaseEI, CachingType, Neighbours, EI}; use graph_impl::multi_graph::plan::operator::hashjoin::probe::Probe; @@ -8,6 +6,8 @@ use graph_impl::multi_graph::plan::operator::hashjoin::probe_multi_vertices::PMV use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; use graph_impl::multi_graph::plan::operator::scan::scan::Scan; use graph_impl::multi_graph::plan::operator::sink::sink::Sink; +use graph_impl::multi_graph::planner::catalog::adj_list_descriptor::AdjListDescriptor; +use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; use std::hash::{BuildHasherDefault, Hash}; diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/build.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/build.rs index 5dd3287e..233bd695 100644 --- a/src/graph_impl/multi_graph/plan/operator/hashjoin/build.rs +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/build.rs @@ -1,9 +1,9 @@ use generic::{GraphTrait, GraphType, IdType}; -use graph_impl::multi_graph::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::plan::operator::hashjoin::hash_table::HashTable; use graph_impl::multi_graph::plan::operator::operator::{ BaseOperator, CommonOperatorTrait, Operator, }; +use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; use std::hash::{BuildHasherDefault, Hash}; diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/hash_join.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/hash_join.rs index b57cdb39..7ef1a07b 100644 --- a/src/graph_impl/multi_graph/plan/operator/hashjoin/hash_join.rs +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/hash_join.rs @@ -1,5 +1,4 @@ use generic::IdType; -use graph_impl::multi_graph::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::plan::operator::extend::EI::EI; use graph_impl::multi_graph::plan::operator::hashjoin::build::Build; use graph_impl::multi_graph::plan::operator::hashjoin::probe::{BaseProbe, Probe}; @@ -12,6 +11,7 @@ use graph_impl::multi_graph::plan::operator::operator::Operator; use graph_impl::multi_graph::plan::operator::scan::scan::Scan; use graph_impl::multi_graph::plan::operator::sink::sink::Sink; use graph_impl::multi_graph::plan::query_plan::QueryPlan; +use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use hashbrown::HashMap; pub struct HashJoin {} diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe.rs index 75b91567..295dbb8a 100644 --- a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe.rs +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe.rs @@ -1,9 +1,9 @@ use generic::{GraphType, IdType}; -use graph_impl::multi_graph::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::plan::operator::hashjoin::hash_table::{BlockInfo, HashTable}; use graph_impl::multi_graph::plan::operator::operator::{ BaseOperator, CommonOperatorTrait, Operator, }; +use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::plan::operator::hashjoin::probe_cartesian::ProbeCartesian; use graph_impl::multi_graph::plan::operator::hashjoin::probe_multi_vertices::PMV; @@ -73,7 +73,7 @@ impl CommonOperatorTrait for BaseProbe { fn process_new_tuple(&mut self) { let hash_vertex = self.base_op.probe_tuple[self.probe_hash_idx].id(); - for mut hash_table in &mut self.hash_tables { + for hash_table in &mut self.hash_tables { let last_chunk_idx = hash_table.num_chunks[hash_vertex]; let mut prev_first_item = -1i32; for chunk_idx in 0..last_chunk_idx { diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_cartesian.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_cartesian.rs index 12e844ee..a75ee165 100644 --- a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_cartesian.rs +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_cartesian.rs @@ -1,8 +1,8 @@ use generic::{GraphTrait, GraphType, IdType}; -use graph_impl::multi_graph::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::plan::operator::hashjoin::hash_table::BlockInfo; use graph_impl::multi_graph::plan::operator::hashjoin::probe::{BaseProbe, Probe}; use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; +use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; use itertools::join; diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices.rs index 6055d3ec..5f748f69 100644 --- a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices.rs +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices.rs @@ -1,8 +1,8 @@ use generic::{GraphType, IdType}; -use graph_impl::multi_graph::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::plan::operator::hashjoin::probe::{BaseProbe, Probe}; use graph_impl::multi_graph::plan::operator::hashjoin::probe_multi_vertices_cartesian::ProbeMultiVerticesCartesian; use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; +use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; use itertools::Itertools; @@ -76,7 +76,7 @@ impl CommonOperatorTrait for ProbeMultiVertices { fn process_new_tuple(&mut self) { let hash_vertex = self.base_probe.base_op.probe_tuple[self.base_probe.probe_hash_idx].id(); - for mut hash_table in self.base_probe.hash_tables.clone() { + for hash_table in self.base_probe.hash_tables.clone() { let last_chunk_idx = hash_table.num_chunks[hash_vertex]; for chunk_idx in 0..last_chunk_idx { hash_table.get_block_and_offsets( @@ -87,7 +87,7 @@ impl CommonOperatorTrait for ProbeMultiVertices { let offset = self.base_probe.block_info.start_offset; (self.base_probe.block_info.start_offset..self.base_probe.block_info.end_offset) .step(self.base_probe.hashed_tuple_len) - .map(|offset| { + .for_each(|offset| { let mut flag = true; for i in 0..self.probe_indices.len() { if self.base_probe.base_op.probe_tuple[self.probe_indices[i]] diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices_cartesian.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices_cartesian.rs index 66760b86..fe2d5b11 100644 --- a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices_cartesian.rs +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices_cartesian.rs @@ -1,11 +1,11 @@ use generic::{GraphTrait, GraphType, IdType}; -use graph_impl::multi_graph::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::plan::operator::hashjoin::hash_table::BlockInfo; use graph_impl::multi_graph::plan::operator::hashjoin::probe::Probe; use graph_impl::multi_graph::plan::operator::hashjoin::probe_multi_vertices::{ ProbeMultiVertices, PMV, }; use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; +use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; use std::hash::{BuildHasherDefault, Hash}; diff --git a/src/graph_impl/multi_graph/plan/operator/mod.rs b/src/graph_impl/multi_graph/plan/operator/mod.rs index dc9d3bb9..6b1076ff 100644 --- a/src/graph_impl/multi_graph/plan/operator/mod.rs +++ b/src/graph_impl/multi_graph/plan/operator/mod.rs @@ -70,7 +70,7 @@ macro_rules! get_scan_as_ref { macro_rules! get_base_op_as_mut { ($item:expr) => { match $item { - Operator::Base(base) => &mut base, + Operator::Base(base) => base, Operator::Sink(sink) => match sink { Sink::BaseSink(base) => &mut base.base_op, Sink::SinkCopy(sc) => &mut sc.base_sink.base_op, @@ -98,6 +98,38 @@ macro_rules! get_base_op_as_mut { }; } +#[macro_export] +macro_rules! get_base_op_as_ref { + ($item:expr) => { + match $item { + Operator::Base(base) => &base, + Operator::Sink(sink) => match sink { + Sink::BaseSink(base) => &base.base_op, + Sink::SinkCopy(sc) => &sc.base_sink.base_op, + Sink::SinkPrint(sp) => &sp.base_sink.base_op, + Sink::SinkLimit(sl) => &sl.base_sink.base_op, + }, + Operator::Scan(scan) => match scan { + Scan::Base(base) => &base.base_op, + Scan::ScanSampling(ss) => &ss.base_scan.base_op, + Scan::ScanBlocking(sb) => &sb.base_scan.base_op, + }, + Operator::EI(ei) => match ei { + EI::Base(base) => &base.base_op, + EI::Extend(base) => &base.base_ei.base_op, + EI::Intersect(base) => &base.base_ei.base_op, + }, + Operator::Build(build) => &build.base_op, + Operator::Probe(probe) => match probe { + Probe::BaseProbe(base) => &base.base_op, + Probe::PC(pc) => &pc.base_probe.base_op, + Probe::PMV(PMV::BasePMV(base)) => &base.base_probe.base_op, + Probe::PMV(PMV::PMVC(pmvc)) => &pmvc.base_pmv.base_probe.base_op, + }, + } + }; +} + /// Get common attributes(Origin) from Operator #[macro_export] macro_rules! get_op_attr { diff --git a/src/graph_impl/multi_graph/plan/operator/operator.rs b/src/graph_impl/multi_graph/plan/operator/operator.rs index b4630bab..c1fd2756 100644 --- a/src/graph_impl/multi_graph/plan/operator/operator.rs +++ b/src/graph_impl/multi_graph/plan/operator/operator.rs @@ -1,5 +1,4 @@ use generic::{GraphType, IdType}; -use graph_impl::multi_graph::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::plan::operator::extend::EI::EI; use graph_impl::multi_graph::plan::operator::hashjoin::build::Build; use graph_impl::multi_graph::plan::operator::hashjoin::probe::Probe; @@ -7,6 +6,7 @@ use graph_impl::multi_graph::plan::operator::hashjoin::probe_multi_vertices::PMV use graph_impl::multi_graph::plan::operator::scan::scan::{BaseScan, Scan}; use graph_impl::multi_graph::plan::operator::scan::scan_sampling::ScanSampling; use graph_impl::multi_graph::plan::operator::sink::sink::Sink; +use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::TypedStaticGraph; use hashbrown::{HashMap, HashSet}; use map::SetMap; diff --git a/src/graph_impl/multi_graph/plan/operator/scan/scan.rs b/src/graph_impl/multi_graph/plan/operator/scan/scan.rs index 6cfbe01c..1d944b17 100644 --- a/src/graph_impl/multi_graph/plan/operator/scan/scan.rs +++ b/src/graph_impl/multi_graph/plan/operator/scan/scan.rs @@ -1,11 +1,11 @@ use generic::{GraphTrait, GraphType, IdType}; -use graph_impl::multi_graph::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::plan::operator::operator::{ BaseOperator, CommonOperatorTrait, Operator, }; use graph_impl::multi_graph::plan::operator::scan::scan_blocking::ScanBlocking; use graph_impl::multi_graph::plan::operator::scan::scan_sampling::ScanSampling; use graph_impl::multi_graph::plan::operator::sink::sink::Sink; +use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::static_graph::sorted_adj_vec::SortedAdjVec; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; @@ -22,8 +22,8 @@ pub enum Scan { #[derive(Clone)] pub struct BaseScan { pub base_op: BaseOperator, - from_query_vertex: String, - to_query_vertex: String, + pub from_query_vertex: String, + pub to_query_vertex: String, pub from_type: usize, pub to_type: usize, pub label_or_to_type: usize, diff --git a/src/graph_impl/multi_graph/plan/operator/scan/scan_blocking.rs b/src/graph_impl/multi_graph/plan/operator/scan/scan_blocking.rs index 2a53b6a8..b8ebae10 100644 --- a/src/graph_impl/multi_graph/plan/operator/scan/scan_blocking.rs +++ b/src/graph_impl/multi_graph/plan/operator/scan/scan_blocking.rs @@ -1,7 +1,7 @@ use generic::{GraphTrait, GraphType, IdType}; -use graph_impl::multi_graph::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; use graph_impl::multi_graph::plan::operator::scan::scan::BaseScan; +use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; use std::hash::{BuildHasherDefault, Hash}; diff --git a/src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs b/src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs index f5426dc2..423912f7 100644 --- a/src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs +++ b/src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs @@ -1,9 +1,9 @@ use generic::{GraphType, IdType}; -use graph_impl::multi_graph::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::plan::operator::operator::{ BaseOperator, CommonOperatorTrait, Operator, }; use graph_impl::multi_graph::plan::operator::scan::scan::{BaseScan, Scan}; +use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; use rand::{thread_rng, Rng}; diff --git a/src/graph_impl/multi_graph/plan/operator/sink/sink.rs b/src/graph_impl/multi_graph/plan/operator/sink/sink.rs index 82844b00..85f12574 100644 --- a/src/graph_impl/multi_graph/plan/operator/sink/sink.rs +++ b/src/graph_impl/multi_graph/plan/operator/sink/sink.rs @@ -1,5 +1,4 @@ use generic::{GraphType, IdType}; -use graph_impl::multi_graph::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::plan::operator::extend::EI::EI; use graph_impl::multi_graph::plan::operator::hashjoin::probe::Probe; use graph_impl::multi_graph::plan::operator::hashjoin::probe_multi_vertices::PMV; @@ -10,11 +9,13 @@ use graph_impl::multi_graph::plan::operator::scan::scan::Scan; use graph_impl::multi_graph::plan::operator::sink::sink_copy::SinkCopy; use graph_impl::multi_graph::plan::operator::sink::sink_limit::SinkLimit; use graph_impl::multi_graph::plan::operator::sink::sink_print::SinkPrint; +use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; use std::hash::{BuildHasherDefault, Hash}; use std::rc::Rc; +#[derive(Clone)] pub enum SinkType { Copy, Print, diff --git a/src/graph_impl/multi_graph/plan/operator/sink/sink_copy.rs b/src/graph_impl/multi_graph/plan/operator/sink/sink_copy.rs index fb8084a6..b63dd39b 100644 --- a/src/graph_impl/multi_graph/plan/operator/sink/sink_copy.rs +++ b/src/graph_impl/multi_graph/plan/operator/sink/sink_copy.rs @@ -1,7 +1,7 @@ use generic::{GraphType, IdType}; -use graph_impl::multi_graph::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; use graph_impl::multi_graph::plan::operator::sink::sink::{BaseSink, Sink}; +use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; use std::hash::{BuildHasherDefault, Hash}; diff --git a/src/graph_impl/multi_graph/plan/operator/sink/sink_limit.rs b/src/graph_impl/multi_graph/plan/operator/sink/sink_limit.rs index 3fc829ce..221f528e 100644 --- a/src/graph_impl/multi_graph/plan/operator/sink/sink_limit.rs +++ b/src/graph_impl/multi_graph/plan/operator/sink/sink_limit.rs @@ -1,11 +1,11 @@ use generic::{GraphType, IdType}; -use graph_impl::multi_graph::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::plan::operator::extend::EI::EI; use graph_impl::multi_graph::plan::operator::hashjoin::probe::Probe; use graph_impl::multi_graph::plan::operator::hashjoin::probe_multi_vertices::PMV; use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; use graph_impl::multi_graph::plan::operator::scan::scan::Scan; use graph_impl::multi_graph::plan::operator::sink::sink::{BaseSink, Sink}; +use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::utils::io_utils::{current_time, get_elapsed_time_in_millis}; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; diff --git a/src/graph_impl/multi_graph/plan/operator/sink/sink_print.rs b/src/graph_impl/multi_graph/plan/operator/sink/sink_print.rs index 1da5c345..c4f107a5 100644 --- a/src/graph_impl/multi_graph/plan/operator/sink/sink_print.rs +++ b/src/graph_impl/multi_graph/plan/operator/sink/sink_print.rs @@ -1,7 +1,7 @@ use generic::{GraphType, IdType}; -use graph_impl::multi_graph::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; use graph_impl::multi_graph::plan::operator::sink::sink::{BaseSink, Sink}; +use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; use std::hash::{BuildHasherDefault, Hash}; diff --git a/src/graph_impl/multi_graph/plan/query_plan.rs b/src/graph_impl/multi_graph/plan/query_plan.rs index 476e29b3..c2ce0ffc 100644 --- a/src/graph_impl/multi_graph/plan/query_plan.rs +++ b/src/graph_impl/multi_graph/plan/query_plan.rs @@ -3,19 +3,20 @@ use graph_impl::multi_graph::plan::operator::extend::EI::EI; use graph_impl::multi_graph::plan::operator::hashjoin::probe::Probe; use graph_impl::multi_graph::plan::operator::hashjoin::probe_multi_vertices::PMV; use graph_impl::multi_graph::plan::operator::operator::{BaseOperator, Operator}; -use graph_impl::multi_graph::plan::operator::scan::scan::Scan; +use graph_impl::multi_graph::plan::operator::scan::scan::{BaseScan, Scan}; use graph_impl::multi_graph::plan::operator::scan::scan_sampling::ScanSampling; use graph_impl::multi_graph::plan::operator::sink::sink::{BaseSink, Sink, SinkType}; use hashbrown::HashMap; use std::ops::Deref; use std::rc::Rc; +#[derive(Clone)] pub struct QueryPlan { sink: Option>, - sink_type: SinkType, + pub sink_type: SinkType, scan_sampling: Option>, - last_operator: Option>>, - out_tuples_limit: usize, + pub last_operator: Option>>, + pub out_tuples_limit: usize, elapsed_time: f64, icost: usize, num_intermediate_tuples: usize, @@ -24,9 +25,9 @@ pub struct QueryPlan { executed: bool, adaptive_enabled: bool, pub subplans: Vec>>, - estimated_icost: f64, - estimated_num_out_tuples: f64, - q_vertex_to_num_out_tuples: HashMap, + pub estimated_icost: f64, + pub estimated_num_out_tuples: f64, + pub q_vertex_to_num_out_tuples: HashMap, } impl QueryPlan { @@ -101,6 +102,18 @@ impl QueryPlan { q_vertex_to_num_out_tuples: HashMap::new(), } } + pub fn new_from_last_op(last_operator: Scan, estimated_num_out_tuples: f64) -> Self { + let mut map = HashMap::new(); + let op = get_scan_as_ref!(&last_operator); + map.insert(op.from_query_vertex.clone(), estimated_num_out_tuples); + map.insert(op.to_query_vertex.clone(), estimated_num_out_tuples); + let mut plan = QueryPlan::new_from_operator(Box::new(Operator::Scan(last_operator))); + plan.estimated_num_out_tuples = estimated_num_out_tuples; + map.into_iter().for_each(|(k, v)| { + plan.q_vertex_to_num_out_tuples.insert(k, v); + }); + plan + } pub fn get_scan_sampling(&mut self) -> Option<&mut ScanSampling> { self.scan_sampling.as_mut() @@ -117,4 +130,14 @@ impl QueryPlan { pub fn shallow_copy(&self) -> QueryPlan { QueryPlan::new_from_subplans(self.subplans.clone()) } + + pub fn append(&mut self, mut new_operator: Operator) { + let mut last_operator = self.last_operator.as_mut().unwrap().as_mut(); + let last_op = get_base_op_as_mut!(&mut last_operator); + last_op.next = Some(vec![new_operator.clone()]); + let new_op = get_base_op_as_mut!(&mut new_operator); + new_op.prev = self.last_operator.clone(); + self.subplans.push(Box::new(new_operator.clone())); + self.last_operator = Some(Box::new(new_operator)); + } } diff --git a/src/graph_impl/multi_graph/catalog/adj_list_descriptor.rs b/src/graph_impl/multi_graph/planner/catalog/adj_list_descriptor.rs similarity index 100% rename from src/graph_impl/multi_graph/catalog/adj_list_descriptor.rs rename to src/graph_impl/multi_graph/planner/catalog/adj_list_descriptor.rs diff --git a/src/graph_impl/multi_graph/catalog/catalog.rs b/src/graph_impl/multi_graph/planner/catalog/catalog.rs similarity index 97% rename from src/graph_impl/multi_graph/catalog/catalog.rs rename to src/graph_impl/multi_graph/planner/catalog/catalog.rs index 331389f3..f63ae7b2 100644 --- a/src/graph_impl/multi_graph/catalog/catalog.rs +++ b/src/graph_impl/multi_graph/planner/catalog/catalog.rs @@ -1,9 +1,4 @@ use generic::{GraphTrait, GraphType, IdType}; -use graph_impl::multi_graph::catalog::adj_list_descriptor::{AdjListDescriptor, Direction}; -use graph_impl::multi_graph::catalog::catalog_plans::{ - CatalogPlans, DEF_MAX_INPUT_NUM_VERTICES, DEF_NUM_EDGES_TO_SAMPLE, -}; -use graph_impl::multi_graph::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::plan::operator::extend::EI::EI; use graph_impl::multi_graph::plan::operator::hashjoin::probe::Probe; use graph_impl::multi_graph::plan::operator::hashjoin::probe_multi_vertices::PMV; @@ -13,6 +8,13 @@ use graph_impl::multi_graph::plan::operator::operator::{ use graph_impl::multi_graph::plan::operator::scan::scan::Scan; use graph_impl::multi_graph::plan::operator::sink::sink::Sink; use graph_impl::multi_graph::plan::query_plan::QueryPlan; +use graph_impl::multi_graph::planner::catalog::adj_list_descriptor::{ + AdjListDescriptor, Direction, +}; +use graph_impl::multi_graph::planner::catalog::catalog_plans::{ + CatalogPlans, DEF_MAX_INPUT_NUM_VERTICES, DEF_NUM_EDGES_TO_SAMPLE, +}; +use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::utils::io_utils; use graph_impl::TypedStaticGraph; use hashbrown::{HashMap, HashSet}; @@ -27,6 +29,11 @@ use std::panic::catch_unwind; use std::ptr::null; use std::thread; +pub static SINGLE_VERTEX_WEIGHT_PROBE_COEF: f64 = 3.0; +pub static SINGLE_VERTEX_WEIGHT_BUILD_COEF: f64 = 12.0; +pub static MULTI_VERTEX_WEIGHT_PROBE_COEF: f64 = 12.0; +pub static MULTI_VERTEX_WEIGHT_BUILD_COEF: f64 = 720.0; + pub struct Catalog { in_subgraphs: Vec, sampled_icost: HashMap>, @@ -73,7 +80,7 @@ impl Catalog { pub fn get_icost( &self, query_graph: &mut QueryGraph, - alds: Vec, + alds: Vec<&AdjListDescriptor>, to_type: usize, ) -> f64 { let mut approx_icost = 0.0; diff --git a/src/graph_impl/multi_graph/catalog/catalog_plans.rs b/src/graph_impl/multi_graph/planner/catalog/catalog_plans.rs similarity index 94% rename from src/graph_impl/multi_graph/catalog/catalog_plans.rs rename to src/graph_impl/multi_graph/planner/catalog/catalog_plans.rs index 50e7f18c..aa9d8931 100644 --- a/src/graph_impl/multi_graph/catalog/catalog_plans.rs +++ b/src/graph_impl/multi_graph/planner/catalog/catalog_plans.rs @@ -1,7 +1,7 @@ use generic::{GraphLabelTrait, GraphTrait, GraphType, IdType}; -use graph_impl::multi_graph::catalog::adj_list_descriptor::AdjListDescriptor; -use graph_impl::multi_graph::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::plan::query_plan::QueryPlan; +use graph_impl::multi_graph::planner::catalog::adj_list_descriptor::AdjListDescriptor; +use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::query::query_graph_set::QueryGraphSet; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; diff --git a/src/graph_impl/multi_graph/catalog/mod.rs b/src/graph_impl/multi_graph/planner/catalog/mod.rs similarity index 91% rename from src/graph_impl/multi_graph/catalog/mod.rs rename to src/graph_impl/multi_graph/planner/catalog/mod.rs index c1a4a663..6b989ce7 100644 --- a/src/graph_impl/multi_graph/catalog/mod.rs +++ b/src/graph_impl/multi_graph/planner/catalog/mod.rs @@ -2,6 +2,7 @@ pub mod adj_list_descriptor; pub mod catalog; pub mod catalog_plans; +pub mod operator; pub mod query_edge; pub mod query_graph; pub mod subgraph_mapping_iterator; diff --git a/src/graph_impl/multi_graph/planner/catalog/operator/intersect_catalog.rs b/src/graph_impl/multi_graph/planner/catalog/operator/intersect_catalog.rs new file mode 100644 index 00000000..fb5d454a --- /dev/null +++ b/src/graph_impl/multi_graph/planner/catalog/operator/intersect_catalog.rs @@ -0,0 +1,193 @@ +use generic::{GraphType, IdType}; +use graph_impl::multi_graph::plan::operator::extend::intersect::Intersect; +use graph_impl::multi_graph::plan::operator::extend::EI::CachingType; +use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; +use graph_impl::multi_graph::planner::catalog::adj_list_descriptor::AdjListDescriptor; +use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; +use graph_impl::TypedStaticGraph; +use hashbrown::HashMap; +use std::hash::{BuildHasherDefault, Hash}; + +pub struct IntersectCatalog { + pub base_intersect: Intersect, + is_adj_list_sorted_by_type: bool, + last_icost: usize, +} + +impl IntersectCatalog { + pub fn new( + to_qvertex: String, + to_type: usize, + alds: Vec, + out_subgraph: QueryGraph, + in_subgraph: QueryGraph, + out_qvertex_to_idx_map: HashMap, + is_adj_list_sorted_by_type: bool, + ) -> IntersectCatalog { + IntersectCatalog { + base_intersect: Intersect::new( + to_qvertex, + to_type, + alds, + Box::new(out_subgraph), + Some(Box::new(in_subgraph)), + out_qvertex_to_idx_map, + ), + is_adj_list_sorted_by_type, + last_icost: 0, + } + } +} + +impl CommonOperatorTrait for IntersectCatalog { + fn init( + &mut self, + probe_tuple: Vec, + graph: &TypedStaticGraph, + ) { + self.base_intersect.init(probe_tuple, graph) + } + + fn process_new_tuple(&mut self) { + let base_ei = &mut self.base_intersect.base_ei; + if 1 == base_ei.alds.len() { + // intersect the adjacency lists and setAdjListSortOrder the output vertex values. + let adj = base_ei.adj_lists_to_cache[0] + [base_ei.base_op.probe_tuple[base_ei.vertex_idx_to_cache[0]].id()] + .as_ref() + .unwrap(); + adj.set_neighbor_ids( + base_ei.labels_or_to_types_to_cache[0], + &mut base_ei.out_neighbours, + ); + base_ei.base_op.icost += + base_ei.out_neighbours.end_idx - base_ei.out_neighbours.start_idx; + } else { + // intersect the adjacency lists and setAdjListSortOrder the output vertex values. + let mut temp; + if base_ei.caching_type == CachingType::None || !base_ei.is_intersection_cached() { + let adj = base_ei.adj_lists_to_cache[0] + [base_ei.base_op.probe_tuple[base_ei.vertex_idx_to_cache[0]].id()] + .as_ref() + .unwrap(); + adj.set_neighbor_ids( + base_ei.labels_or_to_types_to_cache[0], + &mut base_ei.init_neighbours, + ); + self.last_icost = + base_ei.init_neighbours.end_idx - base_ei.init_neighbours.start_idx; + let adj = base_ei.adj_lists_to_cache[1] + [base_ei.base_op.probe_tuple[base_ei.vertex_idx_to_cache[1]].id()] + .as_ref() + .unwrap(); + self.last_icost += adj.intersect( + base_ei.labels_or_to_types_to_cache[1], + &mut base_ei.init_neighbours, + &mut base_ei.cached_neighbours, + ); + if base_ei.to_type != 0 { + let mut curr_end_idx = 0; + for i in base_ei.cached_neighbours.start_idx..base_ei.cached_neighbours.end_idx + { + if base_ei.vertex_types[base_ei.cached_neighbours.ids[i].id()] + == base_ei.to_type + { + base_ei.cached_neighbours.ids[curr_end_idx] = + base_ei.cached_neighbours.ids[i]; + curr_end_idx += 1; + } + } + base_ei.cached_neighbours.end_idx = curr_end_idx; + } + for i in 2..base_ei.adj_lists_to_cache.len() { + temp = base_ei.cached_neighbours.clone(); + base_ei.cached_neighbours = base_ei.temp_neighbours.clone(); + base_ei.temp_neighbours = temp; + let adj = base_ei.adj_lists_to_cache[i] + [base_ei.base_op.probe_tuple[base_ei.vertex_idx_to_cache[i]].id()] + .as_ref() + .unwrap(); + self.last_icost += adj.intersect( + base_ei.labels_or_to_types_to_cache[i], + &mut base_ei.temp_neighbours, + &mut base_ei.cached_neighbours, + ); + } + } + match base_ei.caching_type { + CachingType::None | CachingType::FullCaching => { + base_ei.base_op.icost += self.last_icost; + base_ei.out_neighbours = base_ei.cached_neighbours.clone(); + } + CachingType::PartialCaching => { + let adj = base_ei.adj_lists[0] + [base_ei.base_op.probe_tuple[base_ei.vertex_idx[0]].id()] + .as_ref() + .unwrap(); + base_ei.base_op.icost += adj.intersect( + base_ei.labels_or_to_types[0], + &mut base_ei.cached_neighbours, + &mut base_ei.out_neighbours, + ); + for i in 1..base_ei.adj_lists.len() { + temp = base_ei.out_neighbours.clone(); + base_ei.out_neighbours = base_ei.temp_neighbours.clone(); + base_ei.temp_neighbours = temp; + let adj = base_ei.adj_lists[i] + [base_ei.base_op.probe_tuple[base_ei.vertex_idx[i]].id()] + .as_ref() + .unwrap(); + base_ei.base_op.icost += adj.intersect( + base_ei.labels_or_to_types[i], + &mut base_ei.temp_neighbours, + &mut base_ei.out_neighbours, + ); + } + } + } + } + + for idx in base_ei.out_neighbours.start_idx..base_ei.out_neighbours.end_idx { + base_ei.base_op.probe_tuple[base_ei.out_idx] = base_ei.out_neighbours.ids[idx]; + base_ei.base_op.num_out_tuples += 1; + if self.is_adj_list_sorted_by_type { + base_ei.base_op.next.as_mut().map(|next| { + next.get_mut(0).map(|next_op| { + next_op.process_new_tuple(); + }) + }); + } else { + let ops = base_ei.base_op.next.as_mut().unwrap(); + ops.get_mut( + base_ei.vertex_types[base_ei.base_op.probe_tuple[base_ei.out_idx].id()], + ) + .map(|next_op| next_op.process_new_tuple()); + } + } + } + + fn execute(&mut self) { + self.base_intersect.execute() + } + + fn get_alds_as_string(&self) -> String { + self.base_intersect.get_alds_as_string() + } + + fn update_operator_name(&mut self, query_vertex_to_index_map: HashMap) { + self.base_intersect + .update_operator_name(query_vertex_to_index_map) + } + + fn copy(&self, is_thread_safe: bool) -> Operator { + self.base_intersect.copy(is_thread_safe) + } + + fn is_same_as(&mut self, op: &mut Operator) -> bool { + self.base_intersect.is_same_as(op) + } + + fn get_num_out_tuples(&self) -> usize { + self.base_intersect.get_num_out_tuples() + } +} diff --git a/src/graph_impl/multi_graph/planner/catalog/operator/mod.rs b/src/graph_impl/multi_graph/planner/catalog/operator/mod.rs new file mode 100644 index 00000000..f11094bf --- /dev/null +++ b/src/graph_impl/multi_graph/planner/catalog/operator/mod.rs @@ -0,0 +1,2 @@ +pub mod intersect_catalog; +pub mod noop; diff --git a/src/graph_impl/multi_graph/planner/catalog/operator/noop.rs b/src/graph_impl/multi_graph/planner/catalog/operator/noop.rs new file mode 100644 index 00000000..d717b852 --- /dev/null +++ b/src/graph_impl/multi_graph/planner/catalog/operator/noop.rs @@ -0,0 +1,67 @@ +use generic::{GraphType, IdType}; +use graph_impl::multi_graph::plan::operator::operator::{ + BaseOperator, CommonOperatorTrait, Operator, +}; +use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; +use graph_impl::TypedStaticGraph; +use hashbrown::HashMap; +use itertools::Itertools; +use std::hash::{BuildHasherDefault, Hash}; + +pub struct Noop { + base_op: BaseOperator, +} + +impl Noop { + pub fn new(query_graph: QueryGraph) -> Noop { + Noop { + base_op: BaseOperator::new(Box::new(query_graph.clone()), Some(Box::new(query_graph))), + } + } +} + +impl CommonOperatorTrait for Noop { + fn init( + &mut self, + probe_tuple: Vec, + graph: &TypedStaticGraph, + ) { + self.base_op.probe_tuple = probe_tuple.clone(); + let next = self.base_op.next.as_mut().unwrap(); + for next_op in next { + next_op.init(probe_tuple.clone(), graph); + } + } + + fn process_new_tuple(&mut self) { + self.base_op.num_out_tuples += 1; + self.base_op.next.as_mut().map(|next| { + next.iter_mut() + .for_each(|next_op| next_op.process_new_tuple()) + }); + } + + fn execute(&mut self) { + self.base_op.execute() + } + + fn get_alds_as_string(&self) -> String { + self.base_op.get_alds_as_string() + } + + fn update_operator_name(&mut self, query_vertex_to_index_map: HashMap) { + self.base_op.update_operator_name(query_vertex_to_index_map) + } + + fn copy(&self, is_thread_safe: bool) -> Operator { + self.base_op.copy(is_thread_safe) + } + + fn is_same_as(&mut self, op: &mut Operator) -> bool { + self.base_op.is_same_as(op) + } + + fn get_num_out_tuples(&self) -> usize { + self.base_op.get_num_out_tuples() + } +} diff --git a/src/graph_impl/multi_graph/catalog/query_edge.rs b/src/graph_impl/multi_graph/planner/catalog/query_edge.rs similarity index 100% rename from src/graph_impl/multi_graph/catalog/query_edge.rs rename to src/graph_impl/multi_graph/planner/catalog/query_edge.rs diff --git a/src/graph_impl/multi_graph/catalog/query_graph.rs b/src/graph_impl/multi_graph/planner/catalog/query_graph.rs similarity index 62% rename from src/graph_impl/multi_graph/catalog/query_graph.rs rename to src/graph_impl/multi_graph/planner/catalog/query_graph.rs index 5d9e97b2..715ac418 100644 --- a/src/graph_impl/multi_graph/catalog/query_graph.rs +++ b/src/graph_impl/multi_graph/planner/catalog/query_graph.rs @@ -1,6 +1,6 @@ -use graph_impl::multi_graph::catalog::query_edge::QueryEdge; -use graph_impl::multi_graph::catalog::subgraph_mapping_iterator::SubgraphMappingIterator; -use hashbrown::HashMap; +use graph_impl::multi_graph::planner::catalog::query_edge::QueryEdge; +use graph_impl::multi_graph::planner::catalog::subgraph_mapping_iterator::SubgraphMappingIterator; +use hashbrown::{HashMap, HashSet}; use itertools::Itertools; use std::iter::FromIterator; @@ -13,6 +13,7 @@ pub struct QueryGraph { // Using `Box` here to enable clone. it: Option>, encoding: Option, + pub limit: usize, } impl QueryGraph { @@ -24,6 +25,7 @@ impl QueryGraph { q_edges: vec![], it: None, encoding: None, + limit: 0, } } @@ -59,11 +61,19 @@ impl QueryGraph { } pub fn get_query_vertices(&self) -> Vec { - let mut vec = vec![]; - for key in self.qvertex_to_qedges_map.keys() { - vec.push(key.clone()); - } - vec + self.qvertex_to_qedges_map + .keys() + .map(|x| x.clone()) + .collect() + } + pub fn get_query_vertices_as_set(&self) -> HashSet { + let mut set = HashSet::new(); + self.qvertex_to_qedges_map + .keys() + .for_each(|key| { + set.insert(key.clone()); + }); + set } pub fn get_query_vertex_type(&self, query_vertex: &str) -> usize { @@ -108,6 +118,25 @@ impl QueryGraph { &self.q_edges } + pub fn get_qedges(&self, variable: &String, neighbor_variable: &String) -> Vec { + if !self.qvertex_to_qedges_map.contains_key(variable) { + panic!("The variable '{}' is not present.", variable); + } + let contains_in_qedges = self + .qvertex_to_qedges_map + .get(variable) + .map_or(false, |map| map.contains_key(neighbor_variable)); + if !contains_in_qedges { + return vec![]; + } + self.qvertex_to_qedges_map + .get(variable) + .unwrap() + .get(neighbor_variable) + .unwrap() + .clone() + } + pub fn has_next(&mut self) -> bool { let mut iterator = self.it.take().unwrap(); let res = iterator.has_next(&self); @@ -232,4 +261,95 @@ impl QueryGraph { } None } + + pub fn add_qedges(&mut self, query_edges: &Vec) { + query_edges.iter().for_each(|edge| self.add_qedge(edge.clone())); + } + + pub fn add_qedge(&mut self, query_edge: QueryEdge) { + // Get the vertex IDs. + let from_qvertex = query_edge.from_query_vertex.clone(); + let to_qvertex = query_edge.to_query_vertex.clone(); + let from_type = query_edge.from_type.clone(); + let to_type = query_edge.to_type.clone(); + self.qvertex_to_type_map + .entry(from_qvertex.clone()) + .or_insert(0); + self.qvertex_to_type_map + .entry(to_qvertex.clone()) + .or_insert(0); + if 0 != from_type { + self.qvertex_to_type_map + .insert(from_qvertex.clone(), from_type); + } + if 0 != to_type { + self.qvertex_to_type_map.insert(to_qvertex.clone(), to_type); + } + // Set the in and out degrees for each variable. + if !self.qvertex_to_deg_map.contains_key(&from_qvertex) { + self.qvertex_to_deg_map + .insert(from_qvertex.clone(), vec![0; 2]); + } + *self + .qvertex_to_deg_map + .get_mut(&from_qvertex) + .unwrap() + .get_mut(0) + .unwrap() += 1; + if !self.qvertex_to_deg_map.contains_key(&to_qvertex) { + self.qvertex_to_deg_map + .insert(to_qvertex.clone(), vec![0; 2]); + } + self.qvertex_to_deg_map + .get_mut(&to_qvertex) + .map(|to| to[1] += 1); + // Add fwd edge from_qvertex -> to_qvertex to the qVertexToQEdgesMap. + self.add_qedge_to_qgraph(from_qvertex.clone(), to_qvertex.clone(), query_edge.clone()); + // Add bwd edge to_qvertex <- from_qvertex to the qVertexToQEdgesMap. + self.add_qedge_to_qgraph(to_qvertex.clone(), from_qvertex.clone(), query_edge.clone()); + self.q_edges.push(query_edge); + } + + fn add_qedge_to_qgraph(&mut self, from_qvertex: String, to_qvertex: String, q_edge: QueryEdge) { + self.qvertex_to_qedges_map + .entry(from_qvertex.clone()) + .or_insert(HashMap::new()); + self.qvertex_to_qedges_map + .get_mut(&from_qvertex) + .map(|qedge_map| qedge_map.entry(to_qvertex).or_insert(vec![q_edge])); + } + + pub fn get_neighbors(&self, from_var: Vec) -> HashSet { + let mut to_variables = HashSet::new(); + from_var.iter().for_each(|from| { + if !self.qvertex_to_qedges_map.contains_key(from) { + panic!("The variable '{}' is not present.", from); + } + self.get_neighbours_of_node(from).into_iter().for_each(|n| { + to_variables.insert(n); + }); + }); + from_var.iter().for_each(|from| { + to_variables.remove(from); + }); + to_variables + } + + pub fn get_neighbours_of_node(&self, from: &String) -> Vec { + if !self.qvertex_to_qedges_map.contains_key(from) { + panic!("The variable '{}' is not present.", from); + } + self.qvertex_to_qedges_map + .get(from) + .unwrap() + .keys() + .map(|key| key.clone()) + .collect() + } + + pub fn copy(&self) -> QueryGraph { + let mut q = QueryGraph::empty(); + q.add_qedges(&self.q_edges); + q + } } diff --git a/src/graph_impl/multi_graph/catalog/subgraph_mapping_iterator.rs b/src/graph_impl/multi_graph/planner/catalog/subgraph_mapping_iterator.rs similarity index 99% rename from src/graph_impl/multi_graph/catalog/subgraph_mapping_iterator.rs rename to src/graph_impl/multi_graph/planner/catalog/subgraph_mapping_iterator.rs index 5a85f6cc..11250e00 100644 --- a/src/graph_impl/multi_graph/catalog/subgraph_mapping_iterator.rs +++ b/src/graph_impl/multi_graph/planner/catalog/subgraph_mapping_iterator.rs @@ -1,4 +1,4 @@ -use graph_impl::multi_graph::catalog::query_graph::QueryGraph; +use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use hashbrown::HashMap; // An iterator over a set of possible mappings between two query graphs. diff --git a/src/graph_impl/multi_graph/planner/mod.rs b/src/graph_impl/multi_graph/planner/mod.rs new file mode 100644 index 00000000..ede2a1d7 --- /dev/null +++ b/src/graph_impl/multi_graph/planner/mod.rs @@ -0,0 +1,3 @@ +pub mod catalog; +pub mod query_planner; +pub mod query_planner_big; diff --git a/src/graph_impl/multi_graph/planner/query_planner.rs b/src/graph_impl/multi_graph/planner/query_planner.rs new file mode 100644 index 00000000..54905c9c --- /dev/null +++ b/src/graph_impl/multi_graph/planner/query_planner.rs @@ -0,0 +1,602 @@ +use generic::{GraphType, IdType}; +use graph_impl::multi_graph::plan::operator::extend::EI::{BaseEI, CachingType, EI}; +use graph_impl::multi_graph::plan::operator::hashjoin::hash_join::HashJoin; +use graph_impl::multi_graph::plan::operator::hashjoin::probe::{BaseProbe, Probe}; +use graph_impl::multi_graph::plan::operator::hashjoin::probe_multi_vertices::{ + ProbeMultiVertices, PMV, +}; +use graph_impl::multi_graph::plan::operator::operator::Operator; +use graph_impl::multi_graph::plan::operator::scan::scan::{BaseScan, Scan}; +use graph_impl::multi_graph::plan::operator::sink::sink::Sink; +use graph_impl::multi_graph::plan::operator::sink::sink::SinkType; +use graph_impl::multi_graph::plan::query_plan::QueryPlan; +use graph_impl::multi_graph::planner::catalog::adj_list_descriptor::{ + AdjListDescriptor, Direction, +}; +use graph_impl::multi_graph::planner::catalog::catalog::{ + Catalog, MULTI_VERTEX_WEIGHT_BUILD_COEF, MULTI_VERTEX_WEIGHT_PROBE_COEF, + SINGLE_VERTEX_WEIGHT_BUILD_COEF, SINGLE_VERTEX_WEIGHT_PROBE_COEF, +}; +use graph_impl::multi_graph::planner::catalog::query_edge::QueryEdge; +use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; +use graph_impl::TypedStaticGraph; +use hashbrown::{HashMap, HashSet}; +use itertools::min; +use std::cell::RefCell; +use std::hash::Hash; +use std::ptr::null; + +pub struct QueryPlanner { + subgraph_plans: HashMap>>>, + pub query_graph: QueryGraph, + pub num_qvertices: usize, + pub next_num_qvertices: usize, + graph: TypedStaticGraph, + catalog: Catalog, + has_limit: bool, + computed_selectivities: HashMap>, +} + +impl + QueryPlanner +{ + pub fn new( + query_graph: QueryGraph, + catalog: Catalog, + graph: TypedStaticGraph, + ) -> Self { + QueryPlanner { + subgraph_plans: HashMap::new(), + has_limit: query_graph.limit > 0, + next_num_qvertices: query_graph.get_num_qvertices(), + query_graph, + num_qvertices: 0, + graph, + catalog, + computed_selectivities: HashMap::new(), + } + } + + pub fn plan(&mut self) -> QueryPlan { + if self.num_qvertices == 2 { + return QueryPlan::new_from_operator(Box::new(Operator::Scan(Scan::Base( + BaseScan::new(Box::new(self.query_graph.clone())), + )))); + } + self.consider_all_scan_operators(); + while self.next_num_qvertices <= self.num_qvertices { + self.consider_all_next_query_extensions(); + self.next_num_qvertices += 1; + } + let key = self + .subgraph_plans + .get(&self.num_qvertices) + .unwrap() + .keys() + .next() + .unwrap(); + let mut best_plan = self.get_best_plan(self.num_qvertices, key); + // each operator added only sets its prev pointer (to reuse operator objects). + // the picked plan needs to set the next pointer for each operator in the linear subplans. + self.set_next_pointers(&mut best_plan); + if self.has_limit { + best_plan.sink_type = SinkType::Limit; + best_plan.out_tuples_limit = self.query_graph.limit; + } + best_plan + } + + pub fn set_next_pointers(&self, best_plan: &mut QueryPlan) { + best_plan + .subplans + .iter_mut() + .map(|op| op.as_mut()) + .for_each(|last_op| { + let mut cur_op = last_op; + while get_op_attr_as_ref!(cur_op, prev).is_some() { + let cur_copy = cur_op.clone(); + let prev = get_op_attr_as_mut!(cur_op, prev).as_mut().unwrap().as_mut(); + get_op_attr_as_mut!(prev, next).replace(vec![cur_copy]); + cur_op = get_op_attr_as_mut!(cur_op, prev).as_mut().unwrap().as_mut(); + } + }); + } + + fn consider_all_scan_operators(&mut self) { + self.next_num_qvertices = 2; + self.subgraph_plans + .entry(self.next_num_qvertices) + .or_insert(HashMap::new()); + for query_edge in self.query_graph.get_query_edges() { + let mut out_subgraph = QueryGraph::empty(); + out_subgraph.add_qedge(query_edge.clone()); + let scan = Scan::Base(BaseScan::new(Box::new(out_subgraph))); + let num_edges = self.get_num_edges(&query_edge); + let query_plan = QueryPlan::new_from_last_op(scan, num_edges as f64); + let mut query_plans = vec![]; + query_plans.push(query_plan); + let key = self.get_key(&mut vec![ + query_edge.from_query_vertex.clone(), + query_edge.to_query_vertex.clone(), + ]); + let plan = self + .subgraph_plans + .get_mut(&self.next_num_qvertices) + .unwrap(); + plan.insert(key, query_plans); + } + self.next_num_qvertices = 3; + } + + fn consider_all_next_query_extensions(&mut self) { + self.subgraph_plans + .entry(self.next_num_qvertices) + .or_insert(HashMap::new()); + let plan_map = self + .subgraph_plans + .get(&(self.next_num_qvertices - 1)) + .unwrap(); + let plan_map_keys: Vec = plan_map.keys().map(|v| v.clone()).collect(); + for key in plan_map_keys { + self.consider_all_next_extend_operators(&key); + } + if !self.has_limit && self.next_num_qvertices >= 4 { + let plan_map_keys: Vec = self + .subgraph_plans + .get(&self.next_num_qvertices) + .unwrap() + .keys() + .map(|v| v.clone()) + .collect(); + for plan_map_key in plan_map_keys { + self.consider_all_next_hash_join_operators(&plan_map_key); + } + } + } + + fn consider_all_next_extend_operators(&mut self, key: &String) { + let prev_plan_map = self + .subgraph_plans + .get(&(self.next_num_qvertices - 1)) + .unwrap(); + let prev_query_plans = prev_plan_map.get(key).unwrap(); + let op = prev_query_plans + .get(0) + .unwrap() + .last_operator + .as_ref() + .unwrap() + .as_ref(); + let prev_qvertices = get_op_attr_as_ref!(op, out_subgraph).get_query_vertices(); + let to_qvertices = self.query_graph.get_neighbors(prev_qvertices); + let prev_query_plans_len = prev_plan_map.get(key).unwrap().len(); + let mut plans = vec![]; + for to_qvertex in to_qvertices { + for i in 0..prev_query_plans_len { + plans.push(self.get_plan_with_next_extend_by_index(i, key, &to_qvertex)); + } + } + let plan_map = self + .subgraph_plans + .get_mut(&self.next_num_qvertices) + .unwrap(); + for (key, plan) in plans { + plan_map.entry(key.clone()).or_insert(vec![]); + plan_map.get_mut(&key).unwrap().push(plan); + } + } + + pub fn get_plan_with_next_extend_by_index( + &mut self, + prev_query_plan_index: usize, + key: &String, + to_qvertex: &String, + ) -> (String, QueryPlan) { + let prev_plan_map = self + .subgraph_plans + .get(&(self.next_num_qvertices - 1)) + .unwrap(); + let prev_query_plans = prev_plan_map.get(key).unwrap(); + let prev_query_plan = prev_query_plans.get(prev_query_plan_index).unwrap(); + self.get_plan_with_next_extend(prev_query_plan.clone(), to_qvertex) + } + + pub fn get_plan_with_next_extend( + &mut self, + mut prev_query_plan: QueryPlan, + to_qvertex: &String, + ) -> (String, QueryPlan) { + let last_operator = prev_query_plan.last_operator.as_ref().unwrap().as_ref(); + let base_last_op = get_base_op_as_ref!(last_operator); + let in_subgraph = base_last_op.out_subgraph.as_ref(); + let last_previous_repeated_index = base_last_op.last_repeated_vertex_idx; + let mut alds = vec![]; + let mut next_extend = self.get_next_ei(in_subgraph, to_qvertex, &mut alds, last_operator); + let next_copy = next_extend.clone(); + let base_next_extend = get_ei_as_mut!(&mut next_extend); + base_next_extend.init_caching(last_previous_repeated_index); + let prev_estimated_num_out_tuples = prev_query_plan.estimated_num_out_tuples; + let to_type = base_next_extend + .base_op + .out_subgraph + .get_query_vertex_type(to_qvertex); + let last_operator = prev_query_plan.last_operator.as_mut().unwrap().as_mut(); + let mut base_last_op = get_base_op_as_mut!(last_operator); + let in_subgraph = base_last_op.out_subgraph.as_mut(); + let estimated_selectivity = self.get_selectivity( + in_subgraph, + base_next_extend.base_op.out_subgraph.as_mut(), + &alds, + to_type, + ); + let icost; + if let CachingType::None = base_next_extend.caching_type { + icost = prev_estimated_num_out_tuples + * self.catalog.get_icost( + in_subgraph, + alds.iter().collect(), + base_next_extend.to_type, + ); + } else { + let mut out_tuples_to_process = prev_estimated_num_out_tuples; + if base_last_op.prev.is_some() { + let index = 0; + let mut last_estimated_num_out_tuples_for_extension_qvertex = -1.0; + for ald in alds.iter().filter(|ald| ald.vertex_idx > index) { + last_estimated_num_out_tuples_for_extension_qvertex = prev_query_plan + .q_vertex_to_num_out_tuples + .get(&ald.from_query_vertex) + .unwrap() + .clone(); + } + out_tuples_to_process /= last_estimated_num_out_tuples_for_extension_qvertex; + } + if let CachingType::FullCaching = base_next_extend.caching_type { + icost = out_tuples_to_process * self.catalog.get_icost(in_subgraph, alds.iter().collect(), to_type) + + // added to make caching effect on cost more robust. + (prev_estimated_num_out_tuples - out_tuples_to_process) * estimated_selectivity; + } else { + // cachingType == CachingType.PARTIAL_CACHING + let alds_to_cache = alds + .iter() + .filter(|ald| ald.vertex_idx <= last_previous_repeated_index) + .collect(); + let alds_to_always_intersect = alds + .iter() + .filter(|ald| ald.vertex_idx > last_previous_repeated_index) + .collect(); + let always_intersect_icost = prev_estimated_num_out_tuples + * self + .catalog + .get_icost(in_subgraph, alds_to_always_intersect, to_type); + let cached_intersect_icost = out_tuples_to_process + * self.catalog.get_icost(in_subgraph, alds_to_cache, to_type); + icost = prev_estimated_num_out_tuples * always_intersect_icost + + out_tuples_to_process * cached_intersect_icost + + // added to make caching effect on cost more robust. + (prev_estimated_num_out_tuples - out_tuples_to_process) * estimated_selectivity; + } + } + + let estimated_icost = prev_query_plan.estimated_icost + icost; + let estimated_num_out_tuples = prev_estimated_num_out_tuples * estimated_selectivity; + + let mut q_vertex_to_num_out_tuples = HashMap::new(); + prev_query_plan + .q_vertex_to_num_out_tuples + .iter() + .for_each(|(k, v)| { + q_vertex_to_num_out_tuples.insert(k.clone(), v.clone()); + }); + q_vertex_to_num_out_tuples.insert( + base_next_extend.to_query_vertex.clone(), + estimated_num_out_tuples, + ); + + let mut new_query_plan = prev_query_plan.shallow_copy(); + new_query_plan.estimated_icost = estimated_icost; + new_query_plan.estimated_num_out_tuples = estimated_num_out_tuples; + new_query_plan.append(Operator::EI(next_copy)); + new_query_plan.q_vertex_to_num_out_tuples = q_vertex_to_num_out_tuples; + ( + self.get_key( + &mut (base_next_extend + .base_op + .out_qvertex_to_idx_map + .keys() + .map(|k| k.clone()) + .collect()), + ), + new_query_plan, + ) + } + + fn get_next_ei( + &self, + in_subgraph: &QueryGraph, + to_qvertex: &String, + alds: &mut Vec, + last_operator: &Operator, + ) -> EI { + let mut out_subgraph = in_subgraph.copy(); + in_subgraph + .get_query_vertices() + .iter() + .for_each(|from_qvertex| { + if self + .query_graph + .contains_query_edge(from_qvertex, to_qvertex) + { + // simple query graph so there is only 1 query_edge, so get query_edge at index '0'. + let query_edge = self + .query_graph + .get_qedges(from_qvertex, to_qvertex) + .get(0) + .unwrap() + .clone(); + let index = get_op_attr_as_ref!(last_operator, out_qvertex_to_idx_map) + .get(from_qvertex) + .unwrap() + .clone(); + let direction = if from_qvertex == &query_edge.from_query_vertex { + Direction::Fwd + } else { + Direction::Bwd + }; + let label = query_edge.label; + alds.push(AdjListDescriptor::new( + from_qvertex.clone(), + index, + direction, + label, + )); + out_subgraph.add_qedge(query_edge); + } + }); + let mut output_variable_idx_map = HashMap::new(); + get_op_attr_as_ref!(last_operator, out_qvertex_to_idx_map) + .iter() + .for_each(|(k, v)| { + output_variable_idx_map.insert(k.clone(), v.clone()); + }); + output_variable_idx_map.insert(to_qvertex.clone(), output_variable_idx_map.len()); + EI::make( + to_qvertex.clone(), + self.query_graph.get_query_vertex_type(to_qvertex), + alds.clone(), + out_subgraph, + in_subgraph.clone(), + output_variable_idx_map, + ) + } + + fn get_selectivity( + &mut self, + in_subgraph: &mut QueryGraph, + out_subgraph: &mut QueryGraph, + alds: &Vec, + to_type: usize, + ) -> f64 { + let selectivity; + if self + .computed_selectivities + .contains_key(&out_subgraph.get_encoding()) + { + let computed_selectivity = self + .computed_selectivities + .get_mut(&out_subgraph.get_encoding()) + .unwrap(); + for (graph, selectivity) in computed_selectivity { + if graph.is_isomorphic_to(out_subgraph) { + return selectivity.clone(); + } + } + } else { + self.computed_selectivities + .insert(out_subgraph.get_encoding(), vec![]); + } + selectivity = self.catalog.get_selectivity(in_subgraph, alds, to_type); + self.computed_selectivities + .get_mut(&out_subgraph.get_encoding()) + .map(|selectivities| { + selectivities.push((out_subgraph.clone(), selectivity)); + }); + selectivity + } + + fn consider_all_next_hash_join_operators(&mut self, map_key: &String) { + let plan_map = self.subgraph_plans.get(&self.next_num_qvertices).unwrap(); + let plans = plan_map.get(map_key).unwrap(); + let op = plans[0].last_operator.as_ref().unwrap().as_ref(); + let out_subgraph = get_op_attr_as_ref!(op, out_subgraph).as_ref().clone(); + + let query_vertices = out_subgraph.get_query_vertices(); + let min_size = 3; + let mut max_size = out_subgraph.get_query_vertices().len() - min_size; + if max_size < min_size { + max_size = min_size; + } + for set_size in min_size..=max_size { + let plans = self.subgraph_plans.get(&set_size).unwrap().clone(); + for key in plans.keys() { + let prev_query_plan = self.get_best_plan(set_size, key); + let last_op = prev_query_plan.last_operator.as_ref().unwrap().as_ref(); + let base_last_op = get_base_op_as_ref!(last_op); + let prev_qvertices = base_last_op.out_subgraph.get_query_vertices_as_set(); + let is_subset = query_vertices + .iter() + .map(|v| prev_qvertices.contains(v)) + .filter(|&x| !x) + .count() + == 0; + if !is_subset { + return; + } + let mut other_set: Vec = query_vertices + .iter() + .filter(|&x| !prev_qvertices.contains(x)) + .map(|x| x.clone()) + .collect(); + if other_set.len() == 1 { + return; + } + let join_qvertices = + Self::get_join_qvertices(&out_subgraph, &prev_qvertices, &other_set); + if join_qvertices.len() < 1 + || join_qvertices.len() > 2 + || other_set.len() + join_qvertices.len() > self.next_num_qvertices - 1 + { + return; + } + join_qvertices.iter().for_each(|v| { + other_set.push(v.clone()); + }); + let rest_size = other_set.len(); + let rest_key = self.get_key(&mut other_set); + if !self + .subgraph_plans + .get(&rest_size) + .unwrap() + .contains_key(&rest_key) + { + return; + } + let other_prev_operator = self.get_best_plan(rest_size, &rest_key); + self.consider_hash_join_operator( + &out_subgraph, + query_vertices.clone(), + &prev_query_plan, + &other_prev_operator, + join_qvertices.len(), + ); + } + } + } + + fn get_join_qvertices( + query_graph: &QueryGraph, + vertices: &HashSet, + other_vertices: &Vec, + ) -> Vec { + let mut join_qvertices = HashSet::new(); + vertices.iter().for_each(|cur| { + other_vertices + .iter() + .filter(|&other| query_graph.contains_query_edge(cur, other)) + .for_each(|_other| { + join_qvertices.insert(cur.clone()); + }) + }); + join_qvertices.into_iter().collect() + } + + fn consider_hash_join_operator( + &mut self, + out_subgraph: &QueryGraph, + mut query_vertices: Vec, + subplan: &QueryPlan, + other_subplan: &QueryPlan, + num_join_qvertices: usize, + ) { + let is_plan_build_subplan = + subplan.estimated_num_out_tuples < other_subplan.estimated_num_out_tuples; + let build_subplan = if is_plan_build_subplan { + subplan + } else { + other_subplan + }; + let probe_subplan = if is_plan_build_subplan { + other_subplan + } else { + subplan + }; + let build_coef = if num_join_qvertices == 1 { + SINGLE_VERTEX_WEIGHT_BUILD_COEF + } else { + MULTI_VERTEX_WEIGHT_BUILD_COEF + }; + let probe_coef = if num_join_qvertices == 1 { + SINGLE_VERTEX_WEIGHT_PROBE_COEF + } else { + MULTI_VERTEX_WEIGHT_PROBE_COEF + }; + let icost = build_subplan.estimated_icost + + probe_subplan.estimated_icost + + build_coef * build_subplan.estimated_num_out_tuples + + probe_coef * probe_subplan.estimated_num_out_tuples; + + let key = self.get_key(&mut query_vertices); + let curr_best_query_plan = self.get_best_plan(query_vertices.len(), &key); + if curr_best_query_plan.estimated_icost > icost { + let mut query_plan = HashJoin::make( + out_subgraph.clone(), + build_subplan.clone(), + probe_subplan.clone(), + ); + query_plan.estimated_icost = icost; + query_plan.estimated_num_out_tuples = curr_best_query_plan.estimated_num_out_tuples; + + let mut q_vertex_to_num_out_tuples = HashMap::new(); + probe_subplan + .q_vertex_to_num_out_tuples + .iter() + .for_each(|(k, v)| { + q_vertex_to_num_out_tuples.insert(k.clone(), v.clone()); + }); + + let last_op = build_subplan.last_operator.as_ref().unwrap().as_ref(); + let base_last_op = get_base_op_as_ref!(last_op); + base_last_op + .out_subgraph + .get_query_vertices() + .iter() + .for_each(|v| { + q_vertex_to_num_out_tuples + .entry(v.clone()) + .or_insert(curr_best_query_plan.estimated_num_out_tuples); + }); + query_plan.q_vertex_to_num_out_tuples = q_vertex_to_num_out_tuples; + + let query_plans = self + .subgraph_plans + .get_mut(&query_vertices.len()) + .unwrap() + .get_mut(&key) + .unwrap(); + query_plans.clear(); + query_plans.push(query_plan); + } + } + + fn get_best_plan(&self, num_qvertices: usize, key: &String) -> QueryPlan { + let possible_query_plans = self + .subgraph_plans + .get(&num_qvertices) + .unwrap() + .get(key) + .unwrap(); + let mut best_plan = possible_query_plans.get(0).unwrap(); + possible_query_plans.iter().for_each(|possible_query_plan| { + if possible_query_plan.estimated_icost < best_plan.estimated_icost { + best_plan = possible_query_plan; + } + }); + best_plan.clone() + } + + pub fn get_num_edges(&self, query_edge: &QueryEdge) -> usize { + let from_type = self + .query_graph + .get_query_vertex_type((&query_edge.from_query_vertex)); + let to_type = self + .query_graph + .get_query_vertex_type(&query_edge.to_query_vertex); + let label = query_edge.label; + return self.graph.get_num_edges(from_type, to_type, label); + } + + fn get_key(&self, query_vertices: &mut Vec) -> String { + query_vertices.sort(); + serde_json::to_string(&query_vertices).unwrap() + } +} diff --git a/src/graph_impl/multi_graph/planner/query_planner_big.rs b/src/graph_impl/multi_graph/planner/query_planner_big.rs new file mode 100644 index 00000000..b0aeeb10 --- /dev/null +++ b/src/graph_impl/multi_graph/planner/query_planner_big.rs @@ -0,0 +1,170 @@ +use generic::{GraphType, IdType}; +use graph_impl::multi_graph::plan::operator::extend::EI::{BaseEI, CachingType, EI}; +use graph_impl::multi_graph::plan::operator::hashjoin::probe::{BaseProbe, Probe}; +use graph_impl::multi_graph::plan::operator::hashjoin::probe_multi_vertices::{ + ProbeMultiVertices, PMV, +}; +use graph_impl::multi_graph::plan::operator::operator::Operator; +use graph_impl::multi_graph::plan::operator::scan::scan::{BaseScan, Scan}; +use graph_impl::multi_graph::plan::operator::sink::sink::Sink; +use graph_impl::multi_graph::plan::operator::sink::sink::SinkType; +use graph_impl::multi_graph::plan::query_plan::QueryPlan; +use graph_impl::multi_graph::planner::catalog::catalog::Catalog; +use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; +use graph_impl::multi_graph::planner::query_planner::QueryPlanner; +use graph_impl::TypedStaticGraph; +use hashbrown::{HashMap, HashSet}; +use std::cmp::max; +use std::hash::Hash; + +pub struct QueryPlannerBigger { + base_planner: QueryPlanner, + subgraph_plans: HashMap>>, + num_top_plans_kept: usize, +} + +impl + QueryPlannerBigger +{ + pub fn new( + query_graph: QueryGraph, + catalog: Catalog, + graph: TypedStaticGraph, + ) -> Self { + let mut planner = QueryPlannerBigger { + base_planner: QueryPlanner::new(query_graph, catalog, graph), + subgraph_plans: HashMap::new(), + num_top_plans_kept: 5, + }; + if planner.base_planner.num_qvertices >= 15 { + planner.num_top_plans_kept = 3; + } + planner + } + + pub fn plan(&mut self) -> QueryPlan { + self.consider_least_selective_scans(); + while self.base_planner.next_num_qvertices <= self.base_planner.num_qvertices { + self.consider_next_query_extensions(); + self.base_planner.next_num_qvertices += 1; + } + let plans = self + .subgraph_plans + .get(&self.base_planner.num_qvertices) + .unwrap(); + let mut best_plan = plans.get(0).unwrap(); + for plan in plans { + if best_plan.estimated_icost > plan.estimated_icost { + best_plan = plan; + } + } + let mut best_plan = best_plan.clone(); + // each operator added only sets its prev pointer (to reuse operator objects). + // the picked plan needs to set the next pointer for each operator in the linear subplans. + self.base_planner.set_next_pointers(&mut best_plan); + if self.base_planner.query_graph.limit > 0 { + best_plan.sink_type = SinkType::Limit; + best_plan.out_tuples_limit = self.base_planner.query_graph.limit; + } + best_plan + } + + fn consider_least_selective_scans(&mut self) { + self.base_planner.next_num_qvertices = 2; /* level = 2 for edge scan */ + self.subgraph_plans.entry(2).or_insert(vec![]); + let mut edges_to_scan = vec![]; + let mut num_edges_to_scan = vec![]; + let q_edges = self.base_planner.query_graph.get_query_edges(); + for i in 0..self.num_top_plans_kept { + let edge = q_edges.get(i).unwrap().clone(); + num_edges_to_scan.push(self.base_planner.get_num_edges(&edge)); + edges_to_scan.push(edge); + } + + for i in self.num_top_plans_kept..q_edges.len() { + let num_edges = self.base_planner.get_num_edges(q_edges.get(i).unwrap()); + for j in 0..self.num_top_plans_kept { + if num_edges < num_edges_to_scan[j] { + edges_to_scan[j] = q_edges.get(i).unwrap().clone(); + num_edges_to_scan[j] = num_edges; + break; + } + } + } + for i in 0..self.num_top_plans_kept { + let mut output_subgraph = QueryGraph::empty(); + output_subgraph.add_qedge(edges_to_scan[i].clone()); + let scan = Scan::Base(BaseScan::new(Box::new(output_subgraph))); + let query_plan = QueryPlan::new_from_last_op(scan, num_edges_to_scan[i] as f64); + self.subgraph_plans + .get_mut(&self.base_planner.next_num_qvertices) + .unwrap() + .push(query_plan); + } + self.base_planner.next_num_qvertices = 3; + } + + fn consider_next_query_extensions(&mut self) { + let mut new_query_plans = vec![]; + let plans = self + .subgraph_plans + .get_mut(&(self.base_planner.next_num_qvertices - 1)) + .unwrap(); + for prev_query_plan in plans { + let last_base_op = prev_query_plan.last_operator.as_ref().unwrap().as_ref(); + let last_op = get_base_op_as_ref!(last_base_op); + let prev_qvertices = last_op.out_subgraph.get_query_vertices(); + let to_qvertices = self.base_planner.query_graph.get_neighbors(prev_qvertices); + let in_subgraph = last_op.out_subgraph.as_ref(); + let next_to_qvertices = Self::filter_to_qvertices_by_max_num_alds( + &self.base_planner.query_graph, + to_qvertices, + &in_subgraph, + ); + for to_qvertex in next_to_qvertices { + let (_key, plan) = self + .base_planner + .get_plan_with_next_extend(prev_query_plan.clone(), &to_qvertex); + let icost = plan.estimated_icost; + if new_query_plans.len() < self.num_top_plans_kept { + new_query_plans.push(plan); + } else { + for i in 0..self.num_top_plans_kept { + if new_query_plans.get(i).unwrap().estimated_icost > icost { + new_query_plans.insert(i, plan.clone()); + } + } + } + } + } + self.subgraph_plans + .insert(self.base_planner.next_num_qvertices, new_query_plans); + } + fn filter_to_qvertices_by_max_num_alds( + query_graph: &QueryGraph, + to_qvertices: HashSet, + in_subgraph: &QueryGraph, + ) -> Vec { + let mut max_num_alds = 0; + let mut to_qvertex_to_num_alds_map = HashMap::new(); + to_qvertices.iter().for_each(|to_qvertex| { + let num_alds = in_subgraph + .get_query_vertices() + .iter() + .filter(|&from_qvertex| query_graph.contains_query_edge(from_qvertex, to_qvertex)) + .count(); + max_num_alds = max(max_num_alds, num_alds); + to_qvertex_to_num_alds_map.insert(to_qvertex, num_alds); + }); + let final_max_num_alds = max_num_alds; + to_qvertices + .iter() + .filter(|&to_qvertex| { + to_qvertex_to_num_alds_map + .get(to_qvertex) + .map_or(false, |&to| to == final_max_num_alds) + }) + .map(|x| x.clone()) + .collect() + } +} diff --git a/src/graph_impl/multi_graph/query/query_graph_set.rs b/src/graph_impl/multi_graph/query/query_graph_set.rs index a8482972..254da206 100644 --- a/src/graph_impl/multi_graph/query/query_graph_set.rs +++ b/src/graph_impl/multi_graph/query/query_graph_set.rs @@ -1,4 +1,4 @@ -use graph_impl::multi_graph::catalog::query_graph::QueryGraph; +use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use hashbrown::{HashMap, HashSet}; pub struct QueryGraphSet { diff --git a/src/graph_impl/static_graph/graph.rs b/src/graph_impl/static_graph/graph.rs index 0bbff03d..1e1e841f 100644 --- a/src/graph_impl/static_graph/graph.rs +++ b/src/graph_impl/static_graph/graph.rs @@ -18,8 +18,6 @@ * specific language governing permissions and limitations * under the License. */ -#![feature(test)] - use std::borrow::Cow; use std::hash::{Hash, Hasher}; use std::marker::PhantomData; @@ -29,7 +27,7 @@ use generic::{ GraphLabelTrait, GraphTrait, GraphType, IdType, Iter, MapTrait, MutMapTrait, NodeTrait, NodeType, UnGraphTrait, Undirected, }; -use graph_impl::multi_graph::catalog::adj_list_descriptor::Direction; +use graph_impl::multi_graph::planner::catalog::adj_list_descriptor::Direction; use graph_impl::static_graph::node::StaticNode; use graph_impl::static_graph::sorted_adj_vec::SortedAdjVec; use graph_impl::static_graph::static_edge_iter::StaticEdgeIndexIter; @@ -60,31 +58,32 @@ pub struct TypedStaticGraph, + in_edge_vec: Option>, + // Maintain the node's labels, whose index is aligned with `offsets`. + labels: Option>, + // A marker of thr graph type, namely, directed or undirected. + graph_type: PhantomData, + // A map of node labels. + node_label_map: SetMap, + // A map of edge labels. + edge_label_map: SetMap, + sort_by_node: bool, // node Ids indexed by type and random access to node types. node_ids: Vec, // node_types[node_id] = node_label_id // the node_label_id has been shifted right and id 0 is prepared for no label item. node_types: Vec, node_type_offsets: Vec, - fwd_adj_lists: Vec>>, bwd_adj_lists: Vec>>, - + label_to_num_edges: Vec, label_to_largest_fwd_adj_list_size: Vec, label_to_largest_bwd_adj_list_size: Vec, - - edge_vec: EdgeVec, - in_edge_vec: Option>, - // Maintain the node's labels, whose index is aligned with `offsets`. - labels: Option>, - // A marker of thr graph type, namely, directed or undirected. - graph_type: PhantomData, - // A map of node labels. - node_label_map: SetMap, - // A map of edge labels. - edge_label_map: SetMap, + edge_key_to_num_edges_map: HashMap, + to_type_to_percentage_map: HashMap, + from_type_to_percentage_map: HashMap, } impl PartialEq @@ -221,14 +220,18 @@ impl node_type_offsets: vec![], fwd_adj_lists: vec![], bwd_adj_lists: vec![], + label_to_num_edges: vec![], label_to_largest_fwd_adj_list_size: vec![], label_to_largest_bwd_adj_list_size: vec![], + edge_key_to_num_edges_map: HashMap::new(), + to_type_to_percentage_map: HashMap::new(), edge_vec: edges, in_edge_vec: in_edges, labels: None, node_label_map: SetMap::::new(), edge_label_map: SetMap::::new(), graph_type: PhantomData, + from_type_to_percentage_map: HashMap::new(), }; g.partition_nodes(); g.partition_edges(); @@ -295,14 +298,18 @@ impl node_type_offsets: vec![], fwd_adj_lists: vec![], bwd_adj_lists: vec![], + label_to_num_edges: vec![], label_to_largest_fwd_adj_list_size: vec![], label_to_largest_bwd_adj_list_size: vec![], + edge_key_to_num_edges_map: HashMap::new(), + to_type_to_percentage_map: HashMap::new(), edge_vec: edges, in_edge_vec: in_edges, labels: Some(labels), node_label_map, edge_label_map, graph_type: PhantomData, + from_type_to_percentage_map: HashMap::new(), }; g.partition_nodes(); g.partition_edges(); @@ -375,14 +382,18 @@ impl node_type_offsets: vec![], fwd_adj_lists: vec![], bwd_adj_lists: vec![], + label_to_num_edges: vec![], label_to_largest_fwd_adj_list_size: vec![], label_to_largest_bwd_adj_list_size: vec![], + edge_key_to_num_edges_map: HashMap::new(), + to_type_to_percentage_map: HashMap::new(), edge_vec, in_edge_vec, labels, node_label_map, edge_label_map, graph_type: PhantomData, + from_type_to_percentage_map: HashMap::new(), }; g.partition_nodes(); g.partition_edges(); @@ -479,6 +490,38 @@ impl self.edge_vec.find_edge_index(start, target) } + pub fn get_num_edges(&self, from_type: usize, to_type: usize, label: usize) -> usize { + if from_type == 0 && to_type == 0 { + return self.label_to_num_edges[label]; + } else if from_type != 0 && to_type != 0 { + return self + .edge_key_to_num_edges_map + .get(&Self::get_edge_key(from_type, to_type, label)) + .unwrap() + .clone(); + } else if from_type != 0 { + return self + .from_type_to_percentage_map + .get(&Self::get_edge_key_by_label(from_type, label)) + .unwrap() + .clone(); + } + self.to_type_to_percentage_map + .get(&Self::get_edge_key_by_label(label, to_type)) + .unwrap() + .clone() + } + + pub fn get_edge_key(from_type: usize, to_type: usize, label: usize) -> i64 { + (((from_type & 0xFFFF) << 48) as i64) + | (((to_type & 0x0000FFFF) << 16) as i64) + | ((label & 0xFFFF) as i64) + } + + fn get_edge_key_by_label(from_label: usize, to_label: usize) -> i32 { + (((from_label & 0x0000FFFF) << 16) as i32) | ((to_label & 0xFFFF) as i32) + } + // Partition nodes by type and generating node_ids && offsets for retrieving. fn partition_nodes(&mut self) { if 0 == self.num_of_node_labels() { @@ -1079,8 +1122,11 @@ impl None, @@ -1091,6 +1137,7 @@ impl Date: Thu, 9 Jan 2020 10:55:31 +0800 Subject: [PATCH 13/25] fix:map init --- src/graph_impl/multi_graph/mod.rs | 1 + .../plan/operator/sink/sink_limit.rs | 2 +- .../multi_graph/planner/catalog/catalog.rs | 9 +- .../planner/catalog/query_graph.rs | 24 +-- .../multi_graph/planner/query_planner_big.rs | 13 +- .../multi_graph/runner/catalog_generator.rs | 29 ++++ src/graph_impl/multi_graph/runner/mod.rs | 2 + .../multi_graph/runner/optimizer_executor.rs | 32 ++++ src/graph_impl/multi_graph/utils/mod.rs | 2 +- .../utils/{io_utils.rs => time_utils.rs} | 0 src/graph_impl/static_graph/graph.rs | 144 +++++++++++++----- src/graph_impl/static_graph/sorted_adj_vec.rs | 39 +++-- tests/static_graph.rs | 98 ++++++++++++ 13 files changed, 318 insertions(+), 77 deletions(-) create mode 100644 src/graph_impl/multi_graph/runner/catalog_generator.rs create mode 100644 src/graph_impl/multi_graph/runner/mod.rs create mode 100644 src/graph_impl/multi_graph/runner/optimizer_executor.rs rename src/graph_impl/multi_graph/utils/{io_utils.rs => time_utils.rs} (100%) diff --git a/src/graph_impl/multi_graph/mod.rs b/src/graph_impl/multi_graph/mod.rs index 1cafe0be..5fbc9599 100644 --- a/src/graph_impl/multi_graph/mod.rs +++ b/src/graph_impl/multi_graph/mod.rs @@ -3,4 +3,5 @@ pub mod plan; pub mod node; pub mod planner; pub mod query; +pub mod runner; pub mod utils; diff --git a/src/graph_impl/multi_graph/plan/operator/sink/sink_limit.rs b/src/graph_impl/multi_graph/plan/operator/sink/sink_limit.rs index 221f528e..07432cd1 100644 --- a/src/graph_impl/multi_graph/plan/operator/sink/sink_limit.rs +++ b/src/graph_impl/multi_graph/plan/operator/sink/sink_limit.rs @@ -6,7 +6,7 @@ use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Ope use graph_impl::multi_graph::plan::operator::scan::scan::Scan; use graph_impl::multi_graph::plan::operator::sink::sink::{BaseSink, Sink}; use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; -use graph_impl::multi_graph::utils::io_utils::{current_time, get_elapsed_time_in_millis}; +use graph_impl::multi_graph::utils::time_utils::{current_time, get_elapsed_time_in_millis}; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; use std::hash::{BuildHasherDefault, Hash}; diff --git a/src/graph_impl/multi_graph/planner/catalog/catalog.rs b/src/graph_impl/multi_graph/planner/catalog/catalog.rs index f63ae7b2..854ce576 100644 --- a/src/graph_impl/multi_graph/planner/catalog/catalog.rs +++ b/src/graph_impl/multi_graph/planner/catalog/catalog.rs @@ -15,7 +15,7 @@ use graph_impl::multi_graph::planner::catalog::catalog_plans::{ CatalogPlans, DEF_MAX_INPUT_NUM_VERTICES, DEF_NUM_EDGES_TO_SAMPLE, }; use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; -use graph_impl::multi_graph::utils::io_utils; +use graph_impl::multi_graph::utils::time_utils; use graph_impl::TypedStaticGraph; use hashbrown::{HashMap, HashSet}; use indexmap::Equivalent; @@ -240,11 +240,10 @@ impl Catalog { pub fn populate( &mut self, - graph: TypedStaticGraph, + graph: &TypedStaticGraph, num_threads: usize, - filename: String, ) { - let start_time = io_utils::current_time(); + let start_time = time_utils::current_time(); self.is_sorted_by_node = graph.is_sorted_by_node(); self.sampled_icost = HashMap::new(); self.sampled_selectivity = HashMap::new(); @@ -263,7 +262,7 @@ impl Catalog { self.log_output(&graph, query_plan_arr); query_plan_arr.clear(); } - self.elapsed_time = io_utils::get_elapsed_time_in_millis(start_time); + self.elapsed_time = time_utils::get_elapsed_time_in_millis(start_time); //log here } diff --git a/src/graph_impl/multi_graph/planner/catalog/query_graph.rs b/src/graph_impl/multi_graph/planner/catalog/query_graph.rs index 715ac418..6515374d 100644 --- a/src/graph_impl/multi_graph/planner/catalog/query_graph.rs +++ b/src/graph_impl/multi_graph/planner/catalog/query_graph.rs @@ -6,13 +6,13 @@ use std::iter::FromIterator; #[derive(Clone)] pub struct QueryGraph { - qvertex_to_qedges_map: HashMap>>, - qvertex_to_type_map: HashMap, - qvertex_to_deg_map: HashMap>, - q_edges: Vec, + pub qvertex_to_qedges_map: HashMap>>, + pub qvertex_to_type_map: HashMap, + pub qvertex_to_deg_map: HashMap>, + pub q_edges: Vec, // Using `Box` here to enable clone. - it: Option>, - encoding: Option, + pub it: Option>, + pub encoding: Option, pub limit: usize, } @@ -68,11 +68,9 @@ impl QueryGraph { } pub fn get_query_vertices_as_set(&self) -> HashSet { let mut set = HashSet::new(); - self.qvertex_to_qedges_map - .keys() - .for_each(|key| { - set.insert(key.clone()); - }); + self.qvertex_to_qedges_map.keys().for_each(|key| { + set.insert(key.clone()); + }); set } @@ -263,7 +261,9 @@ impl QueryGraph { } pub fn add_qedges(&mut self, query_edges: &Vec) { - query_edges.iter().for_each(|edge| self.add_qedge(edge.clone())); + query_edges + .iter() + .for_each(|edge| self.add_qedge(edge.clone())); } pub fn add_qedge(&mut self, query_edge: QueryEdge) { diff --git a/src/graph_impl/multi_graph/planner/query_planner_big.rs b/src/graph_impl/multi_graph/planner/query_planner_big.rs index b0aeeb10..15a4e62d 100644 --- a/src/graph_impl/multi_graph/planner/query_planner_big.rs +++ b/src/graph_impl/multi_graph/planner/query_planner_big.rs @@ -17,27 +17,32 @@ use hashbrown::{HashMap, HashSet}; use std::cmp::max; use std::hash::Hash; -pub struct QueryPlannerBigger { +pub struct QueryPlannerBig { base_planner: QueryPlanner, subgraph_plans: HashMap>>, num_top_plans_kept: usize, } impl - QueryPlannerBigger + QueryPlannerBig { pub fn new( query_graph: QueryGraph, catalog: Catalog, graph: TypedStaticGraph, ) -> Self { - let mut planner = QueryPlannerBigger { + let mut planner = QueryPlannerBig { base_planner: QueryPlanner::new(query_graph, catalog, graph), subgraph_plans: HashMap::new(), num_top_plans_kept: 5, }; - if planner.base_planner.num_qvertices >= 15 { + let num_vertices = planner.base_planner.num_qvertices; + if num_vertices >= 15 { planner.num_top_plans_kept = 3; + } else if num_vertices >= 20 && num_vertices <= 25 { + planner.num_top_plans_kept = 5; + } else if num_vertices > 25 { + planner.num_top_plans_kept = 1; } planner } diff --git a/src/graph_impl/multi_graph/runner/catalog_generator.rs b/src/graph_impl/multi_graph/runner/catalog_generator.rs new file mode 100644 index 00000000..560147ae --- /dev/null +++ b/src/graph_impl/multi_graph/runner/catalog_generator.rs @@ -0,0 +1,29 @@ +use generic::{GraphType, IdType}; +use graph_impl::multi_graph::planner::catalog::catalog::Catalog; +use graph_impl::multi_graph::planner::catalog::catalog_plans::{ + CatalogPlans, DEF_MAX_INPUT_NUM_VERTICES, DEF_NUM_EDGES_TO_SAMPLE, +}; +use graph_impl::TypedStaticGraph; +use std::hash::Hash; + +pub fn default( + graph: &TypedStaticGraph, +) -> Catalog { + get( + graph, + DEF_MAX_INPUT_NUM_VERTICES, + DEF_NUM_EDGES_TO_SAMPLE, + 1, + ) +} + +pub fn get( + graph: &TypedStaticGraph, + max_input_num_vertices: usize, + num_sampled_edges: usize, + num_threads: usize, +) -> Catalog { + let mut catalog = Catalog::new(num_sampled_edges, max_input_num_vertices); + catalog.populate(graph, num_threads); + catalog +} diff --git a/src/graph_impl/multi_graph/runner/mod.rs b/src/graph_impl/multi_graph/runner/mod.rs new file mode 100644 index 00000000..bee911f0 --- /dev/null +++ b/src/graph_impl/multi_graph/runner/mod.rs @@ -0,0 +1,2 @@ +pub mod catalog_generator; +pub mod optimizer_executor; diff --git a/src/graph_impl/multi_graph/runner/optimizer_executor.rs b/src/graph_impl/multi_graph/runner/optimizer_executor.rs new file mode 100644 index 00000000..0da4989c --- /dev/null +++ b/src/graph_impl/multi_graph/runner/optimizer_executor.rs @@ -0,0 +1,32 @@ +use generic::{GraphType, IdType}; +use graph_impl::multi_graph::planner::catalog::catalog::Catalog; +use graph_impl::multi_graph::planner::catalog::query_edge::QueryEdge; +use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; +use graph_impl::multi_graph::planner::query_planner::QueryPlanner; +use graph_impl::multi_graph::planner::query_planner_big::QueryPlannerBig; +use graph_impl::multi_graph::runner::catalog_generator; +use graph_impl::multi_graph::utils::time_utils; +use graph_impl::{EdgeVec, TypedStaticGraph}; +use hashbrown::HashMap; +use std::hash::Hash; +use DiStaticGraph; + +pub fn generate_plan( + query_graph: QueryGraph, + catalog: Catalog, + g: TypedStaticGraph, +) { + let num_qvertices = query_graph.get_num_qvertices(); + let start_time = time_utils::current_time(); + let elapsed_time; + if query_graph.get_num_qvertices() <= 8 { + let mut planner = QueryPlanner::new(query_graph, catalog, g); + let query_plan = planner.plan(); + elapsed_time = time_utils::get_elapsed_time_in_millis(start_time); + } else { + let mut planner = QueryPlannerBig::new(query_graph, catalog, g); + let query_plan = planner.plan(); + elapsed_time = time_utils::get_elapsed_time_in_millis(start_time); + } + println!("Optimizer runtime: {} (ms)", elapsed_time); +} diff --git a/src/graph_impl/multi_graph/utils/mod.rs b/src/graph_impl/multi_graph/utils/mod.rs index 53126b15..f8e456e5 100644 --- a/src/graph_impl/multi_graph/utils/mod.rs +++ b/src/graph_impl/multi_graph/utils/mod.rs @@ -1 +1 @@ -pub mod io_utils; +pub mod time_utils; diff --git a/src/graph_impl/multi_graph/utils/io_utils.rs b/src/graph_impl/multi_graph/utils/time_utils.rs similarity index 100% rename from src/graph_impl/multi_graph/utils/io_utils.rs rename to src/graph_impl/multi_graph/utils/time_utils.rs diff --git a/src/graph_impl/static_graph/graph.rs b/src/graph_impl/static_graph/graph.rs index 1e1e841f..7b44d72f 100644 --- a/src/graph_impl/static_graph/graph.rs +++ b/src/graph_impl/static_graph/graph.rs @@ -47,7 +47,7 @@ use test::Options; pub type TypedUnStaticGraph = TypedStaticGraph; pub type TypedDiStaticGraph = TypedStaticGraph; pub type StaticGraph = - TypedStaticGraph; +TypedStaticGraph; pub type UnStaticGraph = StaticGraph; pub type DiStaticGraph = StaticGraph; @@ -87,7 +87,7 @@ pub struct TypedStaticGraph PartialEq - for TypedStaticGraph +for TypedStaticGraph { fn eq(&self, other: &TypedStaticGraph) -> bool { if !self.node_count() == other.node_count() || !self.edge_count() == other.edge_count() { @@ -111,12 +111,11 @@ impl Partial } impl Eq - for TypedStaticGraph -{ -} +for TypedStaticGraph +{} impl Hash - for TypedStaticGraph +for TypedStaticGraph { fn hash(&self, state: &mut H) { { @@ -142,27 +141,25 @@ impl Hash } impl Serialize - for TypedStaticGraph -where - Id: serde::Serialize, - NL: serde::Serialize, - EL: serde::Serialize, - L: serde::Serialize, -{ -} +for TypedStaticGraph + where + Id: serde::Serialize, + NL: serde::Serialize, + EL: serde::Serialize, + L: serde::Serialize, +{} impl Deserialize - for TypedStaticGraph -where - Id: for<'de> serde::Deserialize<'de>, - NL: for<'de> serde::Deserialize<'de>, - EL: for<'de> serde::Deserialize<'de>, - L: for<'de> serde::Deserialize<'de>, -{ -} +for TypedStaticGraph + where + Id: for<'de> serde::Deserialize<'de>, + NL: for<'de> serde::Deserialize<'de>, + EL: for<'de> serde::Deserialize<'de>, + L: for<'de> serde::Deserialize<'de>, +{} impl - TypedStaticGraph +TypedStaticGraph { pub fn empty() -> Self { Self::new(EdgeVec::default(), None, None, None) @@ -233,8 +230,7 @@ impl graph_type: PhantomData, from_type_to_percentage_map: HashMap::new(), }; - g.partition_nodes(); - g.partition_edges(); + g.init_graphflow(); g } @@ -311,8 +307,7 @@ impl graph_type: PhantomData, from_type_to_percentage_map: HashMap::new(), }; - g.partition_nodes(); - g.partition_edges(); + g.init_graphflow(); g } @@ -395,11 +390,85 @@ impl graph_type: PhantomData, from_type_to_percentage_map: HashMap::new(), }; - g.partition_nodes(); - g.partition_edges(); + g.init_graphflow(); g } + pub fn init_graphflow(&mut self) { + self.partition_nodes(); + self.partition_edges(); + let label_cnt = if self.sort_by_node { + self.num_of_node_labels() + } else { + self.num_of_edge_labels() + }; + self.label_to_num_edges = vec![0; label_cnt]; + self.label_to_largest_fwd_adj_list_size = vec![0; label_cnt]; + self.label_to_largest_bwd_adj_list_size = vec![0; label_cnt]; + for i in 0..self.num_nodes { + self.num_edges += self.fwd_adj_lists[i].as_ref().unwrap().len(); + for label in 0..label_cnt { + let fwd_adj_size = self.fwd_adj_lists[i].as_ref().unwrap().sub_len(label); + let bwd_adj_size = self.bwd_adj_lists[i].as_ref().unwrap().sub_len(label); + self.label_to_num_edges[label] += fwd_adj_size; + if fwd_adj_size > self.label_to_largest_fwd_adj_list_size[label] { + self.label_to_largest_fwd_adj_list_size[label] = fwd_adj_size; + } + if bwd_adj_size > self.label_to_largest_bwd_adj_list_size[label] { + self.label_to_largest_fwd_adj_list_size[label] = bwd_adj_size; + } + } + } + let num_vertices = self.num_nodes; + for from in 0..self.node_types.len() { + for to in 0..self.node_types.len() { + for label in 0..self.node_types.len() { + let edge = Self::get_edge_key(from, to, label); + self.edge_key_to_num_edges_map.entry(edge).or_insert(0); + let to_label = Self::get_edge_key_by_label(label, to); + self.to_type_to_percentage_map.entry(to_label).or_insert(0); + let from_label = Self::get_edge_key_by_label(from, label); + self.from_type_to_percentage_map.entry(from_label).or_insert(0); + } + } + } + + for from in 0..num_vertices { + let from_type = self.node_types[from]; + let offsets = self.fwd_adj_lists[from].as_ref().unwrap().get_offsets().clone(); + if self.sort_by_node { + let label = 0; + for to_type in 0..offsets.len() { + let num_edges = offsets[to_type + 1] - offsets[to_type]; + self.add_edge_count(from_type, to_type, label, num_edges); + } + } else { + let neighbours = self.fwd_adj_lists[from].as_ref().unwrap().get_neighbor_ids().clone(); + for label in 0..(offsets.len() - 1) { + for to_idx in offsets[label]..offsets[label + 1] { + let to_type = self.node_types[neighbours[to_idx].id()]; +// if from_type == 1 && to_type == 3 && label == 3 { +// let x = 0; +// } + self.add_edge_count(from_type, to_type, label, 1); + } + } + } + } + } + + fn add_edge_count(&mut self, from_type: usize, to_type: usize, label: usize, num_edges: usize) { + let edge = Self::get_edge_key(from_type, to_type, label); + let num_edges_origin = self.edge_key_to_num_edges_map.get(&edge).unwrap(); + self.edge_key_to_num_edges_map.insert(edge, num_edges_origin + num_edges); + let to_label = Self::get_edge_key_by_label(label, to_type); + let to_percentage = self.to_type_to_percentage_map.get(&to_label).unwrap(); + self.to_type_to_percentage_map.insert(to_label, to_percentage + num_edges); + let from_label = Self::get_edge_key_by_label(from_type, label); + let from_percentage = self.from_type_to_percentage_map.get(&from_label).unwrap(); + self.from_type_to_percentage_map.insert(from_label, from_percentage + num_edges); + } + pub fn is_sorted_by_node(&self) -> bool { self.sort_by_node } @@ -770,7 +839,7 @@ impl } impl GraphTrait - for TypedStaticGraph +for TypedStaticGraph { #[inline] fn get_node(&self, id: Id) -> NodeType { @@ -910,7 +979,7 @@ impl GraphTr } impl - GraphLabelTrait for TypedStaticGraph +GraphLabelTrait for TypedStaticGraph { #[inline(always)] fn get_node_label_map(&self) -> &SetMap { @@ -1007,12 +1076,11 @@ impl } impl UnGraphTrait - for TypedUnStaticGraph -{ -} +for TypedUnStaticGraph +{} impl DiGraphTrait - for TypedDiStaticGraph +for TypedDiStaticGraph { #[inline] fn in_degree(&self, id: Id) -> usize { @@ -1033,7 +1101,7 @@ impl DiGraphTrait } impl GeneralGraph - for TypedUnStaticGraph +for TypedUnStaticGraph { #[inline(always)] fn as_graph(&self) -> &GraphTrait { @@ -1052,7 +1120,7 @@ impl GeneralGraph GeneralGraph - for TypedDiStaticGraph +for TypedDiStaticGraph { #[inline(always)] fn as_graph(&self) -> &GraphTrait { @@ -1098,7 +1166,7 @@ fn _merge_labels(_labels1: Option>, _labels2: Option>) -> Op } impl Add - for TypedStaticGraph +for TypedStaticGraph { type Output = TypedStaticGraph; diff --git a/src/graph_impl/static_graph/sorted_adj_vec.rs b/src/graph_impl/static_graph/sorted_adj_vec.rs index 8bd939c8..3e823de9 100644 --- a/src/graph_impl/static_graph/sorted_adj_vec.rs +++ b/src/graph_impl/static_graph/sorted_adj_vec.rs @@ -23,7 +23,7 @@ use graph_impl::multi_graph::plan::operator::extend::EI::Neighbours; #[derive(Debug, Hash, PartialEq, Eq, Clone, Serialize, Deserialize)] pub struct SortedAdjVec { - label_or_type_offsets: Vec, + label_offset: Vec, neighbour_ids: Vec, } @@ -31,7 +31,7 @@ impl SortedAdjVec { pub fn new(offset: Vec) -> Self { let len = offset[offset.len() - 1]; Self { - label_or_type_offsets: offset, + label_offset: offset, neighbour_ids: vec![IdType::new(0); len], } } @@ -46,12 +46,12 @@ impl SortedAdjVec { pub fn set_neighbor_ids(&self, label_or_type: usize, neighbours: &mut Neighbours) { neighbours.ids = self.neighbour_ids.clone(); - neighbours.start_idx = self.label_or_type_offsets[label_or_type]; - neighbours.end_idx = self.label_or_type_offsets[label_or_type + 1]; + neighbours.start_idx = self.label_offset[label_or_type]; + neighbours.end_idx = self.label_offset[label_or_type + 1]; } pub fn get_offsets(&self) -> &Vec { - self.label_or_type_offsets.as_ref() + self.label_offset.as_ref() } pub fn get_neighbor_ids(&self) -> &Vec { @@ -59,9 +59,9 @@ impl SortedAdjVec { } pub fn sort(&mut self) { - for i in 0..self.label_or_type_offsets.len() - 1 { + for i in 0..self.label_offset.len() - 1 { let block = self.neighbour_ids - [self.label_or_type_offsets[i]..self.label_or_type_offsets[i + 1]] + [self.label_offset[i]..self.label_offset[i + 1]] .as_mut(); block.sort(); } @@ -77,10 +77,10 @@ impl SortedAdjVec { some_neighbours, neighbours, &self.neighbour_ids, - self.label_or_type_offsets[label_or_type], - self.label_or_type_offsets[label_or_type + 1], + self.label_offset[label_or_type], + self.label_offset[label_or_type + 1], ); - self.label_or_type_offsets[label_or_type + 1] - self.label_or_type_offsets[label_or_type] + self.label_offset[label_or_type + 1] - self.label_offset[label_or_type] } fn inner_intersect( @@ -100,16 +100,16 @@ impl SortedAdjVec { this_idx += 1; while this_idx < this_idx_end && neighbour_ids[this_idx] < some_neighbour_ids[some_idx] - { - this_idx += 1; - } + { + this_idx += 1; + } } else if neighbour_ids[this_idx] > some_neighbour_ids[some_idx] { some_idx += 1; while some_idx < some_end_idx && neighbour_ids[this_idx] > some_neighbour_ids[some_idx] - { - some_idx += 1; - } + { + some_idx += 1; + } } else { neighbours.ids[neighbours.end_idx] = neighbour_ids[this_idx]; neighbours.end_idx += 1; @@ -118,4 +118,11 @@ impl SortedAdjVec { } } } + pub fn len(&self) -> usize { + self.neighbour_ids.len() + } + + pub fn sub_len(&self, label: usize) -> usize { + self.label_offset[label + 1] + self.label_offset[label] + } } diff --git a/tests/static_graph.rs b/tests/static_graph.rs index 79ea093b..14f54252 100644 --- a/tests/static_graph.rs +++ b/tests/static_graph.rs @@ -20,11 +20,16 @@ */ #[macro_use] extern crate rust_graph; +extern crate hashbrown; extern crate itertools; extern crate tempfile; +use hashbrown::HashMap; use itertools::Itertools; use rust_graph::generic::DefaultId; +use rust_graph::graph_impl::multi_graph::planner::catalog::query_edge::QueryEdge; +use rust_graph::graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; +use rust_graph::graph_impl::multi_graph::runner::{catalog_generator, optimizer_executor}; use rust_graph::graph_impl::static_graph::StaticNode; use rust_graph::graph_impl::Edge; use rust_graph::graph_impl::EdgeVec; @@ -310,3 +315,96 @@ fn test_clone() { let g = DiStaticGraph::::new(edge_vec, Some(in_edge_vec), None, None); assert_eq!(g, g.clone()); } + +#[test] +fn test_graphflow_planner() { + let edge_vec = EdgeVec::with_labels(vec![0, 2, 3, 4], vec![1, 2, 0, 0], vec![0, 1, 0, 1]); + let in_edge_vec = EdgeVec::new(vec![0, 2, 3, 4], vec![1, 2, 0, 0]); + let labels = vec![1, 0, 1]; + let g = DiStaticGraph::<&str>::with_labels( + edge_vec, + Some(in_edge_vec), + labels, + setmap!["a", "b"], + setmap!["a", "b"], + None, + None, + ); + let catalog = catalog_generator::default(&g); + let mut qvertex_to_qedges_map = HashMap::new(); + let mut qvertex_to_type_map = HashMap::new(); + let mut qvertex_to_deg_map = HashMap::new(); + let q_edges = vec![QueryEdge::default("b".to_owned(), "a".to_owned())]; + + let mut qedges_map = HashMap::new(); + qedges_map.insert( + "a".to_owned(), + vec![QueryEdge::new("a".to_owned(), "b".to_owned(), 1, 0, 0)], + ); + qvertex_to_qedges_map.insert("a".to_owned(), qedges_map); + qedges_map = HashMap::new(); + qedges_map.insert( + "b".to_owned(), + vec![QueryEdge::new("a".to_owned(), "b".to_owned(), 1, 0, 0)], + ); + qvertex_to_qedges_map.insert("b".to_owned(), qedges_map); + qvertex_to_type_map.insert("a".to_owned(), 1); + qvertex_to_type_map.insert("b".to_owned(), 0); + qvertex_to_deg_map.insert("a".to_owned(), vec![1, 0]); + qvertex_to_deg_map.insert("b".to_owned(), vec![0, 1]); + + // let q_edges = vec![ + // QueryEdge::default("a".to_owned(), "b".to_owned()), + // QueryEdge::default("a".to_owned(), "c".to_owned()), + // QueryEdge::default("b".to_owned(), "c".to_owned()), + // QueryEdge::default("c".to_owned(), "e".to_owned()), + // QueryEdge::default("c".to_owned(), "f".to_owned()), + // QueryEdge::default("e".to_owned(), "f".to_owned()), + // ]; + // let mut qedges_map = HashMap::new(); + // qedges_map.insert("b".to_owned(),vec![QueryEdge::default("a".to_owned(),"b".to_owned())]); + // qedges_map.insert("c".to_owned(),vec![QueryEdge::default("a".to_owned(),"c".to_owned())]); + // qvertex_to_qedges_map.insert("a".to_owned(),qedges_map); + // qedges_map = HashMap::new(); + // qedges_map.insert("a".to_owned(),vec![QueryEdge::default("a".to_owned(),"b".to_owned())]); + // qedges_map.insert("c".to_owned(),vec![QueryEdge::default("b".to_owned(),"c".to_owned())]); + // qvertex_to_qedges_map.insert("b".to_owned(),qedges_map); + // qedges_map = HashMap::new(); + // qedges_map.insert("a".to_owned(),vec![QueryEdge::default("a".to_owned(),"c".to_owned())]); + // qedges_map.insert("b".to_owned(),vec![QueryEdge::default("b".to_owned(),"c".to_owned())]); + // qedges_map.insert("e".to_owned(),vec![QueryEdge::default("c".to_owned(),"e".to_owned())]); + // qedges_map.insert("f".to_owned(),vec![QueryEdge::default("c".to_owned(),"f".to_owned())]); + // qvertex_to_qedges_map.insert("c".to_owned(),qedges_map); + // qedges_map = HashMap::new(); + // qedges_map.insert("c".to_owned(),vec![QueryEdge::default("c".to_owned(),"e".to_owned())]); + // qedges_map.insert("f".to_owned(),vec![QueryEdge::default("e".to_owned(),"f".to_owned())]); + // qvertex_to_qedges_map.insert("e".to_owned(),qedges_map); + // qedges_map = HashMap::new(); + // qedges_map.insert("c".to_owned(),vec![QueryEdge::default("c".to_owned(),"f".to_owned())]); + // qedges_map.insert("e".to_owned(),vec![QueryEdge::default("e".to_owned(),"f".to_owned())]); + // qvertex_to_qedges_map.insert("f".to_owned(),qedges_map); + // + // qvertex_to_type_map.insert("a".to_owned(),0); + // qvertex_to_type_map.insert("b".to_owned(),0); + // qvertex_to_type_map.insert("c".to_owned(),0); + // qvertex_to_type_map.insert("e".to_owned(),0); + // qvertex_to_type_map.insert("f".to_owned(),0); + // + // qvertex_to_deg_map.insert("a".to_owned(),vec![2,0]); + // qvertex_to_deg_map.insert("b".to_owned(),vec![1,1]); + // qvertex_to_deg_map.insert("c".to_owned(),vec![2,2]); + // qvertex_to_deg_map.insert("e".to_owned(),vec![1,1]); + // qvertex_to_deg_map.insert("f".to_owned(),vec![0,2]); + + let query_graph = QueryGraph { + qvertex_to_qedges_map, + qvertex_to_type_map, + qvertex_to_deg_map, + q_edges, + it: None, + encoding: None, + limit: 0, + }; + + optimizer_executor::generate_plan(query_graph, catalog, g); +} From 6628ceaaaa9350cf42f1222127fc6e23263d110b Mon Sep 17 00:00:00 2001 From: Michael Chen <920901835@qq.com> Date: Fri, 10 Jan 2020 14:59:46 +0800 Subject: [PATCH 14/25] bugfix:support plan generation --- src/graph_impl/multi_graph/mod.rs | 1 - .../plan/operator/extend/Extend.rs | 4 +- .../plan/operator/extend/Intersect.rs | 6 +- .../plan/operator/extend/extend.rs | 4 +- .../plan/operator/extend/intersect.rs | 6 +- .../plan/operator/hashjoin/hash_join.rs | 13 +- .../operator/hashjoin/probe_multi_vertices.rs | 4 +- .../multi_graph/plan/operator/scan/scan.rs | 4 +- .../plan/operator/sink/sink_limit.rs | 15 +- src/graph_impl/multi_graph/plan/query_plan.rs | 45 ++++- .../multi_graph/planner/catalog/catalog.rs | 46 ++--- .../multi_graph/planner/catalog/query_edge.rs | 2 +- .../planner/catalog/query_graph.rs | 135 +++------------ .../catalog/subgraph_mapping_iterator.rs | 159 ++++++++++-------- .../multi_graph/planner/query_planner.rs | 99 +++-------- .../multi_graph/planner/query_planner_big.rs | 17 +- .../multi_graph/runner/optimizer_executor.rs | 21 ++- src/graph_impl/multi_graph/utils/mod.rs | 1 - .../multi_graph/utils/time_utils.rs | 10 -- src/graph_impl/static_graph/graph.rs | 91 ++++++---- src/graph_impl/static_graph/sorted_adj_vec.rs | 16 +- tests/io.rs | 4 +- tests/static_graph.rs | 143 ++++++++++------ 23 files changed, 399 insertions(+), 447 deletions(-) delete mode 100644 src/graph_impl/multi_graph/utils/mod.rs delete mode 100644 src/graph_impl/multi_graph/utils/time_utils.rs diff --git a/src/graph_impl/multi_graph/mod.rs b/src/graph_impl/multi_graph/mod.rs index 5fbc9599..6beb476f 100644 --- a/src/graph_impl/multi_graph/mod.rs +++ b/src/graph_impl/multi_graph/mod.rs @@ -4,4 +4,3 @@ pub mod node; pub mod planner; pub mod query; pub mod runner; -pub mod utils; diff --git a/src/graph_impl/multi_graph/plan/operator/extend/Extend.rs b/src/graph_impl/multi_graph/plan/operator/extend/Extend.rs index e5afee1b..80185df9 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/Extend.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/Extend.rs @@ -34,7 +34,7 @@ impl Extend { in_subgraph: Option>, out_qvertex_to_idx_map: HashMap, ) -> Extend { - let ald = alds.get(0).unwrap().clone(); + let ald = alds[0].clone(); let mut extend = Extend { base_ei: BaseEI::new(to_qvertex.clone(), to_type, alds, out_subgraph, in_subgraph), vertex_index: ald.vertex_idx, @@ -43,7 +43,7 @@ impl Extend { adj_list: vec![], }; extend.base_ei.base_op.last_repeated_vertex_idx = extend.base_ei.base_op.out_tuple_len - 2; - extend.base_ei.out_idx = out_qvertex_to_idx_map.get(&to_qvertex).unwrap().clone(); + extend.base_ei.out_idx = out_qvertex_to_idx_map[&to_qvertex].clone(); extend.base_ei.base_op.out_qvertex_to_idx_map = out_qvertex_to_idx_map; extend } diff --git a/src/graph_impl/multi_graph/plan/operator/extend/Intersect.rs b/src/graph_impl/multi_graph/plan/operator/extend/Intersect.rs index 347227e3..cab933cd 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/Intersect.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/Intersect.rs @@ -39,11 +39,7 @@ impl Intersect { let base_op = &mut intersect.base_ei.base_op; base_op.last_repeated_vertex_idx = base_op.out_tuple_len - 2; base_op.out_qvertex_to_idx_map = out_qvertex_to_idx_map; - intersect.base_ei.out_idx = base_op - .out_qvertex_to_idx_map - .get(&to_qvertex) - .unwrap() - .clone(); + intersect.base_ei.out_idx = base_op.out_qvertex_to_idx_map[&to_qvertex].clone(); intersect } } diff --git a/src/graph_impl/multi_graph/plan/operator/extend/extend.rs b/src/graph_impl/multi_graph/plan/operator/extend/extend.rs index e5afee1b..80185df9 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/extend.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/extend.rs @@ -34,7 +34,7 @@ impl Extend { in_subgraph: Option>, out_qvertex_to_idx_map: HashMap, ) -> Extend { - let ald = alds.get(0).unwrap().clone(); + let ald = alds[0].clone(); let mut extend = Extend { base_ei: BaseEI::new(to_qvertex.clone(), to_type, alds, out_subgraph, in_subgraph), vertex_index: ald.vertex_idx, @@ -43,7 +43,7 @@ impl Extend { adj_list: vec![], }; extend.base_ei.base_op.last_repeated_vertex_idx = extend.base_ei.base_op.out_tuple_len - 2; - extend.base_ei.out_idx = out_qvertex_to_idx_map.get(&to_qvertex).unwrap().clone(); + extend.base_ei.out_idx = out_qvertex_to_idx_map[&to_qvertex].clone(); extend.base_ei.base_op.out_qvertex_to_idx_map = out_qvertex_to_idx_map; extend } diff --git a/src/graph_impl/multi_graph/plan/operator/extend/intersect.rs b/src/graph_impl/multi_graph/plan/operator/extend/intersect.rs index 347227e3..cab933cd 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/intersect.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/intersect.rs @@ -39,11 +39,7 @@ impl Intersect { let base_op = &mut intersect.base_ei.base_op; base_op.last_repeated_vertex_idx = base_op.out_tuple_len - 2; base_op.out_qvertex_to_idx_map = out_qvertex_to_idx_map; - intersect.base_ei.out_idx = base_op - .out_qvertex_to_idx_map - .get(&to_qvertex) - .unwrap() - .clone(); + intersect.base_ei.out_idx = base_op.out_qvertex_to_idx_map[&to_qvertex].clone(); intersect } } diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/hash_join.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/hash_join.rs index 7ef1a07b..179253dd 100644 --- a/src/graph_impl/multi_graph/plan/operator/hashjoin/hash_join.rs +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/hash_join.rs @@ -45,10 +45,7 @@ impl HashJoin { let build_qvertex_to_idx_map = get_op_attr_as_ref!(&pre_build, out_qvertex_to_idx_map).clone(); let query_vertex_to_hash = &join_qvertices[0]; - let build_hash_idx = build_qvertex_to_idx_map - .get(query_vertex_to_hash) - .unwrap() - .clone(); + let build_hash_idx = build_qvertex_to_idx_map[query_vertex_to_hash].clone(); let mut build = Build::new( get_op_attr_as_ref!(&pre_build, out_subgraph).clone(), query_vertex_to_hash.clone(), @@ -73,14 +70,14 @@ impl HashJoin { ); }); probe_qvertex_to_idx_map.insert( - map.get(&join_qvertices[0]).unwrap().clone(), + map[&join_qvertices[0]].clone(), build_qvertex_to_idx_map.len() - 1, ); } else { probe_qvertex_to_idx_map = get_op_attr_as_ref!(&pre_probe, out_qvertex_to_idx_map).clone(); } - let &probe_hash_idx = probe_qvertex_to_idx_map.get(query_vertex_to_hash).unwrap(); + let probe_hash_idx = probe_qvertex_to_idx_map[query_vertex_to_hash]; let out_qvertex_to_idx_map = HashJoin::compute_out_qvertex_to_idx_map( &join_qvertices, &build_qvertex_to_idx_map, @@ -90,8 +87,8 @@ impl HashJoin { let mut probe_indices = vec![0; join_qvertices.len() - 1]; let mut build_indices = vec![0; join_qvertices.len() - 1]; for (i, join_qvertex) in join_qvertices.iter().enumerate() { - probe_indices[i - 1] = probe_qvertex_to_idx_map.get(join_qvertex).unwrap().clone(); - let mut other_build_idx = build_qvertex_to_idx_map.get(join_qvertex).unwrap().clone(); + probe_indices[i - 1] = probe_qvertex_to_idx_map[join_qvertex].clone(); + let mut other_build_idx = build_qvertex_to_idx_map[join_qvertex]; if build_hash_idx < other_build_idx { other_build_idx -= 1; } diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices.rs index 5f748f69..0039150f 100644 --- a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices.rs +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices.rs @@ -35,7 +35,7 @@ impl ProbeMultiVertices { ) -> ProbeMultiVertices { let mut name = "PROBE ON ".to_owned(); if 1 == join_qvertices.len() { - name = name + "(" + join_qvertices.get(0).unwrap() + ")"; + name = name + "(" + &join_qvertices[0] + ")"; } else { for i in 0..join_qvertices.len() { if i > 0 && i < join_qvertices.len() - 1 { @@ -44,7 +44,7 @@ impl ProbeMultiVertices { if i == join_qvertices.len() - 1 { name += " & " } - name = name + "(" + &join_qvertices.get(i).unwrap() + ")"; + name = name + "(" + &join_qvertices[i] + ")"; } } let mut pmv = ProbeMultiVertices { diff --git a/src/graph_impl/multi_graph/plan/operator/scan/scan.rs b/src/graph_impl/multi_graph/plan/operator/scan/scan.rs index 1d944b17..693d727e 100644 --- a/src/graph_impl/multi_graph/plan/operator/scan/scan.rs +++ b/src/graph_impl/multi_graph/plan/operator/scan/scan.rs @@ -50,10 +50,10 @@ impl BaseScan { from_vertex_end_idx: 0, }; let out_subgraph = scan.base_op.out_subgraph.as_ref(); - if out_subgraph.get_query_edges().len() > 1 { + if out_subgraph.q_edges.len() > 1 { panic!("IllegalArgumentException"); } - let query_edge = &out_subgraph.get_query_edges()[0]; + let query_edge = &out_subgraph.q_edges[0]; scan.from_type = query_edge.from_type; scan.to_type = query_edge.to_type; scan.label_or_to_type = query_edge.label; diff --git a/src/graph_impl/multi_graph/plan/operator/sink/sink_limit.rs b/src/graph_impl/multi_graph/plan/operator/sink/sink_limit.rs index 07432cd1..ad7c1618 100644 --- a/src/graph_impl/multi_graph/plan/operator/sink/sink_limit.rs +++ b/src/graph_impl/multi_graph/plan/operator/sink/sink_limit.rs @@ -6,16 +6,16 @@ use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Ope use graph_impl::multi_graph::plan::operator::scan::scan::Scan; use graph_impl::multi_graph::plan::operator::sink::sink::{BaseSink, Sink}; use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; -use graph_impl::multi_graph::utils::time_utils::{current_time, get_elapsed_time_in_millis}; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; use std::hash::{BuildHasherDefault, Hash}; +use std::time::SystemTime; #[derive(Clone)] pub struct SinkLimit { pub base_sink: BaseSink, - start_time: i64, - elapsed_time: f32, + start_time: SystemTime, + elapsed_time: u128, out_tuples_limit: usize, } @@ -23,8 +23,8 @@ impl SinkLimit { pub fn new(query_graph: Box, out_tuple_limit: usize) -> SinkLimit { SinkLimit { base_sink: BaseSink::new(query_graph), - start_time: 0, - elapsed_time: 0.0, + start_time: SystemTime::now(), + elapsed_time: 0, out_tuples_limit: out_tuple_limit, } } @@ -42,7 +42,10 @@ impl CommonOperatorTrait for SinkLimit { fn process_new_tuple(&mut self) { let prev = self.base_sink.base_op.prev.as_ref().unwrap().as_ref(); if get_op_attr!(prev, num_out_tuples) >= self.out_tuples_limit { - self.elapsed_time = get_elapsed_time_in_millis(self.start_time); + self.elapsed_time = SystemTime::now() + .duration_since(self.start_time.clone()) + .unwrap() + .as_millis(); } } diff --git a/src/graph_impl/multi_graph/plan/query_plan.rs b/src/graph_impl/multi_graph/plan/query_plan.rs index c2ce0ffc..2e9a21f4 100644 --- a/src/graph_impl/multi_graph/plan/query_plan.rs +++ b/src/graph_impl/multi_graph/plan/query_plan.rs @@ -21,7 +21,7 @@ pub struct QueryPlan { icost: usize, num_intermediate_tuples: usize, num_out_tuples: usize, - operator_metrics: Vec<(String, usize, usize)>, + pub operator_metrics: Vec<(String, usize, usize)>, executed: bool, adaptive_enabled: bool, pub subplans: Vec>>, @@ -140,4 +140,47 @@ impl QueryPlan { self.subplans.push(Box::new(new_operator.clone())); self.last_operator = Some(Box::new(new_operator)); } + + pub fn get_output_log(&mut self) -> String { + self.set_stats(); + let mut str_joiner = vec![]; + if self.executed { + str_joiner.push(format!("{}", self.elapsed_time)); + str_joiner.push(format!("{}", self.num_out_tuples)); + str_joiner.push(format!("{}", self.num_intermediate_tuples)); + str_joiner.push(format!("{}", self.icost)); + } + for operator_metric in &self.operator_metrics { + str_joiner.push(format!("{}", operator_metric.0)); /* operator name */ + if self.executed { + if !operator_metric.0.contains("PROBE") + && !operator_metric.0.contains("HASH") + && !operator_metric.0.contains("SCAN") + { + str_joiner.push(format!("{}", operator_metric.1)); /* i-cost */ + } + if !operator_metric.0.contains("HASH") { + str_joiner.push(format!("{}", operator_metric.2)); /* num out tuples */ + } + } + } + str_joiner.join(",") + } + fn set_stats(&mut self) { + for subplan in &self.subplans { + let mut first_operator = subplan.as_ref(); + while get_op_attr_as_ref!(first_operator, prev).is_some() { + first_operator = get_op_attr_as_ref!(first_operator, prev) + .as_ref() + .unwrap() + .as_ref(); + } + first_operator.get_operator_metrics_next_operators(&mut self.operator_metrics); + } + for i in 0..self.operator_metrics.len() - 1 { + self.icost += self.operator_metrics[i].1; + self.num_intermediate_tuples += self.operator_metrics[i].2; + } + self.icost += self.operator_metrics[self.operator_metrics.len() - 1].1; + } } diff --git a/src/graph_impl/multi_graph/planner/catalog/catalog.rs b/src/graph_impl/multi_graph/planner/catalog/catalog.rs index 854ce576..085a8ace 100644 --- a/src/graph_impl/multi_graph/planner/catalog/catalog.rs +++ b/src/graph_impl/multi_graph/planner/catalog/catalog.rs @@ -15,7 +15,6 @@ use graph_impl::multi_graph::planner::catalog::catalog_plans::{ CatalogPlans, DEF_MAX_INPUT_NUM_VERTICES, DEF_NUM_EDGES_TO_SAMPLE, }; use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; -use graph_impl::multi_graph::utils::time_utils; use graph_impl::TypedStaticGraph; use hashbrown::{HashMap, HashSet}; use indexmap::Equivalent; @@ -28,6 +27,7 @@ use std::ops::Deref; use std::panic::catch_unwind; use std::ptr::null; use std::thread; +use std::time::SystemTime; pub static SINGLE_VERTEX_WEIGHT_PROBE_COEF: f64 = 3.0; pub static SINGLE_VERTEX_WEIGHT_BUILD_COEF: f64 = 12.0; @@ -41,7 +41,7 @@ pub struct Catalog { is_sorted_by_node: bool, num_sampled_edge: usize, max_input_num_vertices: usize, - elapsed_time: f32, + elapsed_time: u128, } impl Catalog { @@ -53,7 +53,7 @@ impl Catalog { is_sorted_by_node: false, num_sampled_edge, max_input_num_vertices, - elapsed_time: 0.0, + elapsed_time: 0, } } @@ -72,7 +72,7 @@ impl Catalog { is_sorted_by_node: false, num_sampled_edge: 0, max_input_num_vertices: 0, - elapsed_time: 0.0, + elapsed_time: 0, } } @@ -95,9 +95,8 @@ impl Catalog { if sub_graph.get_num_qvertices() != num_vertices { continue; } - let new_num_edges_matched = query_graph.get_query_edges().len(); - let it = query_graph - .get_subgraph_mapping_iterator(self.in_subgraphs.get(i).unwrap()); + let new_num_edges_matched = query_graph.q_edges.len(); + let it = query_graph.get_subgraph_mapping_iterator(&self.in_subgraphs[i]); if new_num_edges_matched < num_edges_matched { continue; } @@ -108,28 +107,18 @@ impl Catalog { } let sampled_icost; let aldas_str = "(".to_string() - + new_vertex_mapping.get(&ald.from_query_vertex).unwrap() + + &new_vertex_mapping[&ald.from_query_vertex] + ") " + &ald.direction.to_string() + "[" + &ald.label.to_string() + "]"; if self.is_sorted_by_node { - sampled_icost = self - .sampled_selectivity - .get(&i) - .unwrap() - .get(&(aldas_str + "~" + &to_type.to_string())) - .unwrap() + sampled_icost = self.sampled_selectivity[&i] + [&(aldas_str + "~" + &to_type.to_string())] .clone(); } else { - sampled_icost = self - .sampled_icost - .get(&i) - .unwrap() - .get(&aldas_str) - .unwrap() - .clone(); + sampled_icost = self.sampled_icost[&i][&aldas_str].clone(); } if new_num_edges_matched > num_edges_matched || min_icost > sampled_icost { min_icost = sampled_icost; @@ -168,10 +157,9 @@ impl Catalog { if new_num_alds_matched == 0 || new_num_alds_matched < num_alds_matched { continue; } - let selectivity_map = self.sampled_selectivity.get(&i).unwrap(); + let selectivity_map = &self.sampled_selectivity[&i]; let sampled_selectivity = selectivity_map - .get(&self.get_alds_as_str(&alds, Some(vertex_mapping), Some(to_type))) - .unwrap() + [&self.get_alds_as_str(&alds, Some(vertex_mapping), Some(to_type))] .clone(); if new_num_alds_matched > num_alds_matched || sampled_selectivity < approx_selectivity @@ -233,7 +221,7 @@ impl Catalog { vertex_mapping .keys() .filter(|&vertex| { - from_vertices_in_alds.contains(vertex) && vertex_mapping.get(vertex).unwrap() != "" + from_vertices_in_alds.contains(vertex) && vertex_mapping[vertex] != "" }) .count() } @@ -243,7 +231,7 @@ impl Catalog { graph: &TypedStaticGraph, num_threads: usize, ) { - let start_time = time_utils::current_time(); + let start_time = SystemTime::now(); self.is_sorted_by_node = graph.is_sorted_by_node(); self.sampled_icost = HashMap::new(); self.sampled_selectivity = HashMap::new(); @@ -262,8 +250,10 @@ impl Catalog { self.log_output(&graph, query_plan_arr); query_plan_arr.clear(); } - self.elapsed_time = time_utils::get_elapsed_time_in_millis(start_time); - //log here + self.elapsed_time = SystemTime::now() + .duration_since(start_time) + .unwrap() + .as_millis(); } fn init( diff --git a/src/graph_impl/multi_graph/planner/catalog/query_edge.rs b/src/graph_impl/multi_graph/planner/catalog/query_edge.rs index c5f4b918..2c501f4f 100644 --- a/src/graph_impl/multi_graph/planner/catalog/query_edge.rs +++ b/src/graph_impl/multi_graph/planner/catalog/query_edge.rs @@ -1,4 +1,4 @@ -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct QueryEdge { pub from_query_vertex: String, pub to_query_vertex: String, diff --git a/src/graph_impl/multi_graph/planner/catalog/query_graph.rs b/src/graph_impl/multi_graph/planner/catalog/query_graph.rs index 6515374d..93fdd87b 100644 --- a/src/graph_impl/multi_graph/planner/catalog/query_graph.rs +++ b/src/graph_impl/multi_graph/planner/catalog/query_graph.rs @@ -10,7 +10,6 @@ pub struct QueryGraph { pub qvertex_to_type_map: HashMap, pub qvertex_to_deg_map: HashMap>, pub q_edges: Vec, - // Using `Box` here to enable clone. pub it: Option>, pub encoding: Option, pub limit: usize, @@ -33,39 +32,31 @@ impl QueryGraph { self.qvertex_to_qedges_map.len() } - pub fn get_subgraph_mapping_iterator(&mut self, query_graph: &QueryGraph) -> &mut Self { - if self.it.is_none() { - self.it.replace(Box::new(SubgraphMappingIterator::new( + pub fn get_subgraph_mapping_iterator( + &mut self, + query_graph: &QueryGraph, + ) -> &mut Box { + let mut it = self + .it + .take() + .unwrap_or(Box::new(SubgraphMappingIterator::new( self.qvertex_to_qedges_map .keys() .map(|x| x.clone()) - .collect(), + .sorted(), ))); - } - let mut it = self.it.take().unwrap(); - self.init_subgraph_iterator(&mut it, query_graph); + it.init(&self, query_graph); self.it.replace(it); - self - } - - pub fn get_vertex_to_deg_map(&self) -> &HashMap> { - &self.qvertex_to_deg_map - } - - pub fn get_vertex_to_qedges_map(&self) -> &HashMap>> { - &self.qvertex_to_qedges_map - } - - pub fn get_vertex_to_type_map(&self) -> &HashMap { - &self.qvertex_to_type_map + self.it.as_mut().unwrap() } pub fn get_query_vertices(&self) -> Vec { self.qvertex_to_qedges_map .keys() .map(|x| x.clone()) - .collect() + .sorted() } + pub fn get_query_vertices_as_set(&self) -> HashSet { let mut set = HashSet::new(); self.qvertex_to_qedges_map.keys().for_each(|key| { @@ -75,8 +66,7 @@ impl QueryGraph { } pub fn get_query_vertex_type(&self, query_vertex: &str) -> usize { - let vertex_type = self.qvertex_to_type_map.get(query_vertex); - if let Some(vertex_type) = vertex_type { + if let Some(vertex_type) = self.qvertex_to_type_map.get(query_vertex) { return vertex_type.clone(); } 0 @@ -101,21 +91,6 @@ impl QueryGraph { false } - pub fn get_query_edges_by_neighbor( - &self, - variable: &String, - neighbor_variable: &String, - ) -> Option<&Vec> { - if let Some(edges) = self.get_vertex_to_qedges_map().get(variable) { - return edges.get(neighbor_variable); - } - None - } - - pub fn get_query_edges(&self) -> &Vec { - &self.q_edges - } - pub fn get_qedges(&self, variable: &String, neighbor_variable: &String) -> Vec { if !self.qvertex_to_qedges_map.contains_key(variable) { panic!("The variable '{}' is not present.", variable); @@ -127,77 +102,7 @@ impl QueryGraph { if !contains_in_qedges { return vec![]; } - self.qvertex_to_qedges_map - .get(variable) - .unwrap() - .get(neighbor_variable) - .unwrap() - .clone() - } - - pub fn has_next(&mut self) -> bool { - let mut iterator = self.it.take().unwrap(); - let res = iterator.has_next(&self); - self.it.replace(iterator); - res - } - - pub fn next(&mut self) -> Option<&HashMap> { - if let Some(mut iterator) = self.it.take() { - iterator.next(&self); - self.it.replace(iterator); - return Some(&self.it.as_ref().unwrap().next); - } - None - } - - fn init_subgraph_iterator( - &mut self, - it: &mut SubgraphMappingIterator, - o_query_graph: &QueryGraph, - ) { - it.o_qvertices = o_query_graph.get_query_vertices(); - it.current_idx = 0; - it.vertex_indices = vec![0; it.o_qvertices.len()]; - it.curr_mapping.clear(); - for i in 0..it.o_qvertices.len() { - if it.vertices_for_idx.len() <= i { - it.vertices_for_idx.push(vec![]); - } else { - it.vertices_for_idx.get_mut(i).unwrap().clear(); - } - let o_qvertex = it.o_qvertices.get(i).unwrap(); - let o_qvertex_deg = o_query_graph - .get_vertex_to_deg_map() - .get(o_qvertex) - .unwrap(); - for j in 0..it.query_vertices.len() { - let q_vertex = it.query_vertices.get(j).unwrap(); - let vertex_type = self.get_vertex_to_type_map().get(q_vertex).unwrap(); - let q_vertex_deg = self.get_vertex_to_deg_map().get(q_vertex).unwrap(); - if o_query_graph - .get_vertex_to_type_map() - .get(o_qvertex) - .unwrap() - == vertex_type - && o_qvertex_deg.eq(q_vertex_deg) - || (it.o_qvertices.len() < it.query_vertices.len() - && q_vertex_deg[0] >= o_qvertex_deg[0] - && q_vertex_deg[1] >= o_qvertex_deg[1]) - { - it.vertices_for_idx - .get_mut(i) - .unwrap() - .push(q_vertex.clone()); - } - } - if 0 == it.vertices_for_idx.get(i).unwrap().len() { - it.is_next_computed = true; - return; - } - } - it.is_next_computed = false; - it.has_next(self); + self.qvertex_to_qedges_map[variable][neighbor_variable].clone() } pub fn get_encoding(&mut self) -> String { @@ -236,10 +141,10 @@ impl QueryGraph { other_query_graph: &QueryGraph, ) -> Option<&HashMap> { let it = self.get_subgraph_mapping_iterator(other_query_graph); - if it.has_next() { - return it.next(); + if !it.has_next() { + return None; } - None + it.next() } pub fn is_isomorphic_to(&mut self, other_query_graph: &mut QueryGraph) -> bool { @@ -339,9 +244,7 @@ impl QueryGraph { if !self.qvertex_to_qedges_map.contains_key(from) { panic!("The variable '{}' is not present.", from); } - self.qvertex_to_qedges_map - .get(from) - .unwrap() + self.qvertex_to_qedges_map[from] .keys() .map(|key| key.clone()) .collect() diff --git a/src/graph_impl/multi_graph/planner/catalog/subgraph_mapping_iterator.rs b/src/graph_impl/multi_graph/planner/catalog/subgraph_mapping_iterator.rs index 11250e00..8b379106 100644 --- a/src/graph_impl/multi_graph/planner/catalog/subgraph_mapping_iterator.rs +++ b/src/graph_impl/multi_graph/planner/catalog/subgraph_mapping_iterator.rs @@ -1,3 +1,4 @@ +use graph_impl::multi_graph::planner::catalog::query_edge::QueryEdge; use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use hashbrown::HashMap; @@ -6,13 +7,17 @@ use hashbrown::HashMap; pub struct SubgraphMappingIterator { pub query_vertices: Vec, pub o_qvertices: Vec, - o_qgraph: Option, + pub o_qgraph: QueryGraph, pub next: HashMap, pub is_next_computed: bool, pub curr_mapping: Vec, pub current_idx: usize, pub vertex_indices: Vec, pub vertices_for_idx: Vec>, + + pub qvertex_to_qedges_map: HashMap>>, + pub qvertex_to_type_map: HashMap, + pub qvertex_to_deg_map: HashMap>, } impl SubgraphMappingIterator { @@ -24,61 +29,85 @@ impl SubgraphMappingIterator { Self { query_vertices, o_qvertices: vec![], - o_qgraph: None, + o_qgraph: QueryGraph::empty(), next, is_next_computed: false, curr_mapping: vec![], current_idx: 0, vertex_indices: vec![], vertices_for_idx: vec![], + qvertex_to_qedges_map: HashMap::new(), + qvertex_to_type_map: HashMap::new(), + qvertex_to_deg_map: HashMap::new(), + } + } + + pub fn init(&mut self, query_graph: &QueryGraph, o_query_graph: &QueryGraph) { + self.o_qvertices = o_query_graph.get_query_vertices(); + self.o_qgraph = o_query_graph.clone(); + self.current_idx = 0; + self.vertex_indices = vec![0; self.o_qvertices.len()]; + self.curr_mapping.clear(); + self.qvertex_to_qedges_map = query_graph.qvertex_to_qedges_map.clone(); + self.qvertex_to_deg_map = query_graph.qvertex_to_deg_map.clone(); + self.qvertex_to_type_map = query_graph.qvertex_to_type_map.clone(); + for i in 0..self.o_qvertices.len() { + if self.vertices_for_idx.len() <= i { + self.vertices_for_idx.push(vec![]); + } else { + self.vertices_for_idx[i].clear(); + } + let o_qvertex = &self.o_qvertices[i]; + let o_qvertex_deg = &o_query_graph.qvertex_to_deg_map[o_qvertex]; + for j in 0..self.query_vertices.len() { + let q_vertex = &self.query_vertices[j]; + let vertex_type = self.qvertex_to_type_map[q_vertex]; + let q_vertex_deg = &self.qvertex_to_deg_map[q_vertex]; + if o_query_graph.qvertex_to_type_map[o_qvertex] == vertex_type + && (o_qvertex_deg.eq(q_vertex_deg) + || (self.o_qvertices.len() < self.query_vertices.len() + && q_vertex_deg[0] >= o_qvertex_deg[0] + && q_vertex_deg[1] >= o_qvertex_deg[1])) + { + self.vertices_for_idx[i].push(q_vertex.clone()); + } + } + if 0 == self.vertices_for_idx[i].len() { + self.is_next_computed = true; + return; + } } + self.is_next_computed = false; + self.has_next(); } - pub fn has_next(&mut self, parent_graph: &QueryGraph) -> bool { + pub fn has_next(&mut self) -> bool { if !self.is_next_computed { if self.curr_mapping.len() == self.o_qvertices.len() { self.curr_mapping.pop(); } loop { let next_idx = self.curr_mapping.len(); - if next_idx == 0 - && self.vertex_indices[0] < self.vertices_for_idx.get(0).unwrap().len() - { - self.curr_mapping.push( - self.vertices_for_idx - .get(0) - .unwrap() - .get(self.vertex_indices[0]) - .unwrap() - .clone(), - ); + if next_idx == 0 && self.vertex_indices[0] < self.vertices_for_idx[0].len() { + self.curr_mapping + .push(self.vertices_for_idx[0][self.vertex_indices[0]].clone()); self.vertex_indices[0] += 1; - } else if self.vertex_indices[next_idx] - < self.vertices_for_idx.get(next_idx).unwrap().len() - { - let vertices = self.vertices_for_idx.get(next_idx).unwrap(); - let new_var = vertices.get(self.vertex_indices[next_idx]).unwrap(); + } else if self.vertex_indices[next_idx] < self.vertices_for_idx[next_idx].len() { + let vertices = &self.vertices_for_idx[next_idx]; + let new_var = &vertices[self.vertex_indices[next_idx]]; self.vertex_indices[next_idx] += 1; - let other_for_new = self.o_qvertices.get(next_idx).unwrap(); - let mut invalid_map = false; + let other_for_new = &self.o_qvertices[next_idx]; + let mut outer_flag = false; for i in 0..self.curr_mapping.len() { - let prev_var = self.curr_mapping.get(i).unwrap(); + let prev_var = &self.curr_mapping[i]; if prev_var.eq(new_var) { - invalid_map = true; + outer_flag = true; break; } - let other_for_prev = self.o_qvertices.get(i).unwrap(); - let q_edges = parent_graph - .get_vertex_to_qedges_map() - .get(new_var) - .unwrap() - .get(prev_var); - let o_qgraph = self.o_qgraph.as_ref().unwrap(); - let o_qedges = o_qgraph - .get_vertex_to_qedges_map() - .get(other_for_new) - .unwrap() - .get(other_for_prev); + let other_for_prev = &self.o_qvertices[i]; + let q_edges = self.qvertex_to_qedges_map[new_var].get(prev_var); + let o_qedges = + self.o_qgraph.qvertex_to_qedges_map[other_for_new].get(other_for_prev); if q_edges.is_none() && o_qedges.is_none() { continue; } @@ -86,14 +115,14 @@ impl SubgraphMappingIterator { || o_qedges.is_none() || q_edges.unwrap().len() != o_qedges.unwrap().len() { - invalid_map = true; + outer_flag = true; break; } if q_edges.unwrap().len() == 0 { continue; } - let q_edge = q_edges.unwrap().get(0).unwrap(); - let o_qedge = o_qedges.unwrap().get(0).unwrap(); + let q_edge = &q_edges.unwrap()[0]; + let o_qedge = &o_qedges.unwrap()[0]; if q_edge.label != o_qedge.label { continue; } @@ -102,22 +131,20 @@ impl SubgraphMappingIterator { || (q_edge.from_query_vertex.eq(new_var) && o_qedge.from_query_vertex.eq(other_for_new)) { - invalid_map = true; + outer_flag = true; break; } } - if invalid_map { - break; + if outer_flag { + continue; } self.curr_mapping.push(new_var.clone()); - } else if self.vertex_indices[next_idx] - >= self.vertices_for_idx.get(next_idx).unwrap().len() - { + } else if self.vertex_indices[next_idx] >= self.vertices_for_idx[next_idx].len() { self.curr_mapping.pop(); self.vertex_indices[next_idx] = 0; } if self.curr_mapping.len() == self.o_qvertices.len() - || self.vertex_indices[0] < self.vertices_for_idx.get(0).unwrap().len() + || self.vertex_indices[0] < self.vertices_for_idx[0].len() || self.curr_mapping.is_empty() { break; @@ -126,27 +153,20 @@ impl SubgraphMappingIterator { self.is_next_computed = true; } if self.curr_mapping.is_empty() { - return !self.curr_mapping.is_empty(); + return false; } let mut same_edge_labels = true; for i in 0..self.curr_mapping.len() { for j in (i + 1)..self.curr_mapping.len() { - let q_vertex = self.curr_mapping.get(i).unwrap(); - let o_qvertex = self.curr_mapping.get(j).unwrap(); - if !parent_graph.contains_query_edge(q_vertex, o_qvertex) { + let q_vertex = &self.curr_mapping[i]; + let o_qvertex = &self.curr_mapping[j]; + if !self.contains_query_edge(q_vertex, o_qvertex) { continue; } - let q_edge = parent_graph.get_query_edges_by_neighbor(q_vertex, o_qvertex); - let o_graph = self.o_qgraph.as_ref().unwrap(); - let o_qedge = o_graph.get_query_edges_by_neighbor( - self.o_qvertices.get(i).unwrap(), - self.o_qvertices.get(j).unwrap(), - ); - if q_edge.is_none() - || o_qedge.is_none() - || q_edge.unwrap().get(0).unwrap().label - != o_qedge.unwrap().get(0).unwrap().label - { + let q_edge = &self.qvertex_to_qedges_map[q_vertex][o_qvertex][0]; + let o_qedge = &self.o_qgraph.qvertex_to_qedges_map[&self.o_qvertices[i]] + [&self.o_qvertices[j]][0]; + if q_edge.label != o_qedge.label { same_edge_labels = false; break; } @@ -154,23 +174,28 @@ impl SubgraphMappingIterator { } if !same_edge_labels { self.is_next_computed = false; - return self.has_next(parent_graph); + return self.has_next(); } !self.curr_mapping.is_empty() } - pub fn next(&mut self, parent_graph: &QueryGraph) -> Option<&HashMap> { - if !self.has_next(parent_graph) { + pub fn next(&mut self) -> Option<&HashMap> { + if !self.has_next() { return None; } self.is_next_computed = false; self.next.clear(); for i in 0..self.o_qvertices.len() { - self.next.insert( - self.curr_mapping.get(i).unwrap().clone(), - self.query_vertices.get(i).unwrap().clone(), - ); + self.next + .insert(self.curr_mapping[i].clone(), self.query_vertices[i].clone()); } return Some(&self.next); } + + pub fn contains_query_edge(&self, v1: &String, v2: &String) -> bool { + if let Some(map) = self.qvertex_to_qedges_map.get(v1) { + return map.contains_key(v2); + } + false + } } diff --git a/src/graph_impl/multi_graph/planner/query_planner.rs b/src/graph_impl/multi_graph/planner/query_planner.rs index 54905c9c..9f17eb92 100644 --- a/src/graph_impl/multi_graph/planner/query_planner.rs +++ b/src/graph_impl/multi_graph/planner/query_planner.rs @@ -48,9 +48,9 @@ impl QueryPlanner { subgraph_plans: HashMap::new(), has_limit: query_graph.limit > 0, - next_num_qvertices: query_graph.get_num_qvertices(), + num_qvertices: query_graph.get_num_qvertices(), query_graph, - num_qvertices: 0, + next_num_qvertices: 0, graph, catalog, computed_selectivities: HashMap::new(), @@ -68,10 +68,7 @@ impl self.consider_all_next_query_extensions(); self.next_num_qvertices += 1; } - let key = self - .subgraph_plans - .get(&self.num_qvertices) - .unwrap() + let key = self.subgraph_plans[&self.num_qvertices] .keys() .next() .unwrap(); @@ -107,7 +104,7 @@ impl self.subgraph_plans .entry(self.next_num_qvertices) .or_insert(HashMap::new()); - for query_edge in self.query_graph.get_query_edges() { + for query_edge in &self.query_graph.q_edges { let mut out_subgraph = QueryGraph::empty(); out_subgraph.add_qedge(query_edge.clone()); let scan = Scan::Base(BaseScan::new(Box::new(out_subgraph))); @@ -132,19 +129,13 @@ impl self.subgraph_plans .entry(self.next_num_qvertices) .or_insert(HashMap::new()); - let plan_map = self - .subgraph_plans - .get(&(self.next_num_qvertices - 1)) - .unwrap(); + let plan_map = &self.subgraph_plans[&(self.next_num_qvertices - 1)]; let plan_map_keys: Vec = plan_map.keys().map(|v| v.clone()).collect(); for key in plan_map_keys { self.consider_all_next_extend_operators(&key); } if !self.has_limit && self.next_num_qvertices >= 4 { - let plan_map_keys: Vec = self - .subgraph_plans - .get(&self.next_num_qvertices) - .unwrap() + let plan_map_keys: Vec = self.subgraph_plans[&self.next_num_qvertices] .keys() .map(|v| v.clone()) .collect(); @@ -155,21 +146,12 @@ impl } fn consider_all_next_extend_operators(&mut self, key: &String) { - let prev_plan_map = self - .subgraph_plans - .get(&(self.next_num_qvertices - 1)) - .unwrap(); - let prev_query_plans = prev_plan_map.get(key).unwrap(); - let op = prev_query_plans - .get(0) - .unwrap() - .last_operator - .as_ref() - .unwrap() - .as_ref(); + let prev_plan_map = &self.subgraph_plans[&(self.next_num_qvertices - 1)]; + let prev_query_plans = &prev_plan_map[key]; + let op = prev_query_plans[0].last_operator.as_ref().unwrap().as_ref(); let prev_qvertices = get_op_attr_as_ref!(op, out_subgraph).get_query_vertices(); let to_qvertices = self.query_graph.get_neighbors(prev_qvertices); - let prev_query_plans_len = prev_plan_map.get(key).unwrap().len(); + let prev_query_plans_len = prev_plan_map[key].len(); let mut plans = vec![]; for to_qvertex in to_qvertices { for i in 0..prev_query_plans_len { @@ -192,12 +174,9 @@ impl key: &String, to_qvertex: &String, ) -> (String, QueryPlan) { - let prev_plan_map = self - .subgraph_plans - .get(&(self.next_num_qvertices - 1)) - .unwrap(); - let prev_query_plans = prev_plan_map.get(key).unwrap(); - let prev_query_plan = prev_query_plans.get(prev_query_plan_index).unwrap(); + let prev_plan_map = &self.subgraph_plans[&(self.next_num_qvertices - 1)]; + let prev_query_plans = &prev_plan_map[key]; + let prev_query_plan = &prev_query_plans[prev_query_plan_index]; self.get_plan_with_next_extend(prev_query_plan.clone(), to_qvertex) } @@ -243,11 +222,8 @@ impl let index = 0; let mut last_estimated_num_out_tuples_for_extension_qvertex = -1.0; for ald in alds.iter().filter(|ald| ald.vertex_idx > index) { - last_estimated_num_out_tuples_for_extension_qvertex = prev_query_plan - .q_vertex_to_num_out_tuples - .get(&ald.from_query_vertex) - .unwrap() - .clone(); + last_estimated_num_out_tuples_for_extension_qvertex = + prev_query_plan.q_vertex_to_num_out_tuples[&ald.from_query_vertex].clone(); } out_tuples_to_process /= last_estimated_num_out_tuples_for_extension_qvertex; } @@ -328,15 +304,10 @@ impl .contains_query_edge(from_qvertex, to_qvertex) { // simple query graph so there is only 1 query_edge, so get query_edge at index '0'. - let query_edge = self - .query_graph - .get_qedges(from_qvertex, to_qvertex) - .get(0) - .unwrap() - .clone(); + let query_edge = + self.query_graph.get_qedges(from_qvertex, to_qvertex)[0].clone(); let index = get_op_attr_as_ref!(last_operator, out_qvertex_to_idx_map) - .get(from_qvertex) - .unwrap() + [from_qvertex] .clone(); let direction = if from_qvertex == &query_edge.from_query_vertex { Direction::Fwd @@ -378,15 +349,11 @@ impl to_type: usize, ) -> f64 { let selectivity; - if self + let computed_selectivity_op = self .computed_selectivities - .contains_key(&out_subgraph.get_encoding()) - { - let computed_selectivity = self - .computed_selectivities - .get_mut(&out_subgraph.get_encoding()) - .unwrap(); - for (graph, selectivity) in computed_selectivity { + .get_mut(&out_subgraph.get_encoding()); + if computed_selectivity_op.is_some() { + for (graph, selectivity) in computed_selectivity_op.unwrap() { if graph.is_isomorphic_to(out_subgraph) { return selectivity.clone(); } @@ -405,8 +372,8 @@ impl } fn consider_all_next_hash_join_operators(&mut self, map_key: &String) { - let plan_map = self.subgraph_plans.get(&self.next_num_qvertices).unwrap(); - let plans = plan_map.get(map_key).unwrap(); + let plan_map = &self.subgraph_plans[&self.next_num_qvertices]; + let plans = &plan_map[map_key]; let op = plans[0].last_operator.as_ref().unwrap().as_ref(); let out_subgraph = get_op_attr_as_ref!(op, out_subgraph).as_ref().clone(); @@ -417,7 +384,7 @@ impl max_size = min_size; } for set_size in min_size..=max_size { - let plans = self.subgraph_plans.get(&set_size).unwrap().clone(); + let plans = self.subgraph_plans[&set_size].clone(); for key in plans.keys() { let prev_query_plan = self.get_best_plan(set_size, key); let last_op = prev_query_plan.last_operator.as_ref().unwrap().as_ref(); @@ -453,12 +420,7 @@ impl }); let rest_size = other_set.len(); let rest_key = self.get_key(&mut other_set); - if !self - .subgraph_plans - .get(&rest_size) - .unwrap() - .contains_key(&rest_key) - { + if !self.subgraph_plans[&rest_size].contains_key(&rest_key) { return; } let other_prev_operator = self.get_best_plan(rest_size, &rest_key); @@ -569,13 +531,8 @@ impl } fn get_best_plan(&self, num_qvertices: usize, key: &String) -> QueryPlan { - let possible_query_plans = self - .subgraph_plans - .get(&num_qvertices) - .unwrap() - .get(key) - .unwrap(); - let mut best_plan = possible_query_plans.get(0).unwrap(); + let possible_query_plans = &self.subgraph_plans[&num_qvertices][key]; + let mut best_plan = &possible_query_plans[0]; possible_query_plans.iter().for_each(|possible_query_plan| { if possible_query_plan.estimated_icost < best_plan.estimated_icost { best_plan = possible_query_plan; diff --git a/src/graph_impl/multi_graph/planner/query_planner_big.rs b/src/graph_impl/multi_graph/planner/query_planner_big.rs index 15a4e62d..75037a51 100644 --- a/src/graph_impl/multi_graph/planner/query_planner_big.rs +++ b/src/graph_impl/multi_graph/planner/query_planner_big.rs @@ -53,11 +53,8 @@ impl self.consider_next_query_extensions(); self.base_planner.next_num_qvertices += 1; } - let plans = self - .subgraph_plans - .get(&self.base_planner.num_qvertices) - .unwrap(); - let mut best_plan = plans.get(0).unwrap(); + let plans = &self.subgraph_plans[&self.base_planner.num_qvertices]; + let mut best_plan = &plans[0]; for plan in plans { if best_plan.estimated_icost > plan.estimated_icost { best_plan = plan; @@ -79,18 +76,18 @@ impl self.subgraph_plans.entry(2).or_insert(vec![]); let mut edges_to_scan = vec![]; let mut num_edges_to_scan = vec![]; - let q_edges = self.base_planner.query_graph.get_query_edges(); + let q_edges = &self.base_planner.query_graph.q_edges; for i in 0..self.num_top_plans_kept { - let edge = q_edges.get(i).unwrap().clone(); + let edge = q_edges[i].clone(); num_edges_to_scan.push(self.base_planner.get_num_edges(&edge)); edges_to_scan.push(edge); } for i in self.num_top_plans_kept..q_edges.len() { - let num_edges = self.base_planner.get_num_edges(q_edges.get(i).unwrap()); + let num_edges = self.base_planner.get_num_edges(&q_edges[i]); for j in 0..self.num_top_plans_kept { if num_edges < num_edges_to_scan[j] { - edges_to_scan[j] = q_edges.get(i).unwrap().clone(); + edges_to_scan[j] = q_edges[i].clone(); num_edges_to_scan[j] = num_edges; break; } @@ -135,7 +132,7 @@ impl new_query_plans.push(plan); } else { for i in 0..self.num_top_plans_kept { - if new_query_plans.get(i).unwrap().estimated_icost > icost { + if new_query_plans[i].estimated_icost > icost { new_query_plans.insert(i, plan.clone()); } } diff --git a/src/graph_impl/multi_graph/runner/optimizer_executor.rs b/src/graph_impl/multi_graph/runner/optimizer_executor.rs index 0da4989c..a68c4e07 100644 --- a/src/graph_impl/multi_graph/runner/optimizer_executor.rs +++ b/src/graph_impl/multi_graph/runner/optimizer_executor.rs @@ -5,10 +5,10 @@ use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::planner::query_planner::QueryPlanner; use graph_impl::multi_graph::planner::query_planner_big::QueryPlannerBig; use graph_impl::multi_graph::runner::catalog_generator; -use graph_impl::multi_graph::utils::time_utils; use graph_impl::{EdgeVec, TypedStaticGraph}; use hashbrown::HashMap; use std::hash::Hash; +use std::time::SystemTime; use DiStaticGraph; pub fn generate_plan( @@ -17,16 +17,25 @@ pub fn generate_plan, ) { let num_qvertices = query_graph.get_num_qvertices(); - let start_time = time_utils::current_time(); + let start_time = SystemTime::now(); let elapsed_time; - if query_graph.get_num_qvertices() <= 8 { + let mut query_plan = if query_graph.get_num_qvertices() <= 8 { let mut planner = QueryPlanner::new(query_graph, catalog, g); let query_plan = planner.plan(); - elapsed_time = time_utils::get_elapsed_time_in_millis(start_time); + elapsed_time = SystemTime::now() + .duration_since(start_time) + .unwrap() + .as_millis(); + query_plan } else { let mut planner = QueryPlannerBig::new(query_graph, catalog, g); let query_plan = planner.plan(); - elapsed_time = time_utils::get_elapsed_time_in_millis(start_time); - } + elapsed_time = SystemTime::now() + .duration_since(start_time) + .unwrap() + .as_millis(); + query_plan + }; println!("Optimizer runtime: {} (ms)", elapsed_time); + println!("QueryPlan output:{}", query_plan.get_output_log()); } diff --git a/src/graph_impl/multi_graph/utils/mod.rs b/src/graph_impl/multi_graph/utils/mod.rs deleted file mode 100644 index f8e456e5..00000000 --- a/src/graph_impl/multi_graph/utils/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod time_utils; diff --git a/src/graph_impl/multi_graph/utils/time_utils.rs b/src/graph_impl/multi_graph/utils/time_utils.rs deleted file mode 100644 index df851f2d..00000000 --- a/src/graph_impl/multi_graph/utils/time_utils.rs +++ /dev/null @@ -1,10 +0,0 @@ -extern crate time; - -pub fn current_time() -> i64 { - let timespec = time::get_time(); - timespec.sec * 1000 + (timespec.nsec as f64 / 1000.0 / 1000.0) as i64 -} - -pub fn get_elapsed_time_in_millis(begin_time: i64) -> f32 { - (current_time() - begin_time / 1000000.0 as i64) as f32 -} diff --git a/src/graph_impl/static_graph/graph.rs b/src/graph_impl/static_graph/graph.rs index 7b44d72f..d506953e 100644 --- a/src/graph_impl/static_graph/graph.rs +++ b/src/graph_impl/static_graph/graph.rs @@ -47,7 +47,7 @@ use test::Options; pub type TypedUnStaticGraph = TypedStaticGraph; pub type TypedDiStaticGraph = TypedStaticGraph; pub type StaticGraph = -TypedStaticGraph; + TypedStaticGraph; pub type UnStaticGraph = StaticGraph; pub type DiStaticGraph = StaticGraph; @@ -87,7 +87,7 @@ pub struct TypedStaticGraph PartialEq -for TypedStaticGraph + for TypedStaticGraph { fn eq(&self, other: &TypedStaticGraph) -> bool { if !self.node_count() == other.node_count() || !self.edge_count() == other.edge_count() { @@ -111,11 +111,12 @@ for TypedStaticGraph } impl Eq -for TypedStaticGraph -{} + for TypedStaticGraph +{ +} impl Hash -for TypedStaticGraph + for TypedStaticGraph { fn hash(&self, state: &mut H) { { @@ -141,25 +142,27 @@ for TypedStaticGraph } impl Serialize -for TypedStaticGraph - where - Id: serde::Serialize, - NL: serde::Serialize, - EL: serde::Serialize, - L: serde::Serialize, -{} + for TypedStaticGraph +where + Id: serde::Serialize, + NL: serde::Serialize, + EL: serde::Serialize, + L: serde::Serialize, +{ +} impl Deserialize -for TypedStaticGraph - where - Id: for<'de> serde::Deserialize<'de>, - NL: for<'de> serde::Deserialize<'de>, - EL: for<'de> serde::Deserialize<'de>, - L: for<'de> serde::Deserialize<'de>, -{} + for TypedStaticGraph +where + Id: for<'de> serde::Deserialize<'de>, + NL: for<'de> serde::Deserialize<'de>, + EL: for<'de> serde::Deserialize<'de>, + L: for<'de> serde::Deserialize<'de>, +{ +} impl -TypedStaticGraph + TypedStaticGraph { pub fn empty() -> Self { Self::new(EdgeVec::default(), None, None, None) @@ -428,14 +431,20 @@ TypedStaticGraph let to_label = Self::get_edge_key_by_label(label, to); self.to_type_to_percentage_map.entry(to_label).or_insert(0); let from_label = Self::get_edge_key_by_label(from, label); - self.from_type_to_percentage_map.entry(from_label).or_insert(0); + self.from_type_to_percentage_map + .entry(from_label) + .or_insert(0); } } } for from in 0..num_vertices { let from_type = self.node_types[from]; - let offsets = self.fwd_adj_lists[from].as_ref().unwrap().get_offsets().clone(); + let offsets = self.fwd_adj_lists[from] + .as_ref() + .unwrap() + .get_offsets() + .clone(); if self.sort_by_node { let label = 0; for to_type in 0..offsets.len() { @@ -443,13 +452,17 @@ TypedStaticGraph self.add_edge_count(from_type, to_type, label, num_edges); } } else { - let neighbours = self.fwd_adj_lists[from].as_ref().unwrap().get_neighbor_ids().clone(); + let neighbours = self.fwd_adj_lists[from] + .as_ref() + .unwrap() + .get_neighbor_ids() + .clone(); for label in 0..(offsets.len() - 1) { for to_idx in offsets[label]..offsets[label + 1] { let to_type = self.node_types[neighbours[to_idx].id()]; -// if from_type == 1 && to_type == 3 && label == 3 { -// let x = 0; -// } + // if from_type == 1 && to_type == 3 && label == 3 { + // let x = 0; + // } self.add_edge_count(from_type, to_type, label, 1); } } @@ -460,13 +473,16 @@ TypedStaticGraph fn add_edge_count(&mut self, from_type: usize, to_type: usize, label: usize, num_edges: usize) { let edge = Self::get_edge_key(from_type, to_type, label); let num_edges_origin = self.edge_key_to_num_edges_map.get(&edge).unwrap(); - self.edge_key_to_num_edges_map.insert(edge, num_edges_origin + num_edges); + self.edge_key_to_num_edges_map + .insert(edge, num_edges_origin + num_edges); let to_label = Self::get_edge_key_by_label(label, to_type); let to_percentage = self.to_type_to_percentage_map.get(&to_label).unwrap(); - self.to_type_to_percentage_map.insert(to_label, to_percentage + num_edges); + self.to_type_to_percentage_map + .insert(to_label, to_percentage + num_edges); let from_label = Self::get_edge_key_by_label(from_type, label); let from_percentage = self.from_type_to_percentage_map.get(&from_label).unwrap(); - self.from_type_to_percentage_map.insert(from_label, from_percentage + num_edges); + self.from_type_to_percentage_map + .insert(from_label, from_percentage + num_edges); } pub fn is_sorted_by_node(&self) -> bool { @@ -839,7 +855,7 @@ TypedStaticGraph } impl GraphTrait -for TypedStaticGraph + for TypedStaticGraph { #[inline] fn get_node(&self, id: Id) -> NodeType { @@ -979,7 +995,7 @@ for TypedStaticGraph } impl -GraphLabelTrait for TypedStaticGraph + GraphLabelTrait for TypedStaticGraph { #[inline(always)] fn get_node_label_map(&self) -> &SetMap { @@ -1076,11 +1092,12 @@ GraphLabelTrait for TypedStaticGraph } impl UnGraphTrait -for TypedUnStaticGraph -{} + for TypedUnStaticGraph +{ +} impl DiGraphTrait -for TypedDiStaticGraph + for TypedDiStaticGraph { #[inline] fn in_degree(&self, id: Id) -> usize { @@ -1101,7 +1118,7 @@ for TypedDiStaticGraph } impl GeneralGraph -for TypedUnStaticGraph + for TypedUnStaticGraph { #[inline(always)] fn as_graph(&self) -> &GraphTrait { @@ -1120,7 +1137,7 @@ for TypedUnStaticGraph } impl GeneralGraph -for TypedDiStaticGraph + for TypedDiStaticGraph { #[inline(always)] fn as_graph(&self) -> &GraphTrait { @@ -1166,7 +1183,7 @@ fn _merge_labels(_labels1: Option>, _labels2: Option>) -> Op } impl Add -for TypedStaticGraph + for TypedStaticGraph { type Output = TypedStaticGraph; diff --git a/src/graph_impl/static_graph/sorted_adj_vec.rs b/src/graph_impl/static_graph/sorted_adj_vec.rs index 3e823de9..52104c9b 100644 --- a/src/graph_impl/static_graph/sorted_adj_vec.rs +++ b/src/graph_impl/static_graph/sorted_adj_vec.rs @@ -60,9 +60,7 @@ impl SortedAdjVec { pub fn sort(&mut self) { for i in 0..self.label_offset.len() - 1 { - let block = self.neighbour_ids - [self.label_offset[i]..self.label_offset[i + 1]] - .as_mut(); + let block = self.neighbour_ids[self.label_offset[i]..self.label_offset[i + 1]].as_mut(); block.sort(); } } @@ -100,16 +98,16 @@ impl SortedAdjVec { this_idx += 1; while this_idx < this_idx_end && neighbour_ids[this_idx] < some_neighbour_ids[some_idx] - { - this_idx += 1; - } + { + this_idx += 1; + } } else if neighbour_ids[this_idx] > some_neighbour_ids[some_idx] { some_idx += 1; while some_idx < some_end_idx && neighbour_ids[this_idx] > some_neighbour_ids[some_idx] - { - some_idx += 1; - } + { + some_idx += 1; + } } else { neighbours.ids[neighbours.end_idx] = neighbour_ids[this_idx]; neighbours.end_idx += 1; diff --git a/tests/io.rs b/tests/io.rs index 443ccea8..0366e337 100644 --- a/tests/io.rs +++ b/tests/io.rs @@ -75,8 +75,8 @@ fn test_cvs_labeled() { let tmp_dir = TempDir::new().unwrap(); let tmp_dir_path = tmp_dir.path(); - let nodes = 10; - let edges = 20; + let nodes = 10000; + let edges = 200000; let node_labels = &vec!["a".to_owned(), "b".to_owned()]; let edge_labels = &vec![1, 2, 3]; diff --git a/tests/static_graph.rs b/tests/static_graph.rs index 14f54252..0b5ace17 100644 --- a/tests/static_graph.rs +++ b/tests/static_graph.rs @@ -31,11 +31,13 @@ use rust_graph::graph_impl::multi_graph::planner::catalog::query_edge::QueryEdge use rust_graph::graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use rust_graph::graph_impl::multi_graph::runner::{catalog_generator, optimizer_executor}; use rust_graph::graph_impl::static_graph::StaticNode; -use rust_graph::graph_impl::Edge; use rust_graph::graph_impl::EdgeVec; +use rust_graph::graph_impl::{Edge, TypedDiGraphMap, TypedGraphMap}; +use rust_graph::io::read_from_csv; use rust_graph::map::SetMap; use rust_graph::prelude::*; use rust_graph::{DiStaticGraph, UnStaticGraph}; +use std::path::Path; #[test] fn test_directed() { @@ -318,6 +320,19 @@ fn test_clone() { #[test] fn test_graphflow_planner() { + // let mut g_:TypedGraphMap = TypedDiGraphMap::new(); + // let path_to_nodes = Path::new("C:\\Users\\cheny\\OneDrive\\桌面\\rust_graphflow\\human-vertices.csv"); + // let path_to_edges = Path::new("C:\\Users\\cheny\\OneDrive\\桌面\\rust_graphflow\\human-edges.csv"); + // read_from_csv( + // &mut g_, + // vec![path_to_nodes], + // vec![path_to_edges], + // None, + // false, + // false, + // ); + // let g = g_.into_static(); + // println!("load finished."); let edge_vec = EdgeVec::with_labels(vec![0, 2, 3, 4], vec![1, 2, 0, 0], vec![0, 1, 0, 1]); let in_edge_vec = EdgeVec::new(vec![0, 2, 3, 4], vec![1, 2, 0, 0]); let labels = vec![1, 0, 1]; @@ -334,69 +349,87 @@ fn test_graphflow_planner() { let mut qvertex_to_qedges_map = HashMap::new(); let mut qvertex_to_type_map = HashMap::new(); let mut qvertex_to_deg_map = HashMap::new(); - let q_edges = vec![QueryEdge::default("b".to_owned(), "a".to_owned())]; + let q_edges = vec![ + QueryEdge::default("a".to_owned(), "b".to_owned()), + QueryEdge::default("a".to_owned(), "c".to_owned()), + QueryEdge::default("b".to_owned(), "c".to_owned()), + QueryEdge::default("c".to_owned(), "e".to_owned()), + QueryEdge::default("c".to_owned(), "f".to_owned()), + QueryEdge::default("e".to_owned(), "f".to_owned()), + ]; let mut qedges_map = HashMap::new(); qedges_map.insert( - "a".to_owned(), - vec![QueryEdge::new("a".to_owned(), "b".to_owned(), 1, 0, 0)], + "b".to_owned(), + vec![QueryEdge::default("a".to_owned(), "b".to_owned())], + ); + qedges_map.insert( + "c".to_owned(), + vec![QueryEdge::default("a".to_owned(), "c".to_owned())], ); qvertex_to_qedges_map.insert("a".to_owned(), qedges_map); qedges_map = HashMap::new(); qedges_map.insert( - "b".to_owned(), - vec![QueryEdge::new("a".to_owned(), "b".to_owned(), 1, 0, 0)], + "a".to_owned(), + vec![QueryEdge::default("a".to_owned(), "b".to_owned())], + ); + qedges_map.insert( + "c".to_owned(), + vec![QueryEdge::default("b".to_owned(), "c".to_owned())], ); qvertex_to_qedges_map.insert("b".to_owned(), qedges_map); - qvertex_to_type_map.insert("a".to_owned(), 1); + qedges_map = HashMap::new(); + qedges_map.insert( + "a".to_owned(), + vec![QueryEdge::default("a".to_owned(), "c".to_owned())], + ); + qedges_map.insert( + "b".to_owned(), + vec![QueryEdge::default("b".to_owned(), "c".to_owned())], + ); + qedges_map.insert( + "e".to_owned(), + vec![QueryEdge::default("c".to_owned(), "e".to_owned())], + ); + qedges_map.insert( + "f".to_owned(), + vec![QueryEdge::default("c".to_owned(), "f".to_owned())], + ); + qvertex_to_qedges_map.insert("c".to_owned(), qedges_map); + qedges_map = HashMap::new(); + qedges_map.insert( + "c".to_owned(), + vec![QueryEdge::default("c".to_owned(), "e".to_owned())], + ); + qedges_map.insert( + "f".to_owned(), + vec![QueryEdge::default("e".to_owned(), "f".to_owned())], + ); + qvertex_to_qedges_map.insert("e".to_owned(), qedges_map); + qedges_map = HashMap::new(); + qedges_map.insert( + "c".to_owned(), + vec![QueryEdge::default("c".to_owned(), "f".to_owned())], + ); + qedges_map.insert( + "e".to_owned(), + vec![QueryEdge::default("e".to_owned(), "f".to_owned())], + ); + qvertex_to_qedges_map.insert("f".to_owned(), qedges_map); + + qvertex_to_type_map.insert("a".to_owned(), 0); qvertex_to_type_map.insert("b".to_owned(), 0); - qvertex_to_deg_map.insert("a".to_owned(), vec![1, 0]); - qvertex_to_deg_map.insert("b".to_owned(), vec![0, 1]); - - // let q_edges = vec![ - // QueryEdge::default("a".to_owned(), "b".to_owned()), - // QueryEdge::default("a".to_owned(), "c".to_owned()), - // QueryEdge::default("b".to_owned(), "c".to_owned()), - // QueryEdge::default("c".to_owned(), "e".to_owned()), - // QueryEdge::default("c".to_owned(), "f".to_owned()), - // QueryEdge::default("e".to_owned(), "f".to_owned()), - // ]; - // let mut qedges_map = HashMap::new(); - // qedges_map.insert("b".to_owned(),vec![QueryEdge::default("a".to_owned(),"b".to_owned())]); - // qedges_map.insert("c".to_owned(),vec![QueryEdge::default("a".to_owned(),"c".to_owned())]); - // qvertex_to_qedges_map.insert("a".to_owned(),qedges_map); - // qedges_map = HashMap::new(); - // qedges_map.insert("a".to_owned(),vec![QueryEdge::default("a".to_owned(),"b".to_owned())]); - // qedges_map.insert("c".to_owned(),vec![QueryEdge::default("b".to_owned(),"c".to_owned())]); - // qvertex_to_qedges_map.insert("b".to_owned(),qedges_map); - // qedges_map = HashMap::new(); - // qedges_map.insert("a".to_owned(),vec![QueryEdge::default("a".to_owned(),"c".to_owned())]); - // qedges_map.insert("b".to_owned(),vec![QueryEdge::default("b".to_owned(),"c".to_owned())]); - // qedges_map.insert("e".to_owned(),vec![QueryEdge::default("c".to_owned(),"e".to_owned())]); - // qedges_map.insert("f".to_owned(),vec![QueryEdge::default("c".to_owned(),"f".to_owned())]); - // qvertex_to_qedges_map.insert("c".to_owned(),qedges_map); - // qedges_map = HashMap::new(); - // qedges_map.insert("c".to_owned(),vec![QueryEdge::default("c".to_owned(),"e".to_owned())]); - // qedges_map.insert("f".to_owned(),vec![QueryEdge::default("e".to_owned(),"f".to_owned())]); - // qvertex_to_qedges_map.insert("e".to_owned(),qedges_map); - // qedges_map = HashMap::new(); - // qedges_map.insert("c".to_owned(),vec![QueryEdge::default("c".to_owned(),"f".to_owned())]); - // qedges_map.insert("e".to_owned(),vec![QueryEdge::default("e".to_owned(),"f".to_owned())]); - // qvertex_to_qedges_map.insert("f".to_owned(),qedges_map); - // - // qvertex_to_type_map.insert("a".to_owned(),0); - // qvertex_to_type_map.insert("b".to_owned(),0); - // qvertex_to_type_map.insert("c".to_owned(),0); - // qvertex_to_type_map.insert("e".to_owned(),0); - // qvertex_to_type_map.insert("f".to_owned(),0); - // - // qvertex_to_deg_map.insert("a".to_owned(),vec![2,0]); - // qvertex_to_deg_map.insert("b".to_owned(),vec![1,1]); - // qvertex_to_deg_map.insert("c".to_owned(),vec![2,2]); - // qvertex_to_deg_map.insert("e".to_owned(),vec![1,1]); - // qvertex_to_deg_map.insert("f".to_owned(),vec![0,2]); - - let query_graph = QueryGraph { + qvertex_to_type_map.insert("c".to_owned(), 0); + qvertex_to_type_map.insert("e".to_owned(), 0); + qvertex_to_type_map.insert("f".to_owned(), 0); + + qvertex_to_deg_map.insert("a".to_owned(), vec![2, 0]); + qvertex_to_deg_map.insert("b".to_owned(), vec![1, 1]); + qvertex_to_deg_map.insert("c".to_owned(), vec![2, 2]); + qvertex_to_deg_map.insert("e".to_owned(), vec![1, 1]); + qvertex_to_deg_map.insert("f".to_owned(), vec![0, 2]); + + let mut query_graph = QueryGraph { qvertex_to_qedges_map, qvertex_to_type_map, qvertex_to_deg_map, @@ -405,6 +438,6 @@ fn test_graphflow_planner() { encoding: None, limit: 0, }; - optimizer_executor::generate_plan(query_graph, catalog, g); + assert!(false); } From f2b027d80855f588bce5ec53f840b3ba695d8acd Mon Sep 17 00:00:00 2001 From: Michael Chen <920901835@qq.com> Date: Sun, 12 Jan 2020 17:12:01 +0800 Subject: [PATCH 15/25] fix:execute plan --- src/graph_impl/multi_graph/plan/mod.rs | 1 + .../plan/operator/hashjoin/build.rs | 6 +- .../multi_graph/plan/operator/mod.rs | 12 ++ .../plan/operator/scan/scan_blocking.rs | 8 +- .../plan/operator/sink/sink_limit.rs | 4 +- src/graph_impl/multi_graph/plan/query_plan.rs | 160 ++++++++++++++++-- .../multi_graph/plan/query_plan_worker.rs | 144 ++++++++++++++++ .../multi_graph/planner/catalog/query_edge.rs | 2 +- .../planner/catalog/query_graph.rs | 12 ++ .../multi_graph/planner/query_planner.rs | 2 +- src/graph_impl/static_graph/graph.rs | 4 +- tests/static_graph.rs | 48 +++--- 12 files changed, 357 insertions(+), 46 deletions(-) create mode 100644 src/graph_impl/multi_graph/plan/query_plan_worker.rs diff --git a/src/graph_impl/multi_graph/plan/mod.rs b/src/graph_impl/multi_graph/plan/mod.rs index f3b1fffe..49620f2c 100644 --- a/src/graph_impl/multi_graph/plan/mod.rs +++ b/src/graph_impl/multi_graph/plan/mod.rs @@ -1,3 +1,4 @@ #[macro_use] pub mod operator; pub mod query_plan; +pub mod query_plan_worker; diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/build.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/build.rs index 233bd695..1a9d4803 100644 --- a/src/graph_impl/multi_graph/plan/operator/hashjoin/build.rs +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/build.rs @@ -11,11 +11,11 @@ use std::hash::{BuildHasherDefault, Hash}; #[derive(Clone)] pub struct Build { pub base_op: BaseOperator, - hash_table: Option>, + pub hash_table: Option>, pub probing_subgraph: Option>, query_vertex_to_hash: String, - build_hash_idx: usize, - hashed_tuple_len: usize, + pub build_hash_idx: usize, + pub hashed_tuple_len: usize, } impl Build { diff --git a/src/graph_impl/multi_graph/plan/operator/mod.rs b/src/graph_impl/multi_graph/plan/operator/mod.rs index 6b1076ff..5c81c2b0 100644 --- a/src/graph_impl/multi_graph/plan/operator/mod.rs +++ b/src/graph_impl/multi_graph/plan/operator/mod.rs @@ -55,6 +55,18 @@ macro_rules! get_scan_as_mut { }; } +#[macro_export] +macro_rules! get_probe_as_mut { + ($item:expr) => { + match $item { + Probe::BaseProbe(base) => base, + Probe::PC(base) => &mut base.base_probe, + Probe::PMV(PMV::BasePMV(base)) => &mut base.base_probe, + Probe::PMV(PMV::PMVC(pmvc)) => &mut pmvc.base_pmv.base_probe, + } + }; +} + #[macro_export] macro_rules! get_scan_as_ref { ($item:expr) => { diff --git a/src/graph_impl/multi_graph/plan/operator/scan/scan_blocking.rs b/src/graph_impl/multi_graph/plan/operator/scan/scan_blocking.rs index b8ebae10..84de1076 100644 --- a/src/graph_impl/multi_graph/plan/operator/scan/scan_blocking.rs +++ b/src/graph_impl/multi_graph/plan/operator/scan/scan_blocking.rs @@ -9,9 +9,9 @@ use std::hash::{BuildHasherDefault, Hash}; static PARTITION_SIZE: usize = 100; #[derive(Clone)] -struct VertexIdxLimits { - from_variable_index_limit: usize, - to_variable_index_limit: usize, +pub struct VertexIdxLimits { + pub from_variable_index_limit: usize, + pub to_variable_index_limit: usize, } ///TODO:ReentrantLock @@ -24,7 +24,7 @@ pub struct ScanBlocking { to_idx_limit: usize, highest_from_idx: usize, highest_to_idx: usize, - global_vertices_idx_limits: VertexIdxLimits, + pub global_vertices_idx_limits: VertexIdxLimits, } impl ScanBlocking { diff --git a/src/graph_impl/multi_graph/plan/operator/sink/sink_limit.rs b/src/graph_impl/multi_graph/plan/operator/sink/sink_limit.rs index ad7c1618..753a9611 100644 --- a/src/graph_impl/multi_graph/plan/operator/sink/sink_limit.rs +++ b/src/graph_impl/multi_graph/plan/operator/sink/sink_limit.rs @@ -14,8 +14,8 @@ use std::time::SystemTime; #[derive(Clone)] pub struct SinkLimit { pub base_sink: BaseSink, - start_time: SystemTime, - elapsed_time: u128, + pub start_time: SystemTime, + pub elapsed_time: u128, out_tuples_limit: usize, } diff --git a/src/graph_impl/multi_graph/plan/query_plan.rs b/src/graph_impl/multi_graph/plan/query_plan.rs index 2e9a21f4..f42e6f6a 100644 --- a/src/graph_impl/multi_graph/plan/query_plan.rs +++ b/src/graph_impl/multi_graph/plan/query_plan.rs @@ -1,14 +1,24 @@ -use generic::IdType; +use generic::{GraphType, IdType}; use graph_impl::multi_graph::plan::operator::extend::EI::EI; +use graph_impl::multi_graph::plan::operator::hashjoin::hash_table::HashTable; use graph_impl::multi_graph::plan::operator::hashjoin::probe::Probe; use graph_impl::multi_graph::plan::operator::hashjoin::probe_multi_vertices::PMV; -use graph_impl::multi_graph::plan::operator::operator::{BaseOperator, Operator}; +use graph_impl::multi_graph::plan::operator::operator::{ + BaseOperator, CommonOperatorTrait, Operator, +}; use graph_impl::multi_graph::plan::operator::scan::scan::{BaseScan, Scan}; use graph_impl::multi_graph::plan::operator::scan::scan_sampling::ScanSampling; use graph_impl::multi_graph::plan::operator::sink::sink::{BaseSink, Sink, SinkType}; +use graph_impl::multi_graph::plan::operator::sink::sink_copy::SinkCopy; +use graph_impl::multi_graph::plan::operator::sink::sink_limit::SinkLimit; +use graph_impl::multi_graph::plan::operator::sink::sink_print::SinkPrint; +use graph_impl::TypedStaticGraph; use hashbrown::HashMap; +use std::hash::Hash; use std::ops::Deref; use std::rc::Rc; +use std::time::SystemTime; +use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; #[derive(Clone)] pub struct QueryPlan { @@ -17,10 +27,10 @@ pub struct QueryPlan { scan_sampling: Option>, pub last_operator: Option>>, pub out_tuples_limit: usize, - elapsed_time: f64, - icost: usize, - num_intermediate_tuples: usize, - num_out_tuples: usize, + pub elapsed_time: u128, + pub icost: usize, + pub num_intermediate_tuples: usize, + pub num_out_tuples: usize, pub operator_metrics: Vec<(String, usize, usize)>, executed: bool, adaptive_enabled: bool, @@ -49,7 +59,7 @@ impl QueryPlan { scan_sampling: Some(scan_sampling), last_operator: None, out_tuples_limit: 0, - elapsed_time: 0.0, + elapsed_time: 0, icost: 0, num_intermediate_tuples: 0, num_out_tuples: 0, @@ -69,7 +79,7 @@ impl QueryPlan { scan_sampling: None, last_operator: Some(last_operator.clone()), out_tuples_limit: 0, - elapsed_time: 0.0, + elapsed_time: 0, icost: 0, num_intermediate_tuples: 0, num_out_tuples: 0, @@ -89,7 +99,7 @@ impl QueryPlan { scan_sampling: None, last_operator: subplans.get(subplans.len() - 1).map(|x| x.clone()), out_tuples_limit: 0, - elapsed_time: 0.0, + elapsed_time: 0, icost: 0, num_intermediate_tuples: 0, num_out_tuples: 0, @@ -166,7 +176,8 @@ impl QueryPlan { } str_joiner.join(",") } - fn set_stats(&mut self) { + + pub fn set_stats(&mut self) { for subplan in &self.subplans { let mut first_operator = subplan.as_ref(); while get_op_attr_as_ref!(first_operator, prev).is_some() { @@ -183,4 +194,133 @@ impl QueryPlan { } self.icost += self.operator_metrics[self.operator_metrics.len() - 1].1; } + + pub fn copy(&self, is_thread_safe: bool) -> QueryPlan { + let mut subplans = vec![]; + for subplan in &self.subplans { + subplans.push(Box::new(subplan.copy(is_thread_safe))); + } + QueryPlan::new_from_subplans(subplans) + } + + pub fn init( + &mut self, + graph: &TypedStaticGraph, + ) { + let plan_size = self.subplans.len(); + let last_operator = self.subplans[plan_size - 1].as_ref(); + let query_graph = get_op_attr_as_ref!(last_operator, out_subgraph); + let mut sink = match self.sink_type { + SinkType::Copy => Operator::Sink(Sink::SinkCopy(SinkCopy::new( + query_graph.clone(), + get_op_attr!(last_operator, out_tuple_len), + ))), + SinkType::Print => Operator::Sink(Sink::SinkPrint(SinkPrint::new(query_graph.clone()))), + SinkType::Limit => Operator::Sink(Sink::SinkLimit(SinkLimit::new( + query_graph.clone(), + self.out_tuples_limit, + ))), + SinkType::Counter => Operator::Sink(Sink::BaseSink(BaseSink::new(query_graph.clone()))), + }; + let sink_prev = get_op_attr_as_mut!(&mut sink, prev) + .as_mut() + .unwrap() + .as_mut(); + *sink_prev = last_operator.clone(); + *get_op_attr_as_mut!(self.subplans[plan_size - 1].as_mut(), next) = Some(vec![sink]); +// let mut probes = vec![]; +// for operator in &mut self.subplans { +// let mut op = operator.as_mut(); +// while get_op_attr_as_mut!(op, prev).is_some() { +// op = get_op_attr_as_mut!(op, prev).as_mut().unwrap(); +// if let Operator::Probe(p) = op { +// probes.push(op); +// } +// } +// if let Operator::Probe(p) = operator.as_mut() { +// probes.push(operator); +// } +// } + for subplan in &mut self.subplans { + if let Operator::Build(build) = subplan.as_mut() { + let hash_table = HashTable::new(build.build_hash_idx, build.hashed_tuple_len); + build.hash_table = Some(hash_table.clone()); + } + } + let sub_plans = self.subplans.clone(); + for subplan in sub_plans { + if let Operator::Build(build) = subplan.as_ref() { + let hash_table = build.hash_table.as_ref().unwrap(); + let build_insubgrpah = get_op_attr_as_ref!(subplan.as_ref(), in_subgraph) + .as_ref() + .unwrap() + .as_ref(); + self.init_hashtable(build_insubgrpah,hash_table); + } + } + for subplan in &mut self.subplans { + let probe_tuple = vec![]; + let mut first_op = subplan.as_mut(); + while get_op_attr_as_mut!(first_op, prev).is_some() { + first_op = get_op_attr_as_mut!(first_op, prev) + .as_mut() + .unwrap() + .as_mut(); + } + first_op.init(probe_tuple, graph); + } + } + + fn init_hashtable(&mut self,build_insubgrpah:&QueryGraph,hash_table:&HashTable){ + for operator in &mut self.subplans { + if let Operator::Probe(p) = operator.as_mut() { + if Self::check_and_init(build_insubgrpah,operator,hash_table.clone()){ + break; + } + } + let mut op = operator.as_mut(); + while get_op_attr_as_mut!(op, prev).is_some() { + op = get_op_attr_as_mut!(op, prev).as_mut().unwrap(); + if let Operator::Probe(p) = op { + if Self::check_and_init(build_insubgrpah,op,hash_table.clone()){ + return; + } + } + } + } + } + + fn check_and_init(build_insubgrpah:&QueryGraph,probe:&mut Operator,hash_table:HashTable)->bool{ + let prob_insubgraph = get_op_attr_as_ref!(probe, in_subgraph) + .as_ref() + .unwrap() + .as_ref(); + if prob_insubgraph == build_insubgrpah { + if let Operator::Probe(probe_op) = probe { + let mut base_probe = get_probe_as_mut!(probe_op); + base_probe.hash_tables = vec![hash_table.clone()]; + return true; + } + } + false + } + + pub fn execute(&mut self) { + if let SinkType::Limit = self.sink_type { + if let Sink::SinkLimit(sink) = self.sink.as_mut().unwrap() { + sink.start_time = SystemTime::now(); + self.subplans.iter_mut().for_each(|plan| plan.execute()); + self.elapsed_time = sink.elapsed_time; + } + } else { + let start_time = SystemTime::now(); + self.subplans.iter_mut().for_each(|plan| plan.execute()); + self.elapsed_time = SystemTime::now() + .duration_since(start_time) + .unwrap() + .as_millis(); + } + self.executed = true; + self.num_out_tuples = self.sink.as_ref().unwrap().get_num_out_tuples(); + } } diff --git a/src/graph_impl/multi_graph/plan/query_plan_worker.rs b/src/graph_impl/multi_graph/plan/query_plan_worker.rs new file mode 100644 index 00000000..c18115d4 --- /dev/null +++ b/src/graph_impl/multi_graph/plan/query_plan_worker.rs @@ -0,0 +1,144 @@ +use generic::{GraphType, IdType}; +use graph_impl::multi_graph::plan::operator::extend::EI::EI; +use graph_impl::multi_graph::plan::operator::hashjoin::probe::Probe; +use graph_impl::multi_graph::plan::operator::hashjoin::probe_multi_vertices::PMV; +use graph_impl::multi_graph::plan::operator::operator::Operator; +use graph_impl::multi_graph::plan::operator::scan::scan::Scan; +use graph_impl::multi_graph::plan::operator::scan::scan_blocking::VertexIdxLimits; +use graph_impl::multi_graph::plan::operator::sink::sink::Sink; +use graph_impl::multi_graph::plan::query_plan::QueryPlan; +use graph_impl::TypedStaticGraph; +use std::hash::Hash; +use std::thread; +use std::thread::JoinHandle; +use std::time::SystemTime; + +pub struct QPWorkers { + query_plans: Vec>, + elapsed_time: u128, + intersection_cost: usize, + num_intermediate_tuples: usize, + num_out_tuples: usize, + operator_metrics: Vec<(String, usize, usize)>, +} + +impl QPWorkers { + pub fn new(query_plan: QueryPlan, num_threads: usize) -> QPWorkers { + let mut worker = QPWorkers { + query_plans: vec![], + elapsed_time: 0, + intersection_cost: 0, + num_intermediate_tuples: 0, + num_out_tuples: 0, + operator_metrics: vec![], + }; + if num_threads == 1 { + worker.query_plans.push(query_plan); + } else { + // num_threads > 1 + for i in 0..num_threads { + worker.query_plans.push(query_plan.copy(true)); + } + let global_vertex_idx_limits = VertexIdxLimits { + from_variable_index_limit: 0, + to_variable_index_limit: 0, + }; + for query_plan in &mut worker.query_plans { + for last_op in &mut query_plan.subplans { + let mut op = last_op.as_mut(); + while get_op_attr_as_ref!(op, prev).is_some() { + op = get_op_attr_as_mut!(op, prev).as_mut().unwrap().as_mut(); + } + if let Operator::Scan(Scan::ScanBlocking(sb)) = op { + //TODO:Lock need to be fixed + sb.global_vertices_idx_limits = global_vertex_idx_limits.clone(); + } + } + } + } + worker + } + + pub fn init( + &mut self, + graph: &TypedStaticGraph, + ) { + self.query_plans + .iter_mut() + .for_each(|plan| plan.init(graph)); + } + + pub fn execute(&mut self) { + if self.query_plans.len() == 1 { + self.query_plans[0].execute(); + self.elapsed_time = self.query_plans[0].elapsed_time; + } else { + // TODO: update LIMIT for the workers class. + let begin_time = SystemTime::now(); + let mut workers = vec![]; + for plan in &self.query_plans { + let mut plan = plan.clone(); + workers.push(thread::spawn(move || { + plan.execute(); + })); + } + for worker in workers { + worker.join(); + } + self.elapsed_time = SystemTime::now() + .duration_since(begin_time) + .unwrap() + .as_millis(); + } + } + + pub fn get_output_log(&mut self) -> String { + if self.query_plans.len() == 1 { + return self.query_plans[0].get_output_log(); + } + if self.operator_metrics.is_empty() { + self.query_plans + .iter_mut() + .for_each(|plan| plan.set_stats()); + self.aggregate_output(); + } + let mut str_joiner = vec![]; + str_joiner.push(format!("{}", self.elapsed_time)); + str_joiner.push(format!("{}", self.num_out_tuples)); + str_joiner.push(format!("{}", self.num_intermediate_tuples)); + str_joiner.push(format!("{}", self.intersection_cost)); + for operator_metric in &self.operator_metrics { + str_joiner.push(format!("{}", operator_metric.0)); /* operator name */ + if !operator_metric.0.contains("PROBE") + && !operator_metric.0.contains("HASH") + && !operator_metric.0.contains("SCAN") + { + str_joiner.push(format!("{}", operator_metric.1)); /* i-cost */ + } + if !operator_metric.0.contains("HASH") { + str_joiner.push(format!("{}", operator_metric.2)); /* output tuples size */ + } + } + str_joiner.join(",") + } + + fn aggregate_output(&mut self) { + self.operator_metrics = vec![]; + for plan in &mut self.query_plans { + self.intersection_cost += plan.icost; + self.num_intermediate_tuples += plan.num_intermediate_tuples; + self.num_out_tuples += plan.num_out_tuples; + } + let query_plan = &mut self.query_plans[0].operator_metrics; + for metric in &mut self.query_plans[0].operator_metrics { + self.operator_metrics + .push((metric.0.clone(), metric.1, metric.2)); + } + for i in 0..self.query_plans.len() { + for j in 0..self.operator_metrics.len() { + self.operator_metrics[j].1 += self.query_plans[i].operator_metrics[j].1; + self.operator_metrics[j].2 += self.query_plans[i].operator_metrics[j].2; + } + } + } +} diff --git a/src/graph_impl/multi_graph/planner/catalog/query_edge.rs b/src/graph_impl/multi_graph/planner/catalog/query_edge.rs index 2c501f4f..48f1b66f 100644 --- a/src/graph_impl/multi_graph/planner/catalog/query_edge.rs +++ b/src/graph_impl/multi_graph/planner/catalog/query_edge.rs @@ -1,4 +1,4 @@ -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq)] pub struct QueryEdge { pub from_query_vertex: String, pub to_query_vertex: String, diff --git a/src/graph_impl/multi_graph/planner/catalog/query_graph.rs b/src/graph_impl/multi_graph/planner/catalog/query_graph.rs index 93fdd87b..9b77624e 100644 --- a/src/graph_impl/multi_graph/planner/catalog/query_graph.rs +++ b/src/graph_impl/multi_graph/planner/catalog/query_graph.rs @@ -3,6 +3,7 @@ use graph_impl::multi_graph::planner::catalog::subgraph_mapping_iterator::Subgra use hashbrown::{HashMap, HashSet}; use itertools::Itertools; use std::iter::FromIterator; +use std::process::id; #[derive(Clone)] pub struct QueryGraph { @@ -256,3 +257,14 @@ impl QueryGraph { q } } + +impl PartialEq for QueryGraph { + fn eq(&self, other: &Self) -> bool { + self.qvertex_to_qedges_map == other.qvertex_to_qedges_map + && self.qvertex_to_type_map == other.qvertex_to_type_map + && self.qvertex_to_deg_map == other.qvertex_to_deg_map + && self.q_edges.eq(&other.q_edges) + && self.encoding.as_ref().unwrap() == other.encoding.as_ref().unwrap() + && self.limit == other.limit + } +} diff --git a/src/graph_impl/multi_graph/planner/query_planner.rs b/src/graph_impl/multi_graph/planner/query_planner.rs index 9f17eb92..76a5b3eb 100644 --- a/src/graph_impl/multi_graph/planner/query_planner.rs +++ b/src/graph_impl/multi_graph/planner/query_planner.rs @@ -549,7 +549,7 @@ impl .query_graph .get_query_vertex_type(&query_edge.to_query_vertex); let label = query_edge.label; - return self.graph.get_num_edges(from_type, to_type, label); + self.graph.get_num_edges(from_type, to_type, label) } fn get_key(&self, query_vertices: &mut Vec) -> String { diff --git a/src/graph_impl/static_graph/graph.rs b/src/graph_impl/static_graph/graph.rs index d506953e..fb465254 100644 --- a/src/graph_impl/static_graph/graph.rs +++ b/src/graph_impl/static_graph/graph.rs @@ -401,9 +401,9 @@ impl self.partition_nodes(); self.partition_edges(); let label_cnt = if self.sort_by_node { - self.num_of_node_labels() + self.num_of_node_labels() + 1 } else { - self.num_of_edge_labels() + self.num_of_edge_labels() + 1 }; self.label_to_num_edges = vec![0; label_cnt]; self.label_to_largest_fwd_adj_list_size = vec![0; label_cnt]; diff --git a/tests/static_graph.rs b/tests/static_graph.rs index 0b5ace17..c0256e7f 100644 --- a/tests/static_graph.rs +++ b/tests/static_graph.rs @@ -320,31 +320,33 @@ fn test_clone() { #[test] fn test_graphflow_planner() { - // let mut g_:TypedGraphMap = TypedDiGraphMap::new(); - // let path_to_nodes = Path::new("C:\\Users\\cheny\\OneDrive\\桌面\\rust_graphflow\\human-vertices.csv"); - // let path_to_edges = Path::new("C:\\Users\\cheny\\OneDrive\\桌面\\rust_graphflow\\human-edges.csv"); - // read_from_csv( - // &mut g_, - // vec![path_to_nodes], - // vec![path_to_edges], - // None, - // false, - // false, - // ); - // let g = g_.into_static(); - // println!("load finished."); - let edge_vec = EdgeVec::with_labels(vec![0, 2, 3, 4], vec![1, 2, 0, 0], vec![0, 1, 0, 1]); - let in_edge_vec = EdgeVec::new(vec![0, 2, 3, 4], vec![1, 2, 0, 0]); - let labels = vec![1, 0, 1]; - let g = DiStaticGraph::<&str>::with_labels( - edge_vec, - Some(in_edge_vec), - labels, - setmap!["a", "b"], - setmap!["a", "b"], - None, + let mut g_: TypedGraphMap = TypedDiGraphMap::new(); + let path_to_nodes = + Path::new("C:\\Users\\cheny\\OneDrive\\桌面\\rust_graphflow\\human-vertices.csv"); + let path_to_edges = + Path::new("C:\\Users\\cheny\\OneDrive\\桌面\\rust_graphflow\\human-edges.csv"); + read_from_csv( + &mut g_, + vec![path_to_nodes], + vec![path_to_edges], None, + false, + false, ); + let g = g_.into_static(); + println!("load finished."); + // let edge_vec = EdgeVec::with_labels(vec![0, 2, 3, 4], vec![1, 2, 0, 0], vec![0, 1, 0, 1]); + // let in_edge_vec = EdgeVec::new(vec![0, 2, 3, 4], vec![1, 2, 0, 0]); + // let labels = vec![1, 0, 1]; + // let g = DiStaticGraph::<&str>::with_labels( + // edge_vec, + // Some(in_edge_vec), + // labels, + // setmap!["a", "b"], + // setmap!["a", "b"], + // None, + // None, + // ); let catalog = catalog_generator::default(&g); let mut qvertex_to_qedges_map = HashMap::new(); let mut qvertex_to_type_map = HashMap::new(); From ddfd2b1c8ba37bcee84be3a60aceee9ff97f1cd8 Mon Sep 17 00:00:00 2001 From: Michael Chen <920901835@qq.com> Date: Sun, 12 Jan 2020 17:22:30 +0800 Subject: [PATCH 16/25] fix:add runner --- src/graph_impl/multi_graph/runner/optimizer_executor.rs | 5 +++-- tests/static_graph.rs | 8 +++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/graph_impl/multi_graph/runner/optimizer_executor.rs b/src/graph_impl/multi_graph/runner/optimizer_executor.rs index a68c4e07..d1377e03 100644 --- a/src/graph_impl/multi_graph/runner/optimizer_executor.rs +++ b/src/graph_impl/multi_graph/runner/optimizer_executor.rs @@ -10,12 +10,13 @@ use hashbrown::HashMap; use std::hash::Hash; use std::time::SystemTime; use DiStaticGraph; +use graph_impl::multi_graph::plan::query_plan::QueryPlan; pub fn generate_plan( query_graph: QueryGraph, catalog: Catalog, g: TypedStaticGraph, -) { +)->QueryPlan { let num_qvertices = query_graph.get_num_qvertices(); let start_time = SystemTime::now(); let elapsed_time; @@ -37,5 +38,5 @@ pub fn generate_plan Date: Tue, 14 Jan 2020 19:02:13 +0800 Subject: [PATCH 17/25] fix:catalog --- src/graph_impl/multi_graph/mod.rs | 1 + .../multi_graph/plan/operator/extend/EI.rs | 27 +- .../plan/operator/extend/Extend.rs | 8 +- .../plan/operator/extend/Intersect.rs | 93 +++- .../plan/operator/extend/extend.rs | 8 +- .../plan/operator/extend/intersect.rs | 93 +++- .../plan/operator/hashjoin/build.rs | 2 +- .../plan/operator/hashjoin/hash_join.rs | 6 +- .../plan/operator/hashjoin/probe.rs | 18 +- .../operator/hashjoin/probe_multi_vertices.rs | 13 +- .../multi_graph/plan/operator/mod.rs | 40 +- .../multi_graph/plan/operator/operator.rs | 36 +- .../multi_graph/plan/operator/scan/scan.rs | 16 +- .../plan/operator/scan/scan_blocking.rs | 20 +- .../plan/operator/scan/scan_sampling.rs | 28 +- .../multi_graph/plan/operator/sink/sink.rs | 2 + .../plan/operator/sink/sink_limit.rs | 2 + src/graph_impl/multi_graph/plan/query_plan.rs | 45 +- .../multi_graph/plan/query_plan_worker.rs | 3 +- .../multi_graph/planner/catalog/catalog.rs | 216 ++++++++- .../planner/catalog/catalog_plans.rs | 435 +++++++++++++++++- .../catalog/operator/intersect_catalog.rs | 27 +- .../planner/catalog/operator/noop.rs | 14 +- .../multi_graph/planner/query_planner.rs | 4 +- .../multi_graph/planner/query_planner_big.rs | 2 + .../multi_graph/runner/optimizer_executor.rs | 7 +- src/graph_impl/multi_graph/utils/mod.rs | 1 + src/graph_impl/multi_graph/utils/set_utils.rs | 58 +++ tests/static_graph.rs | 6 +- 29 files changed, 1022 insertions(+), 209 deletions(-) create mode 100644 src/graph_impl/multi_graph/utils/mod.rs create mode 100644 src/graph_impl/multi_graph/utils/set_utils.rs diff --git a/src/graph_impl/multi_graph/mod.rs b/src/graph_impl/multi_graph/mod.rs index 6beb476f..5fbc9599 100644 --- a/src/graph_impl/multi_graph/mod.rs +++ b/src/graph_impl/multi_graph/mod.rs @@ -4,3 +4,4 @@ pub mod node; pub mod planner; pub mod query; pub mod runner; +pub mod utils; diff --git a/src/graph_impl/multi_graph/plan/operator/extend/EI.rs b/src/graph_impl/multi_graph/plan/operator/extend/EI.rs index 29e89173..0bc86310 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/EI.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/EI.rs @@ -1,6 +1,8 @@ use generic::{GraphType, IdType}; use graph_impl::multi_graph::plan::operator::extend::extend::Extend; -use graph_impl::multi_graph::plan::operator::extend::intersect::{Intersect, IntersectType}; +use graph_impl::multi_graph::plan::operator::extend::intersect::{ + BaseIntersect, Intersect, IntersectType, +}; use graph_impl::multi_graph::plan::operator::hashjoin::probe::Probe; use graph_impl::multi_graph::plan::operator::hashjoin::probe_multi_vertices::PMV; use graph_impl::multi_graph::plan::operator::operator::{ @@ -289,14 +291,9 @@ impl CommonOperatorTrait for BaseEI { self.init_caching(last_repeated_vertex_idx); self.init_extensions(graph); self.set_alds_and_adj_lists(graph, last_repeated_vertex_idx); - self.base_op - .next - .as_mut() - .unwrap() - .iter_mut() - .foreach(|next_op| { - next_op.init(probe_tuple.clone(), graph); - }); + self.base_op.next.iter_mut().foreach(|next_op| { + next_op.init(probe_tuple.clone(), graph); + }); } fn process_new_tuple(&mut self) { @@ -335,10 +332,10 @@ impl CommonOperatorTrait for BaseEI { self.to_query_vertex.clone(), query_vertex_to_index_map.len(), ); - if let Some(next) = &mut self.base_op.next { - next.iter_mut() - .foreach(|op| op.update_operator_name(query_vertex_to_index_map.clone())) - } + self.base_op + .next + .iter_mut() + .foreach(|op| op.update_operator_name(query_vertex_to_index_map.clone())) } fn copy(&self, is_thread_safe: bool) -> Operator { @@ -384,14 +381,14 @@ impl EI { out_qvertex_to_idx_map, )); } - EI::Intersect(Intersect::new( + EI::Intersect(Intersect::BaseIntersect(BaseIntersect::new( to_qvertex, to_type, alds, Box::new(out_subgraph), Some(Box::new(in_subgraph)), out_qvertex_to_idx_map, - )) + ))) } } diff --git a/src/graph_impl/multi_graph/plan/operator/extend/Extend.rs b/src/graph_impl/multi_graph/plan/operator/extend/Extend.rs index 80185df9..133166ad 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/Extend.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/Extend.rs @@ -1,4 +1,6 @@ use generic::{GraphType, IdType}; +use graph_impl::multi_graph::plan::operator::extend::intersect::BaseIntersect; +use graph_impl::multi_graph::plan::operator::extend::intersect::Intersect; use graph_impl::multi_graph::plan::operator::extend::EI::{ BaseEI, Neighbours, DIFFERENTIATE_FWD_BWD_SINGLE_ALD, EI, }; @@ -68,7 +70,7 @@ impl CommonOperatorTrait for Extend { self.label_or_to_type = self.base_ei.to_type; self.base_ei.to_type = 0; } - for next_operator in self.base_ei.base_op.next.as_mut().unwrap() { + for next_operator in &mut self.base_ei.base_op.next { next_operator.init(probe_tuple.clone(), graph); } } @@ -86,7 +88,7 @@ impl CommonOperatorTrait for Extend { { self.base_ei.base_op.num_out_tuples += 1; self.base_ei.base_op.probe_tuple[self.base_ei.out_idx] = out_neighbour.ids[idx]; - self.base_ei.base_op.next.as_mut().unwrap()[0].process_new_tuple(); + self.base_ei.base_op.next[0].process_new_tuple(); } } } @@ -119,7 +121,7 @@ impl CommonOperatorTrait for Extend { base_op.prev.as_ref().unwrap().copy(is_thread_safe), )); let prev = extend.base_ei.base_op.prev.as_mut().unwrap().as_mut(); - get_op_attr_as_mut!(prev, next).replace(vec![Operator::EI(EI::Extend(extend_copy))]); + *get_op_attr_as_mut!(prev, next) = vec![Operator::EI(EI::Extend(extend_copy))]; let last_repeated_vertex_idx = get_op_attr!(prev, last_repeated_vertex_idx); extend.base_ei.init_caching(last_repeated_vertex_idx); Operator::EI(EI::Extend(extend)) diff --git a/src/graph_impl/multi_graph/plan/operator/extend/Intersect.rs b/src/graph_impl/multi_graph/plan/operator/extend/Intersect.rs index cab933cd..0b6afec7 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/Intersect.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/Intersect.rs @@ -7,6 +7,7 @@ use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Ope use graph_impl::multi_graph::plan::operator::scan::scan::Scan; use graph_impl::multi_graph::plan::operator::sink::sink::Sink; use graph_impl::multi_graph::planner::catalog::adj_list_descriptor::AdjListDescriptor; +use graph_impl::multi_graph::planner::catalog::operator::intersect_catalog::IntersectCatalog; use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; @@ -20,11 +21,17 @@ pub enum IntersectType { } #[derive(Clone)] -pub struct Intersect { +pub enum Intersect { + BaseIntersect(BaseIntersect), + IntersectCatalog(IntersectCatalog), +} + +#[derive(Clone)] +pub struct BaseIntersect { pub base_ei: BaseEI, } -impl Intersect { +impl BaseIntersect { pub fn new( to_qvertex: String, to_type: usize, @@ -32,8 +39,8 @@ impl Intersect { out_subgraph: Box, in_subgraph: Option>, out_qvertex_to_idx_map: HashMap, - ) -> Intersect { - let mut intersect = Intersect { + ) -> BaseIntersect { + let mut intersect = BaseIntersect { base_ei: BaseEI::new(to_qvertex.clone(), to_type, alds, out_subgraph, in_subgraph), }; let base_op = &mut intersect.base_ei.base_op; @@ -44,7 +51,7 @@ impl Intersect { } } -impl CommonOperatorTrait for Intersect { +impl CommonOperatorTrait for BaseIntersect { fn init( &mut self, probe_tuple: Vec, @@ -110,9 +117,7 @@ impl CommonOperatorTrait for Intersect { base_op.num_out_tuples += (out_neighbours.end_idx - out_neighbours.start_idx); for idx in out_neighbours.start_idx..out_neighbours.end_idx { base_op.probe_tuple[base_ei.out_idx] = out_neighbours.ids[idx]; - base_op.next.as_mut().map(|op_vec| { - op_vec.get_mut(0).map(|op| op.process_new_tuple()); - }); + base_op.next[0].process_new_tuple(); } } @@ -131,7 +136,7 @@ impl CommonOperatorTrait for Intersect { fn copy(&self, is_thread_safe: bool) -> Operator { let base_ei = &self.base_ei; let base_op = &base_ei.base_op; - let mut intersect = Intersect::new( + let mut intersect = BaseIntersect::new( base_ei.to_query_vertex.clone(), base_ei.to_type, base_ei.alds.clone(), @@ -144,14 +149,16 @@ impl CommonOperatorTrait for Intersect { base_op.prev.as_ref().unwrap().copy(is_thread_safe), )); let prev = intersect.base_ei.base_op.prev.as_mut().unwrap().as_mut(); - *get_op_attr_as_mut!(prev, next) = Some(vec![Operator::EI(EI::Intersect(intersect_copy))]); + *get_op_attr_as_mut!(prev, next) = vec![Operator::EI(EI::Intersect( + Intersect::BaseIntersect(intersect_copy), + ))]; let last_repeated_vertex_idx = get_op_attr!(prev, last_repeated_vertex_idx); intersect.base_ei.init_caching(last_repeated_vertex_idx); - Operator::EI(EI::Intersect(intersect)) + Operator::EI(EI::Intersect(Intersect::BaseIntersect(intersect))) } fn is_same_as(&mut self, op: &mut Operator) -> bool { - if let Operator::EI(EI::Intersect(intersect)) = op { + if let Operator::EI(EI::Intersect(Intersect::BaseIntersect(intersect))) = op { return self.base_ei.caching_type == intersect.base_ei.caching_type && self.get_alds_as_string() == intersect.get_alds_as_string() && self @@ -186,3 +193,65 @@ impl CommonOperatorTrait for Intersect { self.base_ei.get_num_out_tuples() } } + +impl CommonOperatorTrait for Intersect { + fn init( + &mut self, + probe_tuple: Vec, + graph: &TypedStaticGraph, + ) { + match self { + Intersect::BaseIntersect(base) => base.init(probe_tuple, graph), + Intersect::IntersectCatalog(ic) => ic.init(probe_tuple, graph), + } + } + + fn process_new_tuple(&mut self) { + match self { + Intersect::BaseIntersect(base) => base.process_new_tuple(), + Intersect::IntersectCatalog(ic) => ic.process_new_tuple(), + } + } + + fn execute(&mut self) { + match self { + Intersect::BaseIntersect(base) => base.execute(), + Intersect::IntersectCatalog(ic) => ic.execute(), + } + } + + fn get_alds_as_string(&self) -> String { + match self { + Intersect::BaseIntersect(base) => base.get_alds_as_string(), + Intersect::IntersectCatalog(ic) => ic.get_alds_as_string(), + } + } + + fn update_operator_name(&mut self, query_vertex_to_index_map: HashMap) { + match self { + Intersect::BaseIntersect(base) => base.update_operator_name(query_vertex_to_index_map), + Intersect::IntersectCatalog(ic) => ic.update_operator_name(query_vertex_to_index_map), + } + } + + fn copy(&self, is_thread_safe: bool) -> Operator { + match self { + Intersect::BaseIntersect(base) => base.copy(is_thread_safe), + Intersect::IntersectCatalog(ic) => ic.copy(is_thread_safe), + } + } + + fn is_same_as(&mut self, op: &mut Operator) -> bool { + match self { + Intersect::BaseIntersect(base) => base.is_same_as(op), + Intersect::IntersectCatalog(ic) => ic.is_same_as(op), + } + } + + fn get_num_out_tuples(&self) -> usize { + match self { + Intersect::BaseIntersect(base) => base.get_num_out_tuples(), + Intersect::IntersectCatalog(ic) => ic.get_num_out_tuples(), + } + } +} diff --git a/src/graph_impl/multi_graph/plan/operator/extend/extend.rs b/src/graph_impl/multi_graph/plan/operator/extend/extend.rs index 80185df9..133166ad 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/extend.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/extend.rs @@ -1,4 +1,6 @@ use generic::{GraphType, IdType}; +use graph_impl::multi_graph::plan::operator::extend::intersect::BaseIntersect; +use graph_impl::multi_graph::plan::operator::extend::intersect::Intersect; use graph_impl::multi_graph::plan::operator::extend::EI::{ BaseEI, Neighbours, DIFFERENTIATE_FWD_BWD_SINGLE_ALD, EI, }; @@ -68,7 +70,7 @@ impl CommonOperatorTrait for Extend { self.label_or_to_type = self.base_ei.to_type; self.base_ei.to_type = 0; } - for next_operator in self.base_ei.base_op.next.as_mut().unwrap() { + for next_operator in &mut self.base_ei.base_op.next { next_operator.init(probe_tuple.clone(), graph); } } @@ -86,7 +88,7 @@ impl CommonOperatorTrait for Extend { { self.base_ei.base_op.num_out_tuples += 1; self.base_ei.base_op.probe_tuple[self.base_ei.out_idx] = out_neighbour.ids[idx]; - self.base_ei.base_op.next.as_mut().unwrap()[0].process_new_tuple(); + self.base_ei.base_op.next[0].process_new_tuple(); } } } @@ -119,7 +121,7 @@ impl CommonOperatorTrait for Extend { base_op.prev.as_ref().unwrap().copy(is_thread_safe), )); let prev = extend.base_ei.base_op.prev.as_mut().unwrap().as_mut(); - get_op_attr_as_mut!(prev, next).replace(vec![Operator::EI(EI::Extend(extend_copy))]); + *get_op_attr_as_mut!(prev, next) = vec![Operator::EI(EI::Extend(extend_copy))]; let last_repeated_vertex_idx = get_op_attr!(prev, last_repeated_vertex_idx); extend.base_ei.init_caching(last_repeated_vertex_idx); Operator::EI(EI::Extend(extend)) diff --git a/src/graph_impl/multi_graph/plan/operator/extend/intersect.rs b/src/graph_impl/multi_graph/plan/operator/extend/intersect.rs index cab933cd..0b6afec7 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/intersect.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/intersect.rs @@ -7,6 +7,7 @@ use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Ope use graph_impl::multi_graph::plan::operator::scan::scan::Scan; use graph_impl::multi_graph::plan::operator::sink::sink::Sink; use graph_impl::multi_graph::planner::catalog::adj_list_descriptor::AdjListDescriptor; +use graph_impl::multi_graph::planner::catalog::operator::intersect_catalog::IntersectCatalog; use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; @@ -20,11 +21,17 @@ pub enum IntersectType { } #[derive(Clone)] -pub struct Intersect { +pub enum Intersect { + BaseIntersect(BaseIntersect), + IntersectCatalog(IntersectCatalog), +} + +#[derive(Clone)] +pub struct BaseIntersect { pub base_ei: BaseEI, } -impl Intersect { +impl BaseIntersect { pub fn new( to_qvertex: String, to_type: usize, @@ -32,8 +39,8 @@ impl Intersect { out_subgraph: Box, in_subgraph: Option>, out_qvertex_to_idx_map: HashMap, - ) -> Intersect { - let mut intersect = Intersect { + ) -> BaseIntersect { + let mut intersect = BaseIntersect { base_ei: BaseEI::new(to_qvertex.clone(), to_type, alds, out_subgraph, in_subgraph), }; let base_op = &mut intersect.base_ei.base_op; @@ -44,7 +51,7 @@ impl Intersect { } } -impl CommonOperatorTrait for Intersect { +impl CommonOperatorTrait for BaseIntersect { fn init( &mut self, probe_tuple: Vec, @@ -110,9 +117,7 @@ impl CommonOperatorTrait for Intersect { base_op.num_out_tuples += (out_neighbours.end_idx - out_neighbours.start_idx); for idx in out_neighbours.start_idx..out_neighbours.end_idx { base_op.probe_tuple[base_ei.out_idx] = out_neighbours.ids[idx]; - base_op.next.as_mut().map(|op_vec| { - op_vec.get_mut(0).map(|op| op.process_new_tuple()); - }); + base_op.next[0].process_new_tuple(); } } @@ -131,7 +136,7 @@ impl CommonOperatorTrait for Intersect { fn copy(&self, is_thread_safe: bool) -> Operator { let base_ei = &self.base_ei; let base_op = &base_ei.base_op; - let mut intersect = Intersect::new( + let mut intersect = BaseIntersect::new( base_ei.to_query_vertex.clone(), base_ei.to_type, base_ei.alds.clone(), @@ -144,14 +149,16 @@ impl CommonOperatorTrait for Intersect { base_op.prev.as_ref().unwrap().copy(is_thread_safe), )); let prev = intersect.base_ei.base_op.prev.as_mut().unwrap().as_mut(); - *get_op_attr_as_mut!(prev, next) = Some(vec![Operator::EI(EI::Intersect(intersect_copy))]); + *get_op_attr_as_mut!(prev, next) = vec![Operator::EI(EI::Intersect( + Intersect::BaseIntersect(intersect_copy), + ))]; let last_repeated_vertex_idx = get_op_attr!(prev, last_repeated_vertex_idx); intersect.base_ei.init_caching(last_repeated_vertex_idx); - Operator::EI(EI::Intersect(intersect)) + Operator::EI(EI::Intersect(Intersect::BaseIntersect(intersect))) } fn is_same_as(&mut self, op: &mut Operator) -> bool { - if let Operator::EI(EI::Intersect(intersect)) = op { + if let Operator::EI(EI::Intersect(Intersect::BaseIntersect(intersect))) = op { return self.base_ei.caching_type == intersect.base_ei.caching_type && self.get_alds_as_string() == intersect.get_alds_as_string() && self @@ -186,3 +193,65 @@ impl CommonOperatorTrait for Intersect { self.base_ei.get_num_out_tuples() } } + +impl CommonOperatorTrait for Intersect { + fn init( + &mut self, + probe_tuple: Vec, + graph: &TypedStaticGraph, + ) { + match self { + Intersect::BaseIntersect(base) => base.init(probe_tuple, graph), + Intersect::IntersectCatalog(ic) => ic.init(probe_tuple, graph), + } + } + + fn process_new_tuple(&mut self) { + match self { + Intersect::BaseIntersect(base) => base.process_new_tuple(), + Intersect::IntersectCatalog(ic) => ic.process_new_tuple(), + } + } + + fn execute(&mut self) { + match self { + Intersect::BaseIntersect(base) => base.execute(), + Intersect::IntersectCatalog(ic) => ic.execute(), + } + } + + fn get_alds_as_string(&self) -> String { + match self { + Intersect::BaseIntersect(base) => base.get_alds_as_string(), + Intersect::IntersectCatalog(ic) => ic.get_alds_as_string(), + } + } + + fn update_operator_name(&mut self, query_vertex_to_index_map: HashMap) { + match self { + Intersect::BaseIntersect(base) => base.update_operator_name(query_vertex_to_index_map), + Intersect::IntersectCatalog(ic) => ic.update_operator_name(query_vertex_to_index_map), + } + } + + fn copy(&self, is_thread_safe: bool) -> Operator { + match self { + Intersect::BaseIntersect(base) => base.copy(is_thread_safe), + Intersect::IntersectCatalog(ic) => ic.copy(is_thread_safe), + } + } + + fn is_same_as(&mut self, op: &mut Operator) -> bool { + match self { + Intersect::BaseIntersect(base) => base.is_same_as(op), + Intersect::IntersectCatalog(ic) => ic.is_same_as(op), + } + } + + fn get_num_out_tuples(&self) -> usize { + match self { + Intersect::BaseIntersect(base) => base.get_num_out_tuples(), + Intersect::IntersectCatalog(ic) => ic.get_num_out_tuples(), + } + } +} diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/build.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/build.rs index 1a9d4803..00a011fa 100644 --- a/src/graph_impl/multi_graph/plan/operator/hashjoin/build.rs +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/build.rs @@ -80,7 +80,7 @@ impl CommonOperatorTrait for Build { self.build_hash_idx, ); build.base_op.prev = self.base_op.prev.as_ref().map(|prev| prev.clone()); - build.base_op.next = Some(vec![Operator::Build(build.clone())]); + build.base_op.next = vec![Operator::Build(build.clone())]; build.probing_subgraph = self.probing_subgraph.clone(); Operator::Build(build) } diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/hash_join.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/hash_join.rs index 179253dd..ab445378 100644 --- a/src/graph_impl/multi_graph/plan/operator/hashjoin/hash_join.rs +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/hash_join.rs @@ -1,4 +1,6 @@ use generic::IdType; +use graph_impl::multi_graph::plan::operator::extend::intersect::BaseIntersect; +use graph_impl::multi_graph::plan::operator::extend::intersect::Intersect; use graph_impl::multi_graph::plan::operator::extend::EI::EI; use graph_impl::multi_graph::plan::operator::hashjoin::build::Build; use graph_impl::multi_graph::plan::operator::hashjoin::probe::{BaseProbe, Probe}; @@ -52,7 +54,7 @@ impl HashJoin { build_hash_idx.clone(), ); build.base_op.prev.replace(Box::new(pre_build.clone())); - get_op_attr_as_mut!(&mut pre_build, next).replace(vec![Operator::Build(build.clone())]); + *get_op_attr_as_mut!(&mut pre_build, next) = vec![Operator::Build(build.clone())]; build_subplans.push(Box::new(Operator::Build(build.clone()))); let out_subgraph_probe = get_op_attr_as_mut!(&mut pre_probe, out_subgraph).as_mut(); @@ -149,7 +151,7 @@ impl HashJoin { } get_op_attr_as_mut!(&mut probe, prev).replace(Box::new(pre_probe.clone())); let probe_clone = probe.clone(); - get_op_attr_as_mut!(&mut probe, next).replace(vec![probe_clone]); + *get_op_attr_as_mut!(&mut probe, next) = vec![probe_clone]; let last_index = probe_subplans.len() - 1; probe_subplans[last_index] = Box::new(probe.clone()); } diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe.rs index 295dbb8a..a375e834 100644 --- a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe.rs +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe.rs @@ -64,10 +64,10 @@ impl CommonOperatorTrait for BaseProbe { if self.base_op.probe_tuple.len() == 0 { self.base_op.probe_tuple = probe_tuple.clone(); self.block_info = BlockInfo::empty(); - self.base_op.next.as_mut().map(|next| { - next.iter_mut() - .for_each(|next_op| next_op.init(probe_tuple.clone(), graph)) - }); + self.base_op + .next + .iter_mut() + .for_each(|next_op| next_op.init(probe_tuple.clone(), graph)); } } @@ -98,10 +98,7 @@ impl CommonOperatorTrait for BaseProbe { offset += 1; } } - self.base_op - .next - .as_mut() - .map(|next| next.get_mut(0).map(|next_op| next_op.process_new_tuple())); + self.base_op.next[0].process_new_tuple(); } } } @@ -134,10 +131,7 @@ impl CommonOperatorTrait for BaseProbe { .prev .as_ref() .map(|prev| Box::new(prev.copy(is_thread_safe))); - probe - .base_op - .next - .replace(vec![Operator::Probe(Probe::BaseProbe(probe.clone()))]); + probe.base_op.next = vec![Operator::Probe(Probe::BaseProbe(probe.clone()))]; Operator::Probe(Probe::BaseProbe(probe)) } diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices.rs index 0039150f..8b9bc3d6 100644 --- a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices.rs +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices.rs @@ -115,9 +115,7 @@ impl CommonOperatorTrait for ProbeMultiVertices { out += 1; } } - self.base_probe.base_op.next.as_mut().map(|next| { - next.get_mut(0).map(|next_op| next_op.process_new_tuple()) - }); + self.base_probe.base_op.next[0].process_new_tuple(); } }); } @@ -161,13 +159,8 @@ impl CommonOperatorTrait for ProbeMultiVertices { .prev .as_ref() .map(|prev| Box::new(prev.copy(is_thread_safe))); - probe - .base_probe - .base_op - .next - .replace(vec![Operator::Probe(Probe::PMV(PMV::BasePMV( - probe.clone(), - )))]); + probe.base_probe.base_op.next = + vec![Operator::Probe(Probe::PMV(PMV::BasePMV(probe.clone())))]; Operator::Probe(Probe::PMV(PMV::BasePMV(probe))) } diff --git a/src/graph_impl/multi_graph/plan/operator/mod.rs b/src/graph_impl/multi_graph/plan/operator/mod.rs index 5c81c2b0..0ffde2a7 100644 --- a/src/graph_impl/multi_graph/plan/operator/mod.rs +++ b/src/graph_impl/multi_graph/plan/operator/mod.rs @@ -28,7 +28,10 @@ macro_rules! get_ei_as_mut { match $item { EI::Base(base) => base, EI::Extend(base) => &mut base.base_ei, - EI::Intersect(base) => &mut base.base_ei, + EI::Intersect(base) => match base { + Intersect::BaseIntersect(intersect) => &mut intersect.base_ei, + Intersect::IntersectCatalog(ic) => &mut ic.base_intersect.base_ei, + }, } }; } @@ -39,7 +42,10 @@ macro_rules! get_ei_as_ref { match $item { EI::Base(base) => base, EI::Extend(base) => &base.base_ei, - EI::Intersect(base) => &base.base_ei, + EI::Intersect(base) => match base { + Intersect::BaseIntersect(intersect) => &intersect.base_ei, + Intersect::IntersectCatalog(ic) => &ic.base_intersect.base_ei, + }, } }; } @@ -97,7 +103,10 @@ macro_rules! get_base_op_as_mut { Operator::EI(ei) => match ei { EI::Base(base) => &mut base.base_op, EI::Extend(base) => &mut base.base_ei.base_op, - EI::Intersect(base) => &mut base.base_ei.base_op, + EI::Intersect(base) => match base { + Intersect::BaseIntersect(intersect) => &mut intersect.base_ei.base_op, + Intersect::IntersectCatalog(ic) => &mut ic.base_intersect.base_ei.base_op, + }, }, Operator::Build(build) => &mut build.base_op, Operator::Probe(probe) => match probe { @@ -106,6 +115,7 @@ macro_rules! get_base_op_as_mut { Probe::PMV(PMV::BasePMV(base)) => &mut base.base_probe.base_op, Probe::PMV(PMV::PMVC(pmvc)) => &mut pmvc.base_pmv.base_probe.base_op, }, + Operator::Noop(noop) => &mut noop.base_op, } }; } @@ -129,7 +139,10 @@ macro_rules! get_base_op_as_ref { Operator::EI(ei) => match ei { EI::Base(base) => &base.base_op, EI::Extend(base) => &base.base_ei.base_op, - EI::Intersect(base) => &base.base_ei.base_op, + EI::Intersect(base) => match base { + Intersect::BaseIntersect(intersect) => &intersect.base_ei.base_op, + Intersect::IntersectCatalog(ic) => &ic.base_intersect.base_ei.base_op, + }, }, Operator::Build(build) => &build.base_op, Operator::Probe(probe) => match probe { @@ -138,6 +151,7 @@ macro_rules! get_base_op_as_ref { Probe::PMV(PMV::BasePMV(base)) => &base.base_probe.base_op, Probe::PMV(PMV::PMVC(pmvc)) => &pmvc.base_pmv.base_probe.base_op, }, + Operator::Noop(noop) => &noop.base_op, } }; } @@ -162,7 +176,10 @@ macro_rules! get_op_attr { Operator::EI(ei) => match ei { EI::Base(base) => base.base_op.$name, EI::Extend(base) => base.base_ei.base_op.$name, - EI::Intersect(base) => base.base_ei.base_op.$name, + EI::Intersect(base) => match base { + Intersect::BaseIntersect(intersect) => intersect.base_ei.base_op.$name, + Intersect::IntersectCatalog(ic) => ic.base_intersect.base_ei.base_op.$name, + }, }, Operator::Build(build) => build.base_op.$name, Operator::Probe(probe) => match probe { @@ -171,6 +188,7 @@ macro_rules! get_op_attr { Probe::PMV(PMV::BasePMV(base)) => base.base_probe.base_op.$name, Probe::PMV(PMV::PMVC(pmvc)) => pmvc.base_pmv.base_probe.base_op.$name, }, + Operator::Noop(noop) => noop.base_op.$name, } }; } @@ -195,7 +213,10 @@ macro_rules! get_op_attr_as_ref { Operator::EI(ei) => match ei { EI::Base(base) => &base.base_op.$name, EI::Extend(base) => &base.base_ei.base_op.$name, - EI::Intersect(base) => &base.base_ei.base_op.$name, + EI::Intersect(base) => match base { + Intersect::BaseIntersect(intersect) => &intersect.base_ei.base_op.$name, + Intersect::IntersectCatalog(ic) => &ic.base_intersect.base_ei.base_op.$name, + }, }, Operator::Build(build) => &build.base_op.$name, Operator::Probe(probe) => match probe { @@ -204,6 +225,7 @@ macro_rules! get_op_attr_as_ref { Probe::PMV(PMV::BasePMV(base)) => &base.base_probe.base_op.$name, Probe::PMV(PMV::PMVC(pmvc)) => &pmvc.base_pmv.base_probe.base_op.$name, }, + Operator::Noop(noop) => &noop.base_op.$name, } }; } @@ -228,7 +250,10 @@ macro_rules! get_op_attr_as_mut { Operator::EI(ei) => match ei { EI::Base(base) => &mut base.base_op.$name, EI::Extend(base) => &mut base.base_ei.base_op.$name, - EI::Intersect(base) => &mut base.base_ei.base_op.$name, + EI::Intersect(base) => match base { + Intersect::BaseIntersect(intersect) => &mut intersect.base_ei.base_op.$name, + Intersect::IntersectCatalog(ic) => &mut ic.base_intersect.base_ei.base_op.$name, + }, }, Operator::Build(build) => &mut build.base_op.$name, Operator::Probe(probe) => match probe { @@ -237,6 +262,7 @@ macro_rules! get_op_attr_as_mut { Probe::PMV(PMV::BasePMV(base)) => &mut base.base_probe.base_op.$name, Probe::PMV(PMV::PMVC(pmvc)) => &mut pmvc.base_pmv.base_probe.base_op.$name, }, + Operator::Noop(noop) => &mut noop.base_op.$name, } }; } diff --git a/src/graph_impl/multi_graph/plan/operator/operator.rs b/src/graph_impl/multi_graph/plan/operator/operator.rs index c1fd2756..202a05f3 100644 --- a/src/graph_impl/multi_graph/plan/operator/operator.rs +++ b/src/graph_impl/multi_graph/plan/operator/operator.rs @@ -1,4 +1,6 @@ use generic::{GraphType, IdType}; +use graph_impl::multi_graph::plan::operator::extend::intersect::BaseIntersect; +use graph_impl::multi_graph::plan::operator::extend::intersect::Intersect; use graph_impl::multi_graph::plan::operator::extend::EI::EI; use graph_impl::multi_graph::plan::operator::hashjoin::build::Build; use graph_impl::multi_graph::plan::operator::hashjoin::probe::Probe; @@ -6,6 +8,7 @@ use graph_impl::multi_graph::plan::operator::hashjoin::probe_multi_vertices::PMV use graph_impl::multi_graph::plan::operator::scan::scan::{BaseScan, Scan}; use graph_impl::multi_graph::plan::operator::scan::scan_sampling::ScanSampling; use graph_impl::multi_graph::plan::operator::sink::sink::Sink; +use graph_impl::multi_graph::planner::catalog::operator::noop::Noop; use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::TypedStaticGraph; use hashbrown::{HashMap, HashSet}; @@ -23,13 +26,14 @@ pub enum Operator { EI(EI), Build(Build), Probe(Probe), + Noop(Noop), } /// Basic operator #[derive(Clone)] pub struct BaseOperator { pub name: String, - pub next: Option>>, + pub next: Vec>, pub prev: Option>>, pub probe_tuple: Vec, pub out_tuple_len: usize, @@ -48,7 +52,7 @@ impl BaseOperator { ) -> BaseOperator { BaseOperator { name: "".to_string(), - next: None, + next: vec![], prev: None, probe_tuple: vec![], out_tuple_len: out_subgraph.get_num_qvertices(), @@ -64,7 +68,7 @@ impl BaseOperator { pub fn empty() -> BaseOperator { BaseOperator { name: "".to_string(), - next: None, + next: vec![], prev: None, probe_tuple: vec![], out_tuple_len: 0, @@ -76,14 +80,6 @@ impl BaseOperator { icost: 0, } } - - fn get_next(&self, index: usize) -> &Operator { - let next = self.next.as_ref().unwrap(); - &next[index] - } - fn set_next_vec(&mut self, op: Vec>) { - self.next.replace(op); - } } /// Common operations for every kind of operator @@ -104,8 +100,8 @@ pub trait CommonOperatorTrait { impl Operator { pub fn get_last_operators(&self, last_operators: &mut Vec>) { - if let Some(next) = get_op_attr_as_ref!(self, next) { - for op in next { + if !get_op_attr_as_ref!(self, next).is_empty() { + for op in get_op_attr_as_ref!(self, next) { op.get_last_operators(last_operators); } } else { @@ -121,12 +117,12 @@ impl Operator { let icost = get_op_attr!(self, icost); let num_out_tuples = get_op_attr!(self, num_out_tuples); operator_metrics.push((name.clone(), icost, num_out_tuples)); - if let Some(next) = get_op_attr_as_ref!(self, next) { - next.iter().for_each(|op| match op { + get_op_attr_as_ref!(self, next) + .iter() + .for_each(|op| match op { Operator::Sink(_) => {} _ => op.get_operator_metrics_next_operators(operator_metrics), }); - } } pub fn has_multi_edge_extends(&self) -> bool { @@ -201,6 +197,7 @@ impl CommonOperatorTrait for Operator { Operator::EI(ei) => ei.init(probe_tuple, graph), Operator::Build(build) => build.init(probe_tuple, graph), Operator::Probe(probe) => probe.init(probe_tuple, graph), + Operator::Noop(noop) => noop.init(probe_tuple, graph), } } @@ -212,6 +209,7 @@ impl CommonOperatorTrait for Operator { Operator::EI(ei) => ei.process_new_tuple(), Operator::Build(build) => build.process_new_tuple(), Operator::Probe(probe) => probe.process_new_tuple(), + Operator::Noop(noop) => noop.process_new_tuple(), } } @@ -223,6 +221,7 @@ impl CommonOperatorTrait for Operator { Operator::EI(ei) => ei.execute(), Operator::Build(build) => build.execute(), Operator::Probe(probe) => probe.execute(), + Operator::Noop(noop) => noop.execute(), } } @@ -234,6 +233,7 @@ impl CommonOperatorTrait for Operator { Operator::EI(ei) => ei.get_alds_as_string(), Operator::Build(build) => build.get_alds_as_string(), Operator::Probe(probe) => probe.get_alds_as_string(), + Operator::Noop(noop) => noop.get_alds_as_string(), } } @@ -245,6 +245,7 @@ impl CommonOperatorTrait for Operator { Operator::EI(ei) => ei.update_operator_name(query_vertex_to_index_map), Operator::Build(build) => build.update_operator_name(query_vertex_to_index_map), Operator::Probe(probe) => probe.update_operator_name(query_vertex_to_index_map), + Operator::Noop(noop) => noop.update_operator_name(query_vertex_to_index_map), } } @@ -256,6 +257,7 @@ impl CommonOperatorTrait for Operator { Operator::EI(ei) => ei.copy(is_thread_safe), Operator::Build(build) => build.copy(is_thread_safe), Operator::Probe(probe) => probe.copy(is_thread_safe), + Operator::Noop(noop) => noop.copy(is_thread_safe), } } fn is_same_as(&mut self, op: &mut Operator) -> bool { @@ -266,6 +268,7 @@ impl CommonOperatorTrait for Operator { Operator::EI(ei) => ei.is_same_as(op), Operator::Build(build) => build.is_same_as(op), Operator::Probe(probe) => probe.is_same_as(op), + Operator::Noop(noop) => noop.is_same_as(op), } } @@ -277,6 +280,7 @@ impl CommonOperatorTrait for Operator { Operator::EI(ei) => ei.get_num_out_tuples(), Operator::Build(build) => build.get_num_out_tuples(), Operator::Probe(probe) => probe.get_num_out_tuples(), + Operator::Noop(noop) => noop.get_num_out_tuples(), } } } diff --git a/src/graph_impl/multi_graph/plan/operator/scan/scan.rs b/src/graph_impl/multi_graph/plan/operator/scan/scan.rs index 693d727e..eb9deeb2 100644 --- a/src/graph_impl/multi_graph/plan/operator/scan/scan.rs +++ b/src/graph_impl/multi_graph/plan/operator/scan/scan.rs @@ -93,7 +93,7 @@ impl CommonOperatorTrait for BaseScan { self.label_or_to_type = self.to_type; self.to_type = 0; } - for next_op in self.base_op.next.as_mut().unwrap() { + for next_op in &mut self.base_op.next { next_op.init(probe_tuple.clone(), graph); } } @@ -119,10 +119,7 @@ impl CommonOperatorTrait for BaseScan { || self.vertex_types[self.base_op.probe_tuple[1].id()] == self.to_type { self.base_op.num_out_tuples += 1; - self.base_op - .next - .as_mut() - .map(|next| (&mut next[0]).process_new_tuple()); + self.base_op.next[0].process_new_tuple(); } } } @@ -136,11 +133,10 @@ impl CommonOperatorTrait for BaseScan { query_vertex_to_index_map = HashMap::new(); query_vertex_to_index_map.insert(self.from_query_vertex.clone(), 0); query_vertex_to_index_map.insert(self.to_query_vertex.clone(), 1); - self.base_op.next.as_mut().map(|next| { - for next_op in next { - next_op.update_operator_name(query_vertex_to_index_map.clone()); - } - }); + self.base_op + .next + .iter_mut() + .for_each(|op| op.update_operator_name(query_vertex_to_index_map.clone())); } fn copy(&self, is_thread_safe: bool) -> Operator { diff --git a/src/graph_impl/multi_graph/plan/operator/scan/scan_blocking.rs b/src/graph_impl/multi_graph/plan/operator/scan/scan_blocking.rs index 84de1076..ad07b64c 100644 --- a/src/graph_impl/multi_graph/plan/operator/scan/scan_blocking.rs +++ b/src/graph_impl/multi_graph/plan/operator/scan/scan_blocking.rs @@ -91,10 +91,7 @@ impl ScanBlocking { .unwrap() .get_neighbor_id(Id::new(to_idx)); base_op.num_out_tuples += 1; - base_op - .next - .as_mut() - .map(|next| next.get_mut(0).map(|next_op| next_op.process_new_tuple())); + base_op.next[0].process_new_tuple(); } } @@ -118,11 +115,7 @@ impl ScanBlocking { == self.base_scan.to_type { self.base_scan.base_op.num_out_tuples += 1; - self.base_scan - .base_op - .next - .as_mut() - .map(|next| next.get_mut(0).map(|next_op| next_op.process_new_tuple())); + self.base_scan.base_op.next[0].process_new_tuple(); } } } @@ -154,10 +147,11 @@ impl CommonOperatorTrait for ScanBlocking { .map_or(0, |adj| adj.get_offsets()[label + 1]); self.from_idx_limit = self.curr_from_idx; self.to_idx_limit = self.curr_to_idx; - self.base_scan.base_op.next.as_mut().map(|next| { - next.iter_mut() - .for_each(|next_op| next_op.init(probe_tuple.clone(), graph)) - }); + self.base_scan + .base_op + .next + .iter_mut() + .for_each(|next_op| next_op.init(probe_tuple.clone(), graph)); } fn process_new_tuple(&mut self) { diff --git a/src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs b/src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs index 423912f7..36b96780 100644 --- a/src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs +++ b/src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs @@ -17,7 +17,7 @@ pub struct ScanSampling { } impl ScanSampling { - pub fn new(out_subgraph: Box) -> Self { + pub fn new(out_subgraph: Box) -> ScanSampling { Self { base_scan: BaseScan::new(out_subgraph), edges_queue: vec![], @@ -34,6 +34,18 @@ impl ScanSampling { } } + pub fn set_edge_indices_to_sample_list( + &mut self, + edges: Vec>, + num_edges_to_sample: usize, + ) { + let mut rng = thread_rng(); + while self.edges_queue.len() < num_edges_to_sample { + let edge_idx = rng.gen_range(0, edges.len()); + self.edges_queue.push(edges[edge_idx].clone()); + } + } + pub fn set_edge_indices_to_sample_by_edges( &mut self, edges: Vec>, @@ -46,6 +58,12 @@ impl ScanSampling { self.edges_queue.push(edges[edge_idx].clone()); } } + + pub fn copy_default(&self) -> Operator { + let mut scan_sampling = ScanSampling::new(self.base_scan.base_op.out_subgraph.clone()); + scan_sampling.edges_queue = self.edges_queue.clone(); + Operator::Scan(Scan::ScanSampling(scan_sampling)) + } } impl CommonOperatorTrait for ScanSampling { @@ -56,10 +74,8 @@ impl CommonOperatorTrait for ScanSampling { ) { if self.base_scan.base_op.probe_tuple.is_empty() { self.base_scan.base_op.probe_tuple = probe_tuple.clone(); - self.base_scan.base_op.next.as_mut().map(|next| { - next.iter_mut().for_each(|next_op| { - next_op.init(probe_tuple.clone(), graph); - }) + self.base_scan.base_op.next.iter_mut().for_each(|next_op| { + next_op.init(probe_tuple.clone(), graph); }); } } @@ -74,7 +90,7 @@ impl CommonOperatorTrait for ScanSampling { self.base_scan.base_op.probe_tuple[0] = edge[0]; self.base_scan.base_op.probe_tuple[1] = edge[0]; self.base_scan.base_op.num_out_tuples += 1; - for next_op in self.base_scan.base_op.next.as_mut().unwrap() { + for next_op in &mut self.base_scan.base_op.next { next_op.process_new_tuple(); } } diff --git a/src/graph_impl/multi_graph/plan/operator/sink/sink.rs b/src/graph_impl/multi_graph/plan/operator/sink/sink.rs index 85f12574..b1b962e9 100644 --- a/src/graph_impl/multi_graph/plan/operator/sink/sink.rs +++ b/src/graph_impl/multi_graph/plan/operator/sink/sink.rs @@ -1,4 +1,6 @@ use generic::{GraphType, IdType}; +use graph_impl::multi_graph::plan::operator::extend::intersect::BaseIntersect; +use graph_impl::multi_graph::plan::operator::extend::intersect::Intersect; use graph_impl::multi_graph::plan::operator::extend::EI::EI; use graph_impl::multi_graph::plan::operator::hashjoin::probe::Probe; use graph_impl::multi_graph::plan::operator::hashjoin::probe_multi_vertices::PMV; diff --git a/src/graph_impl/multi_graph/plan/operator/sink/sink_limit.rs b/src/graph_impl/multi_graph/plan/operator/sink/sink_limit.rs index 753a9611..92bab78c 100644 --- a/src/graph_impl/multi_graph/plan/operator/sink/sink_limit.rs +++ b/src/graph_impl/multi_graph/plan/operator/sink/sink_limit.rs @@ -1,4 +1,6 @@ use generic::{GraphType, IdType}; +use graph_impl::multi_graph::plan::operator::extend::intersect::BaseIntersect; +use graph_impl::multi_graph::plan::operator::extend::intersect::Intersect; use graph_impl::multi_graph::plan::operator::extend::EI::EI; use graph_impl::multi_graph::plan::operator::hashjoin::probe::Probe; use graph_impl::multi_graph::plan::operator::hashjoin::probe_multi_vertices::PMV; diff --git a/src/graph_impl/multi_graph/plan/query_plan.rs b/src/graph_impl/multi_graph/plan/query_plan.rs index f42e6f6a..9df3dc77 100644 --- a/src/graph_impl/multi_graph/plan/query_plan.rs +++ b/src/graph_impl/multi_graph/plan/query_plan.rs @@ -1,4 +1,6 @@ use generic::{GraphType, IdType}; +use graph_impl::multi_graph::plan::operator::extend::intersect::BaseIntersect; +use graph_impl::multi_graph::plan::operator::extend::intersect::Intersect; use graph_impl::multi_graph::plan::operator::extend::EI::EI; use graph_impl::multi_graph::plan::operator::hashjoin::hash_table::HashTable; use graph_impl::multi_graph::plan::operator::hashjoin::probe::Probe; @@ -12,13 +14,13 @@ use graph_impl::multi_graph::plan::operator::sink::sink::{BaseSink, Sink, SinkTy use graph_impl::multi_graph::plan::operator::sink::sink_copy::SinkCopy; use graph_impl::multi_graph::plan::operator::sink::sink_limit::SinkLimit; use graph_impl::multi_graph::plan::operator::sink::sink_print::SinkPrint; +use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; use std::hash::Hash; use std::ops::Deref; use std::rc::Rc; use std::time::SystemTime; -use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; #[derive(Clone)] pub struct QueryPlan { @@ -49,8 +51,7 @@ impl QueryPlan { let out_subgraph = Box::new(get_op_attr_as_ref!(op, out_subgraph).as_ref().clone()); let mut sink = BaseSink::new(out_subgraph); for op in last_operators.iter_mut() { - let next = get_op_attr_as_mut!(op, next); - next.replace(vec![Operator::Sink(Sink::BaseSink(sink.clone()))]); + *get_op_attr_as_mut!(op, next) = vec![Operator::Sink(Sink::BaseSink(sink.clone()))]; } sink.previous = Some(last_operators); Self { @@ -144,7 +145,7 @@ impl QueryPlan { pub fn append(&mut self, mut new_operator: Operator) { let mut last_operator = self.last_operator.as_mut().unwrap().as_mut(); let last_op = get_base_op_as_mut!(&mut last_operator); - last_op.next = Some(vec![new_operator.clone()]); + last_op.next = vec![new_operator.clone()]; let new_op = get_base_op_as_mut!(&mut new_operator); new_op.prev = self.last_operator.clone(); self.subplans.push(Box::new(new_operator.clone())); @@ -222,25 +223,9 @@ impl QueryPlan { ))), SinkType::Counter => Operator::Sink(Sink::BaseSink(BaseSink::new(query_graph.clone()))), }; - let sink_prev = get_op_attr_as_mut!(&mut sink, prev) - .as_mut() - .unwrap() - .as_mut(); - *sink_prev = last_operator.clone(); - *get_op_attr_as_mut!(self.subplans[plan_size - 1].as_mut(), next) = Some(vec![sink]); -// let mut probes = vec![]; -// for operator in &mut self.subplans { -// let mut op = operator.as_mut(); -// while get_op_attr_as_mut!(op, prev).is_some() { -// op = get_op_attr_as_mut!(op, prev).as_mut().unwrap(); -// if let Operator::Probe(p) = op { -// probes.push(op); -// } -// } -// if let Operator::Probe(p) = operator.as_mut() { -// probes.push(operator); -// } -// } + let sink_prev = get_op_attr_as_mut!(&mut sink, prev); + sink_prev.replace(Box::new(last_operator.clone())); + *get_op_attr_as_mut!(self.subplans[plan_size - 1].as_mut(), next) = vec![sink]; for subplan in &mut self.subplans { if let Operator::Build(build) = subplan.as_mut() { let hash_table = HashTable::new(build.build_hash_idx, build.hashed_tuple_len); @@ -255,7 +240,7 @@ impl QueryPlan { .as_ref() .unwrap() .as_ref(); - self.init_hashtable(build_insubgrpah,hash_table); + self.init_hashtable(build_insubgrpah, hash_table); } } for subplan in &mut self.subplans { @@ -271,10 +256,10 @@ impl QueryPlan { } } - fn init_hashtable(&mut self,build_insubgrpah:&QueryGraph,hash_table:&HashTable){ + fn init_hashtable(&mut self, build_insubgrpah: &QueryGraph, hash_table: &HashTable) { for operator in &mut self.subplans { if let Operator::Probe(p) = operator.as_mut() { - if Self::check_and_init(build_insubgrpah,operator,hash_table.clone()){ + if Self::check_and_init(build_insubgrpah, operator, hash_table.clone()) { break; } } @@ -282,7 +267,7 @@ impl QueryPlan { while get_op_attr_as_mut!(op, prev).is_some() { op = get_op_attr_as_mut!(op, prev).as_mut().unwrap(); if let Operator::Probe(p) = op { - if Self::check_and_init(build_insubgrpah,op,hash_table.clone()){ + if Self::check_and_init(build_insubgrpah, op, hash_table.clone()) { return; } } @@ -290,7 +275,11 @@ impl QueryPlan { } } - fn check_and_init(build_insubgrpah:&QueryGraph,probe:&mut Operator,hash_table:HashTable)->bool{ + fn check_and_init( + build_insubgrpah: &QueryGraph, + probe: &mut Operator, + hash_table: HashTable, + ) -> bool { let prob_insubgraph = get_op_attr_as_ref!(probe, in_subgraph) .as_ref() .unwrap() diff --git a/src/graph_impl/multi_graph/plan/query_plan_worker.rs b/src/graph_impl/multi_graph/plan/query_plan_worker.rs index c18115d4..eca7125a 100644 --- a/src/graph_impl/multi_graph/plan/query_plan_worker.rs +++ b/src/graph_impl/multi_graph/plan/query_plan_worker.rs @@ -1,4 +1,6 @@ use generic::{GraphType, IdType}; +use graph_impl::multi_graph::plan::operator::extend::intersect::BaseIntersect; +use graph_impl::multi_graph::plan::operator::extend::intersect::Intersect; use graph_impl::multi_graph::plan::operator::extend::EI::EI; use graph_impl::multi_graph::plan::operator::hashjoin::probe::Probe; use graph_impl::multi_graph::plan::operator::hashjoin::probe_multi_vertices::PMV; @@ -73,7 +75,6 @@ impl QPWorkers { self.query_plans[0].execute(); self.elapsed_time = self.query_plans[0].elapsed_time; } else { - // TODO: update LIMIT for the workers class. let begin_time = SystemTime::now(); let mut workers = vec![]; for plan in &self.query_plans { diff --git a/src/graph_impl/multi_graph/planner/catalog/catalog.rs b/src/graph_impl/multi_graph/planner/catalog/catalog.rs index 085a8ace..4f76ee21 100644 --- a/src/graph_impl/multi_graph/planner/catalog/catalog.rs +++ b/src/graph_impl/multi_graph/planner/catalog/catalog.rs @@ -1,4 +1,5 @@ use generic::{GraphTrait, GraphType, IdType}; +use graph_impl::multi_graph::plan::operator::extend::intersect::Intersect; use graph_impl::multi_graph::plan::operator::extend::EI::EI; use graph_impl::multi_graph::plan::operator::hashjoin::probe::Probe; use graph_impl::multi_graph::plan::operator::hashjoin::probe_multi_vertices::PMV; @@ -292,58 +293,227 @@ impl Catalog { graph: &TypedStaticGraph, query_plan_arr: &mut Vec>, ) { - let mut other: Vec<&mut Operator> = query_plan_arr + let mut other: Vec<&Operator> = query_plan_arr .iter_mut() .map(|query_plan| { - let mut base_sink = get_sink_as_mut!(query_plan.get_sink()); - let mut op = &mut base_sink.previous.as_mut().unwrap()[0]; + let base_sink = get_sink_as_ref!(query_plan.get_sink()); + let mut op = &base_sink.previous.as_ref().unwrap()[0]; while if let Operator::Scan(Scan::ScanSampling(sp)) = op.deref() { false } else { true } { - let prev_op = get_op_attr_as_mut!(op, prev).as_mut().unwrap(); - op = prev_op.as_mut(); + let prev_op = get_op_attr_as_ref!(op, prev).as_ref().unwrap(); + op = prev_op.as_ref(); } - &mut get_op_attr_as_mut!(op, next).as_mut().unwrap()[0] + &get_op_attr_as_ref!(op, next)[0] }) .collect(); let op = other.remove(0); if self.is_sorted_by_node { - self.add_icost_and_selectivity_sorted_by_node(op, other, graph.is_directed()); + self.add_icost_and_selectivity_sorted_by_node(op, other, !graph.is_directed()); } else { - self.add_icost_and_selectivity(op, other, graph.is_directed()); + self.add_icost_and_selectivity(op, other, !graph.is_directed()); } } fn add_icost_and_selectivity_sorted_by_node( - &self, - operator: &mut Operator, - other: Vec<&mut Operator>, - is_directed: bool, + &mut self, + operator: &Operator, + other: Vec<&Operator>, + is_undirected: bool, ) { + if let Operator::Sink(sink) = &get_op_attr_as_ref!(operator, next)[0] { + return; + } + let mut num_input_tuples = get_op_attr!(operator, num_out_tuples); + for other_op in &other { + num_input_tuples += get_op_attr!(other_op, num_out_tuples); + } + let in_subgraph = get_op_attr_as_ref!(operator, out_subgraph).as_ref(); + let subgraph_idx = self.get_subgraph_idx(&mut in_subgraph.clone()); + let next = get_op_attr_as_ref!(operator, next); + for i in 0..next.len() { + if let Operator::EI(EI::Intersect(Intersect::IntersectCatalog(intersect))) = &next[i] { + let to_type = intersect.base_intersect.base_ei.to_type; + let mut alds_as_str_list = vec![]; + let alds_str = self.get_alds_as_str( + &intersect.base_intersect.base_ei.alds, + None, + Some(to_type), + ); + if is_undirected { + let splits: Vec<&str> = alds_str.split(", ").collect(); + let direction_patterns = CatalogPlans::::generate_direction_patterns( + splits.len(), + is_undirected, + ); + for pattern in direction_patterns { + let mut alds_str_with_pattern = "".to_owned(); + for j in 0..pattern.len() { + let ok: Vec<&str> = splits[j].split("Bwd").collect(); + if j == pattern.len() - 1 { + alds_str_with_pattern = + alds_str_with_pattern + ok[0] + &pattern[j].to_string() + ok[1]; + } else { + alds_str_with_pattern = alds_str_with_pattern + + ok[0] + + &pattern[j].to_string() + + ok[1] + + ", "; + } + } + alds_as_str_list.push(alds_str_with_pattern); + } + } else { + alds_as_str_list.push(alds_str); + } + let mut selectivity = intersect.base_intersect.base_ei.base_op.num_out_tuples; + for other_op in &other { + let next = &get_op_attr_as_ref!(other_op, next)[i]; + selectivity += get_op_attr!(next, num_out_tuples); + } + self.sampled_selectivity + .entry(subgraph_idx) + .or_insert(HashMap::new()); + for alds_as_str in alds_as_str_list { + if num_input_tuples > 0 { + self.sampled_selectivity + .get_mut(&subgraph_idx) + .unwrap() + .insert( + alds_as_str, + (selectivity as f64) / (num_input_tuples as f64), + ); + } else { + self.sampled_selectivity + .get_mut(&subgraph_idx) + .unwrap() + .insert(alds_as_str, 0.0); + } + } + let noop = &get_op_attr_as_ref!(&next[i], next)[0]; + let mut other_noops = vec![]; + for (j, other) in other.iter().enumerate() { + let next = &get_op_attr_as_ref!(other, next)[i]; + let next_op = &get_op_attr_as_ref!(next, next)[j]; + other_noops.push(next_op); + } + self.add_icost_and_selectivity(noop, other_noops, is_undirected); + } + } } fn add_icost_and_selectivity( &mut self, - operator: &mut Operator, - other: Vec<&mut Operator>, - is_directed: bool, + operator: &Operator, + other: Vec<&Operator>, + is_undirected: bool, ) { - if let Operator::Sink(sink) = &get_op_attr_as_ref!(operator, next).as_ref().unwrap()[0] { + if let Operator::Sink(sink) = &get_op_attr_as_ref!(operator, next)[0] { return; } let mut num_input_tuples = get_op_attr!(operator, num_out_tuples); - for other_op in other { + for other_op in &other { num_input_tuples += get_op_attr!(other_op, num_out_tuples); } - let in_subgraph = get_op_attr_as_mut!(operator, out_subgraph); - let subgraph_idx = self.get_subgraph_idx(in_subgraph.as_mut()); - let next_list = get_op_attr_as_ref!(operator, next).as_ref().unwrap(); - for (i, next) in next_list.iter().enumerate() { - let intersect = next; - //TODO: Implement following code. + let in_subgraph = get_op_attr_as_ref!(operator, out_subgraph).as_ref(); + let subgraph_idx = self.get_subgraph_idx(&mut in_subgraph.clone()); + for (i, next) in get_op_attr_as_ref!(operator, next).iter().enumerate() { + if let Operator::EI(EI::Intersect(Intersect::IntersectCatalog(intersect))) = next { + let alds = &intersect.base_intersect.base_ei.alds; + let mut alds_as_str_list = vec![]; + let alds_str = + self.get_alds_as_str(&intersect.base_intersect.base_ei.alds, None, None); + if is_undirected { + let splits: Vec<&str> = alds_str.split(", ").collect(); + let direction_patterns = CatalogPlans::::generate_direction_patterns( + splits.len(), + is_undirected, + ); + for pattern in direction_patterns { + let mut alds_str_with_pattern = "".to_owned(); + for j in 0..pattern.len() { + let ok: Vec<&str> = splits[j].split("Bwd").collect(); + if j == pattern.len() - 1 { + alds_str_with_pattern = + alds_str_with_pattern + ok[0] + &pattern[j].to_string() + ok[1]; + } else { + alds_str_with_pattern = alds_str_with_pattern + + ok[0] + + &pattern[j].to_string() + + ok[1] + + ", "; + } + } + alds_as_str_list.push(alds_str_with_pattern); + } + } else { + alds_as_str_list.push(alds_str); + } + if 1 == alds.len() { + let mut icost = get_op_attr!(&next, icost); + for other_op in &other { + let next = &get_op_attr_as_ref!(other_op, next)[i]; + icost += get_op_attr!(next, icost); + } + self.sampled_icost + .entry(subgraph_idx) + .or_insert(HashMap::new()); + for alds_as_str in &alds_as_str_list { + if num_input_tuples > 0 { + self.sampled_icost + .get_mut(&subgraph_idx) + .unwrap() + .entry(alds_as_str.clone()) + .or_insert((icost as f64) / (num_input_tuples as f64)); + } else { + self.sampled_icost + .get_mut(&subgraph_idx) + .unwrap() + .entry(alds_as_str.clone()) + .or_insert(0.0); + } + } + } + let noops = get_op_attr_as_ref!(next, next); + for to_type in 0..noops.len() { + let noop = &noops[to_type]; + let mut selectivity = get_op_attr!(noop, num_out_tuples); + for other_op in &other { + let next = &get_op_attr_as_ref!(other_op, next)[i]; + let o_next = &get_op_attr_as_ref!(next, next)[to_type]; + selectivity += get_op_attr!(o_next, num_out_tuples); + } + self.sampled_selectivity + .entry(subgraph_idx) + .or_insert(HashMap::new()); + for alds_as_str in &alds_as_str_list { + if num_input_tuples > 0 { + self.sampled_selectivity + .get_mut(&subgraph_idx) + .unwrap() + .insert( + alds_as_str.to_owned() + "~" + &to_type.to_string(), + (selectivity as f64) / (num_input_tuples as f64), + ); + } else { + self.sampled_selectivity + .get_mut(&subgraph_idx) + .unwrap() + .insert(alds_as_str.to_owned() + "~" + &to_type.to_string(), 0.0); + } + } + let mut other_noops = vec![]; + for other_op in &other { + let next = &get_op_attr_as_ref!(other_op, next)[i]; + let next_op = &get_op_attr_as_ref!(next, next)[to_type]; + other_noops.push(next_op); + } + self.add_icost_and_selectivity(noop, other_noops, is_undirected); + } + } } } diff --git a/src/graph_impl/multi_graph/planner/catalog/catalog_plans.rs b/src/graph_impl/multi_graph/planner/catalog/catalog_plans.rs index aa9d8931..dda8eaf9 100644 --- a/src/graph_impl/multi_graph/planner/catalog/catalog_plans.rs +++ b/src/graph_impl/multi_graph/planner/catalog/catalog_plans.rs @@ -1,8 +1,22 @@ use generic::{GraphLabelTrait, GraphTrait, GraphType, IdType}; +use graph_impl::multi_graph::plan::operator::extend::intersect::Intersect; +use graph_impl::multi_graph::plan::operator::extend::EI::EI; +use graph_impl::multi_graph::plan::operator::hashjoin::probe::Probe; +use graph_impl::multi_graph::plan::operator::hashjoin::probe_multi_vertices::PMV; +use graph_impl::multi_graph::plan::operator::operator::Operator; +use graph_impl::multi_graph::plan::operator::scan::scan::{BaseScan, Scan}; +use graph_impl::multi_graph::plan::operator::scan::scan_sampling::ScanSampling; +use graph_impl::multi_graph::plan::operator::sink::sink::Sink; use graph_impl::multi_graph::plan::query_plan::QueryPlan; -use graph_impl::multi_graph::planner::catalog::adj_list_descriptor::AdjListDescriptor; +use graph_impl::multi_graph::planner::catalog::adj_list_descriptor::{ + AdjListDescriptor, Direction, +}; +use graph_impl::multi_graph::planner::catalog::operator::intersect_catalog::IntersectCatalog; +use graph_impl::multi_graph::planner::catalog::operator::noop::Noop; +use graph_impl::multi_graph::planner::catalog::query_edge::QueryEdge; use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::query::query_graph_set::QueryGraphSet; +use graph_impl::multi_graph::utils::set_utils; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; use std::hash::Hash; @@ -14,8 +28,8 @@ static QUERY_VERTICES: [&str; 7] = ["a", "b", "c", "d", "e", "f", "g"]; pub struct CatalogPlans { num_sampled_edges: usize, max_input_num_vertices: usize, - num_types: usize, - num_labels: usize, + num_node_labels: usize, + num_edge_labels: usize, sorted_by_node: bool, query_graphs_to_extend: QueryGraphSet, query_plans_arrs: Vec>>, @@ -35,8 +49,8 @@ impl CatalogPlans { let mut plans = CatalogPlans { num_sampled_edges, max_input_num_vertices, - num_types: graph.num_of_edge_labels(), - num_labels: graph.num_of_node_labels(), + num_node_labels: graph.num_of_node_labels(), + num_edge_labels: graph.num_of_edge_labels(), sorted_by_node: graph.is_sorted_by_node(), query_graphs_to_extend: QueryGraphSet::new(), query_plans_arrs: vec![], @@ -56,15 +70,418 @@ impl CatalogPlans { for (i, v) in plans.query_vertices.iter().enumerate() { plans.query_vertex_to_idx_map.insert(v.clone(), i); } - // let scans = Vec::new(); - // TODO: Implement scan operator - + let scans = if graph.edge_count() > 1073741823 { + plans.generate_all_scans_for_large_graph(graph) + } else { + plans.generate_all_scans(graph) + }; + for mut scan in scans { + let mut noop = Noop::new(scan.base_scan.base_op.out_subgraph.as_ref().clone()); + scan.base_scan.base_op.next = vec![Operator::Noop(noop.clone())]; + noop.base_op.prev = Some(Box::new(Operator::Scan(Scan::ScanSampling(scan.clone())))); + noop.base_op.out_qvertex_to_idx_map = + scan.base_scan.base_op.out_qvertex_to_idx_map.clone(); + let mut noop_op = Operator::Noop(noop); + plans.set_next_operators(graph, &mut noop_op, false); + let mut query_plans_arr = vec![]; + query_plans_arr.push(QueryPlan::new(scan.clone())); + for i in 1..num_thread { + let mut scan_copy = scan.copy_default(); + let mut base_scan_copy = get_base_op_as_mut!(&mut scan_copy); + let mut another_noop = Noop::new(base_scan_copy.out_subgraph.as_ref().clone()); + base_scan_copy.next = vec![Operator::Noop(another_noop.clone())]; + another_noop.base_op.out_qvertex_to_idx_map = + base_scan_copy.out_qvertex_to_idx_map.clone(); + another_noop.base_op.prev = Some(Box::new(scan_copy.clone())); + let mut another_noop_op = Operator::Noop(another_noop); + plans.set_next_operators(graph, &mut another_noop_op, true); + if let Operator::Scan(Scan::ScanSampling(sc)) = scan_copy { + query_plans_arr.push(QueryPlan::new(sc)); + } + } + plans.query_plans_arrs.push(query_plans_arr); + } plans } pub fn set_next_operators( - graph: TypedStaticGraph, + &mut self, + graph: &TypedStaticGraph, + operator: &mut Operator, + is_none: bool, + ) { + let operator_copy = operator.clone(); + let operator_base = get_base_op_as_ref!(&operator_copy); + let in_subgraph = operator_base.out_subgraph.as_ref(); + if !is_none + && !self + .query_graphs_to_extend + .contains(&mut in_subgraph.clone()) + { + self.query_graphs_to_extend.add(in_subgraph.clone()); + } else if !is_none { + return; + } + + let query_vertices = in_subgraph.get_query_vertices().clone(); + let mut descriptors = vec![]; + for query_vertex_to_extend in set_utils::get_power_set_excluding_empty_set(query_vertices) { + for alds in self.generate_alds(query_vertex_to_extend, self.is_directed) { + descriptors.push(Descriptor { + out_subgraph: self.get_out_subgraph(in_subgraph.copy(), alds.clone()), + alds, + }); + } + } + let to_qvertex = QUERY_VERTICES[in_subgraph.get_num_qvertices()]; + let mut next = vec![]; + let last_repeated_vertex_idx = get_op_attr_as_ref!(operator,last_repeated_vertex_idx).clone(); + if self.sorted_by_node { + for mut descriptor in descriptors { + let mut types = vec![]; + for to_type in 0..self.num_node_labels { + let mut produces_output = true; + for ald in &descriptor.alds { + let from_type = in_subgraph.get_query_vertex_type(&ald.from_query_vertex); + if (ald.direction == Direction::Fwd + && 0 == graph.get_num_edges(from_type, to_type, ald.label)) + || (ald.direction == Direction::Bwd + && 0 == graph.get_num_edges(to_type, from_type, ald.label)) + { + produces_output = false; + break; + } + } + if produces_output { + types.push(to_type); + } else { + self.selectivity_zero.push(( + in_subgraph.clone(), + descriptor.alds.clone(), + to_type, + )); + } + } + let mut out_qvertex_to_idx_map = get_op_attr_as_ref!(operator,out_qvertex_to_idx_map).clone(); + out_qvertex_to_idx_map.insert(to_qvertex.to_owned(), out_qvertex_to_idx_map.len()); + for to_type in types { + descriptor + .out_subgraph + .set_query_vertex_type(to_qvertex.to_owned(), to_type); + let p = to_qvertex.clone(); + let mut intersect = IntersectCatalog::new( + to_qvertex.to_owned(), + to_type, + descriptor.alds.clone(), + descriptor.out_subgraph.clone(), + in_subgraph.clone(), + out_qvertex_to_idx_map.clone(), + self.sorted_by_node, + ); + intersect + .base_intersect + .base_ei + .init_caching(last_repeated_vertex_idx); + next.push(Operator::EI(EI::Intersect(Intersect::IntersectCatalog( + intersect, + )))); + } + } + } else { + for i in 0..descriptors.len() { + let descriptor = &descriptors[i]; + let prev = get_op_attr_as_ref!(operator,prev).as_ref().unwrap().as_ref(); + let mut out_qvertex_to_idx_map = + get_op_attr_as_ref!(prev, out_qvertex_to_idx_map).clone(); + out_qvertex_to_idx_map.insert(to_qvertex.to_owned(), out_qvertex_to_idx_map.len()); + let mut ic = IntersectCatalog::new( + to_qvertex.to_owned(), + 0, + descriptor.alds.clone(), + descriptor.out_subgraph.clone(), + in_subgraph.clone(), + out_qvertex_to_idx_map, + self.sorted_by_node, + ); + ic.base_intersect + .base_ei + .init_caching(last_repeated_vertex_idx); + next.push(Operator::EI(EI::Intersect(Intersect::IntersectCatalog(ic)))); + } + } + Self::set_next_pointer(operator, &mut next); + for mut next_op in next { + let mut next_noops = if self.sorted_by_node { + vec![Noop::new(QueryGraph::empty()); 1] + } else { + vec![Noop::new(QueryGraph::empty()); self.num_node_labels] + }; + self.set_noops( + get_base_op_as_ref!(&next_op).out_subgraph.as_ref(), + to_qvertex.to_owned(), + &mut next_noops, + &get_base_op_as_ref!(&next_op).out_qvertex_to_idx_map, + ); + let mut next_noops = next_noops + .into_iter() + .map(|noop| Operator::Noop(noop)) + .collect(); + Self::set_next_pointer(&mut next_op, &mut next_noops); + if get_base_op_as_ref!(&next_op) + .out_subgraph + .as_ref() + .get_num_qvertices() + <= self.max_input_num_vertices + { + for mut next_noop in next_noops { + *get_op_attr_as_mut!(&mut next_noop, last_repeated_vertex_idx) = + last_repeated_vertex_idx; + self.set_next_operators(graph, &mut next_noop, false) + } + } + } + } + + fn set_next_pointer(operator: &mut Operator, next: &mut Vec>) { + *get_op_attr_as_mut!(operator, next) = next.clone(); + for next_op in next { + *get_op_attr_as_mut!(next_op, prev) = Some(Box::new(operator.clone())); + } + } + + fn set_noops( + &self, + query_graph: &QueryGraph, + to_qvertex: String, + noops: &mut Vec>, + out_qvertex_to_idx_map: &HashMap, + ) { + if self.sorted_by_node { + noops[0] = Noop::new(query_graph.clone()); + noops[0].base_op.out_qvertex_to_idx_map = out_qvertex_to_idx_map.clone(); + } else { + for to_type in 0..self.num_node_labels { + let mut query_graph_copy = query_graph.copy(); + query_graph_copy.set_query_vertex_type(to_qvertex.clone(), to_type); + noops[to_type] = Noop::new(query_graph_copy); + } + } + } + + fn generate_alds( + &self, + qvertices: Vec, + is_direccted: bool, + ) -> Vec> { + let direction_patterns = Self::generate_direction_patterns(qvertices.len(), is_direccted); + let label_patterns = self.generate_labels_patterns(qvertices.len()); + let mut alds_list = vec![]; + for directions in direction_patterns { + for labels in &label_patterns { + let mut alds = vec![]; + for i in 0..directions.len() { + let vertex_idx = self.query_vertex_to_idx_map[&qvertices[i]]; + let to_qvertex = QUERY_VERTICES[vertex_idx]; + alds.push(AdjListDescriptor::new( + to_qvertex.to_owned(), + vertex_idx, + directions[i].clone(), + labels[i].clone(), + )); + } + alds_list.push(alds); + } + } + alds_list + } + + fn generate_labels_patterns(&self, size: usize) -> Vec> { + let mut labels = vec![]; + for label in 0..self.num_edge_labels { + labels.push(label); + } + set_utils::generate_permutations(labels, size) + } + + pub fn generate_direction_patterns(size: usize, is_directed: bool) -> Vec> { + let mut direction_patterns = vec![]; + Self::generate_direction_patterns_inner( + &mut vec![Direction::Fwd; size], + size, + &mut direction_patterns, + is_directed, + ); + direction_patterns + } + + fn generate_direction_patterns_inner( + direction_arr: &mut Vec, + size: usize, + direction_patterns: &mut Vec>, + is_directed: bool, ) { + if size <= 0 { + direction_patterns.push(direction_arr.clone()); + } else { + direction_arr[size - 1] = Direction::Bwd; + Self::generate_direction_patterns_inner( + direction_arr, + size - 1, + direction_patterns, + is_directed, + ); + if is_directed { + direction_arr[size - 1] = Direction::Fwd; + Self::generate_direction_patterns_inner( + direction_arr, + size - 1, + direction_patterns, + is_directed, + ); + } + } + } + + fn get_out_subgraph( + &self, + mut query_graph: QueryGraph, + alds: Vec, + ) -> QueryGraph { + let num_qvertices = query_graph.get_num_qvertices(); + for ald in alds { + let mut query_edge = if let Direction::Fwd = ald.direction { + let mut query_edge = QueryEdge::default( + ald.from_query_vertex.clone(), + self.query_vertices[num_qvertices].clone(), + ); + query_edge.from_type = query_graph.get_query_vertex_type(&ald.from_query_vertex); + query_edge + } else { + let mut query_edge = QueryEdge::default( + self.query_vertices[num_qvertices].clone(), + ald.from_query_vertex.clone(), + ); + query_edge.to_type = query_graph.get_query_vertex_type(&ald.from_query_vertex); + query_edge + }; + query_edge.label = ald.label; + query_graph.add_qedge(query_edge); + } + query_graph + } + + pub fn generate_all_scans_for_large_graph< + NL: Hash + Eq, + EL: Hash + Eq, + Ty: GraphType, + L: IdType, + >( + &mut self, + graph: &TypedStaticGraph, + ) -> Vec> { + let fwd_adj_lists = graph.get_fwd_adj_list(); + let num_vertices = graph.node_count(); + let mut edges = vec![]; + for from_vertex in 0..num_vertices { + for to_vertex in fwd_adj_lists[from_vertex] + .as_ref() + .unwrap() + .get_neighbor_ids() + { + edges.push(vec![Id::new(from_vertex), to_vertex.clone()]); + } + } + let mut out_subgraph = QueryGraph::empty(); + out_subgraph.add_qedge(QueryEdge::new("a".to_owned(), "b".to_owned(), 0, 0, 0)); + let mut scan = ScanSampling::new(Box::new(out_subgraph)); + scan.set_edge_indices_to_sample_list(edges, self.num_sampled_edges); + vec![scan] + } + + pub fn generate_all_scans( + &mut self, + graph: &TypedStaticGraph, + ) -> Vec> { + let fwd_adj_lists = graph.get_fwd_adj_list(); + let vertex_types = graph.get_node_types(); + let num_vertices = graph.node_count(); + let mut key_to_edges_map = HashMap::new(); + let mut key_to_curr_idx = HashMap::new(); + let node_labels = graph.get_node_types(); + for from_type in 0..=self.num_node_labels { + for label in 0..=self.num_edge_labels { + for to_type in 0..=self.num_node_labels { + let edge_key = TypedStaticGraph::::get_edge_key( + from_type, to_type, label, + ); + let num_edges = graph.get_num_edges(from_type, to_type, label); + key_to_edges_map.insert(edge_key, vec![0; num_edges * 2]); + key_to_curr_idx.insert(edge_key, 0); + } + } + } + for from_vertex in 0..num_vertices { + let from_type = vertex_types[from_vertex]; + let offsets = fwd_adj_lists[from_vertex].as_ref().unwrap().get_offsets(); + let neighbours = fwd_adj_lists[from_vertex] + .as_ref() + .unwrap() + .get_neighbor_ids(); + for label_type in 0..offsets.len() - 1 { + for to_idx in offsets[label_type]..offsets[label_type + 1] { + let (to_type, label) = if self.sorted_by_node { + (label_type, 0) + } else { + (vertex_types[neighbours[to_idx].id()], label_type) + }; + let edge_key = TypedStaticGraph::::get_edge_key( + from_type, to_type, label, + ); + let curr_idx = key_to_curr_idx[&edge_key]; + key_to_edges_map.get_mut(&edge_key).unwrap()[curr_idx] = from_vertex; + key_to_edges_map.get_mut(&edge_key).unwrap()[curr_idx + 1] = + neighbours[to_idx].id(); + key_to_curr_idx.insert(edge_key, curr_idx + 2); + } + } + } + let mut scans = vec![]; + for from_type in 0..self.num_node_labels { + for label in 0..self.num_edge_labels { + for to_type in 0..self.num_node_labels { + let mut out_subgraph = QueryGraph::empty(); + out_subgraph.add_qedge(QueryEdge::new( + "a".to_owned(), + "b".to_owned(), + from_type, + to_type, + label, + )); + let edge_key = TypedStaticGraph::::get_edge_key( + from_type, to_type, label, + ); + let actual_num_edges = graph.get_num_edges(from_type, to_type, label); + if actual_num_edges > 0 { + let mut num_edges_to_sample = self.num_sampled_edges + * (graph.get_num_edges(from_type, to_type, label) / graph.edge_count()) + as usize; + let mut scan = ScanSampling::new(Box::new(out_subgraph)); + if self.sorted_by_node && num_edges_to_sample < 1000 { + num_edges_to_sample = actual_num_edges; + } + scan.set_edge_indices_to_sample( + key_to_edges_map[&edge_key] + .iter() + .map(|edge| Id::new(edge.clone())) + .collect(), + num_edges_to_sample, + ); + scans.push(scan); + } + } + } + } + scans } pub fn query_graphs_to_extend(&self) -> &QueryGraphSet { diff --git a/src/graph_impl/multi_graph/planner/catalog/operator/intersect_catalog.rs b/src/graph_impl/multi_graph/planner/catalog/operator/intersect_catalog.rs index fb5d454a..11daf721 100644 --- a/src/graph_impl/multi_graph/planner/catalog/operator/intersect_catalog.rs +++ b/src/graph_impl/multi_graph/planner/catalog/operator/intersect_catalog.rs @@ -1,5 +1,5 @@ use generic::{GraphType, IdType}; -use graph_impl::multi_graph::plan::operator::extend::intersect::Intersect; +use graph_impl::multi_graph::plan::operator::extend::intersect::BaseIntersect; use graph_impl::multi_graph::plan::operator::extend::EI::CachingType; use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; use graph_impl::multi_graph::planner::catalog::adj_list_descriptor::AdjListDescriptor; @@ -8,10 +8,12 @@ use graph_impl::TypedStaticGraph; use hashbrown::HashMap; use std::hash::{BuildHasherDefault, Hash}; +#[derive(Clone)] pub struct IntersectCatalog { - pub base_intersect: Intersect, + pub base_intersect: BaseIntersect, is_adj_list_sorted_by_type: bool, last_icost: usize, + caching_enable: bool, } impl IntersectCatalog { @@ -25,7 +27,7 @@ impl IntersectCatalog { is_adj_list_sorted_by_type: bool, ) -> IntersectCatalog { IntersectCatalog { - base_intersect: Intersect::new( + base_intersect: BaseIntersect::new( to_qvertex, to_type, alds, @@ -35,6 +37,7 @@ impl IntersectCatalog { ), is_adj_list_sorted_by_type, last_icost: 0, + caching_enable: true, } } } @@ -151,17 +154,15 @@ impl CommonOperatorTrait for IntersectCatalog { base_ei.base_op.probe_tuple[base_ei.out_idx] = base_ei.out_neighbours.ids[idx]; base_ei.base_op.num_out_tuples += 1; if self.is_adj_list_sorted_by_type { - base_ei.base_op.next.as_mut().map(|next| { - next.get_mut(0).map(|next_op| { - next_op.process_new_tuple(); - }) - }); + base_ei.base_op.next[0].process_new_tuple(); } else { - let ops = base_ei.base_op.next.as_mut().unwrap(); - ops.get_mut( - base_ei.vertex_types[base_ei.base_op.probe_tuple[base_ei.out_idx].id()], - ) - .map(|next_op| next_op.process_new_tuple()); + base_ei + .base_op + .next + .get_mut( + base_ei.vertex_types[base_ei.base_op.probe_tuple[base_ei.out_idx].id()], + ) + .map(|next_op| next_op.process_new_tuple()); } } } diff --git a/src/graph_impl/multi_graph/planner/catalog/operator/noop.rs b/src/graph_impl/multi_graph/planner/catalog/operator/noop.rs index d717b852..b1cf13a7 100644 --- a/src/graph_impl/multi_graph/planner/catalog/operator/noop.rs +++ b/src/graph_impl/multi_graph/planner/catalog/operator/noop.rs @@ -8,8 +8,9 @@ use hashbrown::HashMap; use itertools::Itertools; use std::hash::{BuildHasherDefault, Hash}; +#[derive(Clone)] pub struct Noop { - base_op: BaseOperator, + pub base_op: BaseOperator, } impl Noop { @@ -27,18 +28,17 @@ impl CommonOperatorTrait for Noop { graph: &TypedStaticGraph, ) { self.base_op.probe_tuple = probe_tuple.clone(); - let next = self.base_op.next.as_mut().unwrap(); - for next_op in next { + for next_op in &mut self.base_op.next { next_op.init(probe_tuple.clone(), graph); } } fn process_new_tuple(&mut self) { self.base_op.num_out_tuples += 1; - self.base_op.next.as_mut().map(|next| { - next.iter_mut() - .for_each(|next_op| next_op.process_new_tuple()) - }); + self.base_op + .next + .iter_mut() + .for_each(|next_op| next_op.process_new_tuple()); } fn execute(&mut self) { diff --git a/src/graph_impl/multi_graph/planner/query_planner.rs b/src/graph_impl/multi_graph/planner/query_planner.rs index 76a5b3eb..c0bed9c8 100644 --- a/src/graph_impl/multi_graph/planner/query_planner.rs +++ b/src/graph_impl/multi_graph/planner/query_planner.rs @@ -1,4 +1,6 @@ use generic::{GraphType, IdType}; +use graph_impl::multi_graph::plan::operator::extend::intersect::BaseIntersect; +use graph_impl::multi_graph::plan::operator::extend::intersect::Intersect; use graph_impl::multi_graph::plan::operator::extend::EI::{BaseEI, CachingType, EI}; use graph_impl::multi_graph::plan::operator::hashjoin::hash_join::HashJoin; use graph_impl::multi_graph::plan::operator::hashjoin::probe::{BaseProbe, Probe}; @@ -93,7 +95,7 @@ impl while get_op_attr_as_ref!(cur_op, prev).is_some() { let cur_copy = cur_op.clone(); let prev = get_op_attr_as_mut!(cur_op, prev).as_mut().unwrap().as_mut(); - get_op_attr_as_mut!(prev, next).replace(vec![cur_copy]); + *get_op_attr_as_mut!(prev, next) = vec![cur_copy]; cur_op = get_op_attr_as_mut!(cur_op, prev).as_mut().unwrap().as_mut(); } }); diff --git a/src/graph_impl/multi_graph/planner/query_planner_big.rs b/src/graph_impl/multi_graph/planner/query_planner_big.rs index 75037a51..2ab6848f 100644 --- a/src/graph_impl/multi_graph/planner/query_planner_big.rs +++ b/src/graph_impl/multi_graph/planner/query_planner_big.rs @@ -1,4 +1,6 @@ use generic::{GraphType, IdType}; +use graph_impl::multi_graph::plan::operator::extend::intersect::BaseIntersect; +use graph_impl::multi_graph::plan::operator::extend::intersect::Intersect; use graph_impl::multi_graph::plan::operator::extend::EI::{BaseEI, CachingType, EI}; use graph_impl::multi_graph::plan::operator::hashjoin::probe::{BaseProbe, Probe}; use graph_impl::multi_graph::plan::operator::hashjoin::probe_multi_vertices::{ diff --git a/src/graph_impl/multi_graph/runner/optimizer_executor.rs b/src/graph_impl/multi_graph/runner/optimizer_executor.rs index d1377e03..f8ceb75c 100644 --- a/src/graph_impl/multi_graph/runner/optimizer_executor.rs +++ b/src/graph_impl/multi_graph/runner/optimizer_executor.rs @@ -1,4 +1,5 @@ use generic::{GraphType, IdType}; +use graph_impl::multi_graph::plan::query_plan::QueryPlan; use graph_impl::multi_graph::planner::catalog::catalog::Catalog; use graph_impl::multi_graph::planner::catalog::query_edge::QueryEdge; use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; @@ -10,17 +11,15 @@ use hashbrown::HashMap; use std::hash::Hash; use std::time::SystemTime; use DiStaticGraph; -use graph_impl::multi_graph::plan::query_plan::QueryPlan; pub fn generate_plan( query_graph: QueryGraph, catalog: Catalog, g: TypedStaticGraph, -)->QueryPlan { - let num_qvertices = query_graph.get_num_qvertices(); +) -> QueryPlan { let start_time = SystemTime::now(); let elapsed_time; - let mut query_plan = if query_graph.get_num_qvertices() <= 8 { + let query_plan = if query_graph.get_num_qvertices() <= 8 { let mut planner = QueryPlanner::new(query_graph, catalog, g); let query_plan = planner.plan(); elapsed_time = SystemTime::now() diff --git a/src/graph_impl/multi_graph/utils/mod.rs b/src/graph_impl/multi_graph/utils/mod.rs new file mode 100644 index 00000000..1ec8e1c3 --- /dev/null +++ b/src/graph_impl/multi_graph/utils/mod.rs @@ -0,0 +1 @@ +pub mod set_utils; diff --git a/src/graph_impl/multi_graph/utils/set_utils.rs b/src/graph_impl/multi_graph/utils/set_utils.rs new file mode 100644 index 00000000..e5fc3753 --- /dev/null +++ b/src/graph_impl/multi_graph/utils/set_utils.rs @@ -0,0 +1,58 @@ +use std::iter::FromIterator; + +pub fn get_power_set_excluding_empty_set(set: Vec) -> Vec> { + let mut res = vec![]; + let len = set.len(); + for sub in generate_power_set(set) { + if sub.len() >= 1 && sub.len() <= len { + res.push(sub); + } + } + res +} + +pub fn generate_permutations(mut set: Vec, len: usize) -> Vec> { + let mut permutations = vec![]; + get_permutations_given_len(&mut set, len, 0, &mut vec![], &mut permutations); + permutations +} + +fn get_permutations_given_len( + set: &mut Vec, + len: usize, + pos: usize, + temp: &mut Vec, + permutation: &mut Vec>, +) { + if len == 0 { + permutation.push(temp.clone()); + return; + } + for i in 0..set.len() { + if temp.len() < pos + 1 { + temp.push(set[i].clone()); + } else { + temp.insert(pos, set[i].clone()); + } + get_permutations_given_len(set, len - 1, pos + 1, temp, permutation); + } +} + +fn generate_power_set(original_set: Vec) -> Vec> { + let mut sets = vec![]; + if original_set.is_empty() { + sets.push(vec![]); + return sets; + } + let list = original_set.clone(); + let head = list.get(0).unwrap(); + let rest = list[1..list.len()].iter().map(|x| x.clone()).collect(); + for set in generate_power_set(rest) { + let mut new_set = vec![]; + new_set.push(head.clone()); + set.iter().for_each(|it| new_set.push(it.clone())); + sets.push(new_set); + sets.push(set); + } + sets +} diff --git a/tests/static_graph.rs b/tests/static_graph.rs index 4d33a6c5..89608a19 100644 --- a/tests/static_graph.rs +++ b/tests/static_graph.rs @@ -27,6 +27,7 @@ extern crate tempfile; use hashbrown::HashMap; use itertools::Itertools; use rust_graph::generic::DefaultId; +use rust_graph::graph_impl::multi_graph::plan::query_plan_worker::QPWorkers; use rust_graph::graph_impl::multi_graph::planner::catalog::query_edge::QueryEdge; use rust_graph::graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use rust_graph::graph_impl::multi_graph::runner::{catalog_generator, optimizer_executor}; @@ -38,7 +39,6 @@ use rust_graph::map::SetMap; use rust_graph::prelude::*; use rust_graph::{DiStaticGraph, UnStaticGraph}; use std::path::Path; -use rust_graph::graph_impl::multi_graph::plan::query_plan_worker::QPWorkers; #[test] fn test_directed() { @@ -335,6 +335,10 @@ fn test_graphflow_planner() { false, ); let g = g_.into_static(); + println!("node_count={}",g.node_count()); + println!("edge_count={}",g.edge_count()); + println!("num_of_node_labels={}",g.num_of_node_labels()); + println!("num_of_edge_labels={}",g.num_of_edge_labels()); println!("load finished."); // let edge_vec = EdgeVec::with_labels(vec![0, 2, 3, 4], vec![1, 2, 0, 0], vec![0, 1, 0, 1]); // let in_edge_vec = EdgeVec::new(vec![0, 2, 3, 4], vec![1, 2, 0, 0]); From b25bad86082c1f0211919a113ab29558f442d021 Mon Sep 17 00:00:00 2001 From: Michael Chen <920901835@qq.com> Date: Thu, 16 Jan 2020 10:20:18 +0800 Subject: [PATCH 18/25] refactor:using Rc and RefCell to reduce memory copy --- .../multi_graph/plan/operator/extend/EI.rs | 44 ++-- .../plan/operator/extend/Extend.rs | 61 ++--- .../plan/operator/extend/Intersect.rs | 61 +++-- .../plan/operator/extend/extend.rs | 61 ++--- .../plan/operator/extend/intersect.rs | 61 +++-- .../plan/operator/hashjoin/build.rs | 26 +- .../plan/operator/hashjoin/hash_join.rs | 94 ++++---- .../plan/operator/hashjoin/hash_table.rs | 1 - .../plan/operator/hashjoin/probe.rs | 37 +-- .../plan/operator/hashjoin/probe_cartesian.rs | 17 +- .../operator/hashjoin/probe_multi_vertices.rs | 31 ++- .../probe_multi_vertices_cartesian.rs | 16 +- .../multi_graph/plan/operator/operator.rs | 51 ++-- .../multi_graph/plan/operator/scan/scan.rs | 28 ++- .../plan/operator/scan/scan_blocking.rs | 22 +- .../plan/operator/scan/scan_sampling.rs | 13 +- .../multi_graph/plan/operator/sink/sink.rs | 39 +-- .../plan/operator/sink/sink_copy.rs | 14 +- .../plan/operator/sink/sink_limit.rs | 13 +- .../plan/operator/sink/sink_print.rs | 13 +- src/graph_impl/multi_graph/plan/query_plan.rs | 222 +++++++++++------- .../multi_graph/plan/query_plan_worker.rs | 39 ++- .../multi_graph/planner/catalog/catalog.rs | 196 +++++++++++----- .../planner/catalog/catalog_plans.rs | 104 ++++---- .../catalog/operator/intersect_catalog.rs | 18 +- .../planner/catalog/operator/noop.rs | 17 +- .../planner/catalog/query_graph.rs | 5 +- .../catalog/subgraph_mapping_iterator.rs | 4 +- .../multi_graph/planner/query_planner.rs | 110 +++++---- .../multi_graph/planner/query_planner_big.rs | 15 +- .../multi_graph/query/query_graph_set.rs | 2 +- .../multi_graph/runner/catalog_generator.rs | 2 +- .../multi_graph/runner/optimizer_executor.rs | 6 +- src/graph_impl/multi_graph/utils/set_utils.rs | 2 - src/graph_impl/static_graph/graph.rs | 3 - tests/static_graph.rs | 8 +- 36 files changed, 856 insertions(+), 600 deletions(-) diff --git a/src/graph_impl/multi_graph/plan/operator/extend/EI.rs b/src/graph_impl/multi_graph/plan/operator/extend/EI.rs index 0bc86310..36c8970b 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/EI.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/EI.rs @@ -17,9 +17,11 @@ use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::static_graph::sorted_adj_vec::SortedAdjVec; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; -use indexmap::Equivalent; use itertools::Itertools; -use std::hash::{BuildHasherDefault, Hash}; +use std::cell::RefCell; +use std::hash::Hash; +use std::ops::{Deref, DerefMut}; +use std::rc::Rc; pub static DIFFERENTIATE_FWD_BWD_SINGLE_ALD: bool = false; @@ -76,8 +78,8 @@ impl BaseEI { to_query_vertex: String, to_type: usize, alds: Vec, - out_subgraph: Box, - in_subgraph: Option>, + out_subgraph: QueryGraph, + in_subgraph: Option, ) -> BaseEI { let mut ei = BaseEI { base_op: BaseOperator::new(out_subgraph, in_subgraph), @@ -286,13 +288,15 @@ impl CommonOperatorTrait for BaseEI { self.base_op.probe_tuple = probe_tuple.clone(); self.caching_type = CachingType::None; self.vertex_types = graph.get_node_types().clone(); - let prev = self.base_op.prev.as_mut().unwrap().as_mut(); - let last_repeated_vertex_idx = get_op_attr_as_mut!(prev, last_repeated_vertex_idx).clone(); + let last_repeated_vertex_idx = { + let mut prev = self.base_op.prev.as_ref().unwrap().borrow(); + get_op_attr!(prev.deref(), last_repeated_vertex_idx) + }; self.init_caching(last_repeated_vertex_idx); self.init_extensions(graph); self.set_alds_and_adj_lists(graph, last_repeated_vertex_idx); - self.base_op.next.iter_mut().foreach(|next_op| { - next_op.init(probe_tuple.clone(), graph); + self.base_op.next.iter_mut().for_each(|next_op| { + next_op.borrow_mut().init(probe_tuple.clone(), graph); }); } @@ -332,17 +336,17 @@ impl CommonOperatorTrait for BaseEI { self.to_query_vertex.clone(), query_vertex_to_index_map.len(), ); - self.base_op - .next - .iter_mut() - .foreach(|op| op.update_operator_name(query_vertex_to_index_map.clone())) + self.base_op.next.iter_mut().foreach(|op| { + op.borrow_mut() + .update_operator_name(query_vertex_to_index_map.clone()) + }) } - fn copy(&self, is_thread_safe: bool) -> Operator { + fn copy(&self, _is_thread_safe: bool) -> Operator { panic!("unsupported operation exception") } - fn is_same_as(&mut self, op: &mut Operator) -> bool { + fn is_same_as(&mut self, _op: &mut Rc>>) -> bool { panic!("unsupported operation exception") } @@ -358,7 +362,7 @@ impl EI { return true; } if let Some(prev) = &base_ei.base_op.prev { - return prev.has_multi_edge_extends(); + return prev.borrow().has_multi_edge_extends(); } false } @@ -376,8 +380,8 @@ impl EI { to_qvertex, to_type, alds, - Box::new(out_subgraph), - Some(Box::new(in_subgraph)), + out_subgraph, + Some(in_subgraph), out_qvertex_to_idx_map, )); } @@ -385,8 +389,8 @@ impl EI { to_qvertex, to_type, alds, - Box::new(out_subgraph), - Some(Box::new(in_subgraph)), + out_subgraph, + Some(in_subgraph), out_qvertex_to_idx_map, ))) } @@ -445,7 +449,7 @@ impl CommonOperatorTrait for EI { } } - fn is_same_as(&mut self, op: &mut Operator) -> bool { + fn is_same_as(&mut self, op: &mut Rc>>) -> bool { match self { EI::Base(base) => base.is_same_as(op), EI::Intersect(intersect) => intersect.is_same_as(op), diff --git a/src/graph_impl/multi_graph/plan/operator/extend/Extend.rs b/src/graph_impl/multi_graph/plan/operator/extend/Extend.rs index 133166ad..00cf9e60 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/Extend.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/Extend.rs @@ -16,7 +16,10 @@ use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::static_graph::sorted_adj_vec::SortedAdjVec; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; -use std::hash::{BuildHasherDefault, Hash}; +use std::cell::RefCell; +use std::hash::Hash; +use std::ops::DerefMut; +use std::rc::Rc; #[derive(Clone)] pub struct Extend { @@ -32,8 +35,8 @@ impl Extend { to_qvertex: String, to_type: usize, alds: Vec, - out_subgraph: Box, - in_subgraph: Option>, + out_subgraph: QueryGraph, + in_subgraph: Option, out_qvertex_to_idx_map: HashMap, ) -> Extend { let ald = alds[0].clone(); @@ -71,7 +74,7 @@ impl CommonOperatorTrait for Extend { self.base_ei.to_type = 0; } for next_operator in &mut self.base_ei.base_op.next { - next_operator.init(probe_tuple.clone(), graph); + next_operator.borrow_mut().init(probe_tuple.clone(), graph); } } @@ -88,7 +91,9 @@ impl CommonOperatorTrait for Extend { { self.base_ei.base_op.num_out_tuples += 1; self.base_ei.base_op.probe_tuple[self.base_ei.out_idx] = out_neighbour.ids[idx]; - self.base_ei.base_op.next[0].process_new_tuple(); + self.base_ei.base_op.next[0] + .borrow_mut() + .process_new_tuple(); } } } @@ -117,45 +122,41 @@ impl CommonOperatorTrait for Extend { base_op.out_qvertex_to_idx_map.clone(), ); let extend_copy = extend.clone(); - extend.base_ei.base_op.prev = Some(Box::new( - base_op.prev.as_ref().unwrap().copy(is_thread_safe), - )); - let prev = extend.base_ei.base_op.prev.as_mut().unwrap().as_mut(); - *get_op_attr_as_mut!(prev, next) = vec![Operator::EI(EI::Extend(extend_copy))]; - let last_repeated_vertex_idx = get_op_attr!(prev, last_repeated_vertex_idx); + extend.base_ei.base_op.prev = Some(Rc::new(RefCell::new( + base_op.prev.as_ref().unwrap().borrow().copy(is_thread_safe), + ))); + + let last_repeated_vertex_idx = { + let mut prev = extend.base_ei.base_op.prev.as_mut().unwrap().borrow_mut(); + *get_op_attr_as_mut!(prev.deref_mut(), next) = + vec![Rc::new(RefCell::new(Operator::EI(EI::Extend(extend_copy))))]; + get_op_attr!(prev.deref_mut(), last_repeated_vertex_idx) + }; extend.base_ei.init_caching(last_repeated_vertex_idx); Operator::EI(EI::Extend(extend)) } - fn is_same_as(&mut self, op: &mut Operator) -> bool { - if let Operator::EI(EI::Extend(extend)) = op { + fn is_same_as(&mut self, op: &mut Rc>>) -> bool { + if let Operator::EI(EI::Extend(extend)) = op.borrow_mut().deref_mut() { + let base_self = &mut self.base_ei.base_op; + let other_op = &mut extend.base_ei.base_op; return (!DIFFERENTIATE_FWD_BWD_SINGLE_ALD || self.dir == extend.dir) && self.label_or_to_type == extend.label_or_to_type && self.base_ei.to_type == extend.base_ei.to_type - && self - .base_ei - .base_op + && base_self .in_subgraph .as_mut() .unwrap() - .is_isomorphic_to( - get_op_attr_as_mut!(op, in_subgraph) - .as_mut() - .unwrap() - .as_mut(), - ) - && self - .base_ei - .base_op + .is_isomorphic_to(other_op.in_subgraph.as_mut().unwrap()) + && base_self .out_subgraph - .is_isomorphic_to(get_op_attr_as_mut!(op, out_subgraph).as_mut()) - && self - .base_ei - .base_op + .is_isomorphic_to(&mut other_op.out_subgraph) + && base_self .prev .as_mut() .unwrap() - .is_same_as(get_op_attr_as_mut!(op, prev).as_mut().unwrap()); + .borrow_mut() + .is_same_as(other_op.prev.as_mut().unwrap()); } false } diff --git a/src/graph_impl/multi_graph/plan/operator/extend/Intersect.rs b/src/graph_impl/multi_graph/plan/operator/extend/Intersect.rs index 0b6afec7..d3fb4565 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/Intersect.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/Intersect.rs @@ -11,7 +11,10 @@ use graph_impl::multi_graph::planner::catalog::operator::intersect_catalog::Inte use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; -use std::hash::{BuildHasherDefault, Hash}; +use std::cell::RefCell; +use std::hash::Hash; +use std::ops::DerefMut; +use std::rc::Rc; pub enum IntersectType { InitCached, @@ -36,8 +39,8 @@ impl BaseIntersect { to_qvertex: String, to_type: usize, alds: Vec, - out_subgraph: Box, - in_subgraph: Option>, + out_subgraph: QueryGraph, + in_subgraph: Option, out_qvertex_to_idx_map: HashMap, ) -> BaseIntersect { let mut intersect = BaseIntersect { @@ -72,7 +75,7 @@ impl CommonOperatorTrait for BaseIntersect { let neighbours = &mut base_ei.init_neighbours; adj_vec.map(|adj| adj.set_neighbor_ids(cache_id, neighbours)); base_ei.base_op.icost += - (base_ei.init_neighbours.end_idx - base_ei.init_neighbours.start_idx); + base_ei.init_neighbours.end_idx - base_ei.init_neighbours.start_idx; base_ei.base_op.icost += base_ei.execute_intersect(1, IntersectType::InitCached); if base_ei.to_type != 0 { @@ -114,10 +117,10 @@ impl CommonOperatorTrait for BaseIntersect { let base_op = &mut base_ei.base_op; // setAdjListSortOrder the initNeighbours ids in the output tuple. let out_neighbours = &mut base_ei.out_neighbours; - base_op.num_out_tuples += (out_neighbours.end_idx - out_neighbours.start_idx); + base_op.num_out_tuples += out_neighbours.end_idx - out_neighbours.start_idx; for idx in out_neighbours.start_idx..out_neighbours.end_idx { base_op.probe_tuple[base_ei.out_idx] = out_neighbours.ids[idx]; - base_op.next[0].process_new_tuple(); + base_op.next[0].borrow_mut().process_new_tuple(); } } @@ -145,46 +148,52 @@ impl CommonOperatorTrait for BaseIntersect { base_op.out_qvertex_to_idx_map.clone(), ); let intersect_copy = intersect.clone(); - intersect.base_ei.base_op.prev = Some(Box::new( - base_op.prev.as_ref().unwrap().copy(is_thread_safe), - )); - let prev = intersect.base_ei.base_op.prev.as_mut().unwrap().as_mut(); - *get_op_attr_as_mut!(prev, next) = vec![Operator::EI(EI::Intersect( - Intersect::BaseIntersect(intersect_copy), - ))]; - let last_repeated_vertex_idx = get_op_attr!(prev, last_repeated_vertex_idx); + intersect.base_ei.base_op.prev = Some(Rc::new(RefCell::new( + base_op.prev.as_ref().unwrap().borrow().copy(is_thread_safe), + ))); + let last_repeated_vertex_idx = { + let mut prev = intersect + .base_ei + .base_op + .prev + .as_mut() + .unwrap() + .borrow_mut(); + *get_op_attr_as_mut!(prev.deref_mut(), next) = vec![Rc::new(RefCell::new( + Operator::EI(EI::Intersect(Intersect::BaseIntersect(intersect_copy))), + ))]; + get_op_attr!(prev.deref_mut(), last_repeated_vertex_idx) + }; intersect.base_ei.init_caching(last_repeated_vertex_idx); Operator::EI(EI::Intersect(Intersect::BaseIntersect(intersect))) } - fn is_same_as(&mut self, op: &mut Operator) -> bool { - if let Operator::EI(EI::Intersect(Intersect::BaseIntersect(intersect))) = op { + fn is_same_as(&mut self, op: &mut Rc>>) -> bool { + if let Operator::EI(EI::Intersect(Intersect::BaseIntersect(intersect))) = + op.borrow_mut().deref_mut() + { return self.base_ei.caching_type == intersect.base_ei.caching_type - && self.get_alds_as_string() == intersect.get_alds_as_string() + && self.get_alds_as_string() == intersect.base_ei.base_op.get_alds_as_string() && self .base_ei .base_op .in_subgraph .as_mut() .unwrap() - .is_isomorphic_to( - get_op_attr_as_mut!(op, in_subgraph) - .as_mut() - .unwrap() - .as_mut(), - ) + .is_isomorphic_to(intersect.base_ei.base_op.in_subgraph.as_mut().unwrap()) && self .base_ei .base_op .out_subgraph - .is_isomorphic_to(get_op_attr_as_mut!(op, out_subgraph).as_mut()) + .is_isomorphic_to(&mut intersect.base_ei.base_op.out_subgraph) && self .base_ei .base_op .prev .as_mut() .unwrap() - .is_same_as(get_op_attr_as_mut!(op, prev).as_mut().unwrap()); + .borrow_mut() + .is_same_as(intersect.base_ei.base_op.prev.as_mut().unwrap()); } false } @@ -241,7 +250,7 @@ impl CommonOperatorTrait for Intersect { } } - fn is_same_as(&mut self, op: &mut Operator) -> bool { + fn is_same_as(&mut self, op: &mut Rc>>) -> bool { match self { Intersect::BaseIntersect(base) => base.is_same_as(op), Intersect::IntersectCatalog(ic) => ic.is_same_as(op), diff --git a/src/graph_impl/multi_graph/plan/operator/extend/extend.rs b/src/graph_impl/multi_graph/plan/operator/extend/extend.rs index 133166ad..00cf9e60 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/extend.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/extend.rs @@ -16,7 +16,10 @@ use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::static_graph::sorted_adj_vec::SortedAdjVec; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; -use std::hash::{BuildHasherDefault, Hash}; +use std::cell::RefCell; +use std::hash::Hash; +use std::ops::DerefMut; +use std::rc::Rc; #[derive(Clone)] pub struct Extend { @@ -32,8 +35,8 @@ impl Extend { to_qvertex: String, to_type: usize, alds: Vec, - out_subgraph: Box, - in_subgraph: Option>, + out_subgraph: QueryGraph, + in_subgraph: Option, out_qvertex_to_idx_map: HashMap, ) -> Extend { let ald = alds[0].clone(); @@ -71,7 +74,7 @@ impl CommonOperatorTrait for Extend { self.base_ei.to_type = 0; } for next_operator in &mut self.base_ei.base_op.next { - next_operator.init(probe_tuple.clone(), graph); + next_operator.borrow_mut().init(probe_tuple.clone(), graph); } } @@ -88,7 +91,9 @@ impl CommonOperatorTrait for Extend { { self.base_ei.base_op.num_out_tuples += 1; self.base_ei.base_op.probe_tuple[self.base_ei.out_idx] = out_neighbour.ids[idx]; - self.base_ei.base_op.next[0].process_new_tuple(); + self.base_ei.base_op.next[0] + .borrow_mut() + .process_new_tuple(); } } } @@ -117,45 +122,41 @@ impl CommonOperatorTrait for Extend { base_op.out_qvertex_to_idx_map.clone(), ); let extend_copy = extend.clone(); - extend.base_ei.base_op.prev = Some(Box::new( - base_op.prev.as_ref().unwrap().copy(is_thread_safe), - )); - let prev = extend.base_ei.base_op.prev.as_mut().unwrap().as_mut(); - *get_op_attr_as_mut!(prev, next) = vec![Operator::EI(EI::Extend(extend_copy))]; - let last_repeated_vertex_idx = get_op_attr!(prev, last_repeated_vertex_idx); + extend.base_ei.base_op.prev = Some(Rc::new(RefCell::new( + base_op.prev.as_ref().unwrap().borrow().copy(is_thread_safe), + ))); + + let last_repeated_vertex_idx = { + let mut prev = extend.base_ei.base_op.prev.as_mut().unwrap().borrow_mut(); + *get_op_attr_as_mut!(prev.deref_mut(), next) = + vec![Rc::new(RefCell::new(Operator::EI(EI::Extend(extend_copy))))]; + get_op_attr!(prev.deref_mut(), last_repeated_vertex_idx) + }; extend.base_ei.init_caching(last_repeated_vertex_idx); Operator::EI(EI::Extend(extend)) } - fn is_same_as(&mut self, op: &mut Operator) -> bool { - if let Operator::EI(EI::Extend(extend)) = op { + fn is_same_as(&mut self, op: &mut Rc>>) -> bool { + if let Operator::EI(EI::Extend(extend)) = op.borrow_mut().deref_mut() { + let base_self = &mut self.base_ei.base_op; + let other_op = &mut extend.base_ei.base_op; return (!DIFFERENTIATE_FWD_BWD_SINGLE_ALD || self.dir == extend.dir) && self.label_or_to_type == extend.label_or_to_type && self.base_ei.to_type == extend.base_ei.to_type - && self - .base_ei - .base_op + && base_self .in_subgraph .as_mut() .unwrap() - .is_isomorphic_to( - get_op_attr_as_mut!(op, in_subgraph) - .as_mut() - .unwrap() - .as_mut(), - ) - && self - .base_ei - .base_op + .is_isomorphic_to(other_op.in_subgraph.as_mut().unwrap()) + && base_self .out_subgraph - .is_isomorphic_to(get_op_attr_as_mut!(op, out_subgraph).as_mut()) - && self - .base_ei - .base_op + .is_isomorphic_to(&mut other_op.out_subgraph) + && base_self .prev .as_mut() .unwrap() - .is_same_as(get_op_attr_as_mut!(op, prev).as_mut().unwrap()); + .borrow_mut() + .is_same_as(other_op.prev.as_mut().unwrap()); } false } diff --git a/src/graph_impl/multi_graph/plan/operator/extend/intersect.rs b/src/graph_impl/multi_graph/plan/operator/extend/intersect.rs index 0b6afec7..d3fb4565 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/intersect.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/intersect.rs @@ -11,7 +11,10 @@ use graph_impl::multi_graph::planner::catalog::operator::intersect_catalog::Inte use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; -use std::hash::{BuildHasherDefault, Hash}; +use std::cell::RefCell; +use std::hash::Hash; +use std::ops::DerefMut; +use std::rc::Rc; pub enum IntersectType { InitCached, @@ -36,8 +39,8 @@ impl BaseIntersect { to_qvertex: String, to_type: usize, alds: Vec, - out_subgraph: Box, - in_subgraph: Option>, + out_subgraph: QueryGraph, + in_subgraph: Option, out_qvertex_to_idx_map: HashMap, ) -> BaseIntersect { let mut intersect = BaseIntersect { @@ -72,7 +75,7 @@ impl CommonOperatorTrait for BaseIntersect { let neighbours = &mut base_ei.init_neighbours; adj_vec.map(|adj| adj.set_neighbor_ids(cache_id, neighbours)); base_ei.base_op.icost += - (base_ei.init_neighbours.end_idx - base_ei.init_neighbours.start_idx); + base_ei.init_neighbours.end_idx - base_ei.init_neighbours.start_idx; base_ei.base_op.icost += base_ei.execute_intersect(1, IntersectType::InitCached); if base_ei.to_type != 0 { @@ -114,10 +117,10 @@ impl CommonOperatorTrait for BaseIntersect { let base_op = &mut base_ei.base_op; // setAdjListSortOrder the initNeighbours ids in the output tuple. let out_neighbours = &mut base_ei.out_neighbours; - base_op.num_out_tuples += (out_neighbours.end_idx - out_neighbours.start_idx); + base_op.num_out_tuples += out_neighbours.end_idx - out_neighbours.start_idx; for idx in out_neighbours.start_idx..out_neighbours.end_idx { base_op.probe_tuple[base_ei.out_idx] = out_neighbours.ids[idx]; - base_op.next[0].process_new_tuple(); + base_op.next[0].borrow_mut().process_new_tuple(); } } @@ -145,46 +148,52 @@ impl CommonOperatorTrait for BaseIntersect { base_op.out_qvertex_to_idx_map.clone(), ); let intersect_copy = intersect.clone(); - intersect.base_ei.base_op.prev = Some(Box::new( - base_op.prev.as_ref().unwrap().copy(is_thread_safe), - )); - let prev = intersect.base_ei.base_op.prev.as_mut().unwrap().as_mut(); - *get_op_attr_as_mut!(prev, next) = vec![Operator::EI(EI::Intersect( - Intersect::BaseIntersect(intersect_copy), - ))]; - let last_repeated_vertex_idx = get_op_attr!(prev, last_repeated_vertex_idx); + intersect.base_ei.base_op.prev = Some(Rc::new(RefCell::new( + base_op.prev.as_ref().unwrap().borrow().copy(is_thread_safe), + ))); + let last_repeated_vertex_idx = { + let mut prev = intersect + .base_ei + .base_op + .prev + .as_mut() + .unwrap() + .borrow_mut(); + *get_op_attr_as_mut!(prev.deref_mut(), next) = vec![Rc::new(RefCell::new( + Operator::EI(EI::Intersect(Intersect::BaseIntersect(intersect_copy))), + ))]; + get_op_attr!(prev.deref_mut(), last_repeated_vertex_idx) + }; intersect.base_ei.init_caching(last_repeated_vertex_idx); Operator::EI(EI::Intersect(Intersect::BaseIntersect(intersect))) } - fn is_same_as(&mut self, op: &mut Operator) -> bool { - if let Operator::EI(EI::Intersect(Intersect::BaseIntersect(intersect))) = op { + fn is_same_as(&mut self, op: &mut Rc>>) -> bool { + if let Operator::EI(EI::Intersect(Intersect::BaseIntersect(intersect))) = + op.borrow_mut().deref_mut() + { return self.base_ei.caching_type == intersect.base_ei.caching_type - && self.get_alds_as_string() == intersect.get_alds_as_string() + && self.get_alds_as_string() == intersect.base_ei.base_op.get_alds_as_string() && self .base_ei .base_op .in_subgraph .as_mut() .unwrap() - .is_isomorphic_to( - get_op_attr_as_mut!(op, in_subgraph) - .as_mut() - .unwrap() - .as_mut(), - ) + .is_isomorphic_to(intersect.base_ei.base_op.in_subgraph.as_mut().unwrap()) && self .base_ei .base_op .out_subgraph - .is_isomorphic_to(get_op_attr_as_mut!(op, out_subgraph).as_mut()) + .is_isomorphic_to(&mut intersect.base_ei.base_op.out_subgraph) && self .base_ei .base_op .prev .as_mut() .unwrap() - .is_same_as(get_op_attr_as_mut!(op, prev).as_mut().unwrap()); + .borrow_mut() + .is_same_as(intersect.base_ei.base_op.prev.as_mut().unwrap()); } false } @@ -241,7 +250,7 @@ impl CommonOperatorTrait for Intersect { } } - fn is_same_as(&mut self, op: &mut Operator) -> bool { + fn is_same_as(&mut self, op: &mut Rc>>) -> bool { match self { Intersect::BaseIntersect(base) => base.is_same_as(op), Intersect::IntersectCatalog(ic) => ic.is_same_as(op), diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/build.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/build.rs index 00a011fa..46130756 100644 --- a/src/graph_impl/multi_graph/plan/operator/hashjoin/build.rs +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/build.rs @@ -6,13 +6,16 @@ use graph_impl::multi_graph::plan::operator::operator::{ use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; -use std::hash::{BuildHasherDefault, Hash}; +use std::cell::RefCell; +use std::hash::Hash; +use std::ops::DerefMut; +use std::rc::Rc; #[derive(Clone)] pub struct Build { pub base_op: BaseOperator, pub hash_table: Option>, - pub probing_subgraph: Option>, + pub probing_subgraph: Option, query_vertex_to_hash: String, pub build_hash_idx: usize, pub hashed_tuple_len: usize, @@ -20,7 +23,7 @@ pub struct Build { impl Build { pub fn new( - in_subgraph: Box, + in_subgraph: QueryGraph, query_vertex_to_hash: String, build_hash_idx: usize, ) -> Build { @@ -80,19 +83,20 @@ impl CommonOperatorTrait for Build { self.build_hash_idx, ); build.base_op.prev = self.base_op.prev.as_ref().map(|prev| prev.clone()); - build.base_op.next = vec![Operator::Build(build.clone())]; + build.base_op.next = vec![Rc::new(RefCell::new(Operator::Build(build.clone())))]; build.probing_subgraph = self.probing_subgraph.clone(); Operator::Build(build) } - fn is_same_as(&mut self, op: &mut Operator) -> bool { - if let Operator::Build(build) = op { - let base_op = &mut self.base_op; - let in_subgraph = base_op.in_subgraph.as_mut().map_or(false, |in_subgraph| { - in_subgraph.is_isomorphic_to(build.base_op.in_subgraph.as_mut().unwrap()) + fn is_same_as(&mut self, op: &mut Rc>>) -> bool { + if let Operator::Build(build) = op.borrow_mut().deref_mut() { + let base_self = &mut self.base_op; + let base_op = &mut build.base_op; + let in_subgraph = base_self.in_subgraph.as_mut().map_or(false, |in_subgraph| { + in_subgraph.is_isomorphic_to(base_op.in_subgraph.as_mut().unwrap()) }); - let prev = base_op.prev.as_mut().map_or(false, |prev| { - prev.is_same_as(build.base_op.prev.as_mut().unwrap().as_mut()) + let prev = base_self.prev.as_ref().map_or(false, |prev| { + prev.borrow_mut().is_same_as(base_op.prev.as_mut().unwrap()) }); return in_subgraph && prev; } diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/hash_join.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/hash_join.rs index ab445378..2501cc68 100644 --- a/src/graph_impl/multi_graph/plan/operator/hashjoin/hash_join.rs +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/hash_join.rs @@ -15,6 +15,9 @@ use graph_impl::multi_graph::plan::operator::sink::sink::Sink; use graph_impl::multi_graph::plan::query_plan::QueryPlan; use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use hashbrown::HashMap; +use std::cell::RefCell; +use std::ops::{Deref, DerefMut}; +use std::rc::Rc; pub struct HashJoin {} @@ -33,35 +36,41 @@ impl HashJoin { fn inner_make( out_subgraph: QueryGraph, - mut build_subplans: Vec>>, - mut probe_subplans: Vec>>, - ) -> Vec>> { - let mut pre_build = *build_subplans.pop().unwrap(); - let mut pre_probe = *probe_subplans.pop().unwrap(); + mut build_subplans: Vec>>>, + mut probe_subplans: Vec>>>, + ) -> Vec>>> { + let mut pre_build = build_subplans.pop().unwrap(); + let mut pre_probe = probe_subplans.pop().unwrap(); let join_qvertices: Vec = pre_build + .borrow() .get_out_query_vertices() - .intersection(&pre_probe.get_out_query_vertices()) + .intersection(&pre_probe.borrow().get_out_query_vertices()) .map(|x| x.clone()) .collect(); let build_qvertex_to_idx_map = - get_op_attr_as_ref!(&pre_build, out_qvertex_to_idx_map).clone(); + get_op_attr_as_ref!(pre_build.borrow().deref(), out_qvertex_to_idx_map).clone(); let query_vertex_to_hash = &join_qvertices[0]; let build_hash_idx = build_qvertex_to_idx_map[query_vertex_to_hash].clone(); let mut build = Build::new( - get_op_attr_as_ref!(&pre_build, out_subgraph).clone(), + get_op_attr_as_ref!(pre_build.borrow().deref(), out_subgraph).clone(), query_vertex_to_hash.clone(), build_hash_idx.clone(), ); - build.base_op.prev.replace(Box::new(pre_build.clone())); - *get_op_attr_as_mut!(&mut pre_build, next) = vec![Operator::Build(build.clone())]; - build_subplans.push(Box::new(Operator::Build(build.clone()))); + build.base_op.prev.replace(pre_build.clone()); + let build = Rc::new(RefCell::new(Operator::Build(build))); + *get_op_attr_as_mut!(pre_build.borrow_mut().deref_mut(), next) = vec![build.clone()]; + build_subplans.push(build.clone()); - let out_subgraph_probe = get_op_attr_as_mut!(&mut pre_probe, out_subgraph).as_mut(); - let out_subgraph_build = get_op_attr_as_mut!(&mut pre_build, out_subgraph).as_mut(); - let mapping = out_subgraph_build.get_isomorphic_mapping_if_any(out_subgraph_probe); + let mapping = { + let mut pre_probe = pre_probe.borrow_mut(); + let mut pre_build = pre_build.borrow_mut(); + let out_subgraph_probe = get_op_attr_as_mut!(pre_probe.deref_mut(), out_subgraph); + let out_subgraph_build = get_op_attr_as_mut!(pre_build.deref_mut(), out_subgraph); + out_subgraph_build.get_isomorphic_mapping_if_any(out_subgraph_probe) + }; let mut probe_qvertex_to_idx_map; - if let Some(map) = mapping { + if let Some(map) = &mapping { probe_qvertex_to_idx_map = HashMap::new(); build_qvertex_to_idx_map .iter() @@ -77,7 +86,7 @@ impl HashJoin { ); } else { probe_qvertex_to_idx_map = - get_op_attr_as_ref!(&pre_probe, out_qvertex_to_idx_map).clone(); + get_op_attr_as_ref!(pre_probe.borrow().deref(), out_qvertex_to_idx_map).clone(); } let probe_hash_idx = probe_qvertex_to_idx_map[query_vertex_to_hash]; let out_qvertex_to_idx_map = HashJoin::compute_out_qvertex_to_idx_map( @@ -96,14 +105,12 @@ impl HashJoin { } build_indices[i - 1] = other_build_idx; } - let mut probe; - let in_subgraph = get_op_attr_as_ref!(&pre_probe, out_subgraph) - .as_ref() - .clone(); - let out_tuple_len = get_op_attr!(&pre_probe, out_tuple_len); - if let Some(map) = mapping { - if probe_indices.len() == 0 { - probe = Operator::Probe(Probe::PC(ProbeCartesian::new( + + let in_subgraph = get_op_attr_as_ref!(pre_probe.borrow().deref(), out_subgraph).clone(); + let out_tuple_len = get_op_attr!(pre_probe.borrow().deref(), out_tuple_len); + let mut probe = if let Some(map) = &mapping { + let probe = if probe_indices.len() == 0 { + Probe::PC(ProbeCartesian::new( out_subgraph, in_subgraph, join_qvertices, @@ -111,9 +118,9 @@ impl HashJoin { hashed_tuple_len, out_tuple_len, out_qvertex_to_idx_map, - ))); + )) } else { - probe = Operator::Probe(Probe::PMV(PMV::PMVC(ProbeMultiVerticesCartesian::new( + Probe::PMV(PMV::PMVC(ProbeMultiVerticesCartesian::new( out_subgraph, in_subgraph, join_qvertices, @@ -123,11 +130,12 @@ impl HashJoin { hashed_tuple_len, out_tuple_len, out_qvertex_to_idx_map, - )))); - } + ))) + }; + Rc::new(RefCell::new(Operator::Probe(probe))) } else { - if probe_indices.len() == 0 { - probe = Operator::Probe(Probe::BaseProbe(BaseProbe::new( + let probe = if probe_indices.len() == 0 { + Probe::BaseProbe(BaseProbe::new( out_subgraph, in_subgraph, join_qvertices, @@ -135,9 +143,9 @@ impl HashJoin { hashed_tuple_len, out_tuple_len, out_qvertex_to_idx_map, - ))); + )) } else { - probe = Operator::Probe(Probe::PMV(PMV::BasePMV(ProbeMultiVertices::new( + Probe::PMV(PMV::BasePMV(ProbeMultiVertices::new( out_subgraph, in_subgraph, join_qvertices, @@ -147,21 +155,25 @@ impl HashJoin { hashed_tuple_len, out_tuple_len, out_qvertex_to_idx_map, - )))); - } - get_op_attr_as_mut!(&mut probe, prev).replace(Box::new(pre_probe.clone())); - let probe_clone = probe.clone(); - *get_op_attr_as_mut!(&mut probe, next) = vec![probe_clone]; + ))) + }; + let probe = Rc::new(RefCell::new(Operator::Probe(probe))); + get_op_attr_as_mut!(probe.borrow_mut().deref_mut(), prev).replace(pre_probe.clone()); + *get_op_attr_as_mut!(probe.borrow_mut().deref_mut(), next) = vec![probe.clone()]; let last_index = probe_subplans.len() - 1; - probe_subplans[last_index] = Box::new(probe.clone()); + probe_subplans[last_index] = probe.clone(); + probe + }; + if let Operator::Build(build) = build.borrow_mut().deref_mut() { + build.probing_subgraph = + get_op_attr_as_ref!(probe.borrow().deref(), in_subgraph).clone(); } - build.probing_subgraph = get_op_attr_as_ref!(&probe, in_subgraph).clone(); let mut subplans = build_subplans.clone(); - if let None = mapping { + if let None = &mapping { subplans.append(&mut probe_subplans); } else { - subplans.push(Box::new(probe)); + subplans.push(probe); } subplans } diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/hash_table.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/hash_table.rs index cd70410c..f0069e3e 100644 --- a/src/graph_impl/multi_graph/plan/operator/hashjoin/hash_table.rs +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/hash_table.rs @@ -1,4 +1,3 @@ -use fxhash::hash; use generic::IdType; use itertools::Itertools; diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe.rs index a375e834..71de5803 100644 --- a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe.rs +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe.rs @@ -9,7 +9,10 @@ use graph_impl::multi_graph::plan::operator::hashjoin::probe_cartesian::ProbeCar use graph_impl::multi_graph::plan::operator::hashjoin::probe_multi_vertices::PMV; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; -use std::hash::{BuildHasherDefault, Hash}; +use std::cell::RefCell; +use std::hash::Hash; +use std::ops::{Deref, DerefMut}; +use std::rc::Rc; #[derive(Clone)] pub enum Probe { @@ -40,7 +43,7 @@ impl BaseProbe { out_qvertex_to_idx_map: HashMap, ) -> BaseProbe { let mut probe = BaseProbe { - base_op: BaseOperator::new(Box::new(out_subgraph), Some(Box::new(in_subgraph))), + base_op: BaseOperator::new(out_subgraph, Some(in_subgraph)), hash_tables: vec![], join_qvertices, probe_hash_idx, @@ -67,7 +70,8 @@ impl CommonOperatorTrait for BaseProbe { self.base_op .next .iter_mut() - .for_each(|next_op| next_op.init(probe_tuple.clone(), graph)); + .map(|next_op| next_op.borrow_mut()) + .for_each(|mut next_op| next_op.deref_mut().init(probe_tuple.clone(), graph)); } } @@ -98,7 +102,10 @@ impl CommonOperatorTrait for BaseProbe { offset += 1; } } - self.base_op.next[0].process_new_tuple(); + self.base_op.next[0] + .borrow_mut() + .deref_mut() + .process_new_tuple(); } } } @@ -119,8 +126,8 @@ impl CommonOperatorTrait for BaseProbe { fn copy(&self, is_thread_safe: bool) -> Operator { let op = &self.base_op; let mut probe = BaseProbe::new( - op.out_subgraph.as_ref().clone(), - op.in_subgraph.as_ref().unwrap().as_ref().clone(), + op.out_subgraph.clone(), + op.in_subgraph.as_ref().unwrap().clone(), self.join_qvertices.clone(), self.probe_hash_idx, self.hashed_tuple_len, @@ -130,13 +137,15 @@ impl CommonOperatorTrait for BaseProbe { probe.base_op.prev = op .prev .as_ref() - .map(|prev| Box::new(prev.copy(is_thread_safe))); - probe.base_op.next = vec![Operator::Probe(Probe::BaseProbe(probe.clone()))]; + .map(|prev| Rc::new(RefCell::new(prev.borrow().deref().copy(is_thread_safe)))); + probe.base_op.next = vec![Rc::new(RefCell::new(Operator::Probe(Probe::BaseProbe( + probe.clone(), + ))))]; Operator::Probe(Probe::BaseProbe(probe)) } - fn is_same_as(&mut self, op: &mut Operator) -> bool { - if let Operator::Probe(Probe::BaseProbe(probe)) = op { + fn is_same_as(&mut self, op: &mut Rc>>) -> bool { + if let Operator::Probe(Probe::BaseProbe(probe)) = op.borrow_mut().deref_mut() { let self_op = &mut self.base_op; let other_op = &mut probe.base_op; let in_subgraph = self_op.in_subgraph.as_mut().map_or(false, |in_subgraph| { @@ -144,10 +153,10 @@ impl CommonOperatorTrait for BaseProbe { }); let out_subgraph = self_op .out_subgraph - .as_mut() - .is_isomorphic_to(other_op.out_subgraph.as_mut()); + .is_isomorphic_to(&mut other_op.out_subgraph); let prev = self_op.prev.as_mut().map_or(false, |prev| { - prev.is_same_as(other_op.prev.as_mut().unwrap().as_mut()) + prev.borrow_mut() + .is_same_as(other_op.prev.as_mut().unwrap()) }); return in_subgraph && out_subgraph && prev; } @@ -212,7 +221,7 @@ impl CommonOperatorTrait for Probe { } } - fn is_same_as(&mut self, op: &mut Operator) -> bool { + fn is_same_as(&mut self, op: &mut Rc>>) -> bool { match self { Probe::BaseProbe(base) => base.is_same_as(op), Probe::PC(pc) => pc.is_same_as(op), diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_cartesian.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_cartesian.rs index a75ee165..4aceb5d3 100644 --- a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_cartesian.rs +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_cartesian.rs @@ -5,8 +5,10 @@ use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Ope use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; -use itertools::join; -use std::hash::{BuildHasherDefault, Hash}; +use std::cell::RefCell; +use std::hash::Hash; +use std::ops::DerefMut; +use std::rc::Rc; #[derive(Clone)] pub struct ProbeCartesian { @@ -111,8 +113,8 @@ impl CommonOperatorTrait for ProbeCartesian { fn copy(&self, is_thread_safe: bool) -> Operator { let op = &self.base_probe.base_op; Operator::Probe(Probe::PC(ProbeCartesian::new( - op.out_subgraph.as_ref().clone(), - op.in_subgraph.as_ref().unwrap().as_ref().clone(), + op.out_subgraph.clone(), + op.in_subgraph.as_ref().unwrap().clone(), self.base_probe.join_qvertices.clone(), self.base_probe.probe_hash_idx, self.base_probe.hashed_tuple_len, @@ -121,8 +123,8 @@ impl CommonOperatorTrait for ProbeCartesian { ))) } - fn is_same_as(&mut self, op: &mut Operator) -> bool { - if let Operator::Probe(Probe::PC(pc)) = op { + fn is_same_as(&mut self, op: &mut Rc>>) -> bool { + if let Operator::Probe(Probe::PC(pc)) = op.borrow_mut().deref_mut() { let self_op = &mut self.base_probe.base_op; let other_op = &mut pc.base_probe.base_op; let in_subgraph = self_op.in_subgraph.as_mut().map_or(false, |in_subgraph| { @@ -130,8 +132,7 @@ impl CommonOperatorTrait for ProbeCartesian { }); let out_subgraph = self_op .out_subgraph - .as_mut() - .is_isomorphic_to(other_op.out_subgraph.as_mut()); + .is_isomorphic_to(&mut other_op.out_subgraph); return in_subgraph && out_subgraph; } false diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices.rs index 8b9bc3d6..29fe78a5 100644 --- a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices.rs +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices.rs @@ -6,7 +6,10 @@ use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; use itertools::Itertools; -use std::hash::{BuildHasherDefault, Hash}; +use std::cell::RefCell; +use std::hash::Hash; +use std::ops::{Deref, DerefMut}; +use std::rc::Rc; #[derive(Clone)] pub enum PMV { @@ -115,7 +118,9 @@ impl CommonOperatorTrait for ProbeMultiVertices { out += 1; } } - self.base_probe.base_op.next[0].process_new_tuple(); + self.base_probe.base_op.next[0] + .borrow_mut() + .process_new_tuple(); } }); } @@ -137,13 +142,12 @@ impl CommonOperatorTrait for ProbeMultiVertices { fn copy(&self, is_thread_safe: bool) -> Operator { let mut probe = ProbeMultiVertices::new( - self.base_probe.base_op.out_subgraph.as_ref().clone(), + self.base_probe.base_op.out_subgraph.clone(), self.base_probe .base_op .in_subgraph .as_ref() .unwrap() - .as_ref() .clone(), self.base_probe.join_qvertices.clone(), self.base_probe.probe_hash_idx, @@ -158,14 +162,15 @@ impl CommonOperatorTrait for ProbeMultiVertices { .base_op .prev .as_ref() - .map(|prev| Box::new(prev.copy(is_thread_safe))); - probe.base_probe.base_op.next = - vec![Operator::Probe(Probe::PMV(PMV::BasePMV(probe.clone())))]; + .map(|prev| Rc::new(RefCell::new(prev.borrow().deref().copy(is_thread_safe)))); + probe.base_probe.base_op.next = vec![Rc::new(RefCell::new(Operator::Probe(Probe::PMV( + PMV::BasePMV(probe.clone()), + ))))]; Operator::Probe(Probe::PMV(PMV::BasePMV(probe))) } - fn is_same_as(&mut self, op: &mut Operator) -> bool { - if let Operator::Probe(Probe::PMV(PMV::BasePMV(probe))) = op { + fn is_same_as(&mut self, op: &mut Rc>>) -> bool { + if let Operator::Probe(Probe::PMV(PMV::BasePMV(probe))) = op.borrow_mut().deref_mut() { let self_op = &mut self.base_probe.base_op; let other_op = &mut probe.base_probe.base_op; let in_subgraph = self_op.in_subgraph.as_mut().map_or(false, |in_subgraph| { @@ -173,10 +178,10 @@ impl CommonOperatorTrait for ProbeMultiVertices { }); let out_subgraph = self_op .out_subgraph - .as_mut() - .is_isomorphic_to(other_op.out_subgraph.as_mut()); + .is_isomorphic_to(&mut other_op.out_subgraph); let prev = self_op.prev.as_mut().map_or(false, |prev| { - prev.is_same_as(other_op.prev.as_mut().unwrap().as_mut()) + prev.borrow_mut() + .is_same_as(other_op.prev.as_mut().unwrap()) }); return in_subgraph && out_subgraph && prev; } @@ -235,7 +240,7 @@ impl CommonOperatorTrait for PMV { } } - fn is_same_as(&mut self, op: &mut Operator) -> bool { + fn is_same_as(&mut self, op: &mut Rc>>) -> bool { match self { PMV::BasePMV(base) => base.is_same_as(op), PMV::PMVC(pmvc) => pmvc.is_same_as(op), diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices_cartesian.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices_cartesian.rs index fe2d5b11..7f6cb5af 100644 --- a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices_cartesian.rs +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices_cartesian.rs @@ -8,7 +8,10 @@ use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Ope use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; -use std::hash::{BuildHasherDefault, Hash}; +use std::cell::RefCell; +use std::hash::Hash; +use std::ops::DerefMut; +use std::rc::Rc; #[derive(Clone)] pub struct ProbeMultiVerticesCartesian { @@ -118,8 +121,8 @@ impl CommonOperatorTrait for ProbeMultiVerticesCartesian { fn copy(&self, is_thread_safe: bool) -> Operator { let op = &self.base_pmv.base_probe.base_op; Operator::Probe(Probe::PMV(PMV::PMVC(ProbeMultiVerticesCartesian::new( - op.out_subgraph.as_ref().clone(), - op.in_subgraph.as_ref().unwrap().as_ref().clone(), + op.out_subgraph.clone(), + op.in_subgraph.as_ref().unwrap().clone(), self.base_pmv.base_probe.join_qvertices.clone(), self.base_pmv.base_probe.probe_hash_idx, self.base_pmv.probe_indices.clone(), @@ -130,8 +133,8 @@ impl CommonOperatorTrait for ProbeMultiVerticesCartesian { )))) } - fn is_same_as(&mut self, op: &mut Operator) -> bool { - if let Operator::Probe(Probe::PMV(PMV::PMVC(pc))) = op { + fn is_same_as(&mut self, op: &mut Rc>>) -> bool { + if let Operator::Probe(Probe::PMV(PMV::PMVC(pc))) = op.borrow_mut().deref_mut() { let self_op = &mut self.base_pmv.base_probe.base_op; let other_op = &mut pc.base_pmv.base_probe.base_op; let in_subgraph = self_op.in_subgraph.as_mut().map_or(false, |in_subgraph| { @@ -139,8 +142,7 @@ impl CommonOperatorTrait for ProbeMultiVerticesCartesian { }); let out_subgraph = self_op .out_subgraph - .as_mut() - .is_isomorphic_to(other_op.out_subgraph.as_mut()); + .is_isomorphic_to(&mut other_op.out_subgraph); return in_subgraph && out_subgraph; } false diff --git a/src/graph_impl/multi_graph/plan/operator/operator.rs b/src/graph_impl/multi_graph/plan/operator/operator.rs index 202a05f3..d187134e 100644 --- a/src/graph_impl/multi_graph/plan/operator/operator.rs +++ b/src/graph_impl/multi_graph/plan/operator/operator.rs @@ -12,9 +12,9 @@ use graph_impl::multi_graph::planner::catalog::operator::noop::Noop; use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::TypedStaticGraph; use hashbrown::{HashMap, HashSet}; -use map::SetMap; -use std::hash::{BuildHasherDefault, Hash}; -use std::iter::FromIterator; +use std::cell::RefCell; +use std::hash::Hash; +use std::ops::Deref; use std::rc::Rc; /// Operator types @@ -33,12 +33,12 @@ pub enum Operator { #[derive(Clone)] pub struct BaseOperator { pub name: String, - pub next: Vec>, - pub prev: Option>>, + pub next: Vec>>>, + pub prev: Option>>>, pub probe_tuple: Vec, pub out_tuple_len: usize, - pub in_subgraph: Option>, - pub out_subgraph: Box, + pub in_subgraph: Option, + pub out_subgraph: QueryGraph, pub out_qvertex_to_idx_map: HashMap, pub last_repeated_vertex_idx: usize, pub num_out_tuples: usize, @@ -46,10 +46,7 @@ pub struct BaseOperator { } impl BaseOperator { - pub fn new( - out_subgraph: Box, - in_subgraph: Option>, - ) -> BaseOperator { + pub fn new(out_subgraph: QueryGraph, in_subgraph: Option) -> BaseOperator { BaseOperator { name: "".to_string(), next: vec![], @@ -73,7 +70,7 @@ impl BaseOperator { probe_tuple: vec![], out_tuple_len: 0, in_subgraph: None, - out_subgraph: Box::new(QueryGraph::empty()), + out_subgraph: QueryGraph::empty(), out_qvertex_to_idx_map: HashMap::new(), last_repeated_vertex_idx: 0, num_out_tuples: 0, @@ -94,18 +91,23 @@ pub trait CommonOperatorTrait { fn get_alds_as_string(&self) -> String; fn update_operator_name(&mut self, query_vertex_to_index_map: HashMap); fn copy(&self, is_thread_safe: bool) -> Operator; - fn is_same_as(&mut self, op: &mut Operator) -> bool; + fn is_same_as(&mut self, op: &mut Rc>>) -> bool; fn get_num_out_tuples(&self) -> usize; } impl Operator { - pub fn get_last_operators(&self, last_operators: &mut Vec>) { - if !get_op_attr_as_ref!(self, next).is_empty() { - for op in get_op_attr_as_ref!(self, next) { - op.get_last_operators(last_operators); + pub fn get_last_operators(&self, last_operators: &mut Vec>>>) { + let next = get_op_attr_as_ref!(self, next); + if next.is_empty() { + return; + } + for op in next { + let next = op.borrow(); + if get_op_attr_as_ref!(next.deref(), next).is_empty() { + last_operators.push(op.clone()); + continue; } - } else { - last_operators.push(self.clone()); + op.borrow().get_last_operators(last_operators); } } @@ -119,7 +121,8 @@ impl Operator { operator_metrics.push((name.clone(), icost, num_out_tuples)); get_op_attr_as_ref!(self, next) .iter() - .for_each(|op| match op { + .map(|op| op.borrow()) + .for_each(|op| match op.deref() { Operator::Sink(_) => {} _ => op.get_operator_metrics_next_operators(operator_metrics), }); @@ -130,7 +133,7 @@ impl Operator { Operator::EI(ei) => ei.has_multi_edge_extends(), _ => { if let Some(prev) = get_op_attr_as_ref!(self, prev) { - return prev.has_multi_edge_extends(); + return prev.borrow().deref().has_multi_edge_extends(); } false } @@ -158,7 +161,7 @@ impl CommonOperatorTrait for BaseOperator { fn execute(&mut self) { if let Some(prev) = self.prev.as_mut() { - prev.execute(); + prev.borrow_mut().execute(); } } @@ -174,7 +177,7 @@ impl CommonOperatorTrait for BaseOperator { panic!("unsupported operation exception") } - fn is_same_as(&mut self, op: &mut Operator) -> bool { + fn is_same_as(&mut self, op: &mut Rc>>) -> bool { panic!("unsupported operation exception") } @@ -260,7 +263,7 @@ impl CommonOperatorTrait for Operator { Operator::Noop(noop) => noop.copy(is_thread_safe), } } - fn is_same_as(&mut self, op: &mut Operator) -> bool { + fn is_same_as(&mut self, op: &mut Rc>>) -> bool { match self { Operator::Base(base) => base.is_same_as(op), Operator::Sink(sink) => sink.is_same_as(op), diff --git a/src/graph_impl/multi_graph/plan/operator/scan/scan.rs b/src/graph_impl/multi_graph/plan/operator/scan/scan.rs index eb9deeb2..3507257a 100644 --- a/src/graph_impl/multi_graph/plan/operator/scan/scan.rs +++ b/src/graph_impl/multi_graph/plan/operator/scan/scan.rs @@ -9,7 +9,9 @@ use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::static_graph::sorted_adj_vec::SortedAdjVec; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; -use std::hash::{BuildHasherDefault, Hash}; +use std::cell::RefCell; +use std::hash::Hash; +use std::ops::DerefMut; use std::rc::Rc; #[derive(Clone)] @@ -35,7 +37,7 @@ pub struct BaseScan { } impl BaseScan { - pub fn new(out_subgraph: Box) -> BaseScan { + pub fn new(out_subgraph: QueryGraph) -> BaseScan { let mut scan = BaseScan { base_op: BaseOperator::new(out_subgraph, None), from_query_vertex: "".to_string(), @@ -49,7 +51,7 @@ impl BaseScan { from_vertex_start_idx: 0, from_vertex_end_idx: 0, }; - let out_subgraph = scan.base_op.out_subgraph.as_ref(); + let out_subgraph = &scan.base_op.out_subgraph; if out_subgraph.q_edges.len() > 1 { panic!("IllegalArgumentException"); } @@ -93,8 +95,8 @@ impl CommonOperatorTrait for BaseScan { self.label_or_to_type = self.to_type; self.to_type = 0; } - for next_op in &mut self.base_op.next { - next_op.init(probe_tuple.clone(), graph); + for next_op in &self.base_op.next { + next_op.borrow_mut().init(probe_tuple.clone(), graph); } } @@ -119,7 +121,7 @@ impl CommonOperatorTrait for BaseScan { || self.vertex_types[self.base_op.probe_tuple[1].id()] == self.to_type { self.base_op.num_out_tuples += 1; - self.base_op.next[0].process_new_tuple(); + self.base_op.next[0].borrow_mut().process_new_tuple(); } } } @@ -133,10 +135,10 @@ impl CommonOperatorTrait for BaseScan { query_vertex_to_index_map = HashMap::new(); query_vertex_to_index_map.insert(self.from_query_vertex.clone(), 0); query_vertex_to_index_map.insert(self.to_query_vertex.clone(), 1); - self.base_op - .next - .iter_mut() - .for_each(|op| op.update_operator_name(query_vertex_to_index_map.clone())); + self.base_op.next.iter_mut().for_each(|op| { + op.borrow_mut() + .update_operator_name(query_vertex_to_index_map.clone()) + }); } fn copy(&self, is_thread_safe: bool) -> Operator { @@ -148,8 +150,8 @@ impl CommonOperatorTrait for BaseScan { Operator::Scan(Scan::Base(BaseScan::new(self.base_op.out_subgraph.clone()))) } - fn is_same_as(&mut self, op: &mut Operator) -> bool { - if let Operator::Scan(scan) = op { + fn is_same_as(&mut self, op: &mut Rc>>) -> bool { + if let Operator::Scan(scan) = op.borrow_mut().deref_mut() { return self.from_type == get_scan_as_ref!(scan).from_type && self.to_type == get_scan_as_ref!(scan).to_type && self.label_or_to_type == get_scan_as_ref!(scan).label_or_to_type; @@ -215,7 +217,7 @@ impl CommonOperatorTrait for Scan { } } - fn is_same_as(&mut self, op: &mut Operator) -> bool { + fn is_same_as(&mut self, op: &mut Rc>>) -> bool { match self { Scan::Base(base) => base.is_same_as(op), Scan::ScanSampling(ss) => ss.is_same_as(op), diff --git a/src/graph_impl/multi_graph/plan/operator/scan/scan_blocking.rs b/src/graph_impl/multi_graph/plan/operator/scan/scan_blocking.rs index ad07b64c..432485d8 100644 --- a/src/graph_impl/multi_graph/plan/operator/scan/scan_blocking.rs +++ b/src/graph_impl/multi_graph/plan/operator/scan/scan_blocking.rs @@ -4,7 +4,9 @@ use graph_impl::multi_graph::plan::operator::scan::scan::BaseScan; use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; -use std::hash::{BuildHasherDefault, Hash}; +use std::cell::RefCell; +use std::hash::Hash; +use std::rc::Rc; static PARTITION_SIZE: usize = 100; @@ -28,7 +30,7 @@ pub struct ScanBlocking { } impl ScanBlocking { - pub fn new(out_subgraph: Box) -> ScanBlocking { + pub fn new(out_subgraph: QueryGraph) -> ScanBlocking { ScanBlocking { base_scan: BaseScan::new(out_subgraph), curr_from_idx: 0, @@ -63,10 +65,10 @@ impl ScanBlocking { .as_mut() .map_or(0, |adj| adj.get_offsets()[label + 1]); if self.to_idx_limit + num_edges_left <= to_limit - 1 { - self.to_idx_limit += (num_edges_left - 1); + self.to_idx_limit += num_edges_left - 1; num_edges_left = 0; } else { - num_edges_left -= (to_limit - 1 - self.to_idx_limit); + num_edges_left -= to_limit - 1 - self.to_idx_limit; self.to_idx_limit = to_limit; if self.from_idx_limit == self.highest_from_idx - 1 { break; @@ -91,7 +93,7 @@ impl ScanBlocking { .unwrap() .get_neighbor_id(Id::new(to_idx)); base_op.num_out_tuples += 1; - base_op.next[0].process_new_tuple(); + base_op.next[0].borrow_mut().process_new_tuple(); } } @@ -115,7 +117,9 @@ impl ScanBlocking { == self.base_scan.to_type { self.base_scan.base_op.num_out_tuples += 1; - self.base_scan.base_op.next[0].process_new_tuple(); + self.base_scan.base_op.next[0] + .borrow_mut() + .process_new_tuple(); } } } @@ -150,8 +154,8 @@ impl CommonOperatorTrait for ScanBlocking { self.base_scan .base_op .next - .iter_mut() - .for_each(|next_op| next_op.init(probe_tuple.clone(), graph)); + .iter() + .for_each(|next_op| next_op.borrow_mut().init(probe_tuple.clone(), graph)); } fn process_new_tuple(&mut self) { @@ -197,7 +201,7 @@ impl CommonOperatorTrait for ScanBlocking { self.base_scan.copy(is_thread_safe) } - fn is_same_as(&mut self, op: &mut Operator) -> bool { + fn is_same_as(&mut self, op: &mut Rc>>) -> bool { self.base_scan.is_same_as(op) } diff --git a/src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs b/src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs index 36b96780..e18b6eed 100644 --- a/src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs +++ b/src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs @@ -7,7 +7,8 @@ use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; use rand::{thread_rng, Rng}; -use std::hash::{BuildHasherDefault, Hash}; +use std::cell::RefCell; +use std::hash::Hash; use std::rc::Rc; #[derive(Clone)] @@ -17,7 +18,7 @@ pub struct ScanSampling { } impl ScanSampling { - pub fn new(out_subgraph: Box) -> ScanSampling { + pub fn new(out_subgraph: QueryGraph) -> ScanSampling { Self { base_scan: BaseScan::new(out_subgraph), edges_queue: vec![], @@ -74,8 +75,8 @@ impl CommonOperatorTrait for ScanSampling { ) { if self.base_scan.base_op.probe_tuple.is_empty() { self.base_scan.base_op.probe_tuple = probe_tuple.clone(); - self.base_scan.base_op.next.iter_mut().for_each(|next_op| { - next_op.init(probe_tuple.clone(), graph); + self.base_scan.base_op.next.iter().for_each(|next_op| { + next_op.borrow_mut().init(probe_tuple.clone(), graph); }); } } @@ -91,7 +92,7 @@ impl CommonOperatorTrait for ScanSampling { self.base_scan.base_op.probe_tuple[1] = edge[0]; self.base_scan.base_op.num_out_tuples += 1; for next_op in &mut self.base_scan.base_op.next { - next_op.process_new_tuple(); + next_op.borrow_mut().process_new_tuple(); } } } @@ -111,7 +112,7 @@ impl CommonOperatorTrait for ScanSampling { Operator::Scan(Scan::ScanSampling(scan_sampling)) } - fn is_same_as(&mut self, op: &mut Operator) -> bool { + fn is_same_as(&mut self, op: &mut Rc>>) -> bool { self.base_scan.is_same_as(op) } diff --git a/src/graph_impl/multi_graph/plan/operator/sink/sink.rs b/src/graph_impl/multi_graph/plan/operator/sink/sink.rs index b1b962e9..ea6b1940 100644 --- a/src/graph_impl/multi_graph/plan/operator/sink/sink.rs +++ b/src/graph_impl/multi_graph/plan/operator/sink/sink.rs @@ -14,7 +14,9 @@ use graph_impl::multi_graph::plan::operator::sink::sink_print::SinkPrint; use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; -use std::hash::{BuildHasherDefault, Hash}; +use std::cell::RefCell; +use std::hash::Hash; +use std::ops::{Deref, DerefMut}; use std::rc::Rc; #[derive(Clone)] @@ -36,14 +38,14 @@ pub enum Sink { #[derive(Clone)] pub struct BaseSink { pub base_op: BaseOperator, - pub previous: Option>>, + pub previous: Vec>>>, } impl BaseSink { - pub fn new(query_graph: Box) -> Self { + pub fn new(query_graph: QueryGraph) -> Self { Self { base_op: BaseOperator::new(query_graph.clone(), Some(query_graph)), - previous: None, + previous: vec![], } } } @@ -60,10 +62,10 @@ impl CommonOperatorTrait for BaseSink { fn process_new_tuple(&mut self) {} fn execute(&mut self) { - if let Some(prev) = self.previous.as_mut() { - prev[0].execute(); + if !self.previous.is_empty() { + self.previous[0].borrow_mut().execute(); } else { - self.base_op.prev.as_mut().unwrap().execute(); + self.base_op.prev.as_mut().unwrap().borrow_mut().execute(); } } @@ -78,27 +80,32 @@ impl CommonOperatorTrait for BaseSink { fn copy(&self, is_thread_safe: bool) -> Operator { let mut sink = BaseSink::new(self.base_op.out_subgraph.clone()); if let Some(prev) = &self.base_op.prev { - sink.base_op.prev = Some(Box::new(prev.copy(is_thread_safe))); + sink.base_op.prev = Some(Rc::new(RefCell::new( + prev.borrow().deref().copy(is_thread_safe), + ))); } else { sink.base_op.prev = None; } Operator::Sink(Sink::BaseSink(sink)) } - fn is_same_as(&mut self, op: &mut Operator) -> bool { - if let Operator::Sink(sink) = op { + fn is_same_as(&mut self, op: &mut Rc>>) -> bool { + if let Operator::Sink(sink) = op.borrow().deref() { if let Some(prev) = &mut self.base_op.prev { - let op_prev = get_op_attr_as_mut!(op, prev).as_mut().unwrap(); - return prev.is_same_as(op_prev.as_mut()); + let mut op = op.borrow_mut(); + let op_prev = get_op_attr_as_mut!(op.deref_mut(), prev).as_mut().unwrap(); + return prev.borrow_mut().is_same_as(op_prev); } } false } fn get_num_out_tuples(&self) -> usize { - if let Some(prev) = &self.previous { - prev.iter() - .map(|op| get_op_attr_as_ref!(op, num_out_tuples)) + if !self.previous.is_empty() { + self.previous + .iter() + .map(|op| op.borrow()) + .map(|op| get_op_attr!(op.deref(), num_out_tuples)) .sum() } else { self.base_op.num_out_tuples @@ -165,7 +172,7 @@ impl CommonOperatorTrait for Sink { } } - fn is_same_as(&mut self, op: &mut Operator) -> bool { + fn is_same_as(&mut self, op: &mut Rc>>) -> bool { match self { Sink::BaseSink(base) => base.is_same_as(op), Sink::SinkCopy(sc) => sc.is_same_as(op), diff --git a/src/graph_impl/multi_graph/plan/operator/sink/sink_copy.rs b/src/graph_impl/multi_graph/plan/operator/sink/sink_copy.rs index b63dd39b..6d33a782 100644 --- a/src/graph_impl/multi_graph/plan/operator/sink/sink_copy.rs +++ b/src/graph_impl/multi_graph/plan/operator/sink/sink_copy.rs @@ -4,8 +4,10 @@ use graph_impl::multi_graph::plan::operator::sink::sink::{BaseSink, Sink}; use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; -use std::hash::{BuildHasherDefault, Hash}; -use std::io::sink; +use std::cell::RefCell; +use std::hash::Hash; +use std::ops::Deref; +use std::rc::Rc; #[derive(Clone)] pub struct SinkCopy { @@ -14,7 +16,7 @@ pub struct SinkCopy { } impl SinkCopy { - pub fn new(query_graph: Box, out_tuple_length: usize) -> SinkCopy { + pub fn new(query_graph: QueryGraph, out_tuple_length: usize) -> SinkCopy { let mut sink_copy = SinkCopy { base_sink: BaseSink::new(query_graph), output_tuple: vec![], @@ -57,11 +59,13 @@ impl CommonOperatorTrait for SinkCopy { let base_op = &self.base_sink.base_op; let mut sink = SinkCopy::new(base_op.out_subgraph.clone(), base_op.out_tuple_len); let origin_prev = base_op.prev.as_ref().unwrap(); - sink.base_sink.base_op.prev = Some(Box::new(origin_prev.copy(is_thread_safe))); + sink.base_sink.base_op.prev = Some(Rc::new(RefCell::new( + origin_prev.borrow().deref().copy(is_thread_safe), + ))); Operator::Sink(Sink::SinkCopy(sink)) } - fn is_same_as(&mut self, op: &mut Operator) -> bool { + fn is_same_as(&mut self, op: &mut Rc>>) -> bool { self.base_sink.is_same_as(op) } diff --git a/src/graph_impl/multi_graph/plan/operator/sink/sink_limit.rs b/src/graph_impl/multi_graph/plan/operator/sink/sink_limit.rs index 92bab78c..4f423818 100644 --- a/src/graph_impl/multi_graph/plan/operator/sink/sink_limit.rs +++ b/src/graph_impl/multi_graph/plan/operator/sink/sink_limit.rs @@ -10,7 +10,10 @@ use graph_impl::multi_graph::plan::operator::sink::sink::{BaseSink, Sink}; use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; -use std::hash::{BuildHasherDefault, Hash}; +use std::cell::RefCell; +use std::hash::Hash; +use std::ops::Deref; +use std::rc::Rc; use std::time::SystemTime; #[derive(Clone)] @@ -22,7 +25,7 @@ pub struct SinkLimit { } impl SinkLimit { - pub fn new(query_graph: Box, out_tuple_limit: usize) -> SinkLimit { + pub fn new(query_graph: QueryGraph, out_tuple_limit: usize) -> SinkLimit { SinkLimit { base_sink: BaseSink::new(query_graph), start_time: SystemTime::now(), @@ -42,8 +45,8 @@ impl CommonOperatorTrait for SinkLimit { } fn process_new_tuple(&mut self) { - let prev = self.base_sink.base_op.prev.as_ref().unwrap().as_ref(); - if get_op_attr!(prev, num_out_tuples) >= self.out_tuples_limit { + let prev = self.base_sink.base_op.prev.as_ref().unwrap().borrow(); + if get_op_attr!(prev.deref(), num_out_tuples) >= self.out_tuples_limit { self.elapsed_time = SystemTime::now() .duration_since(self.start_time.clone()) .unwrap() @@ -68,7 +71,7 @@ impl CommonOperatorTrait for SinkLimit { self.base_sink.copy(is_thread_safe) } - fn is_same_as(&mut self, op: &mut Operator) -> bool { + fn is_same_as(&mut self, op: &mut Rc>>) -> bool { self.base_sink.is_same_as(op) } diff --git a/src/graph_impl/multi_graph/plan/operator/sink/sink_print.rs b/src/graph_impl/multi_graph/plan/operator/sink/sink_print.rs index c4f107a5..8d49fb53 100644 --- a/src/graph_impl/multi_graph/plan/operator/sink/sink_print.rs +++ b/src/graph_impl/multi_graph/plan/operator/sink/sink_print.rs @@ -4,7 +4,10 @@ use graph_impl::multi_graph::plan::operator::sink::sink::{BaseSink, Sink}; use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; -use std::hash::{BuildHasherDefault, Hash}; +use std::cell::RefCell; +use std::hash::Hash; +use std::ops::Deref; +use std::rc::Rc; #[derive(Clone)] pub struct SinkPrint { @@ -12,7 +15,7 @@ pub struct SinkPrint { } impl SinkPrint { - pub fn new(query_graph: Box) -> SinkPrint { + pub fn new(query_graph: QueryGraph) -> SinkPrint { SinkPrint { base_sink: BaseSink::new(query_graph), } @@ -49,11 +52,13 @@ impl CommonOperatorTrait for SinkPrint { let base_op = &self.base_sink.base_op; let mut sink = SinkPrint::new(base_op.out_subgraph.clone()); let origin_prev = base_op.prev.as_ref().unwrap(); - sink.base_sink.base_op.prev = Some(Box::new(origin_prev.copy(is_thread_safe))); + sink.base_sink.base_op.prev = Some(Rc::new(RefCell::new( + origin_prev.borrow().deref().copy(is_thread_safe), + ))); Operator::Sink(Sink::SinkPrint(sink)) } - fn is_same_as(&mut self, op: &mut Operator) -> bool { + fn is_same_as(&mut self, op: &mut Rc>>) -> bool { self.base_sink.is_same_as(op) } diff --git a/src/graph_impl/multi_graph/plan/query_plan.rs b/src/graph_impl/multi_graph/plan/query_plan.rs index 9df3dc77..d31733ab 100644 --- a/src/graph_impl/multi_graph/plan/query_plan.rs +++ b/src/graph_impl/multi_graph/plan/query_plan.rs @@ -17,17 +17,18 @@ use graph_impl::multi_graph::plan::operator::sink::sink_print::SinkPrint; use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; +use std::cell::RefCell; use std::hash::Hash; -use std::ops::Deref; +use std::ops::{Deref, DerefMut}; use std::rc::Rc; use std::time::SystemTime; #[derive(Clone)] pub struct QueryPlan { - sink: Option>, + pub sink: Option>>>, pub sink_type: SinkType, scan_sampling: Option>, - pub last_operator: Option>>, + pub last_operator: Option>>>, pub out_tuples_limit: usize, pub elapsed_time: u128, pub icost: usize, @@ -36,7 +37,7 @@ pub struct QueryPlan { pub operator_metrics: Vec<(String, usize, usize)>, executed: bool, adaptive_enabled: bool, - pub subplans: Vec>>, + pub subplans: Vec>>>, pub estimated_icost: f64, pub estimated_num_out_tuples: f64, pub q_vertex_to_num_out_tuples: HashMap, @@ -47,15 +48,23 @@ impl QueryPlan { let mut last_operators = Vec::new(); let scan_sampling_op = Operator::Scan(Scan::ScanSampling(scan_sampling.clone())); scan_sampling_op.get_last_operators(&mut last_operators); - let op = &last_operators[0]; - let out_subgraph = Box::new(get_op_attr_as_ref!(op, out_subgraph).as_ref().clone()); - let mut sink = BaseSink::new(out_subgraph); + + let out_subgraph = { + let op = last_operators[0].borrow(); + get_op_attr_as_ref!(op.deref(), out_subgraph).clone() + }; + let mut sink = Rc::new(RefCell::new(Operator::Sink(Sink::BaseSink(BaseSink::new( + out_subgraph, + ))))); for op in last_operators.iter_mut() { - *get_op_attr_as_mut!(op, next) = vec![Operator::Sink(Sink::BaseSink(sink.clone()))]; + let mut op = op.borrow_mut(); + *get_op_attr_as_mut!(op.deref_mut(), next) = vec![sink.clone()]; + } + if let Operator::Sink(Sink::BaseSink(sink)) = sink.borrow_mut().deref_mut() { + sink.previous = last_operators.clone(); } - sink.previous = Some(last_operators); Self { - sink: Some(Sink::BaseSink(sink)), + sink: Some(sink), sink_type: SinkType::Counter, scan_sampling: Some(scan_sampling), last_operator: None, @@ -73,7 +82,7 @@ impl QueryPlan { q_vertex_to_num_out_tuples: HashMap::new(), } } - pub fn new_from_operator(last_operator: Box>) -> Self { + pub fn new_from_operator(last_operator: Rc>>) -> Self { Self { sink: None, sink_type: SinkType::Counter, @@ -93,7 +102,7 @@ impl QueryPlan { q_vertex_to_num_out_tuples: HashMap::new(), } } - pub fn new_from_subplans(subplans: Vec>>) -> Self { + pub fn new_from_subplans(subplans: Vec>>>) -> Self { Self { sink: None, sink_type: SinkType::Copy, @@ -118,7 +127,8 @@ impl QueryPlan { let op = get_scan_as_ref!(&last_operator); map.insert(op.from_query_vertex.clone(), estimated_num_out_tuples); map.insert(op.to_query_vertex.clone(), estimated_num_out_tuples); - let mut plan = QueryPlan::new_from_operator(Box::new(Operator::Scan(last_operator))); + let mut plan = + QueryPlan::new_from_operator(Rc::new(RefCell::new(Operator::Scan(last_operator)))); plan.estimated_num_out_tuples = estimated_num_out_tuples; map.into_iter().for_each(|(k, v)| { plan.q_vertex_to_num_out_tuples.insert(k, v); @@ -130,26 +140,22 @@ impl QueryPlan { self.scan_sampling.as_mut() } - pub fn get_sink(&mut self) -> &mut Sink { - self.sink.as_mut().unwrap() - } - - pub fn get_sink_as_ref(&self) -> &Sink { - self.sink.as_ref().unwrap() - } - pub fn shallow_copy(&self) -> QueryPlan { QueryPlan::new_from_subplans(self.subplans.clone()) } - pub fn append(&mut self, mut new_operator: Operator) { - let mut last_operator = self.last_operator.as_mut().unwrap().as_mut(); - let last_op = get_base_op_as_mut!(&mut last_operator); - last_op.next = vec![new_operator.clone()]; - let new_op = get_base_op_as_mut!(&mut new_operator); - new_op.prev = self.last_operator.clone(); - self.subplans.push(Box::new(new_operator.clone())); - self.last_operator = Some(Box::new(new_operator)); + pub fn append(&mut self, mut new_operator: Rc>>) { + { + let mut last_operator = self.last_operator.as_mut().unwrap().borrow_mut(); + *get_op_attr_as_mut!(last_operator.deref_mut(), next) = vec![new_operator.clone()]; + } + { + let mut new_op = new_operator.borrow_mut(); + *get_op_attr_as_mut!(new_op.deref_mut(), prev) = + self.last_operator.as_ref().map(|op| op.clone()); + } + self.subplans.push(new_operator.clone()); + self.last_operator = Some(new_operator); } pub fn get_output_log(&mut self) -> String { @@ -180,14 +186,25 @@ impl QueryPlan { pub fn set_stats(&mut self) { for subplan in &self.subplans { - let mut first_operator = subplan.as_ref(); - while get_op_attr_as_ref!(first_operator, prev).is_some() { - first_operator = get_op_attr_as_ref!(first_operator, prev) - .as_ref() - .unwrap() - .as_ref(); + let mut first_op = subplan.clone(); + loop { + { + let first_op_ref = first_op.borrow(); + if get_op_attr_as_ref!(first_op_ref.deref(), prev).is_none() { + break; + } + } + first_op = { + let first_op_ref = first_op.borrow(); + get_op_attr_as_ref!(first_op_ref.deref(), prev) + .as_ref() + .unwrap() + .clone() + } } - first_operator.get_operator_metrics_next_operators(&mut self.operator_metrics); + first_op + .borrow() + .get_operator_metrics_next_operators(&mut self.operator_metrics); } for i in 0..self.operator_metrics.len() - 1 { self.icost += self.operator_metrics[i].1; @@ -199,7 +216,9 @@ impl QueryPlan { pub fn copy(&self, is_thread_safe: bool) -> QueryPlan { let mut subplans = vec![]; for subplan in &self.subplans { - subplans.push(Box::new(subplan.copy(is_thread_safe))); + subplans.push(Rc::new(RefCell::new( + subplan.borrow().deref().copy(is_thread_safe), + ))); } QueryPlan::new_from_subplans(subplans) } @@ -209,65 +228,97 @@ impl QueryPlan { graph: &TypedStaticGraph, ) { let plan_size = self.subplans.len(); - let last_operator = self.subplans[plan_size - 1].as_ref(); - let query_graph = get_op_attr_as_ref!(last_operator, out_subgraph); - let mut sink = match self.sink_type { - SinkType::Copy => Operator::Sink(Sink::SinkCopy(SinkCopy::new( - query_graph.clone(), - get_op_attr!(last_operator, out_tuple_len), - ))), - SinkType::Print => Operator::Sink(Sink::SinkPrint(SinkPrint::new(query_graph.clone()))), - SinkType::Limit => Operator::Sink(Sink::SinkLimit(SinkLimit::new( - query_graph.clone(), - self.out_tuples_limit, - ))), - SinkType::Counter => Operator::Sink(Sink::BaseSink(BaseSink::new(query_graph.clone()))), + let last_operator = self.subplans[plan_size - 1].clone(); + let query_graph = { + let last_operator = last_operator.borrow(); + get_op_attr_as_ref!(last_operator.deref(), out_subgraph).clone() }; - let sink_prev = get_op_attr_as_mut!(&mut sink, prev); - sink_prev.replace(Box::new(last_operator.clone())); - *get_op_attr_as_mut!(self.subplans[plan_size - 1].as_mut(), next) = vec![sink]; + let mut sink = Rc::new(RefCell::new(Operator::Sink(match self.sink_type { + SinkType::Copy => Sink::SinkCopy(SinkCopy::new( + query_graph.clone(), + get_op_attr!(last_operator.borrow().deref(), out_tuple_len), + )), + SinkType::Print => Sink::SinkPrint(SinkPrint::new(query_graph.clone())), + SinkType::Limit => { + Sink::SinkLimit(SinkLimit::new(query_graph.clone(), self.out_tuples_limit)) + } + SinkType::Counter => Sink::BaseSink(BaseSink::new(query_graph.clone())), + }))); + { + let mut sink = sink.borrow_mut(); + get_op_attr_as_mut!(sink.deref_mut(), prev).replace(last_operator.clone()); + } + *get_op_attr_as_mut!(self.subplans[plan_size - 1].borrow_mut().deref_mut(), next) = + vec![sink]; for subplan in &mut self.subplans { - if let Operator::Build(build) = subplan.as_mut() { + if let Operator::Build(build) = subplan.borrow_mut().deref_mut() { let hash_table = HashTable::new(build.build_hash_idx, build.hashed_tuple_len); build.hash_table = Some(hash_table.clone()); } } let sub_plans = self.subplans.clone(); for subplan in sub_plans { - if let Operator::Build(build) = subplan.as_ref() { + if let Operator::Build(build) = subplan.borrow().deref() { let hash_table = build.hash_table.as_ref().unwrap(); - let build_insubgrpah = get_op_attr_as_ref!(subplan.as_ref(), in_subgraph) + let subplan = subplan.borrow(); + let build_insubgrpah = get_op_attr_as_ref!(subplan.deref(), in_subgraph) .as_ref() - .unwrap() - .as_ref(); + .unwrap(); self.init_hashtable(build_insubgrpah, hash_table); } } for subplan in &mut self.subplans { let probe_tuple = vec![]; - let mut first_op = subplan.as_mut(); - while get_op_attr_as_mut!(first_op, prev).is_some() { - first_op = get_op_attr_as_mut!(first_op, prev) - .as_mut() - .unwrap() - .as_mut(); + let mut first_op = subplan.clone(); + loop { + { + let first_op_ref = first_op.borrow(); + if get_op_attr_as_ref!(first_op_ref.deref(), prev).is_none() { + break; + } + } + first_op = { + let mut first_op_mut = first_op.borrow_mut(); + get_op_attr_as_mut!(first_op_mut.deref_mut(), prev) + .as_mut() + .unwrap() + .clone() + }; } - first_op.init(probe_tuple, graph); + first_op.borrow_mut().deref_mut().init(probe_tuple, graph); } } fn init_hashtable(&mut self, build_insubgrpah: &QueryGraph, hash_table: &HashTable) { for operator in &mut self.subplans { - if let Operator::Probe(p) = operator.as_mut() { - if Self::check_and_init(build_insubgrpah, operator, hash_table.clone()) { + let mut op_mut = operator.borrow_mut(); + if let Operator::Probe(_p) = op_mut.deref_mut() { + if Self::check_and_init(build_insubgrpah, op_mut.deref_mut(), hash_table.clone()) { break; } } - let mut op = operator.as_mut(); - while get_op_attr_as_mut!(op, prev).is_some() { - op = get_op_attr_as_mut!(op, prev).as_mut().unwrap(); - if let Operator::Probe(p) = op { - if Self::check_and_init(build_insubgrpah, op, hash_table.clone()) { + let mut op = operator.clone(); + loop { + { + let op_ref = op.borrow(); + if get_op_attr_as_ref!(op_ref.deref(), prev).is_none() { + break; + } + } + op = { + let op_ref = op.borrow(); + get_op_attr_as_ref!(op_ref.deref(), prev) + .as_ref() + .unwrap() + .clone() + }; + let mut op_mut = op.borrow_mut(); + if let Operator::Probe(p) = op_mut.deref_mut() { + if Self::check_and_init( + build_insubgrpah, + op_mut.deref_mut(), + hash_table.clone(), + ) { return; } } @@ -280,10 +331,7 @@ impl QueryPlan { probe: &mut Operator, hash_table: HashTable, ) -> bool { - let prob_insubgraph = get_op_attr_as_ref!(probe, in_subgraph) - .as_ref() - .unwrap() - .as_ref(); + let prob_insubgraph = get_op_attr_as_ref!(probe, in_subgraph).as_ref().unwrap(); if prob_insubgraph == build_insubgrpah { if let Operator::Probe(probe_op) = probe { let mut base_probe = get_probe_as_mut!(probe_op); @@ -296,20 +344,34 @@ impl QueryPlan { pub fn execute(&mut self) { if let SinkType::Limit = self.sink_type { - if let Sink::SinkLimit(sink) = self.sink.as_mut().unwrap() { + if let Operator::Sink(Sink::SinkLimit(sink)) = + self.sink.as_mut().unwrap().borrow_mut().deref_mut() + { sink.start_time = SystemTime::now(); - self.subplans.iter_mut().for_each(|plan| plan.execute()); + self.subplans + .iter_mut() + .map(|plan| plan.borrow_mut()) + .for_each(|mut plan| plan.execute()); self.elapsed_time = sink.elapsed_time; } } else { let start_time = SystemTime::now(); - self.subplans.iter_mut().for_each(|plan| plan.execute()); + self.subplans + .iter_mut() + .map(|plan| plan.borrow_mut()) + .for_each(|mut plan| plan.execute()); self.elapsed_time = SystemTime::now() .duration_since(start_time) .unwrap() .as_millis(); } self.executed = true; - self.num_out_tuples = self.sink.as_ref().unwrap().get_num_out_tuples(); + self.num_out_tuples = self + .sink + .as_ref() + .unwrap() + .borrow() + .deref() + .get_num_out_tuples(); } } diff --git a/src/graph_impl/multi_graph/plan/query_plan_worker.rs b/src/graph_impl/multi_graph/plan/query_plan_worker.rs index eca7125a..055ea56b 100644 --- a/src/graph_impl/multi_graph/plan/query_plan_worker.rs +++ b/src/graph_impl/multi_graph/plan/query_plan_worker.rs @@ -11,8 +11,7 @@ use graph_impl::multi_graph::plan::operator::sink::sink::Sink; use graph_impl::multi_graph::plan::query_plan::QueryPlan; use graph_impl::TypedStaticGraph; use std::hash::Hash; -use std::thread; -use std::thread::JoinHandle; +use std::ops::{Deref, DerefMut}; use std::time::SystemTime; pub struct QPWorkers { @@ -47,11 +46,24 @@ impl QPWorkers { }; for query_plan in &mut worker.query_plans { for last_op in &mut query_plan.subplans { - let mut op = last_op.as_mut(); - while get_op_attr_as_ref!(op, prev).is_some() { - op = get_op_attr_as_mut!(op, prev).as_mut().unwrap().as_mut(); + let mut op = last_op.clone(); + loop { + { + let op_ref = op.borrow(); + if get_op_attr_as_ref!(op.borrow().deref(), prev).is_none() { + break; + } + } + op = { + let op_ref = op.borrow(); + get_op_attr_as_ref!(op_ref.deref(), prev) + .as_ref() + .unwrap() + .clone() + }; } - if let Operator::Scan(Scan::ScanBlocking(sb)) = op { + let mut op_mut = op.borrow_mut(); + if let Operator::Scan(Scan::ScanBlocking(sb)) = op_mut.deref_mut() { //TODO:Lock need to be fixed sb.global_vertices_idx_limits = global_vertex_idx_limits.clone(); } @@ -76,16 +88,17 @@ impl QPWorkers { self.elapsed_time = self.query_plans[0].elapsed_time; } else { let begin_time = SystemTime::now(); - let mut workers = vec![]; + // let mut workers = vec![]; for plan in &self.query_plans { let mut plan = plan.clone(); - workers.push(thread::spawn(move || { - plan.execute(); - })); - } - for worker in workers { - worker.join(); + // workers.push(thread::spawn(move || { + // plan.execute(); + // })); + plan.execute(); } + // for worker in workers { + // worker.join(); + // } self.elapsed_time = SystemTime::now() .duration_since(begin_time) .unwrap() diff --git a/src/graph_impl/multi_graph/planner/catalog/catalog.rs b/src/graph_impl/multi_graph/planner/catalog/catalog.rs index 4f76ee21..cb27c360 100644 --- a/src/graph_impl/multi_graph/planner/catalog/catalog.rs +++ b/src/graph_impl/multi_graph/planner/catalog/catalog.rs @@ -18,16 +18,11 @@ use graph_impl::multi_graph::planner::catalog::catalog_plans::{ use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::TypedStaticGraph; use hashbrown::{HashMap, HashSet}; -use indexmap::Equivalent; use itertools::Itertools; use std::cell::RefCell; use std::hash::Hash; -use std::iter::FromIterator; -use std::mem::replace; use std::ops::Deref; -use std::panic::catch_unwind; -use std::ptr::null; -use std::thread; +use std::rc::Rc; use std::time::SystemTime; pub static SINGLE_VERTEX_WEIGHT_PROBE_COEF: f64 = 3.0; @@ -154,13 +149,13 @@ impl Catalog { let it = in_subgraph.get_subgraph_mapping_iterator(sub_graph); while it.has_next() { let vertex_mapping = it.next().unwrap(); - let new_num_alds_matched = self.get_num_alds_matched(&alds, vertex_mapping); + let new_num_alds_matched = self.get_num_alds_matched(&alds, &vertex_mapping); if new_num_alds_matched == 0 || new_num_alds_matched < num_alds_matched { continue; } let selectivity_map = &self.sampled_selectivity[&i]; let sampled_selectivity = selectivity_map - [&self.get_alds_as_str(&alds, Some(vertex_mapping), Some(to_type))] + [&self.get_alds_as_str(&alds, Some(&vertex_mapping), Some(to_type))] .clone(); if new_num_alds_matched > num_alds_matched || sampled_selectivity < approx_selectivity @@ -184,11 +179,11 @@ impl Catalog { let mut from_qvertices_and_dirs = alds .iter() .map(|ald| { - let tail = (") ".to_owned() + let tail = ") ".to_owned() + &ald.direction.to_string() + "[" + &ald.label.to_string() - + "]"); + + "]"; if vertex_mapping.is_none() { return Some("(".to_owned() + &ald.from_query_vertex + &tail); } else { @@ -227,6 +222,7 @@ impl Catalog { .count() } + ///TODO: Multi thread catalog building pub fn populate( &mut self, graph: &TypedStaticGraph, @@ -272,18 +268,19 @@ impl Catalog { fn execute(&self, query_plan_arr: &mut Vec>) { if query_plan_arr.len() > 1 { - let mut handlers = vec![]; + // let mut handlers = vec![]; for i in 0..query_plan_arr.len() { - let mut sink = query_plan_arr[i].get_sink().clone(); - handlers.push(thread::spawn(move || { - sink.execute(); - })); - } - for handler in handlers { - handler.join(); + let mut sink = query_plan_arr[i].sink.as_mut().unwrap().borrow_mut(); + sink.execute(); + // handlers.push(thread::spawn(move || { + // sink.execute(); + // })); } + // for handler in handlers { + // handler.join(); + // } } else { - let sink = query_plan_arr[0].get_sink(); + let mut sink = query_plan_arr[0].sink.as_mut().unwrap().borrow_mut(); sink.execute(); } } @@ -293,20 +290,33 @@ impl Catalog { graph: &TypedStaticGraph, query_plan_arr: &mut Vec>, ) { - let mut other: Vec<&Operator> = query_plan_arr + let mut other: Vec>>> = query_plan_arr .iter_mut() + .map(|plan| plan.sink.as_ref().unwrap().borrow()) .map(|query_plan| { - let base_sink = get_sink_as_ref!(query_plan.get_sink()); - let mut op = &base_sink.previous.as_ref().unwrap()[0]; - while if let Operator::Scan(Scan::ScanSampling(sp)) = op.deref() { - false + if let Operator::Sink(sink) = query_plan.deref() { + let base_sink = get_sink_as_ref!(sink); + let mut op = base_sink.previous[0].clone(); + loop { + { + let op_ref = op.borrow(); + if let Operator::Scan(Scan::ScanSampling(sp)) = op_ref.deref() { + break; + } + } + op = { + let op_ref = op.borrow(); + get_op_attr_as_ref!(op_ref.deref(), prev) + .as_ref() + .unwrap() + .clone() + }; + } + let op_ref = op.borrow(); + get_op_attr_as_ref!(op_ref.deref(), next)[0].clone() } else { - true - } { - let prev_op = get_op_attr_as_ref!(op, prev).as_ref().unwrap(); - op = prev_op.as_ref(); + panic!("Sink has not been set.") } - &get_op_attr_as_ref!(op, next)[0] }) .collect(); @@ -320,22 +330,34 @@ impl Catalog { fn add_icost_and_selectivity_sorted_by_node( &mut self, - operator: &Operator, - other: Vec<&Operator>, + operator: Rc>>, + other: Vec>>>, is_undirected: bool, ) { - if let Operator::Sink(sink) = &get_op_attr_as_ref!(operator, next)[0] { + if let Operator::Sink(sink) = get_op_attr_as_ref!(operator.borrow().deref(), next)[0] + .borrow() + .deref() + { return; } - let mut num_input_tuples = get_op_attr!(operator, num_out_tuples); + let mut num_input_tuples = get_op_attr!(operator.borrow().deref(), num_out_tuples); for other_op in &other { - num_input_tuples += get_op_attr!(other_op, num_out_tuples); + num_input_tuples += get_op_attr!(other_op.borrow().deref(), num_out_tuples); } - let in_subgraph = get_op_attr_as_ref!(operator, out_subgraph).as_ref(); - let subgraph_idx = self.get_subgraph_idx(&mut in_subgraph.clone()); - let next = get_op_attr_as_ref!(operator, next); + let mut in_subgraph = { + let op_ref = operator.borrow(); + get_op_attr_as_ref!(op_ref.deref(), out_subgraph).clone() + }; + let subgraph_idx = self.get_subgraph_idx(&mut in_subgraph); + let next = { + let op_ref = operator.borrow(); + get_op_attr_as_ref!(op_ref.deref(), next).clone() + }; for i in 0..next.len() { - if let Operator::EI(EI::Intersect(Intersect::IntersectCatalog(intersect))) = &next[i] { + let next_i = next[i].borrow(); + if let Operator::EI(EI::Intersect(Intersect::IntersectCatalog(intersect))) = + next_i.deref() + { let to_type = intersect.base_intersect.base_ei.to_type; let mut alds_as_str_list = vec![]; let alds_str = self.get_alds_as_str( @@ -371,8 +393,12 @@ impl Catalog { } let mut selectivity = intersect.base_intersect.base_ei.base_op.num_out_tuples; for other_op in &other { - let next = &get_op_attr_as_ref!(other_op, next)[i]; - selectivity += get_op_attr!(next, num_out_tuples); + let next = { + let other_op_ref = other_op.borrow(); + get_op_attr_as_ref!(other_op_ref.deref(), next)[i].clone() + }; + let next_ref = next.borrow(); + selectivity += get_op_attr!(next_ref.deref(), num_out_tuples); } self.sampled_selectivity .entry(subgraph_idx) @@ -393,11 +419,18 @@ impl Catalog { .insert(alds_as_str, 0.0); } } - let noop = &get_op_attr_as_ref!(&next[i], next)[0]; + let noop = { + let next_ref = next[i].borrow(); + get_op_attr_as_ref!(next_ref.deref(), next)[0].clone() + }; let mut other_noops = vec![]; for (j, other) in other.iter().enumerate() { - let next = &get_op_attr_as_ref!(other, next)[i]; - let next_op = &get_op_attr_as_ref!(next, next)[j]; + let next_op = { + let other_ref = other.borrow(); + let next_i = get_op_attr_as_ref!(other_ref.deref(), next)[i].clone(); + let next_ref = next_i.borrow(); + get_op_attr_as_ref!(next_ref.deref(), next)[j].clone() + }; other_noops.push(next_op); } self.add_icost_and_selectivity(noop, other_noops, is_undirected); @@ -407,21 +440,34 @@ impl Catalog { fn add_icost_and_selectivity( &mut self, - operator: &Operator, - other: Vec<&Operator>, + operator: Rc>>, + other: Vec>>>, is_undirected: bool, ) { - if let Operator::Sink(sink) = &get_op_attr_as_ref!(operator, next)[0] { + if let Operator::Sink(sink) = get_op_attr_as_ref!(operator.borrow().deref(), next)[0] + .borrow() + .deref() + { return; } - let mut num_input_tuples = get_op_attr!(operator, num_out_tuples); + let mut num_input_tuples = get_op_attr!(operator.borrow().deref(), num_out_tuples); for other_op in &other { - num_input_tuples += get_op_attr!(other_op, num_out_tuples); + num_input_tuples += get_op_attr!(other_op.borrow().deref(), num_out_tuples); } - let in_subgraph = get_op_attr_as_ref!(operator, out_subgraph).as_ref(); - let subgraph_idx = self.get_subgraph_idx(&mut in_subgraph.clone()); - for (i, next) in get_op_attr_as_ref!(operator, next).iter().enumerate() { - if let Operator::EI(EI::Intersect(Intersect::IntersectCatalog(intersect))) = next { + let mut in_subgraph = { + let op_ref = operator.borrow(); + get_op_attr_as_ref!(op_ref.deref(), out_subgraph).clone() + }; + let subgraph_idx = self.get_subgraph_idx(&mut in_subgraph); + let next_vec = { + let op_ref = operator.borrow(); + get_op_attr_as_ref!(op_ref.deref(), next).clone() + }; + for (i, next) in next_vec.iter().enumerate() { + let next_ref = next.borrow(); + if let Operator::EI(EI::Intersect(Intersect::IntersectCatalog(intersect))) = + next_ref.deref() + { let alds = &intersect.base_intersect.base_ei.alds; let mut alds_as_str_list = vec![]; let alds_str = @@ -453,10 +499,13 @@ impl Catalog { alds_as_str_list.push(alds_str); } if 1 == alds.len() { - let mut icost = get_op_attr!(&next, icost); + let mut icost = get_op_attr!(next.borrow().deref(), icost); for other_op in &other { - let next = &get_op_attr_as_ref!(other_op, next)[i]; - icost += get_op_attr!(next, icost); + let next = { + let other_ref = other_op.borrow(); + get_op_attr_as_ref!(other_ref.deref(), next)[i].clone() + }; + icost += get_op_attr!(next.borrow().deref(), icost); } self.sampled_icost .entry(subgraph_idx) @@ -477,14 +526,29 @@ impl Catalog { } } } - let noops = get_op_attr_as_ref!(next, next); + let noops = { + let next_ref = next.borrow(); + get_op_attr_as_ref!(next_ref.deref(), next).clone() + }; for to_type in 0..noops.len() { - let noop = &noops[to_type]; - let mut selectivity = get_op_attr!(noop, num_out_tuples); + let noop = noops[to_type].clone(); + let mut selectivity = { + let noop_ref = noop.borrow(); + get_op_attr!(noop_ref.deref(), num_out_tuples) + }; for other_op in &other { - let next = &get_op_attr_as_ref!(other_op, next)[i]; - let o_next = &get_op_attr_as_ref!(next, next)[to_type]; - selectivity += get_op_attr!(o_next, num_out_tuples); + let next = { + let other_op_ref = other_op.borrow(); + get_op_attr_as_ref!(other_op_ref.deref(), next)[i].clone() + }; + let o_next = { + let next_ref = next.borrow(); + get_op_attr_as_ref!(next_ref.deref(), next)[to_type].clone() + }; + selectivity += { + let o_next_ref = next.borrow(); + get_op_attr!(o_next_ref.deref(), num_out_tuples) + }; } self.sampled_selectivity .entry(subgraph_idx) @@ -507,8 +571,14 @@ impl Catalog { } let mut other_noops = vec![]; for other_op in &other { - let next = &get_op_attr_as_ref!(other_op, next)[i]; - let next_op = &get_op_attr_as_ref!(next, next)[to_type]; + let next = { + let other_op_ref = other_op.borrow(); + get_op_attr_as_ref!(other_op_ref.deref(), next)[i].clone() + }; + let next_op = { + let next_ref = next.borrow(); + get_op_attr_as_ref!(next_ref.deref(), next)[to_type].clone() + }; other_noops.push(next_op); } self.add_icost_and_selectivity(noop, other_noops, is_undirected); diff --git a/src/graph_impl/multi_graph/planner/catalog/catalog_plans.rs b/src/graph_impl/multi_graph/planner/catalog/catalog_plans.rs index dda8eaf9..6fbf5652 100644 --- a/src/graph_impl/multi_graph/planner/catalog/catalog_plans.rs +++ b/src/graph_impl/multi_graph/planner/catalog/catalog_plans.rs @@ -19,7 +19,10 @@ use graph_impl::multi_graph::query::query_graph_set::QueryGraphSet; use graph_impl::multi_graph::utils::set_utils; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; +use std::cell::RefCell; use std::hash::Hash; +use std::ops::{Deref, DerefMut}; +use std::rc::Rc; pub static DEF_NUM_EDGES_TO_SAMPLE: usize = 1000; pub static DEF_MAX_INPUT_NUM_VERTICES: usize = 3; @@ -76,24 +79,27 @@ impl CatalogPlans { plans.generate_all_scans(graph) }; for mut scan in scans { - let mut noop = Noop::new(scan.base_scan.base_op.out_subgraph.as_ref().clone()); - scan.base_scan.base_op.next = vec![Operator::Noop(noop.clone())]; - noop.base_op.prev = Some(Box::new(Operator::Scan(Scan::ScanSampling(scan.clone())))); + let mut noop = Noop::new(scan.base_scan.base_op.out_subgraph.clone()); + scan.base_scan.base_op.next = vec![Rc::new(RefCell::new(Operator::Noop(noop.clone())))]; + noop.base_op.prev = Some(Rc::new(RefCell::new(Operator::Scan(Scan::ScanSampling( + scan.clone(), + ))))); noop.base_op.out_qvertex_to_idx_map = scan.base_scan.base_op.out_qvertex_to_idx_map.clone(); - let mut noop_op = Operator::Noop(noop); + let mut noop_op = Rc::new(RefCell::new(Operator::Noop(noop))); plans.set_next_operators(graph, &mut noop_op, false); let mut query_plans_arr = vec![]; query_plans_arr.push(QueryPlan::new(scan.clone())); for i in 1..num_thread { let mut scan_copy = scan.copy_default(); let mut base_scan_copy = get_base_op_as_mut!(&mut scan_copy); - let mut another_noop = Noop::new(base_scan_copy.out_subgraph.as_ref().clone()); - base_scan_copy.next = vec![Operator::Noop(another_noop.clone())]; + let mut another_noop = Noop::new(base_scan_copy.out_subgraph.clone()); + base_scan_copy.next = + vec![Rc::new(RefCell::new(Operator::Noop(another_noop.clone())))]; another_noop.base_op.out_qvertex_to_idx_map = base_scan_copy.out_qvertex_to_idx_map.clone(); - another_noop.base_op.prev = Some(Box::new(scan_copy.clone())); - let mut another_noop_op = Operator::Noop(another_noop); + another_noop.base_op.prev = Some(Rc::new(RefCell::new(scan_copy.clone()))); + let mut another_noop_op = Rc::new(RefCell::new(Operator::Noop(another_noop))); plans.set_next_operators(graph, &mut another_noop_op, true); if let Operator::Scan(Scan::ScanSampling(sc)) = scan_copy { query_plans_arr.push(QueryPlan::new(sc)); @@ -107,16 +113,17 @@ impl CatalogPlans { pub fn set_next_operators( &mut self, graph: &TypedStaticGraph, - operator: &mut Operator, + operator: &mut Rc>>, is_none: bool, ) { - let operator_copy = operator.clone(); - let operator_base = get_base_op_as_ref!(&operator_copy); - let in_subgraph = operator_base.out_subgraph.as_ref(); + let mut in_subgraph = { + let op_ref = operator.borrow(); + get_op_attr_as_ref!(op_ref.deref(),out_subgraph).clone() + }; if !is_none && !self - .query_graphs_to_extend - .contains(&mut in_subgraph.clone()) + .query_graphs_to_extend + .contains(&mut in_subgraph) { self.query_graphs_to_extend.add(in_subgraph.clone()); } else if !is_none { @@ -135,7 +142,8 @@ impl CatalogPlans { } let to_qvertex = QUERY_VERTICES[in_subgraph.get_num_qvertices()]; let mut next = vec![]; - let last_repeated_vertex_idx = get_op_attr_as_ref!(operator,last_repeated_vertex_idx).clone(); + let last_repeated_vertex_idx = + get_op_attr_as_ref!(operator.borrow().deref(), last_repeated_vertex_idx).clone(); if self.sorted_by_node { for mut descriptor in descriptors { let mut types = vec![]; @@ -146,7 +154,7 @@ impl CatalogPlans { if (ald.direction == Direction::Fwd && 0 == graph.get_num_edges(from_type, to_type, ald.label)) || (ald.direction == Direction::Bwd - && 0 == graph.get_num_edges(to_type, from_type, ald.label)) + && 0 == graph.get_num_edges(to_type, from_type, ald.label)) { produces_output = false; break; @@ -162,7 +170,8 @@ impl CatalogPlans { )); } } - let mut out_qvertex_to_idx_map = get_op_attr_as_ref!(operator,out_qvertex_to_idx_map).clone(); + let mut out_qvertex_to_idx_map = + get_op_attr_as_ref!(operator.borrow().deref(), out_qvertex_to_idx_map).clone(); out_qvertex_to_idx_map.insert(to_qvertex.to_owned(), out_qvertex_to_idx_map.len()); for to_type in types { descriptor @@ -182,17 +191,23 @@ impl CatalogPlans { .base_intersect .base_ei .init_caching(last_repeated_vertex_idx); - next.push(Operator::EI(EI::Intersect(Intersect::IntersectCatalog( - intersect, - )))); + next.push(Rc::new(RefCell::new(Operator::EI(EI::Intersect( + Intersect::IntersectCatalog(intersect), + ))))); } } } else { for i in 0..descriptors.len() { let descriptor = &descriptors[i]; - let prev = get_op_attr_as_ref!(operator,prev).as_ref().unwrap().as_ref(); - let mut out_qvertex_to_idx_map = - get_op_attr_as_ref!(prev, out_qvertex_to_idx_map).clone(); + let mut out_qvertex_to_idx_map ={ + let op_ref = operator.borrow(); + let prev = get_op_attr_as_ref!(op_ref.deref(), prev) + .as_ref() + .unwrap().clone(); + let prev_ref = prev.borrow(); + get_op_attr_as_ref!(prev_ref.deref(), out_qvertex_to_idx_map).clone() + }; + out_qvertex_to_idx_map.insert(to_qvertex.to_owned(), out_qvertex_to_idx_map.len()); let mut ic = IntersectCatalog::new( to_qvertex.to_owned(), @@ -206,7 +221,9 @@ impl CatalogPlans { ic.base_intersect .base_ei .init_caching(last_repeated_vertex_idx); - next.push(Operator::EI(EI::Intersect(Intersect::IntersectCatalog(ic)))); + next.push(Rc::new(RefCell::new(Operator::EI(EI::Intersect( + Intersect::IntersectCatalog(ic), + ))))); } } Self::set_next_pointer(operator, &mut next); @@ -217,35 +234,37 @@ impl CatalogPlans { vec![Noop::new(QueryGraph::empty()); self.num_node_labels] }; self.set_noops( - get_base_op_as_ref!(&next_op).out_subgraph.as_ref(), + get_op_attr_as_ref!(next_op.borrow().deref(), out_subgraph), to_qvertex.to_owned(), &mut next_noops, - &get_base_op_as_ref!(&next_op).out_qvertex_to_idx_map, + get_op_attr_as_ref!(next_op.borrow().deref(), out_qvertex_to_idx_map), ); let mut next_noops = next_noops .into_iter() - .map(|noop| Operator::Noop(noop)) + .map(|noop| Rc::new(RefCell::new(Operator::Noop(noop)))) .collect(); Self::set_next_pointer(&mut next_op, &mut next_noops); - if get_base_op_as_ref!(&next_op) - .out_subgraph - .as_ref() - .get_num_qvertices() + if get_op_attr_as_ref!(next_op.borrow().deref(), out_subgraph).get_num_qvertices() <= self.max_input_num_vertices { for mut next_noop in next_noops { - *get_op_attr_as_mut!(&mut next_noop, last_repeated_vertex_idx) = - last_repeated_vertex_idx; + *get_op_attr_as_mut!( + next_noop.borrow_mut().deref_mut(), + last_repeated_vertex_idx + ) = last_repeated_vertex_idx; self.set_next_operators(graph, &mut next_noop, false) } } } } - fn set_next_pointer(operator: &mut Operator, next: &mut Vec>) { - *get_op_attr_as_mut!(operator, next) = next.clone(); + fn set_next_pointer( + operator: &mut Rc>>, + next: &mut Vec>>>, + ) { + *get_op_attr_as_mut!(operator.borrow_mut().deref_mut(), next) = next.clone(); for next_op in next { - *get_op_attr_as_mut!(next_op, prev) = Some(Box::new(operator.clone())); + *get_op_attr_as_mut!(next_op.borrow_mut().deref_mut(), prev) = Some(operator.clone()); } } @@ -387,13 +406,13 @@ impl CatalogPlans { .as_ref() .unwrap() .get_neighbor_ids() - { - edges.push(vec![Id::new(from_vertex), to_vertex.clone()]); - } + { + edges.push(vec![Id::new(from_vertex), to_vertex.clone()]); + } } let mut out_subgraph = QueryGraph::empty(); out_subgraph.add_qedge(QueryEdge::new("a".to_owned(), "b".to_owned(), 0, 0, 0)); - let mut scan = ScanSampling::new(Box::new(out_subgraph)); + let mut scan = ScanSampling::new(out_subgraph); scan.set_edge_indices_to_sample_list(edges, self.num_sampled_edges); vec![scan] } @@ -445,6 +464,7 @@ impl CatalogPlans { } } } + //TODO: verify is there need ..=? let mut scans = vec![]; for from_type in 0..self.num_node_labels { for label in 0..self.num_edge_labels { @@ -464,8 +484,8 @@ impl CatalogPlans { if actual_num_edges > 0 { let mut num_edges_to_sample = self.num_sampled_edges * (graph.get_num_edges(from_type, to_type, label) / graph.edge_count()) - as usize; - let mut scan = ScanSampling::new(Box::new(out_subgraph)); + as usize; + let mut scan = ScanSampling::new(out_subgraph); if self.sorted_by_node && num_edges_to_sample < 1000 { num_edges_to_sample = actual_num_edges; } diff --git a/src/graph_impl/multi_graph/planner/catalog/operator/intersect_catalog.rs b/src/graph_impl/multi_graph/planner/catalog/operator/intersect_catalog.rs index 11daf721..4bbd4750 100644 --- a/src/graph_impl/multi_graph/planner/catalog/operator/intersect_catalog.rs +++ b/src/graph_impl/multi_graph/planner/catalog/operator/intersect_catalog.rs @@ -6,7 +6,9 @@ use graph_impl::multi_graph::planner::catalog::adj_list_descriptor::AdjListDescr use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; -use std::hash::{BuildHasherDefault, Hash}; +use std::cell::RefCell; +use std::hash::Hash; +use std::rc::Rc; #[derive(Clone)] pub struct IntersectCatalog { @@ -31,8 +33,8 @@ impl IntersectCatalog { to_qvertex, to_type, alds, - Box::new(out_subgraph), - Some(Box::new(in_subgraph)), + out_subgraph, + Some(in_subgraph), out_qvertex_to_idx_map, ), is_adj_list_sorted_by_type, @@ -154,15 +156,13 @@ impl CommonOperatorTrait for IntersectCatalog { base_ei.base_op.probe_tuple[base_ei.out_idx] = base_ei.out_neighbours.ids[idx]; base_ei.base_op.num_out_tuples += 1; if self.is_adj_list_sorted_by_type { - base_ei.base_op.next[0].process_new_tuple(); + base_ei.base_op.next[0].borrow_mut().process_new_tuple(); } else { base_ei .base_op .next - .get_mut( - base_ei.vertex_types[base_ei.base_op.probe_tuple[base_ei.out_idx].id()], - ) - .map(|next_op| next_op.process_new_tuple()); + .get(base_ei.vertex_types[base_ei.base_op.probe_tuple[base_ei.out_idx].id()]) + .map(|next_op| next_op.borrow_mut().process_new_tuple()); } } } @@ -184,7 +184,7 @@ impl CommonOperatorTrait for IntersectCatalog { self.base_intersect.copy(is_thread_safe) } - fn is_same_as(&mut self, op: &mut Operator) -> bool { + fn is_same_as(&mut self, op: &mut Rc>>) -> bool { self.base_intersect.is_same_as(op) } diff --git a/src/graph_impl/multi_graph/planner/catalog/operator/noop.rs b/src/graph_impl/multi_graph/planner/catalog/operator/noop.rs index b1cf13a7..a90b5d82 100644 --- a/src/graph_impl/multi_graph/planner/catalog/operator/noop.rs +++ b/src/graph_impl/multi_graph/planner/catalog/operator/noop.rs @@ -5,8 +5,9 @@ use graph_impl::multi_graph::plan::operator::operator::{ use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; -use itertools::Itertools; -use std::hash::{BuildHasherDefault, Hash}; +use std::cell::RefCell; +use std::hash::Hash; +use std::rc::Rc; #[derive(Clone)] pub struct Noop { @@ -16,7 +17,7 @@ pub struct Noop { impl Noop { pub fn new(query_graph: QueryGraph) -> Noop { Noop { - base_op: BaseOperator::new(Box::new(query_graph.clone()), Some(Box::new(query_graph))), + base_op: BaseOperator::new(query_graph.clone(), Some(query_graph)), } } } @@ -28,8 +29,8 @@ impl CommonOperatorTrait for Noop { graph: &TypedStaticGraph, ) { self.base_op.probe_tuple = probe_tuple.clone(); - for next_op in &mut self.base_op.next { - next_op.init(probe_tuple.clone(), graph); + for next_op in &self.base_op.next { + next_op.borrow_mut().init(probe_tuple.clone(), graph); } } @@ -37,8 +38,8 @@ impl CommonOperatorTrait for Noop { self.base_op.num_out_tuples += 1; self.base_op .next - .iter_mut() - .for_each(|next_op| next_op.process_new_tuple()); + .iter() + .for_each(|next_op| next_op.borrow_mut().process_new_tuple()); } fn execute(&mut self) { @@ -57,7 +58,7 @@ impl CommonOperatorTrait for Noop { self.base_op.copy(is_thread_safe) } - fn is_same_as(&mut self, op: &mut Operator) -> bool { + fn is_same_as(&mut self, op: &mut Rc>>) -> bool { self.base_op.is_same_as(op) } diff --git a/src/graph_impl/multi_graph/planner/catalog/query_graph.rs b/src/graph_impl/multi_graph/planner/catalog/query_graph.rs index 9b77624e..4637715d 100644 --- a/src/graph_impl/multi_graph/planner/catalog/query_graph.rs +++ b/src/graph_impl/multi_graph/planner/catalog/query_graph.rs @@ -3,7 +3,6 @@ use graph_impl::multi_graph::planner::catalog::subgraph_mapping_iterator::Subgra use hashbrown::{HashMap, HashSet}; use itertools::Itertools; use std::iter::FromIterator; -use std::process::id; #[derive(Clone)] pub struct QueryGraph { @@ -140,7 +139,7 @@ impl QueryGraph { fn get_subgraph_mapping_if_any( &mut self, other_query_graph: &QueryGraph, - ) -> Option<&HashMap> { + ) -> Option> { let it = self.get_subgraph_mapping_iterator(other_query_graph); if !it.has_next() { return None; @@ -159,7 +158,7 @@ impl QueryGraph { pub fn get_isomorphic_mapping_if_any( &mut self, other_query_graph: &mut QueryGraph, - ) -> Option<&HashMap> { + ) -> Option> { if self.is_isomorphic_to(other_query_graph) { return self.get_subgraph_mapping_if_any(other_query_graph); } diff --git a/src/graph_impl/multi_graph/planner/catalog/subgraph_mapping_iterator.rs b/src/graph_impl/multi_graph/planner/catalog/subgraph_mapping_iterator.rs index 8b379106..5243e901 100644 --- a/src/graph_impl/multi_graph/planner/catalog/subgraph_mapping_iterator.rs +++ b/src/graph_impl/multi_graph/planner/catalog/subgraph_mapping_iterator.rs @@ -179,7 +179,7 @@ impl SubgraphMappingIterator { !self.curr_mapping.is_empty() } - pub fn next(&mut self) -> Option<&HashMap> { + pub fn next(&mut self) -> Option> { if !self.has_next() { return None; } @@ -189,7 +189,7 @@ impl SubgraphMappingIterator { self.next .insert(self.curr_mapping[i].clone(), self.query_vertices[i].clone()); } - return Some(&self.next); + return Some(self.next.clone()); } pub fn contains_query_edge(&self, v1: &String, v2: &String) -> bool { diff --git a/src/graph_impl/multi_graph/planner/query_planner.rs b/src/graph_impl/multi_graph/planner/query_planner.rs index c0bed9c8..e6fcf512 100644 --- a/src/graph_impl/multi_graph/planner/query_planner.rs +++ b/src/graph_impl/multi_graph/planner/query_planner.rs @@ -23,10 +23,10 @@ use graph_impl::multi_graph::planner::catalog::query_edge::QueryEdge; use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::TypedStaticGraph; use hashbrown::{HashMap, HashSet}; -use itertools::min; use std::cell::RefCell; use std::hash::Hash; -use std::ptr::null; +use std::ops::{Deref, DerefMut}; +use std::rc::Rc; pub struct QueryPlanner { subgraph_plans: HashMap>>>, @@ -61,8 +61,8 @@ impl pub fn plan(&mut self) -> QueryPlan { if self.num_qvertices == 2 { - return QueryPlan::new_from_operator(Box::new(Operator::Scan(Scan::Base( - BaseScan::new(Box::new(self.query_graph.clone())), + return QueryPlan::new_from_operator(Rc::new(RefCell::new(Operator::Scan( + Scan::Base(BaseScan::new(self.query_graph.clone())), )))); } self.consider_all_scan_operators(); @@ -86,19 +86,22 @@ impl } pub fn set_next_pointers(&self, best_plan: &mut QueryPlan) { - best_plan - .subplans - .iter_mut() - .map(|op| op.as_mut()) - .for_each(|last_op| { - let mut cur_op = last_op; - while get_op_attr_as_ref!(cur_op, prev).is_some() { - let cur_copy = cur_op.clone(); - let prev = get_op_attr_as_mut!(cur_op, prev).as_mut().unwrap().as_mut(); - *get_op_attr_as_mut!(prev, next) = vec![cur_copy]; - cur_op = get_op_attr_as_mut!(cur_op, prev).as_mut().unwrap().as_mut(); + best_plan.subplans.iter_mut().for_each(|last_op| { + let mut cur_op = last_op.clone(); + loop{ + let prev_op = { + let cur_op_ref = cur_op.borrow(); + get_op_attr_as_ref!(cur_op_ref.deref(), prev).as_ref().map(|op|op.clone()) + }; + if prev_op.is_none(){ + break; } - }); + let mut prev = prev_op.unwrap(); + let mut prev_ref = prev.borrow_mut(); + *get_op_attr_as_mut!(prev_ref.deref_mut(), next) = vec![cur_op.clone()]; + cur_op = prev.clone() + } + }); } fn consider_all_scan_operators(&mut self) { @@ -109,7 +112,7 @@ impl for query_edge in &self.query_graph.q_edges { let mut out_subgraph = QueryGraph::empty(); out_subgraph.add_qedge(query_edge.clone()); - let scan = Scan::Base(BaseScan::new(Box::new(out_subgraph))); + let scan = Scan::Base(BaseScan::new(out_subgraph)); let num_edges = self.get_num_edges(&query_edge); let query_plan = QueryPlan::new_from_last_op(scan, num_edges as f64); let mut query_plans = vec![]; @@ -150,8 +153,9 @@ impl fn consider_all_next_extend_operators(&mut self, key: &String) { let prev_plan_map = &self.subgraph_plans[&(self.next_num_qvertices - 1)]; let prev_query_plans = &prev_plan_map[key]; - let op = prev_query_plans[0].last_operator.as_ref().unwrap().as_ref(); - let prev_qvertices = get_op_attr_as_ref!(op, out_subgraph).get_query_vertices(); + let op = prev_query_plans[0].last_operator.as_ref().unwrap(); + let prev_qvertices = + get_op_attr_as_ref!(op.borrow().deref(), out_subgraph).get_query_vertices(); let to_qvertices = self.query_graph.get_neighbors(prev_qvertices); let prev_query_plans_len = prev_plan_map[key].len(); let mut plans = vec![]; @@ -187,12 +191,14 @@ impl mut prev_query_plan: QueryPlan, to_qvertex: &String, ) -> (String, QueryPlan) { - let last_operator = prev_query_plan.last_operator.as_ref().unwrap().as_ref(); - let base_last_op = get_base_op_as_ref!(last_operator); - let in_subgraph = base_last_op.out_subgraph.as_ref(); - let last_previous_repeated_index = base_last_op.last_repeated_vertex_idx; + let last_operator = prev_query_plan.last_operator.as_ref().unwrap(); + let (in_subgraph,last_previous_repeated_index) = { + let last_op_ref = last_operator.borrow(); + let base_op = get_base_op_as_ref!(last_op_ref.deref()); + (base_op.out_subgraph.clone(),base_op.last_repeated_vertex_idx) + }; let mut alds = vec![]; - let mut next_extend = self.get_next_ei(in_subgraph, to_qvertex, &mut alds, last_operator); + let mut next_extend = self.get_next_ei(&in_subgraph, to_qvertex, &mut alds, last_operator); let next_copy = next_extend.clone(); let base_next_extend = get_ei_as_mut!(&mut next_extend); base_next_extend.init_caching(last_previous_repeated_index); @@ -201,12 +207,15 @@ impl .base_op .out_subgraph .get_query_vertex_type(to_qvertex); - let last_operator = prev_query_plan.last_operator.as_mut().unwrap().as_mut(); - let mut base_last_op = get_base_op_as_mut!(last_operator); - let in_subgraph = base_last_op.out_subgraph.as_mut(); + let (mut in_subgraph,last_prev) = { + let last_operator = prev_query_plan.last_operator.as_mut().unwrap(); + let last_ref = last_operator.borrow(); + let base_op = get_base_op_as_ref!(last_ref.deref()); + (base_op.out_subgraph.clone(),base_op.prev.as_ref().map(|op|op.clone())) + }; let estimated_selectivity = self.get_selectivity( - in_subgraph, - base_next_extend.base_op.out_subgraph.as_mut(), + &mut in_subgraph, + &mut base_next_extend.base_op.out_subgraph, &alds, to_type, ); @@ -214,13 +223,13 @@ impl if let CachingType::None = base_next_extend.caching_type { icost = prev_estimated_num_out_tuples * self.catalog.get_icost( - in_subgraph, + &mut in_subgraph, alds.iter().collect(), base_next_extend.to_type, ); } else { let mut out_tuples_to_process = prev_estimated_num_out_tuples; - if base_last_op.prev.is_some() { + if last_prev.is_some() { let index = 0; let mut last_estimated_num_out_tuples_for_extension_qvertex = -1.0; for ald in alds.iter().filter(|ald| ald.vertex_idx > index) { @@ -230,7 +239,7 @@ impl out_tuples_to_process /= last_estimated_num_out_tuples_for_extension_qvertex; } if let CachingType::FullCaching = base_next_extend.caching_type { - icost = out_tuples_to_process * self.catalog.get_icost(in_subgraph, alds.iter().collect(), to_type) + + icost = out_tuples_to_process * self.catalog.get_icost(&mut in_subgraph, alds.iter().collect(), to_type) + // added to make caching effect on cost more robust. (prev_estimated_num_out_tuples - out_tuples_to_process) * estimated_selectivity; } else { @@ -246,9 +255,9 @@ impl let always_intersect_icost = prev_estimated_num_out_tuples * self .catalog - .get_icost(in_subgraph, alds_to_always_intersect, to_type); + .get_icost(&mut in_subgraph, alds_to_always_intersect, to_type); let cached_intersect_icost = out_tuples_to_process - * self.catalog.get_icost(in_subgraph, alds_to_cache, to_type); + * self.catalog.get_icost(&mut in_subgraph, alds_to_cache, to_type); icost = prev_estimated_num_out_tuples * always_intersect_icost + out_tuples_to_process * cached_intersect_icost + // added to make caching effect on cost more robust. @@ -274,7 +283,7 @@ impl let mut new_query_plan = prev_query_plan.shallow_copy(); new_query_plan.estimated_icost = estimated_icost; new_query_plan.estimated_num_out_tuples = estimated_num_out_tuples; - new_query_plan.append(Operator::EI(next_copy)); + new_query_plan.append(Rc::new(RefCell::new(Operator::EI(next_copy)))); new_query_plan.q_vertex_to_num_out_tuples = q_vertex_to_num_out_tuples; ( self.get_key( @@ -294,7 +303,7 @@ impl in_subgraph: &QueryGraph, to_qvertex: &String, alds: &mut Vec, - last_operator: &Operator, + last_operator: &Rc>>, ) -> EI { let mut out_subgraph = in_subgraph.copy(); in_subgraph @@ -308,9 +317,10 @@ impl // simple query graph so there is only 1 query_edge, so get query_edge at index '0'. let query_edge = self.query_graph.get_qedges(from_qvertex, to_qvertex)[0].clone(); - let index = get_op_attr_as_ref!(last_operator, out_qvertex_to_idx_map) - [from_qvertex] - .clone(); + let index = + get_op_attr_as_ref!(last_operator.borrow().deref(), out_qvertex_to_idx_map) + [from_qvertex] + .clone(); let direction = if from_qvertex == &query_edge.from_query_vertex { Direction::Fwd } else { @@ -327,7 +337,7 @@ impl } }); let mut output_variable_idx_map = HashMap::new(); - get_op_attr_as_ref!(last_operator, out_qvertex_to_idx_map) + get_op_attr_as_ref!(last_operator.borrow().deref(), out_qvertex_to_idx_map) .iter() .for_each(|(k, v)| { output_variable_idx_map.insert(k.clone(), v.clone()); @@ -376,8 +386,8 @@ impl fn consider_all_next_hash_join_operators(&mut self, map_key: &String) { let plan_map = &self.subgraph_plans[&self.next_num_qvertices]; let plans = &plan_map[map_key]; - let op = plans[0].last_operator.as_ref().unwrap().as_ref(); - let out_subgraph = get_op_attr_as_ref!(op, out_subgraph).as_ref().clone(); + let op = plans[0].last_operator.as_ref().unwrap(); + let out_subgraph = get_op_attr_as_ref!(op.borrow().deref(), out_subgraph).clone(); let query_vertices = out_subgraph.get_query_vertices(); let min_size = 3; @@ -389,8 +399,9 @@ impl let plans = self.subgraph_plans[&set_size].clone(); for key in plans.keys() { let prev_query_plan = self.get_best_plan(set_size, key); - let last_op = prev_query_plan.last_operator.as_ref().unwrap().as_ref(); - let base_last_op = get_base_op_as_ref!(last_op); + let last_op = prev_query_plan.last_operator.as_ref().unwrap(); + let last_op_ref = last_op.borrow(); + let base_last_op = get_base_op_as_ref!(last_op_ref.deref()); let prev_qvertices = base_last_op.out_subgraph.get_query_vertices_as_set(); let is_subset = query_vertices .iter() @@ -508,10 +519,10 @@ impl q_vertex_to_num_out_tuples.insert(k.clone(), v.clone()); }); - let last_op = build_subplan.last_operator.as_ref().unwrap().as_ref(); - let base_last_op = get_base_op_as_ref!(last_op); - base_last_op - .out_subgraph + let last_op = build_subplan.last_operator.as_ref().unwrap(); + let last_op_ref = last_op.borrow(); + let out_subgraph = get_op_attr_as_ref!(last_op_ref.deref(),out_subgraph); + out_subgraph .get_query_vertices() .iter() .for_each(|v| { @@ -519,6 +530,7 @@ impl .entry(v.clone()) .or_insert(curr_best_query_plan.estimated_num_out_tuples); }); + query_plan.q_vertex_to_num_out_tuples = q_vertex_to_num_out_tuples; let query_plans = self @@ -546,7 +558,7 @@ impl pub fn get_num_edges(&self, query_edge: &QueryEdge) -> usize { let from_type = self .query_graph - .get_query_vertex_type((&query_edge.from_query_vertex)); + .get_query_vertex_type(&query_edge.from_query_vertex); let to_type = self .query_graph .get_query_vertex_type(&query_edge.to_query_vertex); diff --git a/src/graph_impl/multi_graph/planner/query_planner_big.rs b/src/graph_impl/multi_graph/planner/query_planner_big.rs index 2ab6848f..bc0bb257 100644 --- a/src/graph_impl/multi_graph/planner/query_planner_big.rs +++ b/src/graph_impl/multi_graph/planner/query_planner_big.rs @@ -18,6 +18,7 @@ use graph_impl::TypedStaticGraph; use hashbrown::{HashMap, HashSet}; use std::cmp::max; use std::hash::Hash; +use std::ops::Deref; pub struct QueryPlannerBig { base_planner: QueryPlanner, @@ -26,7 +27,7 @@ pub struct QueryPlannerBig - QueryPlannerBig +QueryPlannerBig { pub fn new( query_graph: QueryGraph, @@ -98,7 +99,7 @@ impl for i in 0..self.num_top_plans_kept { let mut output_subgraph = QueryGraph::empty(); output_subgraph.add_qedge(edges_to_scan[i].clone()); - let scan = Scan::Base(BaseScan::new(Box::new(output_subgraph))); + let scan = Scan::Base(BaseScan::new(output_subgraph)); let query_plan = QueryPlan::new_from_last_op(scan, num_edges_to_scan[i] as f64); self.subgraph_plans .get_mut(&self.base_planner.next_num_qvertices) @@ -115,11 +116,13 @@ impl .get_mut(&(self.base_planner.next_num_qvertices - 1)) .unwrap(); for prev_query_plan in plans { - let last_base_op = prev_query_plan.last_operator.as_ref().unwrap().as_ref(); - let last_op = get_base_op_as_ref!(last_base_op); - let prev_qvertices = last_op.out_subgraph.get_query_vertices(); + let (prev_qvertices, in_subgraph) = { + let last_base_op = prev_query_plan.last_operator.as_ref().unwrap(); + let last_base_op_ref = last_base_op.borrow(); + let last_op = get_base_op_as_ref!(last_base_op_ref.deref()); + (last_op.out_subgraph.get_query_vertices(), last_op.out_subgraph.clone()) + }; let to_qvertices = self.base_planner.query_graph.get_neighbors(prev_qvertices); - let in_subgraph = last_op.out_subgraph.as_ref(); let next_to_qvertices = Self::filter_to_qvertices_by_max_num_alds( &self.base_planner.query_graph, to_qvertices, diff --git a/src/graph_impl/multi_graph/query/query_graph_set.rs b/src/graph_impl/multi_graph/query/query_graph_set.rs index 254da206..3c10f5f4 100644 --- a/src/graph_impl/multi_graph/query/query_graph_set.rs +++ b/src/graph_impl/multi_graph/query/query_graph_set.rs @@ -1,5 +1,5 @@ use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; -use hashbrown::{HashMap, HashSet}; +use hashbrown::HashMap; pub struct QueryGraphSet { query_graphs: HashMap>, diff --git a/src/graph_impl/multi_graph/runner/catalog_generator.rs b/src/graph_impl/multi_graph/runner/catalog_generator.rs index 560147ae..4a82aaf1 100644 --- a/src/graph_impl/multi_graph/runner/catalog_generator.rs +++ b/src/graph_impl/multi_graph/runner/catalog_generator.rs @@ -1,7 +1,7 @@ use generic::{GraphType, IdType}; use graph_impl::multi_graph::planner::catalog::catalog::Catalog; use graph_impl::multi_graph::planner::catalog::catalog_plans::{ - CatalogPlans, DEF_MAX_INPUT_NUM_VERTICES, DEF_NUM_EDGES_TO_SAMPLE, + DEF_MAX_INPUT_NUM_VERTICES, DEF_NUM_EDGES_TO_SAMPLE, }; use graph_impl::TypedStaticGraph; use std::hash::Hash; diff --git a/src/graph_impl/multi_graph/runner/optimizer_executor.rs b/src/graph_impl/multi_graph/runner/optimizer_executor.rs index f8ceb75c..1ddaa27f 100644 --- a/src/graph_impl/multi_graph/runner/optimizer_executor.rs +++ b/src/graph_impl/multi_graph/runner/optimizer_executor.rs @@ -1,16 +1,12 @@ use generic::{GraphType, IdType}; use graph_impl::multi_graph::plan::query_plan::QueryPlan; use graph_impl::multi_graph::planner::catalog::catalog::Catalog; -use graph_impl::multi_graph::planner::catalog::query_edge::QueryEdge; use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::planner::query_planner::QueryPlanner; use graph_impl::multi_graph::planner::query_planner_big::QueryPlannerBig; -use graph_impl::multi_graph::runner::catalog_generator; -use graph_impl::{EdgeVec, TypedStaticGraph}; -use hashbrown::HashMap; +use graph_impl::TypedStaticGraph; use std::hash::Hash; use std::time::SystemTime; -use DiStaticGraph; pub fn generate_plan( query_graph: QueryGraph, diff --git a/src/graph_impl/multi_graph/utils/set_utils.rs b/src/graph_impl/multi_graph/utils/set_utils.rs index e5fc3753..1c33a810 100644 --- a/src/graph_impl/multi_graph/utils/set_utils.rs +++ b/src/graph_impl/multi_graph/utils/set_utils.rs @@ -1,5 +1,3 @@ -use std::iter::FromIterator; - pub fn get_power_set_excluding_empty_set(set: Vec) -> Vec> { let mut res = vec![]; let len = set.len(); diff --git a/src/graph_impl/static_graph/graph.rs b/src/graph_impl/static_graph/graph.rs index fb465254..d7f3e184 100644 --- a/src/graph_impl/static_graph/graph.rs +++ b/src/graph_impl/static_graph/graph.rs @@ -38,11 +38,8 @@ use io::serde::{Deserialize, Serialize}; use itertools::Itertools; use map::SetMap; use serde; -use std::any::Any; use std::cmp; use std::ops::Add; -use test::bench::iter; -use test::Options; pub type TypedUnStaticGraph = TypedStaticGraph; pub type TypedDiStaticGraph = TypedStaticGraph; diff --git a/tests/static_graph.rs b/tests/static_graph.rs index 89608a19..6b6131de 100644 --- a/tests/static_graph.rs +++ b/tests/static_graph.rs @@ -335,10 +335,10 @@ fn test_graphflow_planner() { false, ); let g = g_.into_static(); - println!("node_count={}",g.node_count()); - println!("edge_count={}",g.edge_count()); - println!("num_of_node_labels={}",g.num_of_node_labels()); - println!("num_of_edge_labels={}",g.num_of_edge_labels()); + println!("node_count={}", g.node_count()); + println!("edge_count={}", g.edge_count()); + println!("num_of_node_labels={}", g.num_of_node_labels()); + println!("num_of_edge_labels={}", g.num_of_edge_labels()); println!("load finished."); // let edge_vec = EdgeVec::with_labels(vec![0, 2, 3, 4], vec![1, 2, 0, 0], vec![0, 1, 0, 1]); // let in_edge_vec = EdgeVec::new(vec![0, 2, 3, 4], vec![1, 2, 0, 0]); From f07e75a5aa070566fe3406bb55a6aa748dccc4af Mon Sep 17 00:00:00 2001 From: Michael Chen <920901835@qq.com> Date: Sun, 19 Jan 2020 06:39:15 +0800 Subject: [PATCH 19/25] fix:refactor --- .../multi_graph/plan/operator/operator.rs | 11 +- src/graph_impl/multi_graph/plan/query_plan.rs | 109 +++++++----- .../multi_graph/plan/query_plan_worker.rs | 15 +- .../multi_graph/planner/catalog/catalog.rs | 22 +-- .../planner/catalog/catalog_plans.rs | 166 +++++++++--------- .../planner/catalog/query_graph.rs | 39 ++-- .../catalog/subgraph_mapping_iterator.rs | 8 + .../multi_graph/planner/query_planner.rs | 44 +++-- .../multi_graph/planner/query_planner_big.rs | 7 +- .../multi_graph/runner/catalog_generator.rs | 22 +-- src/graph_impl/static_graph/graph.rs | 142 +++++++-------- tests/static_graph.rs | 26 +-- 12 files changed, 311 insertions(+), 300 deletions(-) diff --git a/src/graph_impl/multi_graph/plan/operator/operator.rs b/src/graph_impl/multi_graph/plan/operator/operator.rs index d187134e..d490889f 100644 --- a/src/graph_impl/multi_graph/plan/operator/operator.rs +++ b/src/graph_impl/multi_graph/plan/operator/operator.rs @@ -96,19 +96,24 @@ pub trait CommonOperatorTrait { } impl Operator { - pub fn get_last_operators(&self, last_operators: &mut Vec>>>) { + pub fn get_last_operators(&self) -> Vec>>> { let next = get_op_attr_as_ref!(self, next); if next.is_empty() { - return; + return vec![]; } + let mut last_operators = vec![]; for op in next { let next = op.borrow(); if get_op_attr_as_ref!(next.deref(), next).is_empty() { last_operators.push(op.clone()); continue; } - op.borrow().get_last_operators(last_operators); + op.borrow() + .get_last_operators() + .into_iter() + .for_each(|op| last_operators.push(op)); } + last_operators } pub fn get_operator_metrics_next_operators( diff --git a/src/graph_impl/multi_graph/plan/query_plan.rs b/src/graph_impl/multi_graph/plan/query_plan.rs index d31733ab..adbdb47e 100644 --- a/src/graph_impl/multi_graph/plan/query_plan.rs +++ b/src/graph_impl/multi_graph/plan/query_plan.rs @@ -27,7 +27,7 @@ use std::time::SystemTime; pub struct QueryPlan { pub sink: Option>>>, pub sink_type: SinkType, - scan_sampling: Option>, + pub scan_sampling: Option>>>, pub last_operator: Option>>>, pub out_tuples_limit: usize, pub elapsed_time: u128, @@ -44,10 +44,11 @@ pub struct QueryPlan { } impl QueryPlan { - pub fn new(scan_sampling: ScanSampling) -> Self { - let mut last_operators = Vec::new(); - let scan_sampling_op = Operator::Scan(Scan::ScanSampling(scan_sampling.clone())); - scan_sampling_op.get_last_operators(&mut last_operators); + pub fn new(scan_sampling: Rc>>) -> Self { + let mut last_operators = { + let op_ref = scan_sampling.borrow(); + op_ref.deref().get_last_operators() + }; let out_subgraph = { let op = last_operators[0].borrow(); @@ -57,8 +58,7 @@ impl QueryPlan { out_subgraph, ))))); for op in last_operators.iter_mut() { - let mut op = op.borrow_mut(); - *get_op_attr_as_mut!(op.deref_mut(), next) = vec![sink.clone()]; + *get_op_attr_as_mut!(op.borrow_mut().deref_mut(), next) = vec![sink.clone()]; } if let Operator::Sink(Sink::BaseSink(sink)) = sink.borrow_mut().deref_mut() { sink.previous = last_operators.clone(); @@ -136,10 +136,6 @@ impl QueryPlan { plan } - pub fn get_scan_sampling(&mut self) -> Option<&mut ScanSampling> { - self.scan_sampling.as_mut() - } - pub fn shallow_copy(&self) -> QueryPlan { QueryPlan::new_from_subplans(self.subplans.clone()) } @@ -149,11 +145,8 @@ impl QueryPlan { let mut last_operator = self.last_operator.as_mut().unwrap().borrow_mut(); *get_op_attr_as_mut!(last_operator.deref_mut(), next) = vec![new_operator.clone()]; } - { - let mut new_op = new_operator.borrow_mut(); - *get_op_attr_as_mut!(new_op.deref_mut(), prev) = - self.last_operator.as_ref().map(|op| op.clone()); - } + *get_op_attr_as_mut!(new_operator.borrow_mut().deref_mut(), prev) = + self.last_operator.as_ref().map(|op| op.clone()); self.subplans.push(new_operator.clone()); self.last_operator = Some(new_operator); } @@ -188,19 +181,16 @@ impl QueryPlan { for subplan in &self.subplans { let mut first_op = subplan.clone(); loop { - { - let first_op_ref = first_op.borrow(); - if get_op_attr_as_ref!(first_op_ref.deref(), prev).is_none() { - break; - } - } - first_op = { + let prev = { let first_op_ref = first_op.borrow(); get_op_attr_as_ref!(first_op_ref.deref(), prev) .as_ref() - .unwrap() - .clone() + .map(|op| op.clone()) + }; + if prev.is_none() { + break; } + first_op = prev.as_ref().unwrap().clone(); } first_op .borrow() @@ -250,10 +240,38 @@ impl QueryPlan { } *get_op_attr_as_mut!(self.subplans[plan_size - 1].borrow_mut().deref_mut(), next) = vec![sink]; + let mut probes = vec![]; + for i in 1..self.subplans.len() { + let mut operator = self.subplans[i].clone(); + loop{ + { + let mut op_ref = operator.borrow(); + if let Operator::Probe(pb) = op_ref.deref(){ + probes.push(operator.clone()); + } + } + let prev = { + let mut op_ref = operator.borrow(); + get_op_attr_as_ref!(op_ref.deref(),prev).as_ref().map(|op|op.clone()) + }; + if prev.is_none() {break;} + operator = prev.unwrap(); + } + } for subplan in &mut self.subplans { if let Operator::Build(build) = subplan.borrow_mut().deref_mut() { let hash_table = HashTable::new(build.build_hash_idx, build.hashed_tuple_len); build.hash_table = Some(hash_table.clone()); + for probe in &probes { + let mut probe_mut = probe.borrow_mut(); + if get_op_attr_as_ref!(probe_mut.deref(),in_subgraph).as_ref().unwrap() == build.probing_subgraph.as_ref().unwrap() { + if let Operator::Probe(pb) = probe_mut.deref_mut(){ + let mut base_probe = get_probe_as_mut!(pb); + base_probe.hash_tables = vec![hash_table.clone()]; + break; + } + } + } } } let sub_plans = self.subplans.clone(); @@ -268,24 +286,24 @@ impl QueryPlan { } } for subplan in &mut self.subplans { - let probe_tuple = vec![]; + let probe_tuple = { + let subplan_ref = subplan.borrow(); + vec![Id::new(0);get_op_attr!(subplan_ref.deref(),out_tuple_len)] + }; let mut first_op = subplan.clone(); loop { - { + let prev = { let first_op_ref = first_op.borrow(); - if get_op_attr_as_ref!(first_op_ref.deref(), prev).is_none() { - break; - } - } - first_op = { - let mut first_op_mut = first_op.borrow_mut(); - get_op_attr_as_mut!(first_op_mut.deref_mut(), prev) - .as_mut() - .unwrap() - .clone() + get_op_attr_as_ref!(first_op_ref.deref(), prev) + .as_ref() + .map(|op| op.clone()) }; + if prev.is_none() { + break; + } + first_op = prev.as_ref().unwrap().clone(); } - first_op.borrow_mut().deref_mut().init(probe_tuple, graph); + first_op.borrow_mut().init(probe_tuple, graph); } } @@ -299,19 +317,16 @@ impl QueryPlan { } let mut op = operator.clone(); loop { - { - let op_ref = op.borrow(); - if get_op_attr_as_ref!(op_ref.deref(), prev).is_none() { - break; - } - } - op = { + let prev = { let op_ref = op.borrow(); get_op_attr_as_ref!(op_ref.deref(), prev) .as_ref() - .unwrap() - .clone() + .map(|op| op.clone()) }; + if prev.is_none() { + break; + } + op = prev.as_ref().unwrap().clone(); let mut op_mut = op.borrow_mut(); if let Operator::Probe(p) = op_mut.deref_mut() { if Self::check_and_init( diff --git a/src/graph_impl/multi_graph/plan/query_plan_worker.rs b/src/graph_impl/multi_graph/plan/query_plan_worker.rs index 055ea56b..c5d130cc 100644 --- a/src/graph_impl/multi_graph/plan/query_plan_worker.rs +++ b/src/graph_impl/multi_graph/plan/query_plan_worker.rs @@ -48,19 +48,16 @@ impl QPWorkers { for last_op in &mut query_plan.subplans { let mut op = last_op.clone(); loop { - { - let op_ref = op.borrow(); - if get_op_attr_as_ref!(op.borrow().deref(), prev).is_none() { - break; - } - } - op = { + let prev = { let op_ref = op.borrow(); get_op_attr_as_ref!(op_ref.deref(), prev) .as_ref() - .unwrap() - .clone() + .map(|op| op.clone()) }; + if prev.is_none() { + break; + } + op = prev.as_ref().unwrap().clone(); } let mut op_mut = op.borrow_mut(); if let Operator::Scan(Scan::ScanBlocking(sb)) = op_mut.deref_mut() { diff --git a/src/graph_impl/multi_graph/planner/catalog/catalog.rs b/src/graph_impl/multi_graph/planner/catalog/catalog.rs index cb27c360..198264a8 100644 --- a/src/graph_impl/multi_graph/planner/catalog/catalog.rs +++ b/src/graph_impl/multi_graph/planner/catalog/catalog.rs @@ -21,7 +21,7 @@ use hashbrown::{HashMap, HashSet}; use itertools::Itertools; use std::cell::RefCell; use std::hash::Hash; -use std::ops::Deref; +use std::ops::{Deref, DerefMut}; use std::rc::Rc; use std::time::SystemTime; @@ -37,15 +37,15 @@ pub struct Catalog { is_sorted_by_node: bool, num_sampled_edge: usize, max_input_num_vertices: usize, - elapsed_time: u128, + pub elapsed_time: u128, } impl Catalog { pub fn new(num_sampled_edge: usize, max_input_num_vertices: usize) -> Self { Self { in_subgraphs: vec![], - sampled_icost: Default::default(), - sampled_selectivity: Default::default(), + sampled_icost: HashMap::new(), + sampled_selectivity: HashMap::new(), is_sorted_by_node: false, num_sampled_edge, max_input_num_vertices, @@ -260,8 +260,8 @@ impl Catalog { ) { for query_plan in query_plan_arr { let probe_tuple = vec![Id::new(0); self.max_input_num_vertices + 1]; - if let Some(scan) = query_plan.get_scan_sampling() { - scan.init(probe_tuple, graph); + if let Some(scan) = &mut query_plan.scan_sampling { + scan.borrow_mut().init(probe_tuple, graph); } } } @@ -276,9 +276,9 @@ impl Catalog { // sink.execute(); // })); } - // for handler in handlers { - // handler.join(); - // } + // for handler in handlers { + // handler.join(); + // } } else { let mut sink = query_plan_arr[0].sink.as_mut().unwrap().borrow_mut(); sink.execute(); @@ -356,7 +356,7 @@ impl Catalog { for i in 0..next.len() { let next_i = next[i].borrow(); if let Operator::EI(EI::Intersect(Intersect::IntersectCatalog(intersect))) = - next_i.deref() + next_i.deref() { let to_type = intersect.base_intersect.base_ei.to_type; let mut alds_as_str_list = vec![]; @@ -466,7 +466,7 @@ impl Catalog { for (i, next) in next_vec.iter().enumerate() { let next_ref = next.borrow(); if let Operator::EI(EI::Intersect(Intersect::IntersectCatalog(intersect))) = - next_ref.deref() + next_ref.deref() { let alds = &intersect.base_intersect.base_ei.alds; let mut alds_as_str_list = vec![]; diff --git a/src/graph_impl/multi_graph/planner/catalog/catalog_plans.rs b/src/graph_impl/multi_graph/planner/catalog/catalog_plans.rs index 6fbf5652..b4455d01 100644 --- a/src/graph_impl/multi_graph/planner/catalog/catalog_plans.rs +++ b/src/graph_impl/multi_graph/planner/catalog/catalog_plans.rs @@ -38,7 +38,6 @@ pub struct CatalogPlans { query_plans_arrs: Vec>>, is_directed: bool, selectivity_zero: Vec<(QueryGraph, Vec, usize)>, - query_vertices: Vec, query_vertex_to_idx_map: HashMap, } @@ -59,50 +58,51 @@ impl CatalogPlans { query_plans_arrs: vec![], is_directed: graph.is_directed(), selectivity_zero: vec![], - query_vertices: vec![ - "a".to_string(), - "b".to_string(), - "c".to_string(), - "d".to_string(), - "e".to_string(), - "f".to_string(), - "g".to_string(), - ], query_vertex_to_idx_map: HashMap::new(), }; - for (i, v) in plans.query_vertices.iter().enumerate() { - plans.query_vertex_to_idx_map.insert(v.clone(), i); + for i in 0..QUERY_VERTICES.len() { + plans + .query_vertex_to_idx_map + .insert(QUERY_VERTICES[i].to_owned(), i); } let scans = if graph.edge_count() > 1073741823 { plans.generate_all_scans_for_large_graph(graph) } else { plans.generate_all_scans(graph) }; - for mut scan in scans { + for scan in scans { let mut noop = Noop::new(scan.base_scan.base_op.out_subgraph.clone()); - scan.base_scan.base_op.next = vec![Rc::new(RefCell::new(Operator::Noop(noop.clone())))]; - noop.base_op.prev = Some(Rc::new(RefCell::new(Operator::Scan(Scan::ScanSampling( - scan.clone(), - ))))); - noop.base_op.out_qvertex_to_idx_map = - scan.base_scan.base_op.out_qvertex_to_idx_map.clone(); + let scan_op = Rc::new(RefCell::new(Operator::Scan(Scan::ScanSampling(scan)))); + noop.base_op.prev = Some(scan_op.clone()); + noop.base_op.out_qvertex_to_idx_map = { + let scan_ref = scan_op.borrow(); + get_op_attr_as_ref!(scan_ref.deref(), out_qvertex_to_idx_map).clone() + }; let mut noop_op = Rc::new(RefCell::new(Operator::Noop(noop))); - plans.set_next_operators(graph, &mut noop_op, false); + *get_op_attr_as_mut!(scan_op.borrow_mut().deref_mut(), next) = vec![noop_op.clone()]; + plans.set_next_operators(graph, noop_op, false); let mut query_plans_arr = vec![]; - query_plans_arr.push(QueryPlan::new(scan.clone())); + query_plans_arr.push(QueryPlan::new(scan_op.clone())); for i in 1..num_thread { - let mut scan_copy = scan.copy_default(); - let mut base_scan_copy = get_base_op_as_mut!(&mut scan_copy); - let mut another_noop = Noop::new(base_scan_copy.out_subgraph.clone()); - base_scan_copy.next = - vec![Rc::new(RefCell::new(Operator::Noop(another_noop.clone())))]; + let mut scan_ref = scan_op.borrow(); + let scan_copy = if let Operator::Scan(Scan::ScanSampling(scan)) = scan_ref.deref() { + scan.copy_default() + } else { + panic!("Scan initial failed!"); + }; + let mut another_noop = + Noop::new(get_op_attr_as_ref!(&scan_copy, out_subgraph).clone()); another_noop.base_op.out_qvertex_to_idx_map = - base_scan_copy.out_qvertex_to_idx_map.clone(); - another_noop.base_op.prev = Some(Rc::new(RefCell::new(scan_copy.clone()))); + get_op_attr_as_ref!(&scan_copy, out_qvertex_to_idx_map).clone(); + let mut scan_copy = Rc::new(RefCell::new(scan_copy)); + another_noop.base_op.prev = Some(scan_copy.clone()); let mut another_noop_op = Rc::new(RefCell::new(Operator::Noop(another_noop))); - plans.set_next_operators(graph, &mut another_noop_op, true); - if let Operator::Scan(Scan::ScanSampling(sc)) = scan_copy { - query_plans_arr.push(QueryPlan::new(sc)); + *get_op_attr_as_mut!(scan_copy.borrow_mut().deref_mut(), next) = + vec![another_noop_op.clone()]; + plans.set_next_operators(graph, another_noop_op, true); + let scan_copy_ref = scan_copy.borrow(); + if let Operator::Scan(Scan::ScanSampling(sc)) = scan_copy_ref.deref() { + query_plans_arr.push(QueryPlan::new(scan_copy.clone())); } } plans.query_plans_arrs.push(query_plans_arr); @@ -113,18 +113,14 @@ impl CatalogPlans { pub fn set_next_operators( &mut self, graph: &TypedStaticGraph, - operator: &mut Rc>>, + operator: Rc>>, is_none: bool, ) { let mut in_subgraph = { let op_ref = operator.borrow(); - get_op_attr_as_ref!(op_ref.deref(),out_subgraph).clone() + get_op_attr_as_ref!(op_ref.deref(), out_subgraph).clone() }; - if !is_none - && !self - .query_graphs_to_extend - .contains(&mut in_subgraph) - { + if !is_none && !self.query_graphs_to_extend.contains(&mut in_subgraph) { self.query_graphs_to_extend.add(in_subgraph.clone()); } else if !is_none { return; @@ -132,8 +128,10 @@ impl CatalogPlans { let query_vertices = in_subgraph.get_query_vertices().clone(); let mut descriptors = vec![]; + println!("call here={:?}",set_utils::get_power_set_excluding_empty_set(query_vertices.clone())); for query_vertex_to_extend in set_utils::get_power_set_excluding_empty_set(query_vertices) { for alds in self.generate_alds(query_vertex_to_extend, self.is_directed) { + println!("rec call here={:?}",in_subgraph.qvertex_to_qedges_map.keys().len()); descriptors.push(Descriptor { out_subgraph: self.get_out_subgraph(in_subgraph.copy(), alds.clone()), alds, @@ -144,17 +142,18 @@ impl CatalogPlans { let mut next = vec![]; let last_repeated_vertex_idx = get_op_attr_as_ref!(operator.borrow().deref(), last_repeated_vertex_idx).clone(); + if self.sorted_by_node { for mut descriptor in descriptors { let mut types = vec![]; - for to_type in 0..self.num_node_labels { + for to_type in 0..=self.num_node_labels { let mut produces_output = true; for ald in &descriptor.alds { let from_type = in_subgraph.get_query_vertex_type(&ald.from_query_vertex); if (ald.direction == Direction::Fwd && 0 == graph.get_num_edges(from_type, to_type, ald.label)) || (ald.direction == Direction::Bwd - && 0 == graph.get_num_edges(to_type, from_type, ald.label)) + && 0 == graph.get_num_edges(to_type, from_type, ald.label)) { produces_output = false; break; @@ -177,7 +176,6 @@ impl CatalogPlans { descriptor .out_subgraph .set_query_vertex_type(to_qvertex.to_owned(), to_type); - let p = to_qvertex.clone(); let mut intersect = IntersectCatalog::new( to_qvertex.to_owned(), to_type, @@ -199,11 +197,12 @@ impl CatalogPlans { } else { for i in 0..descriptors.len() { let descriptor = &descriptors[i]; - let mut out_qvertex_to_idx_map ={ + let mut out_qvertex_to_idx_map = { let op_ref = operator.borrow(); let prev = get_op_attr_as_ref!(op_ref.deref(), prev) .as_ref() - .unwrap().clone(); + .unwrap() + .clone(); let prev_ref = prev.borrow(); get_op_attr_as_ref!(prev_ref.deref(), out_qvertex_to_idx_map).clone() }; @@ -226,12 +225,12 @@ impl CatalogPlans { ))))); } } - Self::set_next_pointer(operator, &mut next); - for mut next_op in next { + Self::set_next_pointer(operator.clone(), next.clone()); + for next_op in next { let mut next_noops = if self.sorted_by_node { vec![Noop::new(QueryGraph::empty()); 1] } else { - vec![Noop::new(QueryGraph::empty()); self.num_node_labels] + vec![Noop::new(QueryGraph::empty()); self.num_node_labels+1] }; self.set_noops( get_op_attr_as_ref!(next_op.borrow().deref(), out_subgraph), @@ -239,29 +238,26 @@ impl CatalogPlans { &mut next_noops, get_op_attr_as_ref!(next_op.borrow().deref(), out_qvertex_to_idx_map), ); - let mut next_noops = next_noops + let next_noops: Vec>>> = next_noops .into_iter() .map(|noop| Rc::new(RefCell::new(Operator::Noop(noop)))) .collect(); - Self::set_next_pointer(&mut next_op, &mut next_noops); + Self::set_next_pointer(next_op.clone(), next_noops.clone()); if get_op_attr_as_ref!(next_op.borrow().deref(), out_subgraph).get_num_qvertices() <= self.max_input_num_vertices { - for mut next_noop in next_noops { + for next_noop in next_noops { *get_op_attr_as_mut!( next_noop.borrow_mut().deref_mut(), last_repeated_vertex_idx ) = last_repeated_vertex_idx; - self.set_next_operators(graph, &mut next_noop, false) + self.set_next_operators(graph, next_noop, is_none) } } } } - fn set_next_pointer( - operator: &mut Rc>>, - next: &mut Vec>>>, - ) { + fn set_next_pointer(operator: Rc>>, next: Vec>>>) { *get_op_attr_as_mut!(operator.borrow_mut().deref_mut(), next) = next.clone(); for next_op in next { *get_op_attr_as_mut!(next_op.borrow_mut().deref_mut(), prev) = Some(operator.clone()); @@ -278,8 +274,9 @@ impl CatalogPlans { if self.sorted_by_node { noops[0] = Noop::new(query_graph.clone()); noops[0].base_op.out_qvertex_to_idx_map = out_qvertex_to_idx_map.clone(); + println!("call here!") } else { - for to_type in 0..self.num_node_labels { + for to_type in 0..=self.num_node_labels { let mut query_graph_copy = query_graph.copy(); query_graph_copy.set_query_vertex_type(to_qvertex.clone(), to_type); noops[to_type] = Noop::new(query_graph_copy); @@ -367,23 +364,25 @@ impl CatalogPlans { alds: Vec, ) -> QueryGraph { let num_qvertices = query_graph.get_num_qvertices(); + println!("get_out_sub={},{}",num_qvertices,QUERY_VERTICES[num_qvertices]); for ald in alds { let mut query_edge = if let Direction::Fwd = ald.direction { let mut query_edge = QueryEdge::default( ald.from_query_vertex.clone(), - self.query_vertices[num_qvertices].clone(), + QUERY_VERTICES[num_qvertices].to_owned(), ); query_edge.from_type = query_graph.get_query_vertex_type(&ald.from_query_vertex); query_edge } else { let mut query_edge = QueryEdge::default( - self.query_vertices[num_qvertices].clone(), + QUERY_VERTICES[num_qvertices].to_owned(), ald.from_query_vertex.clone(), ); query_edge.to_type = query_graph.get_query_vertex_type(&ald.from_query_vertex); query_edge }; query_edge.label = ald.label; +// println!("qedge={}->{},query_edge={:?}", query_edge.from_query_vertex, query_edge.to_query_vertex,query_graph.qvertex_to_qedges_map.keys()); query_graph.add_qedge(query_edge); } query_graph @@ -406,9 +405,9 @@ impl CatalogPlans { .as_ref() .unwrap() .get_neighbor_ids() - { - edges.push(vec![Id::new(from_vertex), to_vertex.clone()]); - } + { + edges.push(vec![Id::new(from_vertex), to_vertex.clone()]); + } } let mut out_subgraph = QueryGraph::empty(); out_subgraph.add_qedge(QueryEdge::new("a".to_owned(), "b".to_owned(), 0, 0, 0)); @@ -426,7 +425,6 @@ impl CatalogPlans { let num_vertices = graph.node_count(); let mut key_to_edges_map = HashMap::new(); let mut key_to_curr_idx = HashMap::new(); - let node_labels = graph.get_node_types(); for from_type in 0..=self.num_node_labels { for label in 0..=self.num_edge_labels { for to_type in 0..=self.num_node_labels { @@ -464,40 +462,40 @@ impl CatalogPlans { } } } - //TODO: verify is there need ..=? let mut scans = vec![]; - for from_type in 0..self.num_node_labels { - for label in 0..self.num_edge_labels { - for to_type in 0..self.num_node_labels { + for from_type in 0..=self.num_node_labels { + for edge_label in 0..=self.num_edge_labels { + for to_type in 0..=self.num_node_labels { let mut out_subgraph = QueryGraph::empty(); out_subgraph.add_qedge(QueryEdge::new( "a".to_owned(), "b".to_owned(), from_type, to_type, - label, + edge_label, )); let edge_key = TypedStaticGraph::::get_edge_key( - from_type, to_type, label, + from_type, to_type, edge_label, ); - let actual_num_edges = graph.get_num_edges(from_type, to_type, label); - if actual_num_edges > 0 { - let mut num_edges_to_sample = self.num_sampled_edges - * (graph.get_num_edges(from_type, to_type, label) / graph.edge_count()) - as usize; - let mut scan = ScanSampling::new(out_subgraph); - if self.sorted_by_node && num_edges_to_sample < 1000 { - num_edges_to_sample = actual_num_edges; - } - scan.set_edge_indices_to_sample( - key_to_edges_map[&edge_key] - .iter() - .map(|edge| Id::new(edge.clone())) - .collect(), - num_edges_to_sample, - ); - scans.push(scan); + let actual_num_edges = graph.get_num_edges(from_type, to_type, edge_label); + if actual_num_edges <= 0 { + continue; + } + let mut num_edges_to_sample = self.num_sampled_edges + * (graph.get_num_edges(from_type, to_type, edge_label) / graph.edge_count()) + as usize; + let mut scan = ScanSampling::new(out_subgraph); + if self.sorted_by_node && num_edges_to_sample < 1000 { + num_edges_to_sample = actual_num_edges; } + scan.set_edge_indices_to_sample( + key_to_edges_map[&edge_key] + .iter() + .map(|edge| Id::new(edge.clone())) + .collect(), + num_edges_to_sample, + ); + scans.push(scan); } } } diff --git a/src/graph_impl/multi_graph/planner/catalog/query_graph.rs b/src/graph_impl/multi_graph/planner/catalog/query_graph.rs index 4637715d..e3401bee 100644 --- a/src/graph_impl/multi_graph/planner/catalog/query_graph.rs +++ b/src/graph_impl/multi_graph/planner/catalog/query_graph.rs @@ -175,8 +175,8 @@ impl QueryGraph { // Get the vertex IDs. let from_qvertex = query_edge.from_query_vertex.clone(); let to_qvertex = query_edge.to_query_vertex.clone(); - let from_type = query_edge.from_type.clone(); - let to_type = query_edge.to_type.clone(); + let from_type = query_edge.from_type; + let to_type = query_edge.to_type; self.qvertex_to_type_map .entry(from_qvertex.clone()) .or_insert(0); @@ -195,33 +195,40 @@ impl QueryGraph { self.qvertex_to_deg_map .insert(from_qvertex.clone(), vec![0; 2]); } - *self - .qvertex_to_deg_map - .get_mut(&from_qvertex) - .unwrap() - .get_mut(0) - .unwrap() += 1; + self.qvertex_to_deg_map.get_mut(&from_qvertex).unwrap()[0] += 1; if !self.qvertex_to_deg_map.contains_key(&to_qvertex) { self.qvertex_to_deg_map .insert(to_qvertex.clone(), vec![0; 2]); } - self.qvertex_to_deg_map - .get_mut(&to_qvertex) - .map(|to| to[1] += 1); + self.qvertex_to_deg_map.get_mut(&to_qvertex).unwrap()[1] += 1; // Add fwd edge from_qvertex -> to_qvertex to the qVertexToQEdgesMap. - self.add_qedge_to_qgraph(from_qvertex.clone(), to_qvertex.clone(), query_edge.clone()); + self.add_qedge_to_qgraph(&from_qvertex, &to_qvertex, &query_edge); // Add bwd edge to_qvertex <- from_qvertex to the qVertexToQEdgesMap. - self.add_qedge_to_qgraph(to_qvertex.clone(), from_qvertex.clone(), query_edge.clone()); + self.add_qedge_to_qgraph(&to_qvertex, &from_qvertex, &query_edge); + self.q_edges.push(query_edge); } - fn add_qedge_to_qgraph(&mut self, from_qvertex: String, to_qvertex: String, q_edge: QueryEdge) { + fn add_qedge_to_qgraph( + &mut self, + from_qvertex: &String, + to_qvertex: &String, + q_edge: &QueryEdge, + ) { self.qvertex_to_qedges_map .entry(from_qvertex.clone()) .or_insert(HashMap::new()); self.qvertex_to_qedges_map - .get_mut(&from_qvertex) - .map(|qedge_map| qedge_map.entry(to_qvertex).or_insert(vec![q_edge])); + .get_mut(from_qvertex) + .unwrap() + .entry(to_qvertex.clone()) + .or_insert(vec![]); + self.qvertex_to_qedges_map + .get_mut(from_qvertex) + .unwrap() + .get_mut(to_qvertex) + .unwrap() + .push(q_edge.clone()); } pub fn get_neighbors(&self, from_var: Vec) -> HashSet { diff --git a/src/graph_impl/multi_graph/planner/catalog/subgraph_mapping_iterator.rs b/src/graph_impl/multi_graph/planner/catalog/subgraph_mapping_iterator.rs index 5243e901..33163dd7 100644 --- a/src/graph_impl/multi_graph/planner/catalog/subgraph_mapping_iterator.rs +++ b/src/graph_impl/multi_graph/planner/catalog/subgraph_mapping_iterator.rs @@ -83,6 +83,10 @@ impl SubgraphMappingIterator { pub fn has_next(&mut self) -> bool { if !self.is_next_computed { + println!( + "PRE: curr_mapping={:?},self.o_qvertices={:?}", + self.curr_mapping, self.o_qvertices + ); if self.curr_mapping.len() == self.o_qvertices.len() { self.curr_mapping.pop(); } @@ -176,6 +180,10 @@ impl SubgraphMappingIterator { self.is_next_computed = false; return self.has_next(); } + println!( + "LATER:cur_mapping={:?},self.o_qvertices={:?}", + self.curr_mapping, self.o_qvertices + ); !self.curr_mapping.is_empty() } diff --git a/src/graph_impl/multi_graph/planner/query_planner.rs b/src/graph_impl/multi_graph/planner/query_planner.rs index e6fcf512..9895093b 100644 --- a/src/graph_impl/multi_graph/planner/query_planner.rs +++ b/src/graph_impl/multi_graph/planner/query_planner.rs @@ -88,17 +88,18 @@ impl pub fn set_next_pointers(&self, best_plan: &mut QueryPlan) { best_plan.subplans.iter_mut().for_each(|last_op| { let mut cur_op = last_op.clone(); - loop{ + loop { let prev_op = { let cur_op_ref = cur_op.borrow(); - get_op_attr_as_ref!(cur_op_ref.deref(), prev).as_ref().map(|op|op.clone()) + get_op_attr_as_ref!(cur_op_ref.deref(), prev) + .as_ref() + .map(|op| op.clone()) }; - if prev_op.is_none(){ + if prev_op.is_none() { break; } let mut prev = prev_op.unwrap(); - let mut prev_ref = prev.borrow_mut(); - *get_op_attr_as_mut!(prev_ref.deref_mut(), next) = vec![cur_op.clone()]; + *get_op_attr_as_mut!(prev.borrow_mut().deref_mut(), next) = vec![cur_op.clone()]; cur_op = prev.clone() } }); @@ -192,10 +193,13 @@ impl to_qvertex: &String, ) -> (String, QueryPlan) { let last_operator = prev_query_plan.last_operator.as_ref().unwrap(); - let (in_subgraph,last_previous_repeated_index) = { + let (in_subgraph, last_previous_repeated_index) = { let last_op_ref = last_operator.borrow(); let base_op = get_base_op_as_ref!(last_op_ref.deref()); - (base_op.out_subgraph.clone(),base_op.last_repeated_vertex_idx) + ( + base_op.out_subgraph.clone(), + base_op.last_repeated_vertex_idx, + ) }; let mut alds = vec![]; let mut next_extend = self.get_next_ei(&in_subgraph, to_qvertex, &mut alds, last_operator); @@ -207,11 +211,14 @@ impl .base_op .out_subgraph .get_query_vertex_type(to_qvertex); - let (mut in_subgraph,last_prev) = { + let (mut in_subgraph, last_prev) = { let last_operator = prev_query_plan.last_operator.as_mut().unwrap(); let last_ref = last_operator.borrow(); let base_op = get_base_op_as_ref!(last_ref.deref()); - (base_op.out_subgraph.clone(),base_op.prev.as_ref().map(|op|op.clone())) + ( + base_op.out_subgraph.clone(), + base_op.prev.as_ref().map(|op| op.clone()), + ) }; let estimated_selectivity = self.get_selectivity( &mut in_subgraph, @@ -257,7 +264,9 @@ impl .catalog .get_icost(&mut in_subgraph, alds_to_always_intersect, to_type); let cached_intersect_icost = out_tuples_to_process - * self.catalog.get_icost(&mut in_subgraph, alds_to_cache, to_type); + * self + .catalog + .get_icost(&mut in_subgraph, alds_to_cache, to_type); icost = prev_estimated_num_out_tuples * always_intersect_icost + out_tuples_to_process * cached_intersect_icost + // added to make caching effect on cost more robust. @@ -521,15 +530,12 @@ impl let last_op = build_subplan.last_operator.as_ref().unwrap(); let last_op_ref = last_op.borrow(); - let out_subgraph = get_op_attr_as_ref!(last_op_ref.deref(),out_subgraph); - out_subgraph - .get_query_vertices() - .iter() - .for_each(|v| { - q_vertex_to_num_out_tuples - .entry(v.clone()) - .or_insert(curr_best_query_plan.estimated_num_out_tuples); - }); + let out_subgraph = get_op_attr_as_ref!(last_op_ref.deref(), out_subgraph); + out_subgraph.get_query_vertices().iter().for_each(|v| { + q_vertex_to_num_out_tuples + .entry(v.clone()) + .or_insert(curr_best_query_plan.estimated_num_out_tuples); + }); query_plan.q_vertex_to_num_out_tuples = q_vertex_to_num_out_tuples; diff --git a/src/graph_impl/multi_graph/planner/query_planner_big.rs b/src/graph_impl/multi_graph/planner/query_planner_big.rs index bc0bb257..61307467 100644 --- a/src/graph_impl/multi_graph/planner/query_planner_big.rs +++ b/src/graph_impl/multi_graph/planner/query_planner_big.rs @@ -27,7 +27,7 @@ pub struct QueryPlannerBig -QueryPlannerBig + QueryPlannerBig { pub fn new( query_graph: QueryGraph, @@ -120,7 +120,10 @@ QueryPlannerBig let last_base_op = prev_query_plan.last_operator.as_ref().unwrap(); let last_base_op_ref = last_base_op.borrow(); let last_op = get_base_op_as_ref!(last_base_op_ref.deref()); - (last_op.out_subgraph.get_query_vertices(), last_op.out_subgraph.clone()) + ( + last_op.out_subgraph.get_query_vertices(), + last_op.out_subgraph.clone(), + ) }; let to_qvertices = self.base_planner.query_graph.get_neighbors(prev_qvertices); let next_to_qvertices = Self::filter_to_qvertices_by_max_num_alds( diff --git a/src/graph_impl/multi_graph/runner/catalog_generator.rs b/src/graph_impl/multi_graph/runner/catalog_generator.rs index 4a82aaf1..e60e105a 100644 --- a/src/graph_impl/multi_graph/runner/catalog_generator.rs +++ b/src/graph_impl/multi_graph/runner/catalog_generator.rs @@ -9,21 +9,11 @@ use std::hash::Hash; pub fn default( graph: &TypedStaticGraph, ) -> Catalog { - get( - graph, - DEF_MAX_INPUT_NUM_VERTICES, - DEF_NUM_EDGES_TO_SAMPLE, - 1, - ) -} - -pub fn get( - graph: &TypedStaticGraph, - max_input_num_vertices: usize, - num_sampled_edges: usize, - num_threads: usize, -) -> Catalog { - let mut catalog = Catalog::new(num_sampled_edges, max_input_num_vertices); - catalog.populate(graph, num_threads); + let mut catalog = Catalog::new(DEF_NUM_EDGES_TO_SAMPLE, DEF_MAX_INPUT_NUM_VERTICES); + catalog.populate(graph, 1); + println!( + "Catalog generation finished in {} (ms)", + catalog.elapsed_time + ); catalog } diff --git a/src/graph_impl/static_graph/graph.rs b/src/graph_impl/static_graph/graph.rs index d7f3e184..2e2c2fd5 100644 --- a/src/graph_impl/static_graph/graph.rs +++ b/src/graph_impl/static_graph/graph.rs @@ -70,7 +70,7 @@ pub struct TypedStaticGraph, // node_types[node_id] = node_label_id - // the node_label_id has been shifted right and id 0 is prepared for no label item. + // `node_label_id` has been shifted right and id 0 is prepared for no label item. node_types: Vec, node_type_offsets: Vec, fwd_adj_lists: Vec>>, @@ -78,9 +78,9 @@ pub struct TypedStaticGraph, label_to_largest_fwd_adj_list_size: Vec, label_to_largest_bwd_adj_list_size: Vec, - edge_key_to_num_edges_map: HashMap, - to_type_to_percentage_map: HashMap, - from_type_to_percentage_map: HashMap, + edge_key_to_num_edges_map: HashMap, + to_label_to_percentage_map: HashMap, + from_label_to_percentage_map: HashMap, } impl PartialEq @@ -221,14 +221,14 @@ impl label_to_largest_fwd_adj_list_size: vec![], label_to_largest_bwd_adj_list_size: vec![], edge_key_to_num_edges_map: HashMap::new(), - to_type_to_percentage_map: HashMap::new(), + to_label_to_percentage_map: HashMap::new(), edge_vec: edges, in_edge_vec: in_edges, labels: None, node_label_map: SetMap::::new(), edge_label_map: SetMap::::new(), graph_type: PhantomData, - from_type_to_percentage_map: HashMap::new(), + from_label_to_percentage_map: HashMap::new(), }; g.init_graphflow(); g @@ -298,14 +298,14 @@ impl label_to_largest_fwd_adj_list_size: vec![], label_to_largest_bwd_adj_list_size: vec![], edge_key_to_num_edges_map: HashMap::new(), - to_type_to_percentage_map: HashMap::new(), + to_label_to_percentage_map: HashMap::new(), edge_vec: edges, in_edge_vec: in_edges, labels: Some(labels), node_label_map, edge_label_map, graph_type: PhantomData, - from_type_to_percentage_map: HashMap::new(), + from_label_to_percentage_map: HashMap::new(), }; g.init_graphflow(); g @@ -381,14 +381,14 @@ impl label_to_largest_fwd_adj_list_size: vec![], label_to_largest_bwd_adj_list_size: vec![], edge_key_to_num_edges_map: HashMap::new(), - to_type_to_percentage_map: HashMap::new(), + to_label_to_percentage_map: HashMap::new(), edge_vec, in_edge_vec, labels, node_label_map, edge_label_map, graph_type: PhantomData, - from_type_to_percentage_map: HashMap::new(), + from_label_to_percentage_map: HashMap::new(), }; g.init_graphflow(); g @@ -405,11 +405,17 @@ impl self.label_to_num_edges = vec![0; label_cnt]; self.label_to_largest_fwd_adj_list_size = vec![0; label_cnt]; self.label_to_largest_bwd_adj_list_size = vec![0; label_cnt]; - for i in 0..self.num_nodes { - self.num_edges += self.fwd_adj_lists[i].as_ref().unwrap().len(); + for vertex_id in 0..self.num_nodes { + self.num_edges += self.fwd_adj_lists[vertex_id].as_ref().unwrap().len(); for label in 0..label_cnt { - let fwd_adj_size = self.fwd_adj_lists[i].as_ref().unwrap().sub_len(label); - let bwd_adj_size = self.bwd_adj_lists[i].as_ref().unwrap().sub_len(label); + let fwd_adj_size = self.fwd_adj_lists[vertex_id] + .as_ref() + .unwrap() + .sub_len(label); + let bwd_adj_size = self.bwd_adj_lists[vertex_id] + .as_ref() + .unwrap() + .sub_len(label); self.label_to_num_edges[label] += fwd_adj_size; if fwd_adj_size > self.label_to_largest_fwd_adj_list_size[label] { self.label_to_largest_fwd_adj_list_size[label] = fwd_adj_size; @@ -419,23 +425,24 @@ impl } } } - let num_vertices = self.num_nodes; - for from in 0..self.node_types.len() { - for to in 0..self.node_types.len() { - for label in 0..self.node_types.len() { + + // init count + for from in 0..=self.num_of_node_labels() { + for to in 0..=self.num_of_node_labels() { + for label in 0..=self.num_of_edge_labels() { let edge = Self::get_edge_key(from, to, label); self.edge_key_to_num_edges_map.entry(edge).or_insert(0); let to_label = Self::get_edge_key_by_label(label, to); - self.to_type_to_percentage_map.entry(to_label).or_insert(0); + self.to_label_to_percentage_map.entry(to_label).or_insert(0); let from_label = Self::get_edge_key_by_label(from, label); - self.from_type_to_percentage_map + self.from_label_to_percentage_map .entry(from_label) .or_insert(0); } } } - for from in 0..num_vertices { + for from in 0..self.num_nodes { let from_type = self.node_types[from]; let offsets = self.fwd_adj_lists[from] .as_ref() @@ -444,7 +451,7 @@ impl .clone(); if self.sort_by_node { let label = 0; - for to_type in 0..offsets.len() { + for to_type in 0..(offsets.len() - 1) { let num_edges = offsets[to_type + 1] - offsets[to_type]; self.add_edge_count(from_type, to_type, label, num_edges); } @@ -457,14 +464,12 @@ impl for label in 0..(offsets.len() - 1) { for to_idx in offsets[label]..offsets[label + 1] { let to_type = self.node_types[neighbours[to_idx].id()]; - // if from_type == 1 && to_type == 3 && label == 3 { - // let x = 0; - // } self.add_edge_count(from_type, to_type, label, 1); } } } } + println!("{:?}", self.label_to_num_edges); } fn add_edge_count(&mut self, from_type: usize, to_type: usize, label: usize, num_edges: usize) { @@ -473,12 +478,12 @@ impl self.edge_key_to_num_edges_map .insert(edge, num_edges_origin + num_edges); let to_label = Self::get_edge_key_by_label(label, to_type); - let to_percentage = self.to_type_to_percentage_map.get(&to_label).unwrap(); - self.to_type_to_percentage_map + let to_percentage = self.to_label_to_percentage_map.get(&to_label).unwrap(); + self.to_label_to_percentage_map .insert(to_label, to_percentage + num_edges); let from_label = Self::get_edge_key_by_label(from_type, label); - let from_percentage = self.from_type_to_percentage_map.get(&from_label).unwrap(); - self.from_type_to_percentage_map + let from_percentage = self.from_label_to_percentage_map.get(&from_label).unwrap(); + self.from_label_to_percentage_map .insert(from_label, from_percentage + num_edges); } @@ -576,32 +581,22 @@ impl if from_type == 0 && to_type == 0 { return self.label_to_num_edges[label]; } else if from_type != 0 && to_type != 0 { - return self - .edge_key_to_num_edges_map - .get(&Self::get_edge_key(from_type, to_type, label)) - .unwrap() - .clone(); + return self.edge_key_to_num_edges_map[&Self::get_edge_key(from_type, to_type, label)]; } else if from_type != 0 { - return self - .from_type_to_percentage_map - .get(&Self::get_edge_key_by_label(from_type, label)) - .unwrap() - .clone(); + return self.from_label_to_percentage_map + [&Self::get_edge_key_by_label(from_type, label)]; } - self.to_type_to_percentage_map - .get(&Self::get_edge_key_by_label(label, to_type)) - .unwrap() - .clone() + self.to_label_to_percentage_map[&Self::get_edge_key_by_label(label, to_type)] } - pub fn get_edge_key(from_type: usize, to_type: usize, label: usize) -> i64 { - (((from_type & 0xFFFF) << 48) as i64) - | (((to_type & 0x0000FFFF) << 16) as i64) - | ((label & 0xFFFF) as i64) + pub fn get_edge_key(from_type: usize, to_type: usize, label: usize) -> u64 { + (((from_type & 0xFFFF) << 48) as u64) + | (((label & 0x0000FFFF) << 16) as u64) + | ((to_type & 0xFFFF) as u64) } - fn get_edge_key_by_label(from_label: usize, to_label: usize) -> i32 { - (((from_label & 0x0000FFFF) << 16) as i32) | ((to_label & 0xFFFF) as i32) + fn get_edge_key_by_label(from_label: usize, to_label: usize) -> u32 { + (((from_label & 0x0000FFFF) << 16) as u32) | ((to_label & 0xFFFF) as u32) } // Partition nodes by type and generating node_ids && offsets for retrieving. @@ -640,7 +635,8 @@ impl // Partition edges by edge label or node label(if there did not exist edge labels in graph) fn partition_edges(&mut self) { - self.sort_by_node = self.num_of_edge_labels() == 0 && self.num_of_node_labels() > 0; + // if only one label used + al least 2 vertex type used, then sorted by node + self.sort_by_node = self.num_of_edge_labels() == 1 && self.num_of_node_labels() > 1; let (fwd_adj_meta_data, bwd_adj_meta_data) = self.get_adj_meta_data(); let num_vertices = self.num_nodes; let mut fwd_adj_lists: Vec>> = vec![Option::None; num_vertices]; @@ -672,26 +668,26 @@ impl vec![(from, to)] }) .for_each(|(from, to)| { - let label_id = self + let edge_label_id = self .get_edge(from, to) .get_label_id() .map(|op| op.id() + 1) .unwrap_or(0); - let (from_type_or_label, to_type_or_label) = if self.sort_by_node { + let (from_label_id, to_label_id) = if self.sort_by_node { (self.node_types[from.id()], self.node_types[to.id()]) } else { - (label_id, label_id) + (edge_label_id, edge_label_id) }; - let mut idx = fwd_adj_list_curr_idx.get(&from.id()).unwrap()[to_type_or_label]; - let mut offset = fwd_adj_meta_data.get(&from.id()).unwrap()[to_type_or_label]; - fwd_adj_list_curr_idx.get_mut(&from.id()).unwrap()[to_type_or_label] += 1; + let mut idx = fwd_adj_list_curr_idx[&from.id()][to_label_id]; + let mut offset = fwd_adj_meta_data[&from.id()][to_label_id]; + fwd_adj_list_curr_idx.get_mut(&from.id()).unwrap()[to_label_id] += 1; fwd_adj_lists[from.id()] .as_mut() .unwrap() .set_neighbor_id(to, offset + idx); - idx = bwd_adj_list_curr_idx.get(&to.id()).unwrap()[from_type_or_label]; - offset = bwd_adj_meta_data.get(&to.id()).unwrap()[from_type_or_label]; - bwd_adj_list_curr_idx.get_mut(&to.id()).unwrap()[from_type_or_label] += 1; + idx = bwd_adj_list_curr_idx[&to.id()][from_label_id]; + offset = bwd_adj_meta_data[&to.id()][from_label_id]; + bwd_adj_list_curr_idx.get_mut(&to.id()).unwrap()[from_label_id] += 1; bwd_adj_lists[to.id()] .as_mut() .unwrap() @@ -740,16 +736,14 @@ impl fn get_adj_meta_data(&self) -> (HashMap>, HashMap>) { let mut fwd_adj_list_metadata = HashMap::new(); let mut bwd_adj_list_metadata = HashMap::new(); - let next_node_or_edge = { - if self.sort_by_node { - cmp::max(self.num_of_node_labels(), 1) - } else { - cmp::max(self.num_of_edge_labels(), 1) - } + let next_node_or_edge = if self.sort_by_node { + self.num_of_node_labels() + } else { + self.num_of_edge_labels() }; - for i in 0..self.node_count() { - fwd_adj_list_metadata.insert(i, vec![0; next_node_or_edge + 3]); - bwd_adj_list_metadata.insert(i, vec![0; next_node_or_edge + 3]); + for i in 0..self.num_nodes { + fwd_adj_list_metadata.insert(i, vec![0; next_node_or_edge + 2]); + bwd_adj_list_metadata.insert(i, vec![0; next_node_or_edge + 2]); } self.edge_indices() .flat_map(|(from, to)| { @@ -775,14 +769,14 @@ impl } }); fwd_adj_list_metadata.iter_mut().for_each(|(_id, offsets)| { - for i in 1..next_node_or_edge + 2 { - offsets[next_node_or_edge + 2] += offsets[i]; + for i in 1..offsets.len() - 1 { + offsets[next_node_or_edge + 1] += offsets[i]; offsets[i] += offsets[i - 1]; } }); bwd_adj_list_metadata.iter_mut().for_each(|(_id, offsets)| { - for i in 1..next_node_or_edge + 2 { - offsets[next_node_or_edge + 2] += offsets[i]; + for i in 1..offsets.len() - 1 { + offsets[next_node_or_edge + 1] += offsets[i]; offsets[i] += offsets[i - 1]; } }); @@ -1208,7 +1202,7 @@ impl None, @@ -1219,7 +1213,7 @@ impl = g.neighbors_of_edge_iter(0, None).collect(); @@ -340,18 +340,6 @@ fn test_graphflow_planner() { println!("num_of_node_labels={}", g.num_of_node_labels()); println!("num_of_edge_labels={}", g.num_of_edge_labels()); println!("load finished."); - // let edge_vec = EdgeVec::with_labels(vec![0, 2, 3, 4], vec![1, 2, 0, 0], vec![0, 1, 0, 1]); - // let in_edge_vec = EdgeVec::new(vec![0, 2, 3, 4], vec![1, 2, 0, 0]); - // let labels = vec![1, 0, 1]; - // let g = DiStaticGraph::<&str>::with_labels( - // edge_vec, - // Some(in_edge_vec), - // labels, - // setmap!["a", "b"], - // setmap!["a", "b"], - // None, - // None, - // ); let catalog = catalog_generator::default(&g); let mut qvertex_to_qedges_map = HashMap::new(); let mut qvertex_to_type_map = HashMap::new(); @@ -436,7 +424,7 @@ fn test_graphflow_planner() { qvertex_to_deg_map.insert("e".to_owned(), vec![1, 1]); qvertex_to_deg_map.insert("f".to_owned(), vec![0, 2]); - let mut query_graph = QueryGraph { + let query_graph = QueryGraph { qvertex_to_qedges_map, qvertex_to_type_map, qvertex_to_deg_map, From f359e6de4a8871981b39b3cd6704eb328bb58cda Mon Sep 17 00:00:00 2001 From: Michael Chen <920901835@qq.com> Date: Sun, 2 Feb 2020 15:52:00 +0800 Subject: [PATCH 20/25] fix:catalog generator --- .../multi_graph/plan/operator/extend/EI.rs | 53 +++++++------------ .../multi_graph/plan/operator/operator.rs | 5 +- .../multi_graph/plan/operator/sink/sink.rs | 5 +- .../multi_graph/planner/catalog/catalog.rs | 6 ++- .../planner/catalog/catalog_plans.rs | 11 ++-- .../catalog/operator/intersect_catalog.rs | 8 +-- .../planner/catalog/operator/noop.rs | 4 +- .../catalog/subgraph_mapping_iterator.rs | 16 +++--- .../multi_graph/runner/catalog_generator.rs | 6 ++- src/graph_impl/static_graph/graph.rs | 3 +- src/graph_impl/static_graph/sorted_adj_vec.rs | 8 +-- 11 files changed, 58 insertions(+), 67 deletions(-) diff --git a/src/graph_impl/multi_graph/plan/operator/extend/EI.rs b/src/graph_impl/multi_graph/plan/operator/extend/EI.rs index 36c8970b..3e51e6d0 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/EI.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/EI.rs @@ -114,10 +114,10 @@ impl BaseEI { ald.from_query_vertex.clone() + "[" + if let Direction::Fwd = ald.direction { - "Fwd" - } else { - "Bwd" - } + "Fwd" + } else { + "Bwd" + } + "]" }) .sorted() @@ -204,51 +204,34 @@ impl BaseEI { graph: &TypedStaticGraph, last_repeated_vertex_idx: usize, ) { - let num_cached_alds = if let Some(ids) = &mut self.last_vertex_ids_intersected { - ids.len() - } else { - self.alds.len() - }; - self.vertex_idx_to_cache = vec![0; num_cached_alds]; - self.labels_or_to_types_to_cache = vec![0; num_cached_alds]; - self.adj_lists_to_cache = vec![vec![]; num_cached_alds]; - if let CachingType::PartialCaching = self.caching_type { - self.vertex_idx = vec![0; self.alds.len() - num_cached_alds]; - self.labels_or_to_types = vec![0; self.alds.len() - num_cached_alds]; - self.adj_lists = vec![vec![]; self.alds.len() - num_cached_alds]; - } - let mut idx = 0; - let mut idx_to_cache = 0; for ald in &self.alds { if let CachingType::PartialCaching = self.caching_type { if ald.vertex_idx > last_repeated_vertex_idx { - self.vertex_idx[idx] = ald.vertex_idx; - self.labels_or_to_types[idx] = if graph.is_sorted_by_node() { + self.vertex_idx.push(ald.vertex_idx); + self.labels_or_to_types.push(if graph.is_sorted_by_node() { self.to_type } else { ald.label - }; - self.adj_lists[idx] = if let Direction::Fwd = ald.direction { + }); + self.adj_lists.push(if let Direction::Fwd = ald.direction { graph.get_fwd_adj_list().clone() } else { graph.get_bwd_adj_list().clone() - }; - idx += 1; + }); continue; } } - self.vertex_idx_to_cache[idx_to_cache] = ald.vertex_idx; - self.labels_or_to_types_to_cache[idx_to_cache] = if graph.is_sorted_by_node() { + self.vertex_idx_to_cache.push(ald.vertex_idx); + self.labels_or_to_types_to_cache.push(if graph.is_sorted_by_node() { self.to_type } else { ald.label - }; - self.adj_lists_to_cache[idx_to_cache] = if let Direction::Fwd = ald.direction { + }); + self.adj_lists_to_cache.push(if let Direction::Fwd = ald.direction { graph.get_fwd_adj_list().clone() } else { graph.get_bwd_adj_list().clone() - }; - idx_to_cache += 1; + }); } } @@ -262,7 +245,7 @@ impl BaseEI { _ => ( self.adj_lists_to_cache[idx] [self.base_op.probe_tuple[self.vertex_idx_to_cache[idx]].id()] - .as_ref(), + .as_ref(), self.labels_or_to_types_to_cache[idx], ), }; @@ -288,9 +271,9 @@ impl CommonOperatorTrait for BaseEI { self.base_op.probe_tuple = probe_tuple.clone(); self.caching_type = CachingType::None; self.vertex_types = graph.get_node_types().clone(); - let last_repeated_vertex_idx = { - let mut prev = self.base_op.prev.as_ref().unwrap().borrow(); - get_op_attr!(prev.deref(), last_repeated_vertex_idx) + let last_repeated_vertex_idx = unsafe { + let prev = self.base_op.prev.as_ref().unwrap().as_ptr(); + get_op_attr!(&*prev,last_repeated_vertex_idx) }; self.init_caching(last_repeated_vertex_idx); self.init_extensions(graph); diff --git a/src/graph_impl/multi_graph/plan/operator/operator.rs b/src/graph_impl/multi_graph/plan/operator/operator.rs index d490889f..3a8c7e1f 100644 --- a/src/graph_impl/multi_graph/plan/operator/operator.rs +++ b/src/graph_impl/multi_graph/plan/operator/operator.rs @@ -166,7 +166,10 @@ impl CommonOperatorTrait for BaseOperator { fn execute(&mut self) { if let Some(prev) = self.prev.as_mut() { - prev.borrow_mut().execute(); + let mut op = prev.as_ptr(); + unsafe { + (&mut *op).execute(); + } } } diff --git a/src/graph_impl/multi_graph/plan/operator/sink/sink.rs b/src/graph_impl/multi_graph/plan/operator/sink/sink.rs index ea6b1940..527826d2 100644 --- a/src/graph_impl/multi_graph/plan/operator/sink/sink.rs +++ b/src/graph_impl/multi_graph/plan/operator/sink/sink.rs @@ -63,7 +63,10 @@ impl CommonOperatorTrait for BaseSink { fn execute(&mut self) { if !self.previous.is_empty() { - self.previous[0].borrow_mut().execute(); + let mut prev = self.previous[0].as_ptr(); + unsafe{ + (&mut *prev).execute(); + } } else { self.base_op.prev.as_mut().unwrap().borrow_mut().execute(); } diff --git a/src/graph_impl/multi_graph/planner/catalog/catalog.rs b/src/graph_impl/multi_graph/planner/catalog/catalog.rs index 198264a8..d1295aa6 100644 --- a/src/graph_impl/multi_graph/planner/catalog/catalog.rs +++ b/src/graph_impl/multi_graph/planner/catalog/catalog.rs @@ -280,8 +280,10 @@ impl Catalog { // handler.join(); // } } else { - let mut sink = query_plan_arr[0].sink.as_mut().unwrap().borrow_mut(); - sink.execute(); + let mut sink = query_plan_arr[0].sink.as_mut().unwrap().as_ptr(); + unsafe{ + (&mut *sink).execute(); + } } } diff --git a/src/graph_impl/multi_graph/planner/catalog/catalog_plans.rs b/src/graph_impl/multi_graph/planner/catalog/catalog_plans.rs index b4455d01..51046e7b 100644 --- a/src/graph_impl/multi_graph/planner/catalog/catalog_plans.rs +++ b/src/graph_impl/multi_graph/planner/catalog/catalog_plans.rs @@ -128,10 +128,8 @@ impl CatalogPlans { let query_vertices = in_subgraph.get_query_vertices().clone(); let mut descriptors = vec![]; - println!("call here={:?}",set_utils::get_power_set_excluding_empty_set(query_vertices.clone())); - for query_vertex_to_extend in set_utils::get_power_set_excluding_empty_set(query_vertices) { - for alds in self.generate_alds(query_vertex_to_extend, self.is_directed) { - println!("rec call here={:?}",in_subgraph.qvertex_to_qedges_map.keys().len()); + for query_vertex_to_extend in set_utils::get_power_set_excluding_empty_set(query_vertices){ + for alds in self.generate_alds(&query_vertex_to_extend, self.is_directed){ descriptors.push(Descriptor { out_subgraph: self.get_out_subgraph(in_subgraph.copy(), alds.clone()), alds, @@ -274,7 +272,6 @@ impl CatalogPlans { if self.sorted_by_node { noops[0] = Noop::new(query_graph.clone()); noops[0].base_op.out_qvertex_to_idx_map = out_qvertex_to_idx_map.clone(); - println!("call here!") } else { for to_type in 0..=self.num_node_labels { let mut query_graph_copy = query_graph.copy(); @@ -286,7 +283,7 @@ impl CatalogPlans { fn generate_alds( &self, - qvertices: Vec, + qvertices: &Vec, is_direccted: bool, ) -> Vec> { let direction_patterns = Self::generate_direction_patterns(qvertices.len(), is_direccted); @@ -364,7 +361,6 @@ impl CatalogPlans { alds: Vec, ) -> QueryGraph { let num_qvertices = query_graph.get_num_qvertices(); - println!("get_out_sub={},{}",num_qvertices,QUERY_VERTICES[num_qvertices]); for ald in alds { let mut query_edge = if let Direction::Fwd = ald.direction { let mut query_edge = QueryEdge::default( @@ -382,7 +378,6 @@ impl CatalogPlans { query_edge }; query_edge.label = ald.label; -// println!("qedge={}->{},query_edge={:?}", query_edge.from_query_vertex, query_edge.to_query_vertex,query_graph.qvertex_to_qedges_map.keys()); query_graph.add_qedge(query_edge); } query_graph diff --git a/src/graph_impl/multi_graph/planner/catalog/operator/intersect_catalog.rs b/src/graph_impl/multi_graph/planner/catalog/operator/intersect_catalog.rs index 4bbd4750..d66cf306 100644 --- a/src/graph_impl/multi_graph/planner/catalog/operator/intersect_catalog.rs +++ b/src/graph_impl/multi_graph/planner/catalog/operator/intersect_catalog.rs @@ -13,7 +13,7 @@ use std::rc::Rc; #[derive(Clone)] pub struct IntersectCatalog { pub base_intersect: BaseIntersect, - is_adj_list_sorted_by_type: bool, + is_adj_list_sorted_by_node: bool, last_icost: usize, caching_enable: bool, } @@ -26,7 +26,7 @@ impl IntersectCatalog { out_subgraph: QueryGraph, in_subgraph: QueryGraph, out_qvertex_to_idx_map: HashMap, - is_adj_list_sorted_by_type: bool, + is_adj_list_sorted_by_node: bool, ) -> IntersectCatalog { IntersectCatalog { base_intersect: BaseIntersect::new( @@ -37,7 +37,7 @@ impl IntersectCatalog { Some(in_subgraph), out_qvertex_to_idx_map, ), - is_adj_list_sorted_by_type, + is_adj_list_sorted_by_node, last_icost: 0, caching_enable: true, } @@ -155,7 +155,7 @@ impl CommonOperatorTrait for IntersectCatalog { for idx in base_ei.out_neighbours.start_idx..base_ei.out_neighbours.end_idx { base_ei.base_op.probe_tuple[base_ei.out_idx] = base_ei.out_neighbours.ids[idx]; base_ei.base_op.num_out_tuples += 1; - if self.is_adj_list_sorted_by_type { + if self.is_adj_list_sorted_by_node { base_ei.base_op.next[0].borrow_mut().process_new_tuple(); } else { base_ei diff --git a/src/graph_impl/multi_graph/planner/catalog/operator/noop.rs b/src/graph_impl/multi_graph/planner/catalog/operator/noop.rs index a90b5d82..b6fbcb75 100644 --- a/src/graph_impl/multi_graph/planner/catalog/operator/noop.rs +++ b/src/graph_impl/multi_graph/planner/catalog/operator/noop.rs @@ -39,7 +39,9 @@ impl CommonOperatorTrait for Noop { self.base_op .next .iter() - .for_each(|next_op| next_op.borrow_mut().process_new_tuple()); + .for_each(|next_op| { + next_op.borrow_mut().process_new_tuple(); + }); } fn execute(&mut self) { diff --git a/src/graph_impl/multi_graph/planner/catalog/subgraph_mapping_iterator.rs b/src/graph_impl/multi_graph/planner/catalog/subgraph_mapping_iterator.rs index 33163dd7..21a86d8c 100644 --- a/src/graph_impl/multi_graph/planner/catalog/subgraph_mapping_iterator.rs +++ b/src/graph_impl/multi_graph/planner/catalog/subgraph_mapping_iterator.rs @@ -83,10 +83,10 @@ impl SubgraphMappingIterator { pub fn has_next(&mut self) -> bool { if !self.is_next_computed { - println!( - "PRE: curr_mapping={:?},self.o_qvertices={:?}", - self.curr_mapping, self.o_qvertices - ); +// println!( +// "PRE: curr_mapping={:?},self.o_qvertices={:?}", +// self.curr_mapping, self.o_qvertices +// ); if self.curr_mapping.len() == self.o_qvertices.len() { self.curr_mapping.pop(); } @@ -180,10 +180,10 @@ impl SubgraphMappingIterator { self.is_next_computed = false; return self.has_next(); } - println!( - "LATER:cur_mapping={:?},self.o_qvertices={:?}", - self.curr_mapping, self.o_qvertices - ); +// println!( +// "LATER:cur_mapping={:?},self.o_qvertices={:?}", +// self.curr_mapping, self.o_qvertices +// ); !self.curr_mapping.is_empty() } diff --git a/src/graph_impl/multi_graph/runner/catalog_generator.rs b/src/graph_impl/multi_graph/runner/catalog_generator.rs index e60e105a..1d39001d 100644 --- a/src/graph_impl/multi_graph/runner/catalog_generator.rs +++ b/src/graph_impl/multi_graph/runner/catalog_generator.rs @@ -9,7 +9,11 @@ use std::hash::Hash; pub fn default( graph: &TypedStaticGraph, ) -> Catalog { - let mut catalog = Catalog::new(DEF_NUM_EDGES_TO_SAMPLE, DEF_MAX_INPUT_NUM_VERTICES); + let mut max_input_num_vertex = DEF_MAX_INPUT_NUM_VERTICES; + if graph.is_sorted_by_node() { + max_input_num_vertex = 2; + } + let mut catalog = Catalog::new(DEF_NUM_EDGES_TO_SAMPLE, max_input_num_vertex); catalog.populate(graph, 1); println!( "Catalog generation finished in {} (ms)", diff --git a/src/graph_impl/static_graph/graph.rs b/src/graph_impl/static_graph/graph.rs index 2e2c2fd5..99edd171 100644 --- a/src/graph_impl/static_graph/graph.rs +++ b/src/graph_impl/static_graph/graph.rs @@ -421,7 +421,7 @@ impl self.label_to_largest_fwd_adj_list_size[label] = fwd_adj_size; } if bwd_adj_size > self.label_to_largest_bwd_adj_list_size[label] { - self.label_to_largest_fwd_adj_list_size[label] = bwd_adj_size; + self.label_to_largest_bwd_adj_list_size[label] = bwd_adj_size; } } } @@ -469,7 +469,6 @@ impl } } } - println!("{:?}", self.label_to_num_edges); } fn add_edge_count(&mut self, from_type: usize, to_type: usize, label: usize, num_edges: usize) { diff --git a/src/graph_impl/static_graph/sorted_adj_vec.rs b/src/graph_impl/static_graph/sorted_adj_vec.rs index 52104c9b..ff99c208 100644 --- a/src/graph_impl/static_graph/sorted_adj_vec.rs +++ b/src/graph_impl/static_graph/sorted_adj_vec.rs @@ -90,10 +90,10 @@ impl SortedAdjVec { this_idx_end: usize, ) { neighbours.reset(); - let some_neighbour_ids = some_neighbours.ids.clone(); + let some_neighbour_ids = &some_neighbours.ids; let mut some_idx = some_neighbours.start_idx; - let some_end_idx = some_neighbours.end_idx; - while this_idx < this_idx_end && some_idx < some_end_idx { + let some_idx_end = some_neighbours.end_idx; + while this_idx < this_idx_end && some_idx < some_idx_end { if neighbour_ids[this_idx] < some_neighbour_ids[some_idx] { this_idx += 1; while this_idx < this_idx_end @@ -103,7 +103,7 @@ impl SortedAdjVec { } } else if neighbour_ids[this_idx] > some_neighbour_ids[some_idx] { some_idx += 1; - while some_idx < some_end_idx + while some_idx < some_idx_end && neighbour_ids[this_idx] > some_neighbour_ids[some_idx] { some_idx += 1; From 126df0abf7ec065c2afc72f00731bc7f53daf4e8 Mon Sep 17 00:00:00 2001 From: Michael Chen <920901835@qq.com> Date: Tue, 3 Mar 2020 19:42:49 +0800 Subject: [PATCH 21/25] sync --- .../multi_graph/plan/operator/extend/EI.rs | 50 ++--- .../plan/operator/extend/Extend.rs | 9 +- .../plan/operator/extend/Intersect.rs | 9 +- .../plan/operator/extend/extend.rs | 9 +- .../plan/operator/extend/intersect.rs | 9 +- .../multi_graph/plan/operator/scan/scan.rs | 25 +-- .../plan/operator/scan/scan_blocking.rs | 26 +-- .../plan/operator/scan/scan_sampling.rs | 26 ++- .../multi_graph/plan/operator/sink/sink.rs | 2 +- src/graph_impl/multi_graph/plan/query_plan.rs | 22 +- .../planner/catalog/adj_list_descriptor.rs | 4 +- .../multi_graph/planner/catalog/catalog.rs | 197 +++++++++--------- .../planner/catalog/catalog_plans.rs | 124 +++++------ .../catalog/operator/intersect_catalog.rs | 15 +- .../planner/catalog/operator/noop.rs | 9 +- .../multi_graph/planner/catalog/query_edge.rs | 12 +- .../planner/catalog/query_graph.rs | 16 +- .../catalog/subgraph_mapping_iterator.rs | 90 ++++---- .../multi_graph/planner/query_planner.rs | 127 +++++------ .../multi_graph/query/query_graph_set.rs | 9 +- .../multi_graph/runner/catalog_generator.rs | 3 + src/graph_impl/static_graph/graph.rs | 153 +++++++------- src/graph_impl/static_graph/sorted_adj_vec.rs | 17 +- tests/static_graph.rs | 48 +++-- 24 files changed, 538 insertions(+), 473 deletions(-) diff --git a/src/graph_impl/multi_graph/plan/operator/extend/EI.rs b/src/graph_impl/multi_graph/plan/operator/extend/EI.rs index 3e51e6d0..ba996f4d 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/EI.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/EI.rs @@ -53,15 +53,15 @@ pub enum EI { #[derive(Clone)] pub struct BaseEI { pub base_op: BaseOperator, - pub vertex_types: Vec, - pub to_type: usize, + pub vertex_types: Vec, + pub to_type: i32, pub to_query_vertex: String, pub alds: Vec, pub out_idx: usize, pub vertex_idx: Vec, pub vertex_idx_to_cache: Vec, - pub labels_or_to_types: Vec, - pub labels_or_to_types_to_cache: Vec, + pub labels_or_to_types: Vec, + pub labels_or_to_types_to_cache: Vec, pub adj_lists: Vec>>>, pub adj_lists_to_cache: Vec>>>, pub caching_type: CachingType, @@ -76,7 +76,7 @@ pub struct BaseEI { impl BaseEI { pub fn new( to_query_vertex: String, - to_type: usize, + to_type: i32, alds: Vec, out_subgraph: QueryGraph, in_subgraph: Option, @@ -114,10 +114,10 @@ impl BaseEI { ald.from_query_vertex.clone() + "[" + if let Direction::Fwd = ald.direction { - "Fwd" - } else { - "Bwd" - } + "Fwd" + } else { + "Bwd" + } + "]" }) .sorted() @@ -135,11 +135,11 @@ impl BaseEI { self.is_intersection_cached = true; for i in 0..self.last_vertex_ids_intersected.as_ref().unwrap().len() { if self.last_vertex_ids_intersected.as_ref().unwrap()[i] - != self.base_op.probe_tuple[self.vertex_idx_to_cache[i]] + != self.base_op.probe_tuple[self.vertex_idx_to_cache[i] as usize] { self.is_intersection_cached = false; self.last_vertex_ids_intersected.as_mut().unwrap()[i] = - self.base_op.probe_tuple[self.vertex_idx_to_cache[i]]; + self.base_op.probe_tuple[self.vertex_idx_to_cache[i] as usize]; } } self.is_intersection_cached @@ -222,16 +222,18 @@ impl BaseEI { } } self.vertex_idx_to_cache.push(ald.vertex_idx); - self.labels_or_to_types_to_cache.push(if graph.is_sorted_by_node() { - self.to_type - } else { - ald.label - }); - self.adj_lists_to_cache.push(if let Direction::Fwd = ald.direction { - graph.get_fwd_adj_list().clone() - } else { - graph.get_bwd_adj_list().clone() - }); + self.labels_or_to_types_to_cache + .push(if graph.is_sorted_by_node() { + self.to_type + } else { + ald.label + }); + self.adj_lists_to_cache + .push(if let Direction::Fwd = ald.direction { + graph.get_fwd_adj_list().clone() + } else { + graph.get_bwd_adj_list().clone() + }); } } @@ -245,7 +247,7 @@ impl BaseEI { _ => ( self.adj_lists_to_cache[idx] [self.base_op.probe_tuple[self.vertex_idx_to_cache[idx]].id()] - .as_ref(), + .as_ref(), self.labels_or_to_types_to_cache[idx], ), }; @@ -273,7 +275,7 @@ impl CommonOperatorTrait for BaseEI { self.vertex_types = graph.get_node_types().clone(); let last_repeated_vertex_idx = unsafe { let prev = self.base_op.prev.as_ref().unwrap().as_ptr(); - get_op_attr!(&*prev,last_repeated_vertex_idx) + get_op_attr!(&*prev, last_repeated_vertex_idx) }; self.init_caching(last_repeated_vertex_idx); self.init_extensions(graph); @@ -352,7 +354,7 @@ impl EI { pub fn make( to_qvertex: String, - to_type: usize, + to_type: i32, alds: Vec, out_subgraph: QueryGraph, in_subgraph: QueryGraph, diff --git a/src/graph_impl/multi_graph/plan/operator/extend/Extend.rs b/src/graph_impl/multi_graph/plan/operator/extend/Extend.rs index 00cf9e60..87ecbf35 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/Extend.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/Extend.rs @@ -13,6 +13,7 @@ use graph_impl::multi_graph::planner::catalog::adj_list_descriptor::{ AdjListDescriptor, Direction, }; use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; +use graph_impl::static_graph::graph::KEY_ANY; use graph_impl::static_graph::sorted_adj_vec::SortedAdjVec; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; @@ -25,7 +26,7 @@ use std::rc::Rc; pub struct Extend { pub base_ei: BaseEI, vertex_index: usize, - label_or_to_type: usize, + label_or_to_type: i32, pub dir: Direction, adj_list: Vec>>, } @@ -33,7 +34,7 @@ pub struct Extend { impl Extend { pub fn new( to_qvertex: String, - to_type: usize, + to_type: i32, alds: Vec, out_subgraph: QueryGraph, in_subgraph: Option, @@ -71,7 +72,7 @@ impl CommonOperatorTrait for Extend { .clone(); if graph.is_sorted_by_node() { self.label_or_to_type = self.base_ei.to_type; - self.base_ei.to_type = 0; + self.base_ei.to_type = KEY_ANY; } for next_operator in &mut self.base_ei.base_op.next { next_operator.borrow_mut().init(probe_tuple.clone(), graph); @@ -86,7 +87,7 @@ impl CommonOperatorTrait for Extend { adj_vec.set_neighbor_ids(self.label_or_to_type, out_neighbour); self.base_ei.base_op.icost += out_neighbour.end_idx - out_neighbour.start_idx; for idx in out_neighbour.start_idx..out_neighbour.end_idx { - if self.base_ei.to_type == 0 + if self.base_ei.to_type == KEY_ANY || self.base_ei.to_type == self.base_ei.vertex_types[out_neighbour.ids[idx].id()] { self.base_ei.base_op.num_out_tuples += 1; diff --git a/src/graph_impl/multi_graph/plan/operator/extend/Intersect.rs b/src/graph_impl/multi_graph/plan/operator/extend/Intersect.rs index d3fb4565..4e484c82 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/Intersect.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/Intersect.rs @@ -9,6 +9,7 @@ use graph_impl::multi_graph::plan::operator::sink::sink::Sink; use graph_impl::multi_graph::planner::catalog::adj_list_descriptor::AdjListDescriptor; use graph_impl::multi_graph::planner::catalog::operator::intersect_catalog::IntersectCatalog; use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; +use graph_impl::static_graph::graph::KEY_ANY; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; use std::cell::RefCell; @@ -37,7 +38,7 @@ pub struct BaseIntersect { impl BaseIntersect { pub fn new( to_qvertex: String, - to_type: usize, + to_type: i32, alds: Vec, out_subgraph: QueryGraph, in_subgraph: Option, @@ -68,17 +69,17 @@ impl CommonOperatorTrait for BaseIntersect { if CachingType::None == self.base_ei.caching_type || !self.base_ei.is_intersection_cached() { let base_ei = &mut self.base_ei; - let mut cache_id = base_ei.vertex_idx_to_cache[0]; + let cache_id = base_ei.vertex_idx_to_cache[0]; let to_id = base_ei.base_op.probe_tuple[cache_id].id(); let adj_vec = base_ei.adj_lists_to_cache[0][to_id].as_ref(); - cache_id = base_ei.labels_or_to_types_to_cache[0]; + let cache_id = base_ei.labels_or_to_types_to_cache[0]; let neighbours = &mut base_ei.init_neighbours; adj_vec.map(|adj| adj.set_neighbor_ids(cache_id, neighbours)); base_ei.base_op.icost += base_ei.init_neighbours.end_idx - base_ei.init_neighbours.start_idx; base_ei.base_op.icost += base_ei.execute_intersect(1, IntersectType::InitCached); - if base_ei.to_type != 0 { + if base_ei.to_type != KEY_ANY { let mut curr_end_idx = 0; let cached_neighbours = &mut base_ei.cached_neighbours; for i in cached_neighbours.start_idx..cached_neighbours.end_idx { diff --git a/src/graph_impl/multi_graph/plan/operator/extend/extend.rs b/src/graph_impl/multi_graph/plan/operator/extend/extend.rs index 00cf9e60..87ecbf35 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/extend.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/extend.rs @@ -13,6 +13,7 @@ use graph_impl::multi_graph::planner::catalog::adj_list_descriptor::{ AdjListDescriptor, Direction, }; use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; +use graph_impl::static_graph::graph::KEY_ANY; use graph_impl::static_graph::sorted_adj_vec::SortedAdjVec; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; @@ -25,7 +26,7 @@ use std::rc::Rc; pub struct Extend { pub base_ei: BaseEI, vertex_index: usize, - label_or_to_type: usize, + label_or_to_type: i32, pub dir: Direction, adj_list: Vec>>, } @@ -33,7 +34,7 @@ pub struct Extend { impl Extend { pub fn new( to_qvertex: String, - to_type: usize, + to_type: i32, alds: Vec, out_subgraph: QueryGraph, in_subgraph: Option, @@ -71,7 +72,7 @@ impl CommonOperatorTrait for Extend { .clone(); if graph.is_sorted_by_node() { self.label_or_to_type = self.base_ei.to_type; - self.base_ei.to_type = 0; + self.base_ei.to_type = KEY_ANY; } for next_operator in &mut self.base_ei.base_op.next { next_operator.borrow_mut().init(probe_tuple.clone(), graph); @@ -86,7 +87,7 @@ impl CommonOperatorTrait for Extend { adj_vec.set_neighbor_ids(self.label_or_to_type, out_neighbour); self.base_ei.base_op.icost += out_neighbour.end_idx - out_neighbour.start_idx; for idx in out_neighbour.start_idx..out_neighbour.end_idx { - if self.base_ei.to_type == 0 + if self.base_ei.to_type == KEY_ANY || self.base_ei.to_type == self.base_ei.vertex_types[out_neighbour.ids[idx].id()] { self.base_ei.base_op.num_out_tuples += 1; diff --git a/src/graph_impl/multi_graph/plan/operator/extend/intersect.rs b/src/graph_impl/multi_graph/plan/operator/extend/intersect.rs index d3fb4565..4e484c82 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/intersect.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/intersect.rs @@ -9,6 +9,7 @@ use graph_impl::multi_graph::plan::operator::sink::sink::Sink; use graph_impl::multi_graph::planner::catalog::adj_list_descriptor::AdjListDescriptor; use graph_impl::multi_graph::planner::catalog::operator::intersect_catalog::IntersectCatalog; use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; +use graph_impl::static_graph::graph::KEY_ANY; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; use std::cell::RefCell; @@ -37,7 +38,7 @@ pub struct BaseIntersect { impl BaseIntersect { pub fn new( to_qvertex: String, - to_type: usize, + to_type: i32, alds: Vec, out_subgraph: QueryGraph, in_subgraph: Option, @@ -68,17 +69,17 @@ impl CommonOperatorTrait for BaseIntersect { if CachingType::None == self.base_ei.caching_type || !self.base_ei.is_intersection_cached() { let base_ei = &mut self.base_ei; - let mut cache_id = base_ei.vertex_idx_to_cache[0]; + let cache_id = base_ei.vertex_idx_to_cache[0]; let to_id = base_ei.base_op.probe_tuple[cache_id].id(); let adj_vec = base_ei.adj_lists_to_cache[0][to_id].as_ref(); - cache_id = base_ei.labels_or_to_types_to_cache[0]; + let cache_id = base_ei.labels_or_to_types_to_cache[0]; let neighbours = &mut base_ei.init_neighbours; adj_vec.map(|adj| adj.set_neighbor_ids(cache_id, neighbours)); base_ei.base_op.icost += base_ei.init_neighbours.end_idx - base_ei.init_neighbours.start_idx; base_ei.base_op.icost += base_ei.execute_intersect(1, IntersectType::InitCached); - if base_ei.to_type != 0 { + if base_ei.to_type != KEY_ANY { let mut curr_end_idx = 0; let cached_neighbours = &mut base_ei.cached_neighbours; for i in cached_neighbours.start_idx..cached_neighbours.end_idx { diff --git a/src/graph_impl/multi_graph/plan/operator/scan/scan.rs b/src/graph_impl/multi_graph/plan/operator/scan/scan.rs index 3507257a..d87b2c03 100644 --- a/src/graph_impl/multi_graph/plan/operator/scan/scan.rs +++ b/src/graph_impl/multi_graph/plan/operator/scan/scan.rs @@ -6,6 +6,7 @@ use graph_impl::multi_graph::plan::operator::scan::scan_blocking::ScanBlocking; use graph_impl::multi_graph::plan::operator::scan::scan_sampling::ScanSampling; use graph_impl::multi_graph::plan::operator::sink::sink::Sink; use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; +use graph_impl::static_graph::graph::KEY_ANY; use graph_impl::static_graph::sorted_adj_vec::SortedAdjVec; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; @@ -26,12 +27,12 @@ pub struct BaseScan { pub base_op: BaseOperator, pub from_query_vertex: String, pub to_query_vertex: String, - pub from_type: usize, - pub to_type: usize, - pub label_or_to_type: usize, + pub from_type: i32, + pub to_type: i32, + pub label_or_to_type: i32, pub fwd_adj_list: Vec>>, pub vertex_ids: Vec, - pub vertex_types: Vec, + pub vertex_types: Vec, from_vertex_start_idx: usize, from_vertex_end_idx: usize, } @@ -83,9 +84,9 @@ impl CommonOperatorTrait for BaseScan { self.base_op.probe_tuple = probe_tuple.clone(); self.vertex_ids = graph.get_node_ids().clone(); self.vertex_types = graph.get_node_types().clone(); - if 0 != self.from_type { - self.from_vertex_start_idx = graph.get_node_type_offsets()[self.from_type]; - self.from_vertex_end_idx = graph.get_node_type_offsets()[self.from_type + 1]; + if KEY_ANY != self.from_type { + self.from_vertex_start_idx = graph.get_node_type_offsets()[self.from_type as usize]; + self.from_vertex_end_idx = graph.get_node_type_offsets()[(self.from_type + 1) as usize]; } else { self.from_vertex_start_idx = 0; self.from_vertex_end_idx = graph.node_count(); @@ -93,7 +94,7 @@ impl CommonOperatorTrait for BaseScan { self.fwd_adj_list = graph.get_fwd_adj_list().clone(); if graph.is_sorted_by_node() { self.label_or_to_type = self.to_type; - self.to_type = 0; + self.to_type = KEY_ANY; } for next_op in &self.base_op.next { next_op.borrow_mut().init(probe_tuple.clone(), graph); @@ -108,16 +109,16 @@ impl CommonOperatorTrait for BaseScan { for from_idx in self.from_vertex_start_idx..self.from_vertex_end_idx { let from_vertex = self.vertex_ids[from_idx]; self.base_op.probe_tuple[0] = from_vertex; - let to_vertex_start_idx = - self.fwd_adj_list[from_idx].as_ref().unwrap().get_offsets()[self.label_or_to_type]; + let to_vertex_start_idx = self.fwd_adj_list[from_idx].as_ref().unwrap().get_offsets() + [self.label_or_to_type as usize]; let to_vertex_end_idx = self.fwd_adj_list[from_idx].as_ref().unwrap().get_offsets() - [self.label_or_to_type + 1]; + [(self.label_or_to_type + 1) as usize]; for to_idx in to_vertex_start_idx..to_vertex_end_idx { self.base_op.probe_tuple[1] = self.fwd_adj_list[from_idx] .as_ref() .unwrap() .get_neighbor_id(Id::new(to_idx)); - if self.to_type == 0 + if self.to_type == KEY_ANY || self.vertex_types[self.base_op.probe_tuple[1].id()] == self.to_type { self.base_op.num_out_tuples += 1; diff --git a/src/graph_impl/multi_graph/plan/operator/scan/scan_blocking.rs b/src/graph_impl/multi_graph/plan/operator/scan/scan_blocking.rs index 432485d8..f586a170 100644 --- a/src/graph_impl/multi_graph/plan/operator/scan/scan_blocking.rs +++ b/src/graph_impl/multi_graph/plan/operator/scan/scan_blocking.rs @@ -2,6 +2,7 @@ use generic::{GraphTrait, GraphType, IdType}; use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; use graph_impl::multi_graph::plan::operator::scan::scan::BaseScan; use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; +use graph_impl::static_graph::graph::KEY_ANY; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; use std::cell::RefCell; @@ -63,7 +64,7 @@ impl ScanBlocking { let mut label = self.base_scan.label_or_to_type; let to_limit = self.base_scan.fwd_adj_list[self.from_idx_limit] .as_mut() - .map_or(0, |adj| adj.get_offsets()[label + 1]); + .map_or(0, |adj| adj.get_offsets()[(label + 1) as usize]); if self.to_idx_limit + num_edges_left <= to_limit - 1 { self.to_idx_limit += num_edges_left - 1; num_edges_left = 0; @@ -77,7 +78,7 @@ impl ScanBlocking { label = self.base_scan.label_or_to_type; self.to_idx_limit = self.base_scan.fwd_adj_list[self.from_idx_limit] .as_mut() - .map_or(0, |adj| adj.get_offsets()[label]); + .map_or(0, |adj| adj.get_offsets()[label as usize]); } } self.global_vertices_idx_limits.from_variable_index_limit = self.from_idx_limit; @@ -103,16 +104,16 @@ impl ScanBlocking { self.base_scan.base_op.probe_tuple[0] = self.base_scan.vertex_ids[from_idx]; let to_vertex_idx_start = self.base_scan.fwd_adj_list[from_idx] .as_mut() - .map_or(0, |adj| adj.get_offsets()[label]); + .map_or(0, |adj| adj.get_offsets()[label as usize]); let to_vertex_idx_limit = self.base_scan.fwd_adj_list[from_idx] .as_mut() - .map_or(0, |adj| adj.get_offsets()[label + 1]); + .map_or(0, |adj| adj.get_offsets()[(label + 1) as usize]); for to_idx in to_vertex_idx_start..to_vertex_idx_limit { self.base_scan.base_op.probe_tuple[1] = self.base_scan.fwd_adj_list[from_idx] .as_mut() .unwrap() .get_neighbor_id(Id::new(to_idx)); - if self.base_scan.to_type == 0 + if self.base_scan.to_type == KEY_ANY || self.base_scan.vertex_types[self.base_scan.base_op.probe_tuple[1].id()] == self.base_scan.to_type { @@ -133,9 +134,10 @@ impl CommonOperatorTrait for ScanBlocking { graph: &TypedStaticGraph, ) { self.base_scan.init(probe_tuple.clone(), graph); - if self.base_scan.from_type != 0 { - self.curr_from_idx = graph.get_node_type_offsets()[self.base_scan.from_type]; - self.highest_from_idx = graph.get_node_type_offsets()[self.base_scan.from_type + 1]; + if self.base_scan.from_type != KEY_ANY { + self.curr_from_idx = graph.get_node_type_offsets()[self.base_scan.from_type as usize]; + self.highest_from_idx = + graph.get_node_type_offsets()[(self.base_scan.from_type + 1) as usize]; } else { self.curr_from_idx = 0; self.highest_from_idx = graph.node_count() + 1; @@ -144,11 +146,11 @@ impl CommonOperatorTrait for ScanBlocking { self.curr_to_idx = self.base_scan.fwd_adj_list [self.base_scan.vertex_ids[self.curr_from_idx].id()] .as_mut() - .map_or(0, |adj| adj.get_offsets()[label]); + .map_or(0, |adj| adj.get_offsets()[label as usize]); self.highest_to_idx = self.base_scan.fwd_adj_list [self.base_scan.vertex_ids[self.highest_from_idx - 1].id()] .as_mut() - .map_or(0, |adj| adj.get_offsets()[label + 1]); + .map_or(0, |adj| adj.get_offsets()[(label + 1) as usize]); self.from_idx_limit = self.curr_from_idx; self.to_idx_limit = self.curr_to_idx; self.base_scan @@ -175,13 +177,13 @@ impl CommonOperatorTrait for ScanBlocking { let to_vertex_idx_limit = self.base_scan.fwd_adj_list [self.base_scan.vertex_ids[self.curr_from_idx].id()] .as_mut() - .map_or(0, |adj| adj.get_offsets()[label + 1]); + .map_or(0, |adj| adj.get_offsets()[(label + 1) as usize]); self.produce_new_edges(self.curr_from_idx, self.curr_to_idx, to_vertex_idx_limit); self.produce_new_edges_default(/* startFromIdx: currFromIdx + 1, endFromIdx: fromIdxLimit */); let start_idx = self.base_scan.fwd_adj_list [self.base_scan.vertex_ids[self.from_idx_limit].id()] .as_mut() - .map_or(0, |adj| adj.get_offsets()[label]); + .map_or(0, |adj| adj.get_offsets()[label as usize]); self.produce_new_edges(self.from_idx_limit, start_idx, self.to_idx_limit); } self.update_indices_limits(); diff --git a/src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs b/src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs index e18b6eed..477f674c 100644 --- a/src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs +++ b/src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs @@ -10,6 +10,7 @@ use rand::{thread_rng, Rng}; use std::cell::RefCell; use std::hash::Hash; use std::rc::Rc; +use itertools::Itertools; #[derive(Clone)] pub struct ScanSampling { @@ -17,6 +18,8 @@ pub struct ScanSampling { pub edges_queue: Vec>, } +static mut FLAG: bool = false; + impl ScanSampling { pub fn new(out_subgraph: QueryGraph) -> ScanSampling { Self { @@ -24,14 +27,25 @@ impl ScanSampling { edges_queue: vec![], } } - pub fn set_edge_indices_to_sample(&mut self, edges: Vec, num_edges_to_sample: usize) { let mut rng = thread_rng(); let num_edges = edges.len() / 2; - while self.edges_queue.len() < num_edges_to_sample { - let edge_idx = rng.gen_range(0, num_edges); - self.edges_queue - .push(vec![edges[edge_idx], edges[edge_idx + 1]]); + if unsafe { FLAG == true } { + let vec = vec![60, 88, 49, 47, 35, 83, 71, 21, 69, 74, 77, 17, 83, 82, 35, 14, 4, 15, 41, 20, 13, 18, 74, 77, 62, 10, 33, 52, 72, 53, 65, 65, 77, 47, 32, 2, 15, 83, 58, 15, 20, 25, 35, 20, 28, 21, 64, 16, 82, 42, 49, 57, 86, 66, 87, 16, 88, 17, 3, 7, 17, 88, 25, 78, 48, 87, 61, 8, 88, 18, 38, 22, 17, 11, 26, 60, 77, 72, 85, 10, 7, 33, 27, 49, 45, 72, 79, 43, 21, 61, 240, 28, 189, 287, 315, 93, 91, 1, 159, 54, 277, 37, 273, 102, 215, 44, 4, 195, 121, 120, 263, 248, 44, 187, 12, 220, 143, 282, 132, 303, 225, 125, 37, 307, 82, 182, 205, 293, 178, 75, 0, 215, 75, 290, 78, 171, 74, 136, 272, 302, 279, 157, 16, 206, 7, 96, 8, 57, 103, 147, 67, 148, 185, 138, 288, 277, 251, 278, 118, 318, 98, 72, 17, 311, 166, 100, 217, 262, 55, 100, 147, 173, 77, 229, 65, 32, 29, 243, 101, 161, 103, 300, 228, 314, 236, 83, 197, 22, 125, 51, 216, 125, 239, 173, 307, 107, 171, 97, 88, 203, 290, 245, 178, 21, 157, 197, 126, 1, 50, 231, 71, 294, 210, 25, 266, 244, 64, 202, 312, 10, 28, 193, 89, 245, 221, 110, 205, 262, 94, 139, 237, 271, 271, 303, 17, 58, 76, 111, 235, 261, 266, 113, 273, 157, 238, 78, 216, 278, 120, 35, 155, 5, 28, 309, 155, 315, 113, 190, 1, 286, 258, 152, 318, 101, 67, 69, 19, 111, 222, 281, 237, 1, 298, 261, 104, 283, 189, 192, 180, 53, 152, 275, 213, 91, 193, 2, 260, 34, 174, 18, 115, 262, 29, 188, 238, 290, 0, 236, 93, 202, 214, 54, 102, 8, 149, 22, 239, 114, 195, 290, 193, 100, 37, 270, 72, 32, 270, 283, 54, 62, 280, 286, 72, 145, 305, 233, 33, 88, 77, 280, 146, 222, 185, 186, 59, 303, 272, 217, 10, 55, 134, 296, 34, 248, 289, 8, 88, 295, 136, 246, 87, 164, 244, 149, 163, 213, 3, 212, 186, 94, 235, 124, 215, 126, 11, 217, 144, 137, 157, 194, 159, 62, 200, 198, 125, 132, 211, 180, 106, 198, 179, 71, 266, 232, 5, 44, 297, 36, 23, 242, 95, 262, 288, 141, 271, 256, 272, 263, 73, 298, 130, 68, 79, 218, 271, 240, 309, 157, 316, 135, 240, 28, 189, 287, 315, 93, 91, 1, 159, 54, 277, 37, 273, 102, 215, 44, 4, 195, 121, 120, 263, 248, 44, 187, 12, 220, 143, 282, 132, 303, 225, 125, 37, 307, 82, 182, 205, 293, 178, 75, 0, 215, 75, 290, 78, 171, 74, 136, 272, 302, 279, 157, 16, 206, 7, 96, 8, 57, 103, 147, 67, 148, 185, 138, 288, 277, 251, 278, 118, 318, 98, 72, 17, 311, 166, 100, 217, 262, 55, 100, 147, 173, 77, 229, 65, 32, 29, 243, 101, 161, 103, 300, 228, 314, 236, 83, 197, 22, 125, 51, 216, 125, 239, 173, 307, 107, 171, 97, 88, 203, 290, 245, 178, 21, 157, 197, 126, 1, 50, 231, 71, 294, 210, 25, 266, 244, 64, 202, 312, 10, 28, 193, 89, 245, 221, 110, 205, 262, 94, 139, 237, 271, 271, 303, 17, 58, 76, 111, 235, 261, 266, 113, 273, 157, 238, 78, 216, 278, 120, 35, 155, 5, 28, 309, 155, 315, 113, 190, 1, 286, 258, 152, 318, 101, 67, 69, 19, 111, 222, 281, 237, 1, 298, 261, 104, 283, 189, 192, 180, 53, 152, 275, 213, 91, 193, 2, 260, 34, 174, 18, 115, 262, 29, 188, 238, 290, 0, 236, 93, 202, 214, 54, 102, 8, 149, 22, 239, 114, 195, 290, 193, 100, 37, 270, 72, 32, 270, 283, 54, 62, 280, 286, 72, 145, 305, 233, 33, 88, 77, 280, 146, 222, 185, 186, 59, 303, 272, 217, 10, 55, 134, 296, 34, 248, 289, 8, 88, 295, 136, 246, 87, 164, 244, 149, 163, 213, 3, 212, 186, 94, 235, 124, 215, 126, 11, 217, 144, 137, 157, 194, 159, 62, 200, 198, 125, 132, 211, 180, 106, 198, 179, 71, 266, 232, 5, 44, 297, 36, 23, 242, 95, 262, 288, 141, 271, 256, 272, 263, 73, 298, 130, 68, 79, 218, 271, 240, 309, 157, 316, 135, 560, 188, 733, 383, 571, 317, 187, 129, 255, 790, 885, 357, 337, 70, 983, 716, 132, 611, 185, 952, 903, 440, 812, 699, 556, 764, 943, 890, 356, 975, 129, 637, 645, 915, 146, 278, 461, 165, 434, 491, 704, 503, 651, 546, 910, 107, 106, 424, 400, 846, 567, 829, 16, 910, 263, 608, 232, 793, 647, 787, 739, 308, 185, 234, 608, 629, 795, 886, 54, 638, 418, 840, 689, 663, 806, 324, 249, 550, 823, 356, 915, 621, 461, 5, 833, 512, 573, 819, 869, 865, 583, 108, 196, 762, 332, 723, 869, 278, 221, 19, 792, 605, 239, 13, 339, 971, 875, 865, 664, 779, 34, 821, 306, 117, 157, 357, 254, 897, 466, 583, 775, 710, 370, 249, 682, 20, 832, 650, 600, 170, 60, 673, 665, 661, 125, 302, 205, 518, 510, 139, 749, 975, 239, 367, 977, 26, 12, 175, 235, 773, 618, 785, 849, 93, 238, 846, 24, 118, 728, 163, 283, 5, 380, 85, 731, 347, 849, 126, 609, 190, 258, 536, 126, 581, 739, 901, 755, 207, 318, 537, 877, 673, 810, 837, 616, 795, 765, 224, 52, 981, 376, 947, 245, 955, 801, 610, 644, 802, 238, 402, 499, 294, 381, 540, 78, 354, 384, 780, 413, 362, 534, 54, 390, 232, 853, 342, 399, 498, 483, 674, 577, 4, 229, 526, 8, 160, 46, 603, 982, 702, 408, 798, 296, 113, 209, 649, 897, 248, 781, 760, 338, 382, 633, 858, 219, 335, 816, 857, 522, 759, 710, 840, 706, 536, 449, 488, 952, 423, 616, 118, 535, 420, 532, 821, 291, 437, 515, 980, 154, 446, 587, 796, 791, 990, 75, 57, 112, 521, 509, 322, 703, 926, 904, 70, 669, 548, 819, 404, 42, 966, 595, 647, 234, 968, 613, 108, 873, 740, 375, 146, 127, 454, 672, 429, 239, 0, 304, 295, 265, 554, 706, 356, 431, 666, 463, 912, 341, 829, 860, 807, 385, 911, 691, 128, 82, 668, 655, 311, 94, 307, 48, 150, 979, 548, 495, 255, 703, 406, 972, 568, 362, 679, 131, 788, 129, 193, 956, 890, 654, 405, 610, 820, 962, 983, 459, 86, 21, 758, 502, 41, 261, 854, 852, 211, 666, 716, 316, 389, 426, 213, 930, 577, 472, 820, 194, 837, 278, 838, 790, 504, 716, 835, 616, 688, 369, 915, 161, 894, 118, 970, 127, 663, 642, 578, 869, 767, 376, 659, 108, 212, 85, 178, 87, 949, 661, 323, 677, 429, 981, 892, 352, 351, 915, 408, 242, 175, 977, 501, 567, 875, 281, 619, 365, 373, 314, 265, 680, 366, 209, 813, 482, 320, 547, 31, 559, 263, 454, 330, 303, 30, 352, 313, 637, 386, 231, 234, 772, 709, 39, 210, 754, 881, 610, 45, 196, 470, 145, 671, 32, 882, 9, 524, 834, 540, 441, 337, 814, 973, 624, 829, 772, 940, 475, 431, 210, 757, 716, 365, 739, 421, 758, 734, 341, 68, 948, 229, 392, 47, 42, 890, 563, 289, 676, 103, 752, 221, 858, 284, 280, 891, 725, 975, 265, 876, 43, 947, 451, 396, 6, 884, 210, 964, 67, 493, 390, 248, 405, 166, 958, 595, 170, 813, 259, 941, 390, 527, 721, 407, 875, 435, 240, 515, 762, 724, 801, 242, 307, 25, 979, 955, 633, 627, 150, 309, 465, 306, 71, 866, 840, 164, 393, 568, 276, 108, 917, 652, 371, 634, 232, 733, 688, 460, 347, 282, 836, 769, 496, 466, 344, 630, 552, 311, 393, 447, 81, 845, 804, 3, 189, 84, 337, 468, 462, 207, 333, 810, 653, 945, 847, 700, 249, 804, 844, 538, 989, 693, 757, 137, 859, 927, 958, 668, 36, 594, 122, 587, 361, 416, 450, 179, 107, 580, 948, 874, 463, 335, 88, 284, 398, 144, 286, 359, 553, 255, 675, 592, 601, 647, 819, 513, 324, 430, 456, 767, 524, 962, 464, 651, 961, 961, 269, 21, 407, 901, 949, 856, 466, 170, 58, 887, 94, 541, 580, 978, 571, 836, 922, 964, 981, 484, 160, 860, 177, 658, 794, 981, 356, 410, 367, 532, 350, 453, 297, 415, 835, 843, 696, 693, 122, 629, 118, 557, 863, 109, 133, 750, 467, 516, 661, 360, 979, 800, 34, 238, 6, 673, 378, 145, 783, 846, 123, 442, 771, 288, 571, 195, 788, 586, 737, 434, 620, 35, 49, 627, 694, 179, 518, 721, 469, 790, 375, 625, 504, 143, 859, 44, 69, 268, 777, 486, 946, 50, 569, 47, 564, 293, 550, 982, 558, 566, 21, 805, 491, 266, 270, 381, 102, 482, 969, 655, 806, 492, 37, 563, 520, 262, 273, 873, 404, 938, 525, 330, 754, 634, 619, 810, 693, 649, 719, 684, 789, 68, 662, 769, 475, 539, 370, 831, 565, 112, 499, 315, 521, 123, 258, 962, 390, 503, 615, 153, 950, 333, 362, 335, 176, 155, 950, 28, 43, 96, 794, 601, 812, 902, 425, 59, 544, 941, 705, 77, 421, 944, 8, 713, 290, 23, 115, 722, 88, 632, 96, 721, 144, 266, 195, 11, 467, 871, 926, 699, 25, 61, 42, 785, 345, 370, 434, 742, 383, 370, 766, 123, 155, 599, 45, 589, 150, 243, 891, 535, 192, 604, 770, 517, 741, 436, 317, 272, 350, 628, 8, 730, 259, 501, 355, 360, 382, 603, 469, 151, 634, 863, 13, 380, 923, 279, 100, 669, 136, 836, 927, 325, 419, 407, 944, 328, 153, 640, 251, 915, 820, 53, 22, 814, 269, 220, 202, 434, 255, 181, 87, 161, 504, 135, 627, 198, 872, 69, 339, 426, 700, 989, 898, 656, 964, 925, 622, 494, 784, 418, 642, 840, 538, 702, 206, 970, 988, 869, 451, 738, 962, 757, 634, 216, 314, 927, 884, 300, 648, 726, 988, 586, 278, 947, 585, 235, 585, 611, 96, 459, 974, 781, 566, 515, 243, 484, 114, 675, 947, 805, 337, 974, 553, 749, 387, 721, 934, 507, 659, 353, 605, 189, 644, 176, 283, 218, 663, 12, 948, 987, 937, 980, 120, 855, 985, 184, 261, 601, 808, 428, 758, 368, 240, 268, 13, 461, 215, 425, 431, 471, 357, 380, 23, 323, 407, 172, 35, 194, 274, 249, 473, 38, 247, 126, 308, 41, 170, 46, 51, 160, 414, 377, 11, 245, 5, 155, 194, 398, 159, 65, 400, 285, 218, 205, 89, 272, 424, 93, 82, 106, 460, 294, 463, 345, 410, 282, 69, 196, 340, 89, 21, 367, 125, 340, 313, 150, 408, 87, 295, 278, 196, 0, 218, 472, 449, 407, 80, 330, 149, 162, 355, 64, 277, 249, 99, 157, 189, 468, 277, 133, 165, 133, 219, 178, 84, 210, 308, 229, 159, 12, 135, 135, 398, 279, 221, 311, 57, 139, 15, 213, 270, 193, 148, 263, 190, 187, 127, 349, 96, 235, 300, 447, 447, 398, 320, 91, 238, 194, 220, 482, 184, 282, 450, 383, 369, 257, 469, 436, 71, 212, 436, 117, 241, 407, 113, 191, 325, 52, 54, 383, 467, 119, 430, 271, 341, 257, 104, 204, 382, 38, 158, 321, 191, 477, 54, 473, 245, 35, 101, 184, 365, 88, 336, 184, 260, 243, 25, 199, 315, 391, 384, 19, 347, 357, 148, 179, 100, 41, 367, 38, 290, 361, 132, 49, 421, 203, 61, 116, 278, 100, 402, 58, 391, 182, 375, 86, 174, 360, 14, 46, 59, 250, 90, 308, 324, 456, 441, 284, 201, 352, 229, 410, 337, 446, 277, 466, 320, 162, 358, 195, 448, 464, 68, 148, 96, 399, 483, 263, 159, 390, 383, 390, 148, 74, 221, 158, 267, 237, 28, 263, 230, 101, 352, 120, 278, 230, 323, 20, 364, 109, 310, 108, 117, 0, 166, 87, 423, 151, 331, 300, 102, 162, 409, 392, 389, 92, 79, 275, 214, 77, 219, 448, 471, 306, 202, 472, 279, 382, 83, 64, 212, 276, 69, 17, 258, 162, 325, 224, 171, 340, 445, 94, 55, 330, 80, 383, 425, 424, 162, 415, 225, 210, 12, 340, 323, 258, 221, 360, 101, 249, 48, 431, 157, 193, 235, 346, 128, 302, 435, 175, 258, 277, 100, 312, 345, 110, 459, 231, 123, 220, 420, 354, 478, 415, 271, 176, 269, 63, 462, 294, 450, 269, 416, 262, 378, 27, 3, 72, 193, 70, 236, 97, 225, 78, 280, 93, 118, 406, 348, 177, 484, 479, 186, 391, 66, 438, 388, 135, 104, 316, 122, 98, 54, 87, 228, 382, 453, 407, 63, 76, 354, 154, 99, 403, 126, 322, 227, 311, 96, 129, 376, 248, 289, 198, 181, 445, 37, 189, 287, 173, 399, 168, 468, 319, 373, 142, 438, 429, 43, 345, 377, 101, 335, 171, 477, 255, 130, 65, 168, 182, 261, 129, 340, 446, 331, 123, 323, 277, 216, 182, 369, 456, 94, 167, 215, 268, 175, 178, 396, 73, 5, 336, 130, 263, 318, 469, 430, 166, 289, 183, 368, 52, 395, 140, 126, 226, 351, 431, 404, 385, 430, 341, 96, 78, 189, 297, 374, 51, 126, 385, 69, 339, 278, 402, 121, 312, 246, 195, 0, 28, 19, 17, 5, 23, 11, 21, 9, 14, 17, 17, 23, 22, 5, 14, 4, 15, 11, 20, 13, 18, 14, 17, 2, 10, 3, 22, 12, 23, 0, 28, 19, 17, 5, 23, 11, 21, 9, 14, 17, 17, 23, 22, 5, 14, 4, 15, 11, 20, 13, 18, 14, 17, 2, 10, 3, 22, 12, 23, 60, 88, 49, 47, 35, 83, 71, 21, 69, 74, 77, 17, 83, 82, 35, 14, 4, 15, 41, 20, 13, 18, 74, 77, 62, 10, 33, 52, 72, 53, 65, 65, 77, 47, 32, 2, 15, 83, 58, 15, 20, 25, 35, 20, 28, 21, 64, 16, 82, 42, 49, 57, 86, 66, 87, 16, 88, 17, 3, 7, 17, 88, 25, 78, 48, 87, 61, 8, 88, 18, 38, 22, 17, 11, 26, 60, 77, 72, 85, 10, 7, 33, 27, 49, 45, 72, 79, 43, 21, 61, 0, 0, 96, 44, 179, 177, 39, 35, 163, 15, 55, 16, 29, 23, 21, 2, 95, 0, 122, 175, 125, 154, 1, 80, 178, 151, 142, 58, 13, 180, 114, 69, 49, 73, 51, 49, 88, 136, 101, 175, 122, 15, 70, 61, 3, 84, 76, 141, 124, 56, 6, 42, 161, 167, 104, 68, 11, 66, 80, 109, 37, 9, 105, 82, 41, 118, 152, 83, 13, 96, 66, 38, 24, 0, 69, 59, 156, 118, 67, 150, 161, 66, 61, 111, 163, 29, 75, 4, 163, 49, 139, 53, 59, 48, 6, 44, 168, 77, 139, 136, 13, 37, 92, 65, 139, 73, 153, 129, 153, 25, 170, 129, 60, 123, 62, 75, 9, 79, 92, 155, 48, 133, 35, 142, 54, 157, 14, 148, 102, 172, 146, 88, 2, 145, 173, 17, 81, 80, 181, 178, 64, 69, 167, 5, 179, 105, 11, 58, 16, 93, 17, 145, 6, 95, 13, 97, 26, 42, 40, 92, 70, 99, 179, 41, 114, 165, 67, 9, 167, 104, 125, 174, 16, 80, 54, 43, 179, 109, 95, 43, 40, 37, 37, 163, 12, 16, 25, 23, 11, 35, 23, 15, 27, 2, 29, 23, 35, 16, 11, 14, 10, 21, 41, 14, 1, 24, 38, 11, 2, 16, 27, 40, 30, 41, 35, 17, 23, 35, 32, 38, 3, 35, 10, 15, 14, 19, 12, 2, 11, 9, 11, 7, 9, 1, 13, 2, 1, 9, 7, 2, 12, 2, 11, 9, 11, 7, 9, 1, 13, 2, 1, 9, 7, 2, 12, 16, 25, 23, 11, 35, 23, 15, 27, 2, 29, 23, 35, 16, 11, 14, 10, 21, 41, 14, 1, 24, 38, 11, 2, 16, 27, 40, 30, 41, 35, 17, 23, 35, 32, 38, 3, 35, 10, 15, 14, 19, 0, 4, 1, 5, 5, 5, 0, 1, 1, 0, 1, 1, 12, 2, 11, 9, 11, 7, 9, 1, 13, 2, 1, 9, 7, 2, 0, 1, 1, 0, 12, 2, 11, 9, 11, 7, 9, 1, 13, 2, 1, 9, 7, 2, 0, 1, 1, 0]; + for edge_idx in vec { + self.edges_queue + .push(vec![edges[edge_idx], edges[edge_idx + 1]]); + } + unsafe { + FLAG = false; + } + } else { + while self.edges_queue.len() < num_edges_to_sample { + let edge_idx = rng.gen_range(0, num_edges); + print!("{},", edge_idx); + self.edges_queue + .push(vec![edges[edge_idx], edges[edge_idx + 1]]); + } } } @@ -89,7 +103,7 @@ impl CommonOperatorTrait for ScanSampling { while !self.edges_queue.is_empty() { let edge = self.edges_queue.pop().unwrap(); self.base_scan.base_op.probe_tuple[0] = edge[0]; - self.base_scan.base_op.probe_tuple[1] = edge[0]; + self.base_scan.base_op.probe_tuple[1] = edge[1]; self.base_scan.base_op.num_out_tuples += 1; for next_op in &mut self.base_scan.base_op.next { next_op.borrow_mut().process_new_tuple(); diff --git a/src/graph_impl/multi_graph/plan/operator/sink/sink.rs b/src/graph_impl/multi_graph/plan/operator/sink/sink.rs index 527826d2..a464a9a4 100644 --- a/src/graph_impl/multi_graph/plan/operator/sink/sink.rs +++ b/src/graph_impl/multi_graph/plan/operator/sink/sink.rs @@ -64,7 +64,7 @@ impl CommonOperatorTrait for BaseSink { fn execute(&mut self) { if !self.previous.is_empty() { let mut prev = self.previous[0].as_ptr(); - unsafe{ + unsafe { (&mut *prev).execute(); } } else { diff --git a/src/graph_impl/multi_graph/plan/query_plan.rs b/src/graph_impl/multi_graph/plan/query_plan.rs index adbdb47e..ff45d11c 100644 --- a/src/graph_impl/multi_graph/plan/query_plan.rs +++ b/src/graph_impl/multi_graph/plan/query_plan.rs @@ -243,18 +243,22 @@ impl QueryPlan { let mut probes = vec![]; for i in 1..self.subplans.len() { let mut operator = self.subplans[i].clone(); - loop{ + loop { { let mut op_ref = operator.borrow(); - if let Operator::Probe(pb) = op_ref.deref(){ + if let Operator::Probe(pb) = op_ref.deref() { probes.push(operator.clone()); } } let prev = { let mut op_ref = operator.borrow(); - get_op_attr_as_ref!(op_ref.deref(),prev).as_ref().map(|op|op.clone()) + get_op_attr_as_ref!(op_ref.deref(), prev) + .as_ref() + .map(|op| op.clone()) }; - if prev.is_none() {break;} + if prev.is_none() { + break; + } operator = prev.unwrap(); } } @@ -264,8 +268,12 @@ impl QueryPlan { build.hash_table = Some(hash_table.clone()); for probe in &probes { let mut probe_mut = probe.borrow_mut(); - if get_op_attr_as_ref!(probe_mut.deref(),in_subgraph).as_ref().unwrap() == build.probing_subgraph.as_ref().unwrap() { - if let Operator::Probe(pb) = probe_mut.deref_mut(){ + if get_op_attr_as_ref!(probe_mut.deref(), in_subgraph) + .as_ref() + .unwrap() + == build.probing_subgraph.as_ref().unwrap() + { + if let Operator::Probe(pb) = probe_mut.deref_mut() { let mut base_probe = get_probe_as_mut!(pb); base_probe.hash_tables = vec![hash_table.clone()]; break; @@ -288,7 +296,7 @@ impl QueryPlan { for subplan in &mut self.subplans { let probe_tuple = { let subplan_ref = subplan.borrow(); - vec![Id::new(0);get_op_attr!(subplan_ref.deref(),out_tuple_len)] + vec![Id::new(0); get_op_attr!(subplan_ref.deref(), out_tuple_len)] }; let mut first_op = subplan.clone(); loop { diff --git a/src/graph_impl/multi_graph/planner/catalog/adj_list_descriptor.rs b/src/graph_impl/multi_graph/planner/catalog/adj_list_descriptor.rs index e8188c45..c2acd482 100644 --- a/src/graph_impl/multi_graph/planner/catalog/adj_list_descriptor.rs +++ b/src/graph_impl/multi_graph/planner/catalog/adj_list_descriptor.rs @@ -13,7 +13,7 @@ pub struct AdjListDescriptor { pub from_query_vertex: String, pub vertex_idx: usize, pub direction: Direction, - pub label: usize, + pub label: i32, } impl AdjListDescriptor { @@ -21,7 +21,7 @@ impl AdjListDescriptor { from_query_vertex: String, vertex_idx: usize, direction: Direction, - label: usize, + label: i32, ) -> Self { Self { from_query_vertex, diff --git a/src/graph_impl/multi_graph/planner/catalog/catalog.rs b/src/graph_impl/multi_graph/planner/catalog/catalog.rs index d1295aa6..ca188e27 100644 --- a/src/graph_impl/multi_graph/planner/catalog/catalog.rs +++ b/src/graph_impl/multi_graph/planner/catalog/catalog.rs @@ -30,13 +30,15 @@ pub static SINGLE_VERTEX_WEIGHT_BUILD_COEF: f64 = 12.0; pub static MULTI_VERTEX_WEIGHT_PROBE_COEF: f64 = 12.0; pub static MULTI_VERTEX_WEIGHT_BUILD_COEF: f64 = 720.0; +pub static mut LOGGER_FLAG: bool = false; + pub struct Catalog { - in_subgraphs: Vec, - sampled_icost: HashMap>, - sampled_selectivity: HashMap>, - is_sorted_by_node: bool, - num_sampled_edge: usize, - max_input_num_vertices: usize, + pub in_subgraphs: Vec, + pub sampled_icost: HashMap>, + pub sampled_selectivity: HashMap>, + pub is_sorted_by_node: bool, + pub num_sampled_edge: usize, + pub max_input_num_vertices: usize, pub elapsed_time: u128, } @@ -77,14 +79,12 @@ impl Catalog { &self, query_graph: &mut QueryGraph, alds: Vec<&AdjListDescriptor>, - to_type: usize, + to_type: i32, ) -> f64 { let mut approx_icost = 0.0; let mut min_icost = std::f64::MAX; alds.iter().for_each(|ald| { - // Get each ALD icost by finding the largest subgraph (num vertices then num edges) - // of queryGraph used in stats collection and also minimizing sampledIcost. - for num_vertices in (DEF_NUM_EDGES_TO_SAMPLE - 1)..=2 { + for num_vertices in (2..=(DEF_NUM_EDGES_TO_SAMPLE - 1)).rev() { min_icost = std::f64::MAX; let mut num_edges_matched = 0; for (i, sub_graph) in self.in_subgraphs.iter().enumerate() { @@ -136,7 +136,7 @@ impl Catalog { &self, in_subgraph: &mut QueryGraph, alds: &Vec, - to_type: usize, + to_type: i32, ) -> f64 { let mut approx_selectivity = std::f64::MAX; let mut num_vertices = DEF_MAX_INPUT_NUM_VERTICES - 1; @@ -153,8 +153,10 @@ impl Catalog { if new_num_alds_matched == 0 || new_num_alds_matched < num_alds_matched { continue; } - let selectivity_map = &self.sampled_selectivity[&i]; - let sampled_selectivity = selectivity_map + // for (k, v) in &self.sampled_selectivity[&0] { + // println!("{},{}",k,v); + // } + let sampled_selectivity = self.sampled_selectivity[&i] [&self.get_alds_as_str(&alds, Some(&vertex_mapping), Some(to_type))] .clone(); if new_num_alds_matched > num_alds_matched @@ -174,30 +176,33 @@ impl Catalog { &self, alds: &Vec, vertex_mapping: Option<&HashMap>, - to_type: Option, + to_type: Option, ) -> String { let mut from_qvertices_and_dirs = alds .iter() + .filter(|ald| { + vertex_mapping.is_none() + || vertex_mapping + .unwrap() + .get(&ald.from_query_vertex) + .is_some() + }) .map(|ald| { - let tail = ") ".to_owned() - + &ald.direction.to_string() - + "[" - + &ald.label.to_string() - + "]"; - if vertex_mapping.is_none() { - return Some("(".to_owned() + &ald.from_query_vertex + &tail); + "(".to_owned() + + if vertex_mapping.is_none() { + &ald.from_query_vertex } else { let vertex_mapping = vertex_mapping.unwrap(); - if let Some(from) = vertex_mapping.get(&ald.from_query_vertex) { - return Some("(".to_owned() + from + &tail); - } + vertex_mapping.get(&ald.from_query_vertex).unwrap() } - None + + ") " + + &ald.direction.to_string() + + "[" + + &ald.label.to_string() + + "]" }) - .skip_while(|x| x.is_none()) - .map(|x| x.unwrap()) .sorted() - .join(","); + .join(", "); if to_type.is_some() { from_qvertices_and_dirs += &("~".to_owned() + &to_type.unwrap().to_string()); } @@ -210,7 +215,7 @@ impl Catalog { vertex_mapping: &HashMap, ) -> usize { let mut from_vertices_in_alds = HashSet::new(); - for ald in alds.iter() { + for ald in alds { from_vertices_in_alds.insert(ald.from_query_vertex.clone()); } let num_alds_matched = 0; @@ -238,10 +243,10 @@ impl Catalog { self.num_sampled_edge, self.max_input_num_vertices, ); - self.set_input_subgraphs(plans.query_graphs_to_extend().get_query_graph_set()); + self.set_input_subgraphs(plans.query_graphs_to_extend.get_query_graph_set()); self.add_zero_selectivities(&graph, &mut plans); - for query_plan_arr in plans.get_query_plan_arrs() { + for query_plan_arr in &mut plans.query_plans_arrs { self.init(&graph, query_plan_arr); self.execute(query_plan_arr); self.log_output(&graph, query_plan_arr); @@ -276,17 +281,35 @@ impl Catalog { // sink.execute(); // })); } - // for handler in handlers { - // handler.join(); - // } + // for handler in handlers { + // handler.join(); + // } } else { let mut sink = query_plan_arr[0].sink.as_mut().unwrap().as_ptr(); - unsafe{ + unsafe { (&mut *sink).execute(); +// if let Operator::Sink(Sink::BaseSink(base)) = &*sink{ +// for (i,previous) in base.previous.iter().enumerate() { +// println!("-------i={}",i); +// Self::retrieve_op(previous); +// println!() +// } +// } } } } + fn retrieve_op(op:&Rc>>){ + unsafe{ + print!("{:?}->",op.as_ptr()); + } + let op_ref = op.borrow(); + let base = get_base_op_as_ref!(op_ref.deref()); + if let Some(op)=&base.prev{ + Self::retrieve_op(op); + } + } + fn log_output( &mut self, graph: &TypedStaticGraph, @@ -343,6 +366,7 @@ impl Catalog { return; } let mut num_input_tuples = get_op_attr!(operator.borrow().deref(), num_out_tuples); + for other_op in &other { num_input_tuples += get_op_attr!(other_op.borrow().deref(), num_out_tuples); } @@ -355,10 +379,12 @@ impl Catalog { let op_ref = operator.borrow(); get_op_attr_as_ref!(op_ref.deref(), next).clone() }; + + println!("-----"); for i in 0..next.len() { let next_i = next[i].borrow(); if let Operator::EI(EI::Intersect(Intersect::IntersectCatalog(intersect))) = - next_i.deref() + next_i.deref() { let to_type = intersect.base_intersect.base_ei.to_type; let mut alds_as_str_list = vec![]; @@ -377,15 +403,10 @@ impl Catalog { let mut alds_str_with_pattern = "".to_owned(); for j in 0..pattern.len() { let ok: Vec<&str> = splits[j].split("Bwd").collect(); - if j == pattern.len() - 1 { - alds_str_with_pattern = - alds_str_with_pattern + ok[0] + &pattern[j].to_string() + ok[1]; - } else { - alds_str_with_pattern = alds_str_with_pattern - + ok[0] - + &pattern[j].to_string() - + ok[1] - + ", "; + alds_str_with_pattern = + alds_str_with_pattern + ok[0] + &pattern[j].to_string() + ok[1]; + if j != pattern.len() - 1 { + alds_str_with_pattern += ", "; } } alds_as_str_list.push(alds_str_with_pattern); @@ -394,6 +415,7 @@ impl Catalog { alds_as_str_list.push(alds_str); } let mut selectivity = intersect.base_intersect.base_ei.base_op.num_out_tuples; + println!("add_icost_and_selectivity_sorted_by_node={}",selectivity); for other_op in &other { let next = { let other_op_ref = other_op.borrow(); @@ -406,20 +428,16 @@ impl Catalog { .entry(subgraph_idx) .or_insert(HashMap::new()); for alds_as_str in alds_as_str_list { - if num_input_tuples > 0 { - self.sampled_selectivity - .get_mut(&subgraph_idx) - .unwrap() - .insert( - alds_as_str, - (selectivity as f64) / (num_input_tuples as f64), - ); - } else { - self.sampled_selectivity - .get_mut(&subgraph_idx) - .unwrap() - .insert(alds_as_str, 0.0); - } + self.sampled_selectivity + .get_mut(&subgraph_idx) + .unwrap() + .insert(alds_as_str, + if num_input_tuples > 0 { + (selectivity as f64) / (num_input_tuples as f64) + } else { + 0.0 + }, + ); } let noop = { let next_ref = next[i].borrow(); @@ -427,13 +445,12 @@ impl Catalog { }; let mut other_noops = vec![]; for (j, other) in other.iter().enumerate() { - let next_op = { + other_noops.push({ let other_ref = other.borrow(); let next_i = get_op_attr_as_ref!(other_ref.deref(), next)[i].clone(); let next_ref = next_i.borrow(); get_op_attr_as_ref!(next_ref.deref(), next)[j].clone() - }; - other_noops.push(next_op); + }); } self.add_icost_and_selectivity(noop, other_noops, is_undirected); } @@ -468,7 +485,7 @@ impl Catalog { for (i, next) in next_vec.iter().enumerate() { let next_ref = next.borrow(); if let Operator::EI(EI::Intersect(Intersect::IntersectCatalog(intersect))) = - next_ref.deref() + next_ref.deref() { let alds = &intersect.base_intersect.base_ei.alds; let mut alds_as_str_list = vec![]; @@ -484,15 +501,10 @@ impl Catalog { let mut alds_str_with_pattern = "".to_owned(); for j in 0..pattern.len() { let ok: Vec<&str> = splits[j].split("Bwd").collect(); - if j == pattern.len() - 1 { - alds_str_with_pattern = - alds_str_with_pattern + ok[0] + &pattern[j].to_string() + ok[1]; - } else { - alds_str_with_pattern = alds_str_with_pattern - + ok[0] - + &pattern[j].to_string() - + ok[1] - + ", "; + alds_str_with_pattern = + alds_str_with_pattern + ok[0] + &pattern[j].to_string() + ok[1]; + if j != pattern.len() - 1 { + alds_str_with_pattern += ", "; } } alds_as_str_list.push(alds_str_with_pattern); @@ -556,20 +568,16 @@ impl Catalog { .entry(subgraph_idx) .or_insert(HashMap::new()); for alds_as_str in &alds_as_str_list { - if num_input_tuples > 0 { - self.sampled_selectivity - .get_mut(&subgraph_idx) - .unwrap() - .insert( - alds_as_str.to_owned() + "~" + &to_type.to_string(), - (selectivity as f64) / (num_input_tuples as f64), - ); - } else { - self.sampled_selectivity - .get_mut(&subgraph_idx) - .unwrap() - .insert(alds_as_str.to_owned() + "~" + &to_type.to_string(), 0.0); - } + self.sampled_selectivity + .get_mut(&subgraph_idx) + .unwrap() + .insert(alds_as_str.to_owned() + "~" + &to_type.to_string(), + if num_input_tuples > 0 { + (selectivity as f64) / (num_input_tuples as f64) + } else { + 0.0 + }, + ); } let mut other_noops = vec![]; for other_op in &other { @@ -667,17 +675,17 @@ impl Catalog { graph: &TypedStaticGraph, plans: &mut CatalogPlans, ) { - let selectivity_zero = plans.get_selectivity_zero(); - for select in selectivity_zero { - let subgraph_idx = self.get_subgraph_idx(&mut select.0); + let selectivity_zero = &mut plans.selectivity_zero; + for (q_graph, alds,to_type) in selectivity_zero { + let subgraph_idx = self.get_subgraph_idx(q_graph); if self.sampled_selectivity.get(&subgraph_idx).is_none() { self.sampled_selectivity .insert(subgraph_idx, HashMap::new()); } let mut alds_as_str_list = vec![]; - let alds_str = self.get_alds_as_str(&select.1, None, None); + let alds_str = self.get_alds_as_str(alds, None, None); if !graph.is_directed() { - let splits: Vec<&str> = alds_str.split(',').collect(); + let splits: Vec<&str> = alds_str.split(", ").collect(); let direction_patterns = self.generate_direction_patterns(splits.len(), !graph.is_directed()); for pattern in direction_patterns { @@ -687,7 +695,7 @@ impl Catalog { alds_str_with_pattern = alds_str_with_pattern + ok[0] + &pattern[i].to_string() + &ok[1]; if i != pattern.len() - 1 { - alds_str_with_pattern.push_str(","); + alds_str_with_pattern.push_str(", "); } } alds_as_str_list.push(alds_str_with_pattern); @@ -696,9 +704,8 @@ impl Catalog { alds_as_str_list.push(alds_str); } for alds_as_str in alds_as_str_list { - if let Some(selectivity) = self.sampled_selectivity.get_mut(&subgraph_idx) { - selectivity.insert(alds_as_str + "~" + &select.2.to_string(), 0.00); - } + let selectivity= self.sampled_selectivity.get_mut(&subgraph_idx).unwrap(); + selectivity.insert(alds_as_str + "~" + &to_type.to_string(), 0.00); } } } diff --git a/src/graph_impl/multi_graph/planner/catalog/catalog_plans.rs b/src/graph_impl/multi_graph/planner/catalog/catalog_plans.rs index 51046e7b..030e85e1 100644 --- a/src/graph_impl/multi_graph/planner/catalog/catalog_plans.rs +++ b/src/graph_impl/multi_graph/planner/catalog/catalog_plans.rs @@ -17,12 +17,14 @@ use graph_impl::multi_graph::planner::catalog::query_edge::QueryEdge; use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::multi_graph::query::query_graph_set::QueryGraphSet; use graph_impl::multi_graph::utils::set_utils; +use graph_impl::static_graph::graph::KEY_ANY; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; use std::cell::RefCell; use std::hash::Hash; use std::ops::{Deref, DerefMut}; use std::rc::Rc; +use itertools::Itertools; pub static DEF_NUM_EDGES_TO_SAMPLE: usize = 1000; pub static DEF_MAX_INPUT_NUM_VERTICES: usize = 3; @@ -34,10 +36,10 @@ pub struct CatalogPlans { num_node_labels: usize, num_edge_labels: usize, sorted_by_node: bool, - query_graphs_to_extend: QueryGraphSet, - query_plans_arrs: Vec>>, + pub query_graphs_to_extend: QueryGraphSet, + pub query_plans_arrs: Vec>>, is_directed: bool, - selectivity_zero: Vec<(QueryGraph, Vec, usize)>, + pub selectivity_zero: Vec<(QueryGraph, Vec, i32)>, query_vertex_to_idx_map: HashMap, } @@ -72,19 +74,15 @@ impl CatalogPlans { }; for scan in scans { let mut noop = Noop::new(scan.base_scan.base_op.out_subgraph.clone()); - let scan_op = Rc::new(RefCell::new(Operator::Scan(Scan::ScanSampling(scan)))); - noop.base_op.prev = Some(scan_op.clone()); - noop.base_op.out_qvertex_to_idx_map = { - let scan_ref = scan_op.borrow(); - get_op_attr_as_ref!(scan_ref.deref(), out_qvertex_to_idx_map).clone() - }; - let mut noop_op = Rc::new(RefCell::new(Operator::Noop(noop))); - *get_op_attr_as_mut!(scan_op.borrow_mut().deref_mut(), next) = vec![noop_op.clone()]; - plans.set_next_operators(graph, noop_op, false); - let mut query_plans_arr = vec![]; - query_plans_arr.push(QueryPlan::new(scan_op.clone())); + let scan_pointer = Rc::new(RefCell::new(Operator::Scan(Scan::ScanSampling(scan)))); + noop.base_op.prev = Some(scan_pointer.clone()); + noop.base_op.out_qvertex_to_idx_map = get_op_attr_as_ref!(scan_pointer.borrow().deref(), out_qvertex_to_idx_map).clone(); + let mut noop_pointer = Rc::new(RefCell::new(Operator::Noop(noop))); + *get_op_attr_as_mut!(scan_pointer.borrow_mut().deref_mut(), next) = vec![noop_pointer.clone()]; + plans.set_next_operators(graph, noop_pointer, false); + let mut query_plans_arr = vec![QueryPlan::new(scan_pointer.clone())]; for i in 1..num_thread { - let mut scan_ref = scan_op.borrow(); + let mut scan_ref = scan_pointer.borrow(); let scan_copy = if let Operator::Scan(Scan::ScanSampling(scan)) = scan_ref.deref() { scan.copy_default() } else { @@ -128,8 +126,8 @@ impl CatalogPlans { let query_vertices = in_subgraph.get_query_vertices().clone(); let mut descriptors = vec![]; - for query_vertex_to_extend in set_utils::get_power_set_excluding_empty_set(query_vertices){ - for alds in self.generate_alds(&query_vertex_to_extend, self.is_directed){ + for query_vertex_to_extend in set_utils::get_power_set_excluding_empty_set(query_vertices) { + for alds in self.generate_alds(&query_vertex_to_extend, self.is_directed) { descriptors.push(Descriptor { out_subgraph: self.get_out_subgraph(in_subgraph.copy(), alds.clone()), alds, @@ -144,26 +142,27 @@ impl CatalogPlans { if self.sorted_by_node { for mut descriptor in descriptors { let mut types = vec![]; - for to_type in 0..=self.num_node_labels { + let node_label_cnt = std::cmp::max(self.num_node_labels, 1); + for to_type in 0..node_label_cnt { let mut produces_output = true; for ald in &descriptor.alds { let from_type = in_subgraph.get_query_vertex_type(&ald.from_query_vertex); if (ald.direction == Direction::Fwd - && 0 == graph.get_num_edges(from_type, to_type, ald.label)) + && 0 == graph.get_num_edges(from_type, to_type as i32, ald.label)) || (ald.direction == Direction::Bwd - && 0 == graph.get_num_edges(to_type, from_type, ald.label)) + && 0 == graph.get_num_edges(to_type as i32, from_type, ald.label)) { produces_output = false; break; } } if produces_output { - types.push(to_type); + types.push(to_type as i32); } else { self.selectivity_zero.push(( in_subgraph.clone(), descriptor.alds.clone(), - to_type, + to_type as i32, )); } } @@ -208,7 +207,7 @@ impl CatalogPlans { out_qvertex_to_idx_map.insert(to_qvertex.to_owned(), out_qvertex_to_idx_map.len()); let mut ic = IntersectCatalog::new( to_qvertex.to_owned(), - 0, + KEY_ANY, descriptor.alds.clone(), descriptor.out_subgraph.clone(), in_subgraph.clone(), @@ -228,7 +227,7 @@ impl CatalogPlans { let mut next_noops = if self.sorted_by_node { vec![Noop::new(QueryGraph::empty()); 1] } else { - vec![Noop::new(QueryGraph::empty()); self.num_node_labels+1] + vec![Noop::new(QueryGraph::empty()); self.num_node_labels + 1] }; self.set_noops( get_op_attr_as_ref!(next_op.borrow().deref(), out_subgraph), @@ -244,6 +243,12 @@ impl CatalogPlans { if get_op_attr_as_ref!(next_op.borrow().deref(), out_subgraph).get_num_qvertices() <= self.max_input_num_vertices { + println!( + "next_noops_cnt={},sort_by_node={},max_input_num_vertices={}", + next_noops.len(), + self.sorted_by_node, + self.max_input_num_vertices + ); for next_noop in next_noops { *get_op_attr_as_mut!( next_noop.borrow_mut().deref_mut(), @@ -273,9 +278,10 @@ impl CatalogPlans { noops[0] = Noop::new(query_graph.clone()); noops[0].base_op.out_qvertex_to_idx_map = out_qvertex_to_idx_map.clone(); } else { - for to_type in 0..=self.num_node_labels { + let node_label_cnt = std::cmp::max(self.num_node_labels, 1); + for to_type in 0..node_label_cnt { let mut query_graph_copy = query_graph.copy(); - query_graph_copy.set_query_vertex_type(to_qvertex.clone(), to_type); + query_graph_copy.set_query_vertex_type(to_qvertex.clone(), to_type as i32); noops[to_type] = Noop::new(query_graph_copy); } } @@ -308,10 +314,11 @@ impl CatalogPlans { alds_list } - fn generate_labels_patterns(&self, size: usize) -> Vec> { + fn generate_labels_patterns(&self, size: usize) -> Vec> { let mut labels = vec![]; - for label in 0..self.num_edge_labels { - labels.push(label); + let edge_label_cnt = std::cmp::max(self.num_edge_labels, 1); + for label in 0..edge_label_cnt { + labels.push(label as i32); } set_utils::generate_permutations(labels, size) } @@ -400,9 +407,9 @@ impl CatalogPlans { .as_ref() .unwrap() .get_neighbor_ids() - { - edges.push(vec![Id::new(from_vertex), to_vertex.clone()]); - } + { + edges.push(vec![Id::new(from_vertex), to_vertex.clone()]); + } } let mut out_subgraph = QueryGraph::empty(); out_subgraph.add_qedge(QueryEdge::new("a".to_owned(), "b".to_owned(), 0, 0, 0)); @@ -420,13 +427,16 @@ impl CatalogPlans { let num_vertices = graph.node_count(); let mut key_to_edges_map = HashMap::new(); let mut key_to_curr_idx = HashMap::new(); - for from_type in 0..=self.num_node_labels { - for label in 0..=self.num_edge_labels { - for to_type in 0..=self.num_node_labels { + let node_label_cnt = std::cmp::max(self.num_node_labels, 1); + let edge_label_cnt = std::cmp::max(self.num_edge_labels, 1); + for from_type in 0..node_label_cnt { + for label in 0..edge_label_cnt { + for to_type in 0..node_label_cnt { let edge_key = TypedStaticGraph::::get_edge_key( from_type, to_type, label, ); - let num_edges = graph.get_num_edges(from_type, to_type, label); + let num_edges = + graph.get_num_edges(from_type as i32, to_type as i32, label as i32); key_to_edges_map.insert(edge_key, vec![0; num_edges * 2]); key_to_curr_idx.insert(edge_key, 0); } @@ -444,10 +454,12 @@ impl CatalogPlans { let (to_type, label) = if self.sorted_by_node { (label_type, 0) } else { - (vertex_types[neighbours[to_idx].id()], label_type) + (vertex_types[neighbours[to_idx].id()] as usize, label_type) }; let edge_key = TypedStaticGraph::::get_edge_key( - from_type, to_type, label, + from_type as usize, + to_type, + label, ); let curr_idx = key_to_curr_idx[&edge_key]; key_to_edges_map.get_mut(&edge_key).unwrap()[curr_idx] = from_vertex; @@ -458,31 +470,33 @@ impl CatalogPlans { } } let mut scans = vec![]; - for from_type in 0..=self.num_node_labels { - for edge_label in 0..=self.num_edge_labels { - for to_type in 0..=self.num_node_labels { + for from_type in 0..node_label_cnt { + for edge_label in 0..edge_label_cnt { + for to_type in 0..node_label_cnt { let mut out_subgraph = QueryGraph::empty(); out_subgraph.add_qedge(QueryEdge::new( "a".to_owned(), "b".to_owned(), - from_type, - to_type, - edge_label, + from_type as i32, + to_type as i32, + edge_label as i32, )); let edge_key = TypedStaticGraph::::get_edge_key( from_type, to_type, edge_label, ); - let actual_num_edges = graph.get_num_edges(from_type, to_type, edge_label); + let actual_num_edges = + graph.get_num_edges(from_type as i32, to_type as i32, edge_label as i32); if actual_num_edges <= 0 { continue; } - let mut num_edges_to_sample = self.num_sampled_edges - * (graph.get_num_edges(from_type, to_type, edge_label) / graph.edge_count()) - as usize; + let mut num_edges_to_sample = (self.num_sampled_edges as f64 + * (graph.get_num_edges(from_type as i32, to_type as i32, edge_label as i32) as f64 + / graph.edge_count() as f64)) as usize; let mut scan = ScanSampling::new(out_subgraph); if self.sorted_by_node && num_edges_to_sample < 1000 { num_edges_to_sample = actual_num_edges; } + println!("{:?}",key_to_edges_map[&edge_key]); scan.set_edge_indices_to_sample( key_to_edges_map[&edge_key] .iter() @@ -496,20 +510,6 @@ impl CatalogPlans { } scans } - - pub fn query_graphs_to_extend(&self) -> &QueryGraphSet { - &self.query_graphs_to_extend - } - - pub fn get_selectivity_zero( - &mut self, - ) -> &mut Vec<(QueryGraph, Vec, usize)> { - &mut self.selectivity_zero - } - - pub fn get_query_plan_arrs(&mut self) -> &mut Vec>> { - self.query_plans_arrs.as_mut() - } } pub struct Descriptor { diff --git a/src/graph_impl/multi_graph/planner/catalog/operator/intersect_catalog.rs b/src/graph_impl/multi_graph/planner/catalog/operator/intersect_catalog.rs index d66cf306..3be25b33 100644 --- a/src/graph_impl/multi_graph/planner/catalog/operator/intersect_catalog.rs +++ b/src/graph_impl/multi_graph/planner/catalog/operator/intersect_catalog.rs @@ -4,6 +4,7 @@ use graph_impl::multi_graph::plan::operator::extend::EI::CachingType; use graph_impl::multi_graph::plan::operator::operator::{CommonOperatorTrait, Operator}; use graph_impl::multi_graph::planner::catalog::adj_list_descriptor::AdjListDescriptor; use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; +use graph_impl::static_graph::graph::KEY_ANY; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; use std::cell::RefCell; @@ -21,7 +22,7 @@ pub struct IntersectCatalog { impl IntersectCatalog { pub fn new( to_qvertex: String, - to_type: usize, + to_type: i32, alds: Vec, out_subgraph: QueryGraph, in_subgraph: QueryGraph, @@ -85,12 +86,15 @@ impl CommonOperatorTrait for IntersectCatalog { [base_ei.base_op.probe_tuple[base_ei.vertex_idx_to_cache[1]].id()] .as_ref() .unwrap(); + println!("****"); self.last_icost += adj.intersect( base_ei.labels_or_to_types_to_cache[1], &mut base_ei.init_neighbours, &mut base_ei.cached_neighbours, ); - if base_ei.to_type != 0 { + println!("****"); + + if base_ei.to_type != KEY_ANY { let mut curr_end_idx = 0; for i in base_ei.cached_neighbours.start_idx..base_ei.cached_neighbours.end_idx { @@ -119,6 +123,7 @@ impl CommonOperatorTrait for IntersectCatalog { ); } } + match base_ei.caching_type { CachingType::None | CachingType::FullCaching => { base_ei.base_op.icost += self.last_icost; @@ -152,6 +157,7 @@ impl CommonOperatorTrait for IntersectCatalog { } } +// println!("start_idx={},end_idx={}",base_ei.out_neighbours.start_idx,base_ei.out_neighbours.end_idx); for idx in base_ei.out_neighbours.start_idx..base_ei.out_neighbours.end_idx { base_ei.base_op.probe_tuple[base_ei.out_idx] = base_ei.out_neighbours.ids[idx]; base_ei.base_op.num_out_tuples += 1; @@ -161,7 +167,10 @@ impl CommonOperatorTrait for IntersectCatalog { base_ei .base_op .next - .get(base_ei.vertex_types[base_ei.base_op.probe_tuple[base_ei.out_idx].id()]) + .get( + base_ei.vertex_types[base_ei.base_op.probe_tuple[base_ei.out_idx].id()] + as usize, + ) .map(|next_op| next_op.borrow_mut().process_new_tuple()); } } diff --git a/src/graph_impl/multi_graph/planner/catalog/operator/noop.rs b/src/graph_impl/multi_graph/planner/catalog/operator/noop.rs index b6fbcb75..e819bb71 100644 --- a/src/graph_impl/multi_graph/planner/catalog/operator/noop.rs +++ b/src/graph_impl/multi_graph/planner/catalog/operator/noop.rs @@ -36,12 +36,9 @@ impl CommonOperatorTrait for Noop { fn process_new_tuple(&mut self) { self.base_op.num_out_tuples += 1; - self.base_op - .next - .iter() - .for_each(|next_op| { - next_op.borrow_mut().process_new_tuple(); - }); + self.base_op.next.iter().for_each(|next_op| { + next_op.borrow_mut().process_new_tuple(); + }); } fn execute(&mut self) { diff --git a/src/graph_impl/multi_graph/planner/catalog/query_edge.rs b/src/graph_impl/multi_graph/planner/catalog/query_edge.rs index 48f1b66f..04b6f0e6 100644 --- a/src/graph_impl/multi_graph/planner/catalog/query_edge.rs +++ b/src/graph_impl/multi_graph/planner/catalog/query_edge.rs @@ -2,18 +2,18 @@ pub struct QueryEdge { pub from_query_vertex: String, pub to_query_vertex: String, - pub from_type: usize, - pub to_type: usize, - pub label: usize, + pub from_type: i32, + pub to_type: i32, + pub label: i32, } impl QueryEdge { pub fn new( from_qvertex: String, to_qvertex: String, - from_type: usize, - to_type: usize, - label: usize, + from_type: i32, + to_type: i32, + label: i32, ) -> Self { Self { from_query_vertex: from_qvertex, diff --git a/src/graph_impl/multi_graph/planner/catalog/query_graph.rs b/src/graph_impl/multi_graph/planner/catalog/query_graph.rs index e3401bee..6bf001f3 100644 --- a/src/graph_impl/multi_graph/planner/catalog/query_graph.rs +++ b/src/graph_impl/multi_graph/planner/catalog/query_graph.rs @@ -1,5 +1,7 @@ +use graph_impl::multi_graph::planner::catalog::catalog::LOGGER_FLAG; use graph_impl::multi_graph::planner::catalog::query_edge::QueryEdge; use graph_impl::multi_graph::planner::catalog::subgraph_mapping_iterator::SubgraphMappingIterator; +use graph_impl::static_graph::graph::KEY_ANY; use hashbrown::{HashMap, HashSet}; use itertools::Itertools; use std::iter::FromIterator; @@ -7,7 +9,7 @@ use std::iter::FromIterator; #[derive(Clone)] pub struct QueryGraph { pub qvertex_to_qedges_map: HashMap>>, - pub qvertex_to_type_map: HashMap, + pub qvertex_to_type_map: HashMap, pub qvertex_to_deg_map: HashMap>, pub q_edges: Vec, pub it: Option>, @@ -65,14 +67,14 @@ impl QueryGraph { set } - pub fn get_query_vertex_type(&self, query_vertex: &str) -> usize { + pub fn get_query_vertex_type(&self, query_vertex: &str) -> i32 { if let Some(vertex_type) = self.qvertex_to_type_map.get(query_vertex) { return vertex_type.clone(); } 0 } - pub fn set_query_vertex_type(&mut self, query_vertex: String, to_type: usize) { + pub fn set_query_vertex_type(&mut self, query_vertex: String, to_type: i32) { self.qvertex_to_type_map .insert(query_vertex.clone(), to_type); for edge in self.q_edges.iter_mut() { @@ -179,15 +181,15 @@ impl QueryGraph { let to_type = query_edge.to_type; self.qvertex_to_type_map .entry(from_qvertex.clone()) - .or_insert(0); + .or_insert(KEY_ANY); self.qvertex_to_type_map .entry(to_qvertex.clone()) - .or_insert(0); - if 0 != from_type { + .or_insert(KEY_ANY); + if KEY_ANY != from_type { self.qvertex_to_type_map .insert(from_qvertex.clone(), from_type); } - if 0 != to_type { + if KEY_ANY != to_type { self.qvertex_to_type_map.insert(to_qvertex.clone(), to_type); } // Set the in and out degrees for each variable. diff --git a/src/graph_impl/multi_graph/planner/catalog/subgraph_mapping_iterator.rs b/src/graph_impl/multi_graph/planner/catalog/subgraph_mapping_iterator.rs index 21a86d8c..095b8a0c 100644 --- a/src/graph_impl/multi_graph/planner/catalog/subgraph_mapping_iterator.rs +++ b/src/graph_impl/multi_graph/planner/catalog/subgraph_mapping_iterator.rs @@ -1,3 +1,4 @@ +use graph_impl::multi_graph::planner::catalog::catalog::LOGGER_FLAG; use graph_impl::multi_graph::planner::catalog::query_edge::QueryEdge; use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use hashbrown::HashMap; @@ -16,7 +17,7 @@ pub struct SubgraphMappingIterator { pub vertices_for_idx: Vec>, pub qvertex_to_qedges_map: HashMap>>, - pub qvertex_to_type_map: HashMap, + pub qvertex_to_type_map: HashMap, pub qvertex_to_deg_map: HashMap>, } @@ -26,7 +27,7 @@ impl SubgraphMappingIterator { query_vertices.iter().for_each(|v| { next.insert(v.clone(), String::from("")); }); - Self { + SubgraphMappingIterator { query_vertices, o_qvertices: vec![], o_qgraph: QueryGraph::empty(), @@ -59,11 +60,17 @@ impl SubgraphMappingIterator { } let o_qvertex = &self.o_qvertices[i]; let o_qvertex_deg = &o_query_graph.qvertex_to_deg_map[o_qvertex]; + let o_qvertex_type = o_query_graph.qvertex_to_type_map[o_qvertex]; + for j in 0..self.query_vertices.len() { let q_vertex = &self.query_vertices[j]; let vertex_type = self.qvertex_to_type_map[q_vertex]; let q_vertex_deg = &self.qvertex_to_deg_map[q_vertex]; - if o_query_graph.qvertex_to_type_map[o_qvertex] == vertex_type + // if unsafe { LOGGER_FLAG } { + // println!("q_vertex_type={},q_vertex_deg={:?},q_vertex={},query_vertices={:?}", vertex_type, q_vertex_deg, self.query_vertices.len(),self.query_vertices); + // println!("o_vertex_type={},o_qvertex_deg={:?},o_qvertices={},o_qvertices={:?}", o_qvertex_type, o_qvertex_deg, self.o_qvertices.len(),self.o_qvertices); + // } + if o_qvertex_type == vertex_type && (o_qvertex_deg.eq(q_vertex_deg) || (self.o_qvertices.len() < self.query_vertices.len() && q_vertex_deg[0] >= o_qvertex_deg[0] @@ -83,10 +90,9 @@ impl SubgraphMappingIterator { pub fn has_next(&mut self) -> bool { if !self.is_next_computed { -// println!( -// "PRE: curr_mapping={:?},self.o_qvertices={:?}", -// self.curr_mapping, self.o_qvertices -// ); + // if unsafe{LOGGER_FLAG}{ + // println!("PRE: curr_mapping={:?},self.o_qvertices={:?},vertices_for_idx={:?}",self.curr_mapping,self.o_qvertices,self.vertices_for_idx); + // } if self.curr_mapping.len() == self.o_qvertices.len() { self.curr_mapping.pop(); } @@ -97,14 +103,13 @@ impl SubgraphMappingIterator { .push(self.vertices_for_idx[0][self.vertex_indices[0]].clone()); self.vertex_indices[0] += 1; } else if self.vertex_indices[next_idx] < self.vertices_for_idx[next_idx].len() { - let vertices = &self.vertices_for_idx[next_idx]; - let new_var = &vertices[self.vertex_indices[next_idx]]; + let new_var = &self.vertices_for_idx[next_idx][self.vertex_indices[next_idx]]; self.vertex_indices[next_idx] += 1; let other_for_new = &self.o_qvertices[next_idx]; let mut outer_flag = false; for i in 0..self.curr_mapping.len() { let prev_var = &self.curr_mapping[i]; - if prev_var.eq(new_var) { + if prev_var == new_var { outer_flag = true; break; } @@ -130,10 +135,10 @@ impl SubgraphMappingIterator { if q_edge.label != o_qedge.label { continue; } - if !q_edge.from_query_vertex.eq(prev_var) - && o_qedge.from_query_vertex.eq(other_for_prev) - || (q_edge.from_query_vertex.eq(new_var) - && o_qedge.from_query_vertex.eq(other_for_new)) + if !((&q_edge.from_query_vertex == prev_var + && &o_qedge.from_query_vertex == other_for_prev) + || (&q_edge.from_query_vertex == new_var + && &o_qedge.from_query_vertex == other_for_new)) { outer_flag = true; break; @@ -148,42 +153,37 @@ impl SubgraphMappingIterator { self.vertex_indices[next_idx] = 0; } if self.curr_mapping.len() == self.o_qvertices.len() - || self.vertex_indices[0] < self.vertices_for_idx[0].len() - || self.curr_mapping.is_empty() + || (self.vertex_indices[0] >= self.vertices_for_idx[0].len() + && self.curr_mapping.is_empty()) { break; } } self.is_next_computed = true; } - if self.curr_mapping.is_empty() { - return false; - } - let mut same_edge_labels = true; - for i in 0..self.curr_mapping.len() { - for j in (i + 1)..self.curr_mapping.len() { - let q_vertex = &self.curr_mapping[i]; - let o_qvertex = &self.curr_mapping[j]; - if !self.contains_query_edge(q_vertex, o_qvertex) { - continue; - } - let q_edge = &self.qvertex_to_qedges_map[q_vertex][o_qvertex][0]; - let o_qedge = &self.o_qgraph.qvertex_to_qedges_map[&self.o_qvertices[i]] - [&self.o_qvertices[j]][0]; - if q_edge.label != o_qedge.label { - same_edge_labels = false; - break; + if !self.curr_mapping.is_empty() { + for i in 0..self.curr_mapping.len() { + for j in (i + 1)..self.curr_mapping.len() { + let q_vertex = &self.curr_mapping[i]; + let o_qvertex = &self.curr_mapping[j]; + if !self.contains_query_edge(q_vertex, o_qvertex) { + continue; + } + let q_edge = &self.qvertex_to_qedges_map[q_vertex][o_qvertex][0]; + let o_qedge = &self.o_qgraph.qvertex_to_qedges_map[&self.o_qvertices[i]] + [&self.o_qvertices[j]][0]; + if q_edge.label == o_qedge.label { + continue; + } + self.is_next_computed = false; + return self.has_next(); } } } - if !same_edge_labels { - self.is_next_computed = false; - return self.has_next(); - } -// println!( -// "LATER:cur_mapping={:?},self.o_qvertices={:?}", -// self.curr_mapping, self.o_qvertices -// ); + + // if unsafe{LOGGER_FLAG}{ + // println!("LAT: curr_mapping={:?},self.o_qvertices={:?},vertices_for_idx={:?}",self.curr_mapping,self.o_qvertices,self.vertices_for_idx); + // } !self.curr_mapping.is_empty() } @@ -195,15 +195,13 @@ impl SubgraphMappingIterator { self.next.clear(); for i in 0..self.o_qvertices.len() { self.next - .insert(self.curr_mapping[i].clone(), self.query_vertices[i].clone()); + .insert(self.curr_mapping[i].clone(), self.o_qvertices[i].clone()); } return Some(self.next.clone()); } pub fn contains_query_edge(&self, v1: &String, v2: &String) -> bool { - if let Some(map) = self.qvertex_to_qedges_map.get(v1) { - return map.contains_key(v2); - } - false + self.qvertex_to_qedges_map.contains_key(v1) + && self.qvertex_to_qedges_map[v1].contains_key(v2) } } diff --git a/src/graph_impl/multi_graph/planner/query_planner.rs b/src/graph_impl/multi_graph/planner/query_planner.rs index 9895093b..02a1185c 100644 --- a/src/graph_impl/multi_graph/planner/query_planner.rs +++ b/src/graph_impl/multi_graph/planner/query_planner.rs @@ -16,7 +16,7 @@ use graph_impl::multi_graph::planner::catalog::adj_list_descriptor::{ AdjListDescriptor, Direction, }; use graph_impl::multi_graph::planner::catalog::catalog::{ - Catalog, MULTI_VERTEX_WEIGHT_BUILD_COEF, MULTI_VERTEX_WEIGHT_PROBE_COEF, + Catalog, LOGGER_FLAG, MULTI_VERTEX_WEIGHT_BUILD_COEF, MULTI_VERTEX_WEIGHT_PROBE_COEF, SINGLE_VERTEX_WEIGHT_BUILD_COEF, SINGLE_VERTEX_WEIGHT_PROBE_COEF, }; use graph_impl::multi_graph::planner::catalog::query_edge::QueryEdge; @@ -118,7 +118,7 @@ impl let query_plan = QueryPlan::new_from_last_op(scan, num_edges as f64); let mut query_plans = vec![]; query_plans.push(query_plan); - let key = self.get_key(&mut vec![ + let key = self.get_key(vec![ query_edge.from_query_vertex.clone(), query_edge.to_query_vertex.clone(), ]); @@ -152,58 +152,50 @@ impl } fn consider_all_next_extend_operators(&mut self, key: &String) { - let prev_plan_map = &self.subgraph_plans[&(self.next_num_qvertices - 1)]; - let prev_query_plans = &prev_plan_map[key]; - let op = prev_query_plans[0].last_operator.as_ref().unwrap(); - let prev_qvertices = - get_op_attr_as_ref!(op.borrow().deref(), out_subgraph).get_query_vertices(); + let prev_query_plans = &self.subgraph_plans[&(self.next_num_qvertices - 1)][key]; + let prev_qvertices = { + let op = prev_query_plans[0].last_operator.as_ref().unwrap(); + get_op_attr_as_ref!(op.borrow().deref(), out_subgraph).get_query_vertices() + }; let to_qvertices = self.query_graph.get_neighbors(prev_qvertices); - let prev_query_plans_len = prev_plan_map[key].len(); - let mut plans = vec![]; + let prev_query_plans_len = prev_query_plans.len(); for to_qvertex in to_qvertices { for i in 0..prev_query_plans_len { - plans.push(self.get_plan_with_next_extend_by_index(i, key, &to_qvertex)); + let prev_query_plan = + self.subgraph_plans[&(self.next_num_qvertices - 1)][key][i].clone(); + let (key, plan) = self.get_plan_with_next_extend(prev_query_plan, &to_qvertex); + println!( + "extend_key={}, icost={}, estimated_icost={}", + key, plan.icost, plan.estimated_icost + ); + let plan_map = self + .subgraph_plans + .get_mut(&self.next_num_qvertices) + .unwrap(); + plan_map.entry(key.clone()).or_insert(vec![]); + plan_map.get_mut(&key).unwrap().push(plan); } } - let plan_map = self - .subgraph_plans - .get_mut(&self.next_num_qvertices) - .unwrap(); - for (key, plan) in plans { - plan_map.entry(key.clone()).or_insert(vec![]); - plan_map.get_mut(&key).unwrap().push(plan); - } - } - - pub fn get_plan_with_next_extend_by_index( - &mut self, - prev_query_plan_index: usize, - key: &String, - to_qvertex: &String, - ) -> (String, QueryPlan) { - let prev_plan_map = &self.subgraph_plans[&(self.next_num_qvertices - 1)]; - let prev_query_plans = &prev_plan_map[key]; - let prev_query_plan = &prev_query_plans[prev_query_plan_index]; - self.get_plan_with_next_extend(prev_query_plan.clone(), to_qvertex) + println!("-------"); } pub fn get_plan_with_next_extend( &mut self, - mut prev_query_plan: QueryPlan, + prev_query_plan: QueryPlan, to_qvertex: &String, ) -> (String, QueryPlan) { let last_operator = prev_query_plan.last_operator.as_ref().unwrap(); - let (in_subgraph, last_previous_repeated_index) = { + let (mut in_subgraph, last_previous_repeated_index, last_prev) = { let last_op_ref = last_operator.borrow(); let base_op = get_base_op_as_ref!(last_op_ref.deref()); ( base_op.out_subgraph.clone(), base_op.last_repeated_vertex_idx, + base_op.prev.as_ref().map(|op| op.clone()), ) }; let mut alds = vec![]; let mut next_extend = self.get_next_ei(&in_subgraph, to_qvertex, &mut alds, last_operator); - let next_copy = next_extend.clone(); let base_next_extend = get_ei_as_mut!(&mut next_extend); base_next_extend.init_caching(last_previous_repeated_index); let prev_estimated_num_out_tuples = prev_query_plan.estimated_num_out_tuples; @@ -211,29 +203,35 @@ impl .base_op .out_subgraph .get_query_vertex_type(to_qvertex); - let (mut in_subgraph, last_prev) = { - let last_operator = prev_query_plan.last_operator.as_mut().unwrap(); - let last_ref = last_operator.borrow(); - let base_op = get_base_op_as_ref!(last_ref.deref()); - ( - base_op.out_subgraph.clone(), - base_op.prev.as_ref().map(|op| op.clone()), - ) - }; let estimated_selectivity = self.get_selectivity( &mut in_subgraph, &mut base_next_extend.base_op.out_subgraph, &alds, to_type, ); + println!("estimated_selectivity={}", estimated_selectivity); let icost; if let CachingType::None = base_next_extend.caching_type { + // for (from, edges) in &in_subgraph.qvertex_to_qedges_map { + // for (to, edge) in edges { + // print!("{}->{},", from, to); + // } + // } + // println!(); + // println!("alds.len={},to_type={}", alds.len(), base_next_extend.to_type); + // alds.iter().for_each(|ald|{ + // println!("from={},id={},dir={},label={}",ald.from_query_vertex,ald.vertex_idx,ald.direction,ald.label); + // }); icost = prev_estimated_num_out_tuples * self.catalog.get_icost( &mut in_subgraph, alds.iter().collect(), base_next_extend.to_type, ); + println!( + "prev_estimated_num_out_tuples={},icost={}", + prev_estimated_num_out_tuples, icost + ); } else { let mut out_tuples_to_process = prev_estimated_num_out_tuples; if last_prev.is_some() { @@ -291,20 +289,17 @@ impl let mut new_query_plan = prev_query_plan.shallow_copy(); new_query_plan.estimated_icost = estimated_icost; + println!("new:estimated_num_out_tuples={}", estimated_num_out_tuples); new_query_plan.estimated_num_out_tuples = estimated_num_out_tuples; - new_query_plan.append(Rc::new(RefCell::new(Operator::EI(next_copy)))); + let query_vertices = base_next_extend + .base_op + .out_qvertex_to_idx_map + .keys() + .map(|k| k.clone()) + .collect(); + new_query_plan.append(Rc::new(RefCell::new(Operator::EI(next_extend)))); new_query_plan.q_vertex_to_num_out_tuples = q_vertex_to_num_out_tuples; - ( - self.get_key( - &mut (base_next_extend - .base_op - .out_qvertex_to_idx_map - .keys() - .map(|k| k.clone()) - .collect()), - ), - new_query_plan, - ) + (self.get_key(query_vertices), new_query_plan) } fn get_next_ei( @@ -367,7 +362,7 @@ impl in_subgraph: &mut QueryGraph, out_subgraph: &mut QueryGraph, alds: &Vec, - to_type: usize, + to_type: i32, ) -> f64 { let selectivity; let computed_selectivity_op = self @@ -383,12 +378,20 @@ impl self.computed_selectivities .insert(out_subgraph.get_encoding(), vec![]); } + // for (from, edges) in &in_subgraph.qvertex_to_qedges_map { + // for (to, edge) in edges { + // print!("{}->{},",from,to); + // } + // } + // println!(); + unsafe { LOGGER_FLAG = true }; selectivity = self.catalog.get_selectivity(in_subgraph, alds, to_type); + unsafe { LOGGER_FLAG = false }; + println!("compute selectivity"); self.computed_selectivities .get_mut(&out_subgraph.get_encoding()) - .map(|selectivities| { - selectivities.push((out_subgraph.clone(), selectivity)); - }); + .unwrap() + .push((out_subgraph.clone(), selectivity)); selectivity } @@ -441,7 +444,7 @@ impl other_set.push(v.clone()); }); let rest_size = other_set.len(); - let rest_key = self.get_key(&mut other_set); + let rest_key = self.get_key(other_set); if !self.subgraph_plans[&rest_size].contains_key(&rest_key) { return; } @@ -477,7 +480,7 @@ impl fn consider_hash_join_operator( &mut self, out_subgraph: &QueryGraph, - mut query_vertices: Vec, + query_vertices: Vec, subplan: &QueryPlan, other_subplan: &QueryPlan, num_join_qvertices: usize, @@ -509,7 +512,7 @@ impl + build_coef * build_subplan.estimated_num_out_tuples + probe_coef * probe_subplan.estimated_num_out_tuples; - let key = self.get_key(&mut query_vertices); + let key = self.get_key(query_vertices.clone()); let curr_best_query_plan = self.get_best_plan(query_vertices.len(), &key); if curr_best_query_plan.estimated_icost > icost { let mut query_plan = HashJoin::make( @@ -572,7 +575,7 @@ impl self.graph.get_num_edges(from_type, to_type, label) } - fn get_key(&self, query_vertices: &mut Vec) -> String { + fn get_key(&self, mut query_vertices: Vec) -> String { query_vertices.sort(); serde_json::to_string(&query_vertices).unwrap() } diff --git a/src/graph_impl/multi_graph/query/query_graph_set.rs b/src/graph_impl/multi_graph/query/query_graph_set.rs index 3c10f5f4..6ac6769a 100644 --- a/src/graph_impl/multi_graph/query/query_graph_set.rs +++ b/src/graph_impl/multi_graph/query/query_graph_set.rs @@ -14,10 +14,11 @@ impl QueryGraphSet { pub fn add(&mut self, mut query_graph: QueryGraph) { let encoding = query_graph.get_encoding(); - let graph = self.query_graphs.get_mut(&encoding); - if graph.is_none() { - self.query_graphs.insert(encoding, vec![query_graph]); - } + self.query_graphs.entry(encoding.clone()).or_insert(vec![]); + self.query_graphs + .get_mut(&encoding) + .unwrap() + .push(query_graph); } pub fn contains(&mut self, query_graph: &mut QueryGraph) -> bool { diff --git a/src/graph_impl/multi_graph/runner/catalog_generator.rs b/src/graph_impl/multi_graph/runner/catalog_generator.rs index 1d39001d..4e99ed29 100644 --- a/src/graph_impl/multi_graph/runner/catalog_generator.rs +++ b/src/graph_impl/multi_graph/runner/catalog_generator.rs @@ -15,6 +15,9 @@ pub fn default = TypedStaticGraph; pub type TypedDiStaticGraph = TypedStaticGraph; @@ -50,6 +50,8 @@ pub type DiStaticGraph = StaticGraph { @@ -71,7 +73,7 @@ pub struct TypedStaticGraph, // node_types[node_id] = node_label_id // `node_label_id` has been shifted right and id 0 is prepared for no label item. - node_types: Vec, + node_types: Vec, node_type_offsets: Vec, fwd_adj_lists: Vec>>, bwd_adj_lists: Vec>>, @@ -397,16 +399,18 @@ impl pub fn init_graphflow(&mut self) { self.partition_nodes(); self.partition_edges(); - let label_cnt = if self.sort_by_node { - self.num_of_node_labels() + 1 + let mut label_cnt = if self.sort_by_node { + self.num_of_node_labels() } else { - self.num_of_edge_labels() + 1 + self.num_of_edge_labels() }; + if label_cnt == 0 { + label_cnt = 1; + } self.label_to_num_edges = vec![0; label_cnt]; self.label_to_largest_fwd_adj_list_size = vec![0; label_cnt]; self.label_to_largest_bwd_adj_list_size = vec![0; label_cnt]; for vertex_id in 0..self.num_nodes { - self.num_edges += self.fwd_adj_lists[vertex_id].as_ref().unwrap().len(); for label in 0..label_cnt { let fwd_adj_size = self.fwd_adj_lists[vertex_id] .as_ref() @@ -427,9 +431,11 @@ impl } // init count - for from in 0..=self.num_of_node_labels() { - for to in 0..=self.num_of_node_labels() { - for label in 0..=self.num_of_edge_labels() { + let node_labels_cnt = std::cmp::max(self.num_of_node_labels(), 1); + let edge_labels_cnt = std::cmp::max(self.num_of_edge_labels(), 1); + for from in 0..node_labels_cnt { + for to in 0..node_labels_cnt { + for label in 0..edge_labels_cnt { let edge = Self::get_edge_key(from, to, label); self.edge_key_to_num_edges_map.entry(edge).or_insert(0); let to_label = Self::get_edge_key_by_label(label, to); @@ -453,7 +459,7 @@ impl let label = 0; for to_type in 0..(offsets.len() - 1) { let num_edges = offsets[to_type + 1] - offsets[to_type]; - self.add_edge_count(from_type, to_type, label, num_edges); + self.add_edge_count(from_type as usize, to_type, label, num_edges); } } else { let neighbours = self.fwd_adj_lists[from] @@ -464,7 +470,7 @@ impl for label in 0..(offsets.len() - 1) { for to_idx in offsets[label]..offsets[label + 1] { let to_type = self.node_types[neighbours[to_idx].id()]; - self.add_edge_count(from_type, to_type, label, 1); + self.add_edge_count(from_type as usize, to_type as usize, label, 1); } } } @@ -576,16 +582,18 @@ impl self.edge_vec.find_edge_index(start, target) } - pub fn get_num_edges(&self, from_type: usize, to_type: usize, label: usize) -> usize { - if from_type == 0 && to_type == 0 { - return self.label_to_num_edges[label]; - } else if from_type != 0 && to_type != 0 { - return self.edge_key_to_num_edges_map[&Self::get_edge_key(from_type, to_type, label)]; - } else if from_type != 0 { + pub fn get_num_edges(&self, from_type: i32, to_type: i32, label: i32) -> usize { + if from_type == KEY_ANY && to_type == KEY_ANY { + return self.label_to_num_edges[label as usize]; + } else if from_type != KEY_ANY && to_type != KEY_ANY { + return self.edge_key_to_num_edges_map + [&Self::get_edge_key(from_type as usize, to_type as usize, label as usize)]; + } else if from_type != KEY_ANY { return self.from_label_to_percentage_map - [&Self::get_edge_key_by_label(from_type, label)]; + [&Self::get_edge_key_by_label(from_type as usize, label as usize)]; } - self.to_label_to_percentage_map[&Self::get_edge_key_by_label(label, to_type)] + self.to_label_to_percentage_map + [&Self::get_edge_key_by_label(label as usize, to_type as usize)] } pub fn get_edge_key(from_type: usize, to_type: usize, label: usize) -> u64 { @@ -621,10 +629,11 @@ impl let node_label_id = self .get_node(id) .get_label_id() - .map(|op| op.id() + 1) - .unwrap_or(0); - node_ids[offsets[node_label_id] + curr_idx_by_type[node_label_id]] = id; - curr_idx_by_type[node_label_id] += 1; + .map(|op| op.id()) + .unwrap_or(0) as i32; + node_ids[offsets[node_label_id as usize] + curr_idx_by_type[node_label_id as usize]] = + id; + curr_idx_by_type[node_label_id as usize] += 1; node_types[node_id] = node_label_id; }); self.node_ids = node_ids; @@ -634,7 +643,6 @@ impl // Partition edges by edge label or node label(if there did not exist edge labels in graph) fn partition_edges(&mut self) { - // if only one label used + al least 2 vertex type used, then sorted by node self.sort_by_node = self.num_of_edge_labels() == 1 && self.num_of_node_labels() > 1; let (fwd_adj_meta_data, bwd_adj_meta_data) = self.get_adj_meta_data(); let num_vertices = self.num_nodes; @@ -670,23 +678,23 @@ impl let edge_label_id = self .get_edge(from, to) .get_label_id() - .map(|op| op.id() + 1) + .map(|op| op.id()) .unwrap_or(0); let (from_label_id, to_label_id) = if self.sort_by_node { (self.node_types[from.id()], self.node_types[to.id()]) } else { - (edge_label_id, edge_label_id) + (edge_label_id as i32, edge_label_id as i32) }; - let mut idx = fwd_adj_list_curr_idx[&from.id()][to_label_id]; - let mut offset = fwd_adj_meta_data[&from.id()][to_label_id]; - fwd_adj_list_curr_idx.get_mut(&from.id()).unwrap()[to_label_id] += 1; + let mut idx = fwd_adj_list_curr_idx[&from.id()][to_label_id as usize]; + let mut offset = fwd_adj_meta_data[&from.id()][to_label_id as usize]; + fwd_adj_list_curr_idx.get_mut(&from.id()).unwrap()[to_label_id as usize] += 1; fwd_adj_lists[from.id()] .as_mut() .unwrap() .set_neighbor_id(to, offset + idx); - idx = bwd_adj_list_curr_idx[&to.id()][from_label_id]; - offset = bwd_adj_meta_data[&to.id()][from_label_id]; - bwd_adj_list_curr_idx.get_mut(&to.id()).unwrap()[from_label_id] += 1; + idx = bwd_adj_list_curr_idx[&to.id()][from_label_id as usize]; + offset = bwd_adj_meta_data[&to.id()][from_label_id as usize]; + bwd_adj_list_curr_idx.get_mut(&to.id()).unwrap()[from_label_id as usize] += 1; bwd_adj_lists[to.id()] .as_mut() .unwrap() @@ -708,24 +716,26 @@ impl let label_id = self .get_node(x) .get_label_id() - .map(|op| op.id() + 1) + .map(|op| op.id()) .unwrap_or(0); let default_v = 0; let v = type_to_count_map.get(&label_id).unwrap_or(&default_v); type_to_count_map.insert(label_id, v + 1); }); - let next_node_label_key = self.num_of_node_labels(); - let mut offsets = vec![0; next_node_label_key + 3]; - type_to_count_map.iter().for_each(|(label_id, cnt)| { - let label_id = label_id.to_owned(); - let label_cnt = cnt.to_owned(); - - if label_id < next_node_label_key + 1 { - offsets[label_id + 1] = label_cnt; - } - offsets[next_node_label_key + 2] += label_cnt; - }); + let mut next_node_label_key = self.num_of_node_labels(); + if next_node_label_key == 0 { + next_node_label_key = 1; + } + let mut offsets = vec![0; next_node_label_key + 1]; + type_to_count_map + .into_iter() + .for_each(|(label_id, label_cnt)| { + if label_id < next_node_label_key - 1 { + offsets[label_id + 1] = label_cnt; + } + offsets[next_node_label_key] += label_cnt; + }); for i in 1..offsets.len() - 1 { offsets[i] += offsets[i - 1]; } @@ -735,14 +745,17 @@ impl fn get_adj_meta_data(&self) -> (HashMap>, HashMap>) { let mut fwd_adj_list_metadata = HashMap::new(); let mut bwd_adj_list_metadata = HashMap::new(); - let next_node_or_edge = if self.sort_by_node { + let mut next_node_or_edge = if self.sort_by_node { self.num_of_node_labels() } else { self.num_of_edge_labels() }; + if next_node_or_edge == 0 { + next_node_or_edge = 1; + } for i in 0..self.num_nodes { - fwd_adj_list_metadata.insert(i, vec![0; next_node_or_edge + 2]); - bwd_adj_list_metadata.insert(i, vec![0; next_node_or_edge + 2]); + fwd_adj_list_metadata.insert(i, vec![0; next_node_or_edge + 1]); + bwd_adj_list_metadata.insert(i, vec![0; next_node_or_edge + 1]); } self.edge_indices() .flat_map(|(from, to)| { @@ -755,13 +768,13 @@ impl if self.sort_by_node { let from_type = self.node_types[from.id()]; let to_type = self.node_types[to.id()]; - fwd_adj_list_metadata.get_mut(&from.id()).unwrap()[to_type + 1] += 1; - bwd_adj_list_metadata.get_mut(&to.id()).unwrap()[from_type + 1] += 1; + fwd_adj_list_metadata.get_mut(&from.id()).unwrap()[(to_type + 1) as usize] += 1; + bwd_adj_list_metadata.get_mut(&to.id()).unwrap()[(from_type + 1) as usize] += 1; } else { let label_id = self .get_edge(from, to) .get_label_id() - .map(|op| op.id() + 1) + .map(|op| op.id()) .unwrap_or(0); fwd_adj_list_metadata.get_mut(&from.id()).unwrap()[label_id + 1] += 1; bwd_adj_list_metadata.get_mut(&to.id()).unwrap()[label_id + 1] += 1; @@ -769,13 +782,13 @@ impl }); fwd_adj_list_metadata.iter_mut().for_each(|(_id, offsets)| { for i in 1..offsets.len() - 1 { - offsets[next_node_or_edge + 1] += offsets[i]; + offsets[next_node_or_edge] += offsets[i]; offsets[i] += offsets[i - 1]; } }); bwd_adj_list_metadata.iter_mut().for_each(|(_id, offsets)| { for i in 1..offsets.len() - 1 { - offsets[next_node_or_edge + 1] += offsets[i]; + offsets[next_node_or_edge] += offsets[i]; offsets[i] += offsets[i - 1]; } }); @@ -787,10 +800,7 @@ impl if let Some(label) = label { if let Some(fwd_list) = &self.fwd_adj_lists[id.id()] { let offset = fwd_list.get_offsets(); - let label_id = self - .node_label_map - .find_index(&label) - .map_or(0, |id| id + 1); + let label_id = self.node_label_map.find_index(&label).map_or(0, |id| id); return &fwd_list.get_neighbor_ids()[offset[label_id]..offset[label_id + 1]]; } @@ -802,10 +812,7 @@ impl if let Some(label) = label { if let Some(fwd_list) = &self.fwd_adj_lists[id.id()] { let offset = fwd_list.get_offsets(); - let label_id = self - .edge_label_map - .find_index(&label) - .map_or(0, |id| id + 1); + let label_id = self.edge_label_map.find_index(&label).map_or(0, |id| id); return &fwd_list.get_neighbor_ids()[offset[label_id]..offset[label_id + 1]]; } } @@ -816,7 +823,7 @@ impl &self.node_ids } - pub fn get_node_types(&self) -> &Vec { + pub fn get_node_types(&self) -> &Vec { self.node_types.as_ref() } @@ -834,13 +841,13 @@ impl pub fn get_largest_adj_list_size( &self, - node_or_edge_label: usize, + node_or_edge_label: i32, direction: Direction, ) -> usize { if let Direction::Fwd = direction { - return self.label_to_largest_fwd_adj_list_size[node_or_edge_label]; + return self.label_to_largest_fwd_adj_list_size[node_or_edge_label as usize]; } - self.label_to_largest_bwd_adj_list_size[node_or_edge_label] + self.label_to_largest_bwd_adj_list_size[node_or_edge_label as usize] } } @@ -1035,10 +1042,11 @@ impl fn nodes_with_label(&self, label: Option) -> Iter { if let Some(label) = label { - let label_id = self - .node_label_map - .find_index(&label) - .map_or(0, |id| id + 1); + let label_id = self.node_label_map.find_index(&label); + if label_id.is_none() { + return Iter::new(Box::new(iter::empty::())); + } + let label_id = label_id.unwrap(); return Iter::new(Box::new( self.fwd_adj_lists .iter() @@ -1059,10 +1067,11 @@ impl fn edges_with_label(&self, label: Option) -> Iter<(Id, Id)> { if let Some(label) = label { - let label_id = self - .edge_label_map - .find_index(&label) - .map_or(0, |id| id + 1); + let label_id = self.edge_label_map.find_index(&label); + if label_id.is_none() { + return Iter::new(Box::new(iter::empty::<(Id, Id)>())); + } + let label_id = label_id.unwrap(); return Iter::new(Box::new( self.fwd_adj_lists .iter() diff --git a/src/graph_impl/static_graph/sorted_adj_vec.rs b/src/graph_impl/static_graph/sorted_adj_vec.rs index ff99c208..0a6d6df5 100644 --- a/src/graph_impl/static_graph/sorted_adj_vec.rs +++ b/src/graph_impl/static_graph/sorted_adj_vec.rs @@ -20,6 +20,7 @@ */ use generic::IdType; use graph_impl::multi_graph::plan::operator::extend::EI::Neighbours; +use itertools::Itertools; #[derive(Debug, Hash, PartialEq, Eq, Clone, Serialize, Deserialize)] pub struct SortedAdjVec { @@ -44,10 +45,10 @@ impl SortedAdjVec { self.neighbour_ids[idx] = neighbor_id } - pub fn set_neighbor_ids(&self, label_or_type: usize, neighbours: &mut Neighbours) { + pub fn set_neighbor_ids(&self, label_or_type: i32, neighbours: &mut Neighbours) { neighbours.ids = self.neighbour_ids.clone(); - neighbours.start_idx = self.label_offset[label_or_type]; - neighbours.end_idx = self.label_offset[label_or_type + 1]; + neighbours.start_idx = self.label_offset[label_or_type as usize]; + neighbours.end_idx = self.label_offset[(label_or_type + 1) as usize]; } pub fn get_offsets(&self) -> &Vec { @@ -67,7 +68,7 @@ impl SortedAdjVec { pub fn intersect( &self, - label_or_type: usize, + label_or_type: i32, some_neighbours: &mut Neighbours, neighbours: &mut Neighbours, ) -> usize { @@ -75,10 +76,10 @@ impl SortedAdjVec { some_neighbours, neighbours, &self.neighbour_ids, - self.label_offset[label_or_type], - self.label_offset[label_or_type + 1], + self.label_offset[label_or_type as usize], + self.label_offset[(label_or_type + 1) as usize], ); - self.label_offset[label_or_type + 1] - self.label_offset[label_or_type] + self.label_offset[(label_or_type + 1) as usize] - self.label_offset[label_or_type as usize] } fn inner_intersect( @@ -93,6 +94,8 @@ impl SortedAdjVec { let some_neighbour_ids = &some_neighbours.ids; let mut some_idx = some_neighbours.start_idx; let some_idx_end = some_neighbours.end_idx; + println!("neighbour_ids={:?}",neighbour_ids.iter().map(|x|x.id()).collect_vec()); + println!("some_neighbours={:?}",some_neighbours.ids.iter().map(|x|x.id()).collect_vec()); while this_idx < this_idx_end && some_idx < some_idx_end { if neighbour_ids[this_idx] < some_neighbour_ids[some_idx] { this_idx += 1; diff --git a/tests/static_graph.rs b/tests/static_graph.rs index 6f2cdab9..f69a6f79 100644 --- a/tests/static_graph.rs +++ b/tests/static_graph.rs @@ -28,6 +28,7 @@ use hashbrown::HashMap; use itertools::Itertools; use rust_graph::generic::DefaultId; use rust_graph::graph_impl::multi_graph::plan::query_plan_worker::QPWorkers; +use rust_graph::graph_impl::multi_graph::planner::catalog::catalog::LOGGER_FLAG; use rust_graph::graph_impl::multi_graph::planner::catalog::query_edge::QueryEdge; use rust_graph::graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use rust_graph::graph_impl::multi_graph::runner::{catalog_generator, optimizer_executor}; @@ -51,23 +52,23 @@ fn test_directed() { assert_eq!(g.get_node_type_offsets(), &vec![0, 4]); let fwd_adj_list = g.get_fwd_adj_list()[0].as_ref().unwrap(); - assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 2, 2, 2]); + assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 2]); assert_eq!(fwd_adj_list.get_neighbor_ids(), &vec![1, 2]); let fwd_adj_list = g.get_fwd_adj_list()[1].as_ref().unwrap(); - assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 1, 1, 1]); + assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 1]); assert_eq!(fwd_adj_list.get_neighbor_ids(), &vec![0]); let fwd_adj_list = g.get_fwd_adj_list()[2].as_ref().unwrap(); - assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 1, 1, 1]); + assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 1]); assert_eq!(fwd_adj_list.get_neighbor_ids(), &vec![0]); let bwd_adj_list = g.get_bwd_adj_list()[0].as_ref().unwrap(); - assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 2, 2, 2]); + assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 2]); assert_eq!(bwd_adj_list.get_neighbor_ids(), &vec![1, 2]); let bwd_adj_list = g.get_bwd_adj_list()[1].as_ref().unwrap(); - assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 1, 1, 1]); + assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 1]); assert_eq!(bwd_adj_list.get_neighbor_ids(), &vec![0]); let bwd_adj_list = g.get_bwd_adj_list()[2].as_ref().unwrap(); - assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 1, 1, 1]); + assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 1]); assert_eq!(bwd_adj_list.get_neighbor_ids(), &vec![0]); assert_eq!(g.node_count(), 3); @@ -137,23 +138,23 @@ fn test_undirected() { assert_eq!(g.get_node_type_offsets(), &vec![0, 4]); let fwd_adj_list = g.get_fwd_adj_list()[0].as_ref().unwrap(); - assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 2, 2, 2]); + assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 2]); assert_eq!(fwd_adj_list.get_neighbor_ids(), &vec![1, 2]); let fwd_adj_list = g.get_fwd_adj_list()[1].as_ref().unwrap(); - assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 2, 2, 2]); + assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 2]); assert_eq!(fwd_adj_list.get_neighbor_ids(), &vec![0, 2]); let fwd_adj_list = g.get_fwd_adj_list()[2].as_ref().unwrap(); - assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 2, 2, 2]); + assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 2]); assert_eq!(fwd_adj_list.get_neighbor_ids(), &vec![0, 1]); let bwd_adj_list = g.get_bwd_adj_list()[0].as_ref().unwrap(); - assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 2, 2, 2]); + assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 2]); assert_eq!(bwd_adj_list.get_neighbor_ids(), &vec![1, 2]); let bwd_adj_list = g.get_bwd_adj_list()[1].as_ref().unwrap(); - assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 2, 2, 2]); + assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 2]); assert_eq!(bwd_adj_list.get_neighbor_ids(), &vec![0, 2]); let bwd_adj_list = g.get_bwd_adj_list()[2].as_ref().unwrap(); - assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 2, 2, 2]); + assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 2]); assert_eq!(bwd_adj_list.get_neighbor_ids(), &vec![0, 1]); } @@ -217,27 +218,27 @@ fn test_labeled() { assert!(edges.contains(&g.get_edge(2, 0))); assert_eq!(g.get_node_ids(), &vec![1, 0, 2]); - assert_eq!(g.get_node_types(), &vec![2, 1, 2]); - assert_eq!(g.get_node_type_offsets(), &vec![0, 0, 1, 3, 3]); + assert_eq!(g.get_node_types(), &vec![1, 0, 1]); + assert_eq!(g.get_node_type_offsets(), &vec![0, 1, 3]); let fwd_adj_list = g.get_fwd_adj_list()[0].as_ref().unwrap(); - assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 0, 1, 2]); + assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 1, 2]); assert_eq!(fwd_adj_list.get_neighbor_ids(), &vec![1, 2]); let fwd_adj_list = g.get_fwd_adj_list()[1].as_ref().unwrap(); - assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 0, 1, 1]); + assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 1, 1]); assert_eq!(fwd_adj_list.get_neighbor_ids(), &vec![0]); let fwd_adj_list = g.get_fwd_adj_list()[2].as_ref().unwrap(); - assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 0, 0, 1]); + assert_eq!(fwd_adj_list.get_offsets(), &vec![0, 0, 1]); assert_eq!(fwd_adj_list.get_neighbor_ids(), &vec![0]); let bwd_adj_list = g.get_bwd_adj_list()[0].as_ref().unwrap(); - assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 0, 1, 2]); + assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 1, 2]); assert_eq!(bwd_adj_list.get_neighbor_ids(), &vec![1, 2]); let bwd_adj_list = g.get_bwd_adj_list()[1].as_ref().unwrap(); - assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 0, 1, 1]); + assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 1, 1]); assert_eq!(bwd_adj_list.get_neighbor_ids(), &vec![0]); let bwd_adj_list = g.get_bwd_adj_list()[2].as_ref().unwrap(); - assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 0, 0, 1]); + assert_eq!(bwd_adj_list.get_offsets(), &vec![0, 0, 1]); assert_eq!(bwd_adj_list.get_neighbor_ids(), &vec![0]); let neighbour_edge_no: Vec = g.neighbors_of_edge_iter(0, None).collect(); @@ -340,7 +341,6 @@ fn test_graphflow_planner() { println!("num_of_node_labels={}", g.num_of_node_labels()); println!("num_of_edge_labels={}", g.num_of_edge_labels()); println!("load finished."); - let catalog = catalog_generator::default(&g); let mut qvertex_to_qedges_map = HashMap::new(); let mut qvertex_to_type_map = HashMap::new(); let mut qvertex_to_deg_map = HashMap::new(); @@ -433,11 +433,13 @@ fn test_graphflow_planner() { encoding: None, limit: 0, }; + let catalog = catalog_generator::default(&g); + unsafe { LOGGER_FLAG = true }; let mut query_plan = optimizer_executor::generate_plan(query_graph, catalog, g.clone()); - println!("QueryPlan output:{}", query_plan.get_output_log()); + println!("QueryPlan:{}", query_plan.get_output_log()); let mut workers = QPWorkers::new(query_plan, 1); workers.init(&g); workers.execute(); - println!("QueryPlan output:{}", workers.get_output_log()); + println!("Query result:{}", workers.get_output_log()); assert!(false); } From 1455950edd34d34613d114f231b6796fa29609c1 Mon Sep 17 00:00:00 2001 From: Michael Chen <920901835@qq.com> Date: Wed, 4 Mar 2020 20:41:05 +0800 Subject: [PATCH 22/25] fix:catalog --- .../multi_graph/plan/operator/extend/EI.rs | 12 ++++---- .../plan/operator/extend/Extend.rs | 6 ++-- .../plan/operator/extend/Intersect.rs | 8 +++--- .../plan/operator/extend/extend.rs | 6 ++-- .../plan/operator/extend/intersect.rs | 8 +++--- .../plan/operator/hashjoin/build.rs | 4 +-- .../plan/operator/hashjoin/hash_table.rs | 8 ++++-- .../plan/operator/hashjoin/probe.rs | 14 +++++----- .../plan/operator/hashjoin/probe_cartesian.rs | 12 ++++---- .../operator/hashjoin/probe_multi_vertices.rs | 10 +++---- .../probe_multi_vertices_cartesian.rs | 12 ++++---- .../multi_graph/plan/operator/operator.rs | 12 ++++---- .../multi_graph/plan/operator/scan/scan.rs | 10 +++---- .../plan/operator/scan/scan_blocking.rs | 12 ++++---- .../plan/operator/scan/scan_sampling.rs | 28 ++++++------------- .../multi_graph/plan/operator/sink/sink.rs | 4 +-- .../plan/operator/sink/sink_copy.rs | 4 +-- .../plan/operator/sink/sink_limit.rs | 2 +- .../plan/operator/sink/sink_print.rs | 2 +- src/graph_impl/multi_graph/plan/query_plan.rs | 2 +- .../multi_graph/planner/catalog/catalog.rs | 26 +++++++---------- .../planner/catalog/catalog_plans.rs | 1 - .../catalog/operator/intersect_catalog.rs | 21 +++++++------- .../planner/catalog/operator/noop.rs | 2 +- .../multi_graph/planner/query_planner.rs | 22 --------------- src/graph_impl/static_graph/sorted_adj_vec.rs | 2 -- 26 files changed, 104 insertions(+), 146 deletions(-) diff --git a/src/graph_impl/multi_graph/plan/operator/extend/EI.rs b/src/graph_impl/multi_graph/plan/operator/extend/EI.rs index ba996f4d..df05cf6e 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/EI.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/EI.rs @@ -135,11 +135,11 @@ impl BaseEI { self.is_intersection_cached = true; for i in 0..self.last_vertex_ids_intersected.as_ref().unwrap().len() { if self.last_vertex_ids_intersected.as_ref().unwrap()[i] - != self.base_op.probe_tuple[self.vertex_idx_to_cache[i] as usize] + != self.base_op.probe_tuple.borrow()[self.vertex_idx_to_cache[i] as usize] { self.is_intersection_cached = false; self.last_vertex_ids_intersected.as_mut().unwrap()[i] = - self.base_op.probe_tuple[self.vertex_idx_to_cache[i] as usize]; + self.base_op.probe_tuple.borrow()[self.vertex_idx_to_cache[i] as usize]; } } self.is_intersection_cached @@ -240,13 +240,13 @@ impl BaseEI { pub fn execute_intersect(&mut self, idx: usize, intersect_type: IntersectType) -> usize { let (adj_vec, label_or_type) = match intersect_type { IntersectType::CachedOut | IntersectType::TempOut => ( - self.adj_lists_to_cache[idx][self.base_op.probe_tuple[self.vertex_idx[idx]].id()] + self.adj_lists_to_cache[idx][self.base_op.probe_tuple.borrow()[self.vertex_idx[idx]].id()] .as_ref(), self.labels_or_to_types[idx], ), _ => ( self.adj_lists_to_cache[idx] - [self.base_op.probe_tuple[self.vertex_idx_to_cache[idx]].id()] + [self.base_op.probe_tuple.borrow()[self.vertex_idx_to_cache[idx]].id()] .as_ref(), self.labels_or_to_types_to_cache[idx], ), @@ -267,7 +267,7 @@ impl BaseEI { impl CommonOperatorTrait for BaseEI { fn init( &mut self, - probe_tuple: Vec, + probe_tuple: Rc>>, graph: &TypedStaticGraph, ) { self.base_op.probe_tuple = probe_tuple.clone(); @@ -384,7 +384,7 @@ impl EI { impl CommonOperatorTrait for EI { fn init( &mut self, - probe_tuple: Vec, + probe_tuple:Rc>>, graph: &TypedStaticGraph, ) { match self { diff --git a/src/graph_impl/multi_graph/plan/operator/extend/Extend.rs b/src/graph_impl/multi_graph/plan/operator/extend/Extend.rs index 87ecbf35..1e9dd88e 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/Extend.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/Extend.rs @@ -58,7 +58,7 @@ impl Extend { impl CommonOperatorTrait for Extend { fn init( &mut self, - probe_tuple: Vec, + probe_tuple: Rc>>, graph: &TypedStaticGraph, ) { self.base_ei.out_neighbours = Neighbours::new(); @@ -80,7 +80,7 @@ impl CommonOperatorTrait for Extend { } fn process_new_tuple(&mut self) { - let adj_vec = self.adj_list[self.base_ei.base_op.probe_tuple[self.vertex_index].id()] + let adj_vec = self.adj_list[self.base_ei.base_op.probe_tuple.borrow()[self.vertex_index].id()] .as_mut() .unwrap(); let out_neighbour = &mut self.base_ei.out_neighbours; @@ -91,7 +91,7 @@ impl CommonOperatorTrait for Extend { || self.base_ei.to_type == self.base_ei.vertex_types[out_neighbour.ids[idx].id()] { self.base_ei.base_op.num_out_tuples += 1; - self.base_ei.base_op.probe_tuple[self.base_ei.out_idx] = out_neighbour.ids[idx]; + self.base_ei.base_op.probe_tuple.borrow_mut()[self.base_ei.out_idx] = out_neighbour.ids[idx]; self.base_ei.base_op.next[0] .borrow_mut() .process_new_tuple(); diff --git a/src/graph_impl/multi_graph/plan/operator/extend/Intersect.rs b/src/graph_impl/multi_graph/plan/operator/extend/Intersect.rs index 4e484c82..03cc6160 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/Intersect.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/Intersect.rs @@ -58,7 +58,7 @@ impl BaseIntersect { impl CommonOperatorTrait for BaseIntersect { fn init( &mut self, - probe_tuple: Vec, + probe_tuple: Rc>>, graph: &TypedStaticGraph, ) { self.base_ei.init(probe_tuple, graph) @@ -70,7 +70,7 @@ impl CommonOperatorTrait for BaseIntersect { { let base_ei = &mut self.base_ei; let cache_id = base_ei.vertex_idx_to_cache[0]; - let to_id = base_ei.base_op.probe_tuple[cache_id].id(); + let to_id = base_ei.base_op.probe_tuple.borrow()[cache_id].id(); let adj_vec = base_ei.adj_lists_to_cache[0][to_id].as_ref(); let cache_id = base_ei.labels_or_to_types_to_cache[0]; let neighbours = &mut base_ei.init_neighbours; @@ -120,7 +120,7 @@ impl CommonOperatorTrait for BaseIntersect { let out_neighbours = &mut base_ei.out_neighbours; base_op.num_out_tuples += out_neighbours.end_idx - out_neighbours.start_idx; for idx in out_neighbours.start_idx..out_neighbours.end_idx { - base_op.probe_tuple[base_ei.out_idx] = out_neighbours.ids[idx]; + base_op.probe_tuple.borrow_mut()[base_ei.out_idx] = out_neighbours.ids[idx]; base_op.next[0].borrow_mut().process_new_tuple(); } } @@ -207,7 +207,7 @@ impl CommonOperatorTrait for BaseIntersect { impl CommonOperatorTrait for Intersect { fn init( &mut self, - probe_tuple: Vec, + probe_tuple: Rc>>, graph: &TypedStaticGraph, ) { match self { diff --git a/src/graph_impl/multi_graph/plan/operator/extend/extend.rs b/src/graph_impl/multi_graph/plan/operator/extend/extend.rs index 87ecbf35..1e9dd88e 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/extend.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/extend.rs @@ -58,7 +58,7 @@ impl Extend { impl CommonOperatorTrait for Extend { fn init( &mut self, - probe_tuple: Vec, + probe_tuple: Rc>>, graph: &TypedStaticGraph, ) { self.base_ei.out_neighbours = Neighbours::new(); @@ -80,7 +80,7 @@ impl CommonOperatorTrait for Extend { } fn process_new_tuple(&mut self) { - let adj_vec = self.adj_list[self.base_ei.base_op.probe_tuple[self.vertex_index].id()] + let adj_vec = self.adj_list[self.base_ei.base_op.probe_tuple.borrow()[self.vertex_index].id()] .as_mut() .unwrap(); let out_neighbour = &mut self.base_ei.out_neighbours; @@ -91,7 +91,7 @@ impl CommonOperatorTrait for Extend { || self.base_ei.to_type == self.base_ei.vertex_types[out_neighbour.ids[idx].id()] { self.base_ei.base_op.num_out_tuples += 1; - self.base_ei.base_op.probe_tuple[self.base_ei.out_idx] = out_neighbour.ids[idx]; + self.base_ei.base_op.probe_tuple.borrow_mut()[self.base_ei.out_idx] = out_neighbour.ids[idx]; self.base_ei.base_op.next[0] .borrow_mut() .process_new_tuple(); diff --git a/src/graph_impl/multi_graph/plan/operator/extend/intersect.rs b/src/graph_impl/multi_graph/plan/operator/extend/intersect.rs index 4e484c82..03cc6160 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/intersect.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/intersect.rs @@ -58,7 +58,7 @@ impl BaseIntersect { impl CommonOperatorTrait for BaseIntersect { fn init( &mut self, - probe_tuple: Vec, + probe_tuple: Rc>>, graph: &TypedStaticGraph, ) { self.base_ei.init(probe_tuple, graph) @@ -70,7 +70,7 @@ impl CommonOperatorTrait for BaseIntersect { { let base_ei = &mut self.base_ei; let cache_id = base_ei.vertex_idx_to_cache[0]; - let to_id = base_ei.base_op.probe_tuple[cache_id].id(); + let to_id = base_ei.base_op.probe_tuple.borrow()[cache_id].id(); let adj_vec = base_ei.adj_lists_to_cache[0][to_id].as_ref(); let cache_id = base_ei.labels_or_to_types_to_cache[0]; let neighbours = &mut base_ei.init_neighbours; @@ -120,7 +120,7 @@ impl CommonOperatorTrait for BaseIntersect { let out_neighbours = &mut base_ei.out_neighbours; base_op.num_out_tuples += out_neighbours.end_idx - out_neighbours.start_idx; for idx in out_neighbours.start_idx..out_neighbours.end_idx { - base_op.probe_tuple[base_ei.out_idx] = out_neighbours.ids[idx]; + base_op.probe_tuple.borrow_mut()[base_ei.out_idx] = out_neighbours.ids[idx]; base_op.next[0].borrow_mut().process_new_tuple(); } } @@ -207,7 +207,7 @@ impl CommonOperatorTrait for BaseIntersect { impl CommonOperatorTrait for Intersect { fn init( &mut self, - probe_tuple: Vec, + probe_tuple: Rc>>, graph: &TypedStaticGraph, ) { match self { diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/build.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/build.rs index 46130756..824e98f3 100644 --- a/src/graph_impl/multi_graph/plan/operator/hashjoin/build.rs +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/build.rs @@ -46,10 +46,10 @@ impl Build { impl CommonOperatorTrait for Build { fn init( &mut self, - probe_tuple: Vec, + probe_tuple: Rc>>, graph: &TypedStaticGraph, ) { - if self.base_op.probe_tuple.len() == 0 { + if self.base_op.probe_tuple.borrow().len() == 0 { self.base_op.probe_tuple = probe_tuple; self.hash_table.as_mut().map(|table| { table.allocate_initial_memory(graph.node_count() + 1); diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/hash_table.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/hash_table.rs index f0069e3e..917810b5 100644 --- a/src/graph_impl/multi_graph/plan/operator/hashjoin/hash_table.rs +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/hash_table.rs @@ -1,5 +1,7 @@ use generic::IdType; use itertools::Itertools; +use std::cell::RefCell; +use std::rc::Rc; #[derive(Clone)] pub struct BlockInfo { @@ -68,8 +70,8 @@ impl HashTable { vec![vec![0; self.initial_num_chunks_per_vertex * 3]; highest_vertex_id + 1]; self.num_chunks = vec![0; highest_vertex_id + 1]; } - pub fn insert_tuple(&mut self, build_tuple: Vec) { - let hash_vertex = build_tuple[self.build_hash_idx].id(); + pub fn insert_tuple(&mut self, build_tuple: Rc>>) { + let hash_vertex = build_tuple.borrow()[self.build_hash_idx].id(); let mut last_chunk_idx = self.num_chunks[hash_vertex]; if 0 == last_chunk_idx { self.num_chunks[hash_vertex] += 1; @@ -86,7 +88,7 @@ impl HashTable { }; for i in 0..self.build_tuple_len { if i != self.build_hash_idx { - block[end_offset] = build_tuple[i]; + block[end_offset] = build_tuple.borrow()[i]; end_offset += 1; } } diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe.rs index 71de5803..7c99d208 100644 --- a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe.rs +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe.rs @@ -61,10 +61,10 @@ impl BaseProbe { impl CommonOperatorTrait for BaseProbe { fn init( &mut self, - probe_tuple: Vec, + probe_tuple: Rc>>, graph: &TypedStaticGraph, ) { - if self.base_op.probe_tuple.len() == 0 { + if self.base_op.probe_tuple.borrow().len() == 0 { self.base_op.probe_tuple = probe_tuple.clone(); self.block_info = BlockInfo::empty(); self.base_op @@ -76,7 +76,7 @@ impl CommonOperatorTrait for BaseProbe { } fn process_new_tuple(&mut self) { - let hash_vertex = self.base_op.probe_tuple[self.probe_hash_idx].id(); + let hash_vertex = self.base_op.probe_tuple.borrow()[self.probe_hash_idx].id(); for hash_table in &mut self.hash_tables { let last_chunk_idx = hash_table.num_chunks[hash_vertex]; let mut prev_first_item = -1i32; @@ -89,15 +89,15 @@ impl CommonOperatorTrait for BaseProbe { let first_item = self.block_info.block[offset]; offset += 1; if prev_first_item != first_item.id() as i32 { - self.base_op.probe_tuple[self.probe_tuple_len] = first_item; + self.base_op.probe_tuple.borrow_mut()[self.probe_tuple_len] = first_item; prev_first_item = first_item.id() as i32; } - self.base_op.probe_tuple[self.probe_tuple_len + 1] = + self.base_op.probe_tuple.borrow_mut()[self.probe_tuple_len + 1] = self.block_info.block[offset]; offset += 1; } else { for k in 0..self.hashed_tuple_len { - self.base_op.probe_tuple[self.probe_tuple_len + k] = + self.base_op.probe_tuple.borrow_mut()[self.probe_tuple_len + k] = self.block_info.block[offset]; offset += 1; } @@ -171,7 +171,7 @@ impl CommonOperatorTrait for BaseProbe { impl CommonOperatorTrait for Probe { fn init( &mut self, - probe_tuple: Vec, + probe_tuple: Rc>>, graph: &TypedStaticGraph, ) { match self { diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_cartesian.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_cartesian.rs index 4aceb5d3..8c957a2d 100644 --- a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_cartesian.rs +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_cartesian.rs @@ -48,10 +48,10 @@ impl ProbeCartesian { impl CommonOperatorTrait for ProbeCartesian { fn init( &mut self, - probe_tuple: Vec, + probe_tuple: Rc>>, graph: &TypedStaticGraph, ) { - if self.base_probe.base_op.probe_tuple.len() == 0 { + if self.base_probe.base_op.probe_tuple.borrow().len() == 0 { self.highest_vertex_id = graph.node_count() + 1; self.other_block_info = BlockInfo::empty(); } @@ -61,7 +61,7 @@ impl CommonOperatorTrait for ProbeCartesian { fn process_new_tuple(&mut self) { for a_hash_vertex in 0..self.highest_vertex_id { let base_probe = &mut self.base_probe; - base_probe.base_op.probe_tuple[base_probe.hashed_tuple_len] = Id::new(a_hash_vertex); + base_probe.base_op.probe_tuple.borrow_mut()[base_probe.hashed_tuple_len] = Id::new(a_hash_vertex); for hash_table in base_probe.hash_tables.clone() { let a_last_chunk_idx = hash_table.num_chunks[a_hash_vertex]; let mut a_prev_first_vertex = -1i32; @@ -77,15 +77,15 @@ impl CommonOperatorTrait for ProbeCartesian { let first_vertex = self.other_block_info.block[an_offset]; an_offset += 1; if a_prev_first_vertex != first_vertex.id() as i32 { - base_probe.base_op.probe_tuple[0] = first_vertex; + base_probe.base_op.probe_tuple.borrow_mut()[0] = first_vertex; a_prev_first_vertex = first_vertex.id() as i32; } - base_probe.base_op.probe_tuple[1] = + base_probe.base_op.probe_tuple.borrow_mut()[1] = self.other_block_info.block[an_offset]; an_offset += 1; } else { for k in 0..base_probe.hashed_tuple_len { - base_probe.base_op.probe_tuple[k] = + base_probe.base_op.probe_tuple.borrow_mut()[k] = self.other_block_info.block[an_offset]; an_offset += 1; } diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices.rs index 29fe78a5..a0a550fd 100644 --- a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices.rs +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices.rs @@ -71,14 +71,14 @@ impl ProbeMultiVertices { impl CommonOperatorTrait for ProbeMultiVertices { fn init( &mut self, - probe_tuple: Vec, + probe_tuple: Rc>>, graph: &TypedStaticGraph, ) { self.base_probe.init(probe_tuple, graph); } fn process_new_tuple(&mut self) { - let hash_vertex = self.base_probe.base_op.probe_tuple[self.base_probe.probe_hash_idx].id(); + let hash_vertex = self.base_probe.base_op.probe_tuple.borrow()[self.base_probe.probe_hash_idx].id(); for hash_table in self.base_probe.hash_tables.clone() { let last_chunk_idx = hash_table.num_chunks[hash_vertex]; for chunk_idx in 0..last_chunk_idx { @@ -93,7 +93,7 @@ impl CommonOperatorTrait for ProbeMultiVertices { .for_each(|offset| { let mut flag = true; for i in 0..self.probe_indices.len() { - if self.base_probe.base_op.probe_tuple[self.probe_indices[i]] + if self.base_probe.base_op.probe_tuple.borrow()[self.probe_indices[i]] != self.base_probe.block_info.block[offset + self.build_indices[i]] { flag = false; @@ -112,7 +112,7 @@ impl CommonOperatorTrait for ProbeMultiVertices { } } if copy { - self.base_probe.base_op.probe_tuple + self.base_probe.base_op.probe_tuple.borrow_mut() [self.base_probe.probe_tuple_len + out] = self.base_probe.block_info.block[offset + k]; out += 1; @@ -196,7 +196,7 @@ impl CommonOperatorTrait for ProbeMultiVertices { impl CommonOperatorTrait for PMV { fn init( &mut self, - probe_tuple: Vec, + probe_tuple: Rc>>, graph: &TypedStaticGraph, ) { match self { diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices_cartesian.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices_cartesian.rs index 7f6cb5af..aff24e8d 100644 --- a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices_cartesian.rs +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices_cartesian.rs @@ -56,10 +56,10 @@ impl ProbeMultiVerticesCartesian { impl CommonOperatorTrait for ProbeMultiVerticesCartesian { fn init( &mut self, - probe_tuple: Vec, + probe_tuple: Rc>>, graph: &TypedStaticGraph, ) { - if self.base_pmv.base_probe.base_op.probe_tuple.len() == 0 { + if self.base_pmv.base_probe.base_op.probe_tuple.borrow().len() == 0 { self.highest_vertex_id = graph.node_count() + 1; self.other_block_info = BlockInfo::empty(); } @@ -68,7 +68,7 @@ impl CommonOperatorTrait for ProbeMultiVerticesCartesian { fn process_new_tuple(&mut self) { for a_hash_vertex in 0..self.highest_vertex_id { - self.base_pmv.base_probe.base_op.probe_tuple + self.base_pmv.base_probe.base_op.probe_tuple.borrow_mut() [self.base_pmv.base_probe.hashed_tuple_len] = Id::new(a_hash_vertex); for hash_table in self.base_pmv.base_probe.hash_tables.clone() { let a_last_chunk_idx = hash_table.num_chunks[a_hash_vertex]; @@ -85,15 +85,15 @@ impl CommonOperatorTrait for ProbeMultiVerticesCartesian { let first_vertex = self.other_block_info.block[an_offset]; an_offset += 1; if a_prev_first_vertex != first_vertex.id() as i32 { - self.base_pmv.base_probe.base_op.probe_tuple[0] = first_vertex; + self.base_pmv.base_probe.base_op.probe_tuple.borrow_mut()[0] = first_vertex; a_prev_first_vertex = first_vertex.id() as i32; } - self.base_pmv.base_probe.base_op.probe_tuple[1] = + self.base_pmv.base_probe.base_op.probe_tuple.borrow_mut()[1] = self.other_block_info.block[an_offset]; an_offset += 1; } else { for k in 0..self.base_pmv.base_probe.hashed_tuple_len { - self.base_pmv.base_probe.base_op.probe_tuple[k] = + self.base_pmv.base_probe.base_op.probe_tuple.borrow_mut()[k] = self.other_block_info.block[an_offset]; an_offset += 1; } diff --git a/src/graph_impl/multi_graph/plan/operator/operator.rs b/src/graph_impl/multi_graph/plan/operator/operator.rs index 3a8c7e1f..2839b982 100644 --- a/src/graph_impl/multi_graph/plan/operator/operator.rs +++ b/src/graph_impl/multi_graph/plan/operator/operator.rs @@ -35,7 +35,7 @@ pub struct BaseOperator { pub name: String, pub next: Vec>>>, pub prev: Option>>>, - pub probe_tuple: Vec, + pub probe_tuple: Rc>>, pub out_tuple_len: usize, pub in_subgraph: Option, pub out_subgraph: QueryGraph, @@ -51,7 +51,7 @@ impl BaseOperator { name: "".to_string(), next: vec![], prev: None, - probe_tuple: vec![], + probe_tuple: Rc::new(RefCell::new(vec![])), out_tuple_len: out_subgraph.get_num_qvertices(), in_subgraph, out_subgraph, @@ -67,7 +67,7 @@ impl BaseOperator { name: "".to_string(), next: vec![], prev: None, - probe_tuple: vec![], + probe_tuple: Rc::new(RefCell::new(vec![])), out_tuple_len: 0, in_subgraph: None, out_subgraph: QueryGraph::empty(), @@ -83,7 +83,7 @@ impl BaseOperator { pub trait CommonOperatorTrait { fn init( &mut self, - probe_tuple: Vec, + probe_tuple: Rc>>, graph: &TypedStaticGraph, ); fn process_new_tuple(&mut self); @@ -154,7 +154,7 @@ impl Operator { impl CommonOperatorTrait for BaseOperator { fn init( &mut self, - probe_tuple: Vec, + probe_tuple: Rc>>, graph: &TypedStaticGraph, ) { panic!("unsupported operation exception") @@ -198,7 +198,7 @@ impl CommonOperatorTrait for BaseOperator { impl CommonOperatorTrait for Operator { fn init( &mut self, - probe_tuple: Vec, + probe_tuple: Rc>>, graph: &TypedStaticGraph, ) { match self { diff --git a/src/graph_impl/multi_graph/plan/operator/scan/scan.rs b/src/graph_impl/multi_graph/plan/operator/scan/scan.rs index d87b2c03..f9384b7f 100644 --- a/src/graph_impl/multi_graph/plan/operator/scan/scan.rs +++ b/src/graph_impl/multi_graph/plan/operator/scan/scan.rs @@ -78,7 +78,7 @@ impl BaseScan { impl CommonOperatorTrait for BaseScan { fn init( &mut self, - probe_tuple: Vec, + probe_tuple: Rc>>, graph: &TypedStaticGraph, ) { self.base_op.probe_tuple = probe_tuple.clone(); @@ -108,18 +108,18 @@ impl CommonOperatorTrait for BaseScan { fn execute(&mut self) { for from_idx in self.from_vertex_start_idx..self.from_vertex_end_idx { let from_vertex = self.vertex_ids[from_idx]; - self.base_op.probe_tuple[0] = from_vertex; + self.base_op.probe_tuple.borrow_mut()[0] = from_vertex; let to_vertex_start_idx = self.fwd_adj_list[from_idx].as_ref().unwrap().get_offsets() [self.label_or_to_type as usize]; let to_vertex_end_idx = self.fwd_adj_list[from_idx].as_ref().unwrap().get_offsets() [(self.label_or_to_type + 1) as usize]; for to_idx in to_vertex_start_idx..to_vertex_end_idx { - self.base_op.probe_tuple[1] = self.fwd_adj_list[from_idx] + self.base_op.probe_tuple.borrow_mut()[1] = self.fwd_adj_list[from_idx] .as_ref() .unwrap() .get_neighbor_id(Id::new(to_idx)); if self.to_type == KEY_ANY - || self.vertex_types[self.base_op.probe_tuple[1].id()] == self.to_type + || self.vertex_types[self.base_op.probe_tuple.borrow()[1].id()] == self.to_type { self.base_op.num_out_tuples += 1; self.base_op.next[0].borrow_mut().process_new_tuple(); @@ -168,7 +168,7 @@ impl CommonOperatorTrait for BaseScan { impl CommonOperatorTrait for Scan { fn init( &mut self, - probe_tuple: Vec, + probe_tuple: Rc>>, graph: &TypedStaticGraph, ) { match self { diff --git a/src/graph_impl/multi_graph/plan/operator/scan/scan_blocking.rs b/src/graph_impl/multi_graph/plan/operator/scan/scan_blocking.rs index f586a170..ba1202b4 100644 --- a/src/graph_impl/multi_graph/plan/operator/scan/scan_blocking.rs +++ b/src/graph_impl/multi_graph/plan/operator/scan/scan_blocking.rs @@ -88,8 +88,8 @@ impl ScanBlocking { fn produce_new_edges(&mut self, from_idx: usize, start_to_idx: usize, end_to_idx: usize) { let base_op = &mut self.base_scan.base_op; for to_idx in start_to_idx..end_to_idx { - base_op.probe_tuple[0] = self.base_scan.vertex_ids[from_idx]; - base_op.probe_tuple[1] = self.base_scan.fwd_adj_list[from_idx] + base_op.probe_tuple.borrow_mut()[0] = self.base_scan.vertex_ids[from_idx]; + base_op.probe_tuple.borrow_mut()[1] = self.base_scan.fwd_adj_list[from_idx] .as_mut() .unwrap() .get_neighbor_id(Id::new(to_idx)); @@ -101,7 +101,7 @@ impl ScanBlocking { fn produce_new_edges_default(&mut self) { for from_idx in self.curr_from_idx + 1..self.from_idx_limit { let label = self.base_scan.label_or_to_type; - self.base_scan.base_op.probe_tuple[0] = self.base_scan.vertex_ids[from_idx]; + self.base_scan.base_op.probe_tuple.borrow_mut()[0] = self.base_scan.vertex_ids[from_idx]; let to_vertex_idx_start = self.base_scan.fwd_adj_list[from_idx] .as_mut() .map_or(0, |adj| adj.get_offsets()[label as usize]); @@ -109,12 +109,12 @@ impl ScanBlocking { .as_mut() .map_or(0, |adj| adj.get_offsets()[(label + 1) as usize]); for to_idx in to_vertex_idx_start..to_vertex_idx_limit { - self.base_scan.base_op.probe_tuple[1] = self.base_scan.fwd_adj_list[from_idx] + self.base_scan.base_op.probe_tuple.borrow_mut()[1] = self.base_scan.fwd_adj_list[from_idx] .as_mut() .unwrap() .get_neighbor_id(Id::new(to_idx)); if self.base_scan.to_type == KEY_ANY - || self.base_scan.vertex_types[self.base_scan.base_op.probe_tuple[1].id()] + || self.base_scan.vertex_types[self.base_scan.base_op.probe_tuple.borrow()[1].id()] == self.base_scan.to_type { self.base_scan.base_op.num_out_tuples += 1; @@ -130,7 +130,7 @@ impl ScanBlocking { impl CommonOperatorTrait for ScanBlocking { fn init( &mut self, - probe_tuple: Vec, + probe_tuple: Rc>>, graph: &TypedStaticGraph, ) { self.base_scan.init(probe_tuple.clone(), graph); diff --git a/src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs b/src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs index 477f674c..4bfebf50 100644 --- a/src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs +++ b/src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs @@ -30,22 +30,10 @@ impl ScanSampling { pub fn set_edge_indices_to_sample(&mut self, edges: Vec, num_edges_to_sample: usize) { let mut rng = thread_rng(); let num_edges = edges.len() / 2; - if unsafe { FLAG == true } { - let vec = vec![60, 88, 49, 47, 35, 83, 71, 21, 69, 74, 77, 17, 83, 82, 35, 14, 4, 15, 41, 20, 13, 18, 74, 77, 62, 10, 33, 52, 72, 53, 65, 65, 77, 47, 32, 2, 15, 83, 58, 15, 20, 25, 35, 20, 28, 21, 64, 16, 82, 42, 49, 57, 86, 66, 87, 16, 88, 17, 3, 7, 17, 88, 25, 78, 48, 87, 61, 8, 88, 18, 38, 22, 17, 11, 26, 60, 77, 72, 85, 10, 7, 33, 27, 49, 45, 72, 79, 43, 21, 61, 240, 28, 189, 287, 315, 93, 91, 1, 159, 54, 277, 37, 273, 102, 215, 44, 4, 195, 121, 120, 263, 248, 44, 187, 12, 220, 143, 282, 132, 303, 225, 125, 37, 307, 82, 182, 205, 293, 178, 75, 0, 215, 75, 290, 78, 171, 74, 136, 272, 302, 279, 157, 16, 206, 7, 96, 8, 57, 103, 147, 67, 148, 185, 138, 288, 277, 251, 278, 118, 318, 98, 72, 17, 311, 166, 100, 217, 262, 55, 100, 147, 173, 77, 229, 65, 32, 29, 243, 101, 161, 103, 300, 228, 314, 236, 83, 197, 22, 125, 51, 216, 125, 239, 173, 307, 107, 171, 97, 88, 203, 290, 245, 178, 21, 157, 197, 126, 1, 50, 231, 71, 294, 210, 25, 266, 244, 64, 202, 312, 10, 28, 193, 89, 245, 221, 110, 205, 262, 94, 139, 237, 271, 271, 303, 17, 58, 76, 111, 235, 261, 266, 113, 273, 157, 238, 78, 216, 278, 120, 35, 155, 5, 28, 309, 155, 315, 113, 190, 1, 286, 258, 152, 318, 101, 67, 69, 19, 111, 222, 281, 237, 1, 298, 261, 104, 283, 189, 192, 180, 53, 152, 275, 213, 91, 193, 2, 260, 34, 174, 18, 115, 262, 29, 188, 238, 290, 0, 236, 93, 202, 214, 54, 102, 8, 149, 22, 239, 114, 195, 290, 193, 100, 37, 270, 72, 32, 270, 283, 54, 62, 280, 286, 72, 145, 305, 233, 33, 88, 77, 280, 146, 222, 185, 186, 59, 303, 272, 217, 10, 55, 134, 296, 34, 248, 289, 8, 88, 295, 136, 246, 87, 164, 244, 149, 163, 213, 3, 212, 186, 94, 235, 124, 215, 126, 11, 217, 144, 137, 157, 194, 159, 62, 200, 198, 125, 132, 211, 180, 106, 198, 179, 71, 266, 232, 5, 44, 297, 36, 23, 242, 95, 262, 288, 141, 271, 256, 272, 263, 73, 298, 130, 68, 79, 218, 271, 240, 309, 157, 316, 135, 240, 28, 189, 287, 315, 93, 91, 1, 159, 54, 277, 37, 273, 102, 215, 44, 4, 195, 121, 120, 263, 248, 44, 187, 12, 220, 143, 282, 132, 303, 225, 125, 37, 307, 82, 182, 205, 293, 178, 75, 0, 215, 75, 290, 78, 171, 74, 136, 272, 302, 279, 157, 16, 206, 7, 96, 8, 57, 103, 147, 67, 148, 185, 138, 288, 277, 251, 278, 118, 318, 98, 72, 17, 311, 166, 100, 217, 262, 55, 100, 147, 173, 77, 229, 65, 32, 29, 243, 101, 161, 103, 300, 228, 314, 236, 83, 197, 22, 125, 51, 216, 125, 239, 173, 307, 107, 171, 97, 88, 203, 290, 245, 178, 21, 157, 197, 126, 1, 50, 231, 71, 294, 210, 25, 266, 244, 64, 202, 312, 10, 28, 193, 89, 245, 221, 110, 205, 262, 94, 139, 237, 271, 271, 303, 17, 58, 76, 111, 235, 261, 266, 113, 273, 157, 238, 78, 216, 278, 120, 35, 155, 5, 28, 309, 155, 315, 113, 190, 1, 286, 258, 152, 318, 101, 67, 69, 19, 111, 222, 281, 237, 1, 298, 261, 104, 283, 189, 192, 180, 53, 152, 275, 213, 91, 193, 2, 260, 34, 174, 18, 115, 262, 29, 188, 238, 290, 0, 236, 93, 202, 214, 54, 102, 8, 149, 22, 239, 114, 195, 290, 193, 100, 37, 270, 72, 32, 270, 283, 54, 62, 280, 286, 72, 145, 305, 233, 33, 88, 77, 280, 146, 222, 185, 186, 59, 303, 272, 217, 10, 55, 134, 296, 34, 248, 289, 8, 88, 295, 136, 246, 87, 164, 244, 149, 163, 213, 3, 212, 186, 94, 235, 124, 215, 126, 11, 217, 144, 137, 157, 194, 159, 62, 200, 198, 125, 132, 211, 180, 106, 198, 179, 71, 266, 232, 5, 44, 297, 36, 23, 242, 95, 262, 288, 141, 271, 256, 272, 263, 73, 298, 130, 68, 79, 218, 271, 240, 309, 157, 316, 135, 560, 188, 733, 383, 571, 317, 187, 129, 255, 790, 885, 357, 337, 70, 983, 716, 132, 611, 185, 952, 903, 440, 812, 699, 556, 764, 943, 890, 356, 975, 129, 637, 645, 915, 146, 278, 461, 165, 434, 491, 704, 503, 651, 546, 910, 107, 106, 424, 400, 846, 567, 829, 16, 910, 263, 608, 232, 793, 647, 787, 739, 308, 185, 234, 608, 629, 795, 886, 54, 638, 418, 840, 689, 663, 806, 324, 249, 550, 823, 356, 915, 621, 461, 5, 833, 512, 573, 819, 869, 865, 583, 108, 196, 762, 332, 723, 869, 278, 221, 19, 792, 605, 239, 13, 339, 971, 875, 865, 664, 779, 34, 821, 306, 117, 157, 357, 254, 897, 466, 583, 775, 710, 370, 249, 682, 20, 832, 650, 600, 170, 60, 673, 665, 661, 125, 302, 205, 518, 510, 139, 749, 975, 239, 367, 977, 26, 12, 175, 235, 773, 618, 785, 849, 93, 238, 846, 24, 118, 728, 163, 283, 5, 380, 85, 731, 347, 849, 126, 609, 190, 258, 536, 126, 581, 739, 901, 755, 207, 318, 537, 877, 673, 810, 837, 616, 795, 765, 224, 52, 981, 376, 947, 245, 955, 801, 610, 644, 802, 238, 402, 499, 294, 381, 540, 78, 354, 384, 780, 413, 362, 534, 54, 390, 232, 853, 342, 399, 498, 483, 674, 577, 4, 229, 526, 8, 160, 46, 603, 982, 702, 408, 798, 296, 113, 209, 649, 897, 248, 781, 760, 338, 382, 633, 858, 219, 335, 816, 857, 522, 759, 710, 840, 706, 536, 449, 488, 952, 423, 616, 118, 535, 420, 532, 821, 291, 437, 515, 980, 154, 446, 587, 796, 791, 990, 75, 57, 112, 521, 509, 322, 703, 926, 904, 70, 669, 548, 819, 404, 42, 966, 595, 647, 234, 968, 613, 108, 873, 740, 375, 146, 127, 454, 672, 429, 239, 0, 304, 295, 265, 554, 706, 356, 431, 666, 463, 912, 341, 829, 860, 807, 385, 911, 691, 128, 82, 668, 655, 311, 94, 307, 48, 150, 979, 548, 495, 255, 703, 406, 972, 568, 362, 679, 131, 788, 129, 193, 956, 890, 654, 405, 610, 820, 962, 983, 459, 86, 21, 758, 502, 41, 261, 854, 852, 211, 666, 716, 316, 389, 426, 213, 930, 577, 472, 820, 194, 837, 278, 838, 790, 504, 716, 835, 616, 688, 369, 915, 161, 894, 118, 970, 127, 663, 642, 578, 869, 767, 376, 659, 108, 212, 85, 178, 87, 949, 661, 323, 677, 429, 981, 892, 352, 351, 915, 408, 242, 175, 977, 501, 567, 875, 281, 619, 365, 373, 314, 265, 680, 366, 209, 813, 482, 320, 547, 31, 559, 263, 454, 330, 303, 30, 352, 313, 637, 386, 231, 234, 772, 709, 39, 210, 754, 881, 610, 45, 196, 470, 145, 671, 32, 882, 9, 524, 834, 540, 441, 337, 814, 973, 624, 829, 772, 940, 475, 431, 210, 757, 716, 365, 739, 421, 758, 734, 341, 68, 948, 229, 392, 47, 42, 890, 563, 289, 676, 103, 752, 221, 858, 284, 280, 891, 725, 975, 265, 876, 43, 947, 451, 396, 6, 884, 210, 964, 67, 493, 390, 248, 405, 166, 958, 595, 170, 813, 259, 941, 390, 527, 721, 407, 875, 435, 240, 515, 762, 724, 801, 242, 307, 25, 979, 955, 633, 627, 150, 309, 465, 306, 71, 866, 840, 164, 393, 568, 276, 108, 917, 652, 371, 634, 232, 733, 688, 460, 347, 282, 836, 769, 496, 466, 344, 630, 552, 311, 393, 447, 81, 845, 804, 3, 189, 84, 337, 468, 462, 207, 333, 810, 653, 945, 847, 700, 249, 804, 844, 538, 989, 693, 757, 137, 859, 927, 958, 668, 36, 594, 122, 587, 361, 416, 450, 179, 107, 580, 948, 874, 463, 335, 88, 284, 398, 144, 286, 359, 553, 255, 675, 592, 601, 647, 819, 513, 324, 430, 456, 767, 524, 962, 464, 651, 961, 961, 269, 21, 407, 901, 949, 856, 466, 170, 58, 887, 94, 541, 580, 978, 571, 836, 922, 964, 981, 484, 160, 860, 177, 658, 794, 981, 356, 410, 367, 532, 350, 453, 297, 415, 835, 843, 696, 693, 122, 629, 118, 557, 863, 109, 133, 750, 467, 516, 661, 360, 979, 800, 34, 238, 6, 673, 378, 145, 783, 846, 123, 442, 771, 288, 571, 195, 788, 586, 737, 434, 620, 35, 49, 627, 694, 179, 518, 721, 469, 790, 375, 625, 504, 143, 859, 44, 69, 268, 777, 486, 946, 50, 569, 47, 564, 293, 550, 982, 558, 566, 21, 805, 491, 266, 270, 381, 102, 482, 969, 655, 806, 492, 37, 563, 520, 262, 273, 873, 404, 938, 525, 330, 754, 634, 619, 810, 693, 649, 719, 684, 789, 68, 662, 769, 475, 539, 370, 831, 565, 112, 499, 315, 521, 123, 258, 962, 390, 503, 615, 153, 950, 333, 362, 335, 176, 155, 950, 28, 43, 96, 794, 601, 812, 902, 425, 59, 544, 941, 705, 77, 421, 944, 8, 713, 290, 23, 115, 722, 88, 632, 96, 721, 144, 266, 195, 11, 467, 871, 926, 699, 25, 61, 42, 785, 345, 370, 434, 742, 383, 370, 766, 123, 155, 599, 45, 589, 150, 243, 891, 535, 192, 604, 770, 517, 741, 436, 317, 272, 350, 628, 8, 730, 259, 501, 355, 360, 382, 603, 469, 151, 634, 863, 13, 380, 923, 279, 100, 669, 136, 836, 927, 325, 419, 407, 944, 328, 153, 640, 251, 915, 820, 53, 22, 814, 269, 220, 202, 434, 255, 181, 87, 161, 504, 135, 627, 198, 872, 69, 339, 426, 700, 989, 898, 656, 964, 925, 622, 494, 784, 418, 642, 840, 538, 702, 206, 970, 988, 869, 451, 738, 962, 757, 634, 216, 314, 927, 884, 300, 648, 726, 988, 586, 278, 947, 585, 235, 585, 611, 96, 459, 974, 781, 566, 515, 243, 484, 114, 675, 947, 805, 337, 974, 553, 749, 387, 721, 934, 507, 659, 353, 605, 189, 644, 176, 283, 218, 663, 12, 948, 987, 937, 980, 120, 855, 985, 184, 261, 601, 808, 428, 758, 368, 240, 268, 13, 461, 215, 425, 431, 471, 357, 380, 23, 323, 407, 172, 35, 194, 274, 249, 473, 38, 247, 126, 308, 41, 170, 46, 51, 160, 414, 377, 11, 245, 5, 155, 194, 398, 159, 65, 400, 285, 218, 205, 89, 272, 424, 93, 82, 106, 460, 294, 463, 345, 410, 282, 69, 196, 340, 89, 21, 367, 125, 340, 313, 150, 408, 87, 295, 278, 196, 0, 218, 472, 449, 407, 80, 330, 149, 162, 355, 64, 277, 249, 99, 157, 189, 468, 277, 133, 165, 133, 219, 178, 84, 210, 308, 229, 159, 12, 135, 135, 398, 279, 221, 311, 57, 139, 15, 213, 270, 193, 148, 263, 190, 187, 127, 349, 96, 235, 300, 447, 447, 398, 320, 91, 238, 194, 220, 482, 184, 282, 450, 383, 369, 257, 469, 436, 71, 212, 436, 117, 241, 407, 113, 191, 325, 52, 54, 383, 467, 119, 430, 271, 341, 257, 104, 204, 382, 38, 158, 321, 191, 477, 54, 473, 245, 35, 101, 184, 365, 88, 336, 184, 260, 243, 25, 199, 315, 391, 384, 19, 347, 357, 148, 179, 100, 41, 367, 38, 290, 361, 132, 49, 421, 203, 61, 116, 278, 100, 402, 58, 391, 182, 375, 86, 174, 360, 14, 46, 59, 250, 90, 308, 324, 456, 441, 284, 201, 352, 229, 410, 337, 446, 277, 466, 320, 162, 358, 195, 448, 464, 68, 148, 96, 399, 483, 263, 159, 390, 383, 390, 148, 74, 221, 158, 267, 237, 28, 263, 230, 101, 352, 120, 278, 230, 323, 20, 364, 109, 310, 108, 117, 0, 166, 87, 423, 151, 331, 300, 102, 162, 409, 392, 389, 92, 79, 275, 214, 77, 219, 448, 471, 306, 202, 472, 279, 382, 83, 64, 212, 276, 69, 17, 258, 162, 325, 224, 171, 340, 445, 94, 55, 330, 80, 383, 425, 424, 162, 415, 225, 210, 12, 340, 323, 258, 221, 360, 101, 249, 48, 431, 157, 193, 235, 346, 128, 302, 435, 175, 258, 277, 100, 312, 345, 110, 459, 231, 123, 220, 420, 354, 478, 415, 271, 176, 269, 63, 462, 294, 450, 269, 416, 262, 378, 27, 3, 72, 193, 70, 236, 97, 225, 78, 280, 93, 118, 406, 348, 177, 484, 479, 186, 391, 66, 438, 388, 135, 104, 316, 122, 98, 54, 87, 228, 382, 453, 407, 63, 76, 354, 154, 99, 403, 126, 322, 227, 311, 96, 129, 376, 248, 289, 198, 181, 445, 37, 189, 287, 173, 399, 168, 468, 319, 373, 142, 438, 429, 43, 345, 377, 101, 335, 171, 477, 255, 130, 65, 168, 182, 261, 129, 340, 446, 331, 123, 323, 277, 216, 182, 369, 456, 94, 167, 215, 268, 175, 178, 396, 73, 5, 336, 130, 263, 318, 469, 430, 166, 289, 183, 368, 52, 395, 140, 126, 226, 351, 431, 404, 385, 430, 341, 96, 78, 189, 297, 374, 51, 126, 385, 69, 339, 278, 402, 121, 312, 246, 195, 0, 28, 19, 17, 5, 23, 11, 21, 9, 14, 17, 17, 23, 22, 5, 14, 4, 15, 11, 20, 13, 18, 14, 17, 2, 10, 3, 22, 12, 23, 0, 28, 19, 17, 5, 23, 11, 21, 9, 14, 17, 17, 23, 22, 5, 14, 4, 15, 11, 20, 13, 18, 14, 17, 2, 10, 3, 22, 12, 23, 60, 88, 49, 47, 35, 83, 71, 21, 69, 74, 77, 17, 83, 82, 35, 14, 4, 15, 41, 20, 13, 18, 74, 77, 62, 10, 33, 52, 72, 53, 65, 65, 77, 47, 32, 2, 15, 83, 58, 15, 20, 25, 35, 20, 28, 21, 64, 16, 82, 42, 49, 57, 86, 66, 87, 16, 88, 17, 3, 7, 17, 88, 25, 78, 48, 87, 61, 8, 88, 18, 38, 22, 17, 11, 26, 60, 77, 72, 85, 10, 7, 33, 27, 49, 45, 72, 79, 43, 21, 61, 0, 0, 96, 44, 179, 177, 39, 35, 163, 15, 55, 16, 29, 23, 21, 2, 95, 0, 122, 175, 125, 154, 1, 80, 178, 151, 142, 58, 13, 180, 114, 69, 49, 73, 51, 49, 88, 136, 101, 175, 122, 15, 70, 61, 3, 84, 76, 141, 124, 56, 6, 42, 161, 167, 104, 68, 11, 66, 80, 109, 37, 9, 105, 82, 41, 118, 152, 83, 13, 96, 66, 38, 24, 0, 69, 59, 156, 118, 67, 150, 161, 66, 61, 111, 163, 29, 75, 4, 163, 49, 139, 53, 59, 48, 6, 44, 168, 77, 139, 136, 13, 37, 92, 65, 139, 73, 153, 129, 153, 25, 170, 129, 60, 123, 62, 75, 9, 79, 92, 155, 48, 133, 35, 142, 54, 157, 14, 148, 102, 172, 146, 88, 2, 145, 173, 17, 81, 80, 181, 178, 64, 69, 167, 5, 179, 105, 11, 58, 16, 93, 17, 145, 6, 95, 13, 97, 26, 42, 40, 92, 70, 99, 179, 41, 114, 165, 67, 9, 167, 104, 125, 174, 16, 80, 54, 43, 179, 109, 95, 43, 40, 37, 37, 163, 12, 16, 25, 23, 11, 35, 23, 15, 27, 2, 29, 23, 35, 16, 11, 14, 10, 21, 41, 14, 1, 24, 38, 11, 2, 16, 27, 40, 30, 41, 35, 17, 23, 35, 32, 38, 3, 35, 10, 15, 14, 19, 12, 2, 11, 9, 11, 7, 9, 1, 13, 2, 1, 9, 7, 2, 12, 2, 11, 9, 11, 7, 9, 1, 13, 2, 1, 9, 7, 2, 12, 16, 25, 23, 11, 35, 23, 15, 27, 2, 29, 23, 35, 16, 11, 14, 10, 21, 41, 14, 1, 24, 38, 11, 2, 16, 27, 40, 30, 41, 35, 17, 23, 35, 32, 38, 3, 35, 10, 15, 14, 19, 0, 4, 1, 5, 5, 5, 0, 1, 1, 0, 1, 1, 12, 2, 11, 9, 11, 7, 9, 1, 13, 2, 1, 9, 7, 2, 0, 1, 1, 0, 12, 2, 11, 9, 11, 7, 9, 1, 13, 2, 1, 9, 7, 2, 0, 1, 1, 0]; - for edge_idx in vec { - self.edges_queue - .push(vec![edges[edge_idx], edges[edge_idx + 1]]); - } - unsafe { - FLAG = false; - } - } else { - while self.edges_queue.len() < num_edges_to_sample { - let edge_idx = rng.gen_range(0, num_edges); - print!("{},", edge_idx); - self.edges_queue - .push(vec![edges[edge_idx], edges[edge_idx + 1]]); - } + while self.edges_queue.len() < num_edges_to_sample { + let edge_idx = rng.gen_range(0, num_edges); + self.edges_queue + .push(vec![edges[edge_idx], edges[edge_idx + 1]]); } } @@ -84,10 +72,10 @@ impl ScanSampling { impl CommonOperatorTrait for ScanSampling { fn init( &mut self, - probe_tuple: Vec, + probe_tuple: Rc>>, graph: &TypedStaticGraph, ) { - if self.base_scan.base_op.probe_tuple.is_empty() { + if self.base_scan.base_op.probe_tuple.borrow().is_empty() { self.base_scan.base_op.probe_tuple = probe_tuple.clone(); self.base_scan.base_op.next.iter().for_each(|next_op| { next_op.borrow_mut().init(probe_tuple.clone(), graph); @@ -102,8 +90,8 @@ impl CommonOperatorTrait for ScanSampling { fn execute(&mut self) { while !self.edges_queue.is_empty() { let edge = self.edges_queue.pop().unwrap(); - self.base_scan.base_op.probe_tuple[0] = edge[0]; - self.base_scan.base_op.probe_tuple[1] = edge[1]; + self.base_scan.base_op.probe_tuple.borrow_mut()[0] = edge[0]; + self.base_scan.base_op.probe_tuple.borrow_mut()[1] = edge[1]; self.base_scan.base_op.num_out_tuples += 1; for next_op in &mut self.base_scan.base_op.next { next_op.borrow_mut().process_new_tuple(); diff --git a/src/graph_impl/multi_graph/plan/operator/sink/sink.rs b/src/graph_impl/multi_graph/plan/operator/sink/sink.rs index a464a9a4..d9f8c8e7 100644 --- a/src/graph_impl/multi_graph/plan/operator/sink/sink.rs +++ b/src/graph_impl/multi_graph/plan/operator/sink/sink.rs @@ -53,7 +53,7 @@ impl BaseSink { impl CommonOperatorTrait for BaseSink { fn init( &mut self, - probe_tuple: Vec, + probe_tuple: Rc>>, graph: &TypedStaticGraph, ) { self.base_op.probe_tuple = probe_tuple; @@ -119,7 +119,7 @@ impl CommonOperatorTrait for BaseSink { impl CommonOperatorTrait for Sink { fn init( &mut self, - probe_tuple: Vec, + probe_tuple: Rc>>, graph: &TypedStaticGraph, ) { match self { diff --git a/src/graph_impl/multi_graph/plan/operator/sink/sink_copy.rs b/src/graph_impl/multi_graph/plan/operator/sink/sink_copy.rs index 6d33a782..9dd63534 100644 --- a/src/graph_impl/multi_graph/plan/operator/sink/sink_copy.rs +++ b/src/graph_impl/multi_graph/plan/operator/sink/sink_copy.rs @@ -30,7 +30,7 @@ impl SinkCopy { impl CommonOperatorTrait for SinkCopy { fn init( &mut self, - probe_tuple: Vec, + probe_tuple: Rc>>, graph: &TypedStaticGraph, ) { self.base_sink.init(probe_tuple, graph); @@ -39,7 +39,7 @@ impl CommonOperatorTrait for SinkCopy { fn process_new_tuple(&mut self) { let len = self.output_tuple.len(); self.output_tuple - .clone_from_slice(&self.base_sink.base_op.probe_tuple[0..len]); + .clone_from_slice(&self.base_sink.base_op.probe_tuple.borrow()[0..len]); } fn execute(&mut self) { diff --git a/src/graph_impl/multi_graph/plan/operator/sink/sink_limit.rs b/src/graph_impl/multi_graph/plan/operator/sink/sink_limit.rs index 4f423818..d650b95e 100644 --- a/src/graph_impl/multi_graph/plan/operator/sink/sink_limit.rs +++ b/src/graph_impl/multi_graph/plan/operator/sink/sink_limit.rs @@ -38,7 +38,7 @@ impl SinkLimit { impl CommonOperatorTrait for SinkLimit { fn init( &mut self, - probe_tuple: Vec, + probe_tuple: Rc>>, graph: &TypedStaticGraph, ) { self.base_sink.init(probe_tuple, graph) diff --git a/src/graph_impl/multi_graph/plan/operator/sink/sink_print.rs b/src/graph_impl/multi_graph/plan/operator/sink/sink_print.rs index 8d49fb53..e6bdbd66 100644 --- a/src/graph_impl/multi_graph/plan/operator/sink/sink_print.rs +++ b/src/graph_impl/multi_graph/plan/operator/sink/sink_print.rs @@ -25,7 +25,7 @@ impl SinkPrint { impl CommonOperatorTrait for SinkPrint { fn init( &mut self, - probe_tuple: Vec, + probe_tuple: Rc>>, graph: &TypedStaticGraph, ) { self.base_sink.init(probe_tuple, graph) diff --git a/src/graph_impl/multi_graph/plan/query_plan.rs b/src/graph_impl/multi_graph/plan/query_plan.rs index ff45d11c..3ff62f59 100644 --- a/src/graph_impl/multi_graph/plan/query_plan.rs +++ b/src/graph_impl/multi_graph/plan/query_plan.rs @@ -296,7 +296,7 @@ impl QueryPlan { for subplan in &mut self.subplans { let probe_tuple = { let subplan_ref = subplan.borrow(); - vec![Id::new(0); get_op_attr!(subplan_ref.deref(), out_tuple_len)] + Rc::new(RefCell::new(vec![Id::new(0); get_op_attr!(subplan_ref.deref(), out_tuple_len)])) }; let mut first_op = subplan.clone(); loop { diff --git a/src/graph_impl/multi_graph/planner/catalog/catalog.rs b/src/graph_impl/multi_graph/planner/catalog/catalog.rs index ca188e27..4dcb7a9c 100644 --- a/src/graph_impl/multi_graph/planner/catalog/catalog.rs +++ b/src/graph_impl/multi_graph/planner/catalog/catalog.rs @@ -264,7 +264,7 @@ impl Catalog { query_plan_arr: &mut Vec>, ) { for query_plan in query_plan_arr { - let probe_tuple = vec![Id::new(0); self.max_input_num_vertices + 1]; + let probe_tuple = Rc::new(RefCell::new(vec![Id::new(0); self.max_input_num_vertices + 1])); if let Some(scan) = &mut query_plan.scan_sampling { scan.borrow_mut().init(probe_tuple, graph); } @@ -299,13 +299,13 @@ impl Catalog { } } - fn retrieve_op(op:&Rc>>){ - unsafe{ - print!("{:?}->",op.as_ptr()); + fn retrieve_op(op: &Rc>>) { + unsafe { + print!("{:?}->", op.as_ptr()); } let op_ref = op.borrow(); let base = get_base_op_as_ref!(op_ref.deref()); - if let Some(op)=&base.prev{ + if let Some(op) = &base.prev { Self::retrieve_op(op); } } @@ -323,11 +323,8 @@ impl Catalog { let base_sink = get_sink_as_ref!(sink); let mut op = base_sink.previous[0].clone(); loop { - { - let op_ref = op.borrow(); - if let Operator::Scan(Scan::ScanSampling(sp)) = op_ref.deref() { - break; - } + if let Operator::Scan(Scan::ScanSampling(sp)) = op.borrow().deref() { + break; } op = { let op_ref = op.borrow(); @@ -380,7 +377,6 @@ impl Catalog { get_op_attr_as_ref!(op_ref.deref(), next).clone() }; - println!("-----"); for i in 0..next.len() { let next_i = next[i].borrow(); if let Operator::EI(EI::Intersect(Intersect::IntersectCatalog(intersect))) = @@ -415,7 +411,6 @@ impl Catalog { alds_as_str_list.push(alds_str); } let mut selectivity = intersect.base_intersect.base_ei.base_op.num_out_tuples; - println!("add_icost_and_selectivity_sorted_by_node={}",selectivity); for other_op in &other { let next = { let other_op_ref = other_op.borrow(); @@ -619,8 +614,7 @@ impl Catalog { return idx; } } - //TODO:Fix the case when the given subgraph not found - 0 + panic!("Illegal argument exception.") } fn generate_direction_patterns(&self, size: usize, is_directed: bool) -> Vec> { @@ -676,7 +670,7 @@ impl Catalog { plans: &mut CatalogPlans, ) { let selectivity_zero = &mut plans.selectivity_zero; - for (q_graph, alds,to_type) in selectivity_zero { + for (q_graph, alds, to_type) in selectivity_zero { let subgraph_idx = self.get_subgraph_idx(q_graph); if self.sampled_selectivity.get(&subgraph_idx).is_none() { self.sampled_selectivity @@ -704,7 +698,7 @@ impl Catalog { alds_as_str_list.push(alds_str); } for alds_as_str in alds_as_str_list { - let selectivity= self.sampled_selectivity.get_mut(&subgraph_idx).unwrap(); + let selectivity = self.sampled_selectivity.get_mut(&subgraph_idx).unwrap(); selectivity.insert(alds_as_str + "~" + &to_type.to_string(), 0.00); } } diff --git a/src/graph_impl/multi_graph/planner/catalog/catalog_plans.rs b/src/graph_impl/multi_graph/planner/catalog/catalog_plans.rs index 030e85e1..fc5de0ca 100644 --- a/src/graph_impl/multi_graph/planner/catalog/catalog_plans.rs +++ b/src/graph_impl/multi_graph/planner/catalog/catalog_plans.rs @@ -496,7 +496,6 @@ impl CatalogPlans { if self.sorted_by_node && num_edges_to_sample < 1000 { num_edges_to_sample = actual_num_edges; } - println!("{:?}",key_to_edges_map[&edge_key]); scan.set_edge_indices_to_sample( key_to_edges_map[&edge_key] .iter() diff --git a/src/graph_impl/multi_graph/planner/catalog/operator/intersect_catalog.rs b/src/graph_impl/multi_graph/planner/catalog/operator/intersect_catalog.rs index 3be25b33..5f859cd7 100644 --- a/src/graph_impl/multi_graph/planner/catalog/operator/intersect_catalog.rs +++ b/src/graph_impl/multi_graph/planner/catalog/operator/intersect_catalog.rs @@ -10,6 +10,7 @@ use hashbrown::HashMap; use std::cell::RefCell; use std::hash::Hash; use std::rc::Rc; +use itertools::Itertools; #[derive(Clone)] pub struct IntersectCatalog { @@ -48,7 +49,7 @@ impl IntersectCatalog { impl CommonOperatorTrait for IntersectCatalog { fn init( &mut self, - probe_tuple: Vec, + probe_tuple: Rc>>, graph: &TypedStaticGraph, ) { self.base_intersect.init(probe_tuple, graph) @@ -59,7 +60,7 @@ impl CommonOperatorTrait for IntersectCatalog { if 1 == base_ei.alds.len() { // intersect the adjacency lists and setAdjListSortOrder the output vertex values. let adj = base_ei.adj_lists_to_cache[0] - [base_ei.base_op.probe_tuple[base_ei.vertex_idx_to_cache[0]].id()] + [base_ei.base_op.probe_tuple.borrow()[base_ei.vertex_idx_to_cache[0]].id()] .as_ref() .unwrap(); adj.set_neighbor_ids( @@ -73,7 +74,7 @@ impl CommonOperatorTrait for IntersectCatalog { let mut temp; if base_ei.caching_type == CachingType::None || !base_ei.is_intersection_cached() { let adj = base_ei.adj_lists_to_cache[0] - [base_ei.base_op.probe_tuple[base_ei.vertex_idx_to_cache[0]].id()] + [base_ei.base_op.probe_tuple.borrow()[base_ei.vertex_idx_to_cache[0]].id()] .as_ref() .unwrap(); adj.set_neighbor_ids( @@ -83,16 +84,14 @@ impl CommonOperatorTrait for IntersectCatalog { self.last_icost = base_ei.init_neighbours.end_idx - base_ei.init_neighbours.start_idx; let adj = base_ei.adj_lists_to_cache[1] - [base_ei.base_op.probe_tuple[base_ei.vertex_idx_to_cache[1]].id()] + [base_ei.base_op.probe_tuple.borrow()[base_ei.vertex_idx_to_cache[1]].id()] .as_ref() .unwrap(); - println!("****"); self.last_icost += adj.intersect( base_ei.labels_or_to_types_to_cache[1], &mut base_ei.init_neighbours, &mut base_ei.cached_neighbours, ); - println!("****"); if base_ei.to_type != KEY_ANY { let mut curr_end_idx = 0; @@ -113,7 +112,7 @@ impl CommonOperatorTrait for IntersectCatalog { base_ei.cached_neighbours = base_ei.temp_neighbours.clone(); base_ei.temp_neighbours = temp; let adj = base_ei.adj_lists_to_cache[i] - [base_ei.base_op.probe_tuple[base_ei.vertex_idx_to_cache[i]].id()] + [base_ei.base_op.probe_tuple.borrow()[base_ei.vertex_idx_to_cache[i]].id()] .as_ref() .unwrap(); self.last_icost += adj.intersect( @@ -131,7 +130,7 @@ impl CommonOperatorTrait for IntersectCatalog { } CachingType::PartialCaching => { let adj = base_ei.adj_lists[0] - [base_ei.base_op.probe_tuple[base_ei.vertex_idx[0]].id()] + [base_ei.base_op.probe_tuple.borrow()[base_ei.vertex_idx[0]].id()] .as_ref() .unwrap(); base_ei.base_op.icost += adj.intersect( @@ -144,7 +143,7 @@ impl CommonOperatorTrait for IntersectCatalog { base_ei.out_neighbours = base_ei.temp_neighbours.clone(); base_ei.temp_neighbours = temp; let adj = base_ei.adj_lists[i] - [base_ei.base_op.probe_tuple[base_ei.vertex_idx[i]].id()] + [base_ei.base_op.probe_tuple.borrow()[base_ei.vertex_idx[i]].id()] .as_ref() .unwrap(); base_ei.base_op.icost += adj.intersect( @@ -159,7 +158,7 @@ impl CommonOperatorTrait for IntersectCatalog { // println!("start_idx={},end_idx={}",base_ei.out_neighbours.start_idx,base_ei.out_neighbours.end_idx); for idx in base_ei.out_neighbours.start_idx..base_ei.out_neighbours.end_idx { - base_ei.base_op.probe_tuple[base_ei.out_idx] = base_ei.out_neighbours.ids[idx]; + base_ei.base_op.probe_tuple.borrow_mut()[base_ei.out_idx] = base_ei.out_neighbours.ids[idx]; base_ei.base_op.num_out_tuples += 1; if self.is_adj_list_sorted_by_node { base_ei.base_op.next[0].borrow_mut().process_new_tuple(); @@ -168,7 +167,7 @@ impl CommonOperatorTrait for IntersectCatalog { .base_op .next .get( - base_ei.vertex_types[base_ei.base_op.probe_tuple[base_ei.out_idx].id()] + base_ei.vertex_types[base_ei.base_op.probe_tuple.borrow()[base_ei.out_idx].id()] as usize, ) .map(|next_op| next_op.borrow_mut().process_new_tuple()); diff --git a/src/graph_impl/multi_graph/planner/catalog/operator/noop.rs b/src/graph_impl/multi_graph/planner/catalog/operator/noop.rs index e819bb71..692da1d1 100644 --- a/src/graph_impl/multi_graph/planner/catalog/operator/noop.rs +++ b/src/graph_impl/multi_graph/planner/catalog/operator/noop.rs @@ -25,7 +25,7 @@ impl Noop { impl CommonOperatorTrait for Noop { fn init( &mut self, - probe_tuple: Vec, + probe_tuple: Rc>>, graph: &TypedStaticGraph, ) { self.base_op.probe_tuple = probe_tuple.clone(); diff --git a/src/graph_impl/multi_graph/planner/query_planner.rs b/src/graph_impl/multi_graph/planner/query_planner.rs index 02a1185c..58f04715 100644 --- a/src/graph_impl/multi_graph/planner/query_planner.rs +++ b/src/graph_impl/multi_graph/planner/query_planner.rs @@ -164,10 +164,6 @@ impl let prev_query_plan = self.subgraph_plans[&(self.next_num_qvertices - 1)][key][i].clone(); let (key, plan) = self.get_plan_with_next_extend(prev_query_plan, &to_qvertex); - println!( - "extend_key={}, icost={}, estimated_icost={}", - key, plan.icost, plan.estimated_icost - ); let plan_map = self .subgraph_plans .get_mut(&self.next_num_qvertices) @@ -176,7 +172,6 @@ impl plan_map.get_mut(&key).unwrap().push(plan); } } - println!("-------"); } pub fn get_plan_with_next_extend( @@ -209,29 +204,14 @@ impl &alds, to_type, ); - println!("estimated_selectivity={}", estimated_selectivity); let icost; if let CachingType::None = base_next_extend.caching_type { - // for (from, edges) in &in_subgraph.qvertex_to_qedges_map { - // for (to, edge) in edges { - // print!("{}->{},", from, to); - // } - // } - // println!(); - // println!("alds.len={},to_type={}", alds.len(), base_next_extend.to_type); - // alds.iter().for_each(|ald|{ - // println!("from={},id={},dir={},label={}",ald.from_query_vertex,ald.vertex_idx,ald.direction,ald.label); - // }); icost = prev_estimated_num_out_tuples * self.catalog.get_icost( &mut in_subgraph, alds.iter().collect(), base_next_extend.to_type, ); - println!( - "prev_estimated_num_out_tuples={},icost={}", - prev_estimated_num_out_tuples, icost - ); } else { let mut out_tuples_to_process = prev_estimated_num_out_tuples; if last_prev.is_some() { @@ -289,7 +269,6 @@ impl let mut new_query_plan = prev_query_plan.shallow_copy(); new_query_plan.estimated_icost = estimated_icost; - println!("new:estimated_num_out_tuples={}", estimated_num_out_tuples); new_query_plan.estimated_num_out_tuples = estimated_num_out_tuples; let query_vertices = base_next_extend .base_op @@ -387,7 +366,6 @@ impl unsafe { LOGGER_FLAG = true }; selectivity = self.catalog.get_selectivity(in_subgraph, alds, to_type); unsafe { LOGGER_FLAG = false }; - println!("compute selectivity"); self.computed_selectivities .get_mut(&out_subgraph.get_encoding()) .unwrap() diff --git a/src/graph_impl/static_graph/sorted_adj_vec.rs b/src/graph_impl/static_graph/sorted_adj_vec.rs index 0a6d6df5..0a84fed7 100644 --- a/src/graph_impl/static_graph/sorted_adj_vec.rs +++ b/src/graph_impl/static_graph/sorted_adj_vec.rs @@ -94,8 +94,6 @@ impl SortedAdjVec { let some_neighbour_ids = &some_neighbours.ids; let mut some_idx = some_neighbours.start_idx; let some_idx_end = some_neighbours.end_idx; - println!("neighbour_ids={:?}",neighbour_ids.iter().map(|x|x.id()).collect_vec()); - println!("some_neighbours={:?}",some_neighbours.ids.iter().map(|x|x.id()).collect_vec()); while this_idx < this_idx_end && some_idx < some_idx_end { if neighbour_ids[this_idx] < some_neighbour_ids[some_idx] { this_idx += 1; From 6f59ec871099212032b1abf5691bed89f87d62d4 Mon Sep 17 00:00:00 2001 From: tangtang <761552682@qq.com> Date: Fri, 6 Mar 2020 22:03:38 +0800 Subject: [PATCH 23/25] fix:QueryPlan Generator --- .../plan/operator/hashjoin/hash_join.rs | 11 ++-- src/graph_impl/multi_graph/plan/query_plan.rs | 1 + .../multi_graph/planner/catalog/catalog.rs | 10 --- .../catalog/operator/intersect_catalog.rs | 1 - .../catalog/subgraph_mapping_iterator.rs | 10 --- .../multi_graph/planner/query_planner.rs | 61 +++++++------------ tests/static_graph.rs | 7 ++- 7 files changed, 33 insertions(+), 68 deletions(-) diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/hash_join.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/hash_join.rs index 2501cc68..08bb2755 100644 --- a/src/graph_impl/multi_graph/plan/operator/hashjoin/hash_join.rs +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/hash_join.rs @@ -63,10 +63,10 @@ impl HashJoin { build_subplans.push(build.clone()); let mapping = { - let mut pre_probe = pre_probe.borrow_mut(); - let mut pre_build = pre_build.borrow_mut(); - let out_subgraph_probe = get_op_attr_as_mut!(pre_probe.deref_mut(), out_subgraph); - let out_subgraph_build = get_op_attr_as_mut!(pre_build.deref_mut(), out_subgraph); + let mut pre_probe = pre_probe.borrow().clone(); + let mut pre_build = pre_build.borrow().clone(); + let out_subgraph_probe = get_op_attr_as_mut!(&mut pre_probe, out_subgraph); + let out_subgraph_build = get_op_attr_as_mut!(&mut pre_build, out_subgraph); out_subgraph_build.get_isomorphic_mapping_if_any(out_subgraph_probe) }; let mut probe_qvertex_to_idx_map; @@ -98,6 +98,7 @@ impl HashJoin { let mut probe_indices = vec![0; join_qvertices.len() - 1]; let mut build_indices = vec![0; join_qvertices.len() - 1]; for (i, join_qvertex) in join_qvertices.iter().enumerate() { + if i < 1 { continue; } probe_indices[i - 1] = probe_qvertex_to_idx_map[join_qvertex].clone(); let mut other_build_idx = build_qvertex_to_idx_map[join_qvertex]; if build_hash_idx < other_build_idx { @@ -159,7 +160,7 @@ impl HashJoin { }; let probe = Rc::new(RefCell::new(Operator::Probe(probe))); get_op_attr_as_mut!(probe.borrow_mut().deref_mut(), prev).replace(pre_probe.clone()); - *get_op_attr_as_mut!(probe.borrow_mut().deref_mut(), next) = vec![probe.clone()]; + *get_op_attr_as_mut!(pre_probe.borrow_mut().deref_mut(), next) = vec![probe.clone()]; let last_index = probe_subplans.len() - 1; probe_subplans[last_index] = probe.clone(); probe diff --git a/src/graph_impl/multi_graph/plan/query_plan.rs b/src/graph_impl/multi_graph/plan/query_plan.rs index 3ff62f59..9e7be775 100644 --- a/src/graph_impl/multi_graph/plan/query_plan.rs +++ b/src/graph_impl/multi_graph/plan/query_plan.rs @@ -147,6 +147,7 @@ impl QueryPlan { } *get_op_attr_as_mut!(new_operator.borrow_mut().deref_mut(), prev) = self.last_operator.as_ref().map(|op| op.clone()); + self.subplans.pop(); self.subplans.push(new_operator.clone()); self.last_operator = Some(new_operator); } diff --git a/src/graph_impl/multi_graph/planner/catalog/catalog.rs b/src/graph_impl/multi_graph/planner/catalog/catalog.rs index 4dcb7a9c..ade2fc74 100644 --- a/src/graph_impl/multi_graph/planner/catalog/catalog.rs +++ b/src/graph_impl/multi_graph/planner/catalog/catalog.rs @@ -153,9 +153,6 @@ impl Catalog { if new_num_alds_matched == 0 || new_num_alds_matched < num_alds_matched { continue; } - // for (k, v) in &self.sampled_selectivity[&0] { - // println!("{},{}",k,v); - // } let sampled_selectivity = self.sampled_selectivity[&i] [&self.get_alds_as_str(&alds, Some(&vertex_mapping), Some(to_type))] .clone(); @@ -288,13 +285,6 @@ impl Catalog { let mut sink = query_plan_arr[0].sink.as_mut().unwrap().as_ptr(); unsafe { (&mut *sink).execute(); -// if let Operator::Sink(Sink::BaseSink(base)) = &*sink{ -// for (i,previous) in base.previous.iter().enumerate() { -// println!("-------i={}",i); -// Self::retrieve_op(previous); -// println!() -// } -// } } } } diff --git a/src/graph_impl/multi_graph/planner/catalog/operator/intersect_catalog.rs b/src/graph_impl/multi_graph/planner/catalog/operator/intersect_catalog.rs index 5f859cd7..13528df0 100644 --- a/src/graph_impl/multi_graph/planner/catalog/operator/intersect_catalog.rs +++ b/src/graph_impl/multi_graph/planner/catalog/operator/intersect_catalog.rs @@ -156,7 +156,6 @@ impl CommonOperatorTrait for IntersectCatalog { } } -// println!("start_idx={},end_idx={}",base_ei.out_neighbours.start_idx,base_ei.out_neighbours.end_idx); for idx in base_ei.out_neighbours.start_idx..base_ei.out_neighbours.end_idx { base_ei.base_op.probe_tuple.borrow_mut()[base_ei.out_idx] = base_ei.out_neighbours.ids[idx]; base_ei.base_op.num_out_tuples += 1; diff --git a/src/graph_impl/multi_graph/planner/catalog/subgraph_mapping_iterator.rs b/src/graph_impl/multi_graph/planner/catalog/subgraph_mapping_iterator.rs index 095b8a0c..25e2acea 100644 --- a/src/graph_impl/multi_graph/planner/catalog/subgraph_mapping_iterator.rs +++ b/src/graph_impl/multi_graph/planner/catalog/subgraph_mapping_iterator.rs @@ -66,10 +66,6 @@ impl SubgraphMappingIterator { let q_vertex = &self.query_vertices[j]; let vertex_type = self.qvertex_to_type_map[q_vertex]; let q_vertex_deg = &self.qvertex_to_deg_map[q_vertex]; - // if unsafe { LOGGER_FLAG } { - // println!("q_vertex_type={},q_vertex_deg={:?},q_vertex={},query_vertices={:?}", vertex_type, q_vertex_deg, self.query_vertices.len(),self.query_vertices); - // println!("o_vertex_type={},o_qvertex_deg={:?},o_qvertices={},o_qvertices={:?}", o_qvertex_type, o_qvertex_deg, self.o_qvertices.len(),self.o_qvertices); - // } if o_qvertex_type == vertex_type && (o_qvertex_deg.eq(q_vertex_deg) || (self.o_qvertices.len() < self.query_vertices.len() @@ -90,9 +86,6 @@ impl SubgraphMappingIterator { pub fn has_next(&mut self) -> bool { if !self.is_next_computed { - // if unsafe{LOGGER_FLAG}{ - // println!("PRE: curr_mapping={:?},self.o_qvertices={:?},vertices_for_idx={:?}",self.curr_mapping,self.o_qvertices,self.vertices_for_idx); - // } if self.curr_mapping.len() == self.o_qvertices.len() { self.curr_mapping.pop(); } @@ -181,9 +174,6 @@ impl SubgraphMappingIterator { } } - // if unsafe{LOGGER_FLAG}{ - // println!("LAT: curr_mapping={:?},self.o_qvertices={:?},vertices_for_idx={:?}",self.curr_mapping,self.o_qvertices,self.vertices_for_idx); - // } !self.curr_mapping.is_empty() } diff --git a/src/graph_impl/multi_graph/planner/query_planner.rs b/src/graph_impl/multi_graph/planner/query_planner.rs index 58f04715..a3962461 100644 --- a/src/graph_impl/multi_graph/planner/query_planner.rs +++ b/src/graph_impl/multi_graph/planner/query_planner.rs @@ -27,6 +27,7 @@ use std::cell::RefCell; use std::hash::Hash; use std::ops::{Deref, DerefMut}; use std::rc::Rc; +use itertools::Itertools; pub struct QueryPlanner { subgraph_plans: HashMap>>>, @@ -118,7 +119,7 @@ impl let query_plan = QueryPlan::new_from_last_op(scan, num_edges as f64); let mut query_plans = vec![]; query_plans.push(query_plan); - let key = self.get_key(vec![ + let key = QueryPlanner::::get_key(vec![ query_edge.from_query_vertex.clone(), query_edge.to_query_vertex.clone(), ]); @@ -135,9 +136,8 @@ impl self.subgraph_plans .entry(self.next_num_qvertices) .or_insert(HashMap::new()); - let plan_map = &self.subgraph_plans[&(self.next_num_qvertices - 1)]; - let plan_map_keys: Vec = plan_map.keys().map(|v| v.clone()).collect(); - for key in plan_map_keys { + let keys: Vec = (&self.subgraph_plans[&(self.next_num_qvertices - 1)]).keys().map(|v| v.clone()).collect(); + for key in keys { self.consider_all_next_extend_operators(&key); } if !self.has_limit && self.next_num_qvertices >= 4 { @@ -169,7 +169,7 @@ impl .get_mut(&self.next_num_qvertices) .unwrap(); plan_map.entry(key.clone()).or_insert(vec![]); - plan_map.get_mut(&key).unwrap().push(plan); + plan_map.get_mut(&key).unwrap().push(plan.clone()); } } } @@ -278,7 +278,7 @@ impl .collect(); new_query_plan.append(Rc::new(RefCell::new(Operator::EI(next_extend)))); new_query_plan.q_vertex_to_num_out_tuples = q_vertex_to_num_out_tuples; - (self.get_key(query_vertices), new_query_plan) + (QueryPlanner::::get_key(query_vertices), new_query_plan) } fn get_next_ei( @@ -357,15 +357,7 @@ impl self.computed_selectivities .insert(out_subgraph.get_encoding(), vec![]); } - // for (from, edges) in &in_subgraph.qvertex_to_qedges_map { - // for (to, edge) in edges { - // print!("{}->{},",from,to); - // } - // } - // println!(); - unsafe { LOGGER_FLAG = true }; selectivity = self.catalog.get_selectivity(in_subgraph, alds, to_type); - unsafe { LOGGER_FLAG = false }; self.computed_selectivities .get_mut(&out_subgraph.get_encoding()) .unwrap() @@ -393,14 +385,14 @@ impl let last_op_ref = last_op.borrow(); let base_last_op = get_base_op_as_ref!(last_op_ref.deref()); let prev_qvertices = base_last_op.out_subgraph.get_query_vertices_as_set(); - let is_subset = query_vertices + let is_subset = prev_qvertices .iter() - .map(|v| prev_qvertices.contains(v)) + .map(|v| query_vertices.contains(v)) .filter(|&x| !x) .count() == 0; if !is_subset { - return; + continue; } let mut other_set: Vec = query_vertices .iter() @@ -408,7 +400,7 @@ impl .map(|x| x.clone()) .collect(); if other_set.len() == 1 { - return; + continue; } let join_qvertices = Self::get_join_qvertices(&out_subgraph, &prev_qvertices, &other_set); @@ -416,15 +408,16 @@ impl || join_qvertices.len() > 2 || other_set.len() + join_qvertices.len() > self.next_num_qvertices - 1 { - return; + continue; } join_qvertices.iter().for_each(|v| { other_set.push(v.clone()); }); + let rest_size = other_set.len(); - let rest_key = self.get_key(other_set); + let rest_key = QueryPlanner::::get_key(other_set); if !self.subgraph_plans[&rest_size].contains_key(&rest_key) { - return; + continue; } let other_prev_operator = self.get_best_plan(rest_size, &rest_key); self.consider_hash_join_operator( @@ -465,32 +458,22 @@ impl ) { let is_plan_build_subplan = subplan.estimated_num_out_tuples < other_subplan.estimated_num_out_tuples; - let build_subplan = if is_plan_build_subplan { - subplan - } else { - other_subplan - }; - let probe_subplan = if is_plan_build_subplan { - other_subplan - } else { - subplan - }; - let build_coef = if num_join_qvertices == 1 { - SINGLE_VERTEX_WEIGHT_BUILD_COEF + let (build_subplan,probe_subplan) = if is_plan_build_subplan { + (subplan,other_subplan) } else { - MULTI_VERTEX_WEIGHT_BUILD_COEF + (other_subplan,subplan) }; - let probe_coef = if num_join_qvertices == 1 { - SINGLE_VERTEX_WEIGHT_PROBE_COEF + let (build_coef,probe_coef) = if num_join_qvertices == 1 { + (SINGLE_VERTEX_WEIGHT_BUILD_COEF,SINGLE_VERTEX_WEIGHT_PROBE_COEF) } else { - MULTI_VERTEX_WEIGHT_PROBE_COEF + (MULTI_VERTEX_WEIGHT_BUILD_COEF,MULTI_VERTEX_WEIGHT_PROBE_COEF) }; let icost = build_subplan.estimated_icost + probe_subplan.estimated_icost + build_coef * build_subplan.estimated_num_out_tuples + probe_coef * probe_subplan.estimated_num_out_tuples; - let key = self.get_key(query_vertices.clone()); + let key = QueryPlanner::::get_key(query_vertices.clone()); let curr_best_query_plan = self.get_best_plan(query_vertices.len(), &key); if curr_best_query_plan.estimated_icost > icost { let mut query_plan = HashJoin::make( @@ -553,7 +536,7 @@ impl self.graph.get_num_edges(from_type, to_type, label) } - fn get_key(&self, mut query_vertices: Vec) -> String { + fn get_key(mut query_vertices: Vec) -> String { query_vertices.sort(); serde_json::to_string(&query_vertices).unwrap() } diff --git a/tests/static_graph.rs b/tests/static_graph.rs index f69a6f79..e78b1feb 100644 --- a/tests/static_graph.rs +++ b/tests/static_graph.rs @@ -324,9 +324,11 @@ fn test_clone() { fn test_graphflow_planner() { let mut g_: TypedGraphMap = TypedDiGraphMap::new(); let path_to_nodes = - Path::new("C:\\Users\\cheny\\OneDrive\\桌面\\rust_graphflow\\human-vertices.csv"); + Path::new("C:\\Users\\76155\\Desktop\\rust_graphflow\\human-vertices.csv"); + // Path::new("C:\\Users\\cheny\\OneDrive\\桌面\\rust_graphflow\\human-vertices.csv"); let path_to_edges = - Path::new("C:\\Users\\cheny\\OneDrive\\桌面\\rust_graphflow\\human-edges.csv"); + Path::new("C:\\Users\\76155\\Desktop\\rust_graphflow\\human-edges.csv"); + // Path::new("C:\\Users\\cheny\\OneDrive\\桌面\\rust_graphflow\\human-edges.csv"); read_from_csv( &mut g_, vec![path_to_nodes], @@ -434,7 +436,6 @@ fn test_graphflow_planner() { limit: 0, }; let catalog = catalog_generator::default(&g); - unsafe { LOGGER_FLAG = true }; let mut query_plan = optimizer_executor::generate_plan(query_graph, catalog, g.clone()); println!("QueryPlan:{}", query_plan.get_output_log()); let mut workers = QPWorkers::new(query_plan, 1); From e63c2eaccdb7b3a28c649e2f99c2f30f106453ed Mon Sep 17 00:00:00 2001 From: tangtang <761552682@qq.com> Date: Sat, 7 Mar 2020 15:34:53 +0800 Subject: [PATCH 24/25] fix:execute --- .../multi_graph/plan/operator/extend/EI.rs | 7 +- .../plan/operator/extend/extend.rs | 10 +-- .../plan/operator/hashjoin/build.rs | 8 ++- .../plan/operator/hashjoin/hash_join.rs | 6 +- .../plan/operator/hashjoin/hash_table.rs | 2 +- .../plan/operator/hashjoin/probe.rs | 13 ++-- .../plan/operator/hashjoin/probe_cartesian.rs | 17 ++--- .../operator/hashjoin/probe_multi_vertices.rs | 7 +- .../probe_multi_vertices_cartesian.rs | 9 +-- .../plan/operator/scan/scan_blocking.rs | 9 ++- .../plan/operator/scan/scan_sampling.rs | 2 +- .../multi_graph/plan/operator/sink/sink.rs | 6 +- src/graph_impl/multi_graph/plan/query_plan.rs | 69 +++++++++---------- .../multi_graph/planner/catalog/catalog.rs | 57 ++++++++------- .../planner/catalog/catalog_plans.rs | 14 ++-- .../catalog/operator/intersect_catalog.rs | 8 ++- .../multi_graph/planner/query_planner.rs | 36 ++++++---- tests/static_graph.rs | 14 ++-- 18 files changed, 169 insertions(+), 125 deletions(-) diff --git a/src/graph_impl/multi_graph/plan/operator/extend/EI.rs b/src/graph_impl/multi_graph/plan/operator/extend/EI.rs index df05cf6e..5f4b8d31 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/EI.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/EI.rs @@ -240,8 +240,9 @@ impl BaseEI { pub fn execute_intersect(&mut self, idx: usize, intersect_type: IntersectType) -> usize { let (adj_vec, label_or_type) = match intersect_type { IntersectType::CachedOut | IntersectType::TempOut => ( - self.adj_lists_to_cache[idx][self.base_op.probe_tuple.borrow()[self.vertex_idx[idx]].id()] - .as_ref(), + self.adj_lists_to_cache[idx] + [self.base_op.probe_tuple.borrow()[self.vertex_idx[idx]].id()] + .as_ref(), self.labels_or_to_types[idx], ), _ => ( @@ -384,7 +385,7 @@ impl EI { impl CommonOperatorTrait for EI { fn init( &mut self, - probe_tuple:Rc>>, + probe_tuple: Rc>>, graph: &TypedStaticGraph, ) { match self { diff --git a/src/graph_impl/multi_graph/plan/operator/extend/extend.rs b/src/graph_impl/multi_graph/plan/operator/extend/extend.rs index 1e9dd88e..fb53f71b 100644 --- a/src/graph_impl/multi_graph/plan/operator/extend/extend.rs +++ b/src/graph_impl/multi_graph/plan/operator/extend/extend.rs @@ -80,9 +80,10 @@ impl CommonOperatorTrait for Extend { } fn process_new_tuple(&mut self) { - let adj_vec = self.adj_list[self.base_ei.base_op.probe_tuple.borrow()[self.vertex_index].id()] - .as_mut() - .unwrap(); + let adj_vec = self.adj_list + [self.base_ei.base_op.probe_tuple.borrow()[self.vertex_index].id()] + .as_mut() + .unwrap(); let out_neighbour = &mut self.base_ei.out_neighbours; adj_vec.set_neighbor_ids(self.label_or_to_type, out_neighbour); self.base_ei.base_op.icost += out_neighbour.end_idx - out_neighbour.start_idx; @@ -91,7 +92,8 @@ impl CommonOperatorTrait for Extend { || self.base_ei.to_type == self.base_ei.vertex_types[out_neighbour.ids[idx].id()] { self.base_ei.base_op.num_out_tuples += 1; - self.base_ei.base_op.probe_tuple.borrow_mut()[self.base_ei.out_idx] = out_neighbour.ids[idx]; + self.base_ei.base_op.probe_tuple.borrow_mut()[self.base_ei.out_idx] = + out_neighbour.ids[idx]; self.base_ei.base_op.next[0] .borrow_mut() .process_new_tuple(); diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/build.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/build.rs index 824e98f3..7586ebff 100644 --- a/src/graph_impl/multi_graph/plan/operator/hashjoin/build.rs +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/build.rs @@ -14,7 +14,7 @@ use std::rc::Rc; #[derive(Clone)] pub struct Build { pub base_op: BaseOperator, - pub hash_table: Option>, + pub hash_table: Option>>>, pub probing_subgraph: Option, query_vertex_to_hash: String, pub build_hash_idx: usize, @@ -52,7 +52,9 @@ impl CommonOperatorTrait for Build { if self.base_op.probe_tuple.borrow().len() == 0 { self.base_op.probe_tuple = probe_tuple; self.hash_table.as_mut().map(|table| { - table.allocate_initial_memory(graph.node_count() + 1); + table + .borrow_mut() + .allocate_initial_memory(graph.node_count() + 1); }); } } @@ -60,7 +62,7 @@ impl CommonOperatorTrait for Build { fn process_new_tuple(&mut self) { let probe_tuple = self.base_op.probe_tuple.clone(); self.hash_table.as_mut().map(|table| { - table.insert_tuple(probe_tuple); + table.borrow_mut().insert_tuple(probe_tuple); }); } diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/hash_join.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/hash_join.rs index 08bb2755..c32de07e 100644 --- a/src/graph_impl/multi_graph/plan/operator/hashjoin/hash_join.rs +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/hash_join.rs @@ -76,7 +76,7 @@ impl HashJoin { .iter() .for_each(|(query_vertex, &idx)| { probe_qvertex_to_idx_map.insert( - query_vertex.clone(), + map[query_vertex].clone(), if idx < build_hash_idx { idx } else { idx - 1 }, ); }); @@ -98,7 +98,9 @@ impl HashJoin { let mut probe_indices = vec![0; join_qvertices.len() - 1]; let mut build_indices = vec![0; join_qvertices.len() - 1]; for (i, join_qvertex) in join_qvertices.iter().enumerate() { - if i < 1 { continue; } + if i < 1 { + continue; + } probe_indices[i - 1] = probe_qvertex_to_idx_map[join_qvertex].clone(); let mut other_build_idx = build_qvertex_to_idx_map[join_qvertex]; if build_hash_idx < other_build_idx { diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/hash_table.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/hash_table.rs index 917810b5..51c3f9f1 100644 --- a/src/graph_impl/multi_graph/plan/operator/hashjoin/hash_table.rs +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/hash_table.rs @@ -25,7 +25,7 @@ pub struct HashTable { blocks: Vec>, extra_blocks: Vec>, block_ids_and_chunk_offsets: Vec>, - pub(crate) num_chunks: Vec, + pub num_chunks: Vec, initial_num_blocks: usize, num_tuples_per_chunk: usize, num_chunks_per_block: usize, diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe.rs index 7c99d208..1ecc5585 100644 --- a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe.rs +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe.rs @@ -24,7 +24,7 @@ pub enum Probe { #[derive(Clone)] pub struct BaseProbe { pub base_op: BaseOperator, - pub hash_tables: Vec>, + pub hash_tables: Vec>>>, pub join_qvertices: Vec, pub probe_hash_idx: usize, pub hashed_tuple_len: usize, @@ -78,10 +78,14 @@ impl CommonOperatorTrait for BaseProbe { fn process_new_tuple(&mut self) { let hash_vertex = self.base_op.probe_tuple.borrow()[self.probe_hash_idx].id(); for hash_table in &mut self.hash_tables { - let last_chunk_idx = hash_table.num_chunks[hash_vertex]; + let last_chunk_idx = hash_table.borrow().num_chunks[hash_vertex]; let mut prev_first_item = -1i32; for chunk_idx in 0..last_chunk_idx { - hash_table.get_block_and_offsets(hash_vertex, chunk_idx, &mut self.block_info); + hash_table.borrow().get_block_and_offsets( + hash_vertex, + chunk_idx, + &mut self.block_info, + ); let mut offset = self.block_info.start_offset; while offset < self.block_info.end_offset { self.base_op.num_out_tuples += 1; @@ -89,7 +93,8 @@ impl CommonOperatorTrait for BaseProbe { let first_item = self.block_info.block[offset]; offset += 1; if prev_first_item != first_item.id() as i32 { - self.base_op.probe_tuple.borrow_mut()[self.probe_tuple_len] = first_item; + self.base_op.probe_tuple.borrow_mut()[self.probe_tuple_len] = + first_item; prev_first_item = first_item.id() as i32; } self.base_op.probe_tuple.borrow_mut()[self.probe_tuple_len + 1] = diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_cartesian.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_cartesian.rs index 8c957a2d..9913f0ee 100644 --- a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_cartesian.rs +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_cartesian.rs @@ -52,21 +52,26 @@ impl CommonOperatorTrait for ProbeCartesian { graph: &TypedStaticGraph, ) { if self.base_probe.base_op.probe_tuple.borrow().len() == 0 { - self.highest_vertex_id = graph.node_count() + 1; + self.highest_vertex_id = graph.node_count(); self.other_block_info = BlockInfo::empty(); } self.base_probe.init(probe_tuple, graph); } fn process_new_tuple(&mut self) { + self.base_probe.process_new_tuple(); + } + + fn execute(&mut self) { for a_hash_vertex in 0..self.highest_vertex_id { let base_probe = &mut self.base_probe; - base_probe.base_op.probe_tuple.borrow_mut()[base_probe.hashed_tuple_len] = Id::new(a_hash_vertex); + base_probe.base_op.probe_tuple.borrow_mut()[base_probe.hashed_tuple_len] = + Id::new(a_hash_vertex); for hash_table in base_probe.hash_tables.clone() { - let a_last_chunk_idx = hash_table.num_chunks[a_hash_vertex]; + let a_last_chunk_idx = hash_table.borrow().num_chunks[a_hash_vertex]; let mut a_prev_first_vertex = -1i32; for a_chunk_idx in 0..a_last_chunk_idx { - hash_table.get_block_and_offsets( + hash_table.borrow().get_block_and_offsets( a_hash_vertex, a_chunk_idx, &mut self.other_block_info, @@ -97,10 +102,6 @@ impl CommonOperatorTrait for ProbeCartesian { } } - fn execute(&mut self) { - self.base_probe.execute() - } - fn get_alds_as_string(&self) -> String { self.base_probe.get_alds_as_string() } diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices.rs index a0a550fd..cdfabf8e 100644 --- a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices.rs +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices.rs @@ -78,11 +78,12 @@ impl CommonOperatorTrait for ProbeMultiVertices { } fn process_new_tuple(&mut self) { - let hash_vertex = self.base_probe.base_op.probe_tuple.borrow()[self.base_probe.probe_hash_idx].id(); + let hash_vertex = + self.base_probe.base_op.probe_tuple.borrow()[self.base_probe.probe_hash_idx].id(); for hash_table in self.base_probe.hash_tables.clone() { - let last_chunk_idx = hash_table.num_chunks[hash_vertex]; + let last_chunk_idx = hash_table.borrow().num_chunks[hash_vertex]; for chunk_idx in 0..last_chunk_idx { - hash_table.get_block_and_offsets( + hash_table.borrow().get_block_and_offsets( hash_vertex, chunk_idx, &mut self.base_probe.block_info, diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices_cartesian.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices_cartesian.rs index aff24e8d..52b66b8a 100644 --- a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices_cartesian.rs +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe_multi_vertices_cartesian.rs @@ -60,7 +60,7 @@ impl CommonOperatorTrait for ProbeMultiVerticesCartesian { graph: &TypedStaticGraph, ) { if self.base_pmv.base_probe.base_op.probe_tuple.borrow().len() == 0 { - self.highest_vertex_id = graph.node_count() + 1; + self.highest_vertex_id = graph.node_count(); self.other_block_info = BlockInfo::empty(); } self.base_pmv.init(probe_tuple, graph); @@ -71,10 +71,10 @@ impl CommonOperatorTrait for ProbeMultiVerticesCartesian { self.base_pmv.base_probe.base_op.probe_tuple.borrow_mut() [self.base_pmv.base_probe.hashed_tuple_len] = Id::new(a_hash_vertex); for hash_table in self.base_pmv.base_probe.hash_tables.clone() { - let a_last_chunk_idx = hash_table.num_chunks[a_hash_vertex]; + let a_last_chunk_idx = hash_table.borrow().num_chunks[a_hash_vertex]; let mut a_prev_first_vertex = -1; for a_chunk_idx in 0..a_last_chunk_idx { - hash_table.get_block_and_offsets( + hash_table.borrow().get_block_and_offsets( a_hash_vertex, a_chunk_idx, &mut self.other_block_info, @@ -85,7 +85,8 @@ impl CommonOperatorTrait for ProbeMultiVerticesCartesian { let first_vertex = self.other_block_info.block[an_offset]; an_offset += 1; if a_prev_first_vertex != first_vertex.id() as i32 { - self.base_pmv.base_probe.base_op.probe_tuple.borrow_mut()[0] = first_vertex; + self.base_pmv.base_probe.base_op.probe_tuple.borrow_mut()[0] = + first_vertex; a_prev_first_vertex = first_vertex.id() as i32; } self.base_pmv.base_probe.base_op.probe_tuple.borrow_mut()[1] = diff --git a/src/graph_impl/multi_graph/plan/operator/scan/scan_blocking.rs b/src/graph_impl/multi_graph/plan/operator/scan/scan_blocking.rs index ba1202b4..a05b0e08 100644 --- a/src/graph_impl/multi_graph/plan/operator/scan/scan_blocking.rs +++ b/src/graph_impl/multi_graph/plan/operator/scan/scan_blocking.rs @@ -101,7 +101,8 @@ impl ScanBlocking { fn produce_new_edges_default(&mut self) { for from_idx in self.curr_from_idx + 1..self.from_idx_limit { let label = self.base_scan.label_or_to_type; - self.base_scan.base_op.probe_tuple.borrow_mut()[0] = self.base_scan.vertex_ids[from_idx]; + self.base_scan.base_op.probe_tuple.borrow_mut()[0] = + self.base_scan.vertex_ids[from_idx]; let to_vertex_idx_start = self.base_scan.fwd_adj_list[from_idx] .as_mut() .map_or(0, |adj| adj.get_offsets()[label as usize]); @@ -109,12 +110,14 @@ impl ScanBlocking { .as_mut() .map_or(0, |adj| adj.get_offsets()[(label + 1) as usize]); for to_idx in to_vertex_idx_start..to_vertex_idx_limit { - self.base_scan.base_op.probe_tuple.borrow_mut()[1] = self.base_scan.fwd_adj_list[from_idx] + self.base_scan.base_op.probe_tuple.borrow_mut()[1] = self.base_scan.fwd_adj_list + [from_idx] .as_mut() .unwrap() .get_neighbor_id(Id::new(to_idx)); if self.base_scan.to_type == KEY_ANY - || self.base_scan.vertex_types[self.base_scan.base_op.probe_tuple.borrow()[1].id()] + || self.base_scan.vertex_types + [self.base_scan.base_op.probe_tuple.borrow()[1].id()] == self.base_scan.to_type { self.base_scan.base_op.num_out_tuples += 1; diff --git a/src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs b/src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs index 4bfebf50..bdd47810 100644 --- a/src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs +++ b/src/graph_impl/multi_graph/plan/operator/scan/scan_sampling.rs @@ -6,11 +6,11 @@ use graph_impl::multi_graph::plan::operator::scan::scan::{BaseScan, Scan}; use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; +use itertools::Itertools; use rand::{thread_rng, Rng}; use std::cell::RefCell; use std::hash::Hash; use std::rc::Rc; -use itertools::Itertools; #[derive(Clone)] pub struct ScanSampling { diff --git a/src/graph_impl/multi_graph/plan/operator/sink/sink.rs b/src/graph_impl/multi_graph/plan/operator/sink/sink.rs index d9f8c8e7..ae5736af 100644 --- a/src/graph_impl/multi_graph/plan/operator/sink/sink.rs +++ b/src/graph_impl/multi_graph/plan/operator/sink/sink.rs @@ -111,7 +111,11 @@ impl CommonOperatorTrait for BaseSink { .map(|op| get_op_attr!(op.deref(), num_out_tuples)) .sum() } else { - self.base_op.num_out_tuples + let op = self.base_op.prev.as_ref().unwrap().borrow(); + get_op_attr!( + self.base_op.prev.as_ref().unwrap().borrow().deref(), + num_out_tuples + ) } } } diff --git a/src/graph_impl/multi_graph/plan/query_plan.rs b/src/graph_impl/multi_graph/plan/query_plan.rs index 9e7be775..9092f992 100644 --- a/src/graph_impl/multi_graph/plan/query_plan.rs +++ b/src/graph_impl/multi_graph/plan/query_plan.rs @@ -105,7 +105,7 @@ impl QueryPlan { pub fn new_from_subplans(subplans: Vec>>>) -> Self { Self { sink: None, - sink_type: SinkType::Copy, + sink_type: SinkType::Counter, scan_sampling: None, last_operator: subplans.get(subplans.len() - 1).map(|x| x.clone()), out_tuples_limit: 0, @@ -235,12 +235,9 @@ impl QueryPlan { } SinkType::Counter => Sink::BaseSink(BaseSink::new(query_graph.clone())), }))); - { - let mut sink = sink.borrow_mut(); - get_op_attr_as_mut!(sink.deref_mut(), prev).replace(last_operator.clone()); - } - *get_op_attr_as_mut!(self.subplans[plan_size - 1].borrow_mut().deref_mut(), next) = - vec![sink]; + get_op_attr_as_mut!(sink.borrow_mut().deref_mut(), prev).replace(last_operator.clone()); + *get_op_attr_as_mut!(last_operator.borrow_mut().deref_mut(), next) = vec![sink.clone()]; + self.sink = Some(sink); let mut probes = vec![]; for i in 1..self.subplans.len() { let mut operator = self.subplans[i].clone(); @@ -263,9 +260,13 @@ impl QueryPlan { operator = prev.unwrap(); } } - for subplan in &mut self.subplans { + for i in 0..self.subplans.len() - 1 { + let subplan = self.subplans.get_mut(i).unwrap(); if let Operator::Build(build) = subplan.borrow_mut().deref_mut() { - let hash_table = HashTable::new(build.build_hash_idx, build.hashed_tuple_len); + let hash_table = Rc::new(RefCell::new(HashTable::new( + build.build_hash_idx, + build.hashed_tuple_len, + ))); build.hash_table = Some(hash_table.clone()); for probe in &probes { let mut probe_mut = probe.borrow_mut(); @@ -283,30 +284,20 @@ impl QueryPlan { } } } - let sub_plans = self.subplans.clone(); - for subplan in sub_plans { - if let Operator::Build(build) = subplan.borrow().deref() { - let hash_table = build.hash_table.as_ref().unwrap(); - let subplan = subplan.borrow(); - let build_insubgrpah = get_op_attr_as_ref!(subplan.deref(), in_subgraph) - .as_ref() - .unwrap(); - self.init_hashtable(build_insubgrpah, hash_table); - } - } + for subplan in &mut self.subplans { - let probe_tuple = { - let subplan_ref = subplan.borrow(); - Rc::new(RefCell::new(vec![Id::new(0); get_op_attr!(subplan_ref.deref(), out_tuple_len)])) - }; + let probe_tuple = Rc::new(RefCell::new(vec![ + Id::new(0); + get_op_attr!( + subplan.borrow().deref(), + out_tuple_len + ) + ])); let mut first_op = subplan.clone(); loop { - let prev = { - let first_op_ref = first_op.borrow(); - get_op_attr_as_ref!(first_op_ref.deref(), prev) - .as_ref() - .map(|op| op.clone()) - }; + let prev = get_op_attr_as_ref!(first_op.borrow().deref(), prev) + .as_ref() + .map(|op| op.clone()); if prev.is_none() { break; } @@ -316,7 +307,11 @@ impl QueryPlan { } } - fn init_hashtable(&mut self, build_insubgrpah: &QueryGraph, hash_table: &HashTable) { + fn init_hashtable( + &mut self, + build_insubgrpah: &QueryGraph, + hash_table: Rc>>, + ) { for operator in &mut self.subplans { let mut op_mut = operator.borrow_mut(); if let Operator::Probe(_p) = op_mut.deref_mut() { @@ -353,7 +348,7 @@ impl QueryPlan { fn check_and_init( build_insubgrpah: &QueryGraph, probe: &mut Operator, - hash_table: HashTable, + hash_table: Rc>>, ) -> bool { let prob_insubgraph = get_op_attr_as_ref!(probe, in_subgraph).as_ref().unwrap(); if prob_insubgraph == build_insubgrpah { @@ -380,10 +375,12 @@ impl QueryPlan { } } else { let start_time = SystemTime::now(); - self.subplans - .iter_mut() - .map(|plan| plan.borrow_mut()) - .for_each(|mut plan| plan.execute()); + self.subplans.iter_mut().for_each(|mut plan| { + let mut op = plan.as_ptr(); + unsafe { + (&mut *op).execute(); + } + }); self.elapsed_time = SystemTime::now() .duration_since(start_time) .unwrap() diff --git a/src/graph_impl/multi_graph/planner/catalog/catalog.rs b/src/graph_impl/multi_graph/planner/catalog/catalog.rs index ade2fc74..5d42d0b1 100644 --- a/src/graph_impl/multi_graph/planner/catalog/catalog.rs +++ b/src/graph_impl/multi_graph/planner/catalog/catalog.rs @@ -180,18 +180,18 @@ impl Catalog { .filter(|ald| { vertex_mapping.is_none() || vertex_mapping - .unwrap() - .get(&ald.from_query_vertex) - .is_some() + .unwrap() + .get(&ald.from_query_vertex) + .is_some() }) .map(|ald| { "(".to_owned() + if vertex_mapping.is_none() { - &ald.from_query_vertex - } else { - let vertex_mapping = vertex_mapping.unwrap(); - vertex_mapping.get(&ald.from_query_vertex).unwrap() - } + &ald.from_query_vertex + } else { + let vertex_mapping = vertex_mapping.unwrap(); + vertex_mapping.get(&ald.from_query_vertex).unwrap() + } + ") " + &ald.direction.to_string() + "[" @@ -261,7 +261,10 @@ impl Catalog { query_plan_arr: &mut Vec>, ) { for query_plan in query_plan_arr { - let probe_tuple = Rc::new(RefCell::new(vec![Id::new(0); self.max_input_num_vertices + 1])); + let probe_tuple = Rc::new(RefCell::new(vec![ + Id::new(0); + self.max_input_num_vertices + 1 + ])); if let Some(scan) = &mut query_plan.scan_sampling { scan.borrow_mut().init(probe_tuple, graph); } @@ -278,9 +281,9 @@ impl Catalog { // sink.execute(); // })); } - // for handler in handlers { - // handler.join(); - // } + // for handler in handlers { + // handler.join(); + // } } else { let mut sink = query_plan_arr[0].sink.as_mut().unwrap().as_ptr(); unsafe { @@ -370,7 +373,7 @@ impl Catalog { for i in 0..next.len() { let next_i = next[i].borrow(); if let Operator::EI(EI::Intersect(Intersect::IntersectCatalog(intersect))) = - next_i.deref() + next_i.deref() { let to_type = intersect.base_intersect.base_ei.to_type; let mut alds_as_str_list = vec![]; @@ -416,12 +419,13 @@ impl Catalog { self.sampled_selectivity .get_mut(&subgraph_idx) .unwrap() - .insert(alds_as_str, - if num_input_tuples > 0 { - (selectivity as f64) / (num_input_tuples as f64) - } else { - 0.0 - }, + .insert( + alds_as_str, + if num_input_tuples > 0 { + (selectivity as f64) / (num_input_tuples as f64) + } else { + 0.0 + }, ); } let noop = { @@ -470,7 +474,7 @@ impl Catalog { for (i, next) in next_vec.iter().enumerate() { let next_ref = next.borrow(); if let Operator::EI(EI::Intersect(Intersect::IntersectCatalog(intersect))) = - next_ref.deref() + next_ref.deref() { let alds = &intersect.base_intersect.base_ei.alds; let mut alds_as_str_list = vec![]; @@ -556,12 +560,13 @@ impl Catalog { self.sampled_selectivity .get_mut(&subgraph_idx) .unwrap() - .insert(alds_as_str.to_owned() + "~" + &to_type.to_string(), - if num_input_tuples > 0 { - (selectivity as f64) / (num_input_tuples as f64) - } else { - 0.0 - }, + .insert( + alds_as_str.to_owned() + "~" + &to_type.to_string(), + if num_input_tuples > 0 { + (selectivity as f64) / (num_input_tuples as f64) + } else { + 0.0 + }, ); } let mut other_noops = vec![]; diff --git a/src/graph_impl/multi_graph/planner/catalog/catalog_plans.rs b/src/graph_impl/multi_graph/planner/catalog/catalog_plans.rs index fc5de0ca..860ca15c 100644 --- a/src/graph_impl/multi_graph/planner/catalog/catalog_plans.rs +++ b/src/graph_impl/multi_graph/planner/catalog/catalog_plans.rs @@ -20,11 +20,11 @@ use graph_impl::multi_graph::utils::set_utils; use graph_impl::static_graph::graph::KEY_ANY; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; +use itertools::Itertools; use std::cell::RefCell; use std::hash::Hash; use std::ops::{Deref, DerefMut}; use std::rc::Rc; -use itertools::Itertools; pub static DEF_NUM_EDGES_TO_SAMPLE: usize = 1000; pub static DEF_MAX_INPUT_NUM_VERTICES: usize = 3; @@ -76,9 +76,11 @@ impl CatalogPlans { let mut noop = Noop::new(scan.base_scan.base_op.out_subgraph.clone()); let scan_pointer = Rc::new(RefCell::new(Operator::Scan(Scan::ScanSampling(scan)))); noop.base_op.prev = Some(scan_pointer.clone()); - noop.base_op.out_qvertex_to_idx_map = get_op_attr_as_ref!(scan_pointer.borrow().deref(), out_qvertex_to_idx_map).clone(); + noop.base_op.out_qvertex_to_idx_map = + get_op_attr_as_ref!(scan_pointer.borrow().deref(), out_qvertex_to_idx_map).clone(); let mut noop_pointer = Rc::new(RefCell::new(Operator::Noop(noop))); - *get_op_attr_as_mut!(scan_pointer.borrow_mut().deref_mut(), next) = vec![noop_pointer.clone()]; + *get_op_attr_as_mut!(scan_pointer.borrow_mut().deref_mut(), next) = + vec![noop_pointer.clone()]; plans.set_next_operators(graph, noop_pointer, false); let mut query_plans_arr = vec![QueryPlan::new(scan_pointer.clone())]; for i in 1..num_thread { @@ -490,8 +492,10 @@ impl CatalogPlans { continue; } let mut num_edges_to_sample = (self.num_sampled_edges as f64 - * (graph.get_num_edges(from_type as i32, to_type as i32, edge_label as i32) as f64 - / graph.edge_count() as f64)) as usize; + * (graph.get_num_edges(from_type as i32, to_type as i32, edge_label as i32) + as f64 + / graph.edge_count() as f64)) + as usize; let mut scan = ScanSampling::new(out_subgraph); if self.sorted_by_node && num_edges_to_sample < 1000 { num_edges_to_sample = actual_num_edges; diff --git a/src/graph_impl/multi_graph/planner/catalog/operator/intersect_catalog.rs b/src/graph_impl/multi_graph/planner/catalog/operator/intersect_catalog.rs index 13528df0..64abd453 100644 --- a/src/graph_impl/multi_graph/planner/catalog/operator/intersect_catalog.rs +++ b/src/graph_impl/multi_graph/planner/catalog/operator/intersect_catalog.rs @@ -7,10 +7,10 @@ use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::static_graph::graph::KEY_ANY; use graph_impl::TypedStaticGraph; use hashbrown::HashMap; +use itertools::Itertools; use std::cell::RefCell; use std::hash::Hash; use std::rc::Rc; -use itertools::Itertools; #[derive(Clone)] pub struct IntersectCatalog { @@ -157,7 +157,8 @@ impl CommonOperatorTrait for IntersectCatalog { } for idx in base_ei.out_neighbours.start_idx..base_ei.out_neighbours.end_idx { - base_ei.base_op.probe_tuple.borrow_mut()[base_ei.out_idx] = base_ei.out_neighbours.ids[idx]; + base_ei.base_op.probe_tuple.borrow_mut()[base_ei.out_idx] = + base_ei.out_neighbours.ids[idx]; base_ei.base_op.num_out_tuples += 1; if self.is_adj_list_sorted_by_node { base_ei.base_op.next[0].borrow_mut().process_new_tuple(); @@ -166,7 +167,8 @@ impl CommonOperatorTrait for IntersectCatalog { .base_op .next .get( - base_ei.vertex_types[base_ei.base_op.probe_tuple.borrow()[base_ei.out_idx].id()] + base_ei.vertex_types + [base_ei.base_op.probe_tuple.borrow()[base_ei.out_idx].id()] as usize, ) .map(|next_op| next_op.borrow_mut().process_new_tuple()); diff --git a/src/graph_impl/multi_graph/planner/query_planner.rs b/src/graph_impl/multi_graph/planner/query_planner.rs index a3962461..9092c4d1 100644 --- a/src/graph_impl/multi_graph/planner/query_planner.rs +++ b/src/graph_impl/multi_graph/planner/query_planner.rs @@ -23,11 +23,11 @@ use graph_impl::multi_graph::planner::catalog::query_edge::QueryEdge; use graph_impl::multi_graph::planner::catalog::query_graph::QueryGraph; use graph_impl::TypedStaticGraph; use hashbrown::{HashMap, HashSet}; +use itertools::Itertools; use std::cell::RefCell; use std::hash::Hash; use std::ops::{Deref, DerefMut}; use std::rc::Rc; -use itertools::Itertools; pub struct QueryPlanner { subgraph_plans: HashMap>>>, @@ -119,7 +119,7 @@ impl let query_plan = QueryPlan::new_from_last_op(scan, num_edges as f64); let mut query_plans = vec![]; query_plans.push(query_plan); - let key = QueryPlanner::::get_key(vec![ + let key = QueryPlanner::::get_key(vec![ query_edge.from_query_vertex.clone(), query_edge.to_query_vertex.clone(), ]); @@ -136,7 +136,10 @@ impl self.subgraph_plans .entry(self.next_num_qvertices) .or_insert(HashMap::new()); - let keys: Vec = (&self.subgraph_plans[&(self.next_num_qvertices - 1)]).keys().map(|v| v.clone()).collect(); + let keys: Vec = (&self.subgraph_plans[&(self.next_num_qvertices - 1)]) + .keys() + .map(|v| v.clone()) + .collect(); for key in keys { self.consider_all_next_extend_operators(&key); } @@ -278,7 +281,10 @@ impl .collect(); new_query_plan.append(Rc::new(RefCell::new(Operator::EI(next_extend)))); new_query_plan.q_vertex_to_num_out_tuples = q_vertex_to_num_out_tuples; - (QueryPlanner::::get_key(query_vertices), new_query_plan) + ( + QueryPlanner::::get_key(query_vertices), + new_query_plan, + ) } fn get_next_ei( @@ -415,7 +421,7 @@ impl }); let rest_size = other_set.len(); - let rest_key = QueryPlanner::::get_key(other_set); + let rest_key = QueryPlanner::::get_key(other_set); if !self.subgraph_plans[&rest_size].contains_key(&rest_key) { continue; } @@ -458,22 +464,28 @@ impl ) { let is_plan_build_subplan = subplan.estimated_num_out_tuples < other_subplan.estimated_num_out_tuples; - let (build_subplan,probe_subplan) = if is_plan_build_subplan { - (subplan,other_subplan) + let (build_subplan, probe_subplan) = if is_plan_build_subplan { + (subplan, other_subplan) } else { - (other_subplan,subplan) + (other_subplan, subplan) }; - let (build_coef,probe_coef) = if num_join_qvertices == 1 { - (SINGLE_VERTEX_WEIGHT_BUILD_COEF,SINGLE_VERTEX_WEIGHT_PROBE_COEF) + let (build_coef, probe_coef) = if num_join_qvertices == 1 { + ( + SINGLE_VERTEX_WEIGHT_BUILD_COEF, + SINGLE_VERTEX_WEIGHT_PROBE_COEF, + ) } else { - (MULTI_VERTEX_WEIGHT_BUILD_COEF,MULTI_VERTEX_WEIGHT_PROBE_COEF) + ( + MULTI_VERTEX_WEIGHT_BUILD_COEF, + MULTI_VERTEX_WEIGHT_PROBE_COEF, + ) }; let icost = build_subplan.estimated_icost + probe_subplan.estimated_icost + build_coef * build_subplan.estimated_num_out_tuples + probe_coef * probe_subplan.estimated_num_out_tuples; - let key = QueryPlanner::::get_key(query_vertices.clone()); + let key = QueryPlanner::::get_key(query_vertices.clone()); let curr_best_query_plan = self.get_best_plan(query_vertices.len(), &key); if curr_best_query_plan.estimated_icost > icost { let mut query_plan = HashJoin::make( diff --git a/tests/static_graph.rs b/tests/static_graph.rs index e78b1feb..7486d5ae 100644 --- a/tests/static_graph.rs +++ b/tests/static_graph.rs @@ -323,11 +323,9 @@ fn test_clone() { #[test] fn test_graphflow_planner() { let mut g_: TypedGraphMap = TypedDiGraphMap::new(); - let path_to_nodes = - Path::new("C:\\Users\\76155\\Desktop\\rust_graphflow\\human-vertices.csv"); - // Path::new("C:\\Users\\cheny\\OneDrive\\桌面\\rust_graphflow\\human-vertices.csv"); - let path_to_edges = - Path::new("C:\\Users\\76155\\Desktop\\rust_graphflow\\human-edges.csv"); + let path_to_nodes = Path::new("C:\\Users\\76155\\Desktop\\rust_graphflow\\human-vertices.csv"); + // Path::new("C:\\Users\\cheny\\OneDrive\\桌面\\rust_graphflow\\human-vertices.csv"); + let path_to_edges = Path::new("C:\\Users\\76155\\Desktop\\rust_graphflow\\human-edges.csv"); // Path::new("C:\\Users\\cheny\\OneDrive\\桌面\\rust_graphflow\\human-edges.csv"); read_from_csv( &mut g_, @@ -437,7 +435,11 @@ fn test_graphflow_planner() { }; let catalog = catalog_generator::default(&g); let mut query_plan = optimizer_executor::generate_plan(query_graph, catalog, g.clone()); - println!("QueryPlan:{}", query_plan.get_output_log()); + // println!("QueryPlan:{}", query_plan.get_output_log()); + // println!( + // "num_out_tuple={},icost={}", + // query_plan.estimated_num_out_tuples, query_plan.estimated_icost + // ); let mut workers = QPWorkers::new(query_plan, 1); workers.init(&g); workers.execute(); From 8e5d2e36c3377302445a428a46ca699f91dbc92d Mon Sep 17 00:00:00 2001 From: tangtang <761552682@qq.com> Date: Sat, 7 Mar 2020 21:21:11 +0800 Subject: [PATCH 25/25] fix:execute --- .../multi_graph/plan/operator/hashjoin/build.rs | 2 +- .../multi_graph/plan/operator/hashjoin/probe.rs | 1 + src/graph_impl/multi_graph/plan/operator/scan/scan.rs | 10 +++++----- src/graph_impl/static_graph/graph.rs | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/build.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/build.rs index 7586ebff..60ffb028 100644 --- a/src/graph_impl/multi_graph/plan/operator/hashjoin/build.rs +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/build.rs @@ -8,7 +8,7 @@ use graph_impl::TypedStaticGraph; use hashbrown::HashMap; use std::cell::RefCell; use std::hash::Hash; -use std::ops::DerefMut; +use std::ops::{DerefMut, Deref}; use std::rc::Rc; #[derive(Clone)] diff --git a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe.rs b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe.rs index 1ecc5585..3347dda0 100644 --- a/src/graph_impl/multi_graph/plan/operator/hashjoin/probe.rs +++ b/src/graph_impl/multi_graph/plan/operator/hashjoin/probe.rs @@ -13,6 +13,7 @@ use std::cell::RefCell; use std::hash::Hash; use std::ops::{Deref, DerefMut}; use std::rc::Rc; +use itertools::Itertools; #[derive(Clone)] pub enum Probe { diff --git a/src/graph_impl/multi_graph/plan/operator/scan/scan.rs b/src/graph_impl/multi_graph/plan/operator/scan/scan.rs index f9384b7f..840ecca2 100644 --- a/src/graph_impl/multi_graph/plan/operator/scan/scan.rs +++ b/src/graph_impl/multi_graph/plan/operator/scan/scan.rs @@ -12,7 +12,7 @@ use graph_impl::TypedStaticGraph; use hashbrown::HashMap; use std::cell::RefCell; use std::hash::Hash; -use std::ops::DerefMut; +use std::ops::{DerefMut, Deref}; use std::rc::Rc; #[derive(Clone)] @@ -108,13 +108,13 @@ impl CommonOperatorTrait for BaseScan { fn execute(&mut self) { for from_idx in self.from_vertex_start_idx..self.from_vertex_end_idx { let from_vertex = self.vertex_ids[from_idx]; - self.base_op.probe_tuple.borrow_mut()[0] = from_vertex; - let to_vertex_start_idx = self.fwd_adj_list[from_idx].as_ref().unwrap().get_offsets() + self.base_op.probe_tuple.borrow_mut()[0] = from_vertex.clone(); + let to_vertex_start_idx = self.fwd_adj_list[from_vertex.id()].as_ref().unwrap().get_offsets() [self.label_or_to_type as usize]; - let to_vertex_end_idx = self.fwd_adj_list[from_idx].as_ref().unwrap().get_offsets() + let to_vertex_end_idx = self.fwd_adj_list[from_vertex.id()].as_ref().unwrap().get_offsets() [(self.label_or_to_type + 1) as usize]; for to_idx in to_vertex_start_idx..to_vertex_end_idx { - self.base_op.probe_tuple.borrow_mut()[1] = self.fwd_adj_list[from_idx] + self.base_op.probe_tuple.borrow_mut()[1] = self.fwd_adj_list[from_vertex.id()] .as_ref() .unwrap() .get_neighbor_id(Id::new(to_idx)); diff --git a/src/graph_impl/static_graph/graph.rs b/src/graph_impl/static_graph/graph.rs index 5570af48..bcd3d370 100644 --- a/src/graph_impl/static_graph/graph.rs +++ b/src/graph_impl/static_graph/graph.rs @@ -75,7 +75,7 @@ pub struct TypedStaticGraph, node_type_offsets: Vec, - fwd_adj_lists: Vec>>, + pub fwd_adj_lists: Vec>>, bwd_adj_lists: Vec>>, label_to_num_edges: Vec, label_to_largest_fwd_adj_list_size: Vec,