chore: add ReferenceSerialization for DataValue::Tuple & fix subquery plan on ReferenceSerialization

KKould · KKould · commit 8a2af1d66cf7 · 2024-10-21T23:53:25.000+08:00
diff --git a/src/binder/expr.rs b/src/binder/expr.rs
@@ -231,7 +231,7 @@ impl<'a, 'b, T: Transaction> Binder<'a, 'b, T> {
         sub_query: LogicalPlan,
     ) -> Result<(ScalarExpression, LogicalPlan), DatabaseError> {
         let mut alias_column = ColumnCatalog::clone(&column);
-        alias_column.set_ref_table(self.context.temp_table(), ColumnId::new());
+        alias_column.set_ref_table(self.context.temp_table(), ColumnId::new(), true);
 
         let alias_expr = ScalarExpression::Alias {
             expr: Box::new(ScalarExpression::ColumnRef(column)),
diff --git a/src/binder/select.rs b/src/binder/select.rs
@@ -353,7 +353,11 @@ impl<'a: 'b, 'b, T: Transaction> Binder<'a, 'b, T> {
         for (alias, column) in aliases_with_columns {
             let mut alias_column = ColumnCatalog::clone(&column);
             alias_column.set_name(alias.clone());
-            alias_column.set_ref_table(table_alias.clone(), column.id().unwrap_or(ColumnId::new()));
+            alias_column.set_ref_table(
+                table_alias.clone(),
+                column.id().unwrap_or(ColumnId::new()),
+                true,
+            );
 
             let alias_column_expr = ScalarExpression::Alias {
                 expr: Box::new(ScalarExpression::ColumnRef(column)),
diff --git a/src/catalog/column.rs b/src/catalog/column.rs
@@ -41,6 +41,7 @@ pub enum ColumnRelation {
     Table {
         column_id: ColumnId,
         table_name: TableName,
+        is_temp: bool,
     },
 }
 
@@ -145,10 +146,11 @@ impl ColumnCatalog {
         self.summary.name = name;
     }
 
-    pub fn set_ref_table(&mut self, table_name: TableName, column_id: ColumnId) {
+    pub fn set_ref_table(&mut self, table_name: TableName, column_id: ColumnId, is_temp: bool) {
         self.summary.relation = ColumnRelation::Table {
             column_id,
             table_name,
+            is_temp,
         };
     }
 
diff --git a/src/catalog/table.rs b/src/catalog/table.rs
@@ -100,6 +100,7 @@ impl TableCatalog {
         col.summary_mut().relation = ColumnRelation::Table {
             column_id: col_id,
             table_name: self.name.clone(),
+            is_temp: false,
         };
 
         self.column_idxs
diff --git a/src/db.rs b/src/db.rs
@@ -400,7 +400,7 @@ pub(crate) mod test {
             ColumnDesc::new(LogicalType::Integer, false, false, None).unwrap(),
         );
         let number_column_id = schema[0].id().unwrap();
-        column.set_ref_table(Arc::new("a".to_string()), number_column_id);
+        column.set_ref_table(Arc::new("a".to_string()), number_column_id, true);
 
         debug_assert_eq!(schema, Arc::new(vec![ColumnRef::from(column)]));
         debug_assert_eq!(
diff --git a/src/execution/dml/analyze.rs b/src/execution/dml/analyze.rs
@@ -159,6 +159,7 @@ mod test {
     use crate::errors::DatabaseError;
     use crate::execution::dml::analyze::{DEFAULT_NUM_OF_BUCKETS, DEFAULT_STATISTICS_META_PATH};
     use crate::optimizer::core::statistics_meta::StatisticsMeta;
+    use crate::storage::rocksdb::RocksTransaction;
     use std::ffi::OsStr;
     use std::fs;
     use tempfile::TempDir;
@@ -196,17 +197,17 @@ mod test {
         }
         paths.sort();
 
-        let statistics_meta_pk_index = StatisticsMeta::from_file(&paths[0])?;
+        let statistics_meta_pk_index = StatisticsMeta::from_file::<RocksTransaction>(&paths[0])?;
 
         assert_eq!(statistics_meta_pk_index.index_id(), 0);
         assert_eq!(statistics_meta_pk_index.histogram().values_len(), 101);
 
-        let statistics_meta_b_index = StatisticsMeta::from_file(&paths[1])?;
+        let statistics_meta_b_index = StatisticsMeta::from_file::<RocksTransaction>(&paths[1])?;
 
         assert_eq!(statistics_meta_b_index.index_id(), 1);
         assert_eq!(statistics_meta_b_index.histogram().values_len(), 101);
 
-        let statistics_meta_p_index = StatisticsMeta::from_file(&paths[2])?;
+        let statistics_meta_p_index = StatisticsMeta::from_file::<RocksTransaction>(&paths[2])?;
 
         assert_eq!(statistics_meta_p_index.index_id(), 2);
         assert_eq!(statistics_meta_p_index.histogram().values_len(), 101);
diff --git a/src/execution/dml/copy_from_file.rs b/src/execution/dml/copy_from_file.rs
@@ -137,6 +137,7 @@ mod tests {
                     relation: ColumnRelation::Table {
                         column_id: Ulid::new(),
                         table_name: Arc::new("t1".to_string()),
+                        is_temp: false,
                     },
                 },
                 false,
@@ -149,6 +150,7 @@ mod tests {
                     relation: ColumnRelation::Table {
                         column_id: Ulid::new(),
                         table_name: Arc::new("t1".to_string()),
+                        is_temp: false,
                     },
                 },
                 false,
@@ -161,6 +163,7 @@ mod tests {
                     relation: ColumnRelation::Table {
                         column_id: Ulid::new(),
                         table_name: Arc::new("t1".to_string()),
+                        is_temp: false,
                     },
                 },
                 false,
diff --git a/src/expression/mod.rs b/src/expression/mod.rs
@@ -1382,6 +1382,7 @@ mod test {
                     relation: ColumnRelation::Table {
                         column_id: c3_column_id,
                         table_name: Arc::new("t1".to_string()),
+                        is_temp: false,
                     },
                 },
                 false,
diff --git a/src/optimizer/core/cm_sketch.rs b/src/optimizer/core/cm_sketch.rs
@@ -4,7 +4,6 @@ use crate::serdes::{ReferenceSerialization, ReferenceTables};
 use crate::storage::{TableCache, Transaction};
 use crate::types::value::DataValue;
 use rand::RngCore;
-use serde::{Deserialize, Serialize};
 use siphasher::sip::SipHasher13;
 use std::borrow::Borrow;
 use std::hash::{Hash, Hasher};
@@ -15,7 +14,7 @@ use std::{cmp, mem};
 pub(crate) type FastHasher = SipHasher13;
 
 // https://github.com/jedisct1/rust-count-min-sketch
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone)]
 pub struct CountMinSketch<K> {
     counters: Vec<Vec<usize>>,
     offsets: Vec<usize>,
diff --git a/src/optimizer/core/histogram.rs b/src/optimizer/core/histogram.rs
@@ -8,7 +8,6 @@ use crate::types::index::{IndexId, IndexMeta};
 use crate::types::value::{DataValue, ValueRef};
 use crate::types::LogicalType;
 use ordered_float::OrderedFloat;
-use serde::{Deserialize, Serialize};
 use serde_macros::ReferenceSerialization;
 use std::collections::Bound;
 use std::sync::Arc;
@@ -25,7 +24,7 @@ pub struct HistogramBuilder {
 }
 
 // Equal depth histogram
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ReferenceSerialization)]
+#[derive(Debug, Clone, PartialEq, ReferenceSerialization)]
 pub struct Histogram {
     index_id: IndexId,
 
@@ -40,7 +39,7 @@ pub struct Histogram {
     correlation: f64,
 }
 
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ReferenceSerialization)]
+#[derive(Debug, Clone, PartialEq, ReferenceSerialization)]
 struct Bucket {
     lower: ValueRef,
     upper: ValueRef,
diff --git a/src/optimizer/core/statistics_meta.rs b/src/optimizer/core/statistics_meta.rs
@@ -3,10 +3,10 @@ use crate::errors::DatabaseError;
 use crate::expression::range_detacher::Range;
 use crate::optimizer::core::cm_sketch::CountMinSketch;
 use crate::optimizer::core::histogram::Histogram;
+use crate::serdes::{ReferenceSerialization, ReferenceTables};
 use crate::storage::{StatisticsMetaCache, Transaction};
 use crate::types::index::IndexId;
 use crate::types::value::DataValue;
-use serde::{Deserialize, Serialize};
 use serde_macros::ReferenceSerialization;
 use std::fs::OpenOptions;
 use std::io::Write;
@@ -35,17 +35,16 @@ impl<'a, T: Transaction> StatisticMetaLoader<'a, T> {
             return Ok(Some(statistics_meta));
         }
         if let Some(path) = self.tx.table_meta_path(table_name.as_str(), index_id)? {
-            Ok(Some(
-                self.cache
-                    .get_or_insert(key, |_| StatisticsMeta::from_file(path))?,
-            ))
+            Ok(Some(self.cache.get_or_insert(key, |_| {
+                StatisticsMeta::from_file::<T>(path)
+            })?))
         } else {
             Ok(None)
         }
     }
 }
 
-#[derive(Debug, Serialize, Deserialize, ReferenceSerialization)]
+#[derive(Debug, ReferenceSerialization)]
 pub struct StatisticsMeta {
     index_id: IndexId,
     histogram: Histogram,
@@ -86,20 +85,20 @@ impl StatisticsMeta {
             .read(true)
             .truncate(false)
             .open(path)?;
-        bincode::serialize_into(&mut file, self)?;
+        self.encode(&mut file, true, &mut ReferenceTables::new())?;
         file.flush()?;
 
         Ok(())
     }
 
-    pub fn from_file(path: impl AsRef<Path>) -> Result<Self, DatabaseError> {
-        let file = OpenOptions::new()
+    pub fn from_file<T: Transaction>(path: impl AsRef<Path>) -> Result<Self, DatabaseError> {
+        let mut file = OpenOptions::new()
             .create(true)
             .write(true)
             .read(true)
             .truncate(false)
             .open(path)?;
-        Ok(bincode::deserialize_from(file)?)
+        Self::decode::<T, _>(&mut file, None, &ReferenceTables::new())
     }
 }
 
@@ -108,6 +107,7 @@ mod tests {
     use crate::errors::DatabaseError;
     use crate::optimizer::core::histogram::HistogramBuilder;
     use crate::optimizer::core::statistics_meta::StatisticsMeta;
+    use crate::storage::rocksdb::RocksTransaction;
     use crate::types::index::{IndexMeta, IndexType};
     use crate::types::value::DataValue;
     use crate::types::LogicalType;
@@ -154,7 +154,7 @@ mod tests {
         let path = temp_dir.path().join("meta");
 
         StatisticsMeta::new(histogram.clone(), sketch.clone()).to_file(path.clone())?;
-        let statistics_meta = StatisticsMeta::from_file(path)?;
+        let statistics_meta = StatisticsMeta::from_file::<RocksTransaction>(path)?;
 
         debug_assert_eq!(histogram, statistics_meta.histogram);
         debug_assert_eq!(
diff --git a/src/optimizer/rule/normalization/simplification.rs b/src/optimizer/rule/normalization/simplification.rs
@@ -256,6 +256,7 @@ mod test {
                     relation: ColumnRelation::Table {
                         column_id: *table_state.column_id_by_name("c1"),
                         table_name: Arc::new("t1".to_string()),
+                        is_temp: false,
                     },
                 },
                 false,
@@ -273,6 +274,7 @@ mod test {
                     relation: ColumnRelation::Table {
                         column_id: *table_state.column_id_by_name("c2"),
                         table_name: Arc::new("t1".to_string()),
+                        is_temp: false,
                     },
                 },
                 false,
diff --git a/src/serdes/column.rs b/src/serdes/column.rs
@@ -18,7 +18,12 @@ impl ReferenceSerialization for ColumnRef {
             .then(|| self.nullable())
             .encode(writer, is_direct, reference_tables)?;
 
-        if is_direct || matches!(self.summary().relation, ColumnRelation::None) {
+        if is_direct
+            || !matches!(
+                self.summary().relation,
+                ColumnRelation::Table { is_temp: false, .. }
+            )
+        {
             self.nullable()
                 .encode(writer, is_direct, reference_tables)?;
             self.desc().encode(writer, is_direct, reference_tables)?;
@@ -39,6 +44,7 @@ impl ReferenceSerialization for ColumnRef {
             ColumnRelation::Table {
                 column_id,
                 table_name,
+                is_temp: false,
             },
             Some((transaction, table_cache)),
         ) = (&summary.relation, drive)
@@ -85,10 +91,11 @@ impl ReferenceSerialization for ColumnRelation {
             ColumnRelation::Table {
                 column_id,
                 table_name,
+                is_temp,
             } => {
                 writer.write_all(&[1])?;
-
                 column_id.encode(writer, is_direct, reference_tables)?;
+                is_temp.encode(writer, is_direct, reference_tables)?;
 
                 reference_tables.push_or_replace(table_name).encode(
                     writer,
@@ -113,6 +120,7 @@ impl ReferenceSerialization for ColumnRelation {
             0 => ColumnRelation::None,
             1 => {
                 let column_id = ColumnId::decode(reader, drive, reference_tables)?;
+                let is_temp = bool::decode(reader, drive, reference_tables)?;
                 let table_name = reference_tables
                     .get(<usize as ReferenceSerialization>::decode(
                         reader,
@@ -124,6 +132,7 @@ impl ReferenceSerialization for ColumnRelation {
                 ColumnRelation::Table {
                     column_id,
                     table_name,
+                    is_temp,
                 }
             }
             _ => unreachable!(),
@@ -177,6 +186,7 @@ pub(crate) mod test {
                     relation: ColumnRelation::Table {
                         column_id: c3_column_id,
                         table_name: table_name.clone(),
+                        is_temp: false,
                     },
                 },
                 false,
@@ -253,6 +263,7 @@ pub(crate) mod test {
             relation: ColumnRelation::Table {
                 column_id: Ulid::new(),
                 table_name: Arc::new("t1".to_string()),
+                is_temp: false,
             },
         };
         summary.encode(&mut cursor, false, &mut reference_tables)?;
@@ -288,6 +299,7 @@ pub(crate) mod test {
         let table_relation = ColumnRelation::Table {
             column_id: Ulid::new(),
             table_name: Arc::new("t1".to_string()),
+            is_temp: false,
         };
         table_relation.encode(&mut cursor, false, &mut reference_tables)?;
         cursor.seek(SeekFrom::Start(0))?;
diff --git a/src/serdes/data_value.rs b/src/serdes/data_value.rs
diff --git a/src/storage/mod.rs b/src/storage/mod.rs
diff --git a/src/storage/table_codec.rs b/src/storage/table_codec.rs
diff --git a/src/types/value.rs b/src/types/value.rs

Original file line number	Diff line number	Diff line change
`@@ -41,6 +41,7 @@ pub enum ColumnRelation {`
`41`	`41`	`Table {`
`42`	`42`	`column_id: ColumnId,`
`43`	`43`	`table_name: TableName,`
	`44`	`+ is_temp: bool,`
`44`	`45`	`},`
`45`	`46`	`}`
`46`	`47`
`@@ -145,10 +146,11 @@ impl ColumnCatalog {`
`145`	`146`	`self.summary.name = name;`
`146`	`147`	`}`
`147`	`148`
`148`		`- pub fn set_ref_table(&mut self, table_name: TableName, column_id: ColumnId) {`
	`149`	`+ pub fn set_ref_table(&mut self, table_name: TableName, column_id: ColumnId, is_temp: bool) {`
`149`	`150`	`self.summary.relation = ColumnRelation::Table {`
`150`	`151`	`column_id,`
`151`	`152`	`table_name,`
	`153`	`+ is_temp,`
`152`	`154`	`};`
`153`	`155`	`}`
`154`	`156`