ut: add serialize/deserialize tests for spec

crates/paimon/src/spec/data_file.rs

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use crate::spec::RowType;
+use crate::spec::{BinaryTableStats, RowType};
 use chrono::{DateTime, Utc};
 use serde::{Deserialize, Serialize};
 use std::fmt::{Display, Formatter};
@@ -48,24 +48,6 @@ impl BinaryRow {
     }
 }
 
-/// TODO: implement me.
-/// The statistics for columns, supports the following stats.
-///
-/// Impl References: <https://github.com/apache/paimon/blob/release-0.8.2/paimon-core/src/main/java/org/apache/paimon/stats/SimpleStats.java>
-type SimpleStats = ();
-
-/// The Source of a file.
-/// TODO: move me to the manifest module.
-///
-/// Impl References: <https://github.com/apache/paimon/blob/release-0.8.2/paimon-core/src/main/java/org/apache/paimon/manifest/FileSource.java>
-#[repr(u8)]
-#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub enum FileSource {
-    Append = 0,
-    Compact = 1,
-}
-
 /// Metadata of a data file.
 ///
 /// Impl References: <https://github.com/apache/paimon/blob/release-0.8.2/paimon-core/src/main/java/org/apache/paimon/io/DataFileMeta.java>
@@ -78,8 +60,8 @@ pub struct DataFileMeta {
     pub row_count: i64,
     pub min_key: BinaryRow,
     pub max_key: BinaryRow,
-    pub key_stats: SimpleStats,
-    pub value_stats: SimpleStats,
+    pub key_stats: Option<BinaryTableStats>,
+    pub value_stats: Option<BinaryTableStats>,
     pub min_sequence_number: i64,
     pub max_sequence_number: i64,
     pub schema_id: i64,
@@ -90,16 +72,173 @@ pub struct DataFileMeta {
     pub delete_row_count: Option<i64>,
     // file index filter bytes, if it is small, store in data file meta
     pub embedded_index: Option<Vec<u8>>,
-    pub file_source: Option<FileSource>,
 }
 
 impl Display for DataFileMeta {
-    fn fmt(&self, _: &mut Formatter<'_>) -> std::fmt::Result {
-        todo!()
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "{{fileName: {}, fileSize: {}, rowCount: {}, embeddedIndex: {:?}, \
+            minKey: {:?}, maxKey: {:?}, keyStats: {:?}, valueStats: {:?}, \
+            minSequenceNumber: {}, maxSequenceNumber: {}, \
+            schemaId: {}, level: {}, extraFiles: {:?}, creationTime: {}, deleteRowCount: {:?}}}",
+            self.file_name,
+            self.file_size,
+            self.row_count,
+            self.embedded_index,
+            self.min_key,
+            self.max_key,
+            self.key_stats,
+            self.value_stats,
+            self.min_sequence_number,
+            self.max_sequence_number,
+            self.schema_id,
+            self.level,
+            self.extra_files,
+            self.creation_time,
+            self.delete_row_count
+        )
     }
 }
 
 impl DataFileMeta {
-    // TODO: implement me
     pub const SCHEMA: RowType = RowType::new(vec![]);
+
+    /// Get the file name.
+    pub fn file_name(&self) -> &str {
+        &self.file_name
+    }
+
+    /// Get the file size.
+    pub fn file_size(&self) -> i64 {
+        self.file_size
+    }
+
+    /// Get the row count.
+    pub fn row_count(&self) -> i64 {
+        self.row_count
+    }
+
+    /// Get the min key.
+    pub fn min_key(&self) -> &BinaryRow {
+        &self.min_key
+    }
+
+    /// Get the max key.
+    pub fn max_key(&self) -> &BinaryRow {
+        &self.max_key
+    }
+
+    /// Get the key stats.
+    pub fn key_stats(&self) -> Option<&BinaryTableStats> {
+        self.key_stats.as_ref()
+    }
+
+    /// Get the value stats.
+    pub fn value_stats(&self) -> Option<&BinaryTableStats> {
+        self.value_stats.as_ref()
+    }
+
+    /// Get the min sequence number.
+    pub fn min_sequence_number(&self) -> i64 {
+        self.min_sequence_number
+    }
+
+    /// Get the max sequence number.
+    pub fn max_sequence_number(&self) -> i64 {
+        self.max_sequence_number
+    }
+
+    /// Get the schema id.
+    pub fn schema_id(&self) -> i64 {
+        self.schema_id
+    }
+
+    /// Get the level.
+    pub fn level(&self) -> i32 {
+        self.level
+    }
+
+    /// Get the extra files.
+    pub fn extra_files(&self) -> &[String] {
+        &self.extra_files
+    }
+
+    /// Get the creation time.
+    pub fn creation_time(&self) -> DateTime<Utc> {
+        self.creation_time
+    }
+
+    /// Get the delete row count.
+    pub fn delete_row_count(&self) -> Option<i64> {
+        self.delete_row_count
+    }
+
+    /// Get the embedded index.
+    pub fn embedded_index(&self) -> Option<&[u8]> {
+        self.embedded_index.as_deref()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_data_file_meta_serialize_deserialize() {
+        let json_data = r#"
+        {
+           "fileName":"test.avro",
+           "fileSize":1024,
+           "rowCount":100,
+           "minKey":{
+              "arity":1,
+              "nullBitsSizeInBytes":1
+           },
+           "maxKey":{
+              "arity":10,
+              "nullBitsSizeInBytes":2
+           },
+           "keyStats":null,
+           "valueStats":null,
+           "minSequenceNumber":0,
+           "maxSequenceNumber":100,
+           "schemaId":0,
+           "level":0,
+           "extraFiles":[],
+           "creationTime":"2024-08-13T02:03:03.106490600Z",
+           "deleteRowCount":5,
+           "embeddedIndex":null
+        }
+        "#;
+
+        let data_file_meta: DataFileMeta =
+            serde_json::from_str(json_data).expect("Failed to deserialize DataFileMeta");
+
+        assert_eq!(data_file_meta.file_name, "test.avro");
+        assert_eq!(data_file_meta.file_size, 1024);
+        assert_eq!(data_file_meta.row_count, 100);
+
+        assert_eq!(data_file_meta.min_key.arity, 1);
+        assert_eq!(data_file_meta.min_key.null_bits_size_in_bytes, 1);
+        assert_eq!(data_file_meta.max_key.arity, 10);
+        assert_eq!(data_file_meta.max_key.null_bits_size_in_bytes, 2);
+
+        assert!(data_file_meta.key_stats.is_none());
+        assert!(data_file_meta.value_stats.is_none());
+
+        assert_eq!(data_file_meta.min_sequence_number, 0);
+        assert_eq!(data_file_meta.max_sequence_number, 100);
+        assert_eq!(data_file_meta.schema_id, 0);
+        assert_eq!(data_file_meta.level, 0);
+        assert_eq!(data_file_meta.extra_files.len(), 0);
+        assert_eq!(
+            data_file_meta.creation_time,
+            DateTime::parse_from_rfc3339("2024-08-13T02:03:03.106490600Z")
+                .unwrap()
+                .with_timezone(&Utc)
+        );
+        assert_eq!(data_file_meta.delete_row_count, Some(5));
+        assert!(data_file_meta.embedded_index.is_none());
+    }
 }

crates/paimon/src/spec/manifest_file_meta.rs

@@ -16,7 +16,6 @@
 // under the License.
 
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
-use serde_bytes::Bytes;
 use std::fmt::{Display, Formatter};
 
 /// Metadata of a manifest file.
@@ -159,16 +158,14 @@ where
     for &num in value {
         bytes.extend_from_slice(&num.to_le_bytes());
     }
-
-    let bytes = Bytes::new(bytes.as_slice());
-    serializer.serialize_bytes(bytes)
+    serializer.serialize_bytes(&bytes)
 }
 
 fn deserialize_null_counts<'de, D>(deserializer: D) -> Result<Vec<i64>, D::Error>
 where
     D: Deserializer<'de>,
 {
-    let bytes = Deserialize::deserialize(deserializer).map(Bytes::new)?;
+    let bytes: Vec<u8> = Deserialize::deserialize(deserializer)?;
 
     let size_of_i64 = std::mem::size_of::<i64>();
     let i64_count = bytes.len() / size_of_i64;
@@ -180,3 +177,38 @@ where
     }
     Ok(i64s)
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_manifest_file_meta_serialize_deserialize() {
+        let data_json = r#"
+        {
+           "_FILE_NAME":"manifest_file_meta.avro",
+           "_FILE_SIZE":1024,
+           "_NUM_ADDED_FILES":5,
+           "_NUM_DELETED_FILES":6,
+           "_PARTITION_STATS":{
+              "_MIN_VALUES":[0, 1, 2],
+              "_MAX_VALUES":[3, 4, 5],
+              "_NULL_COUNTS":[6, 7, 8]
+           },
+           "_SCHEMA_ID":1
+        }
+        "#;
+
+        let manifest_file_meta: ManifestFileMeta =
+            serde_json::from_str(data_json).expect("Failed to deserialize ManifestFileMeta.");
+
+        assert_eq!(manifest_file_meta.file_name(), "manifest_file_meta.avro");
+        assert_eq!(manifest_file_meta.file_size, 1024);
+        assert_eq!(manifest_file_meta.num_added_files, 5);
+        assert_eq!(manifest_file_meta.num_deleted_files, 6);
+        assert_eq!(manifest_file_meta.partition_stats.min_values, vec![0, 1, 2]);
+        assert_eq!(manifest_file_meta.partition_stats.max_values, vec![3, 4, 5]);
+        assert_eq!(manifest_file_meta.partition_stats.null_counts.len(), 0);
+        assert_eq!(manifest_file_meta.schema_id, 1);
+    }
+}