fix(storage-proofs): always use temporary files for MTs (#833)

* remove MTs directory, use only temp files

* import new FromIndexedParallelIterator

* bump merkletree dep

Author: schomatis

README.md

@@ -161,7 +161,7 @@ To check that it's working you can inspect the replication log to find `using pa
 
 ### Memory
 
-We try to generate the MTs in parallel to speed up the process but that takes 2 sector sizes per each layer (e.g., a 1 GiB sector may require, in the worst case scenario, up to 20 GiB of memory to hold the MTs alone). To reduce that (at the cost of speed) we have the (experimental) `disk-trees` feature to offload the MTs to disk when we don't use them. For example, to run the `zigzag` example with this feature you'd need to indicate so to `cargo` *and* then indicate to the example (or any other code doing the replication) where they should be stored using the `FIL_PROOFS_REPLICATED_TREES_DIR` environmental variable (if set to a relative path, it will be relative  to the current working directory of the process in which the replication happens, see [`create_dir`](https://doc.rust-lang.org/std/fs/fn.create_dir.html#platform-specific-behavior)),
+We try to generate the MTs in parallel to speed up the process but that takes 2 sector sizes per each layer (e.g., a 1 GiB sector may require, in the worst case scenario, up to 20 GiB of memory to hold the MTs alone). To reduce that (at the cost of speed) we have the (experimental) `disk-trees` feature to offload the MTs to disk when we don't use them. For example, to run the `zigzag` example with this feature you'd need to indicate so to `cargo`,
 
 ```
 # From inside the `storage-proofs` directory, where this feature
@@ -172,15 +172,10 @@ cargo build                                                                   \
   --example zigzag                                                            \
   --features                                                                  \
     disk-trees                                                               &&
-FIL_PROOFS_REPLICATED_TREES_DIR="<tree-dir>"                                  \
 ../target/release/examples/zigzag                                             \
     --size 1048576
 ```
 
-To check if the feature is indeed working you can inspect the replication log to find the text `creating leaves tree mmap-file` indicating the path where each MT file is saved.
-
-Note that at the moment we do *not* clean up `<tree-dir>` so you'll need to **remove old MT files** yourself (otherwise the disk will start to fill up pretty fast). To avoid collisions in the directory we append a random hexadecimal value to each file, to check which files belong to the latest replication run you can (besides checking modification times) inspect the replication log to find out the actual file names generated.
-
 To optimize even more for memory there's another option (used in addition to the `disk-trees` feature) to generate all MTs in sequential order, to make sure we can offload them to disk before we start buildding the next one,
 
 ```

storage-proofs/Cargo.toml

@@ -15,7 +15,7 @@ bench = false
 bitvec = "0.5"
 rand = "0.4"
 libc = "0.2"
-merkletree = "=0.8"
+merkletree = "=0.9"
 failure = "0.1"
 byteorder = "1"
 config = "0.9.3"

storage-proofs/src/drgraph.rs

@@ -10,15 +10,7 @@ use crate::hasher::{Domain, Hasher};
 use crate::merkle::MerkleTree;
 use crate::parameter_cache::ParameterSetMetadata;
 use crate::util::{data_at_node, NODE_SIZE};
-
-#[cfg(feature = "disk-trees")]
-use crate::merkle::DiskStore;
-#[cfg(feature = "disk-trees")]
-use merkletree::merkle::next_pow2;
-#[cfg(feature = "disk-trees")]
-use std::path::Path;
-#[cfg(feature = "disk-trees")]
-use std::path::PathBuf;
+use merkletree::merkle::FromIndexedParallelIterator;
 
 /// The default hasher currently in use.
 pub type DefaultTreeHasher = PedersenHasher;
@@ -70,84 +62,6 @@ pub trait Graph<H: Hasher>: ::std::fmt::Debug + Clone + PartialEq + Eq {
         }
     }
 
-    /// Builds a merkle tree based on the given data and stores it in `path`
-    /// (if set).
-    #[cfg(feature = "disk-trees")]
-    fn merkle_tree_path<'a>(
-        &self,
-        data: &'a [u8],
-        path: Option<&Path>,
-    ) -> Result<MerkleTree<H::Domain, H::Function>> {
-        self.merkle_tree_aux_path(data, PARALLEL_MERKLE, path)
-    }
-
-    #[cfg(feature = "disk-trees")]
-    fn merkle_tree_aux_path<'a>(
-        &self,
-        data: &'a [u8],
-        parallel: bool,
-        path: Option<&Path>,
-    ) -> Result<MerkleTree<H::Domain, H::Function>> {
-        if data.len() != (NODE_SIZE * self.size()) as usize {
-            return Err(Error::InvalidMerkleTreeArgs(
-                data.len(),
-                NODE_SIZE,
-                self.size(),
-            ));
-        }
-
-        let f = |i| {
-            let d = data_at_node(&data, i).expect("data_at_node math failed");
-            // TODO/FIXME: This can panic. FOR NOW, let's leave this since we're experimenting with
-            // optimization paths. However, we need to ensure that bad input will not lead to a panic
-            // that isn't caught by the FPS API.
-            // Unfortunately, it's not clear how to perform this error-handling in the parallel
-            // iterator case.
-            H::Domain::try_from_bytes(d).unwrap()
-        };
-
-        if let Some(path) = path {
-            let path_prefix = path.to_str().expect("couldn't convert path to string");
-            let leaves_path = &PathBuf::from([path_prefix, "leaves"].join("-"));
-            let top_half_path = &PathBuf::from([path_prefix, "top-half"].join("-"));
-            // FIXME: There is probably a more direct way of doing this without
-            // reconverting to string.
-
-            info!(
-                "creating leaves tree mmap-file (path-prefix: {:?})",
-                leaves_path.to_str()
-            );
-            info!(
-                "creating top half tree mmap-file (path-prefix: {:?})",
-                top_half_path.to_str()
-            );
-
-            let leaves_disk_mmap = DiskStore::new_with_path(next_pow2(self.size()), leaves_path);
-            let top_half_disk_mmap =
-                DiskStore::new_with_path(next_pow2(self.size()), top_half_path);
-
-            // FIXME: `new_with_path` is using the `from_iter` implementation,
-            //  instead the `parallel` flag should be passed also as argument
-            //  and decide *there* which code to use (merging this into the
-            //  `if` logic below).
-
-            Ok(MerkleTree::from_data_with_store(
-                (0..self.size()).map(f),
-                leaves_disk_mmap,
-                top_half_disk_mmap,
-            ))
-        // If path is `None` use the existing code that will eventually
-        // call the default `DiskStore::new` creating a temporary
-        // file.
-        } else if parallel {
-            Ok(MerkleTree::from_par_iter(
-                (0..self.size()).into_par_iter().map(f),
-            ))
-        } else {
-            Ok(MerkleTree::new((0..self.size()).map(f)))
-        }
-    }
-
     /// Returns the merkle tree depth.
     fn merkle_tree_depth(&self) -> u64 {
         graph_height(self.size()) as u64

storage-proofs/src/layered_drgporep.rs

@@ -21,15 +21,6 @@ use crate::proof::ProofScheme;
 use crate::settings;
 use crate::vde;
 
-#[cfg(feature = "disk-trees")]
-use rand;
-#[cfg(feature = "disk-trees")]
-use std::fs;
-#[cfg(feature = "disk-trees")]
-use std::io;
-#[cfg(feature = "disk-trees")]
-use std::path::PathBuf;
-
 type Tree<H> = MerkleTree<<H as Hasher>::Domain, <H as Hasher>::Function>;
 
 fn anonymous_mmap(len: usize) -> MmapMut {
@@ -389,7 +380,7 @@ pub trait Layers {
             let mut sorted_trees: Vec<_> = Vec::new();
 
             (0..=layers).fold(graph.clone(), |current_graph, layer| {
-                let tree_d = Self::generate_data_tree(&current_graph, &data, layer);
+                let tree_d = current_graph.merkle_tree(&data).unwrap();
 
                 info!("returning tree (layer: {})", layer);
 
@@ -466,7 +457,7 @@ pub trait Layers {
                                 .recv()
                                 .expect("Failed to receive value through channel");
 
-                            let tree_d = Self::generate_data_tree(&graph, &data_copy, layer);
+                            let tree_d = graph.merkle_tree(&data_copy).unwrap();
 
                             info!("returning tree (layer: {})", layer);
                             return_channel
@@ -518,53 +509,6 @@ pub trait Layers {
 
         Ok((taus, auxs))
     }
-
-    fn generate_data_tree(
-        graph: &Self::Graph,
-        data: &[u8],
-        _layer: usize,
-    ) -> MerkleTree<<Self::Hasher as Hasher>::Domain, <Self::Hasher as Hasher>::Function> {
-        #[cfg(not(feature = "disk-trees"))]
-        return graph.merkle_tree(&data).unwrap();
-
-        #[cfg(feature = "disk-trees")]
-        {
-            let tree_dir = &settings::SETTINGS.lock().unwrap().replicated_trees_dir;
-            // We should always be able to get this configuration
-            // variable (at least as an empty string).
-
-            if tree_dir.is_empty() {
-                // Signal `merkle_tree_path` to create a temporary file.
-                return graph.merkle_tree_path(&data, None).unwrap();
-            } else {
-                // Try to create `tree_dir`, ignore the error if `AlreadyExists`.
-                if let Some(create_error) = fs::create_dir(&tree_dir).err() {
-                    if create_error.kind() != io::ErrorKind::AlreadyExists {
-                        panic!(create_error);
-                    }
-                }
-
-                let tree_d = graph
-                    .merkle_tree_path(
-                        &data,
-                        Some(&PathBuf::from(tree_dir).join(format!(
-                            "tree-{}-{}",
-                            _layer,
-                            // FIXME: This argument is used only with `disk-trees`.
-                            rand::random::<u32>()
-                        ))),
-                    )
-                    .unwrap();
-                // FIXME: The user of `REPLICATED_TREES_DIR` should figure out
-                // how to manage this directory, for now we create every file with
-                // a different random number; the problem being that tests now do
-                // replications many times in the same run so they may end up
-                // reusing the same files with invalid (old) data and failing.
-
-                return tree_d;
-            }
-        }
-    }
 }
 
 impl<'a, L: Layers> ProofScheme<'a> for L {