add compression

src/models/recipev2.rs

@@ -4,7 +4,7 @@ use serde::{Deserialize, Serialize};
 use serde_json::Value;
 use std::collections::BTreeMap;
 use std::fs::File;
-use std::io::Read;
+use std::io::{Read, Write};
 use std::path::PathBuf;
 use std::sync::Arc;
 
@@ -114,8 +114,17 @@ impl ShardingRecipe {
         for cat_key in self.category_map.keys() {
             let shard_out_name = format!("{}_{}.shard", self.shard_name, cat_key);
 
-            let file = File::create(shard_out_name)?;
-            serde_json::to_writer_pretty(file, self.category_map.get(cat_key).unwrap())?;
+            let shard: &Value = self.category_map.get(cat_key).unwrap();
+            let shard_bytes = serde_json::to_vec(shard)?;
+
+            let mut encoder =
+                flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
+            encoder.write_all(&shard_bytes)?;
+            let compressed_data = encoder.finish()?;
+
+            let mut file = File::create(shard_out_name)?;
+            file.write_all(&compressed_data)?;
+            file.flush()?;
         }
         Ok(())
     }