add save shard output
This commit is contained in:
parent
c8351d108c
commit
109e607bee
1 changed files with 20 additions and 4 deletions
|
|
@ -1,7 +1,9 @@
|
|||
use mlua::AsChunk;
|
||||
use rayon::iter::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use std::collections::BTreeMap;
|
||||
use std::fs::File;
|
||||
use std::io::Read;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
|
@ -74,11 +76,12 @@ impl RecipeListChange {
|
|||
}
|
||||
|
||||
pub struct ShardingRecipe {
|
||||
pub shard_name: String,
|
||||
pub category_map: BTreeMap<String, Value>,
|
||||
}
|
||||
|
||||
impl ShardingRecipe {
|
||||
pub fn build(data: BTreeMap<String, Value>) -> Self {
|
||||
pub fn build(name: String, data: BTreeMap<String, Value>) -> Self {
|
||||
let keylist: Vec<String> = data.keys().map(|k| k.to_string()).collect();
|
||||
let mut category_map = BTreeMap::new();
|
||||
|
||||
|
|
@ -101,13 +104,26 @@ impl ShardingRecipe {
|
|||
}
|
||||
}
|
||||
|
||||
Self { category_map }
|
||||
Self {
|
||||
shard_name: name,
|
||||
category_map,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn save_shard(&mut self) -> Result<(), Box<dyn std::error::Error>> {
|
||||
for cat_key in self.category_map.keys() {
|
||||
let shard_out_name = format!("{}_{}.shard", self.shard_name, cat_key);
|
||||
|
||||
let file = File::create(shard_out_name)?;
|
||||
serde_json::to_writer_pretty(file, self.category_map.get(cat_key).unwrap())?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// break down recipe(s) in the folder into mapping,
|
||||
/// tracking diff between fields. (This does not compare other fields than Recipe01)
|
||||
pub fn build_recipe_shardings(source: PathBuf) -> ShardingRecipe {
|
||||
pub fn build_recipe_shardings(name: String, source: PathBuf) -> ShardingRecipe {
|
||||
let files = common::get_all_files_in_directory(source.as_path().to_str().unwrap());
|
||||
let known_latest_version = format!("{}/version", source.as_path().to_str().unwrap());
|
||||
let mut latest_version_string = String::new();
|
||||
|
|
@ -244,5 +260,5 @@ pub fn build_recipe_shardings(source: PathBuf) -> ShardingRecipe {
|
|||
})
|
||||
.collect();
|
||||
|
||||
ShardingRecipe::build(shards)
|
||||
ShardingRecipe::build(name.to_string(), shards)
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue