add save shard output
This commit is contained in:
parent
c8351d108c
commit
109e607bee
1 changed files with 20 additions and 4 deletions
|
|
@ -1,7 +1,9 @@
|
||||||
|
use mlua::AsChunk;
|
||||||
use rayon::iter::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator};
|
use rayon::iter::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
|
use std::fs::File;
|
||||||
use std::io::Read;
|
use std::io::Read;
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
@ -74,11 +76,12 @@ impl RecipeListChange {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct ShardingRecipe {
|
pub struct ShardingRecipe {
|
||||||
|
pub shard_name: String,
|
||||||
pub category_map: BTreeMap<String, Value>,
|
pub category_map: BTreeMap<String, Value>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ShardingRecipe {
|
impl ShardingRecipe {
|
||||||
pub fn build(data: BTreeMap<String, Value>) -> Self {
|
pub fn build(name: String, data: BTreeMap<String, Value>) -> Self {
|
||||||
let keylist: Vec<String> = data.keys().map(|k| k.to_string()).collect();
|
let keylist: Vec<String> = data.keys().map(|k| k.to_string()).collect();
|
||||||
let mut category_map = BTreeMap::new();
|
let mut category_map = BTreeMap::new();
|
||||||
|
|
||||||
|
|
@ -101,13 +104,26 @@ impl ShardingRecipe {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Self { category_map }
|
Self {
|
||||||
|
shard_name: name,
|
||||||
|
category_map,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn save_shard(&mut self) -> Result<(), Box<dyn std::error::Error>> {
|
||||||
|
for cat_key in self.category_map.keys() {
|
||||||
|
let shard_out_name = format!("{}_{}.shard", self.shard_name, cat_key);
|
||||||
|
|
||||||
|
let file = File::create(shard_out_name)?;
|
||||||
|
serde_json::to_writer_pretty(file, self.category_map.get(cat_key).unwrap())?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// break down recipe(s) in the folder into mapping,
|
/// break down recipe(s) in the folder into mapping,
|
||||||
/// tracking diff between fields. (This does not compare other fields than Recipe01)
|
/// tracking diff between fields. (This does not compare other fields than Recipe01)
|
||||||
pub fn build_recipe_shardings(source: PathBuf) -> ShardingRecipe {
|
pub fn build_recipe_shardings(name: String, source: PathBuf) -> ShardingRecipe {
|
||||||
let files = common::get_all_files_in_directory(source.as_path().to_str().unwrap());
|
let files = common::get_all_files_in_directory(source.as_path().to_str().unwrap());
|
||||||
let known_latest_version = format!("{}/version", source.as_path().to_str().unwrap());
|
let known_latest_version = format!("{}/version", source.as_path().to_str().unwrap());
|
||||||
let mut latest_version_string = String::new();
|
let mut latest_version_string = String::new();
|
||||||
|
|
@ -244,5 +260,5 @@ pub fn build_recipe_shardings(source: PathBuf) -> ShardingRecipe {
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
ShardingRecipe::build(shards)
|
ShardingRecipe::build(name.to_string(), shards)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue