add save shard output

This commit is contained in:
Pakin 2025-09-10 16:19:44 +07:00
parent c8351d108c
commit 109e607bee

View file

@ -1,7 +1,9 @@
use mlua::AsChunk;
use rayon::iter::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use std::collections::BTreeMap;
use std::fs::File;
use std::io::Read;
use std::path::PathBuf;
use std::sync::Arc;
@ -74,11 +76,12 @@ impl RecipeListChange {
}
pub struct ShardingRecipe {
pub shard_name: String,
pub category_map: BTreeMap<String, Value>,
}
impl ShardingRecipe {
pub fn build(data: BTreeMap<String, Value>) -> Self {
pub fn build(name: String, data: BTreeMap<String, Value>) -> Self {
let keylist: Vec<String> = data.keys().map(|k| k.to_string()).collect();
let mut category_map = BTreeMap::new();
@ -101,13 +104,26 @@ impl ShardingRecipe {
}
}
Self { category_map }
Self {
shard_name: name,
category_map,
}
}
pub fn save_shard(&mut self) -> Result<(), Box<dyn std::error::Error>> {
for cat_key in self.category_map.keys() {
let shard_out_name = format!("{}_{}.shard", self.shard_name, cat_key);
let file = File::create(shard_out_name)?;
serde_json::to_writer_pretty(file, self.category_map.get(cat_key).unwrap())?;
}
Ok(())
}
}
/// break down recipe(s) in the folder into mapping,
/// tracking diff between fields. (This does not compare other fields than Recipe01)
pub fn build_recipe_shardings(source: PathBuf) -> ShardingRecipe {
pub fn build_recipe_shardings(name: String, source: PathBuf) -> ShardingRecipe {
let files = common::get_all_files_in_directory(source.as_path().to_str().unwrap());
let known_latest_version = format!("{}/version", source.as_path().to_str().unwrap());
let mut latest_version_string = String::new();
@ -244,5 +260,5 @@ pub fn build_recipe_shardings(source: PathBuf) -> ShardingRecipe {
})
.collect();
ShardingRecipe::build(shards)
ShardingRecipe::build(name.to_string(), shards)
}