From 2c042810d8508c1d0e2ee9bd1117ef26ac95154e Mon Sep 17 00:00:00 2001 From: Colin Date: Fri, 27 Nov 2020 22:11:53 -0800 Subject: [PATCH] Implement a 'SerializerRenderer' output format It's CBOR, which seems to pack not so efficiently (250*250*20 * 12 floats takes 400 MB per frame, whereas this many doubles could pack into 120 MB). serde_cbor also seems to be extremely slow: taking multiple minutes per frame. This could be parallelized to be less problematic though (it already is -- just bump the parallelism). Might be worth testing other formats. --- Cargo.toml | 4 +++- examples/toroid25d.rs | 7 ++++--- src/driver.rs | 4 ++++ src/geom/vec.rs | 6 ++++-- src/geom/vecu.rs | 4 +++- src/mat.rs | 4 +++- src/render.rs | 21 +++++++++++++++++++++ src/sim.rs | 9 ++++++--- 8 files changed, 48 insertions(+), 11 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 5635c7a..c6d74d7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,9 +17,11 @@ image = "0.23" imageproc = "0.21" lazy_static = "1.4" log = "0.4" -ndarray = { version = "0.13", features = ["rayon"] } +ndarray = { version = "0.13", features = ["rayon", "serde"] } piecewise-linear = "0.1" plotly = { version = "0.6", features = ["kaleido", "plotly_ndarray"], path = "../plotly/plotly" } +serde = "1.0" +serde_cbor = "0.11" threadpool = "1.8" y4m = "0.7" diff --git a/examples/toroid25d.rs b/examples/toroid25d.rs index 61308ec..3b02c42 100644 --- a/examples/toroid25d.rs +++ b/examples/toroid25d.rs @@ -38,7 +38,7 @@ fn main() { //driver.set_steps_per_frame(120); driver.set_steps_per_frame(160); //driver.set_steps_per_frame(200); - let base = "toroid25d-7"; + let base = "toroid25d-8"; let _ = std::fs::create_dir(base); let prefix = format!("{}/{}-flt{}-{}-feat{}um-{}mA-{}ps--radii{}um-{}um-{}um-{}um", base, @@ -54,8 +54,9 @@ fn main() { m_to_um(ferro_depth), ); let _ = std::fs::create_dir(&prefix); - driver.add_y4m_renderer(&*format!("{}.y4m", prefix)); - driver.add_plotly_renderer(&*format!("{}/frame-", prefix)); + //driver.add_y4m_renderer(&*format!("{}.y4m", prefix)); + //driver.add_plotly_renderer(&*format!("{}/frame-", prefix)); + driver.add_serializer_renderer(&*format!("{}/frame-", prefix)); let conductor_region = CylinderZ::new( Vec2::new(half_width, half_width), conductor_outer_rad); diff --git a/src/driver.rs b/src/driver.rs index 86e42a9..9cc5e86 100644 --- a/src/driver.rs +++ b/src/driver.rs @@ -83,6 +83,10 @@ impl Driver { self.add_renderer(render::ColorTermRenderer, "terminal"); } + pub fn add_serializer_renderer(&mut self, out_base: &str) { + self.add_renderer(render::SerializerRenderer::new(out_base), out_base); + } + // pub fn add_boundary(&mut self, thickness: u32, base_conductivity: Flt) { // for inset in 0..thickness { // let depth = thickness - inset; diff --git a/src/geom/vec.rs b/src/geom/vec.rs index 5508cac..dff80a3 100644 --- a/src/geom/vec.rs +++ b/src/geom/vec.rs @@ -1,5 +1,7 @@ use crate::flt::{Flt, Real}; use super::Vec3u; + +use serde::{Serialize, Deserialize}; use std::convert::From; use std::iter::Sum; use std::ops::{Add, AddAssign, Div, DivAssign, Mul, MulAssign, Neg, Sub}; @@ -8,7 +10,7 @@ fn round(f: Real) -> Real { Real::from_inner(f.into_inner().round()) } -#[derive(Copy, Clone, Debug, Default)] +#[derive(Copy, Clone, Debug, Default, Serialize, Deserialize)] pub struct Vec2 { pub x: Real, pub y: Real, @@ -115,7 +117,7 @@ impl Vec2 { } } -#[derive(Copy, Clone, Debug, Default, PartialEq)] +#[derive(Copy, Clone, Debug, Default, PartialEq, Serialize, Deserialize)] pub struct Vec3 { pub(crate) x: Real, pub(crate) y: Real, diff --git a/src/geom/vecu.rs b/src/geom/vecu.rs index b55760b..d0508b4 100644 --- a/src/geom/vecu.rs +++ b/src/geom/vecu.rs @@ -1,7 +1,9 @@ use super::Vec3; + +use serde::{Serialize, Deserialize}; use std::fmt::{self, Display}; -#[derive(Copy, Clone, Debug, Default, Eq, PartialEq)] +#[derive(Copy, Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] pub struct Vec3u { y: u32, x: u32, diff --git a/src/mat.rs b/src/mat.rs index 2f86a82..cfaf2c1 100644 --- a/src/mat.rs +++ b/src/mat.rs @@ -1,9 +1,11 @@ use crate::{CellState, consts}; use crate::flt::{Flt, Real}; use crate::geom::{Line2d, Vec2, Vec3, Polygon2d}; + use lazy_static::lazy_static; use log::{debug, trace}; use enum_dispatch::enum_dispatch; +use serde::{Serialize, Deserialize}; use std::cmp::Ordering; #[enum_dispatch] @@ -27,7 +29,7 @@ pub trait Material { } } -#[derive(Clone, Default)] +#[derive(Clone, Default, Serialize, Deserialize)] pub struct Static { pub conductivity: Vec3, pub m: Vec3, diff --git a/src/render.rs b/src/render.rs index 787c3a4..480f75c 100644 --- a/src/render.rs +++ b/src/render.rs @@ -462,3 +462,24 @@ impl Renderer for MultiRenderer { } } } + +pub struct SerializerRenderer { + out_base: String +} + +impl SerializerRenderer { + pub fn new(out_base: &str) -> Self { + Self { + out_base: out_base.into(), + } + } +} + +impl Renderer for SerializerRenderer { + fn render(&self, state: &dyn GenericSim, measurements: &[Box]) { + let snap = state.to_static(); + let name = format!("{}{}.cbor", self.out_base, snap.step_no()); + let out = File::create(name).unwrap(); + serde_cbor::to_writer(out, &snap).unwrap(); + } +} diff --git a/src/sim.rs b/src/sim.rs index 4909442..10c19d8 100644 --- a/src/sim.rs +++ b/src/sim.rs @@ -6,6 +6,7 @@ use log::trace; use ndarray::{Array3, Zip}; use ndarray::parallel::prelude::*; +use serde::{Serialize, Deserialize}; use std::convert::From; use std::iter::Sum; @@ -34,6 +35,7 @@ pub trait GenericSim: Send + Sync + DynClone { /// Take a "snapshot" of the simulation, dropping all material-specific information. fn to_static(&self) -> SimState { let mut state = SimState::new(self.size(), self.feature_size()); + state.step_no = self.step_no(); Zip::from(ndarray::indices_of(&state.cells)).par_apply_assign_into( &mut state.cells, |(z, y, x)| { @@ -96,9 +98,10 @@ impl<'a> dyn GenericSim + 'a { } } -#[derive(Default, Clone)] +#[derive(Default, Clone, Serialize, Deserialize)] pub struct SimState { cells: Array3>, + #[serde(skip)] scratch: Array3>, feature_size: Real, step_no: u64, @@ -277,7 +280,7 @@ impl SimState { /// \| | | /// +------------+------------+ /// -#[derive(Clone, Default)] +#[derive(Clone, Default, Serialize, Deserialize)] pub struct Cell { state: CellState, mat: M, @@ -532,7 +535,7 @@ impl Cell { } } -#[derive(Copy, Clone, Debug, Default, PartialEq)] +#[derive(Copy, Clone, Debug, Default, PartialEq, Serialize, Deserialize)] pub struct CellState { e: Vec3, h: Vec3,