feat: gaussian scene asset loader (#179)

* feat: gaussian scene asset loader

* fix: .gscene -> .json

* fix: packed descriptors

Author: mosure

.gitignore

@@ -11,6 +11,7 @@ out/
 *.gcloud
 *.ply
 *.ply4d
+*.json
 
 .DS_Store
 

Cargo.lock

@@ -1,7 +1,7 @@
 [package]
 name = "bevy_gaussian_splatting"
 description = "bevy gaussian splatting render pipeline plugin"
-version = "4.2.0"
+version = "4.3.0"
 edition = "2021"
 authors = ["mosure <[email protected]>"]
 license = "MIT OR Apache-2.0"
@@ -170,6 +170,7 @@ ply-rs = { version = "0.1", optional = true }
 rand = "0.8"
 rayon = { version = "1.8", optional = true }
 serde = "1.0"
+serde_json = "1.0"
 static_assertions = "1.1"
 typenum = "1.17"
 wgpu = "23.0.1"
@@ -189,6 +190,7 @@ features = [
   "bevy_pbr",
   "bevy_render",
   "bevy_winit",
+  "serialize",
   "x11",
 ]
 

Cargo.toml

@@ -37,6 +37,8 @@ bevy_gaussian_splatting --input-file {gaussian_splat_ply_file}
 - [X] 2dgs
 - [X] 3dgs
 - [x] 4dgs
+- [x] multi-cloud scene format
+- [ ] gltf gaussian extensions
 - [ ] 4dgs motion blur
 - [ ] implicit mlp node (isotropic rotation, color)
 - [ ] temporal gaussian hierarchy
@@ -82,7 +84,7 @@ fn setup_gaussian_cloud(
         CloudSettings::default(),
     ));
 
-    commands.spawn(Camera3dBundle::default());
+    commands.spawn(Camera3d::default());
 }
 ```
 

README.md

@@ -0,0 +1,24 @@
+{
+  "bundles": [
+    {
+      "asset_path": "./trellis.ply",
+      "name": "trellis_0",
+      "settings": {},
+      "transform": {
+        "translation": [0.0, 0.0, 0.0],
+        "rotation": [0.0, 0.0, 0.0, 1.0],
+        "scale": [1.0, -1.0, 1.0]
+      }
+    },
+    {
+      "asset_path": "./trellis.ply",
+      "name": "trellis_1",
+      "settings": {},
+      "transform": {
+        "translation": [5.0, 0.0, 0.0],
+        "rotation": [0.0, 0.0, 0.0, 1.0],
+        "scale": [1.0, -1.0, 1.0]
+      }
+    }
+  ]
+}

assets/scene.json

@@ -378,9 +378,9 @@ fn setup_gaussian_cloud(
     if args.gaussian_count > 0 {
         println!("generating {} gaussians", args.gaussian_count);
         cloud = gaussian_assets.add(random_gaussians_3d(args.gaussian_count));
-    } else if !args.input_file.is_empty() {
-        println!("loading {}", args.input_file);
-        cloud = asset_server.load(&args.input_file);
+    } else if args.input_cloud.is_some() && !args.input_cloud.as_ref().unwrap().is_empty() {
+        println!("loading {:?}", args.input_cloud);
+        cloud = asset_server.load(&args.input_cloud.as_ref().unwrap().clone());
     } else {
         cloud = gaussian_assets.add(PlanarGaussian3d::test_model());
     }

examples/headless.rs

@@ -17,6 +17,8 @@ use crate::sort::SortMode;
     Hash,
     PartialEq,
     Reflect,
+    Serialize,
+    Deserialize,
 )]
 pub enum DrawMode {
     #[default]
@@ -93,8 +95,16 @@ pub enum RasterizeMode {
 
 
 // TODO: breakdown into components
-#[derive(Component, Reflect, Clone)]
+#[derive(
+    Component,
+    Clone,
+    Debug,
+    Reflect,
+    Serialize,
+    Deserialize,
+)]
 #[reflect(Component)]
+#[serde(default)]
 pub struct CloudSettings {
     pub aabb: bool,
     pub global_opacity: f32,

src/gaussian/settings.rs

@@ -1,6 +1,21 @@
+use bevy::prelude::*;
+
 pub mod codec;
 pub mod gcloud;
 pub mod loader;
+pub mod scene;
 
 #[cfg(feature = "io_ply")]
 pub mod ply;
+
+
+#[derive(Default)]
+pub struct IoPlugin;
+impl Plugin for IoPlugin {
+    fn build(&self, app: &mut App) {
+        app.init_asset_loader::<loader::Gaussian3dLoader>();
+        app.init_asset_loader::<loader::Gaussian4dLoader>();
+
+        app.add_plugins(scene::GaussianScenePlugin);
+    }
+}

src/io/mod.rs

@@ -0,0 +1,169 @@
+
+#[allow(unused_imports)]
+use std::io::{
+    BufReader,
+    Cursor,
+    ErrorKind,
+};
+
+use bevy::{
+    prelude::*,
+    asset::{
+        AssetLoader,
+        LoadContext,
+        io::Reader,
+    },
+};
+use serde::{
+    Deserialize,
+    Serialize,
+};
+
+use crate::gaussian::{
+    formats::planar_3d::{
+        PlanarGaussian3d,
+        PlanarGaussian3dHandle,
+    },
+    settings::CloudSettings,
+};
+
+
+#[derive(Default)]
+pub struct GaussianScenePlugin;
+impl Plugin for GaussianScenePlugin {
+    fn build(&self, app: &mut App) {
+        app.register_type::<GaussianScene>();
+        app.init_asset::<GaussianScene>();
+
+        app.init_asset_loader::<GaussianSceneLoader>();
+
+        app.add_systems(
+            Update,
+            (
+                spawn_scene,
+            )
+        );
+    }
+}
+
+
+
+#[derive(
+    Clone,
+    Debug,
+    Default,
+    Reflect,
+    Serialize,
+    Deserialize,
+)]
+pub struct CloudBundle {
+    pub asset_path: String,
+    pub name: String,
+    pub settings: CloudSettings,
+    pub transform: Transform,
+}
+
+// TODO: support scene hierarchy with gaussian gltf extension
+#[derive(
+    Asset,
+    Clone,
+    Debug,
+    Default,
+    Reflect,
+    Serialize,
+    Deserialize,
+)]
+pub struct GaussianScene {
+    pub bundles: Vec<CloudBundle>,
+}
+
+#[derive(Component, Clone, Debug, Default, Reflect)]
+#[require(Transform, Visibility)]
+pub struct GaussianSceneHandle(pub Handle<GaussianScene>);
+
+#[derive(Component, Clone, Debug, Default, Reflect)]
+pub struct GaussianSceneLoaded;
+
+
+fn spawn_scene(
+    mut commands: Commands,
+    scene_handles: Query<
+        (
+            Entity,
+            &GaussianSceneHandle,
+        ),
+        Without<GaussianSceneLoaded>,
+    >,
+    asset_server: Res<AssetServer>,
+    scenes: Res<Assets<GaussianScene>>,
+) {
+    for (entity, scene_handle) in scene_handles.iter() {
+        if let Some(load_state) = &asset_server.get_load_state(&scene_handle.0) {
+            if !load_state.is_loaded() {
+                continue;
+            }
+        }
+
+        if scenes.get(&scene_handle.0).is_none() {
+            continue;
+        }
+
+        let scene = scenes.get(&scene_handle.0).unwrap();
+
+        let bundles = scene.bundles
+            .iter()
+            .map(|bundle|(
+                    // TODO: switch between 3d and 4d clouds based on settings
+                    PlanarGaussian3dHandle(
+                        asset_server.load::<PlanarGaussian3d>(bundle.asset_path.clone())
+                    ),
+                    Name::new(bundle.name.clone()),
+                    bundle.settings.clone(),
+                    bundle.transform,
+                )
+            )
+            .collect::<Vec<_>>();
+
+        commands
+            .entity(entity)
+            .with_children(move |builder| {
+                for bundle in bundles {
+                    builder.spawn(bundle);
+                }
+            })
+            .insert(GaussianSceneLoaded);
+    }
+}
+
+
+#[derive(Default)]
+pub struct GaussianSceneLoader;
+
+impl AssetLoader for GaussianSceneLoader {
+    type Asset = GaussianScene;
+    type Settings = ();
+    type Error = std::io::Error;
+
+    async fn load(
+        &self,
+        reader: &mut dyn Reader,
+        _: &Self::Settings,
+        load_context: &mut LoadContext<'_>,
+    ) -> Result<Self::Asset, Self::Error> {
+        let mut bytes = Vec::new();
+        reader.read_to_end(&mut bytes).await?;
+
+        match load_context.path().extension() {
+            Some(ext) if ext == "json" => {
+                let scene: GaussianScene = serde_json::from_slice(&bytes)
+                    .map_err(|err| std::io::Error::new(ErrorKind::InvalidData, err))?;
+                Ok(scene)
+            },
+            _ => Err(std::io::Error::new(ErrorKind::Other, "only .json supported")),
+        }
+    }
+
+    fn extensions(&self) -> &[&str] {
+        &["json"]
+    }
+}