From 9c24c5e6485cf0ca53c33e3f95b31d1623f8d1c1 Mon Sep 17 00:00:00 2001 From: Dan Quinlan Date: Mon, 31 Mar 2025 09:17:53 -0400 Subject: [PATCH 01/11] Initial check-in of audio emitter support into UnityGLTF repo. --- .../VisualScriptingExport/ExportGraph.cs | 19 + .../VisualScriptingExport/ExportGraph.cs.meta | 11 + .../GLTFAudioExportContext.cs | 413 ++++++++++++++++++ .../GLTFAudioExportContext.cs.meta | 11 + .../GLTFAudioExportPlugin.cs | 37 ++ .../GLTFAudioExportPlugin.cs.meta | 11 + .../GltfAudioNodeHelper.cs | 214 +++++++++ .../GltfAudioNodeHelper.cs.meta | 11 + .../Audio/AudioSourcePauseNode.cs | 66 +++ .../Audio/AudioSourcePauseNode.cs.meta | 11 + .../Audio/AudioSourcePlayNode.cs | 79 ++++ .../Audio/AudioSourcePlayNode.cs.meta | 11 + .../Audio/AudioSourceStopNode.cs | 64 +++ .../Audio/AudioSourceStopNode.cs.meta | 11 + .../Audio/AudioSourceUnPauseNode.cs | 65 +++ .../Audio/AudioSourceUnPauseNode.cs.meta | 11 + .../Audio/AudioSourceUnitExport.cs | 107 +++++ .../Audio/AudioSourceUnitExport.cs.meta | 11 + .../VisualScriptingExportContext.cs | 2 +- .../Schema/GltfAudioExtension.cs | 48 ++ .../Schema/GltfAudioExtension.cs.meta | 11 + .../Scripts/Interactivity/Schema/GltfTypes.cs | 4 +- .../Interactivity/Schema/KHRAudioSchemas.cs | 341 +++++++++++++++ .../Schema/KHRAudioSchemas.cs.meta | 11 + .../Schema/Nodes/Audio/Audio_PauseNode.cs | 26 ++ .../Nodes/Audio/Audio_PauseNode.cs.meta | 11 + .../Schema/Nodes/Audio/Audio_StartNode.cs | 26 ++ .../Nodes/Audio/Audio_StartNode.cs.meta | 11 + .../Schema/Nodes/Audio/Audio_StopNode.cs | 19 + .../Schema/Nodes/Audio/Audio_StopNode.cs.meta | 3 + .../Schema/Nodes/Audio/Audio_UnPauseNode.cs | 26 ++ .../Nodes/Audio/Audio_UnPauseNode.cs.meta | 11 + 32 files changed, 1710 insertions(+), 3 deletions(-) create mode 100644 Editor/Scripts/Interactivity/VisualScriptingExport/ExportGraph.cs create mode 100644 Editor/Scripts/Interactivity/VisualScriptingExport/ExportGraph.cs.meta create mode 100644 Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportContext.cs create mode 100644 Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportContext.cs.meta create mode 100644 Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportPlugin.cs create mode 100644 Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportPlugin.cs.meta create mode 100644 Editor/Scripts/Interactivity/VisualScriptingExport/GltfAudioNodeHelper.cs create mode 100644 Editor/Scripts/Interactivity/VisualScriptingExport/GltfAudioNodeHelper.cs.meta create mode 100644 Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourcePauseNode.cs create mode 100644 Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourcePauseNode.cs.meta create mode 100644 Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourcePlayNode.cs create mode 100644 Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourcePlayNode.cs.meta create mode 100644 Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceStopNode.cs create mode 100644 Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceStopNode.cs.meta create mode 100644 Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceUnPauseNode.cs create mode 100644 Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceUnPauseNode.cs.meta create mode 100644 Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceUnitExport.cs create mode 100644 Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceUnitExport.cs.meta create mode 100644 Runtime/Scripts/Interactivity/Schema/GltfAudioExtension.cs create mode 100644 Runtime/Scripts/Interactivity/Schema/GltfAudioExtension.cs.meta create mode 100644 Runtime/Scripts/Interactivity/Schema/KHRAudioSchemas.cs create mode 100644 Runtime/Scripts/Interactivity/Schema/KHRAudioSchemas.cs.meta create mode 100644 Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_PauseNode.cs create mode 100644 Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_PauseNode.cs.meta create mode 100644 Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_StartNode.cs create mode 100644 Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_StartNode.cs.meta create mode 100644 Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_StopNode.cs create mode 100644 Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_StopNode.cs.meta create mode 100644 Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_UnPauseNode.cs create mode 100644 Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_UnPauseNode.cs.meta diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/ExportGraph.cs b/Editor/Scripts/Interactivity/VisualScriptingExport/ExportGraph.cs new file mode 100644 index 000000000..aea24bde4 --- /dev/null +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/ExportGraph.cs @@ -0,0 +1,19 @@ +using System.Collections; +using System.Collections.Generic; +using Unity.VisualScripting; +using UnityEngine; + +using UnityGLTF.Interactivity.VisualScripting.Export; + +namespace UnityGLTF.Interactivity +{ + public class ExportGraph + { + public GameObject gameObject = null; + public ExportGraph parentGraph = null; + public FlowGraph graph; + public Dictionary nodes = new Dictionary(); + internal Dictionary bypasses = new Dictionary(); + public List subGraphs = new List(); + } +} diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/ExportGraph.cs.meta b/Editor/Scripts/Interactivity/VisualScriptingExport/ExportGraph.cs.meta new file mode 100644 index 000000000..2b4ae9664 --- /dev/null +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/ExportGraph.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: ceed7c6d5d63c1843abdff2c2b6317ee +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportContext.cs b/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportContext.cs new file mode 100644 index 000000000..e82fb55c1 --- /dev/null +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportContext.cs @@ -0,0 +1,413 @@ +using GLTF.Schema; +using System.Collections.Generic; +using UnityEngine; +using Unity.VisualScripting; +using System.Linq; +using UnityGLTF.Audio; +using UnityGLTF.Plugins.Experimental; +using System.IO; +using UnityEditor; +using UnityGLTF.Interactivity.VisualScripting.Export; +using UnityGLTF.Interactivity.Schema; + +namespace UnityGLTF.Interactivity.VisualScripting +{ + /// + /// Audio GLTF export context + /// + public class GLTFAudioExportContext : VisualScriptingExportContext + { + /// + /// Contains summary information for an audio clip. + /// + public class AudioDescription + { + public int Id; + public string Name; + public AudioClip Clip; + } + + // container for audio sources by description + private static List _audioSourceIds = new(); + + private const string AudioExtension = ".mp3"; + private const string AudioRelDirectory = "audio"; + private const string MimeTypeString = "audio/mpeg"; + + private bool _saveAudioToFile = false; + + private List addedGraphs = new List(); + private List nodesToExport = new List(); + + internal new ExportGraph currentGraphProcessing { get; private set; } = null; + private GLTFRoot _gltfRoot = null; + + /// + /// Default construction initializes the parent GLTFInteractivity export context. + /// + /// + public GLTFAudioExportContext(GLTFAudioExportPlugin plugin): base(plugin) + { + + } + + /// + /// Constructor sets the save to audio bool using the supplied argument + /// + /// + /// + public GLTFAudioExportContext(GLTFAudioExportPlugin plugin, bool saveToExternalFile) : base(plugin) + { + _saveAudioToFile = saveToExternalFile; + } + + public override void BeforeSceneExport(GLTFSceneExporter exporter, GLTFRoot gltfRoot) { } + + /// + /// Called after the scene has been exported to add khr audio data. + /// + /// This overload of AfterSceneExport exposes the origins as a parameter to simplify tests. + /// + /// GLTFSceneExporter object used to export the scene + /// Root GLTF object for the gltf object tree + /// list of ScriptMachines in the scene. + public override void AfterSceneExport(GLTFSceneExporter exporter, GLTFRoot gltfRoot) + { + var scriptMachines = new List(); + + foreach (var root in exporter.RootTransforms) + { + if (!root) continue; + var machines = root + .GetComponentsInChildren() + .Where(x => x.isActiveAndEnabled && x.graph != null); + scriptMachines.AddRange(machines); + } + + AfterSceneExport(exporter, gltfRoot, scriptMachines); + } + + /// + /// Called after the scene has been exported to add interactivity data. + /// + /// This overload of AfterSceneExport exposes the origins as a parameter to simplify tests. + /// + /// GLTFSceneExporter object used to export the scene + /// Root GLTF object for the gltf object tree + /// list of ScriptMachines in the scene. + internal new void AfterSceneExport(GLTFSceneExporter exporter, GLTFRoot gltfRoot, List visualScriptingComponents) + { + if (visualScriptingComponents.Count == 0) + { + return; + } + this.exporter = exporter; + _gltfRoot = gltfRoot; + foreach (var scriptMachine in visualScriptingComponents) + { + ActiveScriptMachine = scriptMachine; + FlowGraph flowGraph = scriptMachine.graph; + GetAudio(flowGraph); + } + } + + /// + /// Gets the export graph which is not currently being used. + /// + /// + /// + internal ExportGraph GetAudio(FlowGraph graph) + { + var newExportGraph = new ExportGraph(); + newExportGraph.gameObject = ActiveScriptMachine.gameObject; + newExportGraph.parentGraph = currentGraphProcessing; + newExportGraph.graph = graph; + addedGraphs.Add(newExportGraph); + + if (currentGraphProcessing != null) + currentGraphProcessing.subGraphs.Add(newExportGraph); + + var lastCurrentGraph = currentGraphProcessing; + currentGraphProcessing = newExportGraph; + + // Topologically sort the graph to establish the dependency order + LinkedList topologicallySortedNodes = TopologicalSort(graph.units); + IEnumerable sortedNodes = topologicallySortedNodes; + foreach (var unit in sortedNodes) + { + if (unit is Literal literal) + { + AudioSource audio = null; + // If there is a connection, then we can return the value of the literal + if (literal.value is Component component && component is AudioSource) + audio = component.GetComponent(); + if (audio == null) + continue; + + ProcessAudioSource(unit, audio); + } + } + + newExportGraph.nodes = GltfAudioNodeHelper.GetTranslatableNodes(topologicallySortedNodes, this); + + nodesToExport.AddRange(newExportGraph.nodes.Select(g => g.Value)); + + currentGraphProcessing = lastCurrentGraph; + return newExportGraph; + } + + public static AudioDescription AddAudioSource(AudioSource audioSource) + { + AudioClip clip = audioSource.clip; + string name = clip.name; + + foreach (var a in _audioSourceIds) + { + if (name == a.Name && clip == a.Clip) + { + return a; + } + } + AudioDescription ad = new AudioDescription() { Id = _audioSourceIds.Count, Name = clip.name, Clip = clip }; + _audioSourceIds.Add(ad); + return ad; + } + + /// + /// Sets up and process the audio emitter data and sets the extension data for + /// the exporter to write to glb + /// + /// supplied iunit which is not currently used + /// the audio source in the unity scene to parse + internal void ProcessAudioSource(IUnit unit, AudioSource audioSource) + { + List audioDataClips = new List(); + List audioEmitters = new List(); + + var clip = audioSource.clip; + + var audioSourceId = AddAudioSource(audioSource); + + var emitterId = new AudioEmitterId + { + Id = audioSourceId.Id, + Root = _gltfRoot + }; + + var emitter = new KHR_PositionalAudioEmitter() + { + type = "positional", + sources = new List() { new AudioSourceId() { Id = audioSourceId.Id, Root = _gltfRoot } }, + gain = audioSource.volume, + minDistance = audioSource.minDistance, + maxDistance = audioSource.maxDistance, + distanceModel = PositionalAudioDistanceModel.linear + }; + + audioEmitters.Add(emitter); + + var path = AssetDatabase.GetAssetPath(clip); + + var fileName = Path.GetFileName(path); + var settings = GLTFSettings.GetOrCreateSettings(); + + + string savePath = string.Empty; + + var audio = new KHR_AudioData(); + + var fileStream = new FileStream(path, FileMode.Open, FileAccess.Read); + + if (settings != null && !string.IsNullOrEmpty(settings.SaveFolderPath) && _saveAudioToFile) + { + string uriPath; + (uriPath, savePath) = GetRelativePath(settings.SaveFolderPath); + + if (!Directory.Exists(savePath)) + { + Directory.CreateDirectory(savePath); + } + savePath += (Path.DirectorySeparatorChar + fileName); + using (var fileWriteStream = new FileStream(savePath, FileMode.Create)) + { + byte[] data = new byte[fileStream.Length]; + fileStream.Read(data, 0, (int)fileStream.Length); + + fileWriteStream.Write(data, 0, data.Length); + } + + audio.uri = uriPath + fileName; //"." + Path.DirectorySeparatorChar + AudioRelDirectory + Path.DirectorySeparatorChar + fileName; + audio.mimeType = MimeTypeString; + } + else + { + var result = exporter.ExportFile(fileName, "audio/mpeg", fileStream); + audio.uri = result.uri; + audio.mimeType = result.mimeType; + audio.bufferView = result.bufferView; + + } + + var audioData = new List(); + + audioData.Add(audio); + + var audioSources = new List(); + + var khrAudio = new KHR_AudioSource + { + audio = new AudioDataId { Id = audioSourceId.Id, Root = _gltfRoot }, + autoPlay = audioSource.playOnAwake, + loop = audioSource.loop, + gain = audioSource.volume, + sourceName = Path.GetFileNameWithoutExtension(path) + }; + + audioSources.Add(khrAudio); + + var extension = new KHR_audio + { + audio = new List(audioData), + sources = new List(audioSources), + emitters = new List(audioEmitters), + }; + + if (_gltfRoot != null) + { + _gltfRoot.AddExtension(GltfAudioExtension.AudioExtensionName, (IExtension)extension); + exporter.DeclareExtensionUsage(GltfAudioExtension.AudioExtensionName); + } + } + + /// + /// Gets the relative paths and save paths from the supplied full path arg + /// + /// full save path + /// + internal (string, string) GetRelativePath(string fullSavePath) + { + string relPath = "." + Path.DirectorySeparatorChar + AudioRelDirectory + Path.DirectorySeparatorChar; + string savePath = Path.GetFullPath(fullSavePath) + Path.DirectorySeparatorChar + AudioRelDirectory; + + return (relPath, savePath); + } + + /// + /// We are running this, but the converters and graph are not currently used other than + /// the audio source literal elements + /// + /// + /// + private static LinkedList TopologicalSort(IEnumerable nodes) + { + var sorted = new LinkedList(); + var visited = new Dictionary(); + + void Visit(IUnit node) + { + bool inProcess; + bool alreadyVisited = visited.TryGetValue(node, out inProcess); + + if (alreadyVisited) + { + if (inProcess) + { + // TODO: Should quit the topological sort and cancel the export + // throw new ArgumentException("Cyclic dependency found."); + } + } + else + { + visited[node] = true; + + // Get the dependencies from incoming connections and ignore self-references + HashSet dependencies = new HashSet(); + foreach (IUnitConnection connection in node.connections) + { + if (connection.source.unit != node) + { + dependencies.Add(connection.source.unit); + } + } + + foreach (IUnit dependency in dependencies) + { + Visit(dependency); + } + + visited[node] = false; + sorted.AddLast(node); + } + } + + foreach (var node in nodes) + { + Visit(node); + } + + return sorted; + } + + + + //private static LinkedList TopologicalSort(IEnumerable nodes) + //{ + // var sorted = new LinkedList(); + // var visited = new Dictionary(); + + // void Visit(IUnit node) + // { + // bool inProcess; + // bool alreadyVisited = visited.TryGetValue(node, out inProcess); + + // if (alreadyVisited) + // { + // if (inProcess) + // { + // // TODO: Should quit the topological sort and cancel the export + // // throw new ArgumentException("Cyclic dependency found."); + // } + // } + // else + // { + // visited[node] = true; + + // // Get the dependencies from incoming connections and ignore self-references + // HashSet dependencies = new HashSet(); + // foreach (IUnitConnection connection in node.connections) + // { + // if (connection.source.unit != node) + // { + // dependencies.Add(connection.source.unit); + // } + // } + + // foreach (IUnit dependency in dependencies) + // { + // Visit(dependency); + // } + + // visited[node] = false; + // sorted.AddLast(node); + // } + // } + + // foreach (var node in nodes) + // { + // Visit(node); + // } + + // return sorted; + //} + + // remaining context callbacks which are not currently being used. + public override void BeforeNodeExport(GLTFSceneExporter exporter, GLTFRoot gltfRoot, Transform transform, Node node) { } + public override void AfterNodeExport(GLTFSceneExporter exporter, GLTFRoot gltfRoot, Transform transform, Node node){} + public override bool BeforeMaterialExport(GLTFSceneExporter exporter, GLTFRoot gltfRoot, Material material, GLTFMaterial materialNode) => false; + public override void AfterMaterialExport(GLTFSceneExporter exporter, GLTFRoot gltfRoot, Material material, GLTFMaterial materialNode) { } + public override void BeforeTextureExport(GLTFSceneExporter exporter, ref GLTFSceneExporter.UniqueTexture texture, string textureSlot) { } + public override void AfterTextureExport(GLTFSceneExporter exporter, GLTFSceneExporter.UniqueTexture texture, int index, GLTFTexture tex) { } + public override void AfterPrimitiveExport(GLTFSceneExporter exporter, Mesh mesh, MeshPrimitive primitive, int index) { } + public override void AfterMeshExport(GLTFSceneExporter exporter, Mesh mesh, GLTFMesh gltfMesh, int index) { } + } +} diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportContext.cs.meta b/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportContext.cs.meta new file mode 100644 index 000000000..80599f492 --- /dev/null +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportContext.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: c2668537658724f4caa5fc951f317a09 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportPlugin.cs b/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportPlugin.cs new file mode 100644 index 000000000..93d09b878 --- /dev/null +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportPlugin.cs @@ -0,0 +1,37 @@ +using System.Collections; +using System.Collections.Generic; +using UnityEngine; +using UnityGLTF; +using UnityEngine; +using UnityGLTF.Interactivity.Schema; +using UnityGLTF.Plugins; + +namespace UnityGLTF.Interactivity.VisualScripting +{ + /// + /// Pluging for the audio emitter extension + /// +public class GLTFAudioExportPlugin : VisualScriptingExportPlugin + { + /// + /// Plugin name + /// + public override string DisplayName => "GLTF_Audio_Export"; + + /// + /// plugin descriptions + /// + public override string Description => "Exports KHR Audio source nodes(literals) from the visual scripting graph"; + + /// + /// Creates a audio export context with the save to external file arg as true + /// + /// + /// + public override GLTFExportPluginContext CreateInstance(ExportContext context) + { + return new GLTFAudioExportContext(this);//, true); + } + } + +} \ No newline at end of file diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportPlugin.cs.meta b/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportPlugin.cs.meta new file mode 100644 index 000000000..7a880b6b4 --- /dev/null +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportPlugin.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 0e7887da9e2d87d4a83ae85e8d15e005 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/GltfAudioNodeHelper.cs b/Editor/Scripts/Interactivity/VisualScriptingExport/GltfAudioNodeHelper.cs new file mode 100644 index 000000000..4c1b5c166 --- /dev/null +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/GltfAudioNodeHelper.cs @@ -0,0 +1,214 @@ +namespace UnityGLTF.Audio +{ + using System.Collections.Generic; + using System.Linq; + using GLTF.Schema; + using UnityEngine; + using Unity.VisualScripting; + using UnityGLTF.Plugins; + using UnityGLTF.Interactivity.VisualScripting.Export; + using UnityGLTF.Interactivity.VisualScripting; + + public static class GltfAudioNodeHelper + { + public static Dictionary GetTranslatableNodes( + IEnumerable sortedNodes, GLTFAudioExportContext exportContext) + { + Dictionary validNodes = + new Dictionary(); + + foreach (IUnit unit in sortedNodes) + { + if (unit is Literal || unit is This || unit is Null) + continue; + + UnitExporter unitExporter = UnitExporterRegistry.CreateUnitExporter(exportContext, unit); + //if (unitExporter != null) + //{ + // if (unitExporter.IsTranslatable && unitExporter.Nodes.Length > 0) + // validNodes.Add(unit, unitExporter); + //} + //else + // Debug.LogWarning("ExportNode is null for unit: " + Log(unit)+ " of type: " + unit.GetType()); + } + + return validNodes; + } + + static string Log(IUnit unit) + { + if (unit is InvokeMember invokeMember) + return unit.ToString() + " Invoke: " + invokeMember.member.declaringType + "." + invokeMember.member.name; + if (unit is SetMember setMember) + return unit.ToString() + " Set: " + setMember.target + "." + setMember.member.name; + if (unit is GetMember getMember) + return unit.ToString() + " Get: " + getMember.target + "." + getMember.member.name; + if (unit is Expose expose) + return unit.ToString() + " Expose: " + expose.target + " Type: " + expose.type; + return unit.ToString(); + } + + public static readonly string IdPointerNodeIndex = "nodeIndex"; + public static readonly string IdPointerMeshIndex = "meshIndex"; + public static readonly string IdPointerMaterialIndex = "materialIndex"; + public static readonly string IdPointerAnimationIndex = "animationIndex"; + + //public static void AddPointerConfig(GltfInteractivityNode node, string pointer, string gltfType) + //{ + // AddPointerConfig(node, pointer, GltfTypes.TypeIndexByGltfSignature(gltfType)); + //} + + //public static void AddPointerConfig(GltfInteractivityNode node, string pointer, int gltfType) + //{ + // var pointerConfig = node.ConfigurationData[Pointer_SetNode.IdPointer]; + // pointerConfig.Value = pointer; + // var typeConfig = node.ConfigurationData[Pointer_SetNode.IdPointerValueType]; + // typeConfig.Value = gltfType; + //} + + public static bool IsMainCameraInInput(IUnit unit) + { + var target = unit.inputs.FirstOrDefault( i => i.key == "target"); + return IsMainCameraInInput(target as ValueInput); + } + + public static bool IsMainCameraInInput(ValueInput target) + { + if (target == null) + return false; + + if (!target.hasValidConnection) + return false; + + var connection = target.connections.First(); + + if (connection.source != null && connection.source.unit is GetMember getMemberTarget) + { + if (getMemberTarget.member.type == typeof(Camera) && getMemberTarget.member.name == nameof(Camera.main)) + return true; + } + + return false; + } + + //public static void AddPointerTemplateValueInput(GltfInteractivityNode node, string pointerId, int? index = null) + //{ + // node.ValueSocketConnectionData.Add(pointerId, new GltfInteractivityNode.ValueSocketData() + // { + // Value = index, + // Type = GltfTypes.TypeIndexByGltfSignature("int"), + // typeRestriction = TypeRestriction.LimitToInt, + // }); + //} + + // Get the index of a named property that has been exported to GLTF + public static int GetNamedPropertyGltfIndex(string objectName, IEnumerable gltfRootProperties) + { + int i = 0; + foreach (GLTFChildOfRootProperty property in gltfRootProperties) + { + if (objectName == property.Name) + { + return i; + } + i++; + } + + Debug.LogWarning($"Could not find {objectName} in the GLTF List: {gltfRootProperties.ToString()}"); + return -1; + } + + //public static GameObject GetGameObjectFromValueInput(ValueInput value, Dictionary defaultValues, GltfInteractivityExportContext exportContext) + //{ + // if (value.hasValidConnection == false) + // { + // // If there are no connections, then we can return the non-null default value + // if (value.hasDefaultValue && defaultValues.ContainsKey(value.key)) + // { + // GameObject defaultGameObject = null; + // if (defaultValues[value.key] is GameObject) + // defaultGameObject = (GameObject)defaultValues[value.key]; + // else if (defaultValues[value.key] is Component component) + // defaultGameObject = component.gameObject; + + // // If the value is null, then likely it's set to "this" in the UI + // // meaning that it's pointing to the Gameobject that owns the graph + // if (defaultGameObject == null) + // { + // // The value is referencing the gameobject that owns the graph + // return exportContext.ActiveScriptMachine.gameObject; + // } + // else + // { + // return defaultGameObject; + // } + // } + // return null; + // } + // else + // if (value.hasValidConnection && value.connections.First().source.unit is Literal literal) + // { + // // If there is a connection, then we can return the value of the literal + // if (literal.value is GameObject) + // return literal.value as GameObject; + // else if (literal.value is Component component) + // return component.gameObject; + + // return null; + // } + // else if (value.hasValidConnection && value.connections.First().source.unit is GetVariable getVariable) + // { + // // If there is a connection, then we can return the value of the literal + // var getVarValue = exportContext.GetVariableValueRaw(getVariable, out _, out var varType); + // if (getVariable != null && getVarValue is GameObject gameObject) + // return gameObject; + // else if (getVariable != null && getVarValue is Component component) + // return component.gameObject; + + // if (varType != null) + // return null; + + // Debug.LogError("Could not get the default value of the GetVariable node: " + getVariable.ToString()); + // return null; + // } + // else + // { + // // TODO: Parse the input nodes for a value + // Debug.LogWarning("This ValueInput has a valid connection, but we only support self-references"); + // return null; + // } + //} + + public static bool GetDefaultValue (IUnit unit, string key, out T tOut) + { + tOut = default(T); + foreach (ValueInput value in unit.valueInputs) + { + if (value.key == key) + { + if (value.type != typeof(T)) + { + Debug.LogWarning($"{key} value key was found in unit {unit.ToString()}, but it is not of type {typeof(T)}, it is {value.type}"); + return false; + } + + if (value.hasDefaultValue && unit.defaultValues.ContainsKey(value.key)) + { + tOut = (T) unit.defaultValues[value.key]; + return true; + // TODO: Generalize this for the Gameobject edge-case where null means a self-reference + } + else + { + Debug.LogWarning($"{key} value key was found in unit {unit.ToString()}, but it has no default values"); + return false; + // TODO: Check if it has a predictable value instead that can be retrieved from static connections. + } + } + } + + Debug.LogWarning(key + " value key could not be found in the Unit: " + unit.ToString()); + return false; + } + } +} diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/GltfAudioNodeHelper.cs.meta b/Editor/Scripts/Interactivity/VisualScriptingExport/GltfAudioNodeHelper.cs.meta new file mode 100644 index 000000000..5e9709ef6 --- /dev/null +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/GltfAudioNodeHelper.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 5e4d541b70b6a9d4fb18a5fb62079e82 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourcePauseNode.cs b/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourcePauseNode.cs new file mode 100644 index 000000000..f4166aed5 --- /dev/null +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourcePauseNode.cs @@ -0,0 +1,66 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using Unity.VisualScripting; +using UnityEditor; +using UnityEngine; +using UnityGLTF.Interactivity.Schema; + +namespace UnityGLTF.Interactivity.VisualScripting.Export +{ + public class AudioSource_PauseUnitExport : IUnitExporter + { + public System.Type unitType + { + get => typeof(InvokeMember); + } + + [InitializeOnLoadMethod] + private static void Register() + { + InvokeUnitExport.RegisterInvokeExporter(typeof(AudioSource), nameof(AudioSource.Pause), + new AudioSource_PauseUnitExport()); + } + + + public bool InitializeInteractivityNodes(UnitExporter unitExporter) + { + InvokeMember unit = unitExporter.unit as InvokeMember; + + GameObject target = UnitsHelper.GetGameObjectFromValueInput( + unit.target, unit.defaultValues, unitExporter.exportContext); + + if (target == null) + { + UnitExportLogging.AddErrorLog(unit, "Can't resolve target GameObject"); + return false; + } + + var audio = target.GetComponent(); + if (!audio) + { + UnitExportLogging.AddErrorLog(unit, "Target GameObject does not have an Audio component."); + return false; + } + + var clip = audio.clip; + if (clip != null) + { + var name = clip.name; + + GLTFAudioExportContext.AudioDescription description = GLTFAudioExportContext.AddAudioSource(audio); + + var node = unitExporter.CreateNode(new Audio_PauseNode()); + node.ValueSocketConnectionData[Audio_PauseNode.IdValueAudio].Value = description.Id; + node.ValueSocketConnectionData[Audio_PauseNode.IdValueNode].Value = $"audio/node/{description.Id}"; + + unitExporter.MapInputPortToSocketName(unit.enter, Audio_PauseNode.IdFlowIn, node); + // There should only be one output flow from the Animator.Play node + unitExporter.MapOutFlowConnectionWhenValid(unit.exit, Audio_PauseNode.IdFlowOut, node); + } + + return true; + } + + } +} diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourcePauseNode.cs.meta b/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourcePauseNode.cs.meta new file mode 100644 index 000000000..a9f4de520 --- /dev/null +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourcePauseNode.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 45846cb16911f9a4687a6914e8e3a078 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourcePlayNode.cs b/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourcePlayNode.cs new file mode 100644 index 000000000..1973dd54c --- /dev/null +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourcePlayNode.cs @@ -0,0 +1,79 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using Unity.VisualScripting; +using UnityEditor; +using UnityEngine; +using UnityGLTF.Interactivity; +using UnityGLTF.Interactivity.Schema; +using UnityGLTF.Plugins; + +namespace UnityGLTF.Interactivity.VisualScripting.Export +{ + /// + /// Unit Exporte for audio source play node + /// + public class AudioSource_PlayUnitExport : IUnitExporter + { + public System.Type unitType + { + get => typeof(InvokeMember); + } + + /// + /// Register the instance of the play exporter + /// + [InitializeOnLoadMethod] + private static void Register() + { + InvokeUnitExport.RegisterInvokeExporter(typeof(AudioSource), nameof(AudioSource.Play), + new AudioSource_PlayUnitExport()); + } + + + /// + /// Sets up the unitexport with the correct data and associations + /// + /// + /// + public bool InitializeInteractivityNodes(UnitExporter unitExporter) + { + InvokeMember unit = unitExporter.unit as InvokeMember; + + GameObject target = UnitsHelper.GetGameObjectFromValueInput( + unit.target, unit.defaultValues, unitExporter.exportContext); + + if (target == null) + { + UnitExportLogging.AddErrorLog(unit, "Can't resolve target GameObject"); + return false; + } + + var audio = target.GetComponent(); + if (!audio) + { + UnitExportLogging.AddErrorLog(unit, "Target GameObject does not have an Audio component."); + return false; + } + + var clip = audio.clip; + if (clip != null) + { + var name = clip.name; + + GLTFAudioExportContext.AudioDescription description = GLTFAudioExportContext.AddAudioSource(audio); + + var node = unitExporter.CreateNode(new Audio_StartNode()); + node.ValueSocketConnectionData[Audio_StartNode.IdValueAudio].Value = description.Id; + node.ValueSocketConnectionData[Audio_StartNode.IdValueNode].Value = $"audio/node/{description.Id}"; + + unitExporter.MapInputPortToSocketName(unit.enter, Audio_StartNode.IdFlowIn, node); + // There should only be one output flow from the Animator.Play node + unitExporter.MapOutFlowConnectionWhenValid(unit.exit, Audio_StartNode.IdFlowOut, node); + + } + + return true; + } + } +} diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourcePlayNode.cs.meta b/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourcePlayNode.cs.meta new file mode 100644 index 000000000..bdf792a92 --- /dev/null +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourcePlayNode.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 110f279ff07461a41a0fbd8f963a67e3 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceStopNode.cs b/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceStopNode.cs new file mode 100644 index 000000000..8f38f3125 --- /dev/null +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceStopNode.cs @@ -0,0 +1,64 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using Unity.VisualScripting; +using UnityEditor; +using UnityEngine; +using UnityGLTF.Interactivity.Schema; + +namespace UnityGLTF.Interactivity.VisualScripting.Export +{ + public class AudioSource_StopUnitExport : IUnitExporter + { + public System.Type unitType + { + get => typeof(InvokeMember); + } + + [InitializeOnLoadMethod] + private static void Register() + { + InvokeUnitExport.RegisterInvokeExporter(typeof(AudioSource), nameof(AudioSource.Stop), + new AudioSource_StopUnitExport()); + } + + public bool InitializeInteractivityNodes(UnitExporter unitExporter) + { + InvokeMember unit = unitExporter.unit as InvokeMember; + + GameObject target = UnitsHelper.GetGameObjectFromValueInput( + unit.target, unit.defaultValues, unitExporter.exportContext); + + if (target == null) + { + UnitExportLogging.AddErrorLog(unit, "Can't resolve target GameObject"); + return false; + } + + var audio = target.GetComponent(); + if (!audio) + { + UnitExportLogging.AddErrorLog(unit, "Target GameObject does not have an Audio component."); + return false; + } + + var clip = audio.clip; + if (clip != null) + { + var name = clip.name; + + GLTFAudioExportContext.AudioDescription description = GLTFAudioExportContext.AddAudioSource(audio); + + var node = unitExporter.CreateNode(new Audio_StopNode()); + node.ValueSocketConnectionData[Audio_StopNode.IdValueAudio].Value = description.Id; + node.ValueSocketConnectionData[Audio_StopNode.IdValueNode].Value = $"audio/node/{description.Id}"; + + unitExporter.MapInputPortToSocketName(unit.enter, Audio_StartNode.IdFlowIn, node); + // There should only be one output flow from the Animator.Play node + unitExporter.MapOutFlowConnectionWhenValid(unit.exit, Audio_StartNode.IdFlowOut, node); + } + + return true; + } + } +} diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceStopNode.cs.meta b/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceStopNode.cs.meta new file mode 100644 index 000000000..d296ca3c5 --- /dev/null +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceStopNode.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 58e8b3f26951cd44a9c73ffb4cff93f5 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceUnPauseNode.cs b/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceUnPauseNode.cs new file mode 100644 index 000000000..ade497fcb --- /dev/null +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceUnPauseNode.cs @@ -0,0 +1,65 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using Unity.VisualScripting; +using UnityEditor; +using UnityEngine; +using UnityGLTF.Interactivity.Schema; + +namespace UnityGLTF.Interactivity.VisualScripting.Export +{ + public class AudioSource_UnPauseUnitExport : IUnitExporter + { + public System.Type unitType + { + get => typeof(InvokeMember); + } + + [InitializeOnLoadMethod] + private static void Register() + { + InvokeUnitExport.RegisterInvokeExporter(typeof(AudioSource), nameof(AudioSource.UnPause), + new AudioSource_UnPauseUnitExport()); + } + + + public bool InitializeInteractivityNodes(UnitExporter unitExporter) + { + InvokeMember unit = unitExporter.unit as InvokeMember; + + GameObject target = UnitsHelper.GetGameObjectFromValueInput( + unit.target, unit.defaultValues, unitExporter.exportContext); + + if (target == null) + { + UnitExportLogging.AddErrorLog(unit, "Can't resolve target GameObject"); + return false; + } + + var audio = target.GetComponent(); + if (!audio) + { + UnitExportLogging.AddErrorLog(unit, "Target GameObject does not have an Audio component."); + return false; + } + + var clip = audio.clip; + if (clip != null) + { + var name = clip.name; + + GLTFAudioExportContext.AudioDescription description = GLTFAudioExportContext.AddAudioSource(audio); + + var node = unitExporter.CreateNode(new Audio_UnPauseNode()); + node.ValueSocketConnectionData[Audio_UnPauseNode.IdValueAudio].Value = description.Id; + node.ValueSocketConnectionData[Audio_UnPauseNode.IdValueNode].Value = $"audio/node/{description.Id}"; + + unitExporter.MapInputPortToSocketName(unit.enter, Audio_UnPauseNode.IdFlowIn, node); + // There should only be one output flow from the Animator.Play node + unitExporter.MapOutFlowConnectionWhenValid(unit.exit, Audio_UnPauseNode.IdFlowOut, node); + } + + return true; + } + } +} diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceUnPauseNode.cs.meta b/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceUnPauseNode.cs.meta new file mode 100644 index 000000000..a6eeb57c8 --- /dev/null +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceUnPauseNode.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: f629e869b9ea7ae4584623b6a5f59f18 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceUnitExport.cs b/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceUnitExport.cs new file mode 100644 index 000000000..d2ca0caf1 --- /dev/null +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceUnitExport.cs @@ -0,0 +1,107 @@ +using System.Collections; +using System.Collections.Generic; +using UnityEngine; +using UnityGLTF.Interactivity.Schema; +using Unity.VisualScripting; +using System; +using UnityEditor; + + +namespace UnityGLTF.Interactivity.VisualScripting.Export +{ + public class AudioSourceUnitExport : IUnitExporter + { + //public Type unitType { get => typeof(AudioSource); } + public System.Type unitType + { + get => typeof(UnityEngine.AudioSource); + } + + [InitializeOnLoadMethod] + private static void Register() + { +// InvokeUnitExport.RegisterInvokeExporter(typeof(AudioSource), nameof(AudioSource), +// new AudioSourceUnitExport()); + UnitExporterRegistry.RegisterExporter(new AudioSourceUnitExport()); + } + + public bool InitializeInteractivityNodes(UnitExporter unitExporter) + { + return true; + /* + var unit = unitExporter.unit as AudioSource; + + if (unit.target == null) + return false; + + // Regular pointer/set + + var materialTemplate = "/materials/{" + GltfInteractivityNodeHelper.IdPointerMaterialIndex + "}/"; + var template = materialTemplate + "pbrMetallicRoughness/baseColorFactor"; + + if (unit is SetMember setMember) + { + var node = unitExporter.CreateNode(new Pointer_SetNode()); + unitExporter.MapInputPortToSocketName(setMember.assign, Pointer_SetNode.IdFlowIn, node); + unitExporter.MapInputPortToSocketName(setMember.input, Pointer_SetNode.IdValue, node); + unitExporter.MapOutFlowConnectionWhenValid(setMember.assigned, Pointer_SetNode.IdFlowOut, node); + + node.SetupPointerTemplateAndTargetInput(GltfInteractivityNodeHelper.IdPointerMaterialIndex, + setMember.target, template, GltfTypes.Float4); + } + else if (unit is InvokeMember invokeMember) + { + // first parameter is the color property name – so based on that we can determine what pointer to set + // var colorPropertyName = invokeMember.inputParameters[0]; + bool hasAlpha = true; + if (unitExporter.IsInputLiteralOrDefaultValue(invokeMember.inputParameters[0], out var colorPropertyName)) + { + var gltfProperty = MaterialPointerHelper.GetPointer(unitExporter, (string)colorPropertyName, out var map); + if (gltfProperty == null) + { + UnitExportLogging.AddErrorLog(unit, "color property name is not supported."); + return false; + } + + hasAlpha = map.ExportKeepColorAlpha; + template = materialTemplate + gltfProperty; + } + else + { + UnitExportLogging.AddErrorLog(unit, "color property name is not a literal or default value, which is not supported."); + return false; + } + + var node = unitExporter.CreateNode(new Pointer_SetNode()); + node.FlowIn(Pointer_SetNode.IdFlowIn).MapToControlInput(invokeMember.enter); + node.FlowOut(Pointer_SetNode.IdFlowOut).MapToControlOutput(invokeMember.exit); + + if (hasAlpha) + { + node.ValueIn(Pointer_SetNode.IdValue).MapToInputPort(invokeMember.inputParameters[1]).SetType(TypeRestriction.LimitToFloat4); + + node.SetupPointerTemplateAndTargetInput(GltfInteractivityNodeHelper.IdPointerMaterialIndex, + invokeMember.target, template, GltfTypes.Float4); + } + else + { + var extract = unitExporter.CreateNode(new Math_Extract4Node()); + var combine = unitExporter.CreateNode(new Math_Combine3Node()); + extract.ValueIn("a").MapToInputPort(invokeMember.inputParameters[1]) + .SetType(TypeRestriction.LimitToFloat4); + + combine.ValueIn("a").ConnectToSource(extract.ValueOut("0")); + combine.ValueIn("b").ConnectToSource(extract.ValueOut("1")); + combine.ValueIn("c").ConnectToSource(extract.ValueOut("2")); + + node.ValueIn(Pointer_SetNode.IdValue).ConnectToSource(combine.FirstValueOut()); + node.SetupPointerTemplateAndTargetInput(GltfInteractivityNodeHelper.IdPointerMaterialIndex, + invokeMember.target, template, GltfTypes.Float3); + } + } + return true; + */ + } + } + +} diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceUnitExport.cs.meta b/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceUnitExport.cs.meta new file mode 100644 index 000000000..2647b231e --- /dev/null +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceUnitExport.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: e5ab0e7aefc298c43815f4338ff355bd +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/VisualScriptingExportContext.cs b/Editor/Scripts/Interactivity/VisualScriptingExport/VisualScriptingExportContext.cs index 4f999211d..b1efc66e8 100644 --- a/Editor/Scripts/Interactivity/VisualScriptingExport/VisualScriptingExportContext.cs +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/VisualScriptingExportContext.cs @@ -65,7 +65,7 @@ public class ExportGraph public ScriptMachine ActiveScriptMachine = null; public GLTFRoot ActiveGltfRoot = null; - public GLTFSceneExporter exporter { get; private set; } + public GLTFSceneExporter exporter { get; set; } internal List variables = new List(); internal List customEvents = new List(); diff --git a/Runtime/Scripts/Interactivity/Schema/GltfAudioExtension.cs b/Runtime/Scripts/Interactivity/Schema/GltfAudioExtension.cs new file mode 100644 index 000000000..5af836189 --- /dev/null +++ b/Runtime/Scripts/Interactivity/Schema/GltfAudioExtension.cs @@ -0,0 +1,48 @@ +using GLTF.Schema; + +namespace UnityGLTF.Interactivity.Schema +{ + using System; + using System.Linq; + using GLTF.Schema; + using Newtonsoft.Json.Linq; + using UnityGLTF.Interactivity; + + [Serializable] + public class GltfAudioExtension : IExtension + { + public const string AudioExtensionName = "KHR_audio_emitter"; + + public GltfInteractivityGraph[] graphs; + public int graph = 0; + + public GltfAudioExtension() + { + } + + public JProperty Serialize() + { + JObject jo = new JObject + { + new JProperty("graphs", + new JArray( + from gr in graphs + select gr.SerializeObject())), + new JProperty("graph", graph) + }; + + JProperty extension = + new JProperty(GltfAudioExtension.AudioExtensionName, jo); + return extension; + } + + public IExtension Clone(GLTFRoot root) + { + return new GltfAudioExtension() + { + graph = graph, + graphs = graphs + }; + } + } +} diff --git a/Runtime/Scripts/Interactivity/Schema/GltfAudioExtension.cs.meta b/Runtime/Scripts/Interactivity/Schema/GltfAudioExtension.cs.meta new file mode 100644 index 000000000..6bb0246f1 --- /dev/null +++ b/Runtime/Scripts/Interactivity/Schema/GltfAudioExtension.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: c3e5287d067909b47adc4ca1ca9cc9b2 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Scripts/Interactivity/Schema/GltfTypes.cs b/Runtime/Scripts/Interactivity/Schema/GltfTypes.cs index a74c47f77..b5d3ff5c4 100644 --- a/Runtime/Scripts/Interactivity/Schema/GltfTypes.cs +++ b/Runtime/Scripts/Interactivity/Schema/GltfTypes.cs @@ -35,9 +35,9 @@ public class GltfTypes new TypeMapping(Float3, new [] {typeof(Vector3)}), new TypeMapping(Float4, new [] {typeof(Color), typeof(Color32), typeof(Vector4), typeof(Quaternion)}), new TypeMapping(Float4x4, new [] {typeof(Matrix4x4)}), - // new TypeMapping(String, new [] {typeof(string)}), - new TypeMapping("custom", new [] {typeof(string)}, "AMZN_interactivity_string"), new TypeMapping(IntArray, new [] {typeof(int[])}), + new TypeMapping(String, new [] {typeof(string)}), +// new TypeMapping("custom", new [] {typeof(string)}, "AMZN_interactivity_string"), }; public static int GetComponentCount(int typeIndex) diff --git a/Runtime/Scripts/Interactivity/Schema/KHRAudioSchemas.cs b/Runtime/Scripts/Interactivity/Schema/KHRAudioSchemas.cs new file mode 100644 index 000000000..0d945ceef --- /dev/null +++ b/Runtime/Scripts/Interactivity/Schema/KHRAudioSchemas.cs @@ -0,0 +1,341 @@ + +using System; +using System.Collections.Generic; +using GLTF.Schema; +using Newtonsoft.Json.Linq; +using UnityEngine; + +namespace UnityGLTF.Plugins.Experimental +{ + public enum PositionalAudioDistanceModel + { + linear, + inverse, + exponential, + } + + [Serializable] + public class AudioEmitterId : GLTFId { + public AudioEmitterId() + { + } + + public AudioEmitterId(AudioEmitterId id, GLTFRoot newRoot) : base(id, newRoot) + { + } + + public override KHR_AudioEmitter Value + { + get + { + if (Root.Extensions.TryGetValue(KHR_audio.ExtensionName, out IExtension iextension)) + { + KHR_audio extension = iextension as KHR_audio; + return extension.emitters[Id]; + } + else + { + throw new Exception("KHR_audio not found on root object"); + } + } + } + } + + [Serializable] + public class AudioSourceId : GLTFId { + public AudioSourceId() + { + } + + public AudioSourceId(AudioSourceId id, GLTFRoot newRoot) : base(id, newRoot) + { + } + + public override KHR_AudioSource Value + { + get + { + if (Root.Extensions.TryGetValue(KHR_audio.ExtensionName, out IExtension iextension)) + { + KHR_audio extension = iextension as KHR_audio; + return extension.sources[Id]; + } + else + { + throw new Exception("KHR_audio not found on root object"); + } + } + } + } + + [Serializable] + public class AudioDataId : GLTFId { + public AudioDataId() + { + } + + public AudioDataId(AudioDataId id, GLTFRoot newRoot) : base(id, newRoot) + { + } + + public override KHR_AudioData Value + { + get + { + if (Root.Extensions.TryGetValue(KHR_audio.ExtensionName, out IExtension iextension)) + { + KHR_audio extension = iextension as KHR_audio; + return extension.audio[Id]; + } + else + { + throw new Exception("KHR_audio not found on root object"); + } + } + } + } + + [Serializable] + public class KHR_SceneAudioEmittersRef : IExtension { + public List emitters; + + public JProperty Serialize() { + var jo = new JObject(); + JProperty jProperty = new JProperty(KHR_audio.ExtensionName, jo); + + JArray arr = new JArray(); + + foreach (var emitter in emitters) { + arr.Add(emitter.Id); + } + + jo.Add(new JProperty(nameof(emitters), arr)); + + return jProperty; + } + + public IExtension Clone(GLTFRoot root) + { + return new KHR_SceneAudioEmittersRef() { emitters = emitters }; + } + } + + [Serializable] + public class KHR_NodeAudioEmitterRef : IExtension { + public AudioEmitterId emitter; + + public JProperty Serialize() { + var jo = new JObject(); + JProperty jProperty = new JProperty(KHR_audio.ExtensionName, jo); + jo.Add(new JProperty(nameof(emitter), emitter.Id)); + return jProperty; + } + + public IExtension Clone(GLTFRoot root) + { + return new KHR_NodeAudioEmitterRef() { emitter = emitter }; + } + } + + [Serializable] + public class KHR_AudioEmitter : GLTFChildOfRootProperty { + + public string type; + public float gain; + public List sources; + + public virtual JObject Serialize() { + var jo = new JObject(); + + jo.Add(nameof(type), type); + +// if (gain != 1.0f) { + jo.Add(nameof(gain), gain); +// } + + if (sources != null && sources.Count > 0) { + JArray arr = new JArray(); + + foreach (var source in sources) { + arr.Add(source.Id); + } + + jo.Add(new JProperty(nameof(sources), arr)); + } + + return jo; + } + } + + [Serializable] + public class KHR_PositionalAudioEmitter : KHR_AudioEmitter { + + public float coneInnerAngle; + public float coneOuterAngle; + public float coneOuterGain; + public PositionalAudioDistanceModel distanceModel; + public float minDistance; + public float maxDistance; + public float refDistance; + public float rolloffFactor; + + public override JObject Serialize() { + var jo = base.Serialize(); + + var positional = new JObject(); + + //if (!Mathf.Approximately(coneInnerAngle, Mathf.PI * 2)) { + // positional.Add(new JProperty(nameof(coneInnerAngle), coneInnerAngle)); + //} + + //if (!Mathf.Approximately(coneInnerAngle, Mathf.PI * 2)) { + // positional.Add(new JProperty(nameof(coneOuterAngle), coneOuterAngle)); + //} + + //if (coneOuterGain != 0.0f) { + // positional.Add(new JProperty(nameof(coneOuterGain), coneOuterGain)); + //} + + + if (distanceModel != PositionalAudioDistanceModel.inverse) { + positional.Add(new JProperty(nameof(distanceModel), distanceModel.ToString())); + } + + if (minDistance != 0) + { + positional.Add(new JProperty(nameof(minDistance), minDistance)); + } + + if (maxDistance != 10000.0f) { + positional.Add(new JProperty(nameof(maxDistance), maxDistance)); + } + + + //if (refDistance != 1.0f) { + // positional.Add(new JProperty(nameof(refDistance), refDistance)); + //} + + //if (rolloffFactor != 1.0f) { + // positional.Add(new JProperty(nameof(rolloffFactor), rolloffFactor)); + //} + + jo.Add("positional", positional); + + return jo; + } + } + + [Serializable] + public class KHR_AudioSource : GLTFChildOfRootProperty { + public string sourceName; + public bool autoPlay; + public float gain; + public bool loop; + public AudioDataId audio; + + public JObject Serialize() { + var jo = new JObject(); + +// if (autoPlay) { + jo.Add(nameof(autoPlay), autoPlay); + // } + + // if (gain != 1.0f) { + jo.Add(nameof(gain), gain); + // } + + // if (loop) { + jo.Add(nameof(loop), loop); + // } + + if (audio != null) { + jo.Add(nameof(audio), audio.Id); + } + + if (!string.IsNullOrEmpty(sourceName)) + { + jo.Add(nameof(sourceName), sourceName); + } + return jo; + } + } + + [Serializable] + public class KHR_AudioData : GLTFChildOfRootProperty { + + public string uri; + public string mimeType; + public BufferViewId bufferView; + + public JObject Serialize() { + var jo = new JObject(); + + if (uri != null) { + jo.Add(nameof(uri), uri); + } else { + jo.Add(nameof(mimeType), mimeType); + jo.Add(nameof(bufferView), bufferView.Id); + } + + return jo; + } + } + + [Serializable] + public class KHR_audio : IExtension + { + public const string ExtensionName = "KHR_audio_emitter"; + + public List audio; + public List sources; + public List emitters; + + public JProperty Serialize() + { + var jo = new JObject(); + JProperty jProperty = new JProperty(ExtensionName, jo); + + if (audio != null && audio.Count > 0) { + JArray audioArr = new JArray(); + + foreach (var audioData in audio) { + audioArr.Add(audioData.Serialize()); + } + + jo.Add(new JProperty(nameof(audio), audioArr)); + } + + + if (sources != null && sources.Count > 0) { + JArray sourceArr = new JArray(); + + foreach (var source in sources) { + sourceArr.Add(source.Serialize()); + } + + jo.Add(new JProperty(nameof(sources), sourceArr)); + } + + if (emitters != null && emitters.Count > 0) { + JArray emitterArr = new JArray(); + + foreach (var emitter in emitters) { + emitterArr.Add(emitter.Serialize()); + } + + jo.Add(new JProperty(nameof(emitters), emitterArr)); + } + + return jProperty; + } + + public IExtension Clone(GLTFRoot root) + { + return new KHR_audio() { + audio = audio, + sources = sources, + emitters = emitters, + }; + } + } +} diff --git a/Runtime/Scripts/Interactivity/Schema/KHRAudioSchemas.cs.meta b/Runtime/Scripts/Interactivity/Schema/KHRAudioSchemas.cs.meta new file mode 100644 index 000000000..d1e7d2e85 --- /dev/null +++ b/Runtime/Scripts/Interactivity/Schema/KHRAudioSchemas.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 49076143c3c264548a7d3734f637b0f3 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_PauseNode.cs b/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_PauseNode.cs new file mode 100644 index 000000000..2ac96a02e --- /dev/null +++ b/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_PauseNode.cs @@ -0,0 +1,26 @@ +namespace UnityGLTF.Interactivity.Schema +{ + /// + /// The world/startAnimation node plays a global animation + /// + /// See https://github.com/petermart/glTF-InteractivityGraph-AuthoringTool/blob/483da2161aa3d9c01ef47be8cdb2bec2d1dd3a18/src/authoring/AuthoringNodeSpecs.ts#L181 . + /// + /// + + public class Audio_PauseNode : GltfInteractivityNodeSchema + { + public override string Op { get; set; } = "audio/pause"; + + [FlowInSocketDescription()] + public const string IdFlowIn = "in"; + + [FlowOutSocketDescription()] + public const string IdFlowOut = "out"; + + [InputSocketDescription(GltfTypes.Int)] + public const string IdValueAudio = "audioSourceIndex"; + + [InputSocketDescription(GltfTypes.String)] + public const string IdValueNode = "audioSourcePath"; + } +} diff --git a/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_PauseNode.cs.meta b/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_PauseNode.cs.meta new file mode 100644 index 000000000..f43b0390f --- /dev/null +++ b/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_PauseNode.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 57a667287a24daa438a1621ff1df62e9 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_StartNode.cs b/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_StartNode.cs new file mode 100644 index 000000000..de37e4dd2 --- /dev/null +++ b/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_StartNode.cs @@ -0,0 +1,26 @@ +namespace UnityGLTF.Interactivity.Schema +{ + /// + /// The world/startAnimation node plays a global animation + /// + /// See https://github.com/petermart/glTF-InteractivityGraph-AuthoringTool/blob/483da2161aa3d9c01ef47be8cdb2bec2d1dd3a18/src/authoring/AuthoringNodeSpecs.ts#L181 . + /// + /// + + public class Audio_StartNode : GltfInteractivityNodeSchema + { + public override string Op { get; set; } = "audio/start"; + + [FlowInSocketDescription()] + public const string IdFlowIn = "in"; + + [FlowOutSocketDescription()] + public const string IdFlowOut = "out"; + + [InputSocketDescription(GltfTypes.Int)] + public const string IdValueAudio = "audioSourceIndex"; + + [InputSocketDescription(GltfTypes.String)] + public const string IdValueNode = "audioSourcePath"; + } +} diff --git a/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_StartNode.cs.meta b/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_StartNode.cs.meta new file mode 100644 index 000000000..5f5390f36 --- /dev/null +++ b/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_StartNode.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 50ca5f3a313a3114a922dfffc1e84316 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_StopNode.cs b/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_StopNode.cs new file mode 100644 index 000000000..82baaaf49 --- /dev/null +++ b/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_StopNode.cs @@ -0,0 +1,19 @@ +namespace UnityGLTF.Interactivity.Schema +{ + public class Audio_StopNode : GltfInteractivityNodeSchema + { + public override string Op { get; set; } = "audio/stop"; + + [FlowInSocketDescription()] + public const string IdFlowIn = "in"; + + [FlowOutSocketDescription()] + public const string IdFlowOut = "out"; + + [InputSocketDescription(GltfTypes.Int)] + public const string IdValueAudio = "audioSourceIndex"; + + [InputSocketDescription(GltfTypes.String)] + public const string IdValueNode = "audioSourcePath"; + } +} diff --git a/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_StopNode.cs.meta b/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_StopNode.cs.meta new file mode 100644 index 000000000..949855d75 --- /dev/null +++ b/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_StopNode.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: 34586e8f59c9f8e44bd8cd4b7bdd91c8 +timeCreated: 1733310768 \ No newline at end of file diff --git a/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_UnPauseNode.cs b/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_UnPauseNode.cs new file mode 100644 index 000000000..f2ed413b6 --- /dev/null +++ b/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_UnPauseNode.cs @@ -0,0 +1,26 @@ +namespace UnityGLTF.Interactivity.Schema +{ + /// + /// The world/startAnimation node plays a global animation + /// + /// See https://github.com/petermart/glTF-InteractivityGraph-AuthoringTool/blob/483da2161aa3d9c01ef47be8cdb2bec2d1dd3a18/src/authoring/AuthoringNodeSpecs.ts#L181 . + /// + /// + + public class Audio_UnPauseNode : GltfInteractivityNodeSchema + { + public override string Op { get; set; } = "audio/unpause"; + + [FlowInSocketDescription()] + public const string IdFlowIn = "in"; + + [FlowOutSocketDescription()] + public const string IdFlowOut = "out"; + + [InputSocketDescription(GltfTypes.Int)] + public const string IdValueAudio = "audioSourceIndex"; + + [InputSocketDescription(GltfTypes.String)] + public const string IdValueNode = "audioSourcePath"; + } +} diff --git a/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_UnPauseNode.cs.meta b/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_UnPauseNode.cs.meta new file mode 100644 index 000000000..f5c25f9f4 --- /dev/null +++ b/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_UnPauseNode.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 688f359a1e525454f87238752e28d825 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: From 7f69b8acddf68bda7cecf3907fa5b75049012068 Mon Sep 17 00:00:00 2001 From: Dan Quinlan Date: Mon, 31 Mar 2025 21:11:37 -0400 Subject: [PATCH 02/11] Added comments to code and removed most unused/dead code. --- .../VisualScriptingExport/ExportGraph.cs | 3 + .../GLTFAudioExportPlugin.cs | 6 +- .../GltfAudioNodeHelper.cs | 96 +------------------ .../Audio/AudioSourcePauseNode.cs | 11 +++ .../Audio/AudioSourcePlayNode.cs | 2 +- .../Audio/AudioSourceStopNode.cs | 13 ++- .../Audio/AudioSourceUnPauseNode.cs | 14 ++- .../Schema/GltfAudioExtension.cs | 10 ++ .../Schema/Nodes/Audio/Audio_PauseNode.cs | 6 +- .../Schema/Nodes/Audio/Audio_StartNode.cs | 6 +- .../Schema/Nodes/Audio/Audio_StopNode.cs | 3 + .../Schema/Nodes/Audio/Audio_UnPauseNode.cs | 8 +- 12 files changed, 65 insertions(+), 113 deletions(-) diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/ExportGraph.cs b/Editor/Scripts/Interactivity/VisualScriptingExport/ExportGraph.cs index aea24bde4..4e37cb26e 100644 --- a/Editor/Scripts/Interactivity/VisualScriptingExport/ExportGraph.cs +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/ExportGraph.cs @@ -7,6 +7,9 @@ namespace UnityGLTF.Interactivity { + /// + /// Graph containing exporter nodes for use by the interactivity and audio contexts + /// public class ExportGraph { public GameObject gameObject = null; diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportPlugin.cs b/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportPlugin.cs index 93d09b878..6cfeb4c2f 100644 --- a/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportPlugin.cs +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportPlugin.cs @@ -19,12 +19,14 @@ public class GLTFAudioExportPlugin : VisualScriptingExportPlugin public override string DisplayName => "GLTF_Audio_Export"; /// - /// plugin descriptions + /// Plugin descriptions /// public override string Description => "Exports KHR Audio source nodes(literals) from the visual scripting graph"; /// - /// Creates a audio export context with the save to external file arg as true + /// Creates a audio export context with the save to external file arg as false. + /// The second arugment option for the GLTDAudionExportContext tells it to + /// forces a save of the audio to external file by setting to true /// /// /// diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/GltfAudioNodeHelper.cs b/Editor/Scripts/Interactivity/VisualScriptingExport/GltfAudioNodeHelper.cs index 4c1b5c166..0cf4bb3c1 100644 --- a/Editor/Scripts/Interactivity/VisualScriptingExport/GltfAudioNodeHelper.cs +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/GltfAudioNodeHelper.cs @@ -9,6 +9,10 @@ namespace UnityGLTF.Audio using UnityGLTF.Interactivity.VisualScripting.Export; using UnityGLTF.Interactivity.VisualScripting; + /// + /// Copied the GltfInteractivityNodeHelper for now to use with audio. + /// This will need to be cleaned up since a lot of this is not used. + /// public static class GltfAudioNodeHelper { public static Dictionary GetTranslatableNodes( @@ -23,13 +27,6 @@ public static Dictionary GetTranslatableNodes( continue; UnitExporter unitExporter = UnitExporterRegistry.CreateUnitExporter(exportContext, unit); - //if (unitExporter != null) - //{ - // if (unitExporter.IsTranslatable && unitExporter.Nodes.Length > 0) - // validNodes.Add(unit, unitExporter); - //} - //else - // Debug.LogWarning("ExportNode is null for unit: " + Log(unit)+ " of type: " + unit.GetType()); } return validNodes; @@ -53,19 +50,6 @@ static string Log(IUnit unit) public static readonly string IdPointerMaterialIndex = "materialIndex"; public static readonly string IdPointerAnimationIndex = "animationIndex"; - //public static void AddPointerConfig(GltfInteractivityNode node, string pointer, string gltfType) - //{ - // AddPointerConfig(node, pointer, GltfTypes.TypeIndexByGltfSignature(gltfType)); - //} - - //public static void AddPointerConfig(GltfInteractivityNode node, string pointer, int gltfType) - //{ - // var pointerConfig = node.ConfigurationData[Pointer_SetNode.IdPointer]; - // pointerConfig.Value = pointer; - // var typeConfig = node.ConfigurationData[Pointer_SetNode.IdPointerValueType]; - // typeConfig.Value = gltfType; - //} - public static bool IsMainCameraInInput(IUnit unit) { var target = unit.inputs.FirstOrDefault( i => i.key == "target"); @@ -90,17 +74,7 @@ public static bool IsMainCameraInInput(ValueInput target) return false; } - - //public static void AddPointerTemplateValueInput(GltfInteractivityNode node, string pointerId, int? index = null) - //{ - // node.ValueSocketConnectionData.Add(pointerId, new GltfInteractivityNode.ValueSocketData() - // { - // Value = index, - // Type = GltfTypes.TypeIndexByGltfSignature("int"), - // typeRestriction = TypeRestriction.LimitToInt, - // }); - //} - + // Get the index of a named property that has been exported to GLTF public static int GetNamedPropertyGltfIndex(string objectName, IEnumerable gltfRootProperties) { @@ -118,66 +92,6 @@ public static int GetNamedPropertyGltfIndex(string objectName, IEnumerable defaultValues, GltfInteractivityExportContext exportContext) - //{ - // if (value.hasValidConnection == false) - // { - // // If there are no connections, then we can return the non-null default value - // if (value.hasDefaultValue && defaultValues.ContainsKey(value.key)) - // { - // GameObject defaultGameObject = null; - // if (defaultValues[value.key] is GameObject) - // defaultGameObject = (GameObject)defaultValues[value.key]; - // else if (defaultValues[value.key] is Component component) - // defaultGameObject = component.gameObject; - - // // If the value is null, then likely it's set to "this" in the UI - // // meaning that it's pointing to the Gameobject that owns the graph - // if (defaultGameObject == null) - // { - // // The value is referencing the gameobject that owns the graph - // return exportContext.ActiveScriptMachine.gameObject; - // } - // else - // { - // return defaultGameObject; - // } - // } - // return null; - // } - // else - // if (value.hasValidConnection && value.connections.First().source.unit is Literal literal) - // { - // // If there is a connection, then we can return the value of the literal - // if (literal.value is GameObject) - // return literal.value as GameObject; - // else if (literal.value is Component component) - // return component.gameObject; - - // return null; - // } - // else if (value.hasValidConnection && value.connections.First().source.unit is GetVariable getVariable) - // { - // // If there is a connection, then we can return the value of the literal - // var getVarValue = exportContext.GetVariableValueRaw(getVariable, out _, out var varType); - // if (getVariable != null && getVarValue is GameObject gameObject) - // return gameObject; - // else if (getVariable != null && getVarValue is Component component) - // return component.gameObject; - - // if (varType != null) - // return null; - - // Debug.LogError("Could not get the default value of the GetVariable node: " + getVariable.ToString()); - // return null; - // } - // else - // { - // // TODO: Parse the input nodes for a value - // Debug.LogWarning("This ValueInput has a valid connection, but we only support self-references"); - // return null; - // } - //} public static bool GetDefaultValue (IUnit unit, string key, out T tOut) { diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourcePauseNode.cs b/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourcePauseNode.cs index f4166aed5..4cdb14477 100644 --- a/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourcePauseNode.cs +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourcePauseNode.cs @@ -8,6 +8,9 @@ namespace UnityGLTF.Interactivity.VisualScripting.Export { + /// + /// Unit Exporter for audio source play node + /// public class AudioSource_PauseUnitExport : IUnitExporter { public System.Type unitType @@ -15,6 +18,9 @@ public System.Type unitType get => typeof(InvokeMember); } + /// + /// Register the instance of the pause exporter + /// [InitializeOnLoadMethod] private static void Register() { @@ -23,6 +29,11 @@ private static void Register() } + /// + /// Sets up the unitexporter with the correct data and associations + /// + /// + /// public bool InitializeInteractivityNodes(UnitExporter unitExporter) { InvokeMember unit = unitExporter.unit as InvokeMember; diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourcePlayNode.cs b/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourcePlayNode.cs index 1973dd54c..ab5aea519 100644 --- a/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourcePlayNode.cs +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourcePlayNode.cs @@ -11,7 +11,7 @@ namespace UnityGLTF.Interactivity.VisualScripting.Export { /// - /// Unit Exporte for audio source play node + /// Unit Exporter for audio source play node /// public class AudioSource_PlayUnitExport : IUnitExporter { diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceStopNode.cs b/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceStopNode.cs index 8f38f3125..1aea232be 100644 --- a/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceStopNode.cs +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceStopNode.cs @@ -10,11 +10,17 @@ namespace UnityGLTF.Interactivity.VisualScripting.Export { public class AudioSource_StopUnitExport : IUnitExporter { - public System.Type unitType + /// + /// Unit Exporter for audio source stop node + /// + public System.Type unitType { get => typeof(InvokeMember); } + /// + /// Register the instance of stop audio exporter + /// [InitializeOnLoadMethod] private static void Register() { @@ -22,6 +28,11 @@ private static void Register() new AudioSource_StopUnitExport()); } + /// + /// Sets up the unitexporter with the correct data and associations + /// + /// + /// public bool InitializeInteractivityNodes(UnitExporter unitExporter) { InvokeMember unit = unitExporter.unit as InvokeMember; diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceUnPauseNode.cs b/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceUnPauseNode.cs index ade497fcb..8372d4e22 100644 --- a/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceUnPauseNode.cs +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceUnPauseNode.cs @@ -8,13 +8,19 @@ namespace UnityGLTF.Interactivity.VisualScripting.Export { + /// + /// Unit Exporte for audio source unpause node + /// public class AudioSource_UnPauseUnitExport : IUnitExporter { public System.Type unitType { get => typeof(InvokeMember); } - + + /// + /// Register the instance of the unpause audio exporter + /// [InitializeOnLoadMethod] private static void Register() { @@ -22,7 +28,11 @@ private static void Register() new AudioSource_UnPauseUnitExport()); } - + /// + /// Sets up the unpause audio unitexporter with the correct data and associations + /// + /// + /// public bool InitializeInteractivityNodes(UnitExporter unitExporter) { InvokeMember unit = unitExporter.unit as InvokeMember; diff --git a/Runtime/Scripts/Interactivity/Schema/GltfAudioExtension.cs b/Runtime/Scripts/Interactivity/Schema/GltfAudioExtension.cs index 5af836189..78ee88f0f 100644 --- a/Runtime/Scripts/Interactivity/Schema/GltfAudioExtension.cs +++ b/Runtime/Scripts/Interactivity/Schema/GltfAudioExtension.cs @@ -9,6 +9,10 @@ namespace UnityGLTF.Interactivity.Schema using UnityGLTF.Interactivity; [Serializable] + + /// + /// Audio Extension class that is called to serialize the data + /// public class GltfAudioExtension : IExtension { public const string AudioExtensionName = "KHR_audio_emitter"; @@ -20,6 +24,9 @@ public GltfAudioExtension() { } + /// + /// Called when the data is written and serialized to file. + /// public JProperty Serialize() { JObject jo = new JObject @@ -36,6 +43,9 @@ select gr.SerializeObject())), return extension; } + /// + /// Clones the object + /// public IExtension Clone(GLTFRoot root) { return new GltfAudioExtension() diff --git a/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_PauseNode.cs b/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_PauseNode.cs index 2ac96a02e..ac862d05b 100644 --- a/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_PauseNode.cs +++ b/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_PauseNode.cs @@ -1,12 +1,8 @@ namespace UnityGLTF.Interactivity.Schema { /// - /// The world/startAnimation node plays a global animation - /// - /// See https://github.com/petermart/glTF-InteractivityGraph-AuthoringTool/blob/483da2161aa3d9c01ef47be8cdb2bec2d1dd3a18/src/authoring/AuthoringNodeSpecs.ts#L181 . + /// The audio pause node schema / data /// - /// - public class Audio_PauseNode : GltfInteractivityNodeSchema { public override string Op { get; set; } = "audio/pause"; diff --git a/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_StartNode.cs b/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_StartNode.cs index de37e4dd2..bd582d620 100644 --- a/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_StartNode.cs +++ b/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_StartNode.cs @@ -1,12 +1,8 @@ namespace UnityGLTF.Interactivity.Schema { /// - /// The world/startAnimation node plays a global animation - /// - /// See https://github.com/petermart/glTF-InteractivityGraph-AuthoringTool/blob/483da2161aa3d9c01ef47be8cdb2bec2d1dd3a18/src/authoring/AuthoringNodeSpecs.ts#L181 . + /// The audio start node schema / data /// - /// - public class Audio_StartNode : GltfInteractivityNodeSchema { public override string Op { get; set; } = "audio/start"; diff --git a/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_StopNode.cs b/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_StopNode.cs index 82baaaf49..9b9d43830 100644 --- a/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_StopNode.cs +++ b/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_StopNode.cs @@ -1,5 +1,8 @@ namespace UnityGLTF.Interactivity.Schema { + /// + /// The audio stop node schema / data + /// public class Audio_StopNode : GltfInteractivityNodeSchema { public override string Op { get; set; } = "audio/stop"; diff --git a/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_UnPauseNode.cs b/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_UnPauseNode.cs index f2ed413b6..c056aafc1 100644 --- a/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_UnPauseNode.cs +++ b/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_UnPauseNode.cs @@ -1,12 +1,8 @@ namespace UnityGLTF.Interactivity.Schema { - /// - /// The world/startAnimation node plays a global animation - /// - /// See https://github.com/petermart/glTF-InteractivityGraph-AuthoringTool/blob/483da2161aa3d9c01ef47be8cdb2bec2d1dd3a18/src/authoring/AuthoringNodeSpecs.ts#L181 . + /// + /// The audio unpause node schema / data /// - /// - public class Audio_UnPauseNode : GltfInteractivityNodeSchema { public override string Op { get; set; } = "audio/unpause"; From 02a333fa1290fd1a1d9847a2a52a35dc06c9e144 Mon Sep 17 00:00:00 2001 From: dqmachine <33352753+dqmachine@users.noreply.github.com> Date: Tue, 1 Apr 2025 08:07:18 -0400 Subject: [PATCH 03/11] Update Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceStopNode.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../UnitExporters/Audio/AudioSourceStopNode.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceStopNode.cs b/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceStopNode.cs index 1aea232be..d36ad80f7 100644 --- a/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceStopNode.cs +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceStopNode.cs @@ -64,9 +64,9 @@ public bool InitializeInteractivityNodes(UnitExporter unitExporter) node.ValueSocketConnectionData[Audio_StopNode.IdValueAudio].Value = description.Id; node.ValueSocketConnectionData[Audio_StopNode.IdValueNode].Value = $"audio/node/{description.Id}"; - unitExporter.MapInputPortToSocketName(unit.enter, Audio_StartNode.IdFlowIn, node); + unitExporter.MapInputPortToSocketName(unit.enter, Audio_StopNode.IdFlowIn, node); // There should only be one output flow from the Animator.Play node - unitExporter.MapOutFlowConnectionWhenValid(unit.exit, Audio_StartNode.IdFlowOut, node); + unitExporter.MapOutFlowConnectionWhenValid(unit.exit, Audio_StopNode.IdFlowOut, node); } return true; From cd8d78eff82d56f5587fdf7deb558a9c9871f784 Mon Sep 17 00:00:00 2001 From: dqmachine <33352753+dqmachine@users.noreply.github.com> Date: Tue, 1 Apr 2025 08:08:29 -0400 Subject: [PATCH 04/11] Update Editor/Scripts/Interactivity/VisualScriptingExport/GltfAudioNodeHelper.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../Interactivity/VisualScriptingExport/GltfAudioNodeHelper.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/GltfAudioNodeHelper.cs b/Editor/Scripts/Interactivity/VisualScriptingExport/GltfAudioNodeHelper.cs index 0cf4bb3c1..a235189d1 100644 --- a/Editor/Scripts/Interactivity/VisualScriptingExport/GltfAudioNodeHelper.cs +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/GltfAudioNodeHelper.cs @@ -27,6 +27,7 @@ public static Dictionary GetTranslatableNodes( continue; UnitExporter unitExporter = UnitExporterRegistry.CreateUnitExporter(exportContext, unit); + validNodes.Add(unit, unitExporter); } return validNodes; From 331e10dacc70e81d79902d5144e4b0e2e903a873 Mon Sep 17 00:00:00 2001 From: Dan Quinlan Date: Wed, 2 Apr 2025 21:24:27 -0400 Subject: [PATCH 05/11] Cleaned up a little and tried to fix things to match spec. --- .../GLTFAudioExportContext.cs | 8 +++--- .../Interactivity/Schema/KHRAudioSchemas.cs | 27 +++++++++---------- 2 files changed, 16 insertions(+), 19 deletions(-) diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportContext.cs b/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportContext.cs index e82fb55c1..ecca36044 100644 --- a/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportContext.cs +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportContext.cs @@ -201,7 +201,8 @@ internal void ProcessAudioSource(IUnit unit, AudioSource audioSource) gain = audioSource.volume, minDistance = audioSource.minDistance, maxDistance = audioSource.maxDistance, - distanceModel = PositionalAudioDistanceModel.linear + distanceModel = PositionalAudioDistanceModel.linear, + name = "positional emitter" }; audioEmitters.Add(emitter); @@ -237,15 +238,12 @@ internal void ProcessAudioSource(IUnit unit, AudioSource audioSource) } audio.uri = uriPath + fileName; //"." + Path.DirectorySeparatorChar + AudioRelDirectory + Path.DirectorySeparatorChar + fileName; - audio.mimeType = MimeTypeString; } else { var result = exporter.ExportFile(fileName, "audio/mpeg", fileStream); - audio.uri = result.uri; audio.mimeType = result.mimeType; audio.bufferView = result.bufferView; - } var audioData = new List(); @@ -260,7 +258,7 @@ internal void ProcessAudioSource(IUnit unit, AudioSource audioSource) autoPlay = audioSource.playOnAwake, loop = audioSource.loop, gain = audioSource.volume, - sourceName = Path.GetFileNameWithoutExtension(path) + name = Path.GetFileNameWithoutExtension(path) }; audioSources.Add(khrAudio); diff --git a/Runtime/Scripts/Interactivity/Schema/KHRAudioSchemas.cs b/Runtime/Scripts/Interactivity/Schema/KHRAudioSchemas.cs index 0d945ceef..3a0294d2a 100644 --- a/Runtime/Scripts/Interactivity/Schema/KHRAudioSchemas.cs +++ b/Runtime/Scripts/Interactivity/Schema/KHRAudioSchemas.cs @@ -139,26 +139,27 @@ public IExtension Clone(GLTFRoot root) [Serializable] public class KHR_AudioEmitter : GLTFChildOfRootProperty { - + public string name; public string type; public float gain; public List sources; public virtual JObject Serialize() { var jo = new JObject(); - + + if (!string.IsNullOrEmpty(name)){ + jo.Add(nameof(name), name); + } jo.Add(nameof(type), type); -// if (gain != 1.0f) { - jo.Add(nameof(gain), gain); -// } + jo.Add(nameof(gain), gain); if (sources != null && sources.Count > 0) { - JArray arr = new JArray(); + JArray arr = new JArray(); - foreach (var source in sources) { - arr.Add(source.Id); - } + foreach (var source in sources) { + arr.Add(source.Id); + } jo.Add(new JProperty(nameof(sources), arr)); } @@ -169,7 +170,6 @@ public virtual JObject Serialize() { [Serializable] public class KHR_PositionalAudioEmitter : KHR_AudioEmitter { - public float coneInnerAngle; public float coneOuterAngle; public float coneOuterGain; @@ -209,7 +209,6 @@ public override JObject Serialize() { if (maxDistance != 10000.0f) { positional.Add(new JProperty(nameof(maxDistance), maxDistance)); } - //if (refDistance != 1.0f) { // positional.Add(new JProperty(nameof(refDistance), refDistance)); @@ -227,7 +226,7 @@ public override JObject Serialize() { [Serializable] public class KHR_AudioSource : GLTFChildOfRootProperty { - public string sourceName; + public string name; public bool autoPlay; public float gain; public bool loop; @@ -252,9 +251,9 @@ public JObject Serialize() { jo.Add(nameof(audio), audio.Id); } - if (!string.IsNullOrEmpty(sourceName)) + if (!string.IsNullOrEmpty(name)) { - jo.Add(nameof(sourceName), sourceName); + jo.Add(nameof(name), name); } return jo; } From bfec4174b3b80bec52294f3d71d57a2612897dba Mon Sep 17 00:00:00 2001 From: Dan Quinlan Date: Tue, 8 Apr 2025 09:21:20 -0400 Subject: [PATCH 06/11] Removed the string node output placeholder which will be pointers if we ever implement. --- .../VisualScriptingExport/GLTFAudioExportPlugin.cs | 2 +- .../UnitExporters/Audio/AudioSourcePauseNode.cs | 2 +- .../UnitExporters/Audio/AudioSourcePlayNode.cs | 2 +- .../UnitExporters/Audio/AudioSourceStopNode.cs | 6 +++--- .../UnitExporters/Audio/AudioSourceUnPauseNode.cs | 2 +- .../Interactivity/Schema/Nodes/Audio/Audio_PauseNode.cs | 4 ++-- .../Interactivity/Schema/Nodes/Audio/Audio_StartNode.cs | 4 ++-- .../Interactivity/Schema/Nodes/Audio/Audio_StopNode.cs | 4 ++-- .../Interactivity/Schema/Nodes/Audio/Audio_UnPauseNode.cs | 4 ++-- 9 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportPlugin.cs b/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportPlugin.cs index 6cfeb4c2f..fb5f1184e 100644 --- a/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportPlugin.cs +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportPlugin.cs @@ -32,7 +32,7 @@ public class GLTFAudioExportPlugin : VisualScriptingExportPlugin /// public override GLTFExportPluginContext CreateInstance(ExportContext context) { - return new GLTFAudioExportContext(this);//, true); + return new GLTFAudioExportContext(this, false); } } diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourcePauseNode.cs b/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourcePauseNode.cs index 4cdb14477..db6676ef5 100644 --- a/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourcePauseNode.cs +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourcePauseNode.cs @@ -63,7 +63,7 @@ public bool InitializeInteractivityNodes(UnitExporter unitExporter) var node = unitExporter.CreateNode(new Audio_PauseNode()); node.ValueSocketConnectionData[Audio_PauseNode.IdValueAudio].Value = description.Id; - node.ValueSocketConnectionData[Audio_PauseNode.IdValueNode].Value = $"audio/node/{description.Id}"; +// node.ValueSocketConnectionData[Audio_PauseNode.IdValueNode].Value = $"audio/node/{description.Id}"; unitExporter.MapInputPortToSocketName(unit.enter, Audio_PauseNode.IdFlowIn, node); // There should only be one output flow from the Animator.Play node diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourcePlayNode.cs b/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourcePlayNode.cs index ab5aea519..cd65035d7 100644 --- a/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourcePlayNode.cs +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourcePlayNode.cs @@ -65,7 +65,7 @@ public bool InitializeInteractivityNodes(UnitExporter unitExporter) var node = unitExporter.CreateNode(new Audio_StartNode()); node.ValueSocketConnectionData[Audio_StartNode.IdValueAudio].Value = description.Id; - node.ValueSocketConnectionData[Audio_StartNode.IdValueNode].Value = $"audio/node/{description.Id}"; +// node.ValueSocketConnectionData[Audio_StartNode.IdValueNode].Value = $"audio/node/{description.Id}"; unitExporter.MapInputPortToSocketName(unit.enter, Audio_StartNode.IdFlowIn, node); // There should only be one output flow from the Animator.Play node diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceStopNode.cs b/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceStopNode.cs index d36ad80f7..c84f6c7fc 100644 --- a/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceStopNode.cs +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceStopNode.cs @@ -62,11 +62,11 @@ public bool InitializeInteractivityNodes(UnitExporter unitExporter) var node = unitExporter.CreateNode(new Audio_StopNode()); node.ValueSocketConnectionData[Audio_StopNode.IdValueAudio].Value = description.Id; - node.ValueSocketConnectionData[Audio_StopNode.IdValueNode].Value = $"audio/node/{description.Id}"; +// node.ValueSocketConnectionData[Audio_StopNode.IdValueNode].Value = $"audio/node/{description.Id}"; - unitExporter.MapInputPortToSocketName(unit.enter, Audio_StopNode.IdFlowIn, node); + unitExporter.MapInputPortToSocketName(unit.enter, Audio_StartNode.IdFlowIn, node); // There should only be one output flow from the Animator.Play node - unitExporter.MapOutFlowConnectionWhenValid(unit.exit, Audio_StopNode.IdFlowOut, node); + unitExporter.MapOutFlowConnectionWhenValid(unit.exit, Audio_StartNode.IdFlowOut, node); } return true; diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceUnPauseNode.cs b/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceUnPauseNode.cs index 8372d4e22..4b4a66af1 100644 --- a/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceUnPauseNode.cs +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Audio/AudioSourceUnPauseNode.cs @@ -62,7 +62,7 @@ public bool InitializeInteractivityNodes(UnitExporter unitExporter) var node = unitExporter.CreateNode(new Audio_UnPauseNode()); node.ValueSocketConnectionData[Audio_UnPauseNode.IdValueAudio].Value = description.Id; - node.ValueSocketConnectionData[Audio_UnPauseNode.IdValueNode].Value = $"audio/node/{description.Id}"; +// node.ValueSocketConnectionData[Audio_UnPauseNode.IdValueNode].Value = $"audio/node/{description.Id}"; unitExporter.MapInputPortToSocketName(unit.enter, Audio_UnPauseNode.IdFlowIn, node); // There should only be one output flow from the Animator.Play node diff --git a/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_PauseNode.cs b/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_PauseNode.cs index ac862d05b..4d6e4300e 100644 --- a/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_PauseNode.cs +++ b/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_PauseNode.cs @@ -16,7 +16,7 @@ public class Audio_PauseNode : GltfInteractivityNodeSchema [InputSocketDescription(GltfTypes.Int)] public const string IdValueAudio = "audioSourceIndex"; - [InputSocketDescription(GltfTypes.String)] - public const string IdValueNode = "audioSourcePath"; + //[InputSocketDescription(GltfTypes.String)] + //public const string IdValueNode = "audioSourcePath"; } } diff --git a/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_StartNode.cs b/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_StartNode.cs index bd582d620..b7798d431 100644 --- a/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_StartNode.cs +++ b/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_StartNode.cs @@ -16,7 +16,7 @@ public class Audio_StartNode : GltfInteractivityNodeSchema [InputSocketDescription(GltfTypes.Int)] public const string IdValueAudio = "audioSourceIndex"; - [InputSocketDescription(GltfTypes.String)] - public const string IdValueNode = "audioSourcePath"; + //[InputSocketDescription(GltfTypes.String)] + //public const string IdValueNode = "audioSourcePath"; } } diff --git a/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_StopNode.cs b/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_StopNode.cs index 9b9d43830..9c4f015f8 100644 --- a/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_StopNode.cs +++ b/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_StopNode.cs @@ -16,7 +16,7 @@ public class Audio_StopNode : GltfInteractivityNodeSchema [InputSocketDescription(GltfTypes.Int)] public const string IdValueAudio = "audioSourceIndex"; - [InputSocketDescription(GltfTypes.String)] - public const string IdValueNode = "audioSourcePath"; + //[InputSocketDescription(GltfTypes.String)] + //public const string IdValueNode = "audioSourcePath"; } } diff --git a/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_UnPauseNode.cs b/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_UnPauseNode.cs index c056aafc1..70e48d3ae 100644 --- a/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_UnPauseNode.cs +++ b/Runtime/Scripts/Interactivity/Schema/Nodes/Audio/Audio_UnPauseNode.cs @@ -16,7 +16,7 @@ public class Audio_UnPauseNode : GltfInteractivityNodeSchema [InputSocketDescription(GltfTypes.Int)] public const string IdValueAudio = "audioSourceIndex"; - [InputSocketDescription(GltfTypes.String)] - public const string IdValueNode = "audioSourcePath"; + //[InputSocketDescription(GltfTypes.String)] + //public const string IdValueNode = "audioSourcePath"; } } From 2659302ec63585a39699e0e05d4e6cceb0d72dfa Mon Sep 17 00:00:00 2001 From: Dan Quinlan Date: Fri, 11 Apr 2025 15:21:52 -0400 Subject: [PATCH 07/11] Modified to support new abbreviated goog audio emitter extension. --- .../GLTFAudioExportContext.cs | 77 +++++++++++++------ .../GLTFAudioExportPlugin.cs | 4 +- .../Schema/GltfAudioExtension.cs | 2 +- .../Schema/GltfSceneAudioEmitterExtension.cs | 59 ++++++++++++++ .../GltfSceneAudioEmitterExtension.cs.meta | 11 +++ 5 files changed, 127 insertions(+), 26 deletions(-) create mode 100644 Runtime/Scripts/Interactivity/Schema/GltfSceneAudioEmitterExtension.cs create mode 100644 Runtime/Scripts/Interactivity/Schema/GltfSceneAudioEmitterExtension.cs.meta diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportContext.cs b/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportContext.cs index ecca36044..f3cf62a3d 100644 --- a/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportContext.cs +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportContext.cs @@ -61,7 +61,9 @@ public GLTFAudioExportContext(GLTFAudioExportPlugin plugin, bool saveToExternalF _saveAudioToFile = saveToExternalFile; } - public override void BeforeSceneExport(GLTFSceneExporter exporter, GLTFRoot gltfRoot) { } + public override void BeforeSceneExport(GLTFSceneExporter exporter, GLTFRoot gltfRoot) + { + } /// /// Called after the scene has been exported to add khr audio data. @@ -73,6 +75,7 @@ public override void BeforeSceneExport(GLTFSceneExporter exporter, GLTFRoot gltf /// list of ScriptMachines in the scene. public override void AfterSceneExport(GLTFSceneExporter exporter, GLTFRoot gltfRoot) { + var scenes = gltfRoot.Scenes; var scriptMachines = new List(); foreach (var root in exporter.RootTransforms) @@ -85,6 +88,12 @@ public override void AfterSceneExport(GLTFSceneExporter exporter, GLTFRoot gltfR } AfterSceneExport(exporter, gltfRoot, scriptMachines); + + // add new scenes audio emitter extension before process and JSON is written out. + var v = new Dictionary(); + v.Add(GltfAudioExtension.AudioExtensionName, new GltfSceneAudioEmitterExtension() { emitters = GetAudioSourceIndexes() }); + gltfRoot.Scenes.Add(new GLTFScene() { Extensions = v }); + } /// @@ -156,6 +165,11 @@ internal ExportGraph GetAudio(FlowGraph graph) return newExportGraph; } + public static List GetAudioSourceIndexes() + { + return (_audioSourceIds.Select(r => r.Id).ToList()); + } + public static AudioDescription AddAudioSource(AudioSource audioSource) { AudioClip clip = audioSource.clip; @@ -194,25 +208,32 @@ internal void ProcessAudioSource(IUnit unit, AudioSource audioSource) Root = _gltfRoot }; - var emitter = new KHR_PositionalAudioEmitter() - { - type = "positional", - sources = new List() { new AudioSourceId() { Id = audioSourceId.Id, Root = _gltfRoot } }, - gain = audioSource.volume, - minDistance = audioSource.minDistance, - maxDistance = audioSource.maxDistance, - distanceModel = PositionalAudioDistanceModel.linear, - name = "positional emitter" - }; - - audioEmitters.Add(emitter); + //var emitter = new KHR_AudioEmitter() + //{ + // audio = audioSourceId.Id, + // gain = 1, + // autoPlay = true, + // loop = false + //}; + + //var emitter = new KHR_PositionalAudioEmitter() + //{ + // type = "global", + // sources = new List() { new AudioSourceId() { Id = audioSourceId.Id, Root = _gltfRoot } }, + // gain = audioSource.volume, + // minDistance = audioSource.minDistance, + // maxDistance = audioSource.maxDistance, + // distanceModel = PositionalAudioDistanceModel.linear, + // name = "positional emitter" + //}; + +// audioEmitters.Add(emitter); var path = AssetDatabase.GetAssetPath(clip); var fileName = Path.GetFileName(path); var settings = GLTFSettings.GetOrCreateSettings(); - string savePath = string.Empty; var audio = new KHR_AudioData(); @@ -250,23 +271,33 @@ internal void ProcessAudioSource(IUnit unit, AudioSource audioSource) audioData.Add(audio); - var audioSources = new List(); - - var khrAudio = new KHR_AudioSource + var emitter = new KHR_AudioEmitter() { - audio = new AudioDataId { Id = audioSourceId.Id, Root = _gltfRoot }, - autoPlay = audioSource.playOnAwake, - loop = audioSource.loop, + audio = audioSourceId.Id, gain = audioSource.volume, - name = Path.GetFileNameWithoutExtension(path) + autoPlay = audioSource.playOnAwake, + loop = audioSource.loop }; - audioSources.Add(khrAudio); + audioEmitters.Add(emitter); + + //var audioSources = new List(); + + //var khrAudio = new KHR_AudioSource + //{ + // audio = new AudioDataId { Id = audioSourceId.Id, Root = _gltfRoot }, + // autoPlay = audioSource.playOnAwake, + // loop = audioSource.loop, + // gain = audioSource.volume, + // name = Path.GetFileNameWithoutExtension(path) + //}; + + //audioSources.Add(khrAudio); var extension = new KHR_audio { audio = new List(audioData), - sources = new List(audioSources), +// sources = new List(audioSources), emitters = new List(audioEmitters), }; diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportPlugin.cs b/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportPlugin.cs index fb5f1184e..26e058c09 100644 --- a/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportPlugin.cs +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportPlugin.cs @@ -16,7 +16,7 @@ public class GLTFAudioExportPlugin : VisualScriptingExportPlugin /// /// Plugin name /// - public override string DisplayName => "GLTF_Audio_Export"; + public override string DisplayName => "GOOG_audio_emitter"; /// /// Plugin descriptions @@ -32,7 +32,7 @@ public class GLTFAudioExportPlugin : VisualScriptingExportPlugin /// public override GLTFExportPluginContext CreateInstance(ExportContext context) { - return new GLTFAudioExportContext(this, false); + return new GLTFAudioExportContext(this, true); } } diff --git a/Runtime/Scripts/Interactivity/Schema/GltfAudioExtension.cs b/Runtime/Scripts/Interactivity/Schema/GltfAudioExtension.cs index 78ee88f0f..79575fe80 100644 --- a/Runtime/Scripts/Interactivity/Schema/GltfAudioExtension.cs +++ b/Runtime/Scripts/Interactivity/Schema/GltfAudioExtension.cs @@ -15,7 +15,7 @@ namespace UnityGLTF.Interactivity.Schema /// public class GltfAudioExtension : IExtension { - public const string AudioExtensionName = "KHR_audio_emitter"; + public const string AudioExtensionName = "GOOG_audio_emitter"; public GltfInteractivityGraph[] graphs; public int graph = 0; diff --git a/Runtime/Scripts/Interactivity/Schema/GltfSceneAudioEmitterExtension.cs b/Runtime/Scripts/Interactivity/Schema/GltfSceneAudioEmitterExtension.cs new file mode 100644 index 000000000..2ed571988 --- /dev/null +++ b/Runtime/Scripts/Interactivity/Schema/GltfSceneAudioEmitterExtension.cs @@ -0,0 +1,59 @@ +using GLTF.Schema; + +namespace UnityGLTF.Interactivity.Schema +{ + using System; + using System.Collections.Generic; + using System.Linq; + using GLTF.Schema; + using Newtonsoft.Json.Linq; + using UnityGLTF.Interactivity; + + [Serializable] + + /// + /// Audio Emitter Extension class that is called to serialize the data for the scenes node + /// + public class GltfSceneAudioEmitterExtension : IExtension + { + public const string SceneAudioExtensionName = "GOOG_audio_emitter"; + + public List emitters = new(); + + public GltfSceneAudioEmitterExtension() + { + } + + /// + /// Called when the data is written and serialized to file. + /// + public JProperty Serialize() + { + JObject jo = new JObject(); + + JArray arr = new JArray(); + + foreach (var emitter in emitters) { + arr.Add(emitter); + } + + jo.Add(new JProperty(nameof(emitters), arr)); + + + JProperty extension = + new JProperty(GltfSceneAudioEmitterExtension.SceneAudioExtensionName, jo); + return extension; + } + + /// + /// Clones the object + /// + public IExtension Clone(GLTFRoot root) + { + return new GltfSceneAudioEmitterExtension() + { + emitters = emitters + }; + } + } +} diff --git a/Runtime/Scripts/Interactivity/Schema/GltfSceneAudioEmitterExtension.cs.meta b/Runtime/Scripts/Interactivity/Schema/GltfSceneAudioEmitterExtension.cs.meta new file mode 100644 index 000000000..ee3b551a8 --- /dev/null +++ b/Runtime/Scripts/Interactivity/Schema/GltfSceneAudioEmitterExtension.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 851475ba88c79c34c8573ddbe8737193 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: From d383ae80031e0adff23599717f7e334ede8263c9 Mon Sep 17 00:00:00 2001 From: Dan Quinlan Date: Mon, 21 Apr 2025 16:01:10 -0400 Subject: [PATCH 08/11] Added GOOG_video extension --- .../GLTFAudioExportContext.cs | 2 +- .../GLTFVideoExportContext.cs | 349 ++++++++++++++++++ .../GLTFVideoExportContext.cs.meta | 11 + .../GLTFVideoExportPlugin.cs | 37 ++ .../GLTFVideoExportPlugin.cs.meta | 11 + ...eHelper.cs => GltfAudioVideoNodeHelper.cs} | 5 +- ....meta => GltfAudioVideoNodeHelper.cs.meta} | 0 .../Video/VideoPlayerUnitExport.cs | 107 ++++++ .../Video/VideoPlayerUnitExport.cs.meta | 11 + .../Interactivity/Schema/GOOGVideoSchemas.cs | 234 ++++++++++++ .../Schema/GOOGVideoSchemas.cs.meta | 11 + .../Schema/GltfSceneVideoEmitterExtension.cs | 59 +++ .../GltfSceneVideoEmitterExtension.cs.meta | 11 + .../Schema/GltfVideoExtension.cs | 58 +++ .../Schema/GltfVideoExtension.cs.meta | 11 + 15 files changed, 913 insertions(+), 4 deletions(-) create mode 100644 Editor/Scripts/Interactivity/VisualScriptingExport/GLTFVideoExportContext.cs create mode 100644 Editor/Scripts/Interactivity/VisualScriptingExport/GLTFVideoExportContext.cs.meta create mode 100644 Editor/Scripts/Interactivity/VisualScriptingExport/GLTFVideoExportPlugin.cs create mode 100644 Editor/Scripts/Interactivity/VisualScriptingExport/GLTFVideoExportPlugin.cs.meta rename Editor/Scripts/Interactivity/VisualScriptingExport/{GltfAudioNodeHelper.cs => GltfAudioVideoNodeHelper.cs} (96%) rename Editor/Scripts/Interactivity/VisualScriptingExport/{GltfAudioNodeHelper.cs.meta => GltfAudioVideoNodeHelper.cs.meta} (100%) create mode 100644 Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Video/VideoPlayerUnitExport.cs create mode 100644 Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Video/VideoPlayerUnitExport.cs.meta create mode 100644 Runtime/Scripts/Interactivity/Schema/GOOGVideoSchemas.cs create mode 100644 Runtime/Scripts/Interactivity/Schema/GOOGVideoSchemas.cs.meta create mode 100644 Runtime/Scripts/Interactivity/Schema/GltfSceneVideoEmitterExtension.cs create mode 100644 Runtime/Scripts/Interactivity/Schema/GltfSceneVideoEmitterExtension.cs.meta create mode 100644 Runtime/Scripts/Interactivity/Schema/GltfVideoExtension.cs create mode 100644 Runtime/Scripts/Interactivity/Schema/GltfVideoExtension.cs.meta diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportContext.cs b/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportContext.cs index f3cf62a3d..25ca0f90d 100644 --- a/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportContext.cs +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportContext.cs @@ -157,7 +157,7 @@ internal ExportGraph GetAudio(FlowGraph graph) } } - newExportGraph.nodes = GltfAudioNodeHelper.GetTranslatableNodes(topologicallySortedNodes, this); + newExportGraph.nodes = GltfAudioVideoNodeHelper.GetTranslatableNodes(topologicallySortedNodes, this); nodesToExport.AddRange(newExportGraph.nodes.Select(g => g.Value)); diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFVideoExportContext.cs b/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFVideoExportContext.cs new file mode 100644 index 000000000..5c1f86729 --- /dev/null +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFVideoExportContext.cs @@ -0,0 +1,349 @@ +using GLTF.Schema; +using System.Collections.Generic; +using UnityEngine; +using Unity.VisualScripting; +using System.Linq; +using UnityGLTF.Audio; +using UnityGLTF.Plugins.Experimental; +using System.IO; +using UnityEditor; +using UnityGLTF.Interactivity.VisualScripting.Export; +using UnityGLTF.Interactivity.Schema; +using UnityEngine.Video; + +namespace UnityGLTF.Interactivity.VisualScripting +{ + /// + /// Audio GLTF export context + /// + public class GLTFVideoExportContext : VisualScriptingExportContext + { + /// + /// Contains summary information for an audio clip. + /// + public class VideoDescription + { + public int Id; + public string Name; + public VideoClip Clip; + } + + // container for audio sources by description + private static List _videoSourceIds = new(); + + private const string AudioExtension = ".mp4"; + private const string AudioRelDirectory = "video"; + private const string MimeTypeString = "video/mpeg"; + + private bool _saveVideoToFile = false; + + private List addedGraphs = new List(); + private List nodesToExport = new List(); + + internal new ExportGraph currentGraphProcessing { get; private set; } = null; + private GLTFRoot _gltfRoot = null; + + /// + /// Default construction initializes the parent GLTFInteractivity export context. + /// + /// + public GLTFVideoExportContext(GLTFVideoExportPlugin plugin): base(plugin) + { + + } + + /// + /// Constructor sets the save to audio bool using the supplied argument + /// + /// + /// + public GLTFVideoExportContext(GLTFVideoExportPlugin plugin, bool saveToExternalFile) : base(plugin) + { + _saveVideoToFile = saveToExternalFile; + } + + public override void BeforeSceneExport(GLTFSceneExporter exporter, GLTFRoot gltfRoot) + { + } + + /// + /// Called after the scene has been exported to add video data. + /// + /// This overload of AfterSceneExport exposes the origins as a parameter to simplify tests. + /// + /// GLTFSceneExporter object used to export the scene + /// Root GLTF object for the gltf object tree + /// list of ScriptMachines in the scene. + public override void AfterSceneExport(GLTFSceneExporter exporter, GLTFRoot gltfRoot) + { + var scenes = gltfRoot.Scenes; + var scriptMachines = new List(); + + foreach (var root in exporter.RootTransforms) + { + if (!root) continue; + var machines = root + .GetComponentsInChildren() + .Where(x => x.isActiveAndEnabled && x.graph != null); + scriptMachines.AddRange(machines); + } + + AfterSceneExport(exporter, gltfRoot, scriptMachines); + + // add new scenes audio emitter extension before process and JSON is written out. + var v = new Dictionary(); + v.Add(GltfVideoExtension.VideoExtensionName, new GltfSceneVideoEmitterExtension() { videos = GetVideoSourceIndexes() }); + gltfRoot.Scenes.Add(new GLTFScene() { Extensions = v }); + + } + + /// + /// Called after the scene has been exported to add interactivity data. + /// + /// This overload of AfterSceneExport exposes the origins as a parameter to simplify tests. + /// + /// GLTFSceneExporter object used to export the scene + /// Root GLTF object for the gltf object tree + /// list of ScriptMachines in the scene. + internal new void AfterSceneExport(GLTFSceneExporter exporter, GLTFRoot gltfRoot, List visualScriptingComponents) + { + if (visualScriptingComponents.Count == 0) + { + return; + } + this.exporter = exporter; + _gltfRoot = gltfRoot; + foreach (var scriptMachine in visualScriptingComponents) + { + ActiveScriptMachine = scriptMachine; + FlowGraph flowGraph = scriptMachine.graph; + GetVideo(flowGraph); + } + } + + /// + /// Gets the export graph which is not currently being used. + /// + /// + /// + internal ExportGraph GetVideo(FlowGraph graph) + { + var newExportGraph = new ExportGraph(); + newExportGraph.gameObject = ActiveScriptMachine.gameObject; + newExportGraph.parentGraph = currentGraphProcessing; + newExportGraph.graph = graph; + addedGraphs.Add(newExportGraph); + + if (currentGraphProcessing != null) + currentGraphProcessing.subGraphs.Add(newExportGraph); + + var lastCurrentGraph = currentGraphProcessing; + currentGraphProcessing = newExportGraph; + + // Topologically sort the graph to establish the dependency order + LinkedList topologicallySortedNodes = TopologicalSort(graph.units); + IEnumerable sortedNodes = topologicallySortedNodes; + foreach (var unit in sortedNodes) + { + if (unit is Literal literal) + { + VideoPlayer video = null; + // If there is a connection, then we can return the value of the literal + if (literal.value is Component component && component is VideoPlayer) + video = component.GetComponent(); + if (video == null) + continue; + + ProcessVideoSource(unit, video); + } + } + + newExportGraph.nodes = GltfAudioVideoNodeHelper.GetTranslatableNodes(topologicallySortedNodes, this); + + nodesToExport.AddRange(newExportGraph.nodes.Select(g => g.Value)); + + currentGraphProcessing = lastCurrentGraph; + return newExportGraph; + } + + public static List GetVideoSourceIndexes() + { + return (_videoSourceIds.Select(r => r.Id).ToList()); + } + + public static VideoDescription AddVideoSource(VideoPlayer videoPlayer) + { + VideoClip clip = videoPlayer.clip; + string name = clip.name; + + foreach (var v in _videoSourceIds) + { + if (name == v.Name && clip == v.Clip) + { + return v; + } + } + VideoDescription ad = new VideoDescription() { Id = _videoSourceIds.Count, Name = clip.name, Clip = clip }; + _videoSourceIds.Add(ad); + return ad; + } + + /// + /// Sets up and process the audio emitter data and sets the extension data for + /// the exporter to write to glb + /// + /// supplied iunit which is not currently used + /// the audio source in the unity scene to parse + internal void ProcessVideoSource(IUnit unit, VideoPlayer videoPlayer) + { + List videoDataClips = new List(); + + var clip = videoPlayer.clip; + + var audioSourceId = AddVideoSource(videoPlayer); + + var path = AssetDatabase.GetAssetPath(clip); + + var fileName = Path.GetFileName(path); + var settings = GLTFSettings.GetOrCreateSettings(); + + string savePath = string.Empty; + + var videoSource = new GOOG_VideoSource(); + + var fileStream = new FileStream(path, FileMode.Open, FileAccess.Read); + + if (settings != null && !string.IsNullOrEmpty(settings.SaveFolderPath) && _saveVideoToFile) + { + string uriPath; + (uriPath, savePath) = GetRelativePath(settings.SaveFolderPath); + + if (!Directory.Exists(savePath)) + { + Directory.CreateDirectory(savePath); + } + savePath += (Path.DirectorySeparatorChar + fileName); + using (var fileWriteStream = new FileStream(savePath, FileMode.Create)) + { + byte[] data = new byte[fileStream.Length]; + fileStream.Read(data, 0, (int)fileStream.Length); + + fileWriteStream.Write(data, 0, data.Length); + } + + videoSource.uri = uriPath + fileName; //"." + Path.DirectorySeparatorChar + AudioRelDirectory + Path.DirectorySeparatorChar + fileName; + } + else + { + var result = exporter.ExportFile(fileName, "video/mpeg", fileStream); + videoSource.mimeType = result.mimeType; + videoSource.bufferView = result.bufferView; + } + + var videoSources = new List(); + + videoSources.Add(videoSource); + + var videoDatas = new List(); + var videoData = new GOOG_VideoData() + { + name = videoPlayer.clip?.name, + speed = videoPlayer.playbackSpeed, + video = audioSourceId.Id, + autoPlay = videoPlayer.playOnAwake + }; + + videoDatas.Add(videoData); + + var extension = new GOOG_Video + { + videoData = new List(videoDatas), + videoSource = new List(videoSources) + }; + + if (_gltfRoot != null) + { + _gltfRoot.AddExtension(GltfVideoExtension.VideoExtensionName, (IExtension)extension); + exporter.DeclareExtensionUsage(GltfVideoExtension.VideoExtensionName); + } + } + + /// + /// Gets the relative paths and save paths from the supplied full path arg + /// + /// full save path + /// + internal (string, string) GetRelativePath(string fullSavePath) + { + string relPath = "." + Path.DirectorySeparatorChar + AudioRelDirectory + Path.DirectorySeparatorChar; + string savePath = Path.GetFullPath(fullSavePath) + Path.DirectorySeparatorChar + AudioRelDirectory; + + return (relPath, savePath); + } + + /// + /// We are running this, but the converters and graph are not currently used other than + /// the audio source literal elements + /// + /// + /// + private static LinkedList TopologicalSort(IEnumerable nodes) + { + var sorted = new LinkedList(); + var visited = new Dictionary(); + + void Visit(IUnit node) + { + bool inProcess; + bool alreadyVisited = visited.TryGetValue(node, out inProcess); + + if (alreadyVisited) + { + if (inProcess) + { + // TODO: Should quit the topological sort and cancel the export + // throw new ArgumentException("Cyclic dependency found."); + } + } + else + { + visited[node] = true; + + // Get the dependencies from incoming connections and ignore self-references + HashSet dependencies = new HashSet(); + foreach (IUnitConnection connection in node.connections) + { + if (connection.source.unit != node) + { + dependencies.Add(connection.source.unit); + } + } + + foreach (IUnit dependency in dependencies) + { + Visit(dependency); + } + + visited[node] = false; + sorted.AddLast(node); + } + } + + foreach (var node in nodes) + { + Visit(node); + } + + return sorted; + } + + public override void BeforeNodeExport(GLTFSceneExporter exporter, GLTFRoot gltfRoot, Transform transform, Node node) { } + public override void AfterNodeExport(GLTFSceneExporter exporter, GLTFRoot gltfRoot, Transform transform, Node node){} + public override bool BeforeMaterialExport(GLTFSceneExporter exporter, GLTFRoot gltfRoot, Material material, GLTFMaterial materialNode) => false; + public override void AfterMaterialExport(GLTFSceneExporter exporter, GLTFRoot gltfRoot, Material material, GLTFMaterial materialNode) { } + public override void BeforeTextureExport(GLTFSceneExporter exporter, ref GLTFSceneExporter.UniqueTexture texture, string textureSlot) { } + public override void AfterTextureExport(GLTFSceneExporter exporter, GLTFSceneExporter.UniqueTexture texture, int index, GLTFTexture tex) { } + public override void AfterPrimitiveExport(GLTFSceneExporter exporter, Mesh mesh, MeshPrimitive primitive, int index) { } + public override void AfterMeshExport(GLTFSceneExporter exporter, Mesh mesh, GLTFMesh gltfMesh, int index) { } + } +} diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFVideoExportContext.cs.meta b/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFVideoExportContext.cs.meta new file mode 100644 index 000000000..0e26f56fb --- /dev/null +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFVideoExportContext.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: c50a4589c7406fc4c8e43e8b65765b9a +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFVideoExportPlugin.cs b/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFVideoExportPlugin.cs new file mode 100644 index 000000000..535a6f683 --- /dev/null +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFVideoExportPlugin.cs @@ -0,0 +1,37 @@ +using System.Collections; +using System.Collections.Generic; +using UnityEngine; +using UnityGLTF; +using UnityGLTF.Interactivity.Schema; +using UnityGLTF.Plugins; + +namespace UnityGLTF.Interactivity.VisualScripting +{ + /// + /// Pluging for the audio emitter extension + /// + public class GLTFVideoExportPlugin : VisualScriptingExportPlugin + { + /// + /// Plugin name + /// + public override string DisplayName => "GOOG_video"; + + /// + /// Plugin descriptions + /// + public override string Description => "Exports KHR video source nodes(literals) from the visual scripting graph"; + + /// + /// Creates a audio export context with the save to external file arg as false. + /// The second arugment option for the GLTDAudionExportContext tells it to + /// forces a save of the audio to external file by setting to true + /// + /// + /// + public override GLTFExportPluginContext CreateInstance(ExportContext context) + { + return new GLTFVideoExportContext(this, true); + } + } +} \ No newline at end of file diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFVideoExportPlugin.cs.meta b/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFVideoExportPlugin.cs.meta new file mode 100644 index 000000000..d437040d8 --- /dev/null +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFVideoExportPlugin.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: f860cce8bb911f9409190081b30ac67e +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/GltfAudioNodeHelper.cs b/Editor/Scripts/Interactivity/VisualScriptingExport/GltfAudioVideoNodeHelper.cs similarity index 96% rename from Editor/Scripts/Interactivity/VisualScriptingExport/GltfAudioNodeHelper.cs rename to Editor/Scripts/Interactivity/VisualScriptingExport/GltfAudioVideoNodeHelper.cs index a235189d1..eb241f2cc 100644 --- a/Editor/Scripts/Interactivity/VisualScriptingExport/GltfAudioNodeHelper.cs +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/GltfAudioVideoNodeHelper.cs @@ -13,10 +13,10 @@ namespace UnityGLTF.Audio /// Copied the GltfInteractivityNodeHelper for now to use with audio. /// This will need to be cleaned up since a lot of this is not used. /// - public static class GltfAudioNodeHelper + public static class GltfAudioVideoNodeHelper { public static Dictionary GetTranslatableNodes( - IEnumerable sortedNodes, GLTFAudioExportContext exportContext) + IEnumerable sortedNodes, VisualScriptingExportContext exportContext) { Dictionary validNodes = new Dictionary(); @@ -27,7 +27,6 @@ public static Dictionary GetTranslatableNodes( continue; UnitExporter unitExporter = UnitExporterRegistry.CreateUnitExporter(exportContext, unit); - validNodes.Add(unit, unitExporter); } return validNodes; diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/GltfAudioNodeHelper.cs.meta b/Editor/Scripts/Interactivity/VisualScriptingExport/GltfAudioVideoNodeHelper.cs.meta similarity index 100% rename from Editor/Scripts/Interactivity/VisualScriptingExport/GltfAudioNodeHelper.cs.meta rename to Editor/Scripts/Interactivity/VisualScriptingExport/GltfAudioVideoNodeHelper.cs.meta diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Video/VideoPlayerUnitExport.cs b/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Video/VideoPlayerUnitExport.cs new file mode 100644 index 000000000..f0c307eab --- /dev/null +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Video/VideoPlayerUnitExport.cs @@ -0,0 +1,107 @@ +using System.Collections; +using System.Collections.Generic; +using UnityEngine; +using UnityGLTF.Interactivity.Schema; +using Unity.VisualScripting; +using System; +using UnityEditor; + + +namespace UnityGLTF.Interactivity.VisualScripting.Export +{ + public class VideoPlayerUnitExport : IUnitExporter + { + //public Type unitType { get => typeof(AudioSource); } + public System.Type unitType + { + get => typeof(UnityEngine.Video.VideoPlayer); + } + + [InitializeOnLoadMethod] + private static void Register() + { +// InvokeUnitExport.RegisterInvokeExporter(typeof(AudioSource), nameof(AudioSource), +// new AudioSourceUnitExport()); + UnitExporterRegistry.RegisterExporter(new VideoPlayerUnitExport()); + } + + public bool InitializeInteractivityNodes(UnitExporter unitExporter) + { + return true; + /* + var unit = unitExporter.unit as AudioSource; + + if (unit.target == null) + return false; + + // Regular pointer/set + + var materialTemplate = "/materials/{" + GltfInteractivityNodeHelper.IdPointerMaterialIndex + "}/"; + var template = materialTemplate + "pbrMetallicRoughness/baseColorFactor"; + + if (unit is SetMember setMember) + { + var node = unitExporter.CreateNode(new Pointer_SetNode()); + unitExporter.MapInputPortToSocketName(setMember.assign, Pointer_SetNode.IdFlowIn, node); + unitExporter.MapInputPortToSocketName(setMember.input, Pointer_SetNode.IdValue, node); + unitExporter.MapOutFlowConnectionWhenValid(setMember.assigned, Pointer_SetNode.IdFlowOut, node); + + node.SetupPointerTemplateAndTargetInput(GltfInteractivityNodeHelper.IdPointerMaterialIndex, + setMember.target, template, GltfTypes.Float4); + } + else if (unit is InvokeMember invokeMember) + { + // first parameter is the color property name – so based on that we can determine what pointer to set + // var colorPropertyName = invokeMember.inputParameters[0]; + bool hasAlpha = true; + if (unitExporter.IsInputLiteralOrDefaultValue(invokeMember.inputParameters[0], out var colorPropertyName)) + { + var gltfProperty = MaterialPointerHelper.GetPointer(unitExporter, (string)colorPropertyName, out var map); + if (gltfProperty == null) + { + UnitExportLogging.AddErrorLog(unit, "color property name is not supported."); + return false; + } + + hasAlpha = map.ExportKeepColorAlpha; + template = materialTemplate + gltfProperty; + } + else + { + UnitExportLogging.AddErrorLog(unit, "color property name is not a literal or default value, which is not supported."); + return false; + } + + var node = unitExporter.CreateNode(new Pointer_SetNode()); + node.FlowIn(Pointer_SetNode.IdFlowIn).MapToControlInput(invokeMember.enter); + node.FlowOut(Pointer_SetNode.IdFlowOut).MapToControlOutput(invokeMember.exit); + + if (hasAlpha) + { + node.ValueIn(Pointer_SetNode.IdValue).MapToInputPort(invokeMember.inputParameters[1]).SetType(TypeRestriction.LimitToFloat4); + + node.SetupPointerTemplateAndTargetInput(GltfInteractivityNodeHelper.IdPointerMaterialIndex, + invokeMember.target, template, GltfTypes.Float4); + } + else + { + var extract = unitExporter.CreateNode(new Math_Extract4Node()); + var combine = unitExporter.CreateNode(new Math_Combine3Node()); + extract.ValueIn("a").MapToInputPort(invokeMember.inputParameters[1]) + .SetType(TypeRestriction.LimitToFloat4); + + combine.ValueIn("a").ConnectToSource(extract.ValueOut("0")); + combine.ValueIn("b").ConnectToSource(extract.ValueOut("1")); + combine.ValueIn("c").ConnectToSource(extract.ValueOut("2")); + + node.ValueIn(Pointer_SetNode.IdValue).ConnectToSource(combine.FirstValueOut()); + node.SetupPointerTemplateAndTargetInput(GltfInteractivityNodeHelper.IdPointerMaterialIndex, + invokeMember.target, template, GltfTypes.Float3); + } + } + return true; + */ + } + } + +} diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Video/VideoPlayerUnitExport.cs.meta b/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Video/VideoPlayerUnitExport.cs.meta new file mode 100644 index 000000000..2438c0c59 --- /dev/null +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/UnitExporters/Video/VideoPlayerUnitExport.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: da71af5f42093d84ab4d4728a0fc77c0 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Scripts/Interactivity/Schema/GOOGVideoSchemas.cs b/Runtime/Scripts/Interactivity/Schema/GOOGVideoSchemas.cs new file mode 100644 index 000000000..8b6af2a97 --- /dev/null +++ b/Runtime/Scripts/Interactivity/Schema/GOOGVideoSchemas.cs @@ -0,0 +1,234 @@ + +using System; +using System.Collections.Generic; +using GLTF.Schema; +using Newtonsoft.Json.Linq; +using UnityEngine; + +namespace UnityGLTF.Plugins.Experimental +{ + + //[Serializable] + //public class VideoEmitterId : GLTFId { + // public VideoEmitterId() + // { + // } + + // public VideoEmitterId(VideoEmitterId id, GLTFRoot newRoot) : base(id, newRoot) + // { + // } + + // public override GOOG_VideoData Value + // { + // get + // { + // if (Root.Extensions.TryGetValue(KHR_audio.ExtensionName, out IExtension iextension)) + // { + // GOOG_Video extension = iextension as GOOG_Video; + // return extension.videoData[Id]; + // } + // else + // { + // throw new Exception("GOOG_Video not found on root object"); + // } + // } + // } + //} + +// [Serializable] + //public class AudioSourceId : GLTFId { + // public AudioSourceId() + // { + // } + + // public AudioSourceId(AudioSourceId id, GLTFRoot newRoot) : base(id, newRoot) + // { + // } + + // public override KHR_AudioSource Value + // { + // get + // { + // if (Root.Extensions.TryGetValue(KHR_audio.ExtensionName, out IExtension iextension)) + // { + // KHR_audio extension = iextension as KHR_audio; + // return extension.sources[Id]; + // } + // else + // { + // throw new Exception("KHR_audio not found on root object"); + // } + // } + // } + //} + + [Serializable] + public class VideoDataId : GLTFId { + public VideoDataId() + { + } + + public VideoDataId(VideoDataId id, GLTFRoot newRoot) : base(id, newRoot) + { + } + + public override GOOG_VideoData Value + { + get + { + if (Root.Extensions.TryGetValue(KHR_audio.ExtensionName, out IExtension iextension)) + { + GOOG_Video extension = iextension as GOOG_Video; + return extension.videoData[Id]; + } + else + { + throw new Exception("KHR_audio not found on root object"); + } + } + } + } + + //[Serializable] + //public class KHR_SceneAudioEmittersRef : IExtension { + // public List emitters; + + // public JProperty Serialize() { + // var jo = new JObject(); + // JProperty jProperty = new JProperty(KHR_audio.ExtensionName, jo); + + // JArray arr = new JArray(); + + // foreach (var emitter in emitters) { + // arr.Add(emitter.Id); + // } + + // jo.Add(new JProperty(nameof(emitters), arr)); + + // return jProperty; + // } + + // public IExtension Clone(GLTFRoot root) + // { + // return new KHR_SceneAudioEmittersRef() { emitters = emitters }; + // } + //} + + //[Serializable] + //public class KHR_NodeAudioEmitterRef : IExtension { + // public AudioEmitterId emitter; + + // public JProperty Serialize() { + // var jo = new JObject(); + // JProperty jProperty = new JProperty(KHR_audio.ExtensionName, jo); + // jo.Add(new JProperty(nameof(emitter), emitter.Id)); + // return jProperty; + // } + + // public IExtension Clone(GLTFRoot root) + // { + // return new KHR_NodeAudioEmitterRef() { emitter = emitter }; + // } + //} + + + + [Serializable] + public class GOOG_VideoData : GLTFChildOfRootProperty { + public string name; + public bool autoPlay; + public bool loop; + public float speed; + public int video; + + public JObject Serialize() { + var jo = new JObject(); + + if (!string.IsNullOrEmpty(name)) + jo.Add(nameof(name), name); + + jo.Add(nameof(autoPlay), autoPlay); + jo.Add(nameof(loop), loop); + jo.Add(nameof(speed), speed); + jo.Add(nameof(video), video); + + return jo; + } + } + + [Serializable] + public class GOOG_VideoSource : GLTFChildOfRootProperty { + + public string uri; + public string mimeType; + public BufferViewId bufferView; + + public JObject Serialize() + { + var jo = new JObject(); + + if (!string.IsNullOrEmpty(uri)) + { + jo.Add(nameof(uri), uri); + } + else + { + if (!string.IsNullOrEmpty(mimeType)) + jo.Add(nameof(mimeType), mimeType); + jo.Add(nameof(bufferView), bufferView.Id); + } + + return jo; + } + } + + [Serializable] + public class GOOG_Video : IExtension + { + public const string ExtensionName = "GOOG_video"; + + public List videoSource; + public List videoData; + + public JProperty Serialize() + { + var jo = new JObject(); + JProperty jProperty = new JProperty(ExtensionName, jo); + + if (videoSource != null && videoSource.Count > 0) + { + JArray videoArr = new JArray(); + + foreach (var audioData in videoSource) + { + videoArr.Add(audioData.Serialize()); + } + + jo.Add(new JProperty(nameof(videoSource), videoArr)); + } + + if (videoData != null && videoData.Count > 0) + { + JArray videoDataArr = new JArray(); + + foreach (var data in videoData) + { + videoDataArr.Add(data.Serialize()); + } + + jo.Add(new JProperty(nameof(videoData), videoDataArr)); + } + + return jProperty; + } + + public IExtension Clone(GLTFRoot root) + { + return new GOOG_Video() + { + videoData = videoData, + videoSource = videoSource + }; + } + } +} diff --git a/Runtime/Scripts/Interactivity/Schema/GOOGVideoSchemas.cs.meta b/Runtime/Scripts/Interactivity/Schema/GOOGVideoSchemas.cs.meta new file mode 100644 index 000000000..94bd0aae9 --- /dev/null +++ b/Runtime/Scripts/Interactivity/Schema/GOOGVideoSchemas.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: fb42ec2a387e9284f9617f94ea6a3623 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Scripts/Interactivity/Schema/GltfSceneVideoEmitterExtension.cs b/Runtime/Scripts/Interactivity/Schema/GltfSceneVideoEmitterExtension.cs new file mode 100644 index 000000000..826f0d648 --- /dev/null +++ b/Runtime/Scripts/Interactivity/Schema/GltfSceneVideoEmitterExtension.cs @@ -0,0 +1,59 @@ +using GLTF.Schema; + +namespace UnityGLTF.Interactivity.Schema +{ + using System; + using System.Collections.Generic; + using System.Linq; + using GLTF.Schema; + using Newtonsoft.Json.Linq; + using UnityGLTF.Interactivity; + + [Serializable] + + /// + /// Audio Emitter Extension class that is called to serialize the data for the scenes node + /// + public class GltfSceneVideoEmitterExtension : IExtension + { + public const string SceneVideoExtensionName = "GOOG_video"; + + public List videos = new(); + + public GltfSceneVideoEmitterExtension() + { + } + + /// + /// Called when the data is written and serialized to file. + /// + public JProperty Serialize() + { + JObject jo = new JObject(); + + JArray arr = new JArray(); + + foreach (var vid in videos) { + arr.Add(vid); + } + + jo.Add(new JProperty(nameof(videos), arr)); + + + JProperty extension = + new JProperty(GltfSceneVideoEmitterExtension.SceneVideoExtensionName, jo); + return extension; + } + + /// + /// Clones the object + /// + public IExtension Clone(GLTFRoot root) + { + return new GltfSceneVideoEmitterExtension() + { + videos = videos + }; + } + } +} diff --git a/Runtime/Scripts/Interactivity/Schema/GltfSceneVideoEmitterExtension.cs.meta b/Runtime/Scripts/Interactivity/Schema/GltfSceneVideoEmitterExtension.cs.meta new file mode 100644 index 000000000..cf1a29b8c --- /dev/null +++ b/Runtime/Scripts/Interactivity/Schema/GltfSceneVideoEmitterExtension.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 4a386403ce6c1c84a94cc5dc4eba8084 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Scripts/Interactivity/Schema/GltfVideoExtension.cs b/Runtime/Scripts/Interactivity/Schema/GltfVideoExtension.cs new file mode 100644 index 000000000..75729c0a7 --- /dev/null +++ b/Runtime/Scripts/Interactivity/Schema/GltfVideoExtension.cs @@ -0,0 +1,58 @@ +using GLTF.Schema; + +namespace UnityGLTF.Interactivity.Schema +{ + using System; + using System.Linq; + using GLTF.Schema; + using Newtonsoft.Json.Linq; + using UnityGLTF.Interactivity; + + [Serializable] + + /// + /// Audio Extension class that is called to serialize the data + /// + public class GltfVideoExtension : IExtension + { + public const string VideoExtensionName = "GOOG_video"; + + public GltfInteractivityGraph[] graphs; + public int graph = 0; + + public GltfVideoExtension() + { + } + + /// + /// Called when the data is written and serialized to file. + /// + public JProperty Serialize() + { + JObject jo = new JObject + { + new JProperty("graphs", + new JArray( + from gr in graphs + select gr.SerializeObject())), + new JProperty("graph", graph) + }; + + JProperty extension = + new JProperty(GltfVideoExtension.VideoExtensionName, jo); + return extension; + } + + /// + /// Clones the object + /// + public IExtension Clone(GLTFRoot root) + { + return new GltfVideoExtension() + { + graph = graph, + graphs = graphs + }; + } + } +} diff --git a/Runtime/Scripts/Interactivity/Schema/GltfVideoExtension.cs.meta b/Runtime/Scripts/Interactivity/Schema/GltfVideoExtension.cs.meta new file mode 100644 index 000000000..a856cc902 --- /dev/null +++ b/Runtime/Scripts/Interactivity/Schema/GltfVideoExtension.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: fd46feea43bedc147a1f88d160143511 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: From a070d3d43bb6430a9dcf44285b3b4f8ef68a23e7 Mon Sep 17 00:00:00 2001 From: Dan Quinlan Date: Thu, 24 Apr 2025 12:41:39 -0400 Subject: [PATCH 09/11] Fixed multiple video and audio sources. --- .../GLTFAudioExportContext.cs | 51 ++++++++----------- .../GLTFVideoExportContext.cs | 37 ++++++++------ 2 files changed, 41 insertions(+), 47 deletions(-) diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportContext.cs b/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportContext.cs index 25ca0f90d..90bb27d94 100644 --- a/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportContext.cs +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFAudioExportContext.cs @@ -39,6 +39,9 @@ public class AudioDescription private List addedGraphs = new List(); private List nodesToExport = new List(); + private List _audioData = new List(); + private List _audioEmitters = new List(); + internal new ExportGraph currentGraphProcessing { get; private set; } = null; private GLTFRoot _gltfRoot = null; @@ -157,6 +160,21 @@ internal ExportGraph GetAudio(FlowGraph graph) } } + var extension = new KHR_audio + { + audio = new List(_audioData), + emitters = new List(_audioEmitters), + }; + + if (_gltfRoot != null && !_gltfRoot.Extensions.ContainsKey(GltfAudioExtension.AudioExtensionName)) + { + _gltfRoot.AddExtension(GltfAudioExtension.AudioExtensionName, (IExtension)extension); + exporter.DeclareExtensionUsage(GltfAudioExtension.AudioExtensionName); + } + + _audioData.Clear(); + _audioEmitters.Clear(); + newExportGraph.nodes = GltfAudioVideoNodeHelper.GetTranslatableNodes(topologicallySortedNodes, this); nodesToExport.AddRange(newExportGraph.nodes.Select(g => g.Value)); @@ -196,7 +214,6 @@ public static AudioDescription AddAudioSource(AudioSource audioSource) internal void ProcessAudioSource(IUnit unit, AudioSource audioSource) { List audioDataClips = new List(); - List audioEmitters = new List(); var clip = audioSource.clip; @@ -267,9 +284,7 @@ internal void ProcessAudioSource(IUnit unit, AudioSource audioSource) audio.bufferView = result.bufferView; } - var audioData = new List(); - - audioData.Add(audio); + _audioData.Add(audio); var emitter = new KHR_AudioEmitter() { @@ -279,33 +294,7 @@ internal void ProcessAudioSource(IUnit unit, AudioSource audioSource) loop = audioSource.loop }; - audioEmitters.Add(emitter); - - //var audioSources = new List(); - - //var khrAudio = new KHR_AudioSource - //{ - // audio = new AudioDataId { Id = audioSourceId.Id, Root = _gltfRoot }, - // autoPlay = audioSource.playOnAwake, - // loop = audioSource.loop, - // gain = audioSource.volume, - // name = Path.GetFileNameWithoutExtension(path) - //}; - - //audioSources.Add(khrAudio); - - var extension = new KHR_audio - { - audio = new List(audioData), -// sources = new List(audioSources), - emitters = new List(audioEmitters), - }; - - if (_gltfRoot != null) - { - _gltfRoot.AddExtension(GltfAudioExtension.AudioExtensionName, (IExtension)extension); - exporter.DeclareExtensionUsage(GltfAudioExtension.AudioExtensionName); - } + _audioEmitters.Add(emitter); } /// diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFVideoExportContext.cs b/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFVideoExportContext.cs index 5c1f86729..1999f97ad 100644 --- a/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFVideoExportContext.cs +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFVideoExportContext.cs @@ -40,6 +40,10 @@ public class VideoDescription private List addedGraphs = new List(); private List nodesToExport = new List(); + private List _videoDatas = new List(); + private List _videoSources = new List(); + + internal new ExportGraph currentGraphProcessing { get; private set; } = null; private GLTFRoot _gltfRoot = null; @@ -158,6 +162,21 @@ internal ExportGraph GetVideo(FlowGraph graph) } } + var extension = new GOOG_Video + { + videoData = new List(_videoDatas), + videoSource = new List(_videoSources) + }; + + if (_gltfRoot != null && !_gltfRoot.Extensions.ContainsKey(GltfVideoExtension.VideoExtensionName)) + { + _gltfRoot.AddExtension(GltfVideoExtension.VideoExtensionName, (IExtension)extension); + exporter.DeclareExtensionUsage(GltfVideoExtension.VideoExtensionName); + } + + _videoDatas.Clear(); + _videoSources.Clear(); + newExportGraph.nodes = GltfAudioVideoNodeHelper.GetTranslatableNodes(topologicallySortedNodes, this); nodesToExport.AddRange(newExportGraph.nodes.Select(g => g.Value)); @@ -240,11 +259,9 @@ internal void ProcessVideoSource(IUnit unit, VideoPlayer videoPlayer) videoSource.bufferView = result.bufferView; } - var videoSources = new List(); - videoSources.Add(videoSource); + _videoSources.Add(videoSource); - var videoDatas = new List(); var videoData = new GOOG_VideoData() { name = videoPlayer.clip?.name, @@ -253,19 +270,7 @@ internal void ProcessVideoSource(IUnit unit, VideoPlayer videoPlayer) autoPlay = videoPlayer.playOnAwake }; - videoDatas.Add(videoData); - - var extension = new GOOG_Video - { - videoData = new List(videoDatas), - videoSource = new List(videoSources) - }; - - if (_gltfRoot != null) - { - _gltfRoot.AddExtension(GltfVideoExtension.VideoExtensionName, (IExtension)extension); - exporter.DeclareExtensionUsage(GltfVideoExtension.VideoExtensionName); - } + _videoDatas.Add(videoData); } /// From 54f0c3da4f87af275408572a50e4f1057f8805b6 Mon Sep 17 00:00:00 2001 From: Dan Quinlan Date: Fri, 25 Apr 2025 15:21:52 -0400 Subject: [PATCH 10/11] * Changes to output according to the KHR_texture_video spec. Using "GOOG_video" as a placeholder right now. --- .../GLTFVideoExportContext.cs | 25 +-- .../Interactivity/Schema/GOOGVideoSchemas.cs | 202 +++++------------- ...cs => GltfTextureVideoEmitterExtension.cs} | 29 ++- ... GltfTextureVideoEmitterExtension.cs.meta} | 0 4 files changed, 69 insertions(+), 187 deletions(-) rename Runtime/Scripts/Interactivity/Schema/{GltfSceneVideoEmitterExtension.cs => GltfTextureVideoEmitterExtension.cs} (53%) rename Runtime/Scripts/Interactivity/Schema/{GltfSceneVideoEmitterExtension.cs.meta => GltfTextureVideoEmitterExtension.cs.meta} (100%) diff --git a/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFVideoExportContext.cs b/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFVideoExportContext.cs index 1999f97ad..c76cac4d1 100644 --- a/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFVideoExportContext.cs +++ b/Editor/Scripts/Interactivity/VisualScriptingExport/GLTFVideoExportContext.cs @@ -94,11 +94,10 @@ public override void AfterSceneExport(GLTFSceneExporter exporter, GLTFRoot gltfR AfterSceneExport(exporter, gltfRoot, scriptMachines); - // add new scenes audio emitter extension before process and JSON is written out. + // add texture extension here otherwise it will skip over the textures block when writing var v = new Dictionary(); - v.Add(GltfVideoExtension.VideoExtensionName, new GltfSceneVideoEmitterExtension() { videos = GetVideoSourceIndexes() }); - gltfRoot.Scenes.Add(new GLTFScene() { Extensions = v }); - + v.Add(GltfVideoExtension.VideoExtensionName, new GltfTextureVideoEmitterExtension() { videos = _videoDatas.ToArray() }); + exporter.GetRoot().Textures.Add(new GLTFTexture() { Extensions = v }); } /// @@ -164,8 +163,7 @@ internal ExportGraph GetVideo(FlowGraph graph) var extension = new GOOG_Video { - videoData = new List(_videoDatas), - videoSource = new List(_videoSources) + videos = new List(_videoSources) }; if (_gltfRoot != null && !_gltfRoot.Extensions.ContainsKey(GltfVideoExtension.VideoExtensionName)) @@ -174,9 +172,6 @@ internal ExportGraph GetVideo(FlowGraph graph) exporter.DeclareExtensionUsage(GltfVideoExtension.VideoExtensionName); } - _videoDatas.Clear(); - _videoSources.Clear(); - newExportGraph.nodes = GltfAudioVideoNodeHelper.GetTranslatableNodes(topologicallySortedNodes, this); nodesToExport.AddRange(newExportGraph.nodes.Select(g => g.Value)); @@ -185,10 +180,6 @@ internal ExportGraph GetVideo(FlowGraph graph) return newExportGraph; } - public static List GetVideoSourceIndexes() - { - return (_videoSourceIds.Select(r => r.Id).ToList()); - } public static VideoDescription AddVideoSource(VideoPlayer videoPlayer) { @@ -264,9 +255,9 @@ internal void ProcessVideoSource(IUnit unit, VideoPlayer videoPlayer) var videoData = new GOOG_VideoData() { - name = videoPlayer.clip?.name, - speed = videoPlayer.playbackSpeed, - video = audioSourceId.Id, + source = _videoSources.Count - 1, + playhead = 0, + loop = videoPlayer.isLooping, autoPlay = videoPlayer.playOnAwake }; @@ -347,7 +338,7 @@ public override void AfterNodeExport(GLTFSceneExporter exporter, GLTFRoot gltfRo public override bool BeforeMaterialExport(GLTFSceneExporter exporter, GLTFRoot gltfRoot, Material material, GLTFMaterial materialNode) => false; public override void AfterMaterialExport(GLTFSceneExporter exporter, GLTFRoot gltfRoot, Material material, GLTFMaterial materialNode) { } public override void BeforeTextureExport(GLTFSceneExporter exporter, ref GLTFSceneExporter.UniqueTexture texture, string textureSlot) { } - public override void AfterTextureExport(GLTFSceneExporter exporter, GLTFSceneExporter.UniqueTexture texture, int index, GLTFTexture tex) { } + public override void AfterTextureExport(GLTFSceneExporter exporter, GLTFSceneExporter.UniqueTexture texture, int index, GLTFTexture tex){ } public override void AfterPrimitiveExport(GLTFSceneExporter exporter, Mesh mesh, MeshPrimitive primitive, int index) { } public override void AfterMeshExport(GLTFSceneExporter exporter, Mesh mesh, GLTFMesh gltfMesh, int index) { } } diff --git a/Runtime/Scripts/Interactivity/Schema/GOOGVideoSchemas.cs b/Runtime/Scripts/Interactivity/Schema/GOOGVideoSchemas.cs index 8b6af2a97..1ee1c725c 100644 --- a/Runtime/Scripts/Interactivity/Schema/GOOGVideoSchemas.cs +++ b/Runtime/Scripts/Interactivity/Schema/GOOGVideoSchemas.cs @@ -8,150 +8,23 @@ namespace UnityGLTF.Plugins.Experimental { - //[Serializable] - //public class VideoEmitterId : GLTFId { - // public VideoEmitterId() - // { - // } - - // public VideoEmitterId(VideoEmitterId id, GLTFRoot newRoot) : base(id, newRoot) - // { - // } - - // public override GOOG_VideoData Value - // { - // get - // { - // if (Root.Extensions.TryGetValue(KHR_audio.ExtensionName, out IExtension iextension)) - // { - // GOOG_Video extension = iextension as GOOG_Video; - // return extension.videoData[Id]; - // } - // else - // { - // throw new Exception("GOOG_Video not found on root object"); - // } - // } - // } - //} - -// [Serializable] - //public class AudioSourceId : GLTFId { - // public AudioSourceId() - // { - // } - - // public AudioSourceId(AudioSourceId id, GLTFRoot newRoot) : base(id, newRoot) - // { - // } - - // public override KHR_AudioSource Value - // { - // get - // { - // if (Root.Extensions.TryGetValue(KHR_audio.ExtensionName, out IExtension iextension)) - // { - // KHR_audio extension = iextension as KHR_audio; - // return extension.sources[Id]; - // } - // else - // { - // throw new Exception("KHR_audio not found on root object"); - // } - // } - // } - //} - - [Serializable] - public class VideoDataId : GLTFId { - public VideoDataId() - { - } - - public VideoDataId(VideoDataId id, GLTFRoot newRoot) : base(id, newRoot) - { - } - - public override GOOG_VideoData Value - { - get - { - if (Root.Extensions.TryGetValue(KHR_audio.ExtensionName, out IExtension iextension)) - { - GOOG_Video extension = iextension as GOOG_Video; - return extension.videoData[Id]; - } - else - { - throw new Exception("KHR_audio not found on root object"); - } - } - } - } - - //[Serializable] - //public class KHR_SceneAudioEmittersRef : IExtension { - // public List emitters; - - // public JProperty Serialize() { - // var jo = new JObject(); - // JProperty jProperty = new JProperty(KHR_audio.ExtensionName, jo); - - // JArray arr = new JArray(); - - // foreach (var emitter in emitters) { - // arr.Add(emitter.Id); - // } - - // jo.Add(new JProperty(nameof(emitters), arr)); - - // return jProperty; - // } - - // public IExtension Clone(GLTFRoot root) - // { - // return new KHR_SceneAudioEmittersRef() { emitters = emitters }; - // } - //} - - //[Serializable] - //public class KHR_NodeAudioEmitterRef : IExtension { - // public AudioEmitterId emitter; - - // public JProperty Serialize() { - // var jo = new JObject(); - // JProperty jProperty = new JProperty(KHR_audio.ExtensionName, jo); - // jo.Add(new JProperty(nameof(emitter), emitter.Id)); - // return jProperty; - // } - - // public IExtension Clone(GLTFRoot root) - // { - // return new KHR_NodeAudioEmitterRef() { emitter = emitter }; - // } - //} - - - [Serializable] - public class GOOG_VideoData : GLTFChildOfRootProperty { - public string name; + public class GOOG_VideoData {//: GLTFChildOfRootProperty { + public int source; + public int playhead; public bool autoPlay; public bool loop; - public float speed; - public int video; - public JObject Serialize() { - var jo = new JObject(); - if (!string.IsNullOrEmpty(name)) - jo.Add(nameof(name), name); + public JObject SerializeObject() + { + JObject jo = new JObject(); + jo.Add(nameof(source), source); + jo.Add(nameof(playhead), playhead); jo.Add(nameof(autoPlay), autoPlay); jo.Add(nameof(loop), loop); - jo.Add(nameof(speed), speed); - jo.Add(nameof(video), video); - + return jo; } } @@ -167,11 +40,11 @@ public JObject Serialize() { var jo = new JObject(); - if (!string.IsNullOrEmpty(uri)) + if (!string.IsNullOrEmpty(uri)) { jo.Add(nameof(uri), uri); - } - else + } + else { if (!string.IsNullOrEmpty(mimeType)) jo.Add(nameof(mimeType), mimeType); @@ -181,42 +54,64 @@ public JObject Serialize() return jo; } } - [Serializable] - public class GOOG_Video : IExtension + public class GOOG_Video_Data : IExtension { public const string ExtensionName = "GOOG_video"; - public List videoSource; - public List videoData; + public List videoDatas; public JProperty Serialize() { var jo = new JObject(); JProperty jProperty = new JProperty(ExtensionName, jo); - if (videoSource != null && videoSource.Count > 0) + if (videoDatas != null && videoDatas.Count > 0) { JArray videoArr = new JArray(); - foreach (var audioData in videoSource) + foreach (var videoData in videoDatas) { - videoArr.Add(audioData.Serialize()); + videoArr.Add(videoData.SerializeObject()); } - jo.Add(new JProperty(nameof(videoSource), videoArr)); + jo.Add(new JProperty(nameof(videoDatas), videoArr)); } - if (videoData != null && videoData.Count > 0) + return jProperty; + } + + public IExtension Clone(GLTFRoot root) + { + return new GOOG_Video_Data() + { + videoDatas = videoDatas + }; + } + } + + [Serializable] + public class GOOG_Video : IExtension + { + public const string ExtensionName = "GOOG_video"; + + public List videos; + + public JProperty Serialize() + { + var jo = new JObject(); + JProperty jProperty = new JProperty(ExtensionName, jo); + + if (videos != null && videos.Count > 0) { - JArray videoDataArr = new JArray(); + JArray videoArr = new JArray(); - foreach (var data in videoData) + foreach (var video in videos) { - videoDataArr.Add(data.Serialize()); + videoArr.Add(video.Serialize()); } - jo.Add(new JProperty(nameof(videoData), videoDataArr)); + jo.Add(new JProperty(nameof(videos), videoArr)); } return jProperty; @@ -226,8 +121,7 @@ public IExtension Clone(GLTFRoot root) { return new GOOG_Video() { - videoData = videoData, - videoSource = videoSource + videos = videos }; } } diff --git a/Runtime/Scripts/Interactivity/Schema/GltfSceneVideoEmitterExtension.cs b/Runtime/Scripts/Interactivity/Schema/GltfTextureVideoEmitterExtension.cs similarity index 53% rename from Runtime/Scripts/Interactivity/Schema/GltfSceneVideoEmitterExtension.cs rename to Runtime/Scripts/Interactivity/Schema/GltfTextureVideoEmitterExtension.cs index 826f0d648..cad069f3f 100644 --- a/Runtime/Scripts/Interactivity/Schema/GltfSceneVideoEmitterExtension.cs +++ b/Runtime/Scripts/Interactivity/Schema/GltfTextureVideoEmitterExtension.cs @@ -8,19 +8,20 @@ namespace UnityGLTF.Interactivity.Schema using GLTF.Schema; using Newtonsoft.Json.Linq; using UnityGLTF.Interactivity; + using UnityGLTF.Plugins.Experimental; [Serializable] /// - /// Audio Emitter Extension class that is called to serialize the data for the scenes node + /// Video Emitter Extension class that is called to serialize the data for the Texture node /// - public class GltfSceneVideoEmitterExtension : IExtension + public class GltfTextureVideoEmitterExtension : IExtension { - public const string SceneVideoExtensionName = "GOOG_video"; + public const string TextureVideoExtensionName = "GOOG_video"; - public List videos = new(); + public GOOG_VideoData[] videos; - public GltfSceneVideoEmitterExtension() + public GltfTextureVideoEmitterExtension() { } @@ -29,19 +30,15 @@ public GltfSceneVideoEmitterExtension() /// public JProperty Serialize() { - JObject jo = new JObject(); +// JObject jo = new JObject(); + JArray arr = new JArray(); - JArray arr = new JArray(); - - foreach (var vid in videos) { - arr.Add(vid); + foreach (var vid in videos) + { + arr.Add(vid.SerializeObject()); } - - jo.Add(new JProperty(nameof(videos), arr)); - - JProperty extension = - new JProperty(GltfSceneVideoEmitterExtension.SceneVideoExtensionName, jo); + new JProperty(TextureVideoExtensionName, arr); return extension; } @@ -50,7 +47,7 @@ public JProperty Serialize() /// public IExtension Clone(GLTFRoot root) { - return new GltfSceneVideoEmitterExtension() + return new GltfTextureVideoEmitterExtension() { videos = videos }; diff --git a/Runtime/Scripts/Interactivity/Schema/GltfSceneVideoEmitterExtension.cs.meta b/Runtime/Scripts/Interactivity/Schema/GltfTextureVideoEmitterExtension.cs.meta similarity index 100% rename from Runtime/Scripts/Interactivity/Schema/GltfSceneVideoEmitterExtension.cs.meta rename to Runtime/Scripts/Interactivity/Schema/GltfTextureVideoEmitterExtension.cs.meta From c5cd479afe0d1a5c15b42f31eb5d2f48162f0f6d Mon Sep 17 00:00:00 2001 From: Dan Quinlan Date: Mon, 28 Apr 2025 10:42:17 -0400 Subject: [PATCH 11/11] Add an object "data" to the GOOG_Video extension in the interactivity reference which mimics all other types. --- .../GltfTextureVideoEmitterExtension.cs | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/Runtime/Scripts/Interactivity/Schema/GltfTextureVideoEmitterExtension.cs b/Runtime/Scripts/Interactivity/Schema/GltfTextureVideoEmitterExtension.cs index cad069f3f..881513964 100644 --- a/Runtime/Scripts/Interactivity/Schema/GltfTextureVideoEmitterExtension.cs +++ b/Runtime/Scripts/Interactivity/Schema/GltfTextureVideoEmitterExtension.cs @@ -30,16 +30,34 @@ public GltfTextureVideoEmitterExtension() /// public JProperty Serialize() { -// JObject jo = new JObject(); + JObject jo = new JObject(); JArray arr = new JArray(); foreach (var vid in videos) { arr.Add(vid.SerializeObject()); } + jo.Add("data", arr); + // jo.Add("data", arr); JProperty extension = - new JProperty(TextureVideoExtensionName, arr); + new JProperty(TextureVideoExtensionName, jo); return extension; + + //JObject jo = new JObject(); + + //JArray arr = new JArray(); + + //foreach (var video in videos) + //{ + // arr.Add(video); + //} + + //jo.Add(new JProperty(nameof(videos), arr)); + + + //JProperty extension = + // new JProperty(GltfTextureVideoEmitterExtension.TextureVideoExtensionName, jo); + //return extension; } ///