Multi touch event supported (#25)

* multi-touch event supported

* touch event phase support (C#)

* touch event phase support

* MouseWheel Support (C#)

* MouseWheel support (javascript)

Author: karasusan

Assets/Scenes/SampleScene.unity

@@ -120,6 +120,85 @@ NavMeshSettings:
     debug:
       m_Flags: 0
   m_NavMeshData: {fileID: 0}
+--- !u!1 &145210077
+GameObject:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  serializedVersion: 6
+  m_Component:
+  - component: {fileID: 145210078}
+  - component: {fileID: 145210080}
+  - component: {fileID: 145210079}
+  m_Layer: 5
+  m_Name: Text
+  m_TagString: Untagged
+  m_Icon: {fileID: 0}
+  m_NavMeshLayer: 0
+  m_StaticEditorFlags: 0
+  m_IsActive: 1
+--- !u!224 &145210078
+RectTransform:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 145210077}
+  m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
+  m_LocalPosition: {x: 0, y: 0, z: 0}
+  m_LocalScale: {x: 1, y: 1, z: 1}
+  m_Children: []
+  m_Father: {fileID: 1647766022}
+  m_RootOrder: 1
+  m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
+  m_AnchorMin: {x: 0, y: 0}
+  m_AnchorMax: {x: 1, y: 0}
+  m_AnchoredPosition: {x: 50, y: 50}
+  m_SizeDelta: {x: -100, y: 50}
+  m_Pivot: {x: 0, y: 0}
+--- !u!114 &145210079
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 145210077}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 708705254, guid: f70555f144d8491a825f0804e09c671c, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  m_Material: {fileID: 0}
+  m_Color: {r: 1, g: 1, b: 1, a: 1}
+  m_RaycastTarget: 1
+  m_OnCullStateChanged:
+    m_PersistentCalls:
+      m_Calls: []
+    m_TypeName: UnityEngine.UI.MaskableGraphic+CullStateChangedEvent, UnityEngine.UI,
+      Version=1.0.0.0, Culture=neutral, PublicKeyToken=null
+  m_FontData:
+    m_Font: {fileID: 10102, guid: 0000000000000000e000000000000000, type: 0}
+    m_FontSize: 32
+    m_FontStyle: 0
+    m_BestFit: 0
+    m_MinSize: 3
+    m_MaxSize: 40
+    m_Alignment: 6
+    m_AlignByGeometry: 0
+    m_RichText: 1
+    m_HorizontalOverflow: 0
+    m_VerticalOverflow: 0
+    m_LineSpacing: 1
+  m_Text: New Text
+--- !u!222 &145210080
+CanvasRenderer:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 145210077}
+  m_CullTransparentMesh: 0
 --- !u!1 &170076733
 GameObject:
   m_ObjectHideFlags: 0
@@ -389,7 +468,7 @@ MonoBehaviour:
   m_Script: {fileID: 11500000, guid: 045786cf504bd7347842d6948241cbd0, type: 3}
   m_Name: 
   m_EditorClassIdentifier: 
-  urlSignaling: http://192.168.11.2
+  urlSignaling: http://10.81.35.130
   urlSTUN: stun:stun.l.google.com:19302
   interval: 5
 --- !u!4 &1343183829
@@ -513,6 +592,7 @@ MonoBehaviour:
   m_Name: 
   m_EditorClassIdentifier: 
   cursor: {fileID: 1597907846}
+  text: {fileID: 145210079}
 --- !u!114 &1647766019
 MonoBehaviour:
   m_ObjectHideFlags: 0
@@ -585,6 +665,7 @@ RectTransform:
   m_LocalScale: {x: 0, y: 0, z: 0}
   m_Children:
   - {fileID: 1597907846}
+  - {fileID: 145210078}
   m_Father: {fileID: 0}
   m_RootOrder: 3
   m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}

Assets/Scripts/RemoteInput.cs

@@ -5,26 +5,24 @@
 
 namespace Unity.RenderStreaming
 {
-    enum KeyEventType
+    enum KeyboardEventType
     {
-        KeyDown = 0,
-        KeyUp,
-        KeyPress
+        KeyUp = 0,
+        KeyDown = 1,
     }
     enum EventType
     {
         Keyboard = 0,
-        MouseDown = 2,
-        MouseMove = 4,
-        MouseWheel = 5,
-        TouchMove = 6
+        Mouse = 1,
+        MouseWheel = 2,
+        Touch = 3
     }
 
     public static class RemoteInput
     {
-        static Keyboard keyboard;
-        static Mouse mouse;
-        static Touchscreen touch;
+        public static Keyboard Keyboard { get; private set; }
+        public static Mouse Mouse { get; private set; }
+        public static Touchscreen Touch { get; private set; }
 
         static TDevice GetOrAddDevice<TDevice>() where TDevice : InputDevice
         {
@@ -36,59 +34,98 @@ static TDevice GetOrAddDevice<TDevice>() where TDevice : InputDevice
             return InputSystem.AddDevice<TDevice>();
         }
 
-        static RemoteInput()
+        public static void Initialize()
         {
-            keyboard = GetOrAddDevice<Keyboard>();
-            mouse = GetOrAddDevice<Mouse>();
-            touch = GetOrAddDevice<Touchscreen>();
+            Keyboard = GetOrAddDevice<Keyboard>();
+            Mouse = GetOrAddDevice<Mouse>();
+            Touch = GetOrAddDevice<Touchscreen>();
         }
 
         public static void ProcessInput(byte[] bytes)
         {
             switch ((EventType)bytes[0])
             {
                 case EventType.Keyboard:
-                    ProcessKeyEvent((char)bytes[1]);
+                    var type = (KeyboardEventType)bytes[1];
+                    var repeat = bytes[2] == 1;
+                    var key = (char)bytes[3];
+                    ProcessKeyEvent(type, repeat, key);
                     break;
-                case EventType.MouseDown:
-                    ProcessMouseDownEvent(bytes[1]);
-                    break;
-                case EventType.MouseMove:
+                case EventType.Mouse:
                     var deltaX = BitConverter.ToInt16(bytes, 1);
                     var deltaY = BitConverter.ToInt16(bytes, 3);
-                    ProcessMouseMoveEvent(deltaX, deltaY, bytes[5]);
+                    var button = bytes[5];
+                    ProcessMouseMoveEvent(deltaX, deltaY, button);
                     break;
-                case EventType.TouchMove:
-                    var pageX = BitConverter.ToInt16(bytes, 1);
-                    var pageY = BitConverter.ToInt16(bytes, 3);
-                    ProcessTouchMoveEvent(pageX, pageY);
+                case EventType.MouseWheel:
+                    var scrollX = BitConverter.ToSingle(bytes, 1);
+                    var scrollY = BitConverter.ToSingle(bytes, 5);
+                    ProcessMouseWheelEvent(scrollX, scrollY);
+                    break;
+                case EventType.Touch:
+                    var phase = (PointerPhase)bytes[1];
+                    var length = bytes[2];
+                    var index = 3;
+                    for (int i = 0; i < length; i++)
+                    {
+                        var pageX = BitConverter.ToInt16(bytes, index);
+                        var pageY = BitConverter.ToInt16(bytes, index+2);
+                        var force = BitConverter.ToSingle(bytes, index+4);
+                        ProcessTouchMoveEvent(i, phase, pageX, pageY, force);
+                        index += 8;
+                    }
                     break;
-
             }
         }
 
-        static void ProcessKeyEvent(char keyCode)
+        public static void Reset()
         {
-            InputSystem.QueueStateEvent(keyboard, new KeyboardState((Key)keyCode));
-            InputSystem.QueueTextEvent(keyboard, keyCode);
+            InputSystem.QueueStateEvent(Mouse, new MouseState());
+            InputSystem.QueueStateEvent(Keyboard, new KeyboardState());
+            InputSystem.QueueStateEvent(Touch, new TouchState());
+            InputSystem.Update();
+        }
+
+        static void ProcessKeyEvent(KeyboardEventType state, bool repeat, char keyCode)
+        {
+            switch(state)
+            {
+                case KeyboardEventType.KeyDown:
+                    if (!repeat)
+                    {
+                        InputSystem.QueueStateEvent(Keyboard, new KeyboardState((Key)keyCode));
+                    }
+                    InputSystem.QueueTextEvent(Keyboard, keyCode);
+                    break;
+                case KeyboardEventType.KeyUp:
+                    InputSystem.QueueStateEvent(Keyboard, new KeyboardState());
+                    break;
+            }
             InputSystem.Update();
         }
 
         static void ProcessMouseMoveEvent(short deltaX, short deltaY, byte button)
         {
-            InputSystem.QueueStateEvent(mouse, new MouseState { delta = new Vector2Int(deltaX, deltaY), buttons = button });
+            InputSystem.QueueStateEvent(Mouse, new MouseState { delta = new Vector2Int(deltaX, deltaY), buttons = button });
             InputSystem.Update();
         }
 
-        static void ProcessMouseDownEvent(byte button)
+        static void ProcessMouseWheelEvent(float scrollX, float scrollY)
         {
-            InputSystem.QueueStateEvent(mouse, new MouseState { buttons = button });
+            InputSystem.QueueStateEvent(Mouse, new MouseState { scroll = new Vector2(scrollX, scrollY) });
             InputSystem.Update();
         }
 
-        static void ProcessTouchMoveEvent(short pageX, short pageY)
+        static void ProcessTouchMoveEvent(int index, PointerPhase phase, short pageX, short pageY, float force)
         {
-            InputSystem.QueueStateEvent(touch, new TouchState { position = new Vector2Int(pageX, pageY) });
+            InputSystem.QueueDeltaStateEvent(Touch.allTouchControls[index],
+                new TouchState
+                {
+                    touchId = index,
+                    phase = phase,
+                    position = new Vector2Int(pageX, pageY),
+                    pressure = force
+                });
             InputSystem.Update();
         }
     }

Assets/Scripts/RenderStreaming.cs

@@ -174,7 +174,11 @@ void OnDataChannel(RTCPeerConnection pc, RTCDataChannel channel)
             }
             channels.Add(channel.Id, channel);
 
-            channel.OnMessage = new DelegateOnMessage(bytes => { RemoteInput.ProcessInput(bytes); });
+            if(channel.Label == "data")
+            {
+                channel.OnMessage = new DelegateOnMessage(bytes => { RemoteInput.ProcessInput(bytes); });
+                channel.OnClose = new DelegateOnClose(() => { RemoteInput.Reset(); });
+            }
         }
     }
 }

Assets/Scripts/UIController.cs

@@ -5,35 +5,73 @@
 using UnityEngine.InputSystem;
 using UnityEngine.EventSystems;
 
-public class UIController : MonoBehaviour
+namespace Unity.RenderStreaming
 {
-    [SerializeField] RectTransform cursor;
-    Image image;
-    void Start()
+    public class UIController : MonoBehaviour
     {
-        image = cursor.GetComponent<Image>();
-    }
+#pragma warning disable 0649
+        [SerializeField] RectTransform cursor;
+        [SerializeField] Text text;
 
-    void FixedUpdate()
-    {
-        var delta = Mouse.current.delta.ReadValue();
-        cursor.Translate(delta.x, -delta.y, 0);
+#pragma warning restore 0649
 
-        if (Mouse.current.leftButton.ReadValue() > 0)
-        {
-            image.color = Color.red;
-        }
-        else if (Mouse.current.middleButton.ReadValue() > 0)
+        Image image;
+        Vector2 baseSize;
+        float scroll = 1f;
+        float multiplier = 0.01f;
+
+        void Start()
         {
-            image.color = Color.blue;
+            image = cursor.GetComponent<Image>();
+            baseSize = cursor.sizeDelta;
+
+            RemoteInput.Initialize();
+            Keyboard.current.onTextInput += OnTextInput;
         }
-        else if (Mouse.current.rightButton.ReadValue() > 0)
+
+        void OnTextInput(char c)
         {
-            image.color = Color.green;
+            text.text += c;
         }
-        else
+
+        void FixedUpdate()
         {
-            image.color = Color.white;
+            var delta = Mouse.current.delta.ReadValue();
+            cursor.Translate(delta.x, -delta.y, 0);
+
+            if (Touchscreen.current.activeTouches.Count > 0)
+            {
+                cursor.position = Touchscreen.current.activeTouches[0].position.ReadValue();
+            }
+
+            if (!Keyboard.current.anyKey.isPressed)
+            {
+                text.text = string.Empty;
+            }
+
+
+            if (Mouse.current.leftButton.ReadValue() > 0)
+            {
+                image.color = Color.red;
+            }
+            else if (Mouse.current.middleButton.ReadValue() > 0)
+            {
+                image.color = Color.blue;
+            }
+            else if (Mouse.current.rightButton.ReadValue() > 0)
+            {
+                image.color = Color.green;
+            }
+            else
+            {
+                image.color = Color.white;
+            }
+            if(!Mathf.Approximately(Mouse.current.scroll.ReadValue().y, 0f))
+            {
+                scroll += Mouse.current.scroll.ReadValue().y * multiplier;
+                cursor.sizeDelta = baseSize * scroll;
+            }
         }
     }
+
 }

WebApp/public/scripts/app.js

@@ -27,6 +27,7 @@ function showPlayButton() {
 function onClickPlayButton() {
   const playerDiv = document.getElementById('player');
   const element = document.createElement('video');
+  element.style.touchAction = 'none';
   playerDiv.appendChild(element);
   const videoPlayer = setupVideoPlayer(element);
   registerKeyboardEvents(videoPlayer);