Skip to content

Commit 7522d70

Browse files
flame99999karasusan
flame99999
authored and
karasusan
committed
Add MediaStream to Hdrp sample project (#29)
* Add MediaStream support to HDRP sample project * Try to fix test runner issue * Fixed shader not loaded issue * Add missed file * Refactor according to review comments - Add comments - Make MediaStreamTrack member of MediaStream and Adapt MediaStream for multiple tracks * Add missed meta file * Refactor according to review * Refactor according to review
1 parent 5dc88c4 commit 7522d70

File tree

8 files changed

+397
-162
lines changed

8 files changed

+397
-162
lines changed

‎Assets/Scenes/HDRPScene.unity

+205-77
Large diffs are not rendered by default.

‎Assets/Scripts/RenderStreaming.cs

+13-4
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
using System.Collections.Generic;
33
using UnityEngine;
44
using Unity.WebRTC;
5+
using System.Text.RegularExpressions;
56

67
namespace Unity.RenderStreaming
78
{
@@ -15,16 +16,18 @@ public class RenderStreaming : MonoBehaviour
1516

1617
[SerializeField]
1718
private float interval = 5.0f;
18-
1919
private Signaling signaling;
2020
private Dictionary<string, RTCPeerConnection> pcs = new Dictionary<string, RTCPeerConnection>();
2121
private Dictionary<RTCPeerConnection, Dictionary<int, RTCDataChannel>> mapChannels = new Dictionary<RTCPeerConnection, Dictionary<int, RTCDataChannel>>();
2222
private RTCConfiguration conf;
2323
private string sessionId;
24+
[SerializeField]
25+
private Camera cam;
2426

2527
public void Awake()
2628
{
2729
WebRTC.WebRTC.Initialize();
30+
RemoteInput.Initialize();
2831
}
2932

3033
public void OnDestroy()
@@ -90,8 +93,15 @@ IEnumerator GetOffer()
9093
pc.OnDataChannel = new DelegateOnDataChannel(channel => { OnDataChannel(pc, channel); });
9194
pc.SetConfiguration(ref conf);
9295
pc.OnIceCandidate = new DelegateOnIceCandidate(candidate => { StartCoroutine(OnIceCandidate(offer.connectionId, candidate)); });
96+
//make video bit rate starts at 16000kbits, and 160000kbits at max.
97+
string pattern = @"(a=fmtp:\d+ .*level-asymmetry-allowed=.*)\r\n";
98+
_desc.sdp = Regex.Replace(_desc.sdp, pattern, "$1;x-google-start-bitrate=16000;x-google-max-bitrate=160000\r\n");
9399
pc.SetRemoteDescription(ref _desc);
94-
100+
foreach (var track in cam.CaptureStream(1280, 720).GetTracks())
101+
{
102+
pc.AddTrack(track);
103+
}
104+
StartCoroutine(WebRTC.WebRTC.Update());
95105
StartCoroutine(Answer(connectionId));
96106
}
97107
}
@@ -114,7 +124,7 @@ IEnumerator Answer(string connectionId)
114124
Debug.LogError($"Network Error: {opLocalDesc.error}");
115125
yield break;
116126
}
117-
var op3 = signaling.PostAnswer(this.sessionId, connectionId, op.desc.sdp);
127+
var op3 = signaling.PostAnswer(this.sessionId, connectionId, op.desc.sdp);
118128
yield return op3;
119129
if (op3.webRequest.isNetworkError)
120130
{
@@ -163,7 +173,6 @@ IEnumerator OnIceCandidate(string connectionId, RTCIceCandidate candidate)
163173
yield break;
164174
}
165175
}
166-
167176
void OnDataChannel(RTCPeerConnection pc, RTCDataChannel channel)
168177
{
169178
Dictionary<int, RTCDataChannel> channels;

‎Packages/com.unity.webrtc/Runtime/Resources.meta

+8
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Packages/com.unity.webrtc/Runtime/Resources/Flip.shader

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
Shader "Hidden/Flip"
2+
{
3+
Properties
4+
{
5+
_MainTex ("Texture", 2D) = "white" {}
6+
}
7+
SubShader
8+
{
9+
// No culling or depth
10+
Cull Off ZWrite Off ZTest Always
11+
12+
Pass
13+
{
14+
CGPROGRAM
15+
#pragma vertex vert
16+
#pragma fragment frag
17+
18+
#include "UnityCG.cginc"
19+
20+
struct appdata
21+
{
22+
float4 vertex : POSITION;
23+
float2 uv : TEXCOORD0;
24+
};
25+
26+
struct v2f
27+
{
28+
float2 uv : TEXCOORD0;
29+
float4 vertex : SV_POSITION;
30+
};
31+
32+
v2f vert (appdata v)
33+
{
34+
v2f o;
35+
o.vertex = UnityObjectToClipPos(v.vertex);
36+
o.uv = v.uv;
37+
return o;
38+
}
39+
40+
sampler2D _MainTex;
41+
42+
fixed4 frag (v2f i) : SV_Target
43+
{
44+
i.uv.y = 1 - i.uv.y;
45+
return tex2D(_MainTex, i.uv);
46+
}
47+
ENDCG
48+
}
49+
}
50+
}

‎Packages/com.unity.webrtc/Runtime/Resources/Flip.shader.meta

+9
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs

+97-56
Original file line numberDiff line numberDiff line change
@@ -9,81 +9,116 @@ namespace Unity.WebRTC
99
public class MediaStream
1010
{
1111
private IntPtr self;
12-
private RenderTexture rt;
1312
private string id;
1413
public string Id { get => id; private set { } }
1514

16-
public RenderTexture Rt { get => rt; private set => rt = value; }
15+
private Dictionary<MediaStreamTrack, RenderTexture[]> VideoTrackToRts;
16+
private List<MediaStreamTrack> AudioTracks;
1717

18-
public MediaStreamTrack[] GetTracks()
19-
{
20-
int audioTrackSize = 0, videoTrackSize = 0;
21-
IntPtr audioPtr = NativeMethods.MediaStreamGetAudioTracks(self, ref audioTrackSize);
22-
IntPtr videoPtr = NativeMethods.MediaStreamGetVideoTracks(self, ref videoTrackSize);
23-
IntPtr[] tracksPtr = new IntPtr[audioTrackSize + videoTrackSize];
24-
Marshal.Copy(audioPtr, tracksPtr, 0, audioTrackSize);
25-
Marshal.Copy(videoPtr, tracksPtr, audioTrackSize, videoTrackSize);
26-
//TODO: Linux compatibility
27-
Marshal.FreeCoTaskMem(audioPtr);
28-
Marshal.FreeCoTaskMem(videoPtr);
29-
MediaStreamTrack[] tracks = new MediaStreamTrack[audioTrackSize + videoTrackSize];
30-
for (int i = 0; i < audioTrackSize + videoTrackSize; i++)
18+
private void StopTrack(MediaStreamTrack track)
19+
{
20+
21+
if (track.Kind == TrackKind.Video)
3122
{
32-
tracks[i] = new MediaStreamTrack(Rt, tracksPtr[i]);
23+
NativeMethods.StopMediaStreamTrack(track.self);
24+
RenderTexture[] rts = VideoTrackToRts[track];
25+
if (rts != null)
26+
{
27+
CameraExtension.RemoveRt(rts);
28+
rts[0].Release();
29+
rts[1].Release();
30+
UnityEngine.Object.Destroy(rts[0]);
31+
UnityEngine.Object.Destroy(rts[1]);
32+
}
3333
}
34+
else
35+
{
36+
Audio.Stop();
37+
}
38+
39+
}
40+
private RenderTexture[] GetRts(MediaStreamTrack track)
41+
{
42+
return VideoTrackToRts[track];
43+
}
44+
public MediaStreamTrack[] GetTracks()
45+
{
46+
MediaStreamTrack[] tracks = new MediaStreamTrack[VideoTrackToRts.Keys.Count + AudioTracks.Count];
47+
AudioTracks.CopyTo(tracks, 0);
48+
VideoTrackToRts.Keys.CopyTo(tracks, AudioTracks.Count);
3449
return tracks;
3550
}
3651
public MediaStreamTrack[] GetAudioTracks()
3752
{
38-
int trackSize = 0;
39-
IntPtr ptr = NativeMethods.MediaStreamGetAudioTracks(self, ref trackSize);
40-
IntPtr[] tracksPtr = new IntPtr[trackSize];
41-
Marshal.Copy(ptr, tracksPtr, 0, trackSize);
42-
//TODO: Linux compatibility
43-
Marshal.FreeCoTaskMem(ptr);
44-
45-
MediaStreamTrack[] tracks = new MediaStreamTrack[trackSize];
46-
for (int i = 0; i < trackSize; i++)
47-
{
48-
tracks[i] = new MediaStreamTrack(tracksPtr[i]);
49-
}
50-
return tracks;
53+
return AudioTracks.ToArray();
5154
}
5255
public MediaStreamTrack[] GetVideoTracks()
5356
{
54-
int trackSize = 0;
55-
IntPtr ptr = NativeMethods.MediaStreamGetVideoTracks(self, ref trackSize);
56-
IntPtr[] tracksPtr = new IntPtr[trackSize];
57-
Marshal.Copy(ptr, tracksPtr, 0, trackSize);
58-
//TODO: Linux compatibility
59-
Marshal.FreeCoTaskMem(ptr);
60-
61-
MediaStreamTrack[] tracks = new MediaStreamTrack[trackSize];
62-
for (int i = 0; i < trackSize; i++)
63-
{
64-
tracks[i] = new MediaStreamTrack(Rt, tracksPtr[i]);
65-
}
57+
MediaStreamTrack[] tracks = new MediaStreamTrack[VideoTrackToRts.Keys.Count];
58+
VideoTrackToRts.Keys.CopyTo(tracks, 0);
6659
return tracks;
6760
}
6861

6962
public void AddTrack(MediaStreamTrack track)
7063
{
64+
if(track.Kind == TrackKind.Video)
65+
{
66+
VideoTrackToRts[track] = track.getRts(track);
67+
}
68+
else
69+
{
70+
AudioTracks.Add(track);
71+
}
7172
NativeMethods.MediaStreamAddTrack(self, track.self);
7273
}
7374
public void RemoveTrack(MediaStreamTrack track)
7475
{
7576
NativeMethods.MediaStreamRemoveTrack(self, track.self);
7677
}
77-
internal MediaStream(RenderTexture rt, IntPtr ptr)
78+
//for camera CaptureStream
79+
internal MediaStream(RenderTexture[] rts, IntPtr ptr)
7880
{
7981
self = ptr;
8082
id = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamGetID(self));
81-
Rt = rt;
83+
VideoTrackToRts = new Dictionary<MediaStreamTrack, RenderTexture[]>();
84+
AudioTracks = new List<MediaStreamTrack>();
85+
//get initial tracks
86+
int trackSize = 0;
87+
IntPtr tracksNativePtr = NativeMethods.MediaStreamGetVideoTracks(self, ref trackSize);
88+
IntPtr[] tracksPtr = new IntPtr[trackSize];
89+
Marshal.Copy(tracksNativePtr, tracksPtr, 0, trackSize);
90+
//TODO: Linux compatibility
91+
Marshal.FreeCoTaskMem(tracksNativePtr);
92+
for (int i = 0; i < trackSize; i++)
93+
{
94+
MediaStreamTrack track = new MediaStreamTrack(tracksPtr[i]);
95+
track.stopTrack += StopTrack;
96+
track.getRts += GetRts;
97+
VideoTrackToRts[track] = rts;
98+
}
8299
}
100+
//for audio CaptureStream
83101
internal MediaStream(IntPtr ptr)
84102
{
85103
self = ptr;
86104
id = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamGetID(self));
105+
VideoTrackToRts = new Dictionary<MediaStreamTrack, RenderTexture[]>();
106+
AudioTracks = new List<MediaStreamTrack>();
107+
//get initial tracks
108+
int trackSize = 0;
109+
IntPtr trackNativePtr = NativeMethods.MediaStreamGetAudioTracks(self, ref trackSize);
110+
IntPtr[] tracksPtr = new IntPtr[trackSize];
111+
Marshal.Copy(trackNativePtr, tracksPtr, 0, trackSize);
112+
//TODO: Linux compatibility
113+
Marshal.FreeCoTaskMem(trackNativePtr);
114+
115+
for (int i = 0; i < trackSize; i++)
116+
{
117+
MediaStreamTrack track = new MediaStreamTrack(tracksPtr[i]);
118+
track.stopTrack += StopTrack;
119+
track.getRts += GetRts;
120+
AudioTracks.Add(track);
121+
}
87122
}
88123

89124
}
@@ -117,30 +152,36 @@ public static void AddCleanerCallback(this GameObject obj, Action callback)
117152
}
118153
public static class CameraExtension
119154
{
120-
private static List<RenderTexture> camCopyRts = new List<RenderTexture>();
155+
internal static List<RenderTexture[]> camCopyRts = new List<RenderTexture[]>();
121156
internal static bool started = false;
122157
public static MediaStream CaptureStream(this Camera cam, int width, int height)
123158
{
124-
RenderTexture rt = new RenderTexture(width, height, 0, RenderTextureFormat.BGRA32);
125-
rt.Create();
126-
camCopyRts.Add(rt);
127-
cam.targetTexture = rt;
159+
RenderTexture[] rts = new RenderTexture[2];
160+
//rts[0] for render target, rts[1] for flip and WebRTC source
161+
rts[0] = new RenderTexture(width, height, 0, RenderTextureFormat.BGRA32);
162+
rts[1] = new RenderTexture(width, height, 0, RenderTextureFormat.BGRA32);
163+
rts[0].Create();
164+
rts[1].Create();
165+
camCopyRts.Add(rts);
166+
cam.targetTexture = rts[0];
128167
cam.gameObject.AddCleanerCallback(() =>
129168
{
130-
if (rt != null)
169+
if (rts != null)
131170
{
132-
CameraExtension.RemoveRt(rt);
133-
rt.Release();
134-
UnityEngine.Object.Destroy(rt);
171+
CameraExtension.RemoveRt(rts);
172+
rts[0].Release();
173+
rts[1].Release();
174+
UnityEngine.Object.Destroy(rts[0]);
175+
UnityEngine.Object.Destroy(rts[1]);
135176
}
136177
});
137178
started = true;
138-
return new MediaStream(rt, WebRTC.Context.CaptureVideoStream(rt.GetNativeTexturePtr(), width, height));
179+
return new MediaStream(rts, WebRTC.Context.CaptureVideoStream(rts[1].GetNativeTexturePtr(), width, height));
139180
}
140-
public static void RemoveRt(RenderTexture rt)
181+
public static void RemoveRt(RenderTexture[] rts)
141182
{
142-
camCopyRts.Remove(rt);
143-
if(camCopyRts.Count == 0)
183+
camCopyRts.Remove(rts);
184+
if (camCopyRts.Count == 0)
144185
{
145186
started = false;
146187
}

‎Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs

+4-25
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,8 @@ public class MediaStreamTrack
1111
private string id;
1212
private bool enabled;
1313
private TrackState readyState;
14-
private RenderTexture rt;
14+
internal Action<MediaStreamTrack> stopTrack;
15+
internal Func<MediaStreamTrack, RenderTexture[]> getRts;
1516

1617
public bool Enabled
1718
{
@@ -36,16 +37,7 @@ private set { }
3637
public TrackKind Kind { get => kind; private set { } }
3738
public string Id { get => id; private set { } }
3839

39-
public RenderTexture Rt { get => rt; private set => rt = value; }
40-
41-
internal MediaStreamTrack(RenderTexture rt, IntPtr ptr)
42-
{
43-
self = ptr;
44-
kind = NativeMethods.MediaStreamTrackGetKind(self);
45-
id = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamTrackGetID(self));
46-
Rt = rt;
47-
}
48-
internal MediaStreamTrack(IntPtr ptr)
40+
internal MediaStreamTrack(IntPtr ptr)
4941
{
5042
self = ptr;
5143
kind = NativeMethods.MediaStreamTrackGetKind(self);
@@ -54,20 +46,7 @@ internal MediaStreamTrack(IntPtr ptr)
5446
//Disassociate track from its source(video or audio), not for destroying the track
5547
public void Stop()
5648
{
57-
if (kind == TrackKind.Video)
58-
{
59-
NativeMethods.StopMediaStreamTrack(self);
60-
if (Rt != null)
61-
{
62-
CameraExtension.RemoveRt(Rt);
63-
Rt.Release();
64-
UnityEngine.Object.Destroy(Rt);
65-
}
66-
}
67-
else
68-
{
69-
Audio.Stop();
70-
}
49+
stopTrack(this);
7150
}
7251
}
7352

0 commit comments

Comments
 (0)