Skip to content
This repository was archived by the owner on Aug 30, 2024. It is now read-only.

Commit f7646a7

Browse files
4thlabs4thlabs
authored
150 implement screen share from c++ sdk (#152)
* Screen share implementation * Updated yuv to rgb for screenshare * Fixed typo
1 parent aadccc3 commit f7646a7

File tree

15 files changed

+392
-68
lines changed

15 files changed

+392
-68
lines changed

‎samples/CmdLine/Main.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -199,7 +199,7 @@ private static async Task JoinConference(string alias)
199199
await _sdk.Conference.SetSpatialEnvironmentAsync
200200
(
201201
new Vector3(1.0f, 1.0f, 1.0f), // Scale
202-
new Vector3(0.0f, 0.0f, 1.0f), // Forward
202+
new Vector3(0.0f, 0.0f, 1.0f), // Forward
203203
new Vector3(0.0f, 1.0f, 0.0f), // Up
204204
new Vector3(1.0f, 0.0f, 0.0f) // Right
205205
);
@@ -353,7 +353,7 @@ public override void OnFrame(VideoFrame frame)
353353
using(frame)
354354
{
355355

356-
Log.Debug($"OnFrame {frame.Width}x{frame.Height} : {frame.DangerousGetHandle()}");
356+
Log.Debug($"OnFrame {frame.Width}x{frame.Height}");
357357
try
358358
{
359359
var buffer = frame.GetBuffer();

‎src/DolbyIO.Comms.Native/media_device.cc‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,20 @@ extern "C" {
157157
}}.result();
158158
}
159159

160+
EXPORT_API int GetScreenShareSources(int* size, dolbyio::comms::native::screen_share_source** dest) {
161+
return call { [&]() {
162+
auto sources = wait(sdk->device_management().get_screen_share_sources());
163+
(*dest) = (dolbyio::comms::native::screen_share_source*) malloc(sizeof(dolbyio::comms::native::screen_share_source) * sources.size());
164+
165+
std::for_each(sources.begin(), sources.end(), [&sources, dest](const dolbyio::comms::screen_share_source& source) {
166+
int index = &source - &sources[0];
167+
no_alloc_to_c(&(*dest)[index], source);
168+
});
169+
170+
(*size) = sources.size();
171+
}}.result();
172+
}
173+
160174
EXPORT_API bool DeleteDeviceIdentity(dolbyio::comms::audio_device::identity* identity) {
161175
if (identity) {
162176
delete identity;

‎src/DolbyIO.Comms.Native/media_device.h‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,15 @@ namespace dolbyio::comms::native {
2929
char* name;
3030
};
3131

32+
/**
33+
* @brief C# Screen Share Source C Struct
34+
*/
35+
struct screen_share_source {
36+
char* title;
37+
int64_t id;
38+
int type;
39+
};
40+
3241
struct on_audio_device_added {
3342
using event = dolbyio::comms::audio_device_added;
3443
using type = void (*)(audio_device dev);
@@ -97,6 +106,21 @@ namespace dolbyio::comms::native {
97106
}
98107
};
99108

109+
template<typename Traits>
110+
struct translator<dolbyio::comms::native::screen_share_source, dolbyio::comms::screen_share_source, Traits> {
111+
static void to_c(typename Traits::c_type* dest, const typename Traits::cpp_type& src) {
112+
dest->title = strdup(src.title);
113+
dest->id = src.id;
114+
dest->type = to_underlying(src.type);
115+
}
116+
117+
static void to_cpp(typename Traits::cpp_type& dest, typename Traits::c_type* src) {
118+
dest.title = std::string(src->title);
119+
dest.id = src->id;
120+
dest.type = (enum dolbyio::comms::screen_share_source::type)src->type;
121+
}
122+
};
123+
100124
} // namespace dolbyio::comms::native
101125

102126
#endif // _MEDIA_DEVICE_H_

‎src/DolbyIO.Comms.Native/video.cc‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,5 +43,21 @@ extern "C" {
4343
}}.result();
4444
}
4545

46+
EXPORT_API int StartScreenShare(dolbyio::comms::native::screen_share_source src, dolbyio::comms::native::video_frame_handler* handler) {
47+
return call { [&]() {
48+
dolbyio::comms::screen_share_source source;
49+
no_alloc_to_cpp(source, &src);
50+
51+
auto shared = std::shared_ptr<dolbyio::comms::native::video_frame_handler>(handler, null_deleter{});
52+
wait(sdk->conference().start_screen_share(source, shared));
53+
}}.result();
54+
}
55+
56+
EXPORT_API int StopScreenShare() {
57+
return call { []() {
58+
wait(sdk->conference().stop_screen_share());
59+
}}.result();
60+
}
61+
4662
} // extern "C"
4763
} // namespace dolbyio::comms::native

‎src/DolbyIO.Comms.Native/video_sink.h‎

Lines changed: 60 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -33,83 +33,77 @@ namespace dolbyio::comms::native {
3333

3434
void handle_frame(std::unique_ptr<video_frame> frame) {
3535
int bytes_per_pixel = 4;
36+
size_t width, height = 0;
37+
uint8_t* resbuffer = nullptr;
3638

3739
#if defined(__APPLE__)
3840
video_frame_macos *mac_frame = frame->get_native_frame();
39-
CVPixelBufferRef buffer = mac_frame->get_buffer();
40-
int res = CVPixelBufferLockBaseAddress(buffer, kCVPixelBufferLock_ReadOnly);
41-
42-
// Sanity check for ensuring we are capturing NV12 from camera
43-
auto format_type = CVPixelBufferGetPixelFormatType(buffer);
44-
if (format_type != kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange &&
45-
format_type != kCVPixelFormatType_420YpCbCr8BiPlanarFullRange)
46-
return;
47-
48-
size_t width = CVPixelBufferGetWidth(buffer);
49-
size_t height = CVPixelBufferGetHeight(buffer);
50-
51-
uint8_t *y_buffer = (uint8_t*)CVPixelBufferGetBaseAddressOfPlane(buffer, 0);
52-
int y_stride = CVPixelBufferGetBytesPerRowOfPlane(buffer, 0);
53-
54-
uint8_t *uv_buffer = (uint8_t*)CVPixelBufferGetBaseAddressOfPlane(buffer, 1);
55-
int uv_stride = CVPixelBufferGetBytesPerRowOfPlane(buffer, 1);
41+
if (mac_frame) {
42+
CVPixelBufferRef buffer = mac_frame->get_buffer();
43+
int res = CVPixelBufferLockBaseAddress(buffer, kCVPixelBufferLock_ReadOnly);
44+
45+
//Sanity check for ensuring we are capturing NV12 from camera
46+
auto format_type = CVPixelBufferGetPixelFormatType(buffer);
47+
if (format_type != kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange &&
48+
format_type != kCVPixelFormatType_420YpCbCr8BiPlanarFullRange)
49+
return;
50+
51+
width = CVPixelBufferGetWidth(buffer);
52+
height = CVPixelBufferGetHeight(buffer);
53+
54+
uint8_t *y_buffer = (uint8_t*)CVPixelBufferGetBaseAddressOfPlane(buffer, 0);
55+
int y_stride = CVPixelBufferGetBytesPerRowOfPlane(buffer, 0);
56+
57+
uint8_t *uv_buffer = (uint8_t*)CVPixelBufferGetBaseAddressOfPlane(buffer, 1);
58+
int uv_stride = CVPixelBufferGetBytesPerRowOfPlane(buffer, 1);
59+
60+
resbuffer = (uint8_t *)malloc(sizeof(uint8_t) * width * height * bytes_per_pixel);
61+
62+
nv12_rgb24_std(
63+
frame->width(),
64+
frame->height(),
65+
y_buffer,
66+
uv_buffer,
67+
y_stride,
68+
uv_stride,
69+
resbuffer,
70+
width * bytes_per_pixel,
71+
ycbcr_type::ycbcr_jpeg);
72+
73+
CVPixelBufferUnlockBaseAddress(buffer, kCVPixelBufferLock_ReadOnly);
74+
} else {
75+
#endif
5676

77+
auto frame_i420 = frame->get_i420_frame();
5778

58-
uint8_t *resbuffer = (uint8_t *)malloc(sizeof(uint8_t) * width * height * bytes_per_pixel);
79+
width = frame->width();
80+
height = frame->height();
5981

60-
for(int y = 0; y < height; y++) {
61-
uint8_t *rgb_line = &resbuffer[y * width * bytes_per_pixel];
62-
uint8_t *y_addr = &y_buffer[y * y_stride];
63-
uint8_t *uv_addr = &uv_buffer[(y >> 1) * uv_stride];
82+
resbuffer = (uint8_t *)malloc(sizeof(uint8_t) * width * height * bytes_per_pixel);
6483

65-
for(int x = 0; x < width; x++) {
66-
int16_t yy = y_addr[x];
67-
int16_t cb = uv_addr[x & ~1] - 128;
68-
int16_t cr = uv_addr[x | 1] - 128;
84+
const uint8_t* y_addr = frame_i420->get_y();
85+
const uint8_t* u_addr = frame_i420->get_u();
86+
const uint8_t* v_addr = frame_i420->get_v();
6987

70-
uint8_t *rgb_output = &rgb_line[x * bytes_per_pixel];
88+
int y_stride = frame_i420->stride_y();
89+
int u_stride = frame_i420->stride_u();
90+
int v_stride = frame_i420->stride_v();
7191

72-
// BT.601 limited.
73-
int16_t r = (yy - 16) * 1.164 + cr * 1.596 ;
74-
int16_t g = (yy - 16) * 1.164 + cb * -0.391 + cr * -0.813 ;
75-
int16_t b = (yy - 16) * 1.164 + cb * 2.018;
92+
yuv420_rgb24_std(
93+
width,
94+
height,
95+
y_addr,
96+
u_addr,
97+
v_addr,
98+
y_stride,
99+
u_stride,
100+
resbuffer,
101+
width * bytes_per_pixel,
102+
ycbcr_type::ycbcr_jpeg
103+
);
76104

77-
rgb_output[0] = 0xff;
78-
rgb_output[1] = clamp(r);
79-
rgb_output[2] = clamp(g);
80-
rgb_output[3] = clamp(b);
81-
}
105+
#if defined(__APPLE__)
82106
}
83-
84-
CVPixelBufferUnlockBaseAddress(buffer, kCVPixelBufferLock_ReadOnly);
85-
#else
86-
auto frame_i420 = frame->get_i420_frame();
87-
88-
size_t width = frame->width();
89-
size_t height = frame->height();
90-
91-
uint8_t* resbuffer = (uint8_t *)malloc(sizeof(uint8_t) * width * height * bytes_per_pixel);
92-
93-
const uint8_t* y_addr = frame_i420->get_y();
94-
const uint8_t* u_addr = frame_i420->get_u();
95-
const uint8_t* v_addr = frame_i420->get_v();
96-
97-
int y_stride = frame_i420->stride_y();
98-
int u_stride = frame_i420->stride_u();
99-
int v_stride = frame_i420->stride_v();
100-
101-
yuv420_rgb24_std(
102-
width,
103-
height,
104-
y_addr,
105-
u_addr,
106-
v_addr,
107-
y_stride,
108-
u_stride,
109-
resbuffer,
110-
width * bytes_per_pixel,
111-
ycbcr_type::ycbcr_2020_full
112-
);
113107
#endif
114108

115109
delegate_(width, height, resbuffer);

‎src/DolbyIO.Comms.Native/yuv_to_rgba.h‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,3 +143,130 @@ static void yuv420_rgb24_std(
143143
}
144144
}
145145
}
146+
147+
static void nv12_rgb24_std(
148+
uint32_t width, uint32_t height,
149+
const uint8_t* y_addr, const uint8_t* uv_addr, uint32_t y_stride, uint32_t uv_stride,
150+
uint8_t *rgb, uint32_t rgb_stride,
151+
ycbcr_type yuv_type)
152+
{
153+
const yuv_params* const param = &(yuv2rb[(int)yuv_type]);
154+
uint32_t x, y;
155+
for(y=0; y<(height-1); y+=2)
156+
{
157+
const uint8_t *y_ptr1=y_addr+y*y_stride,
158+
*y_ptr2=y_addr+(y+1)*y_stride,
159+
*uv_ptr=uv_addr+(y/2)*uv_stride;
160+
161+
uint8_t *rgb_ptr1=rgb+y*rgb_stride,
162+
*rgb_ptr2=rgb+(y+1)*rgb_stride;
163+
164+
for(x=0; x<(width-1); x+=2)
165+
{
166+
int8_t u_tmp, v_tmp;
167+
u_tmp = uv_ptr[0]-128;
168+
v_tmp = uv_ptr[1]-128;
169+
170+
//compute Cb Cr color offsets, common to four pixels
171+
int16_t b_cb_offset, r_cr_offset, g_cbcr_offset;
172+
b_cb_offset = (param->cb_factor*u_tmp)>>6;
173+
r_cr_offset = (param->cr_factor*v_tmp)>>6;
174+
g_cbcr_offset = (param->g_cb_factor*u_tmp + param->g_cr_factor*v_tmp)>>7;
175+
176+
int16_t y_tmp;
177+
y_tmp = (param->y_factor*(y_ptr1[0]-param->y_offset))>>7;
178+
rgb_ptr1[0] = 0xFF;
179+
rgb_ptr1[1] = clamp(y_tmp + r_cr_offset);
180+
rgb_ptr1[2] = clamp(y_tmp - g_cbcr_offset);
181+
rgb_ptr1[3] = clamp(y_tmp + b_cb_offset);
182+
183+
y_tmp = (param->y_factor*(y_ptr1[1]-param->y_offset))>>7;
184+
rgb_ptr1[4] = 0xFF;
185+
rgb_ptr1[5] = clamp(y_tmp + r_cr_offset);
186+
rgb_ptr1[6] = clamp(y_tmp - g_cbcr_offset);
187+
rgb_ptr1[7] = clamp(y_tmp + b_cb_offset);
188+
189+
y_tmp = (param->y_factor*(y_ptr2[0]-param->y_offset))>>7;
190+
rgb_ptr2[0] = 0xFF;
191+
rgb_ptr2[1] = clamp(y_tmp + r_cr_offset);
192+
rgb_ptr2[2] = clamp(y_tmp - g_cbcr_offset);
193+
rgb_ptr2[3] = clamp(y_tmp + b_cb_offset);
194+
195+
y_tmp = (param->y_factor*(y_ptr2[1]-param->y_offset))>>7;
196+
rgb_ptr2[4] = 0xFF;
197+
rgb_ptr2[5] = clamp(y_tmp + r_cr_offset);
198+
rgb_ptr2[6] = clamp(y_tmp - g_cbcr_offset);
199+
rgb_ptr2[7] = clamp(y_tmp + b_cb_offset);
200+
201+
rgb_ptr1 += 8;
202+
rgb_ptr2 += 8;
203+
y_ptr1 += 2;
204+
y_ptr2 += 2;
205+
uv_ptr += 2;
206+
}
207+
}
208+
}
209+
210+
static void nv21_rgb24_std(
211+
uint32_t width, uint32_t height,
212+
const uint8_t* y_addr, const uint8_t* uv_addr, uint32_t y_stride, uint32_t uv_stride,
213+
uint8_t *rgb, uint32_t rgb_stride,
214+
ycbcr_type yuv_type)
215+
{
216+
const yuv_params* const param = &(yuv2rb[(int)yuv_type]);
217+
218+
uint32_t x, y;
219+
for(y=0; y<(height-1); y+=2)
220+
{
221+
const uint8_t *y_ptr1=y_addr+y*y_stride,
222+
*y_ptr2=y_addr+(y+1)*y_stride,
223+
*uv_ptr=uv_addr+(y/2)*uv_stride;
224+
225+
uint8_t *rgb_ptr1=rgb+y*rgb_stride,
226+
*rgb_ptr2=rgb+(y+1)*rgb_stride;
227+
228+
for(x=0; x<(width-1); x+=2)
229+
{
230+
int8_t u_tmp, v_tmp;
231+
u_tmp = uv_ptr[1]-128;
232+
v_tmp = uv_ptr[0]-128;
233+
234+
//compute Cb Cr color offsets, common to four pixels
235+
int16_t b_cb_offset, r_cr_offset, g_cbcr_offset;
236+
b_cb_offset = (param->cb_factor*u_tmp)>>6;
237+
r_cr_offset = (param->cr_factor*v_tmp)>>6;
238+
g_cbcr_offset = (param->g_cb_factor*u_tmp + param->g_cr_factor*v_tmp)>>7;
239+
240+
int16_t y_tmp;
241+
y_tmp = (param->y_factor*(y_ptr1[0]-param->y_offset))>>7;
242+
rgb_ptr1[0] = 0xFF;
243+
rgb_ptr1[1] = clamp(y_tmp + r_cr_offset);
244+
rgb_ptr1[2] = clamp(y_tmp - g_cbcr_offset);
245+
rgb_ptr1[3] = clamp(y_tmp + b_cb_offset);
246+
247+
y_tmp = (param->y_factor*(y_ptr1[1]-param->y_offset))>>7;
248+
rgb_ptr1[4] = 0xFF;
249+
rgb_ptr1[5] = clamp(y_tmp + r_cr_offset);
250+
rgb_ptr1[6] = clamp(y_tmp - g_cbcr_offset);
251+
rgb_ptr1[7] = clamp(y_tmp + b_cb_offset);
252+
253+
y_tmp = (param->y_factor*(y_ptr2[0]-param->y_offset))>>7;
254+
rgb_ptr2[0] = 0xFF;
255+
rgb_ptr2[1] = clamp(y_tmp + r_cr_offset);
256+
rgb_ptr2[2] = clamp(y_tmp - g_cbcr_offset);
257+
rgb_ptr2[3] = clamp(y_tmp + b_cb_offset);
258+
259+
y_tmp = (param->y_factor*(y_ptr2[1]-param->y_offset))>>7;
260+
rgb_ptr2[4] = 0xFF;
261+
rgb_ptr2[5] = clamp(y_tmp + r_cr_offset);
262+
rgb_ptr2[6] = clamp(y_tmp - g_cbcr_offset);
263+
rgb_ptr2[7] = clamp(y_tmp + b_cb_offset);
264+
265+
rgb_ptr1 += 8;
266+
rgb_ptr2 += 8;
267+
y_ptr1 += 2;
268+
y_ptr2 += 2;
269+
uv_ptr += 2;
270+
}
271+
}
272+
}

‎src/DolbyIO.Comms.Sdk/CMakeLists.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ add_dotnet_library(
1212
Native/Enums/Result.cs
1313
Native/Enums/SpatialAudioStyle.cs
1414
Native/Enums/ListenMode.cs
15+
Native/Enums/ScreenShareType.cs
1516
Native/Structs/Handles/VideoFrame.cs
1617
Native/Structs/Handles/VideoSinkHandle.cs
1718
Native/Structs/Handles/VideoFrameHandlerHandle.cs
@@ -32,6 +33,7 @@ add_dotnet_library(
3233
Native/Structs/VideoSink.cs
3334
Native/Structs/VideoFrameHandler.cs
3435
Native/Structs/VideoTrack.cs
36+
Native/Structs/ScreenShareSource.cs
3537
Native/Callback.cs
3638
Native/Handlers.cs
3739
Native/Native.cs

0 commit comments

Comments
 (0)