Line data Source code
1 : /******************************************************************************************************
2 :
3 : (C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB,
4 : Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,
5 : Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange,
6 : Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other
7 : contributors to this repository. All Rights Reserved.
8 :
9 : This software is protected by copyright law and by international treaties.
10 : The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB,
11 : Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,
12 : Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange,
13 : Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other
14 : contributors to this repository retain full ownership rights in their respective contributions in
15 : the software. This notice grants no license of any kind, including but not limited to patent
16 : license, nor is any license granted by implication, estoppel or otherwise.
17 :
18 : Contributors are required to enter into the IVAS codec Public Collaboration agreement before making
19 : contributions.
20 :
21 : This software is provided "AS IS", without any express or implied warranties. The software is in the
22 : development stage. It is intended exclusively for experts who have experience with such software and
23 : solely for the purpose of inspection. All implied warranties of non-infringement, merchantability
24 : and fitness for a particular purpose are hereby disclaimed and excluded.
25 :
26 : Any dispute, controversy or claim arising under or in relation to providing this software shall be
27 : submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in
28 : accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and
29 : the United Nations Convention on Contracts on the International Sales of Goods.
30 :
31 : *******************************************************************************************************/
32 :
33 : #include <stdint.h>
34 : #include "options.h"
35 : #include "prot.h"
36 : #include "ivas_prot.h"
37 : #include "ivas_prot_rend.h"
38 : #include <math.h>
39 : #include "ivas_rom_com.h"
40 : #ifdef DEBUGGING
41 : #include "debug.h"
42 : #endif
43 : #include "wmc_auto.h"
44 :
45 :
46 : /*---------------------------------------------------------------------*
47 : * ivas_td_binaural_open()
48 : *
49 : * Open and initialize TD Object binaural renderer
50 : *---------------------------------------------------------------------*/
51 :
52 993 : ivas_error ivas_td_binaural_open(
53 : Decoder_Struct *st_ivas /* i/o: IVAS decoder structure */
54 : )
55 : {
56 : int16_t num_src;
57 :
58 993 : num_src = st_ivas->nchan_transport;
59 993 : if ( ( st_ivas->ism_mode == ISM_MASA_MODE_DISC ) || ( st_ivas->ism_mode == ISM_SBA_MODE_DISC ) )
60 : {
61 669 : num_src = st_ivas->nchan_ism;
62 : }
63 :
64 993 : if ( st_ivas->hHrtfTD == NULL && st_ivas->hDecoderConfig->Opt_HRTF_binary )
65 : {
66 0 : return IVAS_ERROR( IVAS_ERR_INTERNAL, "HRTF binary file present but not used in TD renderer" );
67 : }
68 :
69 993 : return ivas_td_binaural_open_unwrap( &st_ivas->hHrtfTD, st_ivas->hDecoderConfig->output_Fs, num_src, st_ivas->ivas_format,
70 993 : st_ivas->transport_config, st_ivas->hRenderConfig->directivity, st_ivas->hRenderConfig->distAtt, st_ivas->hTransSetup, &st_ivas->hBinRendererTd, &st_ivas->binaural_latency_ns );
71 : }
72 :
73 :
74 : /*---------------------------------------------------------------------*
75 : * ivas_td_binaural_renderer_sf()
76 : *
77 : * Receives the current frames for the object streams, updates metadata
78 : * and renders the current frame.
79 : *---------------------------------------------------------------------*/
80 :
81 319600 : ivas_error ivas_td_binaural_renderer_sf(
82 : Decoder_Struct *st_ivas, /* i/o: IVAS decoder structure */
83 : float *output[], /* i/o: SCE channels / Binaural synthesis */
84 : const int16_t n_samples_granularity /* i : granularity of the renderer/buffer */
85 : )
86 : {
87 : int16_t first_sf, last_sf, subframe_idx;
88 : float reverb_signal[BINAURAL_CHANNELS][L_FRAME48k / MAX_PARAM_SPATIAL_SUBFRAMES];
89 : float *p_reverb_signal[BINAURAL_CHANNELS];
90 : float *output_f_local[BINAURAL_CHANNELS];
91 : float *tc_local[MAX_TRANSPORT_CHANNELS];
92 : int16_t ch, slot_size, slots_to_render, output_frame;
93 : ivas_error error;
94 :
95 : int16_t ism_md_subframe_update_jbm;
96 : int16_t c_indx, nS;
97 : int16_t nchan_ism_internal, nchan_ism, ch_offset;
98 :
99 : /* Set the number of ISMs */
100 319600 : if ( st_ivas->ivas_format == MASA_ISM_FORMAT || ( st_ivas->ivas_format == MASA_FORMAT && st_ivas->nchan_ism > 0 ) )
101 : {
102 2308 : nchan_ism_internal = st_ivas->nchan_ism;
103 2308 : nchan_ism = st_ivas->nchan_ism;
104 2308 : ch_offset = CPE_CHANNELS;
105 : }
106 317292 : else if ( st_ivas->ivas_format == SBA_ISM_FORMAT )
107 : {
108 72385 : nchan_ism_internal = st_ivas->nchan_ism;
109 72385 : nchan_ism = st_ivas->nchan_ism;
110 72385 : ch_offset = 0;
111 : }
112 : else
113 : {
114 244907 : nchan_ism_internal = st_ivas->hTcBuffer->nchan_transport_internal;
115 244907 : nchan_ism = st_ivas->nchan_transport;
116 244907 : ch_offset = 0;
117 : }
118 :
119 : /* Number of subframes to delay metadata to sync with audio */
120 319600 : if ( st_ivas->hDecoderConfig->Opt_delay_comp )
121 : {
122 319600 : ism_md_subframe_update_jbm = max( 0, st_ivas->hTcBuffer->nb_subframes - 3 );
123 : }
124 : else
125 : {
126 0 : ism_md_subframe_update_jbm = st_ivas->hTcBuffer->nb_subframes - 2;
127 : }
128 :
129 319600 : if ( st_ivas->ivas_format == MASA_ISM_FORMAT )
130 : {
131 2308 : ism_md_subframe_update_jbm = max( 0, st_ivas->hTcBuffer->nb_subframes - 2 );
132 : }
133 :
134 958800 : for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ )
135 : {
136 639200 : p_reverb_signal[ch] = reverb_signal[ch];
137 : }
138 :
139 1337763 : for ( ch = 0; ch < nchan_ism_internal; ch++ )
140 : {
141 1018163 : tc_local[ch] = st_ivas->hTcBuffer->tc[ch + ch_offset] + st_ivas->hTcBuffer->n_samples_rendered;
142 : }
143 :
144 958800 : for ( ch = 0; ch < st_ivas->hDecoderConfig->nchan_out; ch++ )
145 : {
146 639200 : output_f_local[ch] = output[ch];
147 : }
148 :
149 319600 : slot_size = st_ivas->hTcBuffer->n_samples_granularity;
150 :
151 : /* loop for synthesis, assume we always have to render in multiples of 5ms subframes with spills */
152 319600 : slots_to_render = min( st_ivas->hTcBuffer->num_slots - st_ivas->hTcBuffer->slots_rendered, n_samples_granularity / slot_size );
153 319600 : first_sf = st_ivas->hTcBuffer->subframes_rendered;
154 319600 : last_sf = first_sf;
155 319600 : st_ivas->hTcBuffer->slots_rendered += slots_to_render;
156 :
157 865048 : while ( slots_to_render > 0 )
158 : {
159 545448 : slots_to_render -= st_ivas->hTcBuffer->subframe_nbslots[last_sf];
160 545448 : last_sf++;
161 : }
162 :
163 865048 : for ( subframe_idx = first_sf; subframe_idx < last_sf; subframe_idx++ )
164 : {
165 545448 : output_frame = st_ivas->hTcBuffer->subframe_nbslots[subframe_idx] * st_ivas->hTcBuffer->n_samples_granularity;
166 :
167 : /* Update object position(s) */
168 545448 : c_indx = 0;
169 :
170 2282334 : for ( nS = 0; nS < nchan_ism; nS++ )
171 : {
172 1736886 : if ( !( st_ivas->ivas_format == MC_FORMAT && nS == LFE_CHANNEL ) ) /* Skip LFE for MC */
173 : {
174 1699407 : st_ivas->hBinRendererTd->Sources[c_indx]->InputFrame_p = tc_local[nS];
175 1699407 : st_ivas->hBinRendererTd->Sources[c_indx]->SrcRend_p->InputAvailable = TRUE;
176 1699407 : c_indx++;
177 : }
178 : }
179 :
180 545448 : if ( subframe_idx == ism_md_subframe_update_jbm )
181 : {
182 136578 : if ( st_ivas->ivas_format == ISM_FORMAT || st_ivas->ivas_format == SBA_ISM_FORMAT || st_ivas->ivas_format == MASA_ISM_FORMAT )
183 127206 : {
184 : ISM_METADATA_FRAME ismMetaData[MAX_NUM_OBJECTS];
185 : ISM_METADATA_HANDLE hIsmMetaData[MAX_NUM_OBJECTS];
186 :
187 505221 : for ( nS = 0; nS < nchan_ism; nS++ )
188 : {
189 378015 : ismMetaData[nS].azimuth = st_ivas->hIsmMetaData[nS]->edited_azimuth;
190 378015 : ismMetaData[nS].elevation = st_ivas->hIsmMetaData[nS]->edited_elevation;
191 378015 : ismMetaData[nS].radius = st_ivas->hIsmMetaData[nS]->edited_radius;
192 378015 : ismMetaData[nS].yaw = st_ivas->hIsmMetaData[nS]->edited_yaw;
193 378015 : ismMetaData[nS].pitch = st_ivas->hIsmMetaData[nS]->edited_pitch;
194 378015 : ismMetaData[nS].non_diegetic_flag = st_ivas->hIsmMetaData[nS]->non_diegetic_flag;
195 :
196 378015 : if ( st_ivas->ivas_format == MASA_ISM_FORMAT )
197 : {
198 : /* DISC OMASA ISM gaining with TDREND is done in ivas_dec_prepare_renderer()*/
199 3456 : ismMetaData[nS].gain = 1;
200 : }
201 : else
202 : {
203 374559 : ismMetaData[nS].gain = st_ivas->hIsmMetaData[nS]->edited_gain;
204 : }
205 :
206 378015 : hIsmMetaData[nS] = &ismMetaData[nS];
207 : }
208 :
209 127206 : if ( ( error = TDREND_Update_object_positions( st_ivas->hBinRendererTd, nchan_ism, st_ivas->ivas_format, hIsmMetaData ) ) != IVAS_ERR_OK )
210 : {
211 0 : return error;
212 : }
213 : }
214 : else
215 : {
216 9372 : if ( ( error = TDREND_Update_object_positions( st_ivas->hBinRendererTd, nchan_ism, st_ivas->ivas_format, st_ivas->hIsmMetaData ) ) != IVAS_ERR_OK )
217 : {
218 0 : return error;
219 : }
220 : }
221 : }
222 :
223 : /* Update the listener's location/orientation */
224 1122330 : if ( ( error = TDREND_Update_listener_orientation( st_ivas->hBinRendererTd,
225 545448 : ( st_ivas->hCombinedOrientationData != NULL ) ? st_ivas->hCombinedOrientationData->enableCombinedOrientation[st_ivas->hCombinedOrientationData->subframe_idx] : 0,
226 545448 : ( st_ivas->hCombinedOrientationData != NULL ) ? &st_ivas->hCombinedOrientationData->Quaternions[st_ivas->hCombinedOrientationData->subframe_idx] : NULL,
227 545448 : ( st_ivas->hCombinedOrientationData != NULL ) ? &st_ivas->hCombinedOrientationData->listenerPos[st_ivas->hCombinedOrientationData->subframe_idx] : NULL ) ) != IVAS_ERR_OK )
228 : {
229 0 : return error;
230 : }
231 545448 : if ( st_ivas->hRenderConfig != NULL && st_ivas->hOutSetup.output_config == IVAS_AUDIO_CONFIG_BINAURAL_ROOM_REVERB )
232 : {
233 70188 : if ( ( error = ivas_reverb_process( st_ivas->hReverb, st_ivas->transport_config, 0, tc_local, p_reverb_signal, 0 ) ) != IVAS_ERR_OK )
234 : {
235 0 : return error;
236 : }
237 : }
238 :
239 : /* Render subframe */
240 : /* ism_md_subframe_update_jbm != subframe_idx: trigger update only for ism_md_subframe_update_jbm == subframe_idx,
241 : where then the two TDREND_GetMix()-arguments subframe_idx and ism_md_subframe_update are equal, and we want to enforce the update inside TDREND_GetMix to use subframe_idx == 0 */
242 545448 : if ( ( error = TDREND_GetMix( st_ivas->hBinRendererTd, output_f_local, output_frame, 0 ) ) != IVAS_ERR_OK )
243 : {
244 0 : return error;
245 : }
246 545448 : if ( st_ivas->hRenderConfig != NULL && st_ivas->hOutSetup.output_config == IVAS_AUDIO_CONFIG_BINAURAL_ROOM_REVERB )
247 : {
248 : /* add reverb to rendered signals */
249 70188 : v_add( reverb_signal[0], output_f_local[0], output_f_local[0], output_frame );
250 70188 : v_add( reverb_signal[1], output_f_local[1], output_f_local[1], output_frame );
251 : }
252 :
253 :
254 2282334 : for ( ch = 0; ch < nchan_ism_internal; ch++ )
255 : {
256 1736886 : tc_local[ch] += output_frame;
257 : }
258 :
259 1636344 : for ( ch = 0; ch < st_ivas->hDecoderConfig->nchan_out; ch++ )
260 : {
261 1090896 : output_f_local[ch] += output_frame;
262 : }
263 :
264 : /* update combined orientation access index */
265 545448 : ivas_combined_orientation_update_index( st_ivas->hCombinedOrientationData, output_frame );
266 : }
267 :
268 319600 : st_ivas->hTcBuffer->subframes_rendered = last_sf;
269 :
270 319600 : return IVAS_ERR_OK;
271 : }
272 :
273 :
274 : /*---------------------------------------------------------------------*
275 : * ivas_td_binaural_renderer_sf_splitBinaural()
276 : *
277 : * Render to multiple binaural pairs based on relative head positions for split rendering.
278 : *---------------------------------------------------------------------*/
279 :
280 0 : ivas_error ivas_td_binaural_renderer_sf_splitBinaural(
281 : Decoder_Struct *st_ivas, /* i/o: IVAS decoder structure */
282 : float *output[], /* i/o: SCE channels / Binaural synthesis */
283 : const int16_t nSamplesRendered /* i : number of samples to render */
284 : )
285 : {
286 : int16_t i;
287 : int16_t pos_idx;
288 : IVAS_QUATERNION originalHeadRot[MAX_PARAM_SPATIAL_SUBFRAMES];
289 : MULTI_BIN_REND_POSE_DATA *pMultiBinPoseData;
290 : BINAURAL_TD_OBJECT_RENDERER_HANDLE origTdRendHandle;
291 : ivas_error error;
292 : int16_t original_subframes_rendered;
293 : int16_t original_slots_rendered;
294 : float *p_bin_output[BINAURAL_CHANNELS];
295 : float output_local[MAX_OUTPUT_CHANNELS][L_FRAME48k];
296 :
297 0 : push_wmops( "ivas_td_binaural_renderer_sf_splitBinaural" );
298 0 : pMultiBinPoseData = &st_ivas->hSplitBinRend->splitrend.multiBinPoseData;
299 :
300 : /* If not yet allocated, open additional instances of TD renderer */
301 0 : for ( i = 0; i < pMultiBinPoseData->num_poses - 1; ++i )
302 : {
303 0 : if ( st_ivas->hTdRendHandles[i] != NULL )
304 : {
305 0 : continue;
306 : }
307 :
308 0 : if ( ( error = ivas_td_binaural_open_unwrap( &st_ivas->hHrtfTD,
309 0 : st_ivas->hDecoderConfig->output_Fs,
310 0 : st_ivas->nchan_transport,
311 : st_ivas->ivas_format,
312 : st_ivas->transport_config,
313 0 : st_ivas->hRenderConfig->directivity,
314 0 : st_ivas->hRenderConfig->distAtt,
315 : st_ivas->hTransSetup,
316 0 : &st_ivas->hTdRendHandles[i],
317 : &st_ivas->binaural_latency_ns ) ) != IVAS_ERR_OK )
318 : {
319 0 : return error;
320 : }
321 : }
322 :
323 : /* Save current head positions */
324 0 : for ( i = 0; i < st_ivas->hCombinedOrientationData->num_subframes; ++i )
325 : {
326 0 : originalHeadRot[i] = st_ivas->hCombinedOrientationData->Quaternions[i];
327 : }
328 :
329 0 : original_subframes_rendered = st_ivas->hTcBuffer->subframes_rendered;
330 0 : original_slots_rendered = st_ivas->hTcBuffer->slots_rendered;
331 0 : origTdRendHandle = st_ivas->hBinRendererTd;
332 :
333 0 : for ( pos_idx = 0; pos_idx < pMultiBinPoseData->num_poses; pos_idx++ )
334 : {
335 : /* Update head positions */
336 0 : if ( pos_idx != 0 )
337 : {
338 0 : for ( i = 0; i < st_ivas->hCombinedOrientationData->num_subframes; ++i )
339 : {
340 0 : if ( originalHeadRot[i].w == -3.0f )
341 : {
342 0 : st_ivas->hCombinedOrientationData->Quaternions[i].w = -3.0f;
343 0 : st_ivas->hCombinedOrientationData->Quaternions[i].x = originalHeadRot[i].x + pMultiBinPoseData->relative_head_poses[pos_idx][0];
344 0 : st_ivas->hCombinedOrientationData->Quaternions[i].y = originalHeadRot[i].y + pMultiBinPoseData->relative_head_poses[pos_idx][1];
345 0 : st_ivas->hCombinedOrientationData->Quaternions[i].z = originalHeadRot[i].z + pMultiBinPoseData->relative_head_poses[pos_idx][2];
346 : }
347 : else
348 : {
349 0 : st_ivas->hCombinedOrientationData->Quaternions[i].w = -3.0f;
350 :
351 0 : Quat2EulerDegree( originalHeadRot[i], /* TODO tmu : fix bug with ordering*/
352 0 : &st_ivas->hCombinedOrientationData->Quaternions[i].z,
353 0 : &st_ivas->hCombinedOrientationData->Quaternions[i].y,
354 0 : &st_ivas->hCombinedOrientationData->Quaternions[i].x );
355 :
356 0 : st_ivas->hCombinedOrientationData->Quaternions[i].x += pMultiBinPoseData->relative_head_poses[pos_idx][0];
357 0 : st_ivas->hCombinedOrientationData->Quaternions[i].y += pMultiBinPoseData->relative_head_poses[pos_idx][1];
358 0 : st_ivas->hCombinedOrientationData->Quaternions[i].z += pMultiBinPoseData->relative_head_poses[pos_idx][2];
359 : }
360 : }
361 : }
362 :
363 : /* set output channels */
364 0 : for ( i = 0; i < BINAURAL_CHANNELS; i++ )
365 : {
366 0 : p_bin_output[i] = output_local[pos_idx * BINAURAL_CHANNELS + i];
367 : }
368 0 : st_ivas->hTcBuffer->subframes_rendered = original_subframes_rendered;
369 0 : st_ivas->hTcBuffer->slots_rendered = original_slots_rendered;
370 :
371 : /* update combined orientation access index */
372 0 : ivas_combined_orientation_set_to_start_index( st_ivas->hCombinedOrientationData );
373 :
374 : /* Render */
375 0 : if ( pos_idx != 0 )
376 : {
377 0 : st_ivas->hBinRendererTd = st_ivas->hTdRendHandles[pos_idx - 1];
378 : }
379 :
380 0 : if ( ( error = ivas_td_binaural_renderer_sf( st_ivas, p_bin_output, nSamplesRendered ) ) != IVAS_ERR_OK )
381 : {
382 0 : return error;
383 : }
384 :
385 0 : if ( st_ivas->ivas_format == MC_FORMAT )
386 : {
387 : float *p_tc[MAX_TRANSPORT_CHANNELS];
388 0 : for ( i = 0; i < st_ivas->nchan_transport; i++ )
389 : {
390 0 : p_tc[i] = st_ivas->hTcBuffer->tc[i] + st_ivas->hTcBuffer->n_samples_rendered;
391 : }
392 0 : ivas_binaural_add_LFE( st_ivas, nSamplesRendered, p_tc, p_bin_output );
393 : }
394 : }
395 :
396 0 : for ( i = 0; i < pMultiBinPoseData->num_poses * BINAURAL_CHANNELS; i++ )
397 : {
398 0 : mvr2r( output_local[i], output[i], nSamplesRendered );
399 : }
400 :
401 : /* Restore original head rotation */
402 0 : for ( i = 0; i < st_ivas->hCombinedOrientationData->num_subframes; ++i )
403 : {
404 0 : st_ivas->hCombinedOrientationData->Quaternions[i] = originalHeadRot[i];
405 : }
406 :
407 : /* restore original td renderer handle */
408 0 : st_ivas->hBinRendererTd = origTdRendHandle;
409 :
410 0 : pop_wmops();
411 0 : return IVAS_ERR_OK;
412 : }
|
