Line data Source code
1 : /******************************************************************************************************
2 :
3 : (C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB,
4 : Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,
5 : Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange,
6 : Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other
7 : contributors to this repository. All Rights Reserved.
8 :
9 : This software is protected by copyright law and by international treaties.
10 : The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB,
11 : Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,
12 : Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange,
13 : Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other
14 : contributors to this repository retain full ownership rights in their respective contributions in
15 : the software. This notice grants no license of any kind, including but not limited to patent
16 : license, nor is any license granted by implication, estoppel or otherwise.
17 :
18 : Contributors are required to enter into the IVAS codec Public Collaboration agreement before making
19 : contributions.
20 :
21 : This software is provided "AS IS", without any express or implied warranties. The software is in the
22 : development stage. It is intended exclusively for experts who have experience with such software and
23 : solely for the purpose of inspection. All implied warranties of non-infringement, merchantability
24 : and fitness for a particular purpose are hereby disclaimed and excluded.
25 :
26 : Any dispute, controversy or claim arising under or in relation to providing this software shall be
27 : submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in
28 : accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and
29 : the United Nations Convention on Contracts on the International Sales of Goods.
30 :
31 : *******************************************************************************************************/
32 :
33 :
34 : #include <stdint.h>
35 : #include "options.h"
36 : #include <math.h>
37 : #include "ivas_prot.h"
38 : #include "prot.h"
39 : #include "ivas_cnst.h"
40 : #include "isar_rom_post_rend.h"
41 : #include "lib_isar_pre_rend.h"
42 : #include "isar_prot.h"
43 : #ifdef DEBUGGING
44 : #include "debug.h"
45 : #endif
46 : #include "wmc_auto.h"
47 :
48 :
49 : /*-------------------------------------------------------------------------
50 : * Function ISAR_PRE_REND_open()
51 : *
52 : *
53 : *------------------------------------------------------------------------*/
54 :
55 1032 : ivas_error ISAR_PRE_REND_open(
56 : SPLIT_REND_WRAPPER *hSplitBinRend, /* i/o: Split renderer pre-renderer handle */
57 : ISAR_SPLIT_REND_CONFIG_DATA *pSplitRendConfig, /* i/o: Split renderer pre-renderer config */
58 : const int32_t output_Fs, /* i : output sampling rate */
59 : const int16_t cldfb_in_flag, /* i : Flag to indicate CLDFB or time doamin input */
60 : const int16_t pcm_out_flag, /* i : Flag to indicate PCM output */
61 : const IVAS_RENDER_FRAMESIZE ivas_frame_size, /* i : IVAS frame size */
62 : const int16_t mixed_td_cldfb_flag /* i : Flag to indicate combined TD and CLDFB input */
63 : )
64 : {
65 : ivas_error error, ch, num_ch;
66 1032 : uint8_t isCldfbNeeded = 0;
67 1032 : int16_t cldfb_in_flag_local = cldfb_in_flag;
68 :
69 1032 : if ( ( error = isar_split_rend_choose_default_codec( &( pSplitRendConfig->codec ), &pSplitRendConfig->isar_frame_size_ms, &pSplitRendConfig->codec_frame_size_ms, cldfb_in_flag_local, pcm_out_flag, (int16_t) ivas_frame_size ) ) != IVAS_ERR_OK )
70 : {
71 0 : return error;
72 : }
73 :
74 1032 : if ( mixed_td_cldfb_flag )
75 : {
76 274 : cldfb_in_flag_local = 0;
77 : }
78 :
79 1032 : if ( ( error = isar_split_rend_validate_config( pSplitRendConfig, pcm_out_flag ) ) != IVAS_ERR_OK )
80 : {
81 0 : return error;
82 : }
83 :
84 1032 : if ( cldfb_in_flag_local == 0 )
85 : {
86 500 : isCldfbNeeded = 1;
87 : }
88 532 : else if ( pSplitRendConfig->codec == ISAR_SPLIT_REND_CODEC_LC3PLUS && cldfb_in_flag_local )
89 : {
90 130 : isCldfbNeeded = 1;
91 : }
92 402 : else if ( pcm_out_flag && cldfb_in_flag_local )
93 : {
94 4 : isCldfbNeeded = 1;
95 : }
96 :
97 1032 : if ( isCldfbNeeded && hSplitBinRend->hCldfbHandles == NULL )
98 : {
99 634 : if ( ( hSplitBinRend->hCldfbHandles = (CLDFB_HANDLES_WRAPPER_HANDLE) malloc( sizeof( CLDFB_HANDLES_WRAPPER ) ) ) == NULL )
100 : {
101 0 : return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for CLDFB handles\n" ) );
102 : }
103 634 : num_ch = MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS;
104 10778 : for ( ch = 0; ch < num_ch; ch++ )
105 : {
106 10144 : hSplitBinRend->hCldfbHandles->cldfbAna[ch] = NULL;
107 : }
108 :
109 1902 : for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ )
110 : {
111 1268 : hSplitBinRend->hCldfbHandles->cldfbSyn[ch] = NULL;
112 : }
113 :
114 634 : num_ch = hSplitBinRend->multiBinPoseData.num_poses * BINAURAL_CHANNELS;
115 :
116 5766 : for ( ch = 0; ch < num_ch; ch++ )
117 : {
118 5132 : if ( ( error = openCldfb( &( hSplitBinRend->hCldfbHandles->cldfbAna[ch] ), CLDFB_ANALYSIS, output_Fs, CLDFB_PROTOTYPE_5_00MS ) ) != IVAS_ERR_OK )
119 : {
120 0 : return error;
121 : }
122 : }
123 :
124 1902 : for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ )
125 : {
126 1268 : if ( ( error = openCldfb( &( hSplitBinRend->hCldfbHandles->cldfbSyn[ch] ), CLDFB_SYNTHESIS, output_Fs, CLDFB_PROTOTYPE_5_00MS ) ) != IVAS_ERR_OK )
127 : {
128 0 : return error;
129 : }
130 : }
131 : }
132 :
133 1032 : if ( pSplitRendConfig->poseCorrectionMode == ISAR_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB )
134 : {
135 : #ifdef SPLIT_REND_WITH_HEAD_ROT_DEBUG
136 : if ( ( error = isar_splitBinPreRendOpen( &hSplitBinRend->hBinHrSplitPreRend, &hSplitBinRend->multiBinPoseData, OutSampleRate ) ) != IVAS_ERR_OK )
137 : #else
138 694 : if ( ( error = isar_splitBinPreRendOpen( &hSplitBinRend->hBinHrSplitPreRend, &hSplitBinRend->multiBinPoseData ) ) != IVAS_ERR_OK )
139 : #endif
140 : {
141 0 : return error;
142 : }
143 : }
144 :
145 1032 : if ( pcm_out_flag == 0 )
146 : {
147 1022 : if ( pSplitRendConfig->codec == ISAR_SPLIT_REND_CODEC_LC3PLUS )
148 : {
149 358 : if ( ( error = split_renderer_open_lc3plus( hSplitBinRend, pSplitRendConfig, output_Fs, ivas_frame_size ) ) != IVAS_ERR_OK )
150 : {
151 0 : return error;
152 : }
153 : }
154 : else
155 : {
156 : int16_t iNumBlocksPerFrame;
157 664 : iNumBlocksPerFrame = ( CLDFB_NO_COL_MAX * pSplitRendConfig->codec_frame_size_ms ) / 20;
158 :
159 664 : if ( ( error = isar_splitBinLCLDEncOpen( &hSplitBinRend->hSplitBinLCLDEnc, output_Fs, BINAURAL_CHANNELS, isar_get_lcld_bitrate( pSplitRendConfig->splitRendBitRate, hSplitBinRend->multiBinPoseData.poseCorrectionMode ), iNumBlocksPerFrame, 1 ) ) != IVAS_ERR_OK )
160 : {
161 0 : return error;
162 : }
163 : }
164 : }
165 :
166 1032 : return IVAS_ERR_OK;
167 : }
168 :
169 :
170 : /*-------------------------------------------------------------------------
171 : * Function ISAR_PRE_REND_close()
172 : *
173 : *
174 : *------------------------------------------------------------------------*/
175 :
176 1032 : void ISAR_PRE_REND_close(
177 : SPLIT_REND_WRAPPER *hSplitBinRend, /* i/o: Split renderer pre-renderer handle */
178 : IVAS_REND_AudioBuffer *pSplitRendEncBuffer /* i/o: Split renderer data buffer */
179 : )
180 : {
181 : int16_t i;
182 :
183 1032 : if ( hSplitBinRend->hBinHrSplitPreRend != NULL )
184 : {
185 694 : isar_splitBinPreRendClose( &hSplitBinRend->hBinHrSplitPreRend );
186 : }
187 :
188 1032 : if ( hSplitBinRend->hSplitBinLCLDEnc != NULL )
189 : {
190 664 : isar_splitBinLCLDEncClose( &hSplitBinRend->hSplitBinLCLDEnc );
191 : }
192 :
193 1032 : if ( hSplitBinRend->hCldfbHandles != NULL )
194 : {
195 : int16_t num_ch, ch;
196 634 : num_ch = MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS;
197 10778 : for ( ch = 0; ch < num_ch; ch++ )
198 : {
199 10144 : if ( hSplitBinRend->hCldfbHandles->cldfbAna[ch] != NULL )
200 : {
201 5132 : deleteCldfb( &hSplitBinRend->hCldfbHandles->cldfbAna[ch] );
202 5132 : hSplitBinRend->hCldfbHandles->cldfbAna[ch] = NULL;
203 : }
204 : }
205 :
206 1902 : for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ )
207 : {
208 1268 : if ( hSplitBinRend->hCldfbHandles->cldfbSyn[ch] != NULL )
209 : {
210 1268 : deleteCldfb( &hSplitBinRend->hCldfbHandles->cldfbSyn[ch] );
211 1268 : hSplitBinRend->hCldfbHandles->cldfbSyn[ch] = NULL;
212 : }
213 : }
214 :
215 634 : free( hSplitBinRend->hCldfbHandles );
216 634 : hSplitBinRend->hCldfbHandles = NULL;
217 : }
218 :
219 1032 : if ( hSplitBinRend->hLc3plusEnc != NULL )
220 : {
221 358 : ISAR_LC3PLUS_ENC_Close( &hSplitBinRend->hLc3plusEnc );
222 : }
223 :
224 17544 : for ( i = 0; i < MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS; ++i )
225 : {
226 16512 : if ( hSplitBinRend->lc3plusDelayBuffers[i] != NULL )
227 : {
228 3052 : free( hSplitBinRend->lc3plusDelayBuffers[i] );
229 3052 : hSplitBinRend->lc3plusDelayBuffers[i] = NULL;
230 : }
231 : }
232 :
233 1032 : if ( pSplitRendEncBuffer != NULL )
234 : {
235 :
236 0 : if ( pSplitRendEncBuffer->data != NULL )
237 : {
238 0 : free( pSplitRendEncBuffer->data );
239 0 : pSplitRendEncBuffer->data = NULL;
240 : }
241 :
242 0 : pSplitRendEncBuffer->config.numChannels = 0;
243 0 : pSplitRendEncBuffer->config.numSamplesPerChannel = 0;
244 : }
245 :
246 1032 : return;
247 : }
248 :
249 :
250 : /*-------------------------------------------------------------------------*
251 : * ISAR_PRE_REND_GetMultiBinPoseData()
252 : *
253 : *
254 : *-------------------------------------------------------------------------*/
255 :
256 415998 : void ISAR_PRE_REND_GetMultiBinPoseData(
257 : const ISAR_SPLIT_REND_CONFIG_DATA *pSplit_rend_config, /* i : Split renderer pre-renderer config */
258 : MULTI_BIN_REND_POSE_DATA *pMultiBinPoseData, /* i/o: pose correction data handle */
259 : const ISAR_SPLIT_REND_ROT_AXIS rot_axis /* i : Rotation axis */
260 : )
261 : {
262 415998 : isar_renderSplitGetMultiBinPoseData( pSplit_rend_config, pMultiBinPoseData, rot_axis );
263 :
264 415998 : return;
265 : }
266 :
267 :
268 : /*-------------------------------------------------------------------------
269 : * Function ISAR_PRE_REND_MultiBinToSplitBinaural()
270 : *
271 : *
272 : *------------------------------------------------------------------------*/
273 :
274 220022 : ivas_error ISAR_PRE_REND_MultiBinToSplitBinaural(
275 : SPLIT_REND_WRAPPER *hSplitBin, /* i/o: Split renderer pre-renerer handle */
276 : const IVAS_QUATERNION headPosition, /* i : head rotation QUATERNION */
277 : const int32_t SplitRendBitRate, /* i : Split renderer bitrate */
278 : ISAR_SPLIT_REND_CODEC splitCodec, /* i/o: Split renderer codec */
279 : const int16_t isar_frame_size_ms, /* i : ISAR framesize */
280 : int16_t codec_frame_size_ms, /* i/o: ISAR transport codec framesize */
281 : ISAR_SPLIT_REND_BITS_HANDLE pBits, /* i/o: ISAR bits struct handle */
282 : #ifdef FIX_1119_SPLIT_RENDERING_VOIP
283 : float *Cldfb_In_BinReal[][CLDFB_NO_COL_MAX], /* i/o: CLDFB real buffer */
284 : float *Cldfb_In_BinImag[][CLDFB_NO_COL_MAX], /* i/o: CLDFB imag buffer */
285 : #else
286 : float Cldfb_In_BinReal[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], /* i/o: CLDFB real buffer */
287 : float Cldfb_In_BinImag[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], /* i/o: CLDFB imag buffer */
288 : #endif
289 : const int16_t max_bands, /* i : CLDFB bands */
290 : float *output[], /* i/o: PCM in/out buffer */
291 : const int16_t low_res_pre_rend_rot, /* i : low time resolution pre-renderer flag */
292 : const int16_t cldfb_in_flag, /* i : Flag to indicate CLDFB or time domain input */
293 : const int16_t pcm_out_flag, /* i : Flag to indicate PCM output */
294 : const int16_t ro_md_flag /* i : Flag to indicate real only metadata for yaw */
295 : )
296 : {
297 : ivas_error error;
298 : int32_t bit_len, target_md_bits, available_bits;
299 :
300 220022 : error = IVAS_ERR_OK;
301 220022 : push_wmops( "isar_pre_rend_MultiBinToSplitBinaural" );
302 :
303 220022 : if ( hSplitBin->multiBinPoseData.poseCorrectionMode == ISAR_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB )
304 : {
305 137941 : set_fix_rotation_mat( hSplitBin->hBinHrSplitPreRend->fix_pos_rot_mat, &hSplitBin->multiBinPoseData );
306 137941 : set_pose_types( hSplitBin->hBinHrSplitPreRend->pose_type, &hSplitBin->multiBinPoseData );
307 : }
308 :
309 220022 : if ( cldfb_in_flag == 0 )
310 : {
311 : /*TD input*/
312 : /*if CLDFB handles have been allocated then assume valid multi binaural input in out[][] buffer and perform CLDFB analysis*/
313 59275 : error = isar_renderMultiTDBinToSplitBinaural( hSplitBin, headPosition, SplitRendBitRate, isar_frame_size_ms, codec_frame_size_ms, pBits, max_bands, output, low_res_pre_rend_rot, pcm_out_flag, ro_md_flag );
314 :
315 59275 : pop_wmops();
316 59275 : return error;
317 : }
318 :
319 160747 : if ( splitCodec == ISAR_SPLIT_REND_CODEC_LC3PLUS && hSplitBin->multiBinPoseData.poseCorrectionMode == ISAR_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB )
320 : {
321 : /* Time-align pose correction to delay of LC3plus */
322 33658 : lc3plusTimeAlignCldfbPoseCorr( hSplitBin, Cldfb_In_BinReal, Cldfb_In_BinImag );
323 : }
324 :
325 160747 : if ( hSplitBin->multiBinPoseData.poseCorrectionMode == ISAR_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB )
326 : {
327 104828 : target_md_bits = isar_get_split_rend_md_target_brate( SplitRendBitRate, pcm_out_flag ) * L_FRAME48k / 48000;
328 :
329 :
330 104828 : isar_rend_CldfbSplitPreRendProcess( hSplitBin->hBinHrSplitPreRend, headPosition, &hSplitBin->multiBinPoseData, Cldfb_In_BinReal, Cldfb_In_BinImag, pBits, target_md_bits, low_res_pre_rend_rot, ro_md_flag );
331 : }
332 :
333 160747 : if ( pcm_out_flag == 0 )
334 : {
335 159638 : pBits->codec = splitCodec;
336 159638 : pBits->pose_correction = hSplitBin->multiBinPoseData.poseCorrectionMode;
337 :
338 159638 : if ( splitCodec == ISAR_SPLIT_REND_CODEC_LCLD )
339 : {
340 116925 : available_bits = ( SplitRendBitRate * hSplitBin->hSplitBinLCLDEnc->iNumBlocks * hSplitBin->hSplitBinLCLDEnc->iNumIterations ) / ( 16 * FRAMES_PER_SEC );
341 116925 : available_bits -= pBits->bits_written;
342 116925 : pBits->codec_frame_size_ms = codec_frame_size_ms;
343 116925 : isar_splitBinLCLDEncProcess( hSplitBin->hSplitBinLCLDEnc, Cldfb_In_BinReal, Cldfb_In_BinImag, available_bits, pBits );
344 : }
345 : else
346 : {
347 : int16_t ch, slot_idx, num_slots, ivas_fs;
348 42713 : ivas_fs = (int16_t) hSplitBin->hLc3plusEnc->config.isar_frame_duration_us / 1000;
349 42713 : num_slots = (int16_t) ( CLDFB_NO_COL_MAX * ivas_fs ) / 20;
350 : /* CLDFB synthesis of main pose */
351 128139 : for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ )
352 : {
353 : float *Cldfb_In_BinReal_p[CLDFB_NO_COL_MAX];
354 : float *Cldfb_In_BinImag_p[CLDFB_NO_COL_MAX];
355 :
356 1339546 : for ( slot_idx = 0; slot_idx < num_slots; slot_idx++ )
357 : {
358 1254120 : Cldfb_In_BinReal_p[slot_idx] = Cldfb_In_BinReal[ch][slot_idx];
359 1254120 : Cldfb_In_BinImag_p[slot_idx] = Cldfb_In_BinImag[ch][slot_idx];
360 : }
361 :
362 85426 : cldfbSynthesis( Cldfb_In_BinReal_p, Cldfb_In_BinImag_p, output[ch], hSplitBin->hCldfbHandles->cldfbSyn[0]->no_channels * num_slots, hSplitBin->hCldfbHandles->cldfbSyn[ch] );
363 : }
364 :
365 42713 : if ( pBits->pose_correction == ISAR_SPLIT_REND_POSE_CORRECTION_MODE_NONE )
366 : {
367 9055 : available_bits = isar_get_lc3plus_bitrate( SplitRendBitRate, hSplitBin->multiBinPoseData.poseCorrectionMode, hSplitBin->hLc3plusEnc->config.channels, hSplitBin->hLc3plusEnc->config.lc3plus_frame_duration_us ) / FRAMES_PER_SEC;
368 : }
369 : else
370 : {
371 33658 : available_bits = ( SplitRendBitRate / FRAMES_PER_SEC ) - pBits->bits_written;
372 : }
373 :
374 42713 : if ( ( error = splitRendLc3plusEncodeAndWrite( hSplitBin, pBits, available_bits, output ) ) != IVAS_ERR_OK )
375 : {
376 0 : return error;
377 : }
378 : }
379 : }
380 : else
381 : {
382 : int16_t ch, slot_idx, num_slots;
383 1109 : num_slots = (int16_t) ( isar_frame_size_ms * 1000000 / CLDFB_SLOT_NS );
384 :
385 : /* CLDFB synthesis of main pose */
386 3327 : for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ )
387 : {
388 : float *Cldfb_In_BinReal_p[CLDFB_NO_COL_MAX];
389 : float *Cldfb_In_BinImag_p[CLDFB_NO_COL_MAX];
390 :
391 37706 : for ( slot_idx = 0; slot_idx < num_slots; slot_idx++ )
392 : {
393 35488 : Cldfb_In_BinReal_p[slot_idx] = Cldfb_In_BinReal[ch][slot_idx];
394 35488 : Cldfb_In_BinImag_p[slot_idx] = Cldfb_In_BinImag[ch][slot_idx];
395 : }
396 :
397 2218 : cldfbSynthesis( Cldfb_In_BinReal_p, Cldfb_In_BinImag_p, output[ch], hSplitBin->hCldfbHandles->cldfbSyn[0]->no_channels * num_slots, hSplitBin->hCldfbHandles->cldfbSyn[ch] );
398 : }
399 :
400 1109 : pBits->pose_correction = hSplitBin->multiBinPoseData.poseCorrectionMode;
401 1109 : pBits->codec = ISAR_SPLIT_REND_CODEC_NONE;
402 : }
403 :
404 : /*zero pad*/
405 160747 : if ( pcm_out_flag )
406 : {
407 1109 : bit_len = SplitRendBitRate / FRAMES_PER_SEC;
408 : }
409 : else
410 : {
411 159638 : if ( splitCodec == ISAR_SPLIT_REND_CODEC_LCLD )
412 : {
413 116925 : bit_len = ( SplitRendBitRate * hSplitBin->hSplitBinLCLDEnc->iNumBlocks * hSplitBin->hSplitBinLCLDEnc->iNumIterations ) / ( 16 * FRAMES_PER_SEC );
414 : }
415 : else
416 : {
417 42713 : bit_len = hSplitBin->hLc3plusEnc->config.isar_frame_duration_us / 1000;
418 42713 : bit_len = SplitRendBitRate * bit_len / 1000;
419 : }
420 : }
421 :
422 37664956 : while ( pBits->bits_written < bit_len )
423 : {
424 37504209 : ISAR_SPLIT_REND_BITStream_write_int32( pBits, 0L, 1 );
425 : }
426 :
427 160747 : pop_wmops();
428 :
429 160747 : return error;
430 : }
|
