Line data Source code
1 : /******************************************************************************************************
2 :
3 : (C) 2022-2026 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB,
4 : Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,
5 : Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange,
6 : Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other
7 : contributors to this repository. All Rights Reserved.
8 :
9 : This software is protected by copyright law and by international treaties.
10 : The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB,
11 : Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,
12 : Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange,
13 : Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other
14 : contributors to this repository retain full ownership rights in their respective contributions in
15 : the software. This notice grants no license of any kind, including but not limited to patent
16 : license, nor is any license granted by implication, estoppel or otherwise.
17 :
18 : Contributors are required to enter into the IVAS codec Public Collaboration agreement before making
19 : contributions.
20 :
21 : This software is provided "AS IS", without any express or implied warranties. The software is in the
22 : development stage. It is intended exclusively for experts who have experience with such software and
23 : solely for the purpose of inspection. All implied warranties of non-infringement, merchantability
24 : and fitness for a particular purpose are hereby disclaimed and excluded.
25 :
26 : Any dispute, controversy or claim arising under or in relation to providing this software shall be
27 : submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in
28 : accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and
29 : the United Nations Convention on Contracts on the International Sales of Goods.
30 :
31 : *******************************************************************************************************/
32 :
33 :
34 : #include <stdint.h>
35 : #include "options.h"
36 : #include <math.h>
37 : #include "ivas_prot.h"
38 : #include "prot.h"
39 : #include "ivas_cnst.h"
40 : #include "isar_rom_post_rend.h"
41 : #include "lib_isar_pre_rend.h"
42 : #include "isar_prot.h"
43 : #ifdef DEBUGGING
44 : #include "debug.h"
45 : #endif
46 : #include "wmc_auto.h"
47 :
48 :
49 : /*-------------------------------------------------------------------------
50 : * Function ISAR_PRE_REND_open()
51 : *
52 : *
53 : *------------------------------------------------------------------------*/
54 :
55 3592 : ivas_error ISAR_PRE_REND_open(
56 : SPLIT_REND_WRAPPER *hSplitBinRend, /* i/o: Split renderer pre-renderer handle */
57 : ISAR_SPLIT_REND_CONFIG_DATA *pSplitRendConfig, /* i/o: Split renderer pre-renderer config */
58 : const int32_t output_Fs, /* i : output sampling rate */
59 : const int16_t cldfb_in_flag, /* i : Flag to indicate CLDFB or time doamin input */
60 : const int16_t pcm_out_flag, /* i : Flag to indicate PCM output */
61 : const IVAS_RENDER_NUM_SUBFR render_num_subframes, /* i : rendering number of subframes */
62 : const int16_t mixed_td_cldfb_flag /* i : Flag to indicate combined TD and CLDFB input */
63 : )
64 : {
65 : ivas_error error, ch, num_ch;
66 3592 : uint8_t isCldfbNeeded = 0;
67 3592 : int16_t cldfb_in_flag_local = cldfb_in_flag;
68 :
69 3592 : if ( ( error = isar_split_rend_choose_default_codec( &( pSplitRendConfig->codec ), &pSplitRendConfig->isar_frame_size_ms, &pSplitRendConfig->codec_frame_size_ms, cldfb_in_flag_local, pcm_out_flag, (int16_t) render_num_subframes ) ) != IVAS_ERR_OK )
70 : {
71 0 : return error;
72 : }
73 :
74 3592 : if ( mixed_td_cldfb_flag )
75 : {
76 826 : cldfb_in_flag_local = 0;
77 826 : }
78 :
79 3592 : if ( ( error = isar_split_rend_validate_config( pSplitRendConfig, pcm_out_flag ) ) != IVAS_ERR_OK )
80 : {
81 0 : return error;
82 : }
83 :
84 3592 : if ( cldfb_in_flag_local == 0 )
85 : {
86 1912 : isCldfbNeeded = 1;
87 1912 : }
88 1680 : else if ( pSplitRendConfig->codec == ISAR_SPLIT_REND_CODEC_LC3PLUS && cldfb_in_flag_local )
89 : {
90 406 : isCldfbNeeded = 1;
91 406 : }
92 1274 : else if ( pcm_out_flag && cldfb_in_flag_local )
93 : {
94 18 : isCldfbNeeded = 1;
95 18 : }
96 :
97 3592 : if ( isCldfbNeeded && hSplitBinRend->hCldfbHandles == NULL )
98 : {
99 2336 : if ( ( hSplitBinRend->hCldfbHandles = (CLDFB_HANDLES_WRAPPER_HANDLE) malloc( sizeof( CLDFB_HANDLES_WRAPPER ) ) ) == NULL )
100 : {
101 0 : return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for CLDFB handles\n" ) );
102 : }
103 2336 : num_ch = MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS;
104 39712 : for ( ch = 0; ch < num_ch; ch++ )
105 : {
106 37376 : hSplitBinRend->hCldfbHandles->cldfbAna[ch] = NULL;
107 37376 : }
108 :
109 7008 : for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ )
110 : {
111 4672 : hSplitBinRend->hCldfbHandles->cldfbSyn[ch] = NULL;
112 4672 : }
113 :
114 2336 : num_ch = hSplitBinRend->multiBinPoseData.num_poses * BINAURAL_CHANNELS;
115 :
116 21248 : for ( ch = 0; ch < num_ch; ch++ )
117 : {
118 18912 : if ( ( error = openCldfb( &( hSplitBinRend->hCldfbHandles->cldfbAna[ch] ), CLDFB_ANALYSIS, output_Fs, CLDFB_PROTOTYPE_5_00MS ) ) != IVAS_ERR_OK )
119 : {
120 0 : return error;
121 : }
122 18912 : }
123 :
124 7008 : for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ )
125 : {
126 4672 : if ( ( error = openCldfb( &( hSplitBinRend->hCldfbHandles->cldfbSyn[ch] ), CLDFB_SYNTHESIS, output_Fs, CLDFB_PROTOTYPE_5_00MS ) ) != IVAS_ERR_OK )
127 : {
128 0 : return error;
129 : }
130 4672 : }
131 2336 : }
132 :
133 3592 : if ( pSplitRendConfig->poseCorrectionMode == ISAR_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB )
134 : {
135 : #ifdef SPLIT_REND_WITH_HEAD_ROT_DEBUG
136 : if ( ( error = isar_splitBinPreRendOpen( &hSplitBinRend->hBinHrSplitPreRend, &hSplitBinRend->multiBinPoseData, OutSampleRate ) ) != IVAS_ERR_OK )
137 : #else
138 2434 : if ( ( error = isar_splitBinPreRendOpen( &hSplitBinRend->hBinHrSplitPreRend, &hSplitBinRend->multiBinPoseData ) ) != IVAS_ERR_OK )
139 : #endif
140 : {
141 0 : return error;
142 : }
143 2434 : }
144 :
145 3592 : if ( pcm_out_flag == 0 )
146 : {
147 3540 : if ( pSplitRendConfig->codec == ISAR_SPLIT_REND_CODEC_LC3PLUS )
148 : {
149 1442 : if ( ( error = split_renderer_open_lc3plus( hSplitBinRend, pSplitRendConfig, output_Fs, render_num_subframes ) ) != IVAS_ERR_OK )
150 : {
151 0 : return error;
152 : }
153 1442 : }
154 : else
155 : {
156 : int16_t iNumBlocksPerFrame;
157 2098 : iNumBlocksPerFrame = ( CLDFB_NO_COL_MAX * pSplitRendConfig->codec_frame_size_ms ) / 20;
158 :
159 2098 : if ( ( error = isar_splitBinLCLDEncOpen( &hSplitBinRend->hSplitBinLCLDEnc, output_Fs, BINAURAL_CHANNELS, isar_get_lcld_bitrate( pSplitRendConfig->splitRendBitRate, hSplitBinRend->multiBinPoseData.poseCorrectionMode ), iNumBlocksPerFrame, 1 ) ) != IVAS_ERR_OK )
160 : {
161 0 : return error;
162 : }
163 : }
164 3540 : }
165 :
166 3592 : return IVAS_ERR_OK;
167 3592 : }
168 :
169 :
170 : /*-------------------------------------------------------------------------
171 : * Function ISAR_PRE_REND_close()
172 : *
173 : *
174 : *------------------------------------------------------------------------*/
175 :
176 3592 : void ISAR_PRE_REND_close(
177 : SPLIT_REND_WRAPPER *hSplitBinRend, /* i/o: Split renderer pre-renderer handle */
178 : IVAS_REND_AudioBuffer *pSplitRendEncBuffer /* i/o: Split renderer data buffer */
179 : )
180 : {
181 : int16_t i;
182 :
183 3592 : if ( hSplitBinRend->hBinHrSplitPreRend != NULL )
184 : {
185 2434 : isar_splitBinPreRendClose( &hSplitBinRend->hBinHrSplitPreRend );
186 2434 : }
187 :
188 3592 : if ( hSplitBinRend->hSplitBinLCLDEnc != NULL )
189 : {
190 2098 : isar_splitBinLCLDEncClose( &hSplitBinRend->hSplitBinLCLDEnc );
191 2098 : }
192 :
193 3592 : if ( hSplitBinRend->hCldfbHandles != NULL )
194 : {
195 : int16_t num_ch, ch;
196 2336 : num_ch = MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS;
197 39712 : for ( ch = 0; ch < num_ch; ch++ )
198 : {
199 37376 : if ( hSplitBinRend->hCldfbHandles->cldfbAna[ch] != NULL )
200 : {
201 18912 : deleteCldfb( &hSplitBinRend->hCldfbHandles->cldfbAna[ch] );
202 18912 : hSplitBinRend->hCldfbHandles->cldfbAna[ch] = NULL;
203 18912 : }
204 37376 : }
205 :
206 7008 : for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ )
207 : {
208 4672 : if ( hSplitBinRend->hCldfbHandles->cldfbSyn[ch] != NULL )
209 : {
210 4672 : deleteCldfb( &hSplitBinRend->hCldfbHandles->cldfbSyn[ch] );
211 4672 : hSplitBinRend->hCldfbHandles->cldfbSyn[ch] = NULL;
212 4672 : }
213 4672 : }
214 :
215 2336 : free( hSplitBinRend->hCldfbHandles );
216 2336 : hSplitBinRend->hCldfbHandles = NULL;
217 2336 : }
218 :
219 3592 : if ( hSplitBinRend->hLc3plusEnc != NULL )
220 : {
221 1442 : ISAR_LC3PLUS_ENC_Close( &hSplitBinRend->hLc3plusEnc );
222 1442 : }
223 :
224 61064 : for ( i = 0; i < MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS; ++i )
225 : {
226 57472 : if ( hSplitBinRend->lc3plusDelayBuffers[i] != NULL )
227 : {
228 12084 : free( hSplitBinRend->lc3plusDelayBuffers[i] );
229 12084 : hSplitBinRend->lc3plusDelayBuffers[i] = NULL;
230 12084 : }
231 57472 : }
232 :
233 3592 : if ( pSplitRendEncBuffer != NULL )
234 : {
235 :
236 472 : if ( pSplitRendEncBuffer->data != NULL )
237 : {
238 472 : free( pSplitRendEncBuffer->data );
239 472 : pSplitRendEncBuffer->data = NULL;
240 472 : }
241 :
242 472 : pSplitRendEncBuffer->config.numChannels = 0;
243 472 : pSplitRendEncBuffer->config.numSamplesPerChannel = 0;
244 472 : }
245 :
246 3592 : return;
247 : }
248 :
249 :
250 : /*-------------------------------------------------------------------------*
251 : * ISAR_PRE_REND_GetMultiBinPoseData()
252 : *
253 : *
254 : *-------------------------------------------------------------------------*/
255 :
256 1059322 : void ISAR_PRE_REND_GetMultiBinPoseData(
257 : const ISAR_SPLIT_REND_CONFIG_DATA *pSplit_rend_config, /* i : Split renderer pre-renderer config */
258 : MULTI_BIN_REND_POSE_DATA *pMultiBinPoseData, /* i/o: pose correction data handle */
259 : const ISAR_SPLIT_REND_ROT_AXIS rot_axis /* i : Rotation axis */
260 : )
261 : {
262 1059322 : isar_renderSplitGetMultiBinPoseData( pSplit_rend_config, pMultiBinPoseData, rot_axis );
263 :
264 1059322 : return;
265 : }
266 :
267 :
268 : /*-------------------------------------------------------------------------
269 : * Function ISAR_PRE_REND_MultiBinToSplitBinaural()
270 : *
271 : *
272 : *------------------------------------------------------------------------*/
273 :
274 590697 : ivas_error ISAR_PRE_REND_MultiBinToSplitBinaural(
275 : SPLIT_REND_WRAPPER *hSplitBin, /* i/o: Split renderer pre-renderer handle */
276 : const IVAS_QUATERNION headPosition, /* i : head rotation QUATERNION */
277 : const int32_t SplitRendBitRate, /* i : Split renderer bitrate */
278 : ISAR_SPLIT_REND_CODEC splitCodec, /* i/o: Split renderer codec */
279 : const int16_t isar_frame_size_ms, /* i : ISAR framesize */
280 : int16_t codec_frame_size_ms, /* i/o: ISAR transport codec framesize */
281 : ISAR_SPLIT_REND_BITS_HANDLE pBits, /* i/o: ISAR bits handle */
282 : float *Cldfb_In_BinReal[][CLDFB_NO_COL_MAX], /* i/o: CLDFB real buffer */
283 : float *Cldfb_In_BinImag[][CLDFB_NO_COL_MAX], /* i/o: CLDFB imag buffer */
284 : const int16_t max_bands, /* i : CLDFB bands */
285 : float *output[], /* i/o: PCM in/out buffer */
286 : const int16_t low_res_pre_rend_rot, /* i : low time resolution pre-renderer flag */
287 : const int16_t cldfb_in_flag, /* i : Flag to indicate CLDFB or time domain input */
288 : const int16_t pcm_out_flag, /* i : Flag to indicate PCM output */
289 : const int16_t ro_md_flag /* i : Flag to indicate real only metadata for yaw */
290 : )
291 : {
292 : ivas_error error;
293 : int32_t bit_len, target_md_bits, available_bits;
294 :
295 590697 : error = IVAS_ERR_OK;
296 590697 : push_wmops( "isar_pre_rend_MultiBinToSplitBinaural" );
297 :
298 590697 : if ( hSplitBin->multiBinPoseData.poseCorrectionMode == ISAR_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB )
299 : {
300 379379 : set_fix_rotation_mat( hSplitBin->hBinHrSplitPreRend->fix_pos_rot_mat, &hSplitBin->multiBinPoseData );
301 379379 : set_pose_types( hSplitBin->hBinHrSplitPreRend->pose_type, &hSplitBin->multiBinPoseData );
302 379379 : }
303 :
304 590697 : if ( cldfb_in_flag == 0 )
305 : {
306 : /*TD input*/
307 : /*if CLDFB handles have been allocated then assume valid multi binaural input in out[][] buffer and perform CLDFB analysis*/
308 198994 : error = isar_renderMultiTDBinToSplitBinaural( hSplitBin, headPosition, SplitRendBitRate, isar_frame_size_ms, codec_frame_size_ms, pBits, max_bands, output, low_res_pre_rend_rot, pcm_out_flag, ro_md_flag );
309 :
310 198994 : pop_wmops();
311 198994 : return error;
312 : }
313 :
314 391703 : if ( splitCodec == ISAR_SPLIT_REND_CODEC_LC3PLUS && hSplitBin->multiBinPoseData.poseCorrectionMode == ISAR_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB )
315 : {
316 : /* Time-align pose correction to delay of LC3plus */
317 84550 : lc3plusTimeAlignCldfbPoseCorr( hSplitBin, Cldfb_In_BinReal, Cldfb_In_BinImag );
318 84550 : }
319 :
320 391703 : if ( hSplitBin->multiBinPoseData.poseCorrectionMode == ISAR_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB )
321 : {
322 258741 : target_md_bits = isar_get_split_rend_md_target_brate( SplitRendBitRate, pcm_out_flag ) * L_FRAME48k / 48000;
323 :
324 :
325 258741 : isar_rend_CldfbSplitPreRendProcess( hSplitBin->hBinHrSplitPreRend, headPosition, &hSplitBin->multiBinPoseData, Cldfb_In_BinReal, Cldfb_In_BinImag, pBits, target_md_bits, low_res_pre_rend_rot, ro_md_flag );
326 258741 : }
327 :
328 391703 : if ( pcm_out_flag == 0 )
329 : {
330 387513 : pBits->codec = splitCodec;
331 387513 : pBits->pose_correction = hSplitBin->multiBinPoseData.poseCorrectionMode;
332 :
333 387513 : if ( splitCodec == ISAR_SPLIT_REND_CODEC_LCLD )
334 : {
335 280555 : available_bits = ( SplitRendBitRate * hSplitBin->hSplitBinLCLDEnc->iNumBlocks * hSplitBin->hSplitBinLCLDEnc->iNumIterations ) / ( 16 * FRAMES_PER_SEC );
336 280555 : available_bits -= pBits->bits_written;
337 280555 : pBits->codec_frame_size_ms = codec_frame_size_ms;
338 280555 : isar_splitBinLCLDEncProcess( hSplitBin->hSplitBinLCLDEnc, Cldfb_In_BinReal, Cldfb_In_BinImag, available_bits, pBits );
339 280555 : }
340 : else
341 : {
342 : int16_t ch, slot_idx, num_slots, ivas_fs;
343 106958 : ivas_fs = (int16_t) hSplitBin->hLc3plusEnc->config.isar_frame_duration_us / 1000;
344 106958 : num_slots = (int16_t) ( CLDFB_NO_COL_MAX * ivas_fs ) / 20;
345 : /* CLDFB synthesis of main pose */
346 320874 : for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ )
347 : {
348 : float *Cldfb_In_BinReal_p[CLDFB_NO_COL_MAX];
349 : float *Cldfb_In_BinImag_p[CLDFB_NO_COL_MAX];
350 :
351 3384076 : for ( slot_idx = 0; slot_idx < num_slots; slot_idx++ )
352 : {
353 3170160 : Cldfb_In_BinReal_p[slot_idx] = Cldfb_In_BinReal[ch][slot_idx];
354 3170160 : Cldfb_In_BinImag_p[slot_idx] = Cldfb_In_BinImag[ch][slot_idx];
355 3170160 : }
356 :
357 213916 : cldfbSynthesis( Cldfb_In_BinReal_p, Cldfb_In_BinImag_p, output[ch], hSplitBin->hCldfbHandles->cldfbSyn[0]->no_channels * num_slots, hSplitBin->hCldfbHandles->cldfbSyn[ch] );
358 213916 : }
359 :
360 106958 : if ( pBits->pose_correction == ISAR_SPLIT_REND_POSE_CORRECTION_MODE_NONE )
361 : {
362 22408 : available_bits = isar_get_lc3plus_bitrate( SplitRendBitRate, hSplitBin->multiBinPoseData.poseCorrectionMode, hSplitBin->hLc3plusEnc->config.channels, hSplitBin->hLc3plusEnc->config.lc3plus_frame_duration_us ) / FRAMES_PER_SEC;
363 22408 : }
364 : else
365 : {
366 84550 : available_bits = ( SplitRendBitRate / FRAMES_PER_SEC ) - pBits->bits_written;
367 : }
368 :
369 106958 : if ( ( error = splitRendLc3plusEncodeAndWrite( hSplitBin, pBits, available_bits, output ) ) != IVAS_ERR_OK )
370 : {
371 0 : return error;
372 : }
373 : }
374 387513 : }
375 : else
376 : {
377 : int16_t ch, slot_idx, num_slots;
378 4190 : num_slots = (int16_t) ( isar_frame_size_ms * 1000000 / CLDFB_SLOT_NS );
379 :
380 : /* CLDFB synthesis of main pose */
381 12570 : for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ )
382 : {
383 : float *Cldfb_In_BinReal_p[CLDFB_NO_COL_MAX];
384 : float *Cldfb_In_BinImag_p[CLDFB_NO_COL_MAX];
385 :
386 142460 : for ( slot_idx = 0; slot_idx < num_slots; slot_idx++ )
387 : {
388 134080 : Cldfb_In_BinReal_p[slot_idx] = Cldfb_In_BinReal[ch][slot_idx];
389 134080 : Cldfb_In_BinImag_p[slot_idx] = Cldfb_In_BinImag[ch][slot_idx];
390 134080 : }
391 :
392 8380 : cldfbSynthesis( Cldfb_In_BinReal_p, Cldfb_In_BinImag_p, output[ch], hSplitBin->hCldfbHandles->cldfbSyn[0]->no_channels * num_slots, hSplitBin->hCldfbHandles->cldfbSyn[ch] );
393 8380 : }
394 :
395 4190 : pBits->pose_correction = hSplitBin->multiBinPoseData.poseCorrectionMode;
396 4190 : pBits->codec = ISAR_SPLIT_REND_CODEC_NONE;
397 : }
398 :
399 : /*zero pad*/
400 391703 : if ( pcm_out_flag )
401 : {
402 4190 : bit_len = SplitRendBitRate / FRAMES_PER_SEC;
403 4190 : }
404 : else
405 : {
406 387513 : if ( splitCodec == ISAR_SPLIT_REND_CODEC_LCLD )
407 : {
408 280555 : bit_len = ( SplitRendBitRate * hSplitBin->hSplitBinLCLDEnc->iNumBlocks * hSplitBin->hSplitBinLCLDEnc->iNumIterations ) / ( 16 * FRAMES_PER_SEC );
409 280555 : }
410 : else
411 : {
412 106958 : bit_len = hSplitBin->hLc3plusEnc->config.isar_frame_duration_us / 1000;
413 106958 : bit_len = SplitRendBitRate * bit_len / 1000;
414 : }
415 : }
416 :
417 115459001 : while ( pBits->bits_written < bit_len )
418 : {
419 115067298 : ISAR_SPLIT_REND_BITStream_write_int32( pBits, 0L, 1 );
420 : }
421 :
422 391703 : pop_wmops();
423 :
424 391703 : return error;
425 590697 : }
|
