;******************************************************************************
;* SIMD-optimized MLP DSP functions
;* Copyright (c) 2014 James Almer <jamrial@gmail.com>
;*
;* This file is part of FFmpeg.
;*
;* FFmpeg is free software; you can redistribute it and/or
;* modify it under the terms of the GNU Lesser General Public
;* License as published by the Free Software Foundation; either
;* version 2.1 of the License, or (at your option) any later version.
;*
;* FFmpeg is distributed in the hope that it will be useful,
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
;* Lesser General Public License for more details.
;*
;* You should have received a copy of the GNU Lesser General Public
;* License along with FFmpeg; if not, write to the Free Software
;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
;******************************************************************************
%include "libavutil/x86/x86util.asm"
SECTION .text
%if ARCH_X86_64
%macro SHLX 2
%if cpuflag(bmi2)
shlx %1, %1, %2q
%else
shl %1, %2b
%endif
%endmacro
%macro REMATRIX 0
movdqa m0, [samplesq]
movdqa m1, [coeffsq ]
pshufd m2, m0, q2301
pshufd m3, m1, q2301
pmuldq m0, m1
pmuldq m3, m2
paddq m0, m3
%if notcpuflag(avx2)
movdqa m1, [samplesq + 16]
movdqa m2, [coeffsq + 16]
pshufd m3, m1, q2301
pshufd m4, m2, q2301
pmuldq m1, m2
pmuldq m4, m3
paddq m0, m1
paddq m0, m4
%else
vextracti128 xm1, m0, 1
paddq xm0, xm1
%endif
%endmacro
%macro LOOP_END 0
pshufd xm1, xm0, q0032
paddq xm0, xm1
movq accumq, xm0
movzx blsbsd, byte [blsbs_ptrq] ; load *bypassed_lsbs
sar accumq, 14 ; accum >>= 14
and accumd, maskd ; accum &= mask
add accumd, blsbsd ; accum += *bypassed_lsbs
mov [samplesq + dest_chq], accumd ; samples[dest_ch] = accum
add blsbs_ptrq, 8 ; bypassed_lsbs += MAX_CHANNELS;
add samplesq, 32 ; samples += MAX_CHANNELS;
cmp blsbs_ptrq, cntq
%endmacro
%macro LOOP_SHIFT_END 0
pshufd xm1, xm0, q0032
paddq xm0, xm1
movq accumq, xm0
and indexd, auspd ; index &= access_unit_size_pow2;
movsx noiseq, byte [noise_bufferq + indexq] ; load noise_buffer[index]
add indexd, index2d ; index += index2
SHLX noiseq, mns ; noise_buffer[index] <<= matrix_noise_shift
add accumq, noiseq ; accum += noise_buffer[index]
movzx noised, byte [blsbs_ptrq] ; load *bypassed_lsbs (reuse tmp noise register)
sar accumq, 14 ; accum >>= 14
and accumd, maskd ; accum &= mask
add accumd, noised ; accum += *bypassed_lsbs
mov [samplesq + dest_chq], accumd ; samples[dest_ch] = accum
add blsbs_ptrq, 8 ; bypassed_lsbs += MAX_CHANNELS;
add samplesq, 32 ; samples += MAX_CHANNELS;
cmp blsbs_ptrq, cntq
%endmacro
;void ff_mlp_rematrix_channel(int32_t *samples, const int32_t *coeffs,
; const uint8_t *bypassed_lsbs, const int8_t *noise_buffer,
; int index, unsigned int dest_ch, uint16_t blockpos,
; unsigned int maxchan, int matrix_noise_shift,
; int access_unit_size_pow2, int32_t mask)
%macro MLP_REMATRIX_CHANNEL 0
cglobal mlp_rematrix_channel, 0, 13, 5, samples, coeffs, blsbs_ptr, blsbs, \
index, dest_ch, blockpos, maxchan, mns, \
accum, mask, cnt
mov mnsd, mnsm ; load matrix_noise_shift
movzx blockposq, word blockposm ; load and zero extend blockpos (16bit)
mov maxchand, maxchanm ; load maxchan
mov maskd, maskm ; load mask
%if WIN64
mov dest_chd, dest_chm ; load dest_chd (not needed on UNIX64)
%endif
shl dest_chd, 2
lea cntq, [blsbs_ptrq + blockposq*8]
test mnsd, mnsd ; is matrix_noise_shift != 0?
jne .shift ; jump if true
cmp maxchand, 4 ; is maxchan < 4?
jl .loop4 ; jump if true
align 16
.loop8:
; Process 5 or more channels
REMATRIX
LOOP_END
jne .loop8
RET
align 16
.loop4:
; Process up to 4 channels
movdqa xm0, [samplesq]
movdqa xm1, [coeffsq ]
pshufd xm2, xm0, q2301
pshufd xm3, xm1, q2301
pmuldq xm0, xm1
pmuldq xm3, xm2
paddq xm0, xm3
LOOP_END
jne .loop4
RET
.shift:
%if WIN64
mov indexd, indexm ; load index (not needed on UNIX64)
%endif
mov r9d, r9m ; load access_unit_size_pow2
%if cpuflag(bmi2)
; bmi2 has shift functions that accept any gpr, not just cl, so keep things in place.
DEFINE_ARGS samples, coeffs, blsbs_ptr, noise_buffer, \
index, dest_ch, accum, index2, mns, \
ausp, mask, cnt, noise
add mnsd, 7 ; matrix_noise_shift += 7
%else ; sse4
mov r6, rcx ; move rcx elsewhere so we can use cl for matrix_noise_shift
%if WIN64
; r0 = rcx
DEFINE_ARGS mns, coeffs, blsbs_ptr, noise_buffer, index, dest_ch, samples, \
index2, accum, ausp, mask, cnt, noise
%else ; UNIX64
; r3 = rcx
DEFINE_ARGS samples, coeffs, blsbs_ptr, mns, index, dest_ch, noise_buffer, \
index2, accum, ausp, mask, cnt, noise
%endif
lea mnsd, [r8 + 7] ; rcx = matrix_noise_shift + 7
%endif ; cpuflag
sub auspd, 1 ; access_unit_size_pow2 -= 1
cmp r7d, 4 ; is maxchan < 4?
lea index2q, [indexq*2 + 1] ; index2 = 2 * index + 1;
jl .loop4_shift ; jump if maxchan < 4
align 16
.loop8_shift:
; Process 5 or more channels
REMATRIX
LOOP_SHIFT_END
jne .loop8_shift
RET
align 16
.loop4_shift:
; Process up to 4 channels
movdqa xm0, [samplesq]
movdqa xm1, [coeffsq ]
pshufd xm2, xm0, q2301
pshufd xm3, xm1, q2301
pmuldq xm0, xm1
pmuldq xm3, xm2
paddq xm0, xm3
LOOP_SHIFT_END
jne .loop4_shift
RET
%endmacro
INIT_XMM sse4
MLP_REMATRIX_CHANNEL
%if HAVE_AVX2_EXTERNAL
INIT_YMM avx2, bmi2
MLP_REMATRIX_CHANNEL
%endif
%endif ; ARCH_X86_64