i2p/Zerin-2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

initial commit

Browse Source
This commit is contained in:
i2p
2026-08-27 11:03:10 -06:00
commit d164820ea9
282 changed files with 90944 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/libjpeg-turbo/simd/x86_64/jccolext-sse2.asm
Executable
+541
View File
@@ -0,0 +1,541 @@
;
; Colorspace conversion (64-bit SSE2)
;
; Copyright (C) 2009, 2016, 2024-2025, D. R. Commander.
; Copyright (C) 2018, Matthias Räncker.
; Copyright (C) 2023, Aliaksiej Kandracienka.
;
; Based on the x86 SIMD extension for IJG JPEG library
; Copyright (C) 1999-2006, MIYASAKA Masaru.
; For conditions of distribution and use, see copyright notice in jsimdext.inc
;
; This file should be assembled with NASM (Netwide Assembler) or Yasm.
%include "jcolsamp.inc"
; --------------------------------------------------------------------------
;
; Convert some rows of samples to the JPEG colorspace.
;
; GLOBAL(void)
; jsimd_rgb_ycc_convert_sse2(JDIMENSION img_width, JSAMPARRAY input_buf,
; JSAMPIMAGE output_buf, JDIMENSION output_row,
; int num_rows)
;
; r10d = JDIMENSION img_width
; r11 = JSAMPARRAY input_buf
; r12 = JSAMPIMAGE output_buf
; r13d = JDIMENSION output_row
; r14d = int num_rows
%define wk(i) r15 - (WK_NUM - (i)) * SIZEOF_XMMWORD ; xmmword wk[WK_NUM]
%define WK_NUM 8
align 32
GLOBAL_FUNCTION(jsimd_rgb_ycc_convert_sse2)
EXTN(jsimd_rgb_ycc_convert_sse2):
ENDBR64
push rbp
mov rbp, rsp
push r15
and rsp, byte (-SIZEOF_XMMWORD) ; align to 128 bits
; Allocate stack space for wk array. r15 is used to access it.
mov r15, rsp
sub rsp, (SIZEOF_XMMWORD * WK_NUM)
COLLECT_ARGS 5
push rbx
mov ecx, r10d
test rcx, rcx
jz near .return
push rcx
mov rsi, r12
mov ecx, r13d
mov rdip, JSAMPARRAY [rsi + 0 * SIZEOF_JSAMPARRAY]
mov rbxp, JSAMPARRAY [rsi + 1 * SIZEOF_JSAMPARRAY]
mov rdxp, JSAMPARRAY [rsi + 2 * SIZEOF_JSAMPARRAY]
lea rdi, [rdi + rcx * SIZEOF_JSAMPROW]
lea rbx, [rbx + rcx * SIZEOF_JSAMPROW]
lea rdx, [rdx + rcx * SIZEOF_JSAMPROW]
pop rcx
mov rsi, r11
mov eax, r14d
test rax, rax
jle near .return
.rowloop:
push rdx
push rbx
push rdi
push rsi
push rcx ; col
mov rsip, JSAMPROW [rsi] ; inptr
mov rdip, JSAMPROW [rdi] ; outptr0
mov rbxp, JSAMPROW [rbx] ; outptr1
mov rdxp, JSAMPROW [rdx] ; outptr2
cmp rcx, byte SIZEOF_XMMWORD
jae near .columnloop
%if RGB_PIXELSIZE == 3 ; ---------------
.column_ld1:
push rax
push rdx
lea rcx, [rcx + rcx * 2] ; imul ecx, RGB_PIXELSIZE
test cl, SIZEOF_BYTE
jz short .column_ld2
sub rcx, byte SIZEOF_BYTE
movzx rax, byte [rsi + rcx]
.column_ld2:
test cl, SIZEOF_WORD
jz short .column_ld4
sub rcx, byte SIZEOF_WORD
movzx rdx, word [rsi + rcx]
shl rax, WORD_BIT
or rax, rdx
.column_ld4:
movd xmmA, eax
pop rdx
pop rax
test cl, SIZEOF_DWORD
jz short .column_ld8
sub rcx, byte SIZEOF_DWORD
movd xmmF, XMM_DWORD [rsi + rcx]
pslldq xmmA, SIZEOF_DWORD
por xmmA, xmmF
.column_ld8:
test cl, SIZEOF_MMWORD
jz short .column_ld16
sub rcx, byte SIZEOF_MMWORD
movq xmmB, XMM_MMWORD [rsi + rcx]
pslldq xmmA, SIZEOF_MMWORD
por xmmA, xmmB
.column_ld16:
test cl, SIZEOF_XMMWORD
jz short .column_ld32
movdqa xmmF, xmmA
movdqu xmmA, XMMWORD [rsi + 0 * SIZEOF_XMMWORD]
mov rcx, SIZEOF_XMMWORD
jmp short .rgb_ycc_cnv
.column_ld32:
test cl, 2 * SIZEOF_XMMWORD
mov rcx, SIZEOF_XMMWORD
jz short .rgb_ycc_cnv
movdqa xmmB, xmmA
movdqu xmmA, XMMWORD [rsi + 0 * SIZEOF_XMMWORD]
movdqu xmmF, XMMWORD [rsi + 1 * SIZEOF_XMMWORD]
jmp short .rgb_ycc_cnv
.columnloop:
movdqu xmmA, XMMWORD [rsi + 0 * SIZEOF_XMMWORD]
movdqu xmmF, XMMWORD [rsi + 1 * SIZEOF_XMMWORD]
movdqu xmmB, XMMWORD [rsi + 2 * SIZEOF_XMMWORD]
.rgb_ycc_cnv:
; NOTE: The values of RGB_RED, RGB_GREEN, and RGB_BLUE determine the
; mapping of components A, B, and C to red, green, and blue.
;
; xmmA = (A0 B0 C0 A1 B1 C1 A2 B2 C2 A3 B3 C3 A4 B4 C4 A5)
; xmmF = (B5 C5 A6 B6 C6 A7 B7 C7 A8 B8 C8 A9 B9 C9 Aa Ba)
; xmmB = (Ca Ab Bb Cb Ac Bc Cc Ad Bd Cd Ae Be Ce Af Bf Cf)
movdqa xmmG, xmmA
pslldq xmmA, 8
; xmmA = (-- -- -- -- -- -- -- -- A0 B0 C0 A1 B1 C1 A2 B2)
psrldq xmmG, 8
; xmmG = (C2 A3 B3 C3 A4 B4 C4 A5 -- -- -- -- -- -- -- --)
punpckhbw xmmA, xmmF
; xmmA = (A0 A8 B0 B8 C0 C8 A1 A9 B1 B9 C1 C9 A2 Aa B2 Ba)
pslldq xmmF, 8
; xmmF = (-- -- -- -- -- -- -- -- B5 C5 A6 B6 C6 A7 B7 C7)
punpcklbw xmmG, xmmB
; xmmG = (C2 Ca A3 Ab B3 Bb C3 Cb A4 Ac B4 Bc C4 Cc A5 Ad)
punpckhbw xmmF, xmmB
; xmmF = (B5 Bd C5 Cd A6 Ae B6 Be C6 Ce A7 Af B7 Bf C7 Cf)
movdqa xmmD, xmmA
pslldq xmmA, 8
; xmmA = (-- -- -- -- -- -- -- -- A0 A8 B0 B8 C0 C8 A1 A9)
psrldq xmmD, 8
; xmmD = (B1 B9 C1 C9 A2 Aa B2 Ba -- -- -- -- -- -- -- --)
punpckhbw xmmA, xmmG
; xmmA = (A0 A4 A8 Ac B0 B4 B8 Bc C0 C4 C8 Cc A1 A5 A9 Ad)
pslldq xmmG, 8
; xmmG = (-- -- -- -- -- -- -- -- C2 Ca A3 Ab B3 Bb C3 Cb)
punpcklbw xmmD, xmmF
; xmmD = (B1 B5 B9 Bd C1 C5 C9 Cd A2 A6 Aa Ae B2 B6 Ba Be)
punpckhbw xmmG, xmmF
; xmmG = (C2 C6 Ca Ce A3 A7 Ab Af B3 B7 Bb Bf C3 C7 Cb Cf)
movdqa xmmE, xmmA
pslldq xmmA, 8
; xmmA = (-- -- -- -- -- -- -- -- A0 A4 A8 Ac B0 B4 B8 Bc)
psrldq xmmE, 8
; xmmE = (C0 C4 C8 Cc A1 A5 A9 Ad -- -- -- -- -- -- -- --)
punpckhbw xmmA, xmmD
; xmmA = (A0 A2 A4 A6 A8 Aa Ac Ae B0 B2 B4 B6 B8 Ba Bc Be)
pslldq xmmD, 8
; xmmD = (-- -- -- -- -- -- -- -- B1 B5 B9 Bd C1 C5 C9 Cd)
punpcklbw xmmE, xmmG
; xmmE = (C0 C2 C4 C6 C8 Ca Cc Ce A1 A3 A5 A7 A9 Ab Ad Af)
punpckhbw xmmD, xmmG
; xmmD = (B1 B3 B5 B7 B9 Bb Bd Bf C1 C3 C5 C7 C9 Cb Cd Cf)
pxor xmmH, xmmH
movdqa xmmC, xmmA
punpcklbw xmmA, xmmH ; xmmA = (A0 A2 A4 A6 A8 Aa Ac Ae) = AE
punpckhbw xmmC, xmmH ; xmmC = (B0 B2 B4 B6 B8 Ba Bc Be) = BE
movdqa xmmB, xmmE
punpcklbw xmmE, xmmH ; xmmE = (C0 C2 C4 C6 C8 Ca Cc Ce) = CE
punpckhbw xmmB, xmmH ; xmmB = (A1 A3 A5 A7 A9 Ab Ad Af) = AO
movdqa xmmF, xmmD
punpcklbw xmmD, xmmH ; xmmD = (B1 B3 B5 B7 B9 Bb Bd Bf) = BO
punpckhbw xmmF, xmmH ; xmmF = (C1 C3 C5 C7 C9 Cb Cd Cf) = CO
%else ; RGB_PIXELSIZE == 4 ; -----------
.column_ld1:
test cl, SIZEOF_XMMWORD / 16
jz short .column_ld2
sub rcx, byte SIZEOF_XMMWORD / 16
movd xmmA, XMM_DWORD [rsi + rcx * RGB_PIXELSIZE]
.column_ld2:
test cl, SIZEOF_XMMWORD / 8
jz short .column_ld4
sub rcx, byte SIZEOF_XMMWORD / 8
movq xmmE, XMM_MMWORD [rsi + rcx * RGB_PIXELSIZE]
pslldq xmmA, SIZEOF_MMWORD
por xmmA, xmmE
.column_ld4:
test cl, SIZEOF_XMMWORD / 4
jz short .column_ld8
sub rcx, byte SIZEOF_XMMWORD / 4
movdqa xmmE, xmmA
movdqu xmmA, XMMWORD [rsi + rcx * RGB_PIXELSIZE]
.column_ld8:
test cl, SIZEOF_XMMWORD / 2
mov rcx, SIZEOF_XMMWORD
jz short .rgb_ycc_cnv
movdqa xmmF, xmmA
movdqa xmmH, xmmE
movdqu xmmA, XMMWORD [rsi + 0 * SIZEOF_XMMWORD]
movdqu xmmE, XMMWORD [rsi + 1 * SIZEOF_XMMWORD]
jmp short .rgb_ycc_cnv
.columnloop:
movdqu xmmA, XMMWORD [rsi + 0 * SIZEOF_XMMWORD]
movdqu xmmE, XMMWORD [rsi + 1 * SIZEOF_XMMWORD]
movdqu xmmF, XMMWORD [rsi + 2 * SIZEOF_XMMWORD]
movdqu xmmH, XMMWORD [rsi + 3 * SIZEOF_XMMWORD]
.rgb_ycc_cnv:
; NOTE: The values of RGB_RED, RGB_GREEN, and RGB_BLUE determine the
; mapping of components A, B, C, and D to red, green, and blue.
;
; xmmA = (A0 B0 C0 D0 A1 B1 C1 D1 A2 B2 C2 D2 A3 B3 C3 D3)
; xmmE = (A4 B4 C4 D4 A5 B5 C5 D5 A6 B6 C6 D6 A7 B7 C7 D7)
; xmmF = (A8 B8 C8 D8 A9 B9 C9 D9 Aa Ba Ca Da Ab Bb Cb Db)
; xmmH = (Ac Bc Cc Dc Ad Bd Cd Dd Ae Be Ce De Af Bf Cf Df)
movdqa xmmD, xmmA
punpcklbw xmmA, xmmE
; xmmA = (A0 A4 B0 B4 C0 C4 D0 D4 A1 A5 B1 B5 C1 C5 D1 D5)
punpckhbw xmmD, xmmE
; xmmD = (A2 A6 B2 B6 C2 C6 D2 D6 A3 A7 B3 B7 C3 C7 D3 D7)
movdqa xmmC, xmmF
punpcklbw xmmF, xmmH
; xmmF = (A8 Ac B8 Bc C8 Cc D8 Dc A9 Ad B9 Bd C9 Cd D9 Dd)
punpckhbw xmmC, xmmH
; xmmC = (Aa Ae Ba Be Ca Ce Da De Ab Af Bb Bf Cb Cf Db Df)
movdqa xmmB, xmmA
punpcklwd xmmA, xmmF
; xmmA = (A0 A4 A8 Ac B0 B4 B8 Bc C0 C4 C8 Cc D0 D4 D8 Dc)
punpckhwd xmmB, xmmF
; xmmB = (A1 A5 A9 Ad B1 B5 B9 Bd C1 C5 C9 Cd D1 D5 D9 Dd)
movdqa xmmG, xmmD
punpcklwd xmmD, xmmC
; xmmD = (A2 A6 Aa Ae B2 B6 Ba Be C2 C6 Ca Ce D2 D6 Da De)
punpckhwd xmmG, xmmC
; xmmG = (A3 A7 Ab Af B3 B7 Bb Bf C3 C7 Cb Cf D3 D7 Db Df)
movdqa xmmE, xmmA
punpcklbw xmmA, xmmD
; xmmA = (A0 A2 A4 A6 A8 Aa Ac Ae B0 B2 B4 B6 B8 Ba Bc Be)
punpckhbw xmmE, xmmD
; xmmE = (C0 C2 C4 C6 C8 Ca Cc Ce D0 D2 D4 D6 D8 Da Dc De)
movdqa xmmH, xmmB
punpcklbw xmmB, xmmG
; xmmB = (A1 A3 A5 A7 A9 Ab Ad Af B1 B3 B5 B7 B9 Bb Bd Bf)
punpckhbw xmmH, xmmG
; xmmH = (C1 C3 C5 C7 C9 Cb Cd Cf D1 D3 D5 D7 D9 Db Dd Df)
pxor xmmF, xmmF
movdqa xmmC, xmmA
punpcklbw xmmA, xmmF ; xmmA = (A0 A2 A4 A6 A8 Aa Ac Ae) = AE
punpckhbw xmmC, xmmF ; xmmC = (B0 B2 B4 B6 B8 Ba Bc Be) = BE
movdqa xmmD, xmmB
punpcklbw xmmB, xmmF ; xmmB = (A1 A3 A5 A7 A9 Ab Ad Af) = AO
punpckhbw xmmD, xmmF ; xmmD = (B1 B3 B5 B7 B9 Bb Bd Bf) = BO
movdqa xmmG, xmmE
punpcklbw xmmE, xmmF ; xmmE = (C0 C2 C4 C6 C8 Ca Cc Ce) = CE
punpckhbw xmmG, xmmF ; xmmG = (D0 D2 D4 D6 D8 Da Dc De) = DE
punpcklbw xmmF, xmmH
punpckhbw xmmH, xmmH
psrlw xmmF, BYTE_BIT ; xmmF = (C1 C3 C5 C7 C9 Cb Cd Cf) = CO
psrlw xmmH, BYTE_BIT ; xmmH = (D1 D3 D5 D7 D9 Db Dd Df) = DO
%endif ; RGB_PIXELSIZE ; ---------------
; xmm0 = (R0 R2 R4 R6 R8 Ra Rc Re) = RE
; xmm2 = (G0 G2 G4 G6 G8 Ga Gc Ge) = GE
; xmm4 = (B0 B2 B4 B6 B8 Ba Bc Be) = BE
; xmm1 = (R1 R3 R5 R7 R9 Rb Rd Rf) = RO
; xmm3 = (G1 G3 G5 G7 G9 Gb Gd Gf) = GO
; xmm5 = (B1 B3 B5 B7 B9 Bb Bd Bf) = BO
;
; (Original)
; Y = 0.29900 * R + 0.58700 * G + 0.11400 * B
; Cb = -0.16874 * R - 0.33126 * G + 0.50000 * B + CENTERJSAMPLE
; Cr = 0.50000 * R - 0.41869 * G - 0.08131 * B + CENTERJSAMPLE
;
; (This implementation)
; Y = 0.29900 * R + 0.33700 * G + 0.11400 * B + 0.25000 * G
; Cb = -0.16874 * R - 0.33126 * G + 0.50000 * B + CENTERJSAMPLE
; Cr = 0.50000 * R - 0.41869 * G - 0.08131 * B + CENTERJSAMPLE
movdqa XMMWORD [wk(0)], xmm0 ; wk(0) = RE
movdqa XMMWORD [wk(1)], xmm1 ; wk(1) = RO
movdqa XMMWORD [wk(2)], xmm4 ; wk(2) = BE
movdqa XMMWORD [wk(3)], xmm5 ; wk(3) = BO
movdqa xmm6, xmm1
punpcklwd xmm1, xmm3
punpckhwd xmm6, xmm3
movdqa xmm7, xmm1
movdqa xmm4, xmm6
pmaddwd xmm1, [rel PW_F0299_F0337]
; xmm1 = ROL * FIX(0.299) + GOL * FIX(0.337)
pmaddwd xmm6, [rel PW_F0299_F0337]
; xmm6 = ROH * FIX(0.299) + GOH * FIX(0.337)
pmaddwd xmm7, [rel PW_MF016_MF033]
; xmm7 = ROL * -FIX(0.168) + GOL * -FIX(0.331)
pmaddwd xmm4, [rel PW_MF016_MF033]
; xmm4 = ROH * -FIX(0.168) + GOH * -FIX(0.331)
movdqa XMMWORD [wk(4)], xmm1
; wk(4) = ROL * FIX(0.299) + GOL * FIX(0.337)
movdqa XMMWORD [wk(5)], xmm6
; wk(5) = ROH * FIX(0.299) + GOH * FIX(0.337)
pxor xmm1, xmm1
pxor xmm6, xmm6
punpcklwd xmm1, xmm5 ; xmm1 = BOL
punpckhwd xmm6, xmm5 ; xmm6 = BOH
psrld xmm1, 1 ; xmm1 = BOL * FIX(0.500)
psrld xmm6, 1 ; xmm6 = BOH * FIX(0.500)
movdqa xmm5, [rel PD_ONEHALFM1_CJ] ; xmm5 = [PD_ONEHALFM1_CJ]
paddd xmm7, xmm1
paddd xmm4, xmm6
paddd xmm7, xmm5
paddd xmm4, xmm5
psrld xmm7, SCALEBITS ; xmm7 = CbOL
psrld xmm4, SCALEBITS ; xmm4 = CbOH
packssdw xmm7, xmm4 ; xmm7 = CbO
movdqa xmm1, XMMWORD [wk(2)] ; xmm1 = BE
movdqa xmm6, xmm0
punpcklwd xmm0, xmm2
punpckhwd xmm6, xmm2
movdqa xmm5, xmm0
movdqa xmm4, xmm6
pmaddwd xmm0, [rel PW_F0299_F0337]
; xmm0 = REL * FIX(0.299) + GEL * FIX(0.337)
pmaddwd xmm6, [rel PW_F0299_F0337]
; xmm6 = REH * FIX(0.299) + GEH * FIX(0.337)
pmaddwd xmm5, [rel PW_MF016_MF033]
; xmm5 = REL * -FIX(0.168) + GEL * -FIX(0.331)
pmaddwd xmm4, [rel PW_MF016_MF033]
; xmm4 = REH * -FIX(0.168) + GEH * -FIX(0.331)
movdqa XMMWORD [wk(6)], xmm0
; wk(6) = REL * FIX(0.299) + GEL * FIX(0.337)
movdqa XMMWORD [wk(7)], xmm6
; wk(7) = REH * FIX(0.299) + GEH * FIX(0.337)
pxor xmm0, xmm0
pxor xmm6, xmm6
punpcklwd xmm0, xmm1 ; xmm0 = BEL
punpckhwd xmm6, xmm1 ; xmm6 = BEH
psrld xmm0, 1 ; xmm0 = BEL * FIX(0.500)
psrld xmm6, 1 ; xmm6 = BEH * FIX(0.500)
movdqa xmm1, [rel PD_ONEHALFM1_CJ] ; xmm1 = [PD_ONEHALFM1_CJ]
paddd xmm5, xmm0
paddd xmm4, xmm6
paddd xmm5, xmm1
paddd xmm4, xmm1
psrld xmm5, SCALEBITS ; xmm5 = CbEL
psrld xmm4, SCALEBITS ; xmm4 = CbEH
packssdw xmm5, xmm4 ; xmm5 = CbE
psllw xmm7, BYTE_BIT
por xmm5, xmm7 ; xmm5 = Cb
movdqa XMMWORD [rbx], xmm5 ; Save Cb
movdqa xmm0, XMMWORD [wk(3)] ; xmm0 = BO
movdqa xmm6, XMMWORD [wk(2)] ; xmm6 = BE
movdqa xmm1, XMMWORD [wk(1)] ; xmm1 = RO
movdqa xmm4, xmm0
punpcklwd xmm0, xmm3
punpckhwd xmm4, xmm3
movdqa xmm7, xmm0
movdqa xmm5, xmm4
pmaddwd xmm0, [rel PW_F0114_F0250]
; xmm0 = BOL * FIX(0.114) + GOL * FIX(0.250)
pmaddwd xmm4, [rel PW_F0114_F0250]
; xmm4 = BOH * FIX(0.114) + GOH * FIX(0.250)
pmaddwd xmm7, [rel PW_MF008_MF041]
; xmm7 = BOL * -FIX(0.081) + GOL * -FIX(0.418)
pmaddwd xmm5, [rel PW_MF008_MF041]
; xmm5 = BOH * -FIX(0.081) + GOH * -FIX(0.418)
movdqa xmm3, [rel PD_ONEHALF] ; xmm3 = [PD_ONEHALF]
paddd xmm0, XMMWORD [wk(4)]
paddd xmm4, XMMWORD [wk(5)]
paddd xmm0, xmm3
paddd xmm4, xmm3
psrld xmm0, SCALEBITS ; xmm0 = YOL
psrld xmm4, SCALEBITS ; xmm4 = YOH
packssdw xmm0, xmm4 ; xmm0 = YO
pxor xmm3, xmm3
pxor xmm4, xmm4
punpcklwd xmm3, xmm1 ; xmm3 = ROL
punpckhwd xmm4, xmm1 ; xmm4 = ROH
psrld xmm3, 1 ; xmm3 = ROL * FIX(0.500)
psrld xmm4, 1 ; xmm4 = ROH * FIX(0.500)
movdqa xmm1, [rel PD_ONEHALFM1_CJ] ; xmm1 = [PD_ONEHALFM1_CJ]
paddd xmm7, xmm3
paddd xmm5, xmm4
paddd xmm7, xmm1
paddd xmm5, xmm1
psrld xmm7, SCALEBITS ; xmm7 = CrOL
psrld xmm5, SCALEBITS ; xmm5 = CrOH
packssdw xmm7, xmm5 ; xmm7 = CrO
movdqa xmm3, XMMWORD [wk(0)] ; xmm3 = RE
movdqa xmm4, xmm6
punpcklwd xmm6, xmm2
punpckhwd xmm4, xmm2
movdqa xmm1, xmm6
movdqa xmm5, xmm4
pmaddwd xmm6, [rel PW_F0114_F0250]
; xmm6 = BEL * FIX(0.114) + GEL * FIX(0.250)
pmaddwd xmm4, [rel PW_F0114_F0250]
; xmm4 = BEH * FIX(0.114) + GEH * FIX(0.250)
pmaddwd xmm1, [rel PW_MF008_MF041]
; xmm1 = BEL * -FIX(0.081) + GEL * -FIX(0.418)
pmaddwd xmm5, [rel PW_MF008_MF041]
; xmm5 = BEH * -FIX(0.081) + GEH * -FIX(0.418)
movdqa xmm2, [rel PD_ONEHALF] ; xmm2 = [PD_ONEHALF]
paddd xmm6, XMMWORD [wk(6)]
paddd xmm4, XMMWORD [wk(7)]
paddd xmm6, xmm2
paddd xmm4, xmm2
psrld xmm6, SCALEBITS ; xmm6 = YEL
psrld xmm4, SCALEBITS ; xmm4 = YEH
packssdw xmm6, xmm4 ; xmm6 = YE
psllw xmm0, BYTE_BIT
por xmm6, xmm0 ; xmm6 = Y
movdqa XMMWORD [rdi], xmm6 ; Save Y
pxor xmm2, xmm2
pxor xmm4, xmm4
punpcklwd xmm2, xmm3 ; xmm2 = REL
punpckhwd xmm4, xmm3 ; xmm4 = REH
psrld xmm2, 1 ; xmm2 = REL * FIX(0.500)
psrld xmm4, 1 ; xmm4 = REH * FIX(0.500)
movdqa xmm0, [rel PD_ONEHALFM1_CJ] ; xmm0 = [PD_ONEHALFM1_CJ]
paddd xmm1, xmm2
paddd xmm5, xmm4
paddd xmm1, xmm0
paddd xmm5, xmm0
psrld xmm1, SCALEBITS ; xmm1 = CrEL
psrld xmm5, SCALEBITS ; xmm5 = CrEH
packssdw xmm1, xmm5 ; xmm1 = CrE
psllw xmm7, BYTE_BIT
por xmm1, xmm7 ; xmm1 = Cr
movdqa XMMWORD [rdx], xmm1 ; Save Cr
sub rcx, byte SIZEOF_XMMWORD
add rsi, byte RGB_PIXELSIZE * SIZEOF_XMMWORD ; inptr
add rdi, byte SIZEOF_XMMWORD ; outptr0
add rbx, byte SIZEOF_XMMWORD ; outptr1
add rdx, byte SIZEOF_XMMWORD ; outptr2
cmp rcx, byte SIZEOF_XMMWORD
jae near .columnloop
test rcx, rcx
jnz near .column_ld1
pop rcx ; col
pop rsi
pop rdi
pop rbx
pop rdx
add rsi, byte SIZEOF_JSAMPROW ; input_buf
add rdi, byte SIZEOF_JSAMPROW
add rbx, byte SIZEOF_JSAMPROW
add rdx, byte SIZEOF_JSAMPROW
dec rax ; num_rows
jg near .rowloop
.return:
pop rbx
UNCOLLECT_ARGS 5
lea rsp, [rbp - 8]
pop r15
pop rbp
ret
; For some reason, the OS X linker does not honor the request to align the
; segment unless we do this.
align 32
lib/libjpeg-turbo/simd/x86_64/jccolor-avx2.asm
Executable
+117
View File
@@ -0,0 +1,117 @@
;
; Colorspace conversion (64-bit AVX2)
;
; Copyright (C) 2009, 2016, 2024, D. R. Commander.
; Copyright (C) 2015, Intel Corporation.
;
; Based on the x86 SIMD extension for IJG JPEG library
; Copyright (C) 1999-2006, MIYASAKA Masaru.
; For conditions of distribution and use, see copyright notice in jsimdext.inc
;
; This file should be assembled with NASM (Netwide Assembler) or Yasm.
%include "jsimdext.inc"
; --------------------------------------------------------------------------
%define SCALEBITS 16
F_0_081 equ 5329 ; FIX(0.08131)
F_0_114 equ 7471 ; FIX(0.11400)
F_0_168 equ 11059 ; FIX(0.16874)
F_0_250 equ 16384 ; FIX(0.25000)
F_0_299 equ 19595 ; FIX(0.29900)
F_0_331 equ 21709 ; FIX(0.33126)
F_0_418 equ 27439 ; FIX(0.41869)
F_0_587 equ 38470 ; FIX(0.58700)
F_0_337 equ (F_0_587 - F_0_250) ; FIX(0.58700) - FIX(0.25000)
; --------------------------------------------------------------------------
SECTION SEG_CONST
ALIGNZ 32
GLOBAL_DATA(jconst_rgb_ycc_convert_avx2)
EXTN(jconst_rgb_ycc_convert_avx2):
PW_F0299_F0337 times 8 dw F_0_299, F_0_337
PW_F0114_F0250 times 8 dw F_0_114, F_0_250
PW_MF016_MF033 times 8 dw -F_0_168, -F_0_331
PW_MF008_MF041 times 8 dw -F_0_081, -F_0_418
PD_ONEHALFM1_CJ times 8 dd (1 << (SCALEBITS - 1)) - 1 + \
(CENTERJSAMPLE << SCALEBITS)
PD_ONEHALF times 8 dd (1 << (SCALEBITS - 1))
ALIGNZ 32
; --------------------------------------------------------------------------
SECTION SEG_TEXT
BITS 64
%include "jccolext-avx2.asm"
%undef RGB_RED
%undef RGB_GREEN
%undef RGB_BLUE
%undef RGB_PIXELSIZE
%define RGB_RED EXT_RGB_RED
%define RGB_GREEN EXT_RGB_GREEN
%define RGB_BLUE EXT_RGB_BLUE
%define RGB_PIXELSIZE EXT_RGB_PIXELSIZE
%define jsimd_rgb_ycc_convert_avx2 jsimd_extrgb_ycc_convert_avx2
%include "jccolext-avx2.asm"
%undef RGB_RED
%undef RGB_GREEN
%undef RGB_BLUE
%undef RGB_PIXELSIZE
%define RGB_RED EXT_RGBX_RED
%define RGB_GREEN EXT_RGBX_GREEN
%define RGB_BLUE EXT_RGBX_BLUE
%define RGB_PIXELSIZE EXT_RGBX_PIXELSIZE
%define jsimd_rgb_ycc_convert_avx2 jsimd_extrgbx_ycc_convert_avx2
%include "jccolext-avx2.asm"
%undef RGB_RED
%undef RGB_GREEN
%undef RGB_BLUE
%undef RGB_PIXELSIZE
%define RGB_RED EXT_BGR_RED
%define RGB_GREEN EXT_BGR_GREEN
%define RGB_BLUE EXT_BGR_BLUE
%define RGB_PIXELSIZE EXT_BGR_PIXELSIZE
%define jsimd_rgb_ycc_convert_avx2 jsimd_extbgr_ycc_convert_avx2
%include "jccolext-avx2.asm"
%undef RGB_RED
%undef RGB_GREEN
%undef RGB_BLUE
%undef RGB_PIXELSIZE
%define RGB_RED EXT_BGRX_RED
%define RGB_GREEN EXT_BGRX_GREEN
%define RGB_BLUE EXT_BGRX_BLUE
%define RGB_PIXELSIZE EXT_BGRX_PIXELSIZE
%define jsimd_rgb_ycc_convert_avx2 jsimd_extbgrx_ycc_convert_avx2
%include "jccolext-avx2.asm"
%undef RGB_RED
%undef RGB_GREEN
%undef RGB_BLUE
%undef RGB_PIXELSIZE
%define RGB_RED EXT_XBGR_RED
%define RGB_GREEN EXT_XBGR_GREEN
%define RGB_BLUE EXT_XBGR_BLUE
%define RGB_PIXELSIZE EXT_XBGR_PIXELSIZE
%define jsimd_rgb_ycc_convert_avx2 jsimd_extxbgr_ycc_convert_avx2
%include "jccolext-avx2.asm"
%undef RGB_RED
%undef RGB_GREEN
%undef RGB_BLUE
%undef RGB_PIXELSIZE
%define RGB_RED EXT_XRGB_RED
%define RGB_GREEN EXT_XRGB_GREEN
%define RGB_BLUE EXT_XRGB_BLUE
%define RGB_PIXELSIZE EXT_XRGB_PIXELSIZE
%define jsimd_rgb_ycc_convert_avx2 jsimd_extxrgb_ycc_convert_avx2
%include "jccolext-avx2.asm"
lib/libjpeg-turbo/simd/x86_64/jccolor-sse2.asm
Executable
+116
View File
@@ -0,0 +1,116 @@
;
; Colorspace conversion (64-bit SSE2)
;
; Copyright (C) 2009, 2016, 2024, D. R. Commander.
;
; Based on the x86 SIMD extension for IJG JPEG library
; Copyright (C) 1999-2006, MIYASAKA Masaru.
; For conditions of distribution and use, see copyright notice in jsimdext.inc
;
; This file should be assembled with NASM (Netwide Assembler) or Yasm.
%include "jsimdext.inc"
; --------------------------------------------------------------------------
%define SCALEBITS 16
F_0_081 equ 5329 ; FIX(0.08131)
F_0_114 equ 7471 ; FIX(0.11400)
F_0_168 equ 11059 ; FIX(0.16874)
F_0_250 equ 16384 ; FIX(0.25000)
F_0_299 equ 19595 ; FIX(0.29900)
F_0_331 equ 21709 ; FIX(0.33126)
F_0_418 equ 27439 ; FIX(0.41869)
F_0_587 equ 38470 ; FIX(0.58700)
F_0_337 equ (F_0_587 - F_0_250) ; FIX(0.58700) - FIX(0.25000)
; --------------------------------------------------------------------------
SECTION SEG_CONST
ALIGNZ 32
GLOBAL_DATA(jconst_rgb_ycc_convert_sse2)
EXTN(jconst_rgb_ycc_convert_sse2):
PW_F0299_F0337 times 4 dw F_0_299, F_0_337
PW_F0114_F0250 times 4 dw F_0_114, F_0_250
PW_MF016_MF033 times 4 dw -F_0_168, -F_0_331
PW_MF008_MF041 times 4 dw -F_0_081, -F_0_418
PD_ONEHALFM1_CJ times 4 dd (1 << (SCALEBITS - 1)) - 1 + \
(CENTERJSAMPLE << SCALEBITS)
PD_ONEHALF times 4 dd (1 << (SCALEBITS - 1))
ALIGNZ 32
; --------------------------------------------------------------------------
SECTION SEG_TEXT
BITS 64
%include "jccolext-sse2.asm"
%undef RGB_RED
%undef RGB_GREEN
%undef RGB_BLUE
%undef RGB_PIXELSIZE
%define RGB_RED EXT_RGB_RED
%define RGB_GREEN EXT_RGB_GREEN
%define RGB_BLUE EXT_RGB_BLUE
%define RGB_PIXELSIZE EXT_RGB_PIXELSIZE
%define jsimd_rgb_ycc_convert_sse2 jsimd_extrgb_ycc_convert_sse2
%include "jccolext-sse2.asm"
%undef RGB_RED
%undef RGB_GREEN
%undef RGB_BLUE
%undef RGB_PIXELSIZE
%define RGB_RED EXT_RGBX_RED
%define RGB_GREEN EXT_RGBX_GREEN
%define RGB_BLUE EXT_RGBX_BLUE
%define RGB_PIXELSIZE EXT_RGBX_PIXELSIZE
%define jsimd_rgb_ycc_convert_sse2 jsimd_extrgbx_ycc_convert_sse2
%include "jccolext-sse2.asm"
%undef RGB_RED
%undef RGB_GREEN
%undef RGB_BLUE
%undef RGB_PIXELSIZE
%define RGB_RED EXT_BGR_RED
%define RGB_GREEN EXT_BGR_GREEN
%define RGB_BLUE EXT_BGR_BLUE
%define RGB_PIXELSIZE EXT_BGR_PIXELSIZE
%define jsimd_rgb_ycc_convert_sse2 jsimd_extbgr_ycc_convert_sse2
%include "jccolext-sse2.asm"
%undef RGB_RED
%undef RGB_GREEN
%undef RGB_BLUE
%undef RGB_PIXELSIZE
%define RGB_RED EXT_BGRX_RED
%define RGB_GREEN EXT_BGRX_GREEN
%define RGB_BLUE EXT_BGRX_BLUE
%define RGB_PIXELSIZE EXT_BGRX_PIXELSIZE
%define jsimd_rgb_ycc_convert_sse2 jsimd_extbgrx_ycc_convert_sse2
%include "jccolext-sse2.asm"
%undef RGB_RED
%undef RGB_GREEN
%undef RGB_BLUE
%undef RGB_PIXELSIZE
%define RGB_RED EXT_XBGR_RED
%define RGB_GREEN EXT_XBGR_GREEN
%define RGB_BLUE EXT_XBGR_BLUE
%define RGB_PIXELSIZE EXT_XBGR_PIXELSIZE
%define jsimd_rgb_ycc_convert_sse2 jsimd_extxbgr_ycc_convert_sse2
%include "jccolext-sse2.asm"
%undef RGB_RED
%undef RGB_GREEN
%undef RGB_BLUE
%undef RGB_PIXELSIZE
%define RGB_RED EXT_XRGB_RED
%define RGB_GREEN EXT_XRGB_GREEN
%define RGB_BLUE EXT_XRGB_BLUE
%define RGB_PIXELSIZE EXT_XRGB_PIXELSIZE
%define jsimd_rgb_ycc_convert_sse2 jsimd_extxrgb_ycc_convert_sse2
%include "jccolext-sse2.asm"
lib/libjpeg-turbo/simd/x86_64/jcgray-avx2.asm
Executable
+109
View File
@@ -0,0 +1,109 @@
;
; Grayscale colorspace conversion (64-bit AVX2)
;
; Copyright (C) 2011, 2016, 2024, D. R. Commander.
; Copyright (C) 2015, Intel Corporation.
;
; Based on the x86 SIMD extension for IJG JPEG library
; Copyright (C) 1999-2006, MIYASAKA Masaru.
; For conditions of distribution and use, see copyright notice in jsimdext.inc
;
; This file should be assembled with NASM (Netwide Assembler) or Yasm.
%include "jsimdext.inc"
; --------------------------------------------------------------------------
%define SCALEBITS 16
F_0_114 equ 7471 ; FIX(0.11400)
F_0_250 equ 16384 ; FIX(0.25000)
F_0_299 equ 19595 ; FIX(0.29900)
F_0_587 equ 38470 ; FIX(0.58700)
F_0_337 equ (F_0_587 - F_0_250) ; FIX(0.58700) - FIX(0.25000)
; --------------------------------------------------------------------------
SECTION SEG_CONST
ALIGNZ 32
GLOBAL_DATA(jconst_rgb_gray_convert_avx2)
EXTN(jconst_rgb_gray_convert_avx2):
PW_F0299_F0337 times 8 dw F_0_299, F_0_337
PW_F0114_F0250 times 8 dw F_0_114, F_0_250
PD_ONEHALF times 8 dd (1 << (SCALEBITS - 1))
ALIGNZ 32
; --------------------------------------------------------------------------
SECTION SEG_TEXT
BITS 64
%include "jcgryext-avx2.asm"
%undef RGB_RED
%undef RGB_GREEN
%undef RGB_BLUE
%undef RGB_PIXELSIZE
%define RGB_RED EXT_RGB_RED
%define RGB_GREEN EXT_RGB_GREEN
%define RGB_BLUE EXT_RGB_BLUE
%define RGB_PIXELSIZE EXT_RGB_PIXELSIZE
%define jsimd_rgb_gray_convert_avx2 jsimd_extrgb_gray_convert_avx2
%include "jcgryext-avx2.asm"
%undef RGB_RED
%undef RGB_GREEN
%undef RGB_BLUE
%undef RGB_PIXELSIZE
%define RGB_RED EXT_RGBX_RED
%define RGB_GREEN EXT_RGBX_GREEN
%define RGB_BLUE EXT_RGBX_BLUE
%define RGB_PIXELSIZE EXT_RGBX_PIXELSIZE
%define jsimd_rgb_gray_convert_avx2 jsimd_extrgbx_gray_convert_avx2
%include "jcgryext-avx2.asm"
%undef RGB_RED
%undef RGB_GREEN
%undef RGB_BLUE
%undef RGB_PIXELSIZE
%define RGB_RED EXT_BGR_RED
%define RGB_GREEN EXT_BGR_GREEN
%define RGB_BLUE EXT_BGR_BLUE
%define RGB_PIXELSIZE EXT_BGR_PIXELSIZE
%define jsimd_rgb_gray_convert_avx2 jsimd_extbgr_gray_convert_avx2
%include "jcgryext-avx2.asm"
%undef RGB_RED
%undef RGB_GREEN
%undef RGB_BLUE
%undef RGB_PIXELSIZE
%define RGB_RED EXT_BGRX_RED
%define RGB_GREEN EXT_BGRX_GREEN
%define RGB_BLUE EXT_BGRX_BLUE
%define RGB_PIXELSIZE EXT_BGRX_PIXELSIZE
%define jsimd_rgb_gray_convert_avx2 jsimd_extbgrx_gray_convert_avx2
%include "jcgryext-avx2.asm"
%undef RGB_RED
%undef RGB_GREEN
%undef RGB_BLUE
%undef RGB_PIXELSIZE
%define RGB_RED EXT_XBGR_RED
%define RGB_GREEN EXT_XBGR_GREEN
%define RGB_BLUE EXT_XBGR_BLUE
%define RGB_PIXELSIZE EXT_XBGR_PIXELSIZE
%define jsimd_rgb_gray_convert_avx2 jsimd_extxbgr_gray_convert_avx2
%include "jcgryext-avx2.asm"
%undef RGB_RED
%undef RGB_GREEN
%undef RGB_BLUE
%undef RGB_PIXELSIZE
%define RGB_RED EXT_XRGB_RED
%define RGB_GREEN EXT_XRGB_GREEN
%define RGB_BLUE EXT_XRGB_BLUE
%define RGB_PIXELSIZE EXT_XRGB_PIXELSIZE
%define jsimd_rgb_gray_convert_avx2 jsimd_extxrgb_gray_convert_avx2
%include "jcgryext-avx2.asm"
lib/libjpeg-turbo/simd/x86_64/jcgryext-sse2.asm
Executable
+411
View File
@@ -0,0 +1,411 @@
;
; Grayscale colorspace conversion (64-bit SSE2)
;
; Copyright (C) 2011, 2016, 2024-2025, D. R. Commander.
; Copyright (C) 2018, Matthias Räncker.
; Copyright (C) 2023, Aliaksiej Kandracienka.
;
; Based on the x86 SIMD extension for IJG JPEG library
; Copyright (C) 1999-2006, MIYASAKA Masaru.
; For conditions of distribution and use, see copyright notice in jsimdext.inc
;
; This file should be assembled with NASM (Netwide Assembler) or Yasm.
%include "jcolsamp.inc"
; --------------------------------------------------------------------------
;
; Convert some rows of samples to the JPEG colorspace.
;
; GLOBAL(void)
; jsimd_rgb_gray_convert_sse2(JDIMENSION img_width, JSAMPARRAY input_buf,
; JSAMPIMAGE output_buf, JDIMENSION output_row,
; int num_rows)
;
; r10d = JDIMENSION img_width
; r11 = JSAMPARRAY input_buf
; r12 = JSAMPIMAGE output_buf
; r13d = JDIMENSION output_row
; r14d = int num_rows
%define wk(i) r15 - (WK_NUM - (i)) * SIZEOF_XMMWORD ; xmmword wk[WK_NUM]
%define WK_NUM 2
align 32
GLOBAL_FUNCTION(jsimd_rgb_gray_convert_sse2)
EXTN(jsimd_rgb_gray_convert_sse2):
ENDBR64
push rbp
mov rbp, rsp
push r15
and rsp, byte (-SIZEOF_XMMWORD) ; align to 128 bits
; Allocate stack space for wk array. r15 is used to access it.
mov r15, rsp
sub rsp, byte (SIZEOF_XMMWORD * WK_NUM)
COLLECT_ARGS 5
push rbx
mov ecx, r10d
test rcx, rcx
jz near .return
push rcx
mov rsi, r12
mov ecx, r13d
mov rdip, JSAMPARRAY [rsi + 0 * SIZEOF_JSAMPARRAY]
lea rdi, [rdi + rcx * SIZEOF_JSAMPROW]
pop rcx
mov rsi, r11
mov eax, r14d
test rax, rax
jle near .return
.rowloop:
push rdi
push rsi
push rcx ; col
mov rsip, JSAMPROW [rsi] ; inptr
mov rdip, JSAMPROW [rdi] ; outptr0
cmp rcx, byte SIZEOF_XMMWORD
jae near .columnloop
%if RGB_PIXELSIZE == 3 ; ---------------
.column_ld1:
push rax
push rdx
lea rcx, [rcx + rcx * 2] ; imul ecx, RGB_PIXELSIZE
test cl, SIZEOF_BYTE
jz short .column_ld2
sub rcx, byte SIZEOF_BYTE
movzx rax, byte [rsi + rcx]
.column_ld2:
test cl, SIZEOF_WORD
jz short .column_ld4
sub rcx, byte SIZEOF_WORD
movzx rdx, word [rsi + rcx]
shl rax, WORD_BIT
or rax, rdx
.column_ld4:
movd xmmA, eax
pop rdx
pop rax
test cl, SIZEOF_DWORD
jz short .column_ld8
sub rcx, byte SIZEOF_DWORD
movd xmmF, XMM_DWORD [rsi + rcx]
pslldq xmmA, SIZEOF_DWORD
por xmmA, xmmF
.column_ld8:
test cl, SIZEOF_MMWORD
jz short .column_ld16
sub rcx, byte SIZEOF_MMWORD
movq xmmB, XMM_MMWORD [rsi + rcx]
pslldq xmmA, SIZEOF_MMWORD
por xmmA, xmmB
.column_ld16:
test cl, SIZEOF_XMMWORD
jz short .column_ld32
movdqa xmmF, xmmA
movdqu xmmA, XMMWORD [rsi + 0 * SIZEOF_XMMWORD]
mov rcx, SIZEOF_XMMWORD
jmp short .rgb_gray_cnv
.column_ld32:
test cl, 2 * SIZEOF_XMMWORD
mov rcx, SIZEOF_XMMWORD
jz short .rgb_gray_cnv
movdqa xmmB, xmmA
movdqu xmmA, XMMWORD [rsi + 0 * SIZEOF_XMMWORD]
movdqu xmmF, XMMWORD [rsi + 1 * SIZEOF_XMMWORD]
jmp short .rgb_gray_cnv
.columnloop:
movdqu xmmA, XMMWORD [rsi + 0 * SIZEOF_XMMWORD]
movdqu xmmF, XMMWORD [rsi + 1 * SIZEOF_XMMWORD]
movdqu xmmB, XMMWORD [rsi + 2 * SIZEOF_XMMWORD]
.rgb_gray_cnv:
; NOTE: The values of RGB_RED, RGB_GREEN, and RGB_BLUE determine the
; mapping of components A, B, and C to red, green, and blue.
;
; xmmA = (A0 B0 C0 A1 B1 C1 A2 B2 C2 A3 B3 C3 A4 B4 C4 A5)
; xmmF = (B5 C5 A6 B6 C6 A7 B7 C7 A8 B8 C8 A9 B9 C9 Aa Ba)
; xmmB = (Ca Ab Bb Cb Ac Bc Cc Ad Bd Cd Ae Be Ce Af Bf Cf)
movdqa xmmG, xmmA
pslldq xmmA, 8
; xmmA = (-- -- -- -- -- -- -- -- A0 B0 C0 A1 B1 C1 A2 B2)
psrldq xmmG, 8
; xmmG = (C2 A3 B3 C3 A4 B4 C4 A5 -- -- -- -- -- -- -- --)
punpckhbw xmmA, xmmF
; xmmA = (A0 A8 B0 B8 C0 C8 A1 A9 B1 B9 C1 C9 A2 Aa B2 Ba)
pslldq xmmF, 8
; xmmF = (-- -- -- -- -- -- -- -- B5 C5 A6 B6 C6 A7 B7 C7)
punpcklbw xmmG, xmmB
; xmmG = (C2 Ca A3 Ab B3 Bb C3 Cb A4 Ac B4 Bc C4 Cc A5 Ad)
punpckhbw xmmF, xmmB
; xmmF = (B5 Bd C5 Cd A6 Ae B6 Be C6 Ce A7 Af B7 Bf C7 Cf)
movdqa xmmD, xmmA
pslldq xmmA, 8
; xmmA = (-- -- -- -- -- -- -- -- A0 A8 B0 B8 C0 C8 A1 A9)
psrldq xmmD, 8
; xmmD = (B1 B9 C1 C9 A2 Aa B2 Ba -- -- -- -- -- -- -- --)
punpckhbw xmmA, xmmG
; xmmA = (A0 A4 A8 Ac B0 B4 B8 Bc C0 C4 C8 Cc A1 A5 A9 Ad)
pslldq xmmG, 8
; xmmG = (-- -- -- -- -- -- -- -- C2 Ca A3 Ab B3 Bb C3 Cb)
punpcklbw xmmD, xmmF
; xmmD = (B1 B5 B9 Bd C1 C5 C9 Cd A2 A6 Aa Ae B2 B6 Ba Be)
punpckhbw xmmG, xmmF
; xmmG = (C2 C6 Ca Ce A3 A7 Ab Af B3 B7 Bb Bf C3 C7 Cb Cf)
movdqa xmmE, xmmA
pslldq xmmA, 8
; xmmA = (-- -- -- -- -- -- -- -- A0 A4 A8 Ac B0 B4 B8 Bc)
psrldq xmmE, 8
; xmmE = (C0 C4 C8 Cc A1 A5 A9 Ad -- -- -- -- -- -- -- --)
punpckhbw xmmA, xmmD
; xmmA = (A0 A2 A4 A6 A8 Aa Ac Ae B0 B2 B4 B6 B8 Ba Bc Be)
pslldq xmmD, 8
; xmmD = (-- -- -- -- -- -- -- -- B1 B5 B9 Bd C1 C5 C9 Cd)
punpcklbw xmmE, xmmG
; xmmE = (C0 C2 C4 C6 C8 Ca Cc Ce A1 A3 A5 A7 A9 Ab Ad Af)
punpckhbw xmmD, xmmG
; xmmD = (B1 B3 B5 B7 B9 Bb Bd Bf C1 C3 C5 C7 C9 Cb Cd Cf)
pxor xmmH, xmmH
movdqa xmmC, xmmA
punpcklbw xmmA, xmmH ; xmmA = (A0 A2 A4 A6 A8 Aa Ac Ae) = AE
punpckhbw xmmC, xmmH ; xmmC = (B0 B2 B4 B6 B8 Ba Bc Be) = BE
movdqa xmmB, xmmE
punpcklbw xmmE, xmmH ; xmmE = (C0 C2 C4 C6 C8 Ca Cc Ce) = CE
punpckhbw xmmB, xmmH ; xmmB = (A1 A3 A5 A7 A9 Ab Ad Af) = AO
movdqa xmmF, xmmD
punpcklbw xmmD, xmmH ; xmmD = (B1 B3 B5 B7 B9 Bb Bd Bf) = BO
punpckhbw xmmF, xmmH ; xmmF = (C1 C3 C5 C7 C9 Cb Cd Cf) = CO
%else ; RGB_PIXELSIZE == 4 ; -----------
.column_ld1:
test cl, SIZEOF_XMMWORD / 16
jz short .column_ld2
sub rcx, byte SIZEOF_XMMWORD / 16
movd xmmA, XMM_DWORD [rsi + rcx * RGB_PIXELSIZE]
.column_ld2:
test cl, SIZEOF_XMMWORD / 8
jz short .column_ld4
sub rcx, byte SIZEOF_XMMWORD / 8
movq xmmE, XMM_MMWORD [rsi + rcx * RGB_PIXELSIZE]
pslldq xmmA, SIZEOF_MMWORD
por xmmA, xmmE
.column_ld4:
test cl, SIZEOF_XMMWORD / 4
jz short .column_ld8
sub rcx, byte SIZEOF_XMMWORD / 4
movdqa xmmE, xmmA
movdqu xmmA, XMMWORD [rsi + rcx * RGB_PIXELSIZE]
.column_ld8:
test cl, SIZEOF_XMMWORD / 2
mov rcx, SIZEOF_XMMWORD
jz short .rgb_gray_cnv
movdqa xmmF, xmmA
movdqa xmmH, xmmE
movdqu xmmA, XMMWORD [rsi + 0 * SIZEOF_XMMWORD]
movdqu xmmE, XMMWORD [rsi + 1 * SIZEOF_XMMWORD]
jmp short .rgb_gray_cnv
.columnloop:
movdqu xmmA, XMMWORD [rsi + 0 * SIZEOF_XMMWORD]
movdqu xmmE, XMMWORD [rsi + 1 * SIZEOF_XMMWORD]
movdqu xmmF, XMMWORD [rsi + 2 * SIZEOF_XMMWORD]
movdqu xmmH, XMMWORD [rsi + 3 * SIZEOF_XMMWORD]
.rgb_gray_cnv:
; NOTE: The values of RGB_RED, RGB_GREEN, and RGB_BLUE determine the
; mapping of components A, B, C, and D to red, green, and blue.
;
; xmmA = (A0 B0 C0 D0 A1 B1 C1 D1 A2 B2 C2 D2 A3 B3 C3 D3)
; xmmE = (A4 B4 C4 D4 A5 B5 C5 D5 A6 B6 C6 D6 A7 B7 C7 D7)
; xmmF = (A8 B8 C8 D8 A9 B9 C9 D9 Aa Ba Ca Da Ab Bb Cb Db)
; xmmH = (Ac Bc Cc Dc Ad Bd Cd Dd Ae Be Ce De Af Bf Cf Df)
movdqa xmmD, xmmA
punpcklbw xmmA, xmmE
; xmmA = (A0 A4 B0 B4 C0 C4 D0 D4 A1 A5 B1 B5 C1 C5 D1 D5)
punpckhbw xmmD, xmmE
; xmmD = (A2 A6 B2 B6 C2 C6 D2 D6 A3 A7 B3 B7 C3 C7 D3 D7)
movdqa xmmC, xmmF
punpcklbw xmmF, xmmH
; xmmF = (A8 Ac B8 Bc C8 Cc D8 Dc A9 Ad B9 Bd C9 Cd D9 Dd)
punpckhbw xmmC, xmmH
; xmmC = (Aa Ae Ba Be Ca Ce Da De Ab Af Bb Bf Cb Cf Db Df)
movdqa xmmB, xmmA
punpcklwd xmmA, xmmF
; xmmA = (A0 A4 A8 Ac B0 B4 B8 Bc C0 C4 C8 Cc D0 D4 D8 Dc)
punpckhwd xmmB, xmmF
; xmmB = (A1 A5 A9 Ad B1 B5 B9 Bd C1 C5 C9 Cd D1 D5 D9 Dd)
movdqa xmmG, xmmD
punpcklwd xmmD, xmmC
; xmmD = (A2 A6 Aa Ae B2 B6 Ba Be C2 C6 Ca Ce D2 D6 Da De)
punpckhwd xmmG, xmmC
; xmmG = (A3 A7 Ab Af B3 B7 Bb Bf C3 C7 Cb Cf D3 D7 Db Df)
movdqa xmmE, xmmA
punpcklbw xmmA, xmmD
; xmmA = (A0 A2 A4 A6 A8 Aa Ac Ae B0 B2 B4 B6 B8 Ba Bc Be)
punpckhbw xmmE, xmmD
; xmmE = (C0 C2 C4 C6 C8 Ca Cc Ce D0 D2 D4 D6 D8 Da Dc De)
movdqa xmmH, xmmB
punpcklbw xmmB, xmmG
; xmmB = (A1 A3 A5 A7 A9 Ab Ad Af B1 B3 B5 B7 B9 Bb Bd Bf)
punpckhbw xmmH, xmmG
; xmmH = (C1 C3 C5 C7 C9 Cb Cd Cf D1 D3 D5 D7 D9 Db Dd Df)
pxor xmmF, xmmF
movdqa xmmC, xmmA
punpcklbw xmmA, xmmF ; xmmA = (A0 A2 A4 A6 A8 Aa Ac Ae) = AE
punpckhbw xmmC, xmmF ; xmmC = (B0 B2 B4 B6 B8 Ba Bc Be) = BE
movdqa xmmD, xmmB
punpcklbw xmmB, xmmF ; xmmB = (A1 A3 A5 A7 A9 Ab Ad Af) = AO
punpckhbw xmmD, xmmF ; xmmD = (B1 B3 B5 B7 B9 Bb Bd Bf) = BO
movdqa xmmG, xmmE
punpcklbw xmmE, xmmF ; xmmE = (C0 C2 C4 C6 C8 Ca Cc Ce) = CE
punpckhbw xmmG, xmmF ; xmmG = (D0 D2 D4 D6 D8 Da Dc De) = DE
punpcklbw xmmF, xmmH
punpckhbw xmmH, xmmH
psrlw xmmF, BYTE_BIT ; xmmF = (C1 C3 C5 C7 C9 Cb Cd Cf) = CO
psrlw xmmH, BYTE_BIT ; xmmH = (D1 D3 D5 D7 D9 Db Dd Df) = DO
%endif ; RGB_PIXELSIZE ; ---------------
; xmm0 = (R0 R2 R4 R6 R8 Ra Rc Re) = RE
; xmm2 = (G0 G2 G4 G6 G8 Ga Gc Ge) = GE
; xmm4 = (B0 B2 B4 B6 B8 Ba Bc Be) = BE
; xmm1 = (R1 R3 R5 R7 R9 Rb Rd Rf) = RO
; xmm3 = (G1 G3 G5 G7 G9 Gb Gd Gf) = GO
; xmm5 = (B1 B3 B5 B7 B9 Bb Bd Bf) = BO
;
; (Original)
; Y = 0.29900 * R + 0.58700 * G + 0.11400 * B
;
; (This implementation)
; Y = 0.29900 * R + 0.33700 * G + 0.11400 * B + 0.25000 * G
movdqa xmm6, xmm1
punpcklwd xmm1, xmm3
punpckhwd xmm6, xmm3
pmaddwd xmm1, [rel PW_F0299_F0337]
; xmm1 = ROL * FIX(0.299) + GOL * FIX(0.337)
pmaddwd xmm6, [rel PW_F0299_F0337]
; xmm6 = ROH * FIX(0.299) + GOH * FIX(0.337)
movdqa xmm7, xmm6
; xmm7 = ROH * FIX(0.299) + GOH * FIX(0.337)
movdqa xmm6, xmm0
punpcklwd xmm0, xmm2
punpckhwd xmm6, xmm2
pmaddwd xmm0, [rel PW_F0299_F0337]
; xmm0 = REL * FIX(0.299) + GEL * FIX(0.337)
pmaddwd xmm6, [rel PW_F0299_F0337]
; xmm6 = REH * FIX(0.299) + GEH * FIX(0.337)
movdqa XMMWORD [wk(0)], xmm0
; wk(0) = REL * FIX(0.299) + GEL * FIX(0.337)
movdqa XMMWORD [wk(1)], xmm6
; wk(1) = REH * FIX(0.299) + GEH * FIX(0.337)
movdqa xmm0, xmm5 ; xmm0 = BO
movdqa xmm6, xmm4 ; xmm6 = BE
movdqa xmm4, xmm0
punpcklwd xmm0, xmm3
punpckhwd xmm4, xmm3
pmaddwd xmm0, [rel PW_F0114_F0250]
; xmm0 = BOL * FIX(0.114) + GOL * FIX(0.250)
pmaddwd xmm4, [rel PW_F0114_F0250]
; xmm4 = BOH * FIX(0.114) + GOH * FIX(0.250)
movdqa xmm3, [rel PD_ONEHALF] ; xmm3 = [PD_ONEHALF]
paddd xmm0, xmm1
paddd xmm4, xmm7
paddd xmm0, xmm3
paddd xmm4, xmm3
psrld xmm0, SCALEBITS ; xmm0 = YOL
psrld xmm4, SCALEBITS ; xmm4 = YOH
packssdw xmm0, xmm4 ; xmm0 = YO
movdqa xmm4, xmm6
punpcklwd xmm6, xmm2
punpckhwd xmm4, xmm2
pmaddwd xmm6, [rel PW_F0114_F0250]
; xmm6 = BEL * FIX(0.114) + GEL * FIX(0.250)
pmaddwd xmm4, [rel PW_F0114_F0250]
; xmm4 = BEH * FIX(0.114) + GEH * FIX(0.250)
movdqa xmm2, [rel PD_ONEHALF] ; xmm2 = [PD_ONEHALF]
paddd xmm6, XMMWORD [wk(0)]
paddd xmm4, XMMWORD [wk(1)]
paddd xmm6, xmm2
paddd xmm4, xmm2
psrld xmm6, SCALEBITS ; xmm6 = YEL
psrld xmm4, SCALEBITS ; xmm4 = YEH
packssdw xmm6, xmm4 ; xmm6 = YE
psllw xmm0, BYTE_BIT
por xmm6, xmm0 ; xmm6 = Y
movdqa XMMWORD [rdi], xmm6 ; Save Y
sub rcx, byte SIZEOF_XMMWORD
add rsi, byte RGB_PIXELSIZE * SIZEOF_XMMWORD ; inptr
add rdi, byte SIZEOF_XMMWORD ; outptr0
cmp rcx, byte SIZEOF_XMMWORD
jae near .columnloop
test rcx, rcx
jnz near .column_ld1
pop rcx ; col
pop rsi
pop rdi
add rsi, byte SIZEOF_JSAMPROW ; input_buf
add rdi, byte SIZEOF_JSAMPROW
dec rax ; num_rows
jg near .rowloop
.return:
pop rbx
UNCOLLECT_ARGS 5
lea rsp, [rbp - 8]
pop r15
pop rbp
ret
; For some reason, the OS X linker does not honor the request to align the
; segment unless we do this.
align 32
lib/libjpeg-turbo/simd/x86_64/jcphuff-sse2.asm
Executable
+626
View File
@@ -0,0 +1,626 @@
;
; Prepare data for progressive Huffman encoding (64-bit SSE2)
;
; Copyright (C) 2016, 2018, Matthieu Darbois.
; Copyright (C) 2023, Aliaksiej Kandracienka.
; Copyright (C) 2024-2025, D. R. Commander.
;
; Based on the x86 SIMD extension for IJG JPEG library
; Copyright (C) 1999-2006, MIYASAKA Masaru.
; For conditions of distribution and use, see copyright notice in jsimdext.inc
;
; This file should be assembled with NASM (Netwide Assembler) or Yasm.
;
; This file contains an SSE2 implementation of data preparation for progressive
; Huffman encoding. See jcphuff.c for more details.
%include "jsimdext.inc"
; --------------------------------------------------------------------------
SECTION SEG_TEXT
BITS 64
; --------------------------------------------------------------------------
; Macros to load data for jsimd_encode_mcu_AC_first_prepare_sse2() and
; jsimd_encode_mcu_AC_refine_prepare_sse2()
%macro LOAD16 0
pxor N0, N0
pxor N1, N1
mov T0d, INT [LUT + 0 * SIZEOF_INT]
mov T1d, INT [LUT + 8 * SIZEOF_INT]
pinsrw X0, word [BLOCK + T0 * 2], 0
pinsrw X1, word [BLOCK + T1 * 2], 0
mov T0d, INT [LUT + 1 * SIZEOF_INT]
mov T1d, INT [LUT + 9 * SIZEOF_INT]
pinsrw X0, word [BLOCK + T0 * 2], 1
pinsrw X1, word [BLOCK + T1 * 2], 1
mov T0d, INT [LUT + 2 * SIZEOF_INT]
mov T1d, INT [LUT + 10 * SIZEOF_INT]
pinsrw X0, word [BLOCK + T0 * 2], 2
pinsrw X1, word [BLOCK + T1 * 2], 2
mov T0d, INT [LUT + 3 * SIZEOF_INT]
mov T1d, INT [LUT + 11 * SIZEOF_INT]
pinsrw X0, word [BLOCK + T0 * 2], 3
pinsrw X1, word [BLOCK + T1 * 2], 3
mov T0d, INT [LUT + 4 * SIZEOF_INT]
mov T1d, INT [LUT + 12 * SIZEOF_INT]
pinsrw X0, word [BLOCK + T0 * 2], 4
pinsrw X1, word [BLOCK + T1 * 2], 4
mov T0d, INT [LUT + 5 * SIZEOF_INT]
mov T1d, INT [LUT + 13 * SIZEOF_INT]
pinsrw X0, word [BLOCK + T0 * 2], 5
pinsrw X1, word [BLOCK + T1 * 2], 5
mov T0d, INT [LUT + 6 * SIZEOF_INT]
mov T1d, INT [LUT + 14 * SIZEOF_INT]
pinsrw X0, word [BLOCK + T0 * 2], 6
pinsrw X1, word [BLOCK + T1 * 2], 6
mov T0d, INT [LUT + 7 * SIZEOF_INT]
mov T1d, INT [LUT + 15 * SIZEOF_INT]
pinsrw X0, word [BLOCK + T0 * 2], 7
pinsrw X1, word [BLOCK + T1 * 2], 7
%endmacro
%macro LOAD15 0
pxor N0, N0
pxor N1, N1
pxor X1, X1
mov T0d, INT [LUT + 0 * SIZEOF_INT]
mov T1d, INT [LUT + 8 * SIZEOF_INT]
pinsrw X0, word [BLOCK + T0 * 2], 0
pinsrw X1, word [BLOCK + T1 * 2], 0
mov T0d, INT [LUT + 1 * SIZEOF_INT]
pinsrw X0, word [BLOCK + T0 * 2], 1
mov T0d, INT [LUT + 2 * SIZEOF_INT]
pinsrw X0, word [BLOCK + T0 * 2], 2
mov T0d, INT [LUT + 3 * SIZEOF_INT]
pinsrw X0, word [BLOCK + T0 * 2], 3
mov T0d, INT [LUT + 4 * SIZEOF_INT]
pinsrw X0, word [BLOCK + T0 * 2], 4
mov T0d, INT [LUT + 5 * SIZEOF_INT]
pinsrw X0, word [BLOCK + T0 * 2], 5
mov T0d, INT [LUT + 6 * SIZEOF_INT]
pinsrw X0, word [BLOCK + T0 * 2], 6
mov T0d, INT [LUT + 7 * SIZEOF_INT]
pinsrw X0, word [BLOCK + T0 * 2], 7
cmp LENEND, 2
jl %%.ELOAD15
mov T1d, INT [LUT + 9 * SIZEOF_INT]
pinsrw X1, word [BLOCK + T1 * 2], 1
cmp LENEND, 3
jl %%.ELOAD15
mov T1d, INT [LUT + 10 * SIZEOF_INT]
pinsrw X1, word [BLOCK + T1 * 2], 2
cmp LENEND, 4
jl %%.ELOAD15
mov T1d, INT [LUT + 11 * SIZEOF_INT]
pinsrw X1, word [BLOCK + T1 * 2], 3
cmp LENEND, 5
jl %%.ELOAD15
mov T1d, INT [LUT + 12 * SIZEOF_INT]
pinsrw X1, word [BLOCK + T1 * 2], 4
cmp LENEND, 6
jl %%.ELOAD15
mov T1d, INT [LUT + 13 * SIZEOF_INT]
pinsrw X1, word [BLOCK + T1 * 2], 5
cmp LENEND, 7
jl %%.ELOAD15
mov T1d, INT [LUT + 14 * SIZEOF_INT]
pinsrw X1, word [BLOCK + T1 * 2], 6
%%.ELOAD15:
%endmacro
%macro LOAD8 0
pxor N0, N0
mov T0d, INT [LUT + 0 * SIZEOF_INT]
pinsrw X0, word [BLOCK + T0 * 2], 0
mov T0d, INT [LUT + 1 * SIZEOF_INT]
pinsrw X0, word [BLOCK + T0 * 2], 1
mov T0d, INT [LUT + 2 * SIZEOF_INT]
pinsrw X0, word [BLOCK + T0 * 2], 2
mov T0d, INT [LUT + 3 * SIZEOF_INT]
pinsrw X0, word [BLOCK + T0 * 2], 3
mov T0d, INT [LUT + 4 * SIZEOF_INT]
pinsrw X0, word [BLOCK + T0 * 2], 4
mov T0d, INT [LUT + 5 * SIZEOF_INT]
pinsrw X0, word [BLOCK + T0 * 2], 5
mov T0d, INT [LUT + 6 * SIZEOF_INT]
pinsrw X0, word [BLOCK + T0 * 2], 6
mov T0d, INT [LUT + 7 * SIZEOF_INT]
pinsrw X0, word [BLOCK + T0 * 2], 7
%endmacro
%macro LOAD7 0
pxor N0, N0
pxor X0, X0
mov T1d, INT [LUT + 0 * SIZEOF_INT]
pinsrw X0, word [BLOCK + T1 * 2], 0
cmp LENEND, 2
jl %%.ELOAD7
mov T1d, INT [LUT + 1 * SIZEOF_INT]
pinsrw X0, word [BLOCK + T1 * 2], 1
cmp LENEND, 3
jl %%.ELOAD7
mov T1d, INT [LUT + 2 * SIZEOF_INT]
pinsrw X0, word [BLOCK + T1 * 2], 2
cmp LENEND, 4
jl %%.ELOAD7
mov T1d, INT [LUT + 3 * SIZEOF_INT]
pinsrw X0, word [BLOCK + T1 * 2], 3
cmp LENEND, 5
jl %%.ELOAD7
mov T1d, INT [LUT + 4 * SIZEOF_INT]
pinsrw X0, word [BLOCK + T1 * 2], 4
cmp LENEND, 6
jl %%.ELOAD7
mov T1d, INT [LUT + 5 * SIZEOF_INT]
pinsrw X0, word [BLOCK + T1 * 2], 5
cmp LENEND, 7
jl %%.ELOAD7
mov T1d, INT [LUT + 6 * SIZEOF_INT]
pinsrw X0, word [BLOCK + T1 * 2], 6
%%.ELOAD7:
%endmacro
%macro REDUCE0 0
movdqa xmm0, XMMWORD [VALUES + ( 0 * 2)]
movdqa xmm1, XMMWORD [VALUES + ( 8 * 2)]
movdqa xmm2, XMMWORD [VALUES + (16 * 2)]
movdqa xmm3, XMMWORD [VALUES + (24 * 2)]
movdqa xmm4, XMMWORD [VALUES + (32 * 2)]
movdqa xmm5, XMMWORD [VALUES + (40 * 2)]
movdqa xmm6, XMMWORD [VALUES + (48 * 2)]
movdqa xmm7, XMMWORD [VALUES + (56 * 2)]
pcmpeqw xmm0, ZERO
pcmpeqw xmm1, ZERO
pcmpeqw xmm2, ZERO
pcmpeqw xmm3, ZERO
pcmpeqw xmm4, ZERO
pcmpeqw xmm5, ZERO
pcmpeqw xmm6, ZERO
pcmpeqw xmm7, ZERO
packsswb xmm0, xmm1
packsswb xmm2, xmm3
packsswb xmm4, xmm5
packsswb xmm6, xmm7
pmovmskb eax, xmm0
pmovmskb ecx, xmm2
pmovmskb edx, xmm4
pmovmskb esi, xmm6
shl rcx, 16
shl rdx, 32
shl rsi, 48
or rax, rcx
or rdx, rsi
or rax, rdx
not rax
mov MMWORD [r15], rax
%endmacro
; Prepare data for jsimd_encode_mcu_AC_first().
;
; GLOBAL(void)
; jsimd_encode_mcu_AC_first_prepare_sse2(const JCOEF *block,
; const int *jpeg_natural_order_start,
; int Sl, int Al, JCOEF *values,
; size_t *zerobits)
;
; r10 = const JCOEF *block
; r11 = const int *jpeg_natural_order_start
; r12 = int Sl
; r13 = int Al
; r14 = JCOEF *values
; r15 = size_t *zerobits
%define ZERO xmm9
%define X0 xmm0
%define X1 xmm1
%define N0 xmm2
%define N1 xmm3
%define AL xmm4
%define K eax
%define LUT r11
%define T0 rcx
%define T0d ecx
%define T1 rdx
%define T1d edx
%define BLOCK r10
%define VALUES r14
%define LEN r12d
%define LENEND r13d
align 32
GLOBAL_FUNCTION(jsimd_encode_mcu_AC_first_prepare_sse2)
EXTN(jsimd_encode_mcu_AC_first_prepare_sse2):
ENDBR64
push rbp
mov rbp, rsp
and rsp, byte (-SIZEOF_XMMWORD) ; align to 128 bits
sub rsp, SIZEOF_XMMWORD
movdqa XMMWORD [rsp], ZERO
COLLECT_ARGS 6
movd AL, r13d
pxor ZERO, ZERO
mov K, LEN
mov LENEND, LEN
and K, -16
and LENEND, 7
shr K, 4
jz .ELOOP16
.BLOOP16:
LOAD16
pcmpgtw N0, X0
pcmpgtw N1, X1
paddw X0, N0
paddw X1, N1
pxor X0, N0
pxor X1, N1
psrlw X0, AL
psrlw X1, AL
pxor N0, X0
pxor N1, X1
movdqa XMMWORD [VALUES + (0) * 2], X0
movdqa XMMWORD [VALUES + (8) * 2], X1
movdqa XMMWORD [VALUES + (0 + DCTSIZE2) * 2], N0
movdqa XMMWORD [VALUES + (8 + DCTSIZE2) * 2], N1
add VALUES, 16 * 2
add LUT, 16 * SIZEOF_INT
dec K
jnz .BLOOP16
test LEN, 15
je .PADDING
.ELOOP16:
test LEN, 8
jz .TRY7
test LEN, 7
jz .TRY8
LOAD15
pcmpgtw N0, X0
pcmpgtw N1, X1
paddw X0, N0
paddw X1, N1
pxor X0, N0
pxor X1, N1
psrlw X0, AL
psrlw X1, AL
pxor N0, X0
pxor N1, X1
movdqa XMMWORD [VALUES + (0) * 2], X0
movdqa XMMWORD [VALUES + (8) * 2], X1
movdqa XMMWORD [VALUES + (0 + DCTSIZE2) * 2], N0
movdqa XMMWORD [VALUES + (8 + DCTSIZE2) * 2], N1
add VALUES, 16 * 2
jmp .PADDING
.TRY8:
LOAD8
pcmpgtw N0, X0
paddw X0, N0
pxor X0, N0
psrlw X0, AL
pxor N0, X0
movdqa XMMWORD [VALUES + (0) * 2], X0
movdqa XMMWORD [VALUES + (0 + DCTSIZE2) * 2], N0
add VALUES, 8 * 2
jmp .PADDING
.TRY7:
LOAD7
pcmpgtw N0, X0
paddw X0, N0
pxor X0, N0
psrlw X0, AL
pxor N0, X0
movdqa XMMWORD [VALUES + (0) * 2], X0
movdqa XMMWORD [VALUES + (0 + DCTSIZE2) * 2], N0
add VALUES, 8 * 2
.PADDING:
mov K, LEN
add K, 7
and K, -8
shr K, 3
sub K, DCTSIZE2 / 8
jz .EPADDING
align 16
.ZEROLOOP:
movdqa XMMWORD [VALUES + 0], ZERO
add VALUES, 8 * 2
inc K
jnz .ZEROLOOP
.EPADDING:
sub VALUES, DCTSIZE2 * 2
REDUCE0
UNCOLLECT_ARGS 6
movdqa ZERO, XMMWORD [rsp]
mov rsp, rbp
pop rbp
ret
%undef ZERO
%undef X0
%undef X1
%undef N0
%undef N1
%undef AL
%undef K
%undef LUT
%undef T0
%undef T0d
%undef T1
%undef T1d
%undef BLOCK
%undef VALUES
%undef LEN
%undef LENEND
; Prepare data for jsimd_encode_mcu_AC_refine().
;
; GLOBAL(int)
; jsimd_encode_mcu_AC_refine_prepare_sse2(const JCOEF *block,
; const int *jpeg_natural_order_start,
; int Sl, int Al, JCOEF *absvalues,
; size_t *bits)
;
; r10 = const JCOEF *block
; r11 = const int *jpeg_natural_order_start
; r12 = int Sl
; r13 = int Al
; r14 = JCOEF *values
; r15 = size_t *bits
%define ZERO xmm9
%define ONE xmm5
%define X0 xmm0
%define X1 xmm1
%define N0 xmm2
%define N1 xmm3
%define AL xmm4
%define K eax
%define KK r9d
%define EOB r8d
%define SIGN rdi
%define LUT r11
%define T0 rcx
%define T0d ecx
%define T1 rdx
%define T1d edx
%define BLOCK r10
%define VALUES r14
%define LEN r12d
%define LENEND r13d
align 32
GLOBAL_FUNCTION(jsimd_encode_mcu_AC_refine_prepare_sse2)
EXTN(jsimd_encode_mcu_AC_refine_prepare_sse2):
ENDBR64
push rbp
mov rbp, rsp
and rsp, byte (-SIZEOF_XMMWORD) ; align to 128 bits
sub rsp, SIZEOF_XMMWORD
movdqa XMMWORD [rsp], ZERO
COLLECT_ARGS 6
xor SIGN, SIGN
xor EOB, EOB
xor KK, KK
movd AL, r13d
pxor ZERO, ZERO
pcmpeqw ONE, ONE
psrlw ONE, 15
mov K, LEN
mov LENEND, LEN
and K, -16
and LENEND, 7
shr K, 4
jz .ELOOPR16
.BLOOPR16:
LOAD16
pcmpgtw N0, X0
pcmpgtw N1, X1
paddw X0, N0
paddw X1, N1
pxor X0, N0
pxor X1, N1
psrlw X0, AL
psrlw X1, AL
movdqa XMMWORD [VALUES + (0) * 2], X0
movdqa XMMWORD [VALUES + (8) * 2], X1
pcmpeqw X0, ONE
pcmpeqw X1, ONE
packsswb N0, N1
packsswb X0, X1
pmovmskb T0d, N0 ; lsignbits.val16u[k >> 4] = _mm_movemask_epi8(neg);
pmovmskb T1d, X0 ; idx = _mm_movemask_epi8(x1);
shr SIGN, 16 ; make room for sizebits
shl T0, 48
or SIGN, T0
bsr T1d, T1d ; idx = 16 - (__builtin_clz(idx) >> 1);
jz .CONTINUER16 ; if (idx) {
mov EOB, KK
add EOB, T1d ; EOB = k + idx;
.CONTINUER16:
add VALUES, 16 * 2
add LUT, 16 * SIZEOF_INT
add KK, 16
dec K
jnz .BLOOPR16
test LEN, 15
je .PADDINGR
.ELOOPR16:
test LEN, 8
jz .TRYR7
test LEN, 7
jz .TRYR8
LOAD15
pcmpgtw N0, X0
pcmpgtw N1, X1
paddw X0, N0
paddw X1, N1
pxor X0, N0
pxor X1, N1
psrlw X0, AL
psrlw X1, AL
movdqa XMMWORD [VALUES + (0) * 2], X0
movdqa XMMWORD [VALUES + (8) * 2], X1
pcmpeqw X0, ONE
pcmpeqw X1, ONE
packsswb N0, N1
packsswb X0, X1
pmovmskb T0d, N0 ; lsignbits.val16u[k >> 4] = _mm_movemask_epi8(neg);
pmovmskb T1d, X0 ; idx = _mm_movemask_epi8(x1);
shr SIGN, 16 ; make room for sizebits
shl T0, 48
or SIGN, T0
bsr T1d, T1d ; idx = 16 - (__builtin_clz(idx) >> 1);
jz .CONTINUER15 ; if (idx) {
mov EOB, KK
add EOB, T1d ; EOB = k + idx;
.CONTINUER15:
add VALUES, 16 * 2
jmp .PADDINGR
.TRYR8:
LOAD8
pcmpgtw N0, X0
paddw X0, N0
pxor X0, N0
psrlw X0, AL
movdqa XMMWORD [VALUES + (0) * 2], X0
pcmpeqw X0, ONE
packsswb N0, ZERO
packsswb X0, ZERO
pmovmskb T0d, N0 ; lsignbits.val16u[k >> 4] = _mm_movemask_epi8(neg);
pmovmskb T1d, X0 ; idx = _mm_movemask_epi8(x1);
shr SIGN, 8 ; make room for sizebits
shl T0, 56
or SIGN, T0
bsr T1d, T1d ; idx = 16 - (__builtin_clz(idx) >> 1);
jz .CONTINUER8 ; if (idx) {
mov EOB, KK
add EOB, T1d ; EOB = k + idx;
.CONTINUER8:
add VALUES, 8 * 2
jmp .PADDINGR
.TRYR7:
LOAD7
pcmpgtw N0, X0
paddw X0, N0
pxor X0, N0
psrlw X0, AL
movdqa XMMWORD [VALUES + (0) * 2], X0
pcmpeqw X0, ONE
packsswb N0, ZERO
packsswb X0, ZERO
pmovmskb T0d, N0 ; lsignbits.val16u[k >> 4] = _mm_movemask_epi8(neg);
pmovmskb T1d, X0 ; idx = _mm_movemask_epi8(x1);
shr SIGN, 8 ; make room for sizebits
shl T0, 56
or SIGN, T0
bsr T1d, T1d ; idx = 16 - (__builtin_clz(idx) >> 1);
jz .CONTINUER7 ; if (idx) {
mov EOB, KK
add EOB, T1d ; EOB = k + idx;
.CONTINUER7:
add VALUES, 8 * 2
.PADDINGR:
mov K, LEN
add K, 7
and K, -8
shr K, 3
sub K, DCTSIZE2 / 8
jz .EPADDINGR
align 16
.ZEROLOOPR:
movdqa XMMWORD [VALUES + 0], ZERO
shr SIGN, 8
add VALUES, 8 * 2
inc K
jnz .ZEROLOOPR
.EPADDINGR:
not SIGN
sub VALUES, DCTSIZE2 * 2
mov MMWORD [r15 + SIZEOF_MMWORD], SIGN
REDUCE0
mov eax, EOB
UNCOLLECT_ARGS 6
movdqa ZERO, XMMWORD [rsp]
mov rsp, rbp
pop rbp
ret
%undef ZERO
%undef ONE
%undef X0
%undef X1
%undef N0
%undef N1
%undef AL
%undef K
%undef KK
%undef EOB
%undef SIGN
%undef LUT
%undef T0
%undef T0d
%undef T1
%undef T1d
%undef BLOCK
%undef VALUES
%undef LEN
%undef LENEND
; For some reason, the OS X linker does not honor the request to align the
; segment unless we do this.
align 32
lib/libjpeg-turbo/simd/x86_64/jdcolor-avx2.asm
Executable
+114
View File
@@ -0,0 +1,114 @@
;
; Colorspace conversion (64-bit AVX2)
;
; Copyright 2009 Pierre Ossman <[email protected]> for Cendio AB
; Copyright (C) 2009, 2016, 2024, D. R. Commander.
; Copyright (C) 2015, Intel Corporation.
;
; Based on the x86 SIMD extension for IJG JPEG library
; Copyright (C) 1999-2006, MIYASAKA Masaru.
; For conditions of distribution and use, see copyright notice in jsimdext.inc
;
; This file should be assembled with NASM (Netwide Assembler) or Yasm.
%include "jsimdext.inc"
; --------------------------------------------------------------------------
%define SCALEBITS 16
F_0_344 equ 22554 ; FIX(0.34414)
F_0_714 equ 46802 ; FIX(0.71414)
F_1_402 equ 91881 ; FIX(1.40200)
F_1_772 equ 116130 ; FIX(1.77200)
F_0_402 equ (F_1_402 - 65536) ; FIX(1.40200) - FIX(1)
F_0_285 equ ( 65536 - F_0_714) ; FIX(1) - FIX(0.71414)
F_0_228 equ (131072 - F_1_772) ; FIX(2) - FIX(1.77200)
; --------------------------------------------------------------------------
SECTION SEG_CONST
ALIGNZ 32
GLOBAL_DATA(jconst_ycc_rgb_convert_avx2)
EXTN(jconst_ycc_rgb_convert_avx2):
PW_F0402 times 16 dw F_0_402
PW_MF0228 times 16 dw -F_0_228
PW_MF0344_F0285 times 8 dw -F_0_344, F_0_285
PW_ONE times 16 dw 1
PD_ONEHALF times 8 dd 1 << (SCALEBITS - 1)
ALIGNZ 32
; --------------------------------------------------------------------------
SECTION SEG_TEXT
BITS 64
%include "jdcolext-avx2.asm"
%undef RGB_RED
%undef RGB_GREEN
%undef RGB_BLUE
%undef RGB_PIXELSIZE
%define RGB_RED EXT_RGB_RED
%define RGB_GREEN EXT_RGB_GREEN
%define RGB_BLUE EXT_RGB_BLUE
%define RGB_PIXELSIZE EXT_RGB_PIXELSIZE
%define jsimd_ycc_rgb_convert_avx2 jsimd_ycc_extrgb_convert_avx2
%include "jdcolext-avx2.asm"
%undef RGB_RED
%undef RGB_GREEN
%undef RGB_BLUE
%undef RGB_PIXELSIZE
%define RGB_RED EXT_RGBX_RED
%define RGB_GREEN EXT_RGBX_GREEN
%define RGB_BLUE EXT_RGBX_BLUE
%define RGB_PIXELSIZE EXT_RGBX_PIXELSIZE
%define jsimd_ycc_rgb_convert_avx2 jsimd_ycc_extrgbx_convert_avx2
%include "jdcolext-avx2.asm"
%undef RGB_RED
%undef RGB_GREEN
%undef RGB_BLUE
%undef RGB_PIXELSIZE
%define RGB_RED EXT_BGR_RED
%define RGB_GREEN EXT_BGR_GREEN
%define RGB_BLUE EXT_BGR_BLUE
%define RGB_PIXELSIZE EXT_BGR_PIXELSIZE
%define jsimd_ycc_rgb_convert_avx2 jsimd_ycc_extbgr_convert_avx2
%include "jdcolext-avx2.asm"
%undef RGB_RED
%undef RGB_GREEN
%undef RGB_BLUE
%undef RGB_PIXELSIZE
%define RGB_RED EXT_BGRX_RED
%define RGB_GREEN EXT_BGRX_GREEN
%define RGB_BLUE EXT_BGRX_BLUE
%define RGB_PIXELSIZE EXT_BGRX_PIXELSIZE
%define jsimd_ycc_rgb_convert_avx2 jsimd_ycc_extbgrx_convert_avx2
%include "jdcolext-avx2.asm"
%undef RGB_RED
%undef RGB_GREEN
%undef RGB_BLUE
%undef RGB_PIXELSIZE
%define RGB_RED EXT_XBGR_RED
%define RGB_GREEN EXT_XBGR_GREEN
%define RGB_BLUE EXT_XBGR_BLUE
%define RGB_PIXELSIZE EXT_XBGR_PIXELSIZE
%define jsimd_ycc_rgb_convert_avx2 jsimd_ycc_extxbgr_convert_avx2
%include "jdcolext-avx2.asm"
%undef RGB_RED
%undef RGB_GREEN
%undef RGB_BLUE
%undef RGB_PIXELSIZE
%define RGB_RED EXT_XRGB_RED
%define RGB_GREEN EXT_XRGB_GREEN
%define RGB_BLUE EXT_XRGB_BLUE
%define RGB_PIXELSIZE EXT_XRGB_PIXELSIZE
%define jsimd_ycc_rgb_convert_avx2 jsimd_ycc_extxrgb_convert_avx2
%include "jdcolext-avx2.asm"
lib/libjpeg-turbo/simd/x86_64/jdcolor-sse2.asm
Executable
+113
View File
@@ -0,0 +1,113 @@
;
; Colorspace conversion (64-bit SSE2)
;
; Copyright 2009 Pierre Ossman <[email protected]> for Cendio AB
; Copyright (C) 2009, 2016, 2024, D. R. Commander.
;
; Based on the x86 SIMD extension for IJG JPEG library
; Copyright (C) 1999-2006, MIYASAKA Masaru.
; For conditions of distribution and use, see copyright notice in jsimdext.inc
;
; This file should be assembled with NASM (Netwide Assembler) or Yasm.
%include "jsimdext.inc"
; --------------------------------------------------------------------------
%define SCALEBITS 16
F_0_344 equ 22554 ; FIX(0.34414)
F_0_714 equ 46802 ; FIX(0.71414)
F_1_402 equ 91881 ; FIX(1.40200)
F_1_772 equ 116130 ; FIX(1.77200)
F_0_402 equ (F_1_402 - 65536) ; FIX(1.40200) - FIX(1)
F_0_285 equ ( 65536 - F_0_714) ; FIX(1) - FIX(0.71414)
F_0_228 equ (131072 - F_1_772) ; FIX(2) - FIX(1.77200)
; --------------------------------------------------------------------------
SECTION SEG_CONST
ALIGNZ 32
GLOBAL_DATA(jconst_ycc_rgb_convert_sse2)
EXTN(jconst_ycc_rgb_convert_sse2):
PW_F0402 times 8 dw F_0_402
PW_MF0228 times 8 dw -F_0_228
PW_MF0344_F0285 times 4 dw -F_0_344, F_0_285
PW_ONE times 8 dw 1
PD_ONEHALF times 4 dd 1 << (SCALEBITS - 1)
ALIGNZ 32
; --------------------------------------------------------------------------
SECTION SEG_TEXT
BITS 64
%include "jdcolext-sse2.asm"
%undef RGB_RED
%undef RGB_GREEN
%undef RGB_BLUE
%undef RGB_PIXELSIZE
%define RGB_RED EXT_RGB_RED
%define RGB_GREEN EXT_RGB_GREEN
%define RGB_BLUE EXT_RGB_BLUE
%define RGB_PIXELSIZE EXT_RGB_PIXELSIZE
%define jsimd_ycc_rgb_convert_sse2 jsimd_ycc_extrgb_convert_sse2
%include "jdcolext-sse2.asm"
%undef RGB_RED
%undef RGB_GREEN
%undef RGB_BLUE
%undef RGB_PIXELSIZE
%define RGB_RED EXT_RGBX_RED
%define RGB_GREEN EXT_RGBX_GREEN
%define RGB_BLUE EXT_RGBX_BLUE
%define RGB_PIXELSIZE EXT_RGBX_PIXELSIZE
%define jsimd_ycc_rgb_convert_sse2 jsimd_ycc_extrgbx_convert_sse2
%include "jdcolext-sse2.asm"
%undef RGB_RED
%undef RGB_GREEN
%undef RGB_BLUE
%undef RGB_PIXELSIZE
%define RGB_RED EXT_BGR_RED
%define RGB_GREEN EXT_BGR_GREEN
%define RGB_BLUE EXT_BGR_BLUE
%define RGB_PIXELSIZE EXT_BGR_PIXELSIZE
%define jsimd_ycc_rgb_convert_sse2 jsimd_ycc_extbgr_convert_sse2
%include "jdcolext-sse2.asm"
%undef RGB_RED
%undef RGB_GREEN
%undef RGB_BLUE
%undef RGB_PIXELSIZE
%define RGB_RED EXT_BGRX_RED
%define RGB_GREEN EXT_BGRX_GREEN
%define RGB_BLUE EXT_BGRX_BLUE
%define RGB_PIXELSIZE EXT_BGRX_PIXELSIZE
%define jsimd_ycc_rgb_convert_sse2 jsimd_ycc_extbgrx_convert_sse2
%include "jdcolext-sse2.asm"
%undef RGB_RED
%undef RGB_GREEN
%undef RGB_BLUE
%undef RGB_PIXELSIZE
%define RGB_RED EXT_XBGR_RED
%define RGB_GREEN EXT_XBGR_GREEN
%define RGB_BLUE EXT_XBGR_BLUE
%define RGB_PIXELSIZE EXT_XBGR_PIXELSIZE
%define jsimd_ycc_rgb_convert_sse2 jsimd_ycc_extxbgr_convert_sse2
%include "jdcolext-sse2.asm"
%undef RGB_RED
%undef RGB_GREEN
%undef RGB_BLUE
%undef RGB_PIXELSIZE
%define RGB_RED EXT_XRGB_RED
%define RGB_GREEN EXT_XRGB_GREEN
%define RGB_BLUE EXT_XRGB_BLUE
%define RGB_PIXELSIZE EXT_XRGB_PIXELSIZE
%define jsimd_ycc_rgb_convert_sse2 jsimd_ycc_extxrgb_convert_sse2
%include "jdcolext-sse2.asm"
lib/libjpeg-turbo/simd/x86_64/jdmerge-avx2.asm
Executable
+132
View File
@@ -0,0 +1,132 @@
;
; Merged upsampling/color conversion (64-bit AVX2)
;
; Copyright 2009 Pierre Ossman <[email protected]> for Cendio AB
; Copyright (C) 2009, 2016, 2024, D. R. Commander.
; Copyright (C) 2015, Intel Corporation.
;
; Based on the x86 SIMD extension for IJG JPEG library
; Copyright (C) 1999-2006, MIYASAKA Masaru.
; For conditions of distribution and use, see copyright notice in jsimdext.inc
;
; This file should be assembled with NASM (Netwide Assembler) or Yasm.
%include "jsimdext.inc"
; --------------------------------------------------------------------------
%define SCALEBITS 16
F_0_344 equ 22554 ; FIX(0.34414)
F_0_714 equ 46802 ; FIX(0.71414)
F_1_402 equ 91881 ; FIX(1.40200)
F_1_772 equ 116130 ; FIX(1.77200)
F_0_402 equ (F_1_402 - 65536) ; FIX(1.40200) - FIX(1)
F_0_285 equ ( 65536 - F_0_714) ; FIX(1) - FIX(0.71414)
F_0_228 equ (131072 - F_1_772) ; FIX(2) - FIX(1.77200)
; --------------------------------------------------------------------------
SECTION SEG_CONST
ALIGNZ 32
GLOBAL_DATA(jconst_merged_upsample_avx2)
EXTN(jconst_merged_upsample_avx2):
PW_F0402 times 16 dw F_0_402
PW_MF0228 times 16 dw -F_0_228
PW_MF0344_F0285 times 8 dw -F_0_344, F_0_285
PW_ONE times 16 dw 1
PD_ONEHALF times 8 dd 1 << (SCALEBITS - 1)
ALIGNZ 32
; --------------------------------------------------------------------------
SECTION SEG_TEXT
BITS 64
%include "jdmrgext-avx2.asm"
%undef RGB_RED
%undef RGB_GREEN
%undef RGB_BLUE
%undef RGB_PIXELSIZE
%define RGB_RED EXT_RGB_RED
%define RGB_GREEN EXT_RGB_GREEN
%define RGB_BLUE EXT_RGB_BLUE
%define RGB_PIXELSIZE EXT_RGB_PIXELSIZE
%define jsimd_h2v1_merged_upsample_avx2 \
jsimd_h2v1_extrgb_merged_upsample_avx2
%define jsimd_h2v2_merged_upsample_avx2 \
jsimd_h2v2_extrgb_merged_upsample_avx2
%include "jdmrgext-avx2.asm"
%undef RGB_RED
%undef RGB_GREEN
%undef RGB_BLUE
%undef RGB_PIXELSIZE
%define RGB_RED EXT_RGBX_RED
%define RGB_GREEN EXT_RGBX_GREEN
%define RGB_BLUE EXT_RGBX_BLUE
%define RGB_PIXELSIZE EXT_RGBX_PIXELSIZE
%define jsimd_h2v1_merged_upsample_avx2 \
jsimd_h2v1_extrgbx_merged_upsample_avx2
%define jsimd_h2v2_merged_upsample_avx2 \
jsimd_h2v2_extrgbx_merged_upsample_avx2
%include "jdmrgext-avx2.asm"
%undef RGB_RED
%undef RGB_GREEN
%undef RGB_BLUE
%undef RGB_PIXELSIZE
%define RGB_RED EXT_BGR_RED
%define RGB_GREEN EXT_BGR_GREEN
%define RGB_BLUE EXT_BGR_BLUE
%define RGB_PIXELSIZE EXT_BGR_PIXELSIZE
%define jsimd_h2v1_merged_upsample_avx2 \
jsimd_h2v1_extbgr_merged_upsample_avx2
%define jsimd_h2v2_merged_upsample_avx2 \
jsimd_h2v2_extbgr_merged_upsample_avx2
%include "jdmrgext-avx2.asm"
%undef RGB_RED
%undef RGB_GREEN
%undef RGB_BLUE
%undef RGB_PIXELSIZE
%define RGB_RED EXT_BGRX_RED
%define RGB_GREEN EXT_BGRX_GREEN
%define RGB_BLUE EXT_BGRX_BLUE
%define RGB_PIXELSIZE EXT_BGRX_PIXELSIZE
%define jsimd_h2v1_merged_upsample_avx2 \
jsimd_h2v1_extbgrx_merged_upsample_avx2
%define jsimd_h2v2_merged_upsample_avx2 \
jsimd_h2v2_extbgrx_merged_upsample_avx2
%include "jdmrgext-avx2.asm"
%undef RGB_RED
%undef RGB_GREEN
%undef RGB_BLUE
%undef RGB_PIXELSIZE
%define RGB_RED EXT_XBGR_RED
%define RGB_GREEN EXT_XBGR_GREEN
%define RGB_BLUE EXT_XBGR_BLUE
%define RGB_PIXELSIZE EXT_XBGR_PIXELSIZE
%define jsimd_h2v1_merged_upsample_avx2 \
jsimd_h2v1_extxbgr_merged_upsample_avx2
%define jsimd_h2v2_merged_upsample_avx2 \
jsimd_h2v2_extxbgr_merged_upsample_avx2
%include "jdmrgext-avx2.asm"
%undef RGB_RED
%undef RGB_GREEN
%undef RGB_BLUE
%undef RGB_PIXELSIZE
%define RGB_RED EXT_XRGB_RED
%define RGB_GREEN EXT_XRGB_GREEN
%define RGB_BLUE EXT_XRGB_BLUE
%define RGB_PIXELSIZE EXT_XRGB_PIXELSIZE
%define jsimd_h2v1_merged_upsample_avx2 \
jsimd_h2v1_extxrgb_merged_upsample_avx2
%define jsimd_h2v2_merged_upsample_avx2 \
jsimd_h2v2_extxrgb_merged_upsample_avx2
%include "jdmrgext-avx2.asm"
lib/libjpeg-turbo/simd/x86_64/jdmrgext-avx2.asm
Executable
+658
View File
@@ -0,0 +1,658 @@
;
; Merged upsampling/color conversion (64-bit AVX2)
;
; Copyright 2009, 2012 Pierre Ossman <[email protected]> for Cendio AB
; Copyright (C) 2009, 2012, 2016, 2024-2025, D. R. Commander.
; Copyright (C) 2015, Intel Corporation.
; Copyright (C) 2018, Matthias Räncker.
; Copyright (C) 2023, Aliaksiej Kandracienka.
;
; Based on the x86 SIMD extension for IJG JPEG library
; Copyright (C) 1999-2006, MIYASAKA Masaru.
; For conditions of distribution and use, see copyright notice in jsimdext.inc
;
; This file should be assembled with NASM (Netwide Assembler) or Yasm.
%include "jcolsamp.inc"
; --------------------------------------------------------------------------
;
; Upsample and color convert for the case of 2:1 horizontal and 1:1 vertical.
;
; GLOBAL(void)
; jsimd_h2v1_merged_upsample_avx2(JDIMENSION output_width,
; JSAMPIMAGE input_buf,
; JDIMENSION in_row_group_ctr,
; JSAMPARRAY output_buf)
;
; r10d = JDIMENSION output_width
; r11 = JSAMPIMAGE input_buf
; r12d = JDIMENSION in_row_group_ctr
; r13 = JSAMPARRAY output_buf
%define wk(i) r15 - (WK_NUM - (i)) * SIZEOF_YMMWORD ; ymmword wk[WK_NUM]
%define WK_NUM 3
align 32
GLOBAL_FUNCTION(jsimd_h2v1_merged_upsample_avx2)
EXTN(jsimd_h2v1_merged_upsample_avx2):
ENDBR64
push rbp
mov rbp, rsp
push r15
and rsp, byte (-SIZEOF_YMMWORD) ; align to 256 bits
; Allocate stack space for wk array. r15 is used to access it.
mov r15, rsp
sub rsp, SIZEOF_YMMWORD * WK_NUM
COLLECT_ARGS 4
push rbx
mov ecx, r10d ; col
test rcx, rcx
jz near .return
push rcx
mov rdi, r11
mov ecx, r12d
mov rsip, JSAMPARRAY [rdi + 0 * SIZEOF_JSAMPARRAY]
mov rbxp, JSAMPARRAY [rdi + 1 * SIZEOF_JSAMPARRAY]
mov rdxp, JSAMPARRAY [rdi + 2 * SIZEOF_JSAMPARRAY]
mov rdi, r13
mov rsip, JSAMPROW [rsi + rcx * SIZEOF_JSAMPROW] ; inptr0
mov rbxp, JSAMPROW [rbx + rcx * SIZEOF_JSAMPROW] ; inptr1
mov rdxp, JSAMPROW [rdx + rcx * SIZEOF_JSAMPROW] ; inptr2
mov rdip, JSAMPROW [rdi] ; outptr
pop rcx ; col
.columnloop:
vmovdqu ymm6, YMMWORD [rbx]
; ymm6 = Cb(0123456789abcdefghijklmnopqrstuv)
vmovdqu ymm7, YMMWORD [rdx]
; ymm7 = Cr(0123456789abcdefghijklmnopqrstuv)
vpxor ymm1, ymm1, ymm1 ; ymm1 = (all 0's)
vpcmpeqw ymm3, ymm3, ymm3
vpsllw ymm3, ymm3, 7 ; ymm3 = { 0xFF80 0xFF80 0xFF80 0xFF80 .. }
vpermq ymm6, ymm6, 0xd8 ; ymm6 = Cb(01234567ghijklmn89abcdefopqrstuv)
vpermq ymm7, ymm7, 0xd8 ; ymm7 = Cr(01234567ghijklmn89abcdefopqrstuv)
vpunpcklbw ymm4, ymm6, ymm1 ; ymm4 = Cb(0123456789abcdef) = CbL
vpunpckhbw ymm6, ymm6, ymm1 ; ymm6 = Cb(ghijklmnopqrstuv) = CbH
vpunpcklbw ymm0, ymm7, ymm1 ; ymm0 = Cr(0123456789abcdef) = CrL
vpunpckhbw ymm7, ymm7, ymm1 ; ymm7 = Cr(ghijklmnopqrstuv) = CrH
vpaddw ymm5, ymm6, ymm3
vpaddw ymm2, ymm4, ymm3
vpaddw ymm1, ymm7, ymm3
vpaddw ymm3, ymm0, ymm3
; (Original)
; R = Y + 1.40200 * Cr
; G = Y - 0.34414 * Cb - 0.71414 * Cr
; B = Y + 1.77200 * Cb
;
; (This implementation)
; R = Y + 0.40200 * Cr + Cr
; G = Y - 0.34414 * Cb + 0.28586 * Cr - Cr
; B = Y - 0.22800 * Cb + Cb + Cb
vpaddw ymm6, ymm5, ymm5 ; ymm6 = 2 * CbH
vpaddw ymm4, ymm2, ymm2 ; ymm4 = 2 * CbL
vpaddw ymm7, ymm1, ymm1 ; ymm7 = 2 * CrH
vpaddw ymm0, ymm3, ymm3 ; ymm0 = 2 * CrL
vpmulhw ymm6, ymm6, [rel PW_MF0228] ; ymm6 = (2 * CbH * -FIX(0.22800))
vpmulhw ymm4, ymm4, [rel PW_MF0228] ; ymm4 = (2 * CbL * -FIX(0.22800))
vpmulhw ymm7, ymm7, [rel PW_F0402] ; ymm7 = (2 * CrH * FIX(0.40200))
vpmulhw ymm0, ymm0, [rel PW_F0402] ; ymm0 = (2 * CrL * FIX(0.40200))
vpaddw ymm6, ymm6, [rel PW_ONE]
vpaddw ymm4, ymm4, [rel PW_ONE]
vpsraw ymm6, ymm6, 1 ; ymm6 = (CbH * -FIX(0.22800))
vpsraw ymm4, ymm4, 1 ; ymm4 = (CbL * -FIX(0.22800))
vpaddw ymm7, ymm7, [rel PW_ONE]
vpaddw ymm0, ymm0, [rel PW_ONE]
vpsraw ymm7, ymm7, 1 ; ymm7 = (CrH * FIX(0.40200))
vpsraw ymm0, ymm0, 1 ; ymm0 = (CrL * FIX(0.40200))
vpaddw ymm6, ymm6, ymm5
vpaddw ymm4, ymm4, ymm2
vpaddw ymm6, ymm6, ymm5 ; ymm6 = (CbH * FIX(1.77200)) = (B - Y)H
vpaddw ymm4, ymm4, ymm2 ; ymm4 = (CbL * FIX(1.77200)) = (B - Y)L
vpaddw ymm7, ymm7, ymm1 ; ymm7 = (CrH * FIX(1.40200)) = (R - Y)H
vpaddw ymm0, ymm0, ymm3 ; ymm0 = (CrL * FIX(1.40200)) = (R - Y)L
vmovdqa YMMWORD [wk(0)], ymm6 ; wk(0) = (B - Y)H
vmovdqa YMMWORD [wk(1)], ymm7 ; wk(1) = (R - Y)H
vpunpckhwd ymm6, ymm5, ymm1
vpunpcklwd ymm5, ymm5, ymm1
vpmaddwd ymm5, ymm5, [rel PW_MF0344_F0285]
vpmaddwd ymm6, ymm6, [rel PW_MF0344_F0285]
vpunpckhwd ymm7, ymm2, ymm3
vpunpcklwd ymm2, ymm2, ymm3
vpmaddwd ymm2, ymm2, [rel PW_MF0344_F0285]
vpmaddwd ymm7, ymm7, [rel PW_MF0344_F0285]
vpaddd ymm5, ymm5, [rel PD_ONEHALF]
vpaddd ymm6, ymm6, [rel PD_ONEHALF]
vpsrad ymm5, ymm5, SCALEBITS
vpsrad ymm6, ymm6, SCALEBITS
vpaddd ymm2, ymm2, [rel PD_ONEHALF]
vpaddd ymm7, ymm7, [rel PD_ONEHALF]
vpsrad ymm2, ymm2, SCALEBITS
vpsrad ymm7, ymm7, SCALEBITS
vpackssdw ymm5, ymm5, ymm6
; ymm5 = CbH * -FIX(0.344) + CrH * FIX(0.285)
vpackssdw ymm2, ymm2, ymm7
; ymm2 = CbL * -FIX(0.344) + CrL * FIX(0.285)
vpsubw ymm5, ymm5, ymm1
; ymm5 = CbH * -FIX(0.344) + CrH * -FIX(0.714) = (G - Y)H
vpsubw ymm2, ymm2, ymm3
; ymm2 = CbL * -FIX(0.344) + CrL * -FIX(0.714) = (G - Y)L
vmovdqa YMMWORD [wk(2)], ymm5 ; wk(2) = (G - Y)H
mov al, 2 ; Yctr
jmp short .Yloop_1st
.Yloop_2nd:
vmovdqa ymm0, YMMWORD [wk(1)] ; ymm0 = (R - Y)H
vmovdqa ymm2, YMMWORD [wk(2)] ; ymm2 = (G - Y)H
vmovdqa ymm4, YMMWORD [wk(0)] ; ymm4 = (B - Y)H
.Yloop_1st:
vmovdqu ymm7, YMMWORD [rsi]
; ymm7 = Y(0123456789abcdefghijklmnopqrstuv)
vpcmpeqw ymm6, ymm6, ymm6
vpsrlw ymm6, ymm6, BYTE_BIT ; ymm6 = { 0xFF 0x00 0xFF 0x00 .. }
vpand ymm6, ymm6, ymm7 ; ymm6 = Y(02468acegikmoqsu) = YE
vpsrlw ymm7, ymm7, BYTE_BIT ; ymm7 = Y(13579bdfhjlnprtv) = YO
vmovdqa ymm1, ymm0 ; ymm1 = ymm0 = (R - Y)(L / H)
vmovdqa ymm3, ymm2 ; ymm3 = ymm2 = (G - Y)(L / H)
vmovdqa ymm5, ymm4 ; ymm5 = ymm4 = (B - Y)(L / H)
vpaddw ymm0, ymm0, ymm6
; ymm0 = ((R - Y) + YE) = RE = R(02468acegikmoqsu)
vpaddw ymm1, ymm1, ymm7
; ymm1 = ((R - Y) + YO) = RO = R(13579bdfhjlnprtv)
vpackuswb ymm0, ymm0, ymm0 ; ymm0 = R(02468ace********gikmoqsu********)
vpackuswb ymm1, ymm1, ymm1 ; ymm1 = R(13579bdf********hjlnprtv********)
vpaddw ymm2, ymm2, ymm6
; ymm2 = ((G - Y) + YE) = GE = G(02468acegikmoqsu)
vpaddw ymm3, ymm3, ymm7
; ymm3 = ((G - Y) + YO) = GO = G(13579bdfhjlnprtv)
vpackuswb ymm2, ymm2, ymm2 ; ymm2 = G(02468ace********gikmoqsu********)
vpackuswb ymm3, ymm3, ymm3 ; ymm3 = G(13579bdf********hjlnprtv********)
vpaddw ymm4, ymm4, ymm6
; ymm4 = ((B - Y) + YE) = BE = B(02468acegikmoqsu)
vpaddw ymm5, ymm5, ymm7
; ymm5 = ((B - Y) + YO) = BO = B(13579bdfhjlnprtv)
vpackuswb ymm4, ymm4, ymm4 ; ymm4 = B(02468ace********gikmoqsu********)
vpackuswb ymm5, ymm5, ymm5 ; ymm5 = B(13579bdf********hjlnprtv********)
%if RGB_PIXELSIZE == 3 ; ---------------
; NOTE: The values of RGB_RED, RGB_GREEN, and RGB_BLUE determine the
; mapping of components A, B, and C to red, green, and blue.
;
; ymmA = (A0 A2 A4 A6 A8 Aa Ac Ae Ag Ai Ak Am Ao Aq As Au) = AE
; ymmB = (A1 A3 A5 A7 A9 Ab Ad Af Ah Aj Al An Ap Ar At Av) = AO
; ymmC = (B0 B2 B4 B6 B8 Ba Bc Be Bg Bi Bk Bm Bo Bq Bs Bu) = BE
; ymmD = (B1 B3 B5 B7 B9 Bb Bd Bf Bh Bj Bl Bn Bp Br Bt Bv) = BO
; ymmE = (C0 C2 C4 C6 C8 Ca Cc Ce Cg Ci Ck Cm Co Cq Cs Cu) = CE
; ymmF = (C1 C3 C5 C7 C9 Cb Cd Cf Ch Cj Cl Cn Cp Cr Ct Cv) = CO
; ymmG = (** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **)
; ymmH = (** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **)
vpunpcklbw ymmA, ymmA, ymmC
; ymmA = (A0 B0 A2 B2 A4 B4 A6 B6 A8 B8 Aa Ba Ac Bc Ae Be
; Ag Bg Ai Bi Ak Bk Am Bm Ao Bo Aq Bq As Bs Au Bu)
vpunpcklbw ymmE, ymmE, ymmB
; ymmE = (C0 A1 C2 A3 C4 A5 C6 A7 C8 A9 Ca Ab Cc Ad Ce Af
; Cg Ah Ci Aj Ck Al Cm An Co Ap Cq Ar Cs At Cu Av)
vpunpcklbw ymmD, ymmD, ymmF
; ymmD = (B1 C1 B3 C3 B5 C5 B7 C7 B9 C9 Bb Cb Bd Cd Bf Cf
; Bh Ch Bj Cj Bl Cl Bn Cn Bp Cp Br Cr Bt Ct Bv Cv)
vpsrldq ymmH, ymmA, 2
; ymmH = (A2 B2 A4 B4 A6 B6 A8 B8 Aa Ba Ac Bc Ae Be Ag Bg
; Ai Bi Ak Bk Am Bm Ao Bo Aq Bq As Bs Au Bu -- --)
vpunpckhwd ymmG, ymmA, ymmE
; ymmG = (A8 B8 C8 A9 Aa Ba Ca Ab Ac Bc Cc Ad Ae Be Ce Af
; Ao Bo Co Ap Aq Bq Cq Ar As Bs Cs At Au Bu Cu Av)
vpunpcklwd ymmA, ymmA, ymmE
; ymmA = (A0 B0 C0 A1 A2 B2 C2 A3 A4 B4 C4 A5 A6 B6 C6 A7
; Ag Bg Cg Ah Ai Bi Ci Aj Ak Bk Ck Al Am Bm Cm An)
vpsrldq ymmE, ymmE, 2
; ymmE = (C2 A3 C4 A5 C6 A7 C8 A9 Ca Ab Cc Ad Ce Af Cg Ah
; Ci Aj Ck Al Cm An Co Ap Cq Ar Cs At Cu Av -- --)
vpsrldq ymmB, ymmD, 2
; ymmB = (B3 C3 B5 C5 B7 C7 B9 C9 Bb Cb Bd Cd Bf Cf Bh Ch
; Bj Cj Bl Cl Bn Cn Bp Cp Br Cr Bt Ct Bv Cv -- --)
vpunpckhwd ymmC, ymmD, ymmH
; ymmC = (B9 C9 Aa Ba Bb Cb Ac Bc Bd Cd Ae Be Bf Cf Ag Bg
; Bp Cp Aq Bq Br Cr As Bs Bt Ct Au Bu Bv Cv -- --)
vpunpcklwd ymmD, ymmD, ymmH
; ymmD = (B1 C1 A2 B2 B3 C3 A4 B4 B5 C5 A6 B6 B7 C7 A8 B8
; Bh Ch Ai Bi Bj Cj Ak Bk Bl Cl Am Bm Bn Cn Ao Bo)
vpunpckhwd ymmF, ymmE, ymmB
; ymmF = (Ca Ab Bb Cb Cc Ad Bd Cd Ce Af Bf Cf Cg Ah Bh Ch
; Cq Ar Br Cr Cs At Bt Ct Cu Av Bv Cv -- -- -- --)
vpunpcklwd ymmE, ymmE, ymmB
; ymmE = (C2 A3 B3 C3 C4 A5 B5 C5 C6 A7 B7 C7 C8 A9 B9 C9
; Ci Aj Bj Cj Ck Al Bl Cl Cm An Bn Cn Co Ap Bp Cp)
vpshufd ymmH, ymmA, 0x4E
; ymmH = (A4 B4 C4 A5 A6 B6 C6 A7 A0 B0 C0 A1 A2 B2 C2 A3
; Ak Bk Ck Al Am Bm Cm An Ag Bg Cg Ah Ai Bi Ci Aj)
vpunpckldq ymmA, ymmA, ymmD
; ymmA = (A0 B0 C0 A1 B1 C1 A2 B2 A2 B2 C2 A3 B3 C3 A4 B4
; Ag Bg Cg Ah Bh Ch Ai Bi Ai Bi Ci Aj Bj Cj Ak Bk)
vpunpckhdq ymmD, ymmD, ymmE
; ymmD = (B5 C5 A6 B6 C6 A7 B7 C7 B7 C7 A8 B8 C8 A9 B9 C9
; Bl Cl Am Bm Cm An Bn Cn Bn Cn Ao Bo Co Ap Bp Cp)
vpunpckldq ymmE, ymmE, ymmH
; ymmE = (C2 A3 B3 C3 A4 B4 C4 A5 C4 A5 B5 C5 A6 B6 C6 A7
; Ci Aj Bj Cj Ak Bk Ck Al Ck Al Bl Cl Am Bm Cm An)
vpshufd ymmH, ymmG, 0x4E
; ymmH = (Ac Bc Cc Ad Ae Be Ce Af A8 B8 C8 A9 Aa Ba Ca Ab
; As Bs Cs At Au Bu Cu Av Ao Bo Co Ap Aq Bq Cq Ar)
vpunpckldq ymmG, ymmG, ymmC
; ymmG = (A8 B8 C8 A9 B9 C9 Aa Ba Aa Ba Ca Ab Bb Cb Ac Bc
; Ao Bo Co Ap Bp Cp Aq Bq Aq Bq Cq Ar Br Cr As Bs)
vpunpckhdq ymmC, ymmC, ymmF
; ymmC = (Bd Cd Ae Be Ce Af Bf Cf Bf Cf Ag Bg Cg Ah Bh Ch
; Bt Ct Au Bu Cu Av Bv Cv Bv Cv -- -- -- -- -- --)
vpunpckldq ymmF, ymmF, ymmH
; ymmF = (Ca Ab Bb Cb Ac Bc Cc Ad Cc Ad Bd Cd Ae Be Ce Af
; Cq Ar Br Cr As Bs Cs At Cs At Bt Ct Au Bu Cu Av)
vpunpcklqdq ymmH, ymmA, ymmE
; ymmH = (A0 B0 C0 A1 B1 C1 A2 B2 C2 A3 B3 C3 A4 B4 C4 A5
; Ag Bg Cg Ah Bh Ch Ai Bi Ci Aj Bj Cj Ak Bk Ck Al)
vpunpcklqdq ymmG, ymmD, ymmG
; ymmG = (B5 C5 A6 B6 C6 A7 B7 C7 A8 B8 C8 A9 B9 C9 Aa Ba
; Bl Cl Am Bm Cm An Bn Cn Ao Bo Co Ap Bp Cp Aq Bq)
vpunpcklqdq ymmC, ymmF, ymmC
; ymmC = (Ca Ab Bb Cb Ac Bc Cc Ad Bd Cd Ae Be Ce Af Bf Cf
; Cq Ar Br Cr As Bs Cs At Bt Ct Au Bu Cu Av Bv Cv)
vperm2i128 ymmA, ymmH, ymmG, 0x20
; ymmA = (A0 B0 C0 A1 B1 C1 A2 B2 C2 A3 B3 C3 A4 B4 C4 A5
; B5 C5 A6 B6 C6 A7 B7 C7 A8 B8 C8 A9 B9 C9 Aa Ba)
vperm2i128 ymmD, ymmC, ymmH, 0x30
; ymmD = (Ca Ab Bb Cb Ac Bc Cc Ad Bd Cd Ae Be Ce Af Bf Cf
; Ag Bg Cg Ah Bh Ch Ai Bi Ci Aj Bj Cj Ak Bk Ck Al)
vperm2i128 ymmF, ymmG, ymmC, 0x31
; ymmF = (Bl Cl Am Bm Cm An Bn Cn Ao Bo Co Ap Bp Cp Aq Bq
; Cq Ar Br Cr As Bs Cs At Bt Ct Au Bu Cu Av Bv Cv)
cmp rcx, byte SIZEOF_YMMWORD
jb short .column_st64
test rdi, SIZEOF_YMMWORD - 1
jnz short .out1
; --(aligned)-------------------
vmovntdq YMMWORD [rdi + 0 * SIZEOF_YMMWORD], ymmA
vmovntdq YMMWORD [rdi + 1 * SIZEOF_YMMWORD], ymmD
vmovntdq YMMWORD [rdi + 2 * SIZEOF_YMMWORD], ymmF
jmp short .out0
.out1: ; --(unaligned)-----------------
vmovdqu YMMWORD [rdi + 0 * SIZEOF_YMMWORD], ymmA
vmovdqu YMMWORD [rdi + 1 * SIZEOF_YMMWORD], ymmD
vmovdqu YMMWORD [rdi + 2 * SIZEOF_YMMWORD], ymmF
.out0:
add rdi, byte RGB_PIXELSIZE * SIZEOF_YMMWORD ; outptr
sub rcx, byte SIZEOF_YMMWORD
jz near .endcolumn
add rsi, byte SIZEOF_YMMWORD ; inptr0
dec al ; Yctr
jnz near .Yloop_2nd
add rbx, byte SIZEOF_YMMWORD ; inptr1
add rdx, byte SIZEOF_YMMWORD ; inptr2
jmp near .columnloop
.column_st64:
lea rcx, [rcx + rcx * 2] ; imul ecx, RGB_PIXELSIZE
cmp rcx, byte 2 * SIZEOF_YMMWORD
jb short .column_st32
vmovdqu YMMWORD [rdi + 0 * SIZEOF_YMMWORD], ymmA
vmovdqu YMMWORD [rdi + 1 * SIZEOF_YMMWORD], ymmD
add rdi, byte 2 * SIZEOF_YMMWORD ; outptr
vmovdqa ymmA, ymmF
sub rcx, byte 2 * SIZEOF_YMMWORD
jmp short .column_st31
.column_st32:
cmp rcx, byte SIZEOF_YMMWORD
jb short .column_st31
vmovdqu YMMWORD [rdi + 0 * SIZEOF_YMMWORD], ymmA
add rdi, byte SIZEOF_YMMWORD ; outptr
vmovdqa ymmA, ymmD
sub rcx, byte SIZEOF_YMMWORD
jmp short .column_st31
.column_st31:
cmp rcx, byte SIZEOF_XMMWORD
jb short .column_st15
vmovdqu XMMWORD [rdi + 0 * SIZEOF_XMMWORD], xmmA
add rdi, byte SIZEOF_XMMWORD ; outptr
vperm2i128 ymmA, ymmA, ymmA, 1
sub rcx, byte SIZEOF_XMMWORD
.column_st15:
; Store the lower 8 bytes of xmmA to the output when it has enough
; space.
cmp rcx, byte SIZEOF_MMWORD
jb short .column_st7
vmovq XMM_MMWORD [rdi], xmmA
add rdi, byte SIZEOF_MMWORD
sub rcx, byte SIZEOF_MMWORD
vpsrldq xmmA, xmmA, SIZEOF_MMWORD
.column_st7:
; Store the lower 4 bytes of xmmA to the output when it has enough
; space.
cmp rcx, byte SIZEOF_DWORD
jb short .column_st3
vmovd XMM_DWORD [rdi], xmmA
add rdi, byte SIZEOF_DWORD
sub rcx, byte SIZEOF_DWORD
vpsrldq xmmA, xmmA, SIZEOF_DWORD
.column_st3:
; Store the lower 2 bytes of rax to the output when it has enough
; space.
vmovd eax, xmmA
cmp rcx, byte SIZEOF_WORD
jb short .column_st1
mov word [rdi], ax
add rdi, byte SIZEOF_WORD
sub rcx, byte SIZEOF_WORD
shr rax, 16
.column_st1:
; Store the lower 1 byte of rax to the output when it has enough
; space.
test rcx, rcx
jz short .endcolumn
mov byte [rdi], al
%else ; RGB_PIXELSIZE == 4 ; -----------
%ifdef RGBX_FILLER_0XFF
vpcmpeqb ymm6, ymm6, ymm6
; ymm6 = XE = X(02468ace********gikmoqsu********)
vpcmpeqb ymm7, ymm7, ymm7
; ymm7 = XO = X(13579bdf********hjlnprtv********)
%else
vpxor ymm6, ymm6, ymm6
; ymm6 = XE = X(02468ace********gikmoqsu********)
vpxor ymm7, ymm7, ymm7
; ymm7 = XO = X(13579bdf********hjlnprtv********)
%endif
; NOTE: The values of RGB_RED, RGB_GREEN, and RGB_BLUE determine the
; mapping of components A, B, C, and D to red, green, and blue.
;
; ymmA = (A0 A2 A4 A6 A8 Aa Ac Ae Ag Ai Ak Am Ao Aq As Au) = AE
; ymmB = (A1 A3 A5 A7 A9 Ab Ad Af Ah Aj Al An Ap Ar At Av) = AO
; ymmC = (B0 B2 B4 B6 B8 Ba Bc Be Bg Bi Bk Bm Bo Bq Bs Bu) = BE
; ymmD = (B1 B3 B5 B7 B9 Bb Bd Bf Bh Bj Bl Bn Bp Br Bt Bv) = BO
; ymmE = (C0 C2 C4 C6 C8 Ca Cc Ce Cg Ci Ck Cm Co Cq Cs Cu) = CE
; ymmF = (C1 C3 C5 C7 C9 Cb Cd Cf Ch Cj Cl Cn Cp Cr Ct Cv) = CO
; ymmG = (D0 D2 D4 D6 D8 Da Dc De Dg Di Dk Dm Do Dq Ds Du) = DE
; ymmH = (D1 D3 D5 D7 D9 Db Dd Df Dh Dj Dl Dn Dp Dr Dt Dv) = DO
vpunpcklbw ymmA, ymmA, ymmC
; ymmA = (A0 B0 A2 B2 A4 B4 A6 B6 A8 B8 Aa Ba Ac Bc Ae Be
; Ag Bg Ai Bi Ak Bk Am Bm Ao Bo Aq Bq As Bs Au Bu)
vpunpcklbw ymmE, ymmE, ymmG
; ymmE = (C0 D0 C2 D2 C4 D4 C6 D6 C8 D8 Ca Da Cc Dc Ce De
; Cg Dg Ci Di Ck Dk Cm Dm Co Do Cq Dq Cs Ds Cu Du)
vpunpcklbw ymmB, ymmB, ymmD
; ymmB = (A1 B1 A3 B3 A5 B5 A7 B7 A9 B9 Ab Bb Ad Bd Af Bf
; Ah Bh Aj Bj Al Bl An Bn Ap Bp Ar Br At Bt Av Bv)
vpunpcklbw ymmF, ymmF, ymmH
; ymmF = (C1 D1 C3 D3 C5 D5 C7 D7 C9 D9 Cb Db Cd Dd Cf Df
; Ch Dh Cj Dj Cl Dl Cn Dn Cp Dp Cr Dr Ct Dt Cv Dv)
vpunpckhwd ymmC, ymmA, ymmE
; ymmC = (A8 B8 C8 D8 Aa Ba Ca Da Ac Bc Cc Dc Ae Be Ce De
; Ao Bo Co Do Aq Bq Cq Dq As Bs Cs Ds Au Bu Cu Du)
vpunpcklwd ymmA, ymmA, ymmE
; ymmA = (A0 B0 C0 D0 A2 B2 C2 D2 A4 B4 C4 D4 A6 B6 C6 D6
; Ag Bg Cg Dg Ai Bi Ci Di Ak Bk Ck Dk Am Bm Cm Dm)
vpunpckhwd ymmG, ymmB, ymmF
; ymmG = (A9 B9 C9 D9 Ab Bb Cb Db Ad Bd Cd Dd Af Bf Cf Df
; Ap Bp Cp Dp Ar Br Cr Dr At Bt Ct Dt Av Bv Cv Dv)
vpunpcklwd ymmB, ymmB, ymmF
; ymmB = (A1 B1 C1 D1 A3 B3 C3 D3 A5 B5 C5 D5 A7 B7 C7 D7
; Ah Bh Ch Dh Aj Bj Cj Dj Al Bl Cl Dl An Bn Cn Dn)
vpunpckhdq ymmE, ymmA, ymmB
; ymmE = (A4 B4 C4 D4 A5 B5 C5 D5 A6 B6 C6 D6 A7 B7 C7 D7
; Ak Bk Ck Dk Al Bl Cl Dl Am Bm Cm Dm An Bn Cn Dn)
vpunpckldq ymmB, ymmA, ymmB
; ymmB = (A0 B0 C0 D0 A1 B1 C1 D1 A2 B2 C2 D2 A3 B3 C3 D3
; Ag Bg Cg Dg Ah Bh Ch Dh Ai Bi Ci Di Aj Bj Cj Dj)
vpunpckhdq ymmF, ymmC, ymmG
; ymmF = (Ac Bc Cc Dc Ad Bd Cd Dd Ae Be Ce De Af Bf Cf Df
; As Bs Cs Ds At Bt Ct Dt Au Bu Cu Du Av Bv Cv Dv)
vpunpckldq ymmG, ymmC, ymmG
; ymmG = (A8 B8 C8 D8 A9 B9 C9 D9 Aa Ba Ca Da Ab Bb Cb Db
; Ao Bo Co Do Ap Bp Cp Dp Aq Bq Cq Dq Ar Br Cr Dr)
vperm2i128 ymmA, ymmB, ymmE, 0x20
; ymmA = (A0 B0 C0 D0 A1 B1 C1 D1 A2 B2 C2 D2 A3 B3 C3 D3
; A4 B4 C4 D4 A5 B5 C5 D5 A6 B6 C6 D6 A7 B7 C7 D7)
vperm2i128 ymmD, ymmG, ymmF, 0x20
; ymmD = (A8 B8 C8 D8 A9 B9 C9 D9 Aa Ba Ca Da Ab Bb Cb Db
; Ac Bc Cc Dc Ad Bd Cd Dd Ae Be Ce De Af Bf Cf Df)
vperm2i128 ymmC, ymmB, ymmE, 0x31
; ymmC = (Ag Bg Cg Dg Ah Bh Ch Dh Ai Bi Ci Di Aj Bj Cj Dj
; Ak Bk Ck Dk Al Bl Cl Dl Am Bm Cm Dm An Bn Cn Dn)
vperm2i128 ymmH, ymmG, ymmF, 0x31
; ymmH = (Ao Bo Co Do Ap Bp Cp Dp Aq Bq Cq Dq Ar Br Cr Dr
; As Bs Cs Ds At Bt Ct Dt Au Bu Cu Du Av Bv Cv Dv)
cmp rcx, byte SIZEOF_YMMWORD
jb short .column_st64
test rdi, SIZEOF_YMMWORD - 1
jnz short .out1
; --(aligned)-------------------
vmovntdq YMMWORD [rdi + 0 * SIZEOF_YMMWORD], ymmA
vmovntdq YMMWORD [rdi + 1 * SIZEOF_YMMWORD], ymmD
vmovntdq YMMWORD [rdi + 2 * SIZEOF_YMMWORD], ymmC
vmovntdq YMMWORD [rdi + 3 * SIZEOF_YMMWORD], ymmH
jmp short .out0
.out1: ; --(unaligned)-----------------
vmovdqu YMMWORD [rdi + 0 * SIZEOF_YMMWORD], ymmA
vmovdqu YMMWORD [rdi + 1 * SIZEOF_YMMWORD], ymmD
vmovdqu YMMWORD [rdi + 2 * SIZEOF_YMMWORD], ymmC
vmovdqu YMMWORD [rdi + 3 * SIZEOF_YMMWORD], ymmH
.out0:
add rdi, RGB_PIXELSIZE * SIZEOF_YMMWORD ; outptr
sub rcx, byte SIZEOF_YMMWORD
jz near .endcolumn
add rsi, byte SIZEOF_YMMWORD ; inptr0
dec al
jnz near .Yloop_2nd
add rbx, byte SIZEOF_YMMWORD ; inptr1
add rdx, byte SIZEOF_YMMWORD ; inptr2
jmp near .columnloop
.column_st64:
cmp rcx, byte SIZEOF_YMMWORD / 2
jb short .column_st32
vmovdqu YMMWORD [rdi + 0 * SIZEOF_YMMWORD], ymmA
vmovdqu YMMWORD [rdi + 1 * SIZEOF_YMMWORD], ymmD
add rdi, byte 2 * SIZEOF_YMMWORD ; outptr
vmovdqa ymmA, ymmC
vmovdqa ymmD, ymmH
sub rcx, byte SIZEOF_YMMWORD / 2
.column_st32:
cmp rcx, byte SIZEOF_YMMWORD / 4
jb short .column_st16
vmovdqu YMMWORD [rdi + 0 * SIZEOF_YMMWORD], ymmA
add rdi, byte SIZEOF_YMMWORD ; outptr
vmovdqa ymmA, ymmD
sub rcx, byte SIZEOF_YMMWORD / 4
.column_st16:
cmp rcx, byte SIZEOF_YMMWORD / 8
jb short .column_st15
vmovdqu XMMWORD [rdi + 0 * SIZEOF_XMMWORD], xmmA
add rdi, byte SIZEOF_XMMWORD ; outptr
vperm2i128 ymmA, ymmA, ymmA, 1
sub rcx, byte SIZEOF_YMMWORD / 8
.column_st15:
; Store two pixels (8 bytes) of ymmA to the output when it has enough
; space.
cmp rcx, byte SIZEOF_YMMWORD / 16
jb short .column_st7
vmovq MMWORD [rdi], xmmA
add rdi, byte SIZEOF_YMMWORD / 16 * 4
sub rcx, byte SIZEOF_YMMWORD / 16
vpsrldq xmmA, SIZEOF_YMMWORD / 16 * 4
.column_st7:
; Store one pixel (4 bytes) of ymmA to the output when it has enough
; space.
test rcx, rcx
jz short .endcolumn
vmovd XMM_DWORD [rdi], xmmA
%endif ; RGB_PIXELSIZE ; ---------------
.endcolumn:
sfence ; flush the write buffer
.return:
pop rbx
vzeroupper
UNCOLLECT_ARGS 4
lea rsp, [rbp - 8]
pop r15
pop rbp
ret
; --------------------------------------------------------------------------
;
; Upsample and color convert for the case of 2:1 horizontal and 2:1 vertical.
;
; GLOBAL(void)
; jsimd_h2v2_merged_upsample_avx2(JDIMENSION output_width,
; JSAMPIMAGE input_buf,
; JDIMENSION in_row_group_ctr,
; JSAMPARRAY output_buf)
;
; r10d = JDIMENSION output_width
; r11 = JSAMPIMAGE input_buf
; r12d = JDIMENSION in_row_group_ctr
; r13 = JSAMPARRAY output_buf
align 32
GLOBAL_FUNCTION(jsimd_h2v2_merged_upsample_avx2)
EXTN(jsimd_h2v2_merged_upsample_avx2):
ENDBR64
push rbp
mov rbp, rsp
COLLECT_ARGS 4
push rbx
mov eax, r10d
mov rdi, r11
mov ecx, r12d
mov rsip, JSAMPARRAY [rdi + 0 * SIZEOF_JSAMPARRAY]
mov rbxp, JSAMPARRAY [rdi + 1 * SIZEOF_JSAMPARRAY]
mov rdxp, JSAMPARRAY [rdi + 2 * SIZEOF_JSAMPARRAY]
mov rdi, r13
lea rsi, [rsi + rcx * SIZEOF_JSAMPROW]
sub rsp, SIZEOF_JSAMPARRAY * 4
mov JSAMPARRAY [rsp + 0 * SIZEOF_JSAMPARRAY], rsip ; intpr00
mov JSAMPARRAY [rsp + 1 * SIZEOF_JSAMPARRAY], rbxp ; intpr1
mov JSAMPARRAY [rsp + 2 * SIZEOF_JSAMPARRAY], rdxp ; intpr2
mov rbx, rsp
push rdi
push rcx
push rax
%ifdef WIN64
mov r8, rcx
mov r9, rdi
mov rcx, rax
mov rdx, rbx
%else
mov rdx, rcx
mov rcx, rdi
mov rdi, rax
mov rsi, rbx
%endif
call EXTN(jsimd_h2v1_merged_upsample_avx2)
pop rax
pop rcx
pop rdi
mov rsip, JSAMPARRAY [rsp + 0 * SIZEOF_JSAMPARRAY]
mov rbxp, JSAMPARRAY [rsp + 1 * SIZEOF_JSAMPARRAY]
mov rdxp, JSAMPARRAY [rsp + 2 * SIZEOF_JSAMPARRAY]
add rdi, byte SIZEOF_JSAMPROW ; outptr1
add rsi, byte SIZEOF_JSAMPROW ; inptr01
mov JSAMPARRAY [rsp + 0 * SIZEOF_JSAMPARRAY], rsip ; intpr00
mov JSAMPARRAY [rsp + 1 * SIZEOF_JSAMPARRAY], rbxp ; intpr1
mov JSAMPARRAY [rsp + 2 * SIZEOF_JSAMPARRAY], rdxp ; intpr2
mov rbx, rsp
push rdi
push rcx
push rax
%ifdef WIN64
mov r8, rcx
mov r9, rdi
mov rcx, rax
mov rdx, rbx
%else
mov rdx, rcx
mov rcx, rdi
mov rdi, rax
mov rsi, rbx
%endif
call EXTN(jsimd_h2v1_merged_upsample_avx2)
pop rax
pop rcx
pop rdi
mov rsip, JSAMPARRAY [rsp + 0 * SIZEOF_JSAMPARRAY]
mov rbxp, JSAMPARRAY [rsp + 1 * SIZEOF_JSAMPARRAY]
mov rdxp, JSAMPARRAY [rsp + 2 * SIZEOF_JSAMPARRAY]
add rsp, SIZEOF_JSAMPARRAY * 4
pop rbx
UNCOLLECT_ARGS 4
pop rbp
ret
; For some reason, the OS X linker does not honor the request to align the
; segment unless we do this.
align 32
lib/libjpeg-turbo/simd/x86_64/jdmrgext-sse2.asm
Executable
+587
View File
@@ -0,0 +1,587 @@
;
; Merged upsampling/color conversion (64-bit SSE2)
;
; Copyright 2009, 2012 Pierre Ossman <[email protected]> for Cendio AB
; Copyright (C) 2009, 2012, 2016, 2024-2025, D. R. Commander.
; Copyright (C) 2018, Matthias Räncker.
; Copyright (C) 2023, Aliaksiej Kandracienka.
;
; Based on the x86 SIMD extension for IJG JPEG library
; Copyright (C) 1999-2006, MIYASAKA Masaru.
; For conditions of distribution and use, see copyright notice in jsimdext.inc
;
; This file should be assembled with NASM (Netwide Assembler) or Yasm.
%include "jcolsamp.inc"
; --------------------------------------------------------------------------
;
; Upsample and color convert for the case of 2:1 horizontal and 1:1 vertical.
;
; GLOBAL(void)
; jsimd_h2v1_merged_upsample_sse2(JDIMENSION output_width,
; JSAMPIMAGE input_buf,
; JDIMENSION in_row_group_ctr,
; JSAMPARRAY output_buf)
;
; r10d = JDIMENSION output_width
; r11 = JSAMPIMAGE input_buf
; r12d = JDIMENSION in_row_group_ctr
; r13 = JSAMPARRAY output_buf
%define wk(i) r15 - (WK_NUM - (i)) * SIZEOF_XMMWORD ; xmmword wk[WK_NUM]
%define WK_NUM 3
align 32
GLOBAL_FUNCTION(jsimd_h2v1_merged_upsample_sse2)
EXTN(jsimd_h2v1_merged_upsample_sse2):
ENDBR64
push rbp
mov rbp, rsp
push r15
and rsp, byte (-SIZEOF_XMMWORD) ; align to 128 bits
; Allocate stack space for wk array. r15 is used to access it.
mov r15, rsp
sub rsp, byte (SIZEOF_XMMWORD * WK_NUM)
COLLECT_ARGS 4
push rbx
mov ecx, r10d ; col
test rcx, rcx
jz near .return
push rcx
mov rdi, r11
mov ecx, r12d
mov rsip, JSAMPARRAY [rdi + 0 * SIZEOF_JSAMPARRAY]
mov rbxp, JSAMPARRAY [rdi + 1 * SIZEOF_JSAMPARRAY]
mov rdxp, JSAMPARRAY [rdi + 2 * SIZEOF_JSAMPARRAY]
mov rdi, r13
mov rsip, JSAMPROW [rsi + rcx * SIZEOF_JSAMPROW] ; inptr0
mov rbxp, JSAMPROW [rbx + rcx * SIZEOF_JSAMPROW] ; inptr1
mov rdxp, JSAMPROW [rdx + rcx * SIZEOF_JSAMPROW] ; inptr2
mov rdip, JSAMPROW [rdi] ; outptr
pop rcx ; col
.columnloop:
movdqa xmm6, XMMWORD [rbx] ; xmm6 = Cb(0123456789abcdef)
movdqa xmm7, XMMWORD [rdx] ; xmm7 = Cr(0123456789abcdef)
pxor xmm1, xmm1 ; xmm1 = (all 0's)
pcmpeqw xmm3, xmm3
psllw xmm3, 7 ; xmm3 = { 0xFF80 0xFF80 0xFF80 0xFF80 .. }
movdqa xmm4, xmm6
punpckhbw xmm6, xmm1 ; xmm6 = Cb(89abcdef) = CbH
punpcklbw xmm4, xmm1 ; xmm4 = Cb(01234567) = CbL
movdqa xmm0, xmm7
punpckhbw xmm7, xmm1 ; xmm7 = Cr(89abcdef) = CrH
punpcklbw xmm0, xmm1 ; xmm0 = Cr(01234567) = CrL
paddw xmm6, xmm3
paddw xmm4, xmm3
paddw xmm7, xmm3
paddw xmm0, xmm3
; (Original)
; R = Y + 1.40200 * Cr
; G = Y - 0.34414 * Cb - 0.71414 * Cr
; B = Y + 1.77200 * Cb
;
; (This implementation)
; R = Y + 0.40200 * Cr + Cr
; G = Y - 0.34414 * Cb + 0.28586 * Cr - Cr
; B = Y - 0.22800 * Cb + Cb + Cb
movdqa xmm5, xmm6 ; xmm5 = CbH
movdqa xmm2, xmm4 ; xmm2 = CbL
paddw xmm6, xmm6 ; xmm6 = 2 * CbH
paddw xmm4, xmm4 ; xmm4 = 2 * CbL
movdqa xmm1, xmm7 ; xmm1 = CrH
movdqa xmm3, xmm0 ; xmm3 = CrL
paddw xmm7, xmm7 ; xmm7 = 2 * CrH
paddw xmm0, xmm0 ; xmm0 = 2 * CrL
pmulhw xmm6, [rel PW_MF0228] ; xmm6 = (2 * CbH * -FIX(0.22800))
pmulhw xmm4, [rel PW_MF0228] ; xmm4 = (2 * CbL * -FIX(0.22800))
pmulhw xmm7, [rel PW_F0402] ; xmm7 = (2 * CrH * FIX(0.40200))
pmulhw xmm0, [rel PW_F0402] ; xmm0 = (2 * CrL * FIX(0.40200))
paddw xmm6, [rel PW_ONE]
paddw xmm4, [rel PW_ONE]
psraw xmm6, 1 ; xmm6 = (CbH * -FIX(0.22800))
psraw xmm4, 1 ; xmm4 = (CbL * -FIX(0.22800))
paddw xmm7, [rel PW_ONE]
paddw xmm0, [rel PW_ONE]
psraw xmm7, 1 ; xmm7 = (CrH * FIX(0.40200))
psraw xmm0, 1 ; xmm0 = (CrL * FIX(0.40200))
paddw xmm6, xmm5
paddw xmm4, xmm2
paddw xmm6, xmm5 ; xmm6 = (CbH * FIX(1.77200)) = (B - Y)H
paddw xmm4, xmm2 ; xmm4 = (CbL * FIX(1.77200)) = (B - Y)L
paddw xmm7, xmm1 ; xmm7 = (CrH * FIX(1.40200)) = (R - Y)H
paddw xmm0, xmm3 ; xmm0 = (CrL * FIX(1.40200)) = (R - Y)L
movdqa XMMWORD [wk(0)], xmm6 ; wk(0) = (B - Y)H
movdqa XMMWORD [wk(1)], xmm7 ; wk(1) = (R - Y)H
movdqa xmm6, xmm5
movdqa xmm7, xmm2
punpcklwd xmm5, xmm1
punpckhwd xmm6, xmm1
pmaddwd xmm5, [rel PW_MF0344_F0285]
pmaddwd xmm6, [rel PW_MF0344_F0285]
punpcklwd xmm2, xmm3
punpckhwd xmm7, xmm3
pmaddwd xmm2, [rel PW_MF0344_F0285]
pmaddwd xmm7, [rel PW_MF0344_F0285]
paddd xmm5, [rel PD_ONEHALF]
paddd xmm6, [rel PD_ONEHALF]
psrad xmm5, SCALEBITS
psrad xmm6, SCALEBITS
paddd xmm2, [rel PD_ONEHALF]
paddd xmm7, [rel PD_ONEHALF]
psrad xmm2, SCALEBITS
psrad xmm7, SCALEBITS
packssdw xmm5, xmm6
; xmm5 = CbH * -FIX(0.344) + CrH * FIX(0.285)
packssdw xmm2, xmm7
; xmm2 = CbL * -FIX(0.344) + CrL * FIX(0.285)
psubw xmm5, xmm1
; xmm5 = CbH * -FIX(0.344) + CrH * -FIX(0.714) = (G - Y)H
psubw xmm2, xmm3
; xmm2 = CbL * -FIX(0.344) + CrL * -FIX(0.714) = (G - Y)L
movdqa XMMWORD [wk(2)], xmm5 ; wk(2) = (G - Y)H
mov al, 2 ; Yctr
jmp short .Yloop_1st
.Yloop_2nd:
movdqa xmm0, XMMWORD [wk(1)] ; xmm0 = (R - Y)H
movdqa xmm2, XMMWORD [wk(2)] ; xmm2 = (G - Y)H
movdqa xmm4, XMMWORD [wk(0)] ; xmm4 = (B - Y)H
.Yloop_1st:
movdqa xmm7, XMMWORD [rsi] ; xmm7 = Y(0123456789abcdef)
pcmpeqw xmm6, xmm6
psrlw xmm6, BYTE_BIT ; xmm6 = { 0xFF 0x00 0xFF 0x00 .. }
pand xmm6, xmm7 ; xmm6 = Y(02468ace) = YE
psrlw xmm7, BYTE_BIT ; xmm7 = Y(13579bdf) = YO
movdqa xmm1, xmm0 ; xmm1 = xmm0 = (R - Y)(L / H)
movdqa xmm3, xmm2 ; xmm3 = xmm2 = (G - Y)(L / H)
movdqa xmm5, xmm4 ; xmm5 = xmm4 = (B - Y)(L / H)
paddw xmm0, xmm6 ; xmm0 = ((R - Y) + YE) = RE = R(02468ace)
paddw xmm1, xmm7 ; xmm1 = ((R - Y) + YO) = RO = R(13579bdf)
packuswb xmm0, xmm0 ; xmm0 = R(02468ace********)
packuswb xmm1, xmm1 ; xmm1 = R(13579bdf********)
paddw xmm2, xmm6 ; xmm2 = ((G - Y) + YE) = GE = G(02468ace)
paddw xmm3, xmm7 ; xmm3 = ((G - Y) + YO) = GO = G(13579bdf)
packuswb xmm2, xmm2 ; xmm2 = G(02468ace********)
packuswb xmm3, xmm3 ; xmm3 = G(13579bdf********)
paddw xmm4, xmm6 ; xmm4 = ((B - Y) + YE) = BE = B(02468ace)
paddw xmm5, xmm7 ; xmm5 = ((B - Y) + YO) = BO = B(13579bdf)
packuswb xmm4, xmm4 ; xmm4 = B(02468ace********)
packuswb xmm5, xmm5 ; xmm5 = B(13579bdf********)
%if RGB_PIXELSIZE == 3 ; ---------------
; NOTE: The values of RGB_RED, RGB_GREEN, and RGB_BLUE determine the
; mapping of components A, B, and C to red, green, and blue.
;
; xmmA = (A0 A2 A4 A6 A8 Aa Ac Ae) = AE
; xmmB = (A1 A3 A5 A7 A9 Ab Ad Af) = AO
; xmmC = (B0 B2 B4 B6 B8 Ba Bc Be) = BE
; xmmD = (B1 B3 B5 B7 B9 Bb Bd Bf) = BO
; xmmE = (C0 C2 C4 C6 C8 Ca Cc Ce) = CE
; xmmF = (C1 C3 C5 C7 C9 Cb Cd Cf) = CO
; xmmG = (** ** ** ** ** ** ** **)
; xmmH = (** ** ** ** ** ** ** **)
punpcklbw xmmA, xmmC
; xmmA = (A0 B0 A2 B2 A4 B4 A6 B6 A8 B8 Aa Ba Ac Bc Ae Be)
punpcklbw xmmE, xmmB
; xmmE = (C0 A1 C2 A3 C4 A5 C6 A7 C8 A9 Ca Ab Cc Ad Ce Af)
punpcklbw xmmD, xmmF
; xmmD = (B1 C1 B3 C3 B5 C5 B7 C7 B9 C9 Bb Cb Bd Cd Bf Cf)
movdqa xmmG, xmmA
movdqa xmmH, xmmA
punpcklwd xmmA, xmmE
; xmmA = (A0 B0 C0 A1 A2 B2 C2 A3 A4 B4 C4 A5 A6 B6 C6 A7)
punpckhwd xmmG, xmmE
; xmmG = (A8 B8 C8 A9 Aa Ba Ca Ab Ac Bc Cc Ad Ae Be Ce Af)
psrldq xmmH, 2
; xmmH = (A2 B2 A4 B4 A6 B6 A8 B8 Aa Ba Ac Bc Ae Be -- --)
psrldq xmmE, 2
; xmmE = (C2 A3 C4 A5 C6 A7 C8 A9 Ca Ab Cc Ad Ce Af -- --)
movdqa xmmC, xmmD
movdqa xmmB, xmmD
punpcklwd xmmD, xmmH
; xmmD = (B1 C1 A2 B2 B3 C3 A4 B4 B5 C5 A6 B6 B7 C7 A8 B8)
punpckhwd xmmC, xmmH
; xmmC = (B9 C9 Aa Ba Bb Cb Ac Bc Bd Cd Ae Be Bf Cf -- --)
psrldq xmmB, 2
; xmmB = (B3 C3 B5 C5 B7 C7 B9 C9 Bb Cb Bd Cd Bf Cf -- --)
movdqa xmmF, xmmE
punpcklwd xmmE, xmmB
; xmmE = (C2 A3 B3 C3 C4 A5 B5 C5 C6 A7 B7 C7 C8 A9 B9 C9)
punpckhwd xmmF, xmmB
; xmmF = (Ca Ab Bb Cb Cc Ad Bd Cd Ce Af Bf Cf -- -- -- --)
pshufd xmmH, xmmA, 0x4E
; xmmH = (A4 B4 C4 A5 A6 B6 C6 A7 A0 B0 C0 A1 A2 B2 C2 A3)
movdqa xmmB, xmmE
punpckldq xmmA, xmmD
; xmmA = (A0 B0 C0 A1 B1 C1 A2 B2 A2 B2 C2 A3 B3 C3 A4 B4)
punpckldq xmmE, xmmH
; xmmE = (C2 A3 B3 C3 A4 B4 C4 A5 C4 A5 B5 C5 A6 B6 C6 A7)
punpckhdq xmmD, xmmB
; xmmD = (B5 C5 A6 B6 C6 A7 B7 C7 B7 C7 A8 B8 C8 A9 B9 C9)
pshufd xmmH, xmmG, 0x4E
; xmmH = (Ac Bc Cc Ad Ae Be Ce Af A8 B8 C8 A9 Aa Ba Ca Ab)
movdqa xmmB, xmmF
punpckldq xmmG, xmmC
; xmmG = (A8 B8 C8 A9 B9 C9 Aa Ba Aa Ba Ca Ab Bb Cb Ac Bc)
punpckldq xmmF, xmmH
; xmmF = (Ca Ab Bb Cb Ac Bc Cc Ad Cc Ad Bd Cd Ae Be Ce Af)
punpckhdq xmmC, xmmB
; xmmC = (Bd Cd Ae Be Ce Af Bf Cf Bf Cf -- -- -- -- -- --)
punpcklqdq xmmA, xmmE
; xmmA = (A0 B0 C0 A1 B1 C1 A2 B2 C2 A3 B3 C3 A4 B4 C4 A5)
punpcklqdq xmmD, xmmG
; xmmD = (B5 C5 A6 B6 C6 A7 B7 C7 A8 B8 C8 A9 B9 C9 Aa Ba)
punpcklqdq xmmF, xmmC
; xmmF = (Ca Ab Bb Cb Ac Bc Cc Ad Bd Cd Ae Be Ce Af Bf Cf)
cmp rcx, byte SIZEOF_XMMWORD
jb short .column_st32
test rdi, SIZEOF_XMMWORD - 1
jnz short .out1
; --(aligned)-------------------
movntdq XMMWORD [rdi + 0 * SIZEOF_XMMWORD], xmmA
movntdq XMMWORD [rdi + 1 * SIZEOF_XMMWORD], xmmD
movntdq XMMWORD [rdi + 2 * SIZEOF_XMMWORD], xmmF
jmp short .out0
.out1: ; --(unaligned)-----------------
movdqu XMMWORD [rdi + 0 * SIZEOF_XMMWORD], xmmA
movdqu XMMWORD [rdi + 1 * SIZEOF_XMMWORD], xmmD
movdqu XMMWORD [rdi + 2 * SIZEOF_XMMWORD], xmmF
.out0:
add rdi, byte RGB_PIXELSIZE * SIZEOF_XMMWORD ; outptr
sub rcx, byte SIZEOF_XMMWORD
jz near .endcolumn
add rsi, byte SIZEOF_XMMWORD ; inptr0
dec al ; Yctr
jnz near .Yloop_2nd
add rbx, byte SIZEOF_XMMWORD ; inptr1
add rdx, byte SIZEOF_XMMWORD ; inptr2
jmp near .columnloop
.column_st32:
lea rcx, [rcx + rcx * 2] ; imul ecx, RGB_PIXELSIZE
cmp rcx, byte 2 * SIZEOF_XMMWORD
jb short .column_st16
movdqu XMMWORD [rdi + 0 * SIZEOF_XMMWORD], xmmA
movdqu XMMWORD [rdi + 1 * SIZEOF_XMMWORD], xmmD
add rdi, byte 2 * SIZEOF_XMMWORD ; outptr
movdqa xmmA, xmmF
sub rcx, byte 2 * SIZEOF_XMMWORD
jmp short .column_st15
.column_st16:
cmp rcx, byte SIZEOF_XMMWORD
jb short .column_st15
movdqu XMMWORD [rdi + 0 * SIZEOF_XMMWORD], xmmA
add rdi, byte SIZEOF_XMMWORD ; outptr
movdqa xmmA, xmmD
sub rcx, byte SIZEOF_XMMWORD
.column_st15:
; Store the lower 8 bytes of xmmA to the output when it has enough
; space.
cmp rcx, byte SIZEOF_MMWORD
jb short .column_st7
movq XMM_MMWORD [rdi], xmmA
add rdi, byte SIZEOF_MMWORD
sub rcx, byte SIZEOF_MMWORD
psrldq xmmA, SIZEOF_MMWORD
.column_st7:
; Store the lower 4 bytes of xmmA to the output when it has enough
; space.
cmp rcx, byte SIZEOF_DWORD
jb short .column_st3
movd XMM_DWORD [rdi], xmmA
add rdi, byte SIZEOF_DWORD
sub rcx, byte SIZEOF_DWORD
psrldq xmmA, SIZEOF_DWORD
.column_st3:
; Store the lower 2 bytes of rax to the output when it has enough
; space.
movd eax, xmmA
cmp rcx, byte SIZEOF_WORD
jb short .column_st1
mov word [rdi], ax
add rdi, byte SIZEOF_WORD
sub rcx, byte SIZEOF_WORD
shr rax, 16
.column_st1:
; Store the lower 1 byte of rax to the output when it has enough
; space.
test rcx, rcx
jz short .endcolumn
mov byte [rdi], al
%else ; RGB_PIXELSIZE == 4 ; -----------
%ifdef RGBX_FILLER_0XFF
pcmpeqb xmm6, xmm6 ; xmm6 = XE = X(02468ace********)
pcmpeqb xmm7, xmm7 ; xmm7 = XO = X(13579bdf********)
%else
pxor xmm6, xmm6 ; xmm6 = XE = X(02468ace********)
pxor xmm7, xmm7 ; xmm7 = XO = X(13579bdf********)
%endif
; NOTE: The values of RGB_RED, RGB_GREEN, and RGB_BLUE determine the
; mapping of components A, B, C, and D to red, green, and blue.
;
; xmmA = (A0 A2 A4 A6 A8 Aa Ac Ae) = AE
; xmmB = (A1 A3 A5 A7 A9 Ab Ad Af) = AO
; xmmC = (B0 B2 B4 B6 B8 Ba Bc Be) = BE
; xmmD = (B1 B3 B5 B7 B9 Bb Bd Bf) = BO
; xmmE = (C0 C2 C4 C6 C8 Ca Cc Ce) = CE
; xmmF = (C1 C3 C5 C7 C9 Cb Cd Cf) = CO
; xmmG = (D0 D2 D4 D6 D8 Da Dc De) = DE
; xmmH = (D1 D3 D5 D7 D9 Db Dd Df) = DO
punpcklbw xmmA, xmmC
; xmmA = (A0 B0 A2 B2 A4 B4 A6 B6 A8 B8 Aa Ba Ac Bc Ae Be)
punpcklbw xmmE, xmmG
; xmmE = (C0 D0 C2 D2 C4 D4 C6 D6 C8 D8 Ca Da Cc Dc Ce De)
punpcklbw xmmB, xmmD
; xmmB = (A1 B1 A3 B3 A5 B5 A7 B7 A9 B9 Ab Bb Ad Bd Af Bf)
punpcklbw xmmF, xmmH
; xmmF = (C1 D1 C3 D3 C5 D5 C7 D7 C9 D9 Cb Db Cd Dd Cf Df)
movdqa xmmC, xmmA
punpcklwd xmmA, xmmE
; xmmA = (A0 B0 C0 D0 A2 B2 C2 D2 A4 B4 C4 D4 A6 B6 C6 D6)
punpckhwd xmmC, xmmE
; xmmC = (A8 B8 C8 D8 Aa Ba Ca Da Ac Bc Cc Dc Ae Be Ce De)
movdqa xmmG, xmmB
punpcklwd xmmB, xmmF
; xmmB = (A1 B1 C1 D1 A3 B3 C3 D3 A5 B5 C5 D5 A7 B7 C7 D7)
punpckhwd xmmG, xmmF
; xmmG = (A9 B9 C9 D9 Ab Bb Cb Db Ad Bd Cd Dd Af Bf Cf Df)
movdqa xmmD, xmmA
punpckldq xmmA, xmmB
; xmmA = (A0 B0 C0 D0 A1 B1 C1 D1 A2 B2 C2 D2 A3 B3 C3 D3)
punpckhdq xmmD, xmmB
; xmmD = (A4 B4 C4 D4 A5 B5 C5 D5 A6 B6 C6 D6 A7 B7 C7 D7)
movdqa xmmH, xmmC
punpckldq xmmC, xmmG
; xmmC = (A8 B8 C8 D8 A9 B9 C9 D9 Aa Ba Ca Da Ab Bb Cb Db)
punpckhdq xmmH, xmmG
; xmmH = (Ac Bc Cc Dc Ad Bd Cd Dd Ae Be Ce De Af Bf Cf Df)
cmp rcx, byte SIZEOF_XMMWORD
jb short .column_st32
test rdi, SIZEOF_XMMWORD - 1
jnz short .out1
; --(aligned)-------------------
movntdq XMMWORD [rdi + 0 * SIZEOF_XMMWORD], xmmA
movntdq XMMWORD [rdi + 1 * SIZEOF_XMMWORD], xmmD
movntdq XMMWORD [rdi + 2 * SIZEOF_XMMWORD], xmmC
movntdq XMMWORD [rdi + 3 * SIZEOF_XMMWORD], xmmH
jmp short .out0
.out1: ; --(unaligned)-----------------
movdqu XMMWORD [rdi + 0 * SIZEOF_XMMWORD], xmmA
movdqu XMMWORD [rdi + 1 * SIZEOF_XMMWORD], xmmD
movdqu XMMWORD [rdi + 2 * SIZEOF_XMMWORD], xmmC
movdqu XMMWORD [rdi + 3 * SIZEOF_XMMWORD], xmmH
.out0:
add rdi, byte RGB_PIXELSIZE * SIZEOF_XMMWORD ; outptr
sub rcx, byte SIZEOF_XMMWORD
jz near .endcolumn
add rsi, byte SIZEOF_XMMWORD ; inptr0
dec al ; Yctr
jnz near .Yloop_2nd
add rbx, byte SIZEOF_XMMWORD ; inptr1
add rdx, byte SIZEOF_XMMWORD ; inptr2
jmp near .columnloop
.column_st32:
cmp rcx, byte SIZEOF_XMMWORD / 2
jb short .column_st16
movdqu XMMWORD [rdi + 0 * SIZEOF_XMMWORD], xmmA
movdqu XMMWORD [rdi + 1 * SIZEOF_XMMWORD], xmmD
add rdi, byte 2 * SIZEOF_XMMWORD ; outptr
movdqa xmmA, xmmC
movdqa xmmD, xmmH
sub rcx, byte SIZEOF_XMMWORD / 2
.column_st16:
cmp rcx, byte SIZEOF_XMMWORD / 4
jb short .column_st15
movdqu XMMWORD [rdi + 0 * SIZEOF_XMMWORD], xmmA
add rdi, byte SIZEOF_XMMWORD ; outptr
movdqa xmmA, xmmD
sub rcx, byte SIZEOF_XMMWORD / 4
.column_st15:
; Store two pixels (8 bytes) of xmmA to the output when it has enough
; space.
cmp rcx, byte SIZEOF_XMMWORD / 8
jb short .column_st7
movq XMM_MMWORD [rdi], xmmA
add rdi, byte SIZEOF_XMMWORD / 8 * 4
sub rcx, byte SIZEOF_XMMWORD / 8
psrldq xmmA, SIZEOF_XMMWORD / 8 * 4
.column_st7:
; Store one pixel (4 bytes) of xmmA to the output when it has enough
; space.
test rcx, rcx
jz short .endcolumn
movd XMM_DWORD [rdi], xmmA
%endif ; RGB_PIXELSIZE ; ---------------
.endcolumn:
sfence ; flush the write buffer
.return:
pop rbx
UNCOLLECT_ARGS 4
lea rsp, [rbp - 8]
pop r15
pop rbp
ret
; --------------------------------------------------------------------------
;
; Upsample and color convert for the case of 2:1 horizontal and 2:1 vertical.
;
; GLOBAL(void)
; jsimd_h2v2_merged_upsample_sse2(JDIMENSION output_width,
; JSAMPIMAGE input_buf,
; JDIMENSION in_row_group_ctr,
; JSAMPARRAY output_buf)
;
; r10d = JDIMENSION output_width
; r11 = JSAMPIMAGE input_buf
; r12d = JDIMENSION in_row_group_ctr
; r13 = JSAMPARRAY output_buf
align 32
GLOBAL_FUNCTION(jsimd_h2v2_merged_upsample_sse2)
EXTN(jsimd_h2v2_merged_upsample_sse2):
ENDBR64
push rbp
mov rbp, rsp
COLLECT_ARGS 4
push rbx
mov eax, r10d
mov rdi, r11
mov ecx, r12d
mov rsip, JSAMPARRAY [rdi + 0 * SIZEOF_JSAMPARRAY]
mov rbxp, JSAMPARRAY [rdi + 1 * SIZEOF_JSAMPARRAY]
mov rdxp, JSAMPARRAY [rdi + 2 * SIZEOF_JSAMPARRAY]
mov rdi, r13
lea rsi, [rsi + rcx * SIZEOF_JSAMPROW]
sub rsp, SIZEOF_JSAMPARRAY * 4
mov JSAMPARRAY [rsp + 0 * SIZEOF_JSAMPARRAY], rsip ; intpr00
mov JSAMPARRAY [rsp + 1 * SIZEOF_JSAMPARRAY], rbxp ; intpr1
mov JSAMPARRAY [rsp + 2 * SIZEOF_JSAMPARRAY], rdxp ; intpr2
mov rbx, rsp
push rdi
push rcx
push rax
%ifdef WIN64
mov r8, rcx
mov r9, rdi
mov rcx, rax
mov rdx, rbx
%else
mov rdx, rcx
mov rcx, rdi
mov rdi, rax
mov rsi, rbx
%endif
call EXTN(jsimd_h2v1_merged_upsample_sse2)
pop rax
pop rcx
pop rdi
mov rsip, JSAMPARRAY [rsp + 0 * SIZEOF_JSAMPARRAY]
mov rbxp, JSAMPARRAY [rsp + 1 * SIZEOF_JSAMPARRAY]
mov rdxp, JSAMPARRAY [rsp + 2 * SIZEOF_JSAMPARRAY]
add rdi, byte SIZEOF_JSAMPROW ; outptr1
add rsi, byte SIZEOF_JSAMPROW ; inptr01
mov JSAMPARRAY [rsp + 0 * SIZEOF_JSAMPARRAY], rsip ; intpr00
mov JSAMPARRAY [rsp + 1 * SIZEOF_JSAMPARRAY], rbxp ; intpr1
mov JSAMPARRAY [rsp + 2 * SIZEOF_JSAMPARRAY], rdxp ; intpr2
mov rbx, rsp
push rdi
push rcx
push rax
%ifdef WIN64
mov r8, rcx
mov r9, rdi
mov rcx, rax
mov rdx, rbx
%else
mov rdx, rcx
mov rcx, rdi
mov rdi, rax
mov rsi, rbx
%endif
call EXTN(jsimd_h2v1_merged_upsample_sse2)
pop rax
pop rcx
pop rdi
mov rsip, JSAMPARRAY [rsp + 0 * SIZEOF_JSAMPARRAY]
mov rbxp, JSAMPARRAY [rsp + 1 * SIZEOF_JSAMPARRAY]
mov rdxp, JSAMPARRAY [rsp + 2 * SIZEOF_JSAMPARRAY]
add rsp, SIZEOF_JSAMPARRAY * 4
pop rbx
UNCOLLECT_ARGS 4
pop rbp
ret
; For some reason, the OS X linker does not honor the request to align the
; segment unless we do this.
align 32
lib/libjpeg-turbo/simd/x86_64/jdsample-sse2.asm
Executable
+663
View File
@@ -0,0 +1,663 @@
;
; Upsampling (64-bit SSE2)
;
; Copyright 2009 Pierre Ossman <[email protected]> for Cendio AB
; Copyright (C) 2009, 2016, 2024-2026, D. R. Commander.
; Copyright (C) 2018, Matthias Räncker.
; Copyright (C) 2023, Aliaksiej Kandracienka.
;
; Based on the x86 SIMD extension for IJG JPEG library
; Copyright (C) 1999-2006, MIYASAKA Masaru.
; For conditions of distribution and use, see copyright notice in jsimdext.inc
;
; This file should be assembled with NASM (Netwide Assembler) or Yasm.
%include "jsimdext.inc"
; --------------------------------------------------------------------------
SECTION SEG_CONST
ALIGNZ 32
GLOBAL_DATA(jconst_fancy_upsample_sse2)
EXTN(jconst_fancy_upsample_sse2):
PW_ONE times 8 dw 1
PW_TWO times 8 dw 2
PW_THREE times 8 dw 3
PW_SEVEN times 8 dw 7
PW_EIGHT times 8 dw 8
ALIGNZ 32
; --------------------------------------------------------------------------
SECTION SEG_TEXT
BITS 64
; Fancy processing for the common case of 2:1 horizontal and 1:1 vertical.
;
; The upsampling algorithm is linear interpolation between component centers,
; also known as a "triangle filter". This is a good compromise between speed
; and visual quality. The centers of the output components are 1/4 and 3/4 of
; the way between input component centers.
;
; GLOBAL(void)
; jsimd_h2v1_fancy_upsample_sse2(int max_v_samp_factor,
; JDIMENSION downsampled_width,
; JSAMPARRAY input_data,
; JSAMPARRAY *output_data_ptr)
;
; r10 = int max_v_samp_factor
; r11d = JDIMENSION downsampled_width
; r12 = JSAMPARRAY input_data
; r13 = JSAMPARRAY *output_data_ptr
align 32
GLOBAL_FUNCTION(jsimd_h2v1_fancy_upsample_sse2)
EXTN(jsimd_h2v1_fancy_upsample_sse2):
ENDBR64
push rbp
mov rbp, rsp
COLLECT_ARGS 4
mov eax, r11d ; colctr
test rax, rax
jz near .return
mov rcx, r10 ; rowctr
test rcx, rcx
jz near .return
mov rsi, r12 ; input_data
mov rdi, r13
mov rdip, JSAMPARRAY [rdi] ; output_data
.rowloop:
push rax ; colctr
push rdi
push rsi
mov rsip, JSAMPROW [rsi] ; inptr
mov rdip, JSAMPROW [rdi] ; outptr
test rax, SIZEOF_XMMWORD - 1
jz short .skip
mov dl, JSAMPLE [rsi + (rax - 1) * SIZEOF_JSAMPLE]
mov JSAMPLE [rsi + rax * SIZEOF_JSAMPLE], dl
; insert a dummy sample
.skip:
pxor xmm0, xmm0 ; xmm0 = (all 0's)
pcmpeqb xmm7, xmm7
psrldq xmm7, (SIZEOF_XMMWORD - 1)
pand xmm7, XMMWORD [rsi + 0 * SIZEOF_XMMWORD]
add rax, byte SIZEOF_XMMWORD - 1
and rax, byte -SIZEOF_XMMWORD
cmp rax, byte SIZEOF_XMMWORD
ja short .columnloop
.columnloop_last:
pcmpeqb xmm6, xmm6
pslldq xmm6, (SIZEOF_XMMWORD - 1)
pand xmm6, XMMWORD [rsi + 0 * SIZEOF_XMMWORD]
jmp short .upsample
.columnloop:
movdqa xmm6, XMMWORD [rsi + 1 * SIZEOF_XMMWORD]
pslldq xmm6, (SIZEOF_XMMWORD - 1)
.upsample:
movdqa xmm1, XMMWORD [rsi + 0 * SIZEOF_XMMWORD]
movdqa xmm2, xmm1
movdqa xmm3, xmm1 ; xmm1 = ( 0 1 2 ... 13 14 15)
pslldq xmm2, 1 ; xmm2 = (-- 0 1 ... 12 13 14)
psrldq xmm3, 1 ; xmm3 = ( 1 2 3 ... 14 15 --)
por xmm2, xmm7 ; xmm2 = (-1 0 1 ... 12 13 14)
por xmm3, xmm6 ; xmm3 = ( 1 2 3 ... 14 15 16)
movdqa xmm7, xmm1
psrldq xmm7, (SIZEOF_XMMWORD - 1) ; xmm7 = (15 -- -- ... -- -- --)
movdqa xmm4, xmm1
punpcklbw xmm1, xmm0 ; xmm1 = ( 0 1 2 3 4 5 6 7)
punpckhbw xmm4, xmm0 ; xmm4 = ( 8 9 10 11 12 13 14 15)
movdqa xmm5, xmm2
punpcklbw xmm2, xmm0 ; xmm2 = (-1 0 1 2 3 4 5 6)
punpckhbw xmm5, xmm0 ; xmm5 = ( 7 8 9 10 11 12 13 14)
movdqa xmm6, xmm3
punpcklbw xmm3, xmm0 ; xmm3 = ( 1 2 3 4 5 6 7 8)
punpckhbw xmm6, xmm0 ; xmm6 = ( 9 10 11 12 13 14 15 16)
pmullw xmm1, [rel PW_THREE]
pmullw xmm4, [rel PW_THREE]
paddw xmm2, [rel PW_ONE]
paddw xmm5, [rel PW_ONE]
paddw xmm3, [rel PW_TWO]
paddw xmm6, [rel PW_TWO]
paddw xmm2, xmm1
paddw xmm5, xmm4
psrlw xmm2, 2 ; xmm2 = OutLE = ( 0 2 4 6 8 10 12 14)
psrlw xmm5, 2 ; xmm5 = OutHE = (16 18 20 22 24 26 28 30)
paddw xmm3, xmm1
paddw xmm6, xmm4
psrlw xmm3, 2 ; xmm3 = OutLO = ( 1 3 5 7 9 11 13 15)
psrlw xmm6, 2 ; xmm6 = OutHO = (17 19 21 23 25 27 29 31)
psllw xmm3, BYTE_BIT
psllw xmm6, BYTE_BIT
por xmm2, xmm3 ; xmm2 = OutL = ( 0 1 2 ... 13 14 15)
por xmm5, xmm6 ; xmm5 = OutH = (16 17 18 ... 29 30 31)
movdqa XMMWORD [rdi + 0 * SIZEOF_XMMWORD], xmm2
movdqa XMMWORD [rdi + 1 * SIZEOF_XMMWORD], xmm5
sub rax, byte SIZEOF_XMMWORD
add rsi, byte 1 * SIZEOF_XMMWORD ; inptr
add rdi, byte 2 * SIZEOF_XMMWORD ; outptr
cmp rax, byte SIZEOF_XMMWORD
ja near .columnloop
test eax, eax
jnz near .columnloop_last
pop rsi
pop rdi
pop rax
add rsi, byte SIZEOF_JSAMPROW ; input_data
add rdi, byte SIZEOF_JSAMPROW ; output_data
dec rcx ; rowctr
jg near .rowloop
.return:
UNCOLLECT_ARGS 4
pop rbp
ret
; --------------------------------------------------------------------------
;
; Fancy processing for the common case of 2:1 horizontal and 2:1 vertical.
; Again a triangle filter; see comments for h2v1 case, above.
;
; GLOBAL(void)
; jsimd_h2v2_fancy_upsample_sse2(int max_v_samp_factor,
; JDIMENSION downsampled_width,
; JSAMPARRAY input_data,
; JSAMPARRAY *output_data_ptr)
;
; r10 = int max_v_samp_factor
; r11d = JDIMENSION downsampled_width
; r12 = JSAMPARRAY input_data
; r13 = JSAMPARRAY *output_data_ptr
%define wk(i) r15 - (WK_NUM - (i)) * SIZEOF_XMMWORD ; xmmword wk[WK_NUM]
%define WK_NUM 4
align 32
GLOBAL_FUNCTION(jsimd_h2v2_fancy_upsample_sse2)
EXTN(jsimd_h2v2_fancy_upsample_sse2):
ENDBR64
push rbp
mov rbp, rsp
push r15
and rsp, byte (-SIZEOF_XMMWORD) ; align to 128 bits
; Allocate stack space for wk array. r15 is used to access it.
mov r15, rsp
sub rsp, byte (SIZEOF_XMMWORD * WK_NUM)
COLLECT_ARGS 4
push rbx
mov eax, r11d ; colctr
test rax, rax
jz near .return
mov rcx, r10 ; rowctr
test rcx, rcx
jz near .return
mov rsi, r12 ; input_data
mov rdi, r13
mov rdip, JSAMPARRAY [rdi] ; output_data
.rowloop:
push rax ; colctr
push rcx
push rdi
push rsi
mov rcxp, JSAMPROW [rsi - 1 * SIZEOF_JSAMPROW] ; inptr1(above)
mov rbxp, JSAMPROW [rsi + 0 * SIZEOF_JSAMPROW] ; inptr0
mov rsip, JSAMPROW [rsi + 1 * SIZEOF_JSAMPROW] ; inptr1(below)
mov rdxp, JSAMPROW [rdi + 0 * SIZEOF_JSAMPROW] ; outptr0
mov rdip, JSAMPROW [rdi + 1 * SIZEOF_JSAMPROW] ; outptr1
test rax, SIZEOF_XMMWORD - 1
jz short .skip
push rdx
mov dl, JSAMPLE [rcx + (rax - 1) * SIZEOF_JSAMPLE]
mov JSAMPLE [rcx + rax * SIZEOF_JSAMPLE], dl
mov dl, JSAMPLE [rbx + (rax - 1) * SIZEOF_JSAMPLE]
mov JSAMPLE [rbx + rax * SIZEOF_JSAMPLE], dl
mov dl, JSAMPLE [rsi + (rax - 1) * SIZEOF_JSAMPLE]
mov JSAMPLE [rsi + rax * SIZEOF_JSAMPLE], dl
; insert a dummy sample
pop rdx
.skip:
; -- process the first column block
movdqa xmm0, XMMWORD [rbx + 0 * SIZEOF_XMMWORD] ; xmm0 = row[ 0][0]
movdqa xmm1, XMMWORD [rcx + 0 * SIZEOF_XMMWORD] ; xmm1 = row[-1][0]
movdqa xmm2, XMMWORD [rsi + 0 * SIZEOF_XMMWORD] ; xmm2 = row[+1][0]
pxor xmm3, xmm3 ; xmm3 = (all 0's)
movdqa xmm4, xmm0
punpcklbw xmm0, xmm3 ; xmm0 = row[ 0]( 0 1 2 3 4 5 6 7)
punpckhbw xmm4, xmm3 ; xmm4 = row[ 0]( 8 9 10 11 12 13 14 15)
movdqa xmm5, xmm1
punpcklbw xmm1, xmm3 ; xmm1 = row[-1]( 0 1 2 3 4 5 6 7)
punpckhbw xmm5, xmm3 ; xmm5 = row[-1]( 8 9 10 11 12 13 14 15)
movdqa xmm6, xmm2
punpcklbw xmm2, xmm3 ; xmm2 = row[+1]( 0 1 2 3 4 5 6 7)
punpckhbw xmm6, xmm3 ; xmm6 = row[+1]( 8 9 10 11 12 13 14 15)
pmullw xmm0, [rel PW_THREE]
pmullw xmm4, [rel PW_THREE]
pcmpeqb xmm7, xmm7
psrldq xmm7, (SIZEOF_XMMWORD - 2)
paddw xmm1, xmm0 ; xmm1 = Int0L = ( 0 1 2 3 4 5 6 7)
paddw xmm5, xmm4 ; xmm5 = Int0H = ( 8 9 10 11 12 13 14 15)
paddw xmm2, xmm0 ; xmm2 = Int1L = ( 0 1 2 3 4 5 6 7)
paddw xmm6, xmm4 ; xmm6 = Int1H = ( 8 9 10 11 12 13 14 15)
; temporarily save the intermediate data
movdqa XMMWORD [rdx + 0 * SIZEOF_XMMWORD], xmm1
movdqa XMMWORD [rdx + 1 * SIZEOF_XMMWORD], xmm5
movdqa XMMWORD [rdi + 0 * SIZEOF_XMMWORD], xmm2
movdqa XMMWORD [rdi + 1 * SIZEOF_XMMWORD], xmm6
pand xmm1, xmm7 ; xmm1 = ( 0 -- -- -- -- -- -- --)
pand xmm2, xmm7 ; xmm2 = ( 0 -- -- -- -- -- -- --)
movdqa XMMWORD [wk(0)], xmm1
movdqa XMMWORD [wk(1)], xmm2
add rax, byte SIZEOF_XMMWORD - 1
and rax, byte -SIZEOF_XMMWORD
cmp rax, byte SIZEOF_XMMWORD
ja short .columnloop
.columnloop_last:
; -- process the last column block
pcmpeqb xmm1, xmm1
pslldq xmm1, (SIZEOF_XMMWORD - 2)
movdqa xmm2, xmm1
pand xmm1, XMMWORD [rdx + 1 * SIZEOF_XMMWORD]
pand xmm2, XMMWORD [rdi + 1 * SIZEOF_XMMWORD]
movdqa XMMWORD [wk(2)], xmm1 ; xmm1 = (-- -- -- -- -- -- -- 15)
movdqa XMMWORD [wk(3)], xmm2 ; xmm2 = (-- -- -- -- -- -- -- 15)
jmp near .upsample
.columnloop:
; -- process the next column block
movdqa xmm0, XMMWORD [rbx + 1 * SIZEOF_XMMWORD] ; xmm0 = row[ 0][1]
movdqa xmm1, XMMWORD [rcx + 1 * SIZEOF_XMMWORD] ; xmm1 = row[-1][1]
movdqa xmm2, XMMWORD [rsi + 1 * SIZEOF_XMMWORD] ; xmm2 = row[+1][1]
pxor xmm3, xmm3 ; xmm3 = (all 0's)
movdqa xmm4, xmm0
punpcklbw xmm0, xmm3 ; xmm0 = row[ 0]( 0 1 2 3 4 5 6 7)
punpckhbw xmm4, xmm3 ; xmm4 = row[ 0]( 8 9 10 11 12 13 14 15)
movdqa xmm5, xmm1
punpcklbw xmm1, xmm3 ; xmm1 = row[-1]( 0 1 2 3 4 5 6 7)
punpckhbw xmm5, xmm3 ; xmm5 = row[-1]( 8 9 10 11 12 13 14 15)
movdqa xmm6, xmm2
punpcklbw xmm2, xmm3 ; xmm2 = row[+1]( 0 1 2 3 4 5 6 7)
punpckhbw xmm6, xmm3 ; xmm6 = row[+1]( 8 9 10 11 12 13 14 15)
pmullw xmm0, [rel PW_THREE]
pmullw xmm4, [rel PW_THREE]
paddw xmm1, xmm0 ; xmm1 = Int0L = ( 0 1 2 3 4 5 6 7)
paddw xmm5, xmm4 ; xmm5 = Int0H = ( 8 9 10 11 12 13 14 15)
paddw xmm2, xmm0 ; xmm2 = Int1L = ( 0 1 2 3 4 5 6 7)
paddw xmm6, xmm4 ; xmm6 = Int1H = ( 8 9 10 11 12 13 14 15)
; temporarily save the intermediate data
movdqa XMMWORD [rdx + 2 * SIZEOF_XMMWORD], xmm1
movdqa XMMWORD [rdx + 3 * SIZEOF_XMMWORD], xmm5
movdqa XMMWORD [rdi + 2 * SIZEOF_XMMWORD], xmm2
movdqa XMMWORD [rdi + 3 * SIZEOF_XMMWORD], xmm6
pslldq xmm1, (SIZEOF_XMMWORD - 2) ; xmm1 = (-- -- -- -- -- -- -- 0)
pslldq xmm2, (SIZEOF_XMMWORD - 2) ; xmm2 = (-- -- -- -- -- -- -- 0)
movdqa XMMWORD [wk(2)], xmm1
movdqa XMMWORD [wk(3)], xmm2
.upsample:
; -- process the upper row
movdqa xmm7, XMMWORD [rdx + 0 * SIZEOF_XMMWORD]
movdqa xmm3, XMMWORD [rdx + 1 * SIZEOF_XMMWORD]
movdqa xmm0, xmm7 ; xmm7 = Int0L = ( 0 1 2 3 4 5 6 7)
movdqa xmm4, xmm3 ; xmm3 = Int0H = ( 8 9 10 11 12 13 14 15)
psrldq xmm0, 2 ; xmm0 = ( 1 2 3 4 5 6 7 --)
pslldq xmm4, (SIZEOF_XMMWORD - 2) ; xmm4 = (-- -- -- -- -- -- -- 8)
movdqa xmm5, xmm7
movdqa xmm6, xmm3
psrldq xmm5, (SIZEOF_XMMWORD - 2) ; xmm5 = ( 7 -- -- -- -- -- -- --)
pslldq xmm6, 2 ; xmm6 = (-- 8 9 10 11 12 13 14)
por xmm0, xmm4 ; xmm0 = ( 1 2 3 4 5 6 7 8)
por xmm5, xmm6 ; xmm5 = ( 7 8 9 10 11 12 13 14)
movdqa xmm1, xmm7
movdqa xmm2, xmm3
pslldq xmm1, 2 ; xmm1 = (-- 0 1 2 3 4 5 6)
psrldq xmm2, 2 ; xmm2 = ( 9 10 11 12 13 14 15 --)
movdqa xmm4, xmm3
psrldq xmm4, (SIZEOF_XMMWORD - 2) ; xmm4 = (15 -- -- -- -- -- -- --)
por xmm1, XMMWORD [wk(0)] ; xmm1 = (-1 0 1 2 3 4 5 6)
por xmm2, XMMWORD [wk(2)] ; xmm2 = ( 9 10 11 12 13 14 15 16)
movdqa XMMWORD [wk(0)], xmm4
pmullw xmm7, [rel PW_THREE]
pmullw xmm3, [rel PW_THREE]
paddw xmm1, [rel PW_EIGHT]
paddw xmm5, [rel PW_EIGHT]
paddw xmm0, [rel PW_SEVEN]
paddw xmm2, [rel PW_SEVEN]
paddw xmm1, xmm7
paddw xmm5, xmm3
psrlw xmm1, 4 ; xmm1 = Out0LE = ( 0 2 4 6 8 10 12 14)
psrlw xmm5, 4 ; xmm5 = Out0HE = (16 18 20 22 24 26 28 30)
paddw xmm0, xmm7
paddw xmm2, xmm3
psrlw xmm0, 4 ; xmm0 = Out0LO = ( 1 3 5 7 9 11 13 15)
psrlw xmm2, 4 ; xmm2 = Out0HO = (17 19 21 23 25 27 29 31)
psllw xmm0, BYTE_BIT
psllw xmm2, BYTE_BIT
por xmm1, xmm0 ; xmm1 = Out0L = ( 0 1 2 ... 13 14 15)
por xmm5, xmm2 ; xmm5 = Out0H = (16 17 18 ... 29 30 31)
movdqa XMMWORD [rdx + 0 * SIZEOF_XMMWORD], xmm1
movdqa XMMWORD [rdx + 1 * SIZEOF_XMMWORD], xmm5
; -- process the lower row
movdqa xmm6, XMMWORD [rdi + 0 * SIZEOF_XMMWORD]
movdqa xmm4, XMMWORD [rdi + 1 * SIZEOF_XMMWORD]
movdqa xmm7, xmm6 ; xmm6 = Int1L = ( 0 1 2 3 4 5 6 7)
movdqa xmm3, xmm4 ; xmm4 = Int1H = ( 8 9 10 11 12 13 14 15)
psrldq xmm7, 2 ; xmm7 = ( 1 2 3 4 5 6 7 --)
pslldq xmm3, (SIZEOF_XMMWORD - 2) ; xmm3 = (-- -- -- -- -- -- -- 8)
movdqa xmm0, xmm6
movdqa xmm2, xmm4
psrldq xmm0, (SIZEOF_XMMWORD - 2) ; xmm0 = ( 7 -- -- -- -- -- -- --)
pslldq xmm2, 2 ; xmm2 = (-- 8 9 10 11 12 13 14)
por xmm7, xmm3 ; xmm7 = ( 1 2 3 4 5 6 7 8)
por xmm0, xmm2 ; xmm0 = ( 7 8 9 10 11 12 13 14)
movdqa xmm1, xmm6
movdqa xmm5, xmm4
pslldq xmm1, 2 ; xmm1 = (-- 0 1 2 3 4 5 6)
psrldq xmm5, 2 ; xmm5 = ( 9 10 11 12 13 14 15 --)
movdqa xmm3, xmm4
psrldq xmm3, (SIZEOF_XMMWORD - 2) ; xmm3 = (15 -- -- -- -- -- -- --)
por xmm1, XMMWORD [wk(1)] ; xmm1 = (-1 0 1 2 3 4 5 6)
por xmm5, XMMWORD [wk(3)] ; xmm5 = ( 9 10 11 12 13 14 15 16)
movdqa XMMWORD [wk(1)], xmm3
pmullw xmm6, [rel PW_THREE]
pmullw xmm4, [rel PW_THREE]
paddw xmm1, [rel PW_EIGHT]
paddw xmm0, [rel PW_EIGHT]
paddw xmm7, [rel PW_SEVEN]
paddw xmm5, [rel PW_SEVEN]
paddw xmm1, xmm6
paddw xmm0, xmm4
psrlw xmm1, 4 ; xmm1 = Out1LE = ( 0 2 4 6 8 10 12 14)
psrlw xmm0, 4 ; xmm0 = Out1HE = (16 18 20 22 24 26 28 30)
paddw xmm7, xmm6
paddw xmm5, xmm4
psrlw xmm7, 4 ; xmm7 = Out1LO = ( 1 3 5 7 9 11 13 15)
psrlw xmm5, 4 ; xmm5 = Out1HO = (17 19 21 23 25 27 29 31)
psllw xmm7, BYTE_BIT
psllw xmm5, BYTE_BIT
por xmm1, xmm7 ; xmm1 = Out1L = ( 0 1 2 ... 13 14 15)
por xmm0, xmm5 ; xmm0 = Out1H = (16 17 18 ... 29 30 31)
movdqa XMMWORD [rdi + 0 * SIZEOF_XMMWORD], xmm1
movdqa XMMWORD [rdi + 1 * SIZEOF_XMMWORD], xmm0
sub rax, byte SIZEOF_XMMWORD
add rcx, byte 1 * SIZEOF_XMMWORD ; inptr1(above)
add rbx, byte 1 * SIZEOF_XMMWORD ; inptr0
add rsi, byte 1 * SIZEOF_XMMWORD ; inptr1(below)
add rdx, byte 2 * SIZEOF_XMMWORD ; outptr0
add rdi, byte 2 * SIZEOF_XMMWORD ; outptr1
cmp rax, byte SIZEOF_XMMWORD
ja near .columnloop
test rax, rax
jnz near .columnloop_last
pop rsi
pop rdi
pop rcx
pop rax
add rsi, byte 1 * SIZEOF_JSAMPROW ; input_data
add rdi, byte 2 * SIZEOF_JSAMPROW ; output_data
sub rcx, byte 2 ; rowctr
jg near .rowloop
.return:
pop rbx
UNCOLLECT_ARGS 4
lea rsp, [rbp - 8]
pop r15
pop rbp
ret
; --------------------------------------------------------------------------
;
; Fast processing for the common case of 2:1 horizontal and 1:1 vertical.
; It's still a box filter.
;
; GLOBAL(void)
; jsimd_h2v1_upsample_sse2(int max_v_samp_factor, JDIMENSION output_width,
; JSAMPARRAY input_data, JSAMPARRAY *output_data_ptr)
;
; r10 = int max_v_samp_factor
; r11d = JDIMENSION output_width
; r12 = JSAMPARRAY input_data
; r13 = JSAMPARRAY *output_data_ptr
align 32
GLOBAL_FUNCTION(jsimd_h2v1_upsample_sse2)
EXTN(jsimd_h2v1_upsample_sse2):
ENDBR64
push rbp
mov rbp, rsp
COLLECT_ARGS 4
mov edx, r11d
add rdx, byte (2 * SIZEOF_XMMWORD) - 1
and rdx, byte -(2 * SIZEOF_XMMWORD)
jz near .return
mov rcx, r10 ; rowctr
test rcx, rcx
jz short .return
mov rsi, r12 ; input_data
mov rdi, r13
mov rdip, JSAMPARRAY [rdi] ; output_data
.rowloop:
push rdi
push rsi
mov rsip, JSAMPROW [rsi] ; inptr
mov rdip, JSAMPROW [rdi] ; outptr
mov rax, rdx ; colctr
.columnloop:
movdqa xmm0, XMMWORD [rsi + 0 * SIZEOF_XMMWORD]
movdqa xmm1, xmm0
punpcklbw xmm0, xmm0
punpckhbw xmm1, xmm1
movdqa XMMWORD [rdi + 0 * SIZEOF_XMMWORD], xmm0
movdqa XMMWORD [rdi + 1 * SIZEOF_XMMWORD], xmm1
sub rax, byte 2 * SIZEOF_XMMWORD
jz short .nextrow
movdqa xmm2, XMMWORD [rsi + 1 * SIZEOF_XMMWORD]
movdqa xmm3, xmm2
punpcklbw xmm2, xmm2
punpckhbw xmm3, xmm3
movdqa XMMWORD [rdi + 2 * SIZEOF_XMMWORD], xmm2
movdqa XMMWORD [rdi + 3 * SIZEOF_XMMWORD], xmm3
sub rax, byte 2 * SIZEOF_XMMWORD
jz short .nextrow
add rsi, byte 2 * SIZEOF_XMMWORD ; inptr
add rdi, byte 4 * SIZEOF_XMMWORD ; outptr
jmp short .columnloop
.nextrow:
pop rsi
pop rdi
add rsi, byte SIZEOF_JSAMPROW ; input_data
add rdi, byte SIZEOF_JSAMPROW ; output_data
dec rcx ; rowctr
jg short .rowloop
.return:
UNCOLLECT_ARGS 4
pop rbp
ret
; --------------------------------------------------------------------------
;
; Fast processing for the common case of 2:1 horizontal and 2:1 vertical.
; It's still a box filter.
;
; GLOBAL(void)
; jsimd_h2v2_upsample_sse2(int max_v_samp_factor, JDIMENSION output_width,
; JSAMPARRAY input_data, JSAMPARRAY *output_data_ptr)
;
; r10 = int max_v_samp_factor
; r11d = JDIMENSION output_width
; r12 = JSAMPARRAY input_data
; r13 = JSAMPARRAY *output_data_ptr
align 32
GLOBAL_FUNCTION(jsimd_h2v2_upsample_sse2)
EXTN(jsimd_h2v2_upsample_sse2):
ENDBR64
push rbp
mov rbp, rsp
COLLECT_ARGS 4
push rbx
mov edx, r11d
add rdx, byte (2 * SIZEOF_XMMWORD) - 1
and rdx, byte -(2 * SIZEOF_XMMWORD)
jz near .return
mov rcx, r10 ; rowctr
test rcx, rcx
jz near .return
mov rsi, r12 ; input_data
mov rdi, r13
mov rdip, JSAMPARRAY [rdi] ; output_data
.rowloop:
push rdi
push rsi
mov rsip, JSAMPROW [rsi] ; inptr
mov rbxp, JSAMPROW [rdi + 0 * SIZEOF_JSAMPROW] ; outptr0
mov rdip, JSAMPROW [rdi + 1 * SIZEOF_JSAMPROW] ; outptr1
mov rax, rdx ; colctr
.columnloop:
movdqa xmm0, XMMWORD [rsi + 0 * SIZEOF_XMMWORD]
movdqa xmm1, xmm0
punpcklbw xmm0, xmm0
punpckhbw xmm1, xmm1
movdqa XMMWORD [rbx + 0 * SIZEOF_XMMWORD], xmm0
movdqa XMMWORD [rbx + 1 * SIZEOF_XMMWORD], xmm1
movdqa XMMWORD [rdi + 0 * SIZEOF_XMMWORD], xmm0
movdqa XMMWORD [rdi + 1 * SIZEOF_XMMWORD], xmm1
sub rax, byte 2 * SIZEOF_XMMWORD
jz short .nextrow
movdqa xmm2, XMMWORD [rsi + 1 * SIZEOF_XMMWORD]
movdqa xmm3, xmm2
punpcklbw xmm2, xmm2
punpckhbw xmm3, xmm3
movdqa XMMWORD [rbx + 2 * SIZEOF_XMMWORD], xmm2
movdqa XMMWORD [rbx + 3 * SIZEOF_XMMWORD], xmm3
movdqa XMMWORD [rdi + 2 * SIZEOF_XMMWORD], xmm2
movdqa XMMWORD [rdi + 3 * SIZEOF_XMMWORD], xmm3
sub rax, byte 2 * SIZEOF_XMMWORD
jz short .nextrow
add rsi, byte 2 * SIZEOF_XMMWORD ; inptr
add rbx, byte 4 * SIZEOF_XMMWORD ; outptr0
add rdi, byte 4 * SIZEOF_XMMWORD ; outptr1
jmp short .columnloop
.nextrow:
pop rsi
pop rdi
add rsi, byte 1 * SIZEOF_JSAMPROW ; input_data
add rdi, byte 2 * SIZEOF_JSAMPROW ; output_data
sub rcx, byte 2 ; rowctr
jg near .rowloop
.return:
pop rbx
UNCOLLECT_ARGS 4
pop rbp
ret
; For some reason, the OS X linker does not honor the request to align the
; segment unless we do this.
align 32
lib/libjpeg-turbo/simd/x86_64/jfdctflt-sse.asm
Executable
+360
View File
@@ -0,0 +1,360 @@
;
; Floating-point FDCT (64-bit SSE)
;
; Copyright 2009 Pierre Ossman <[email protected]> for Cendio AB
; Copyright (C) 2009, 2016, 2024-2025, D. R. Commander.
; Copyright (C) 2023, Aliaksiej Kandracienka.
;
; Based on the x86 SIMD extension for IJG JPEG library
; Copyright (C) 1999-2006, MIYASAKA Masaru.
; For conditions of distribution and use, see copyright notice in jsimdext.inc
;
; This file should be assembled with NASM (Netwide Assembler) or Yasm.
;
; This file contains a floating-point implementation of the forward DCT
; (Discrete Cosine Transform). The following code is based directly on the
; IJG's original jfdctflt.c; see jfdctflt.c for more details.
%include "jsimdext.inc"
%include "jdct.inc"
; --------------------------------------------------------------------------
%macro unpcklps2 2 ; %1 = (0 1 2 3) / %2 = (4 5 6 7) => %1 = (0 1 4 5)
shufps %1, %2, 0x44
%endmacro
%macro unpckhps2 2 ; %1 = (0 1 2 3) / %2 = (4 5 6 7) => %1 = (2 3 6 7)
shufps %1, %2, 0xEE
%endmacro
; --------------------------------------------------------------------------
SECTION SEG_CONST
ALIGNZ 32
GLOBAL_DATA(jconst_fdct_float_sse)
EXTN(jconst_fdct_float_sse):
PD_0_382 times 4 dd 0.382683432365089771728460
PD_0_707 times 4 dd 0.707106781186547524400844
PD_0_541 times 4 dd 0.541196100146196984399723
PD_1_306 times 4 dd 1.306562964876376527856643
ALIGNZ 32
; --------------------------------------------------------------------------
SECTION SEG_TEXT
BITS 64
; Perform the forward DCT on one block of samples.
;
; GLOBAL(void)
; jsimd_fdct_float_sse(FAST_FLOAT *data)
;
; r10 = FAST_FLOAT *data
%define wk(i) r15 - (WK_NUM - (i)) * SIZEOF_XMMWORD ; xmmword wk[WK_NUM]
%define WK_NUM 2
align 32
GLOBAL_FUNCTION(jsimd_fdct_float_sse)
EXTN(jsimd_fdct_float_sse):
ENDBR64
push rbp
mov rbp, rsp
push r15
and rsp, byte (-SIZEOF_XMMWORD) ; align to 128 bits
; Allocate stack space for wk array. r15 is used to access it.
mov r15, rsp
sub rsp, byte (SIZEOF_XMMWORD * WK_NUM)
COLLECT_ARGS 1
; ---- Pass 1: process rows.
mov rdx, r10 ; (FAST_FLOAT *)
mov rcx, DCTSIZE / 4
.rowloop:
movaps xmm0, XMMWORD [XMMBLOCK(2, 0, rdx, SIZEOF_FAST_FLOAT)]
movaps xmm1, XMMWORD [XMMBLOCK(3, 0, rdx, SIZEOF_FAST_FLOAT)]
movaps xmm2, XMMWORD [XMMBLOCK(2, 1, rdx, SIZEOF_FAST_FLOAT)]
movaps xmm3, XMMWORD [XMMBLOCK(3, 1, rdx, SIZEOF_FAST_FLOAT)]
; xmm0 = (20 21 22 23)
; xmm2 = (24 25 26 27)
; xmm1 = (30 31 32 33)
; xmm3 = (34 35 36 37)
movaps xmm4, xmm0 ; transpose coefficients(phase 1)
unpcklps xmm0, xmm1 ; xmm0 = (20 30 21 31)
unpckhps xmm4, xmm1 ; xmm4 = (22 32 23 33)
movaps xmm5, xmm2 ; transpose coefficients(phase 1)
unpcklps xmm2, xmm3 ; xmm2 = (24 34 25 35)
unpckhps xmm5, xmm3 ; xmm5 = (26 36 27 37)
movaps xmm6, XMMWORD [XMMBLOCK(0, 0, rdx, SIZEOF_FAST_FLOAT)]
movaps xmm7, XMMWORD [XMMBLOCK(1, 0, rdx, SIZEOF_FAST_FLOAT)]
movaps xmm1, XMMWORD [XMMBLOCK(0, 1, rdx, SIZEOF_FAST_FLOAT)]
movaps xmm3, XMMWORD [XMMBLOCK(1, 1, rdx, SIZEOF_FAST_FLOAT)]
; xmm6 = (00 01 02 03)
; xmm1 = (04 05 06 07)
; xmm7 = (10 11 12 13)
; xmm3 = (14 15 16 17)
movaps XMMWORD [wk(0)], xmm4 ; wk(0) = (22 32 23 33)
movaps XMMWORD [wk(1)], xmm2 ; wk(1) = (24 34 25 35)
movaps xmm4, xmm6 ; transpose coefficients(phase 1)
unpcklps xmm6, xmm7 ; xmm6 = (00 10 01 11)
unpckhps xmm4, xmm7 ; xmm4 = (02 12 03 13)
movaps xmm2, xmm1 ; transpose coefficients(phase 1)
unpcklps xmm1, xmm3 ; xmm1 = (04 14 05 15)
unpckhps xmm2, xmm3 ; xmm2 = (06 16 07 17)
movaps xmm7, xmm6 ; transpose coefficients(phase 2)
unpcklps2 xmm6, xmm0 ; xmm6 = (00 10 20 30) = data0
unpckhps2 xmm7, xmm0 ; xmm7 = (01 11 21 31) = data1
movaps xmm3, xmm2 ; transpose coefficients(phase 2)
unpcklps2 xmm2, xmm5 ; xmm2 = (06 16 26 36) = data6
unpckhps2 xmm3, xmm5 ; xmm3 = (07 17 27 37) = data7
movaps xmm0, xmm7
movaps xmm5, xmm6
subps xmm7, xmm2 ; xmm7 = data1 - data6 = tmp6
subps xmm6, xmm3 ; xmm6 = data0 - data7 = tmp7
addps xmm0, xmm2 ; xmm0 = data1 + data6 = tmp1
addps xmm5, xmm3 ; xmm5 = data0 + data7 = tmp0
movaps xmm2, XMMWORD [wk(0)] ; xmm2 = (22 32 23 33)
movaps xmm3, XMMWORD [wk(1)] ; xmm3 = (24 34 25 35)
movaps XMMWORD [wk(0)], xmm7 ; wk(0) = tmp6
movaps XMMWORD [wk(1)], xmm6 ; wk(1) = tmp7
movaps xmm7, xmm4 ; transpose coefficients(phase 2)
unpcklps2 xmm4, xmm2 ; xmm4 = (02 12 22 32) = data2
unpckhps2 xmm7, xmm2 ; xmm7 = (03 13 23 33) = data3
movaps xmm6, xmm1 ; transpose coefficients(phase 2)
unpcklps2 xmm1, xmm3 ; xmm1 = (04 14 24 34) = data4
unpckhps2 xmm6, xmm3 ; xmm6 = (05 15 25 35) = data5
movaps xmm2, xmm7
movaps xmm3, xmm4
addps xmm7, xmm1 ; xmm7 = data3 + data4 = tmp3
addps xmm4, xmm6 ; xmm4 = data2 + data5 = tmp2
subps xmm2, xmm1 ; xmm2 = data3 - data4 = tmp4
subps xmm3, xmm6 ; xmm3 = data2 - data5 = tmp5
; -- Even part
movaps xmm1, xmm5
movaps xmm6, xmm0
subps xmm5, xmm7 ; xmm5 = tmp13
subps xmm0, xmm4 ; xmm0 = tmp12
addps xmm1, xmm7 ; xmm1 = tmp10
addps xmm6, xmm4 ; xmm6 = tmp11
addps xmm0, xmm5
mulps xmm0, [rel PD_0_707] ; xmm0 = z1
movaps xmm7, xmm1
movaps xmm4, xmm5
subps xmm1, xmm6 ; xmm1 = data4
subps xmm5, xmm0 ; xmm5 = data6
addps xmm7, xmm6 ; xmm7 = data0
addps xmm4, xmm0 ; xmm4 = data2
movaps XMMWORD [XMMBLOCK(0, 1, rdx, SIZEOF_FAST_FLOAT)], xmm1
movaps XMMWORD [XMMBLOCK(2, 1, rdx, SIZEOF_FAST_FLOAT)], xmm5
movaps XMMWORD [XMMBLOCK(0, 0, rdx, SIZEOF_FAST_FLOAT)], xmm7
movaps XMMWORD [XMMBLOCK(2, 0, rdx, SIZEOF_FAST_FLOAT)], xmm4
; -- Odd part
movaps xmm6, XMMWORD [wk(0)] ; xmm6 = tmp6
movaps xmm0, XMMWORD [wk(1)] ; xmm0 = tmp7
addps xmm2, xmm3 ; xmm2 = tmp10
addps xmm3, xmm6 ; xmm3 = tmp11
addps xmm6, xmm0 ; xmm6 = tmp12, xmm0 = tmp7
mulps xmm3, [rel PD_0_707] ; xmm3 = z3
movaps xmm1, xmm2 ; xmm1 = tmp10
subps xmm2, xmm6
mulps xmm2, [rel PD_0_382] ; xmm2 = z5
mulps xmm1, [rel PD_0_541] ; xmm1 = MULTIPLY(tmp10, FIX_0_541196)
mulps xmm6, [rel PD_1_306] ; xmm6 = MULTIPLY(tmp12, FIX_1_306562)
addps xmm1, xmm2 ; xmm1 = z2
addps xmm6, xmm2 ; xmm6 = z4
movaps xmm5, xmm0
subps xmm0, xmm3 ; xmm0 = z13
addps xmm5, xmm3 ; xmm5 = z11
movaps xmm7, xmm0
movaps xmm4, xmm5
subps xmm0, xmm1 ; xmm0 = data3
subps xmm5, xmm6 ; xmm5 = data7
addps xmm7, xmm1 ; xmm7 = data5
addps xmm4, xmm6 ; xmm4 = data1
movaps XMMWORD [XMMBLOCK(3, 0, rdx, SIZEOF_FAST_FLOAT)], xmm0
movaps XMMWORD [XMMBLOCK(3, 1, rdx, SIZEOF_FAST_FLOAT)], xmm5
movaps XMMWORD [XMMBLOCK(1, 1, rdx, SIZEOF_FAST_FLOAT)], xmm7
movaps XMMWORD [XMMBLOCK(1, 0, rdx, SIZEOF_FAST_FLOAT)], xmm4
add rdx, 4 * DCTSIZE * SIZEOF_FAST_FLOAT
dec rcx
jnz near .rowloop
; ---- Pass 2: process columns.
mov rdx, r10 ; (FAST_FLOAT *)
mov rcx, DCTSIZE / 4
.columnloop:
movaps xmm0, XMMWORD [XMMBLOCK(2, 0, rdx, SIZEOF_FAST_FLOAT)]
movaps xmm1, XMMWORD [XMMBLOCK(3, 0, rdx, SIZEOF_FAST_FLOAT)]
movaps xmm2, XMMWORD [XMMBLOCK(6, 0, rdx, SIZEOF_FAST_FLOAT)]
movaps xmm3, XMMWORD [XMMBLOCK(7, 0, rdx, SIZEOF_FAST_FLOAT)]
; xmm0 = (02 12 22 32)
; xmm2 = (42 52 62 72)
; xmm1 = (03 13 23 33)
; xmm3 = (43 53 63 73)
movaps xmm4, xmm0 ; transpose coefficients(phase 1)
unpcklps xmm0, xmm1 ; xmm0 = (02 03 12 13)
unpckhps xmm4, xmm1 ; xmm4 = (22 23 32 33)
movaps xmm5, xmm2 ; transpose coefficients(phase 1)
unpcklps xmm2, xmm3 ; xmm2 = (42 43 52 53)
unpckhps xmm5, xmm3 ; xmm5 = (62 63 72 73)
movaps xmm6, XMMWORD [XMMBLOCK(0, 0, rdx, SIZEOF_FAST_FLOAT)]
movaps xmm7, XMMWORD [XMMBLOCK(1, 0, rdx, SIZEOF_FAST_FLOAT)]
movaps xmm1, XMMWORD [XMMBLOCK(4, 0, rdx, SIZEOF_FAST_FLOAT)]
movaps xmm3, XMMWORD [XMMBLOCK(5, 0, rdx, SIZEOF_FAST_FLOAT)]
; xmm6 = (00 10 20 30)
; xmm1 = (40 50 60 70)
; xmm7 = (01 11 21 31)
; xmm3 = (41 51 61 71)
movaps XMMWORD [wk(0)], xmm4 ; wk(0) = (22 23 32 33)
movaps XMMWORD [wk(1)], xmm2 ; wk(1) = (42 43 52 53)
movaps xmm4, xmm6 ; transpose coefficients(phase 1)
unpcklps xmm6, xmm7 ; xmm6 = (00 01 10 11)
unpckhps xmm4, xmm7 ; xmm4 = (20 21 30 31)
movaps xmm2, xmm1 ; transpose coefficients(phase 1)
unpcklps xmm1, xmm3 ; xmm1 = (40 41 50 51)
unpckhps xmm2, xmm3 ; xmm2 = (60 61 70 71)
movaps xmm7, xmm6 ; transpose coefficients(phase 2)
unpcklps2 xmm6, xmm0 ; xmm6 = (00 01 02 03) = data0
unpckhps2 xmm7, xmm0 ; xmm7 = (10 11 12 13) = data1
movaps xmm3, xmm2 ; transpose coefficients(phase 2)
unpcklps2 xmm2, xmm5 ; xmm2 = (60 61 62 63) = data6
unpckhps2 xmm3, xmm5 ; xmm3 = (70 71 72 73) = data7
movaps xmm0, xmm7
movaps xmm5, xmm6
subps xmm7, xmm2 ; xmm7 = data1 - data6 = tmp6
subps xmm6, xmm3 ; xmm6 = data0 - data7 = tmp7
addps xmm0, xmm2 ; xmm0 = data1 + data6 = tmp1
addps xmm5, xmm3 ; xmm5 = data0 + data7 = tmp0
movaps xmm2, XMMWORD [wk(0)] ; xmm2 = (22 23 32 33)
movaps xmm3, XMMWORD [wk(1)] ; xmm3 = (42 43 52 53)
movaps XMMWORD [wk(0)], xmm7 ; wk(0) = tmp6
movaps XMMWORD [wk(1)], xmm6 ; wk(1) = tmp7
movaps xmm7, xmm4 ; transpose coefficients(phase 2)
unpcklps2 xmm4, xmm2 ; xmm4 = (20 21 22 23) = data2
unpckhps2 xmm7, xmm2 ; xmm7 = (30 31 32 33) = data3
movaps xmm6, xmm1 ; transpose coefficients(phase 2)
unpcklps2 xmm1, xmm3 ; xmm1 = (40 41 42 43) = data4
unpckhps2 xmm6, xmm3 ; xmm6 = (50 51 52 53) = data5
movaps xmm2, xmm7
movaps xmm3, xmm4
addps xmm7, xmm1 ; xmm7 = data3 + data4 = tmp3
addps xmm4, xmm6 ; xmm4 = data2 + data5 = tmp2
subps xmm2, xmm1 ; xmm2 = data3 - data4 = tmp4
subps xmm3, xmm6 ; xmm3 = data2 - data5 = tmp5
; -- Even part
movaps xmm1, xmm5
movaps xmm6, xmm0
subps xmm5, xmm7 ; xmm5 = tmp13
subps xmm0, xmm4 ; xmm0 = tmp12
addps xmm1, xmm7 ; xmm1 = tmp10
addps xmm6, xmm4 ; xmm6 = tmp11
addps xmm0, xmm5
mulps xmm0, [rel PD_0_707] ; xmm0 = z1
movaps xmm7, xmm1
movaps xmm4, xmm5
subps xmm1, xmm6 ; xmm1 = data4
subps xmm5, xmm0 ; xmm5 = data6
addps xmm7, xmm6 ; xmm7 = data0
addps xmm4, xmm0 ; xmm4 = data2
movaps XMMWORD [XMMBLOCK(4, 0, rdx, SIZEOF_FAST_FLOAT)], xmm1
movaps XMMWORD [XMMBLOCK(6, 0, rdx, SIZEOF_FAST_FLOAT)], xmm5
movaps XMMWORD [XMMBLOCK(0, 0, rdx, SIZEOF_FAST_FLOAT)], xmm7
movaps XMMWORD [XMMBLOCK(2, 0, rdx, SIZEOF_FAST_FLOAT)], xmm4
; -- Odd part
movaps xmm6, XMMWORD [wk(0)] ; xmm6 = tmp6
movaps xmm0, XMMWORD [wk(1)] ; xmm0 = tmp7
addps xmm2, xmm3 ; xmm2 = tmp10
addps xmm3, xmm6 ; xmm3 = tmp11
addps xmm6, xmm0 ; xmm6 = tmp12, xmm0 = tmp7
mulps xmm3, [rel PD_0_707] ; xmm3 = z3
movaps xmm1, xmm2 ; xmm1 = tmp10
subps xmm2, xmm6
mulps xmm2, [rel PD_0_382] ; xmm2 = z5
mulps xmm1, [rel PD_0_541] ; xmm1 = MULTIPLY(tmp10, FIX_0_541196)
mulps xmm6, [rel PD_1_306] ; xmm6 = MULTIPLY(tmp12, FIX_1_306562)
addps xmm1, xmm2 ; xmm1 = z2
addps xmm6, xmm2 ; xmm6 = z4
movaps xmm5, xmm0
subps xmm0, xmm3 ; xmm0 = z13
addps xmm5, xmm3 ; xmm5 = z11
movaps xmm7, xmm0
movaps xmm4, xmm5
subps xmm0, xmm1 ; xmm0 = data3
subps xmm5, xmm6 ; xmm5 = data7
addps xmm7, xmm1 ; xmm7 = data5
addps xmm4, xmm6 ; xmm4 = data1
movaps XMMWORD [XMMBLOCK(3, 0, rdx, SIZEOF_FAST_FLOAT)], xmm0
movaps XMMWORD [XMMBLOCK(7, 0, rdx, SIZEOF_FAST_FLOAT)], xmm5
movaps XMMWORD [XMMBLOCK(5, 0, rdx, SIZEOF_FAST_FLOAT)], xmm7
movaps XMMWORD [XMMBLOCK(1, 0, rdx, SIZEOF_FAST_FLOAT)], xmm4
add rdx, byte 4 * SIZEOF_FAST_FLOAT
dec rcx
jnz near .columnloop
UNCOLLECT_ARGS 1
lea rsp, [rbp - 8]
pop r15
pop rbp
ret
; For some reason, the OS X linker does not honor the request to align the
; segment unless we do this.
align 32
lib/libjpeg-turbo/simd/x86_64/jfdctint-sse2.asm
Executable
+623
View File
@@ -0,0 +1,623 @@
;
; Accurate integer FDCT (64-bit SSE2)
;
; Copyright 2009 Pierre Ossman <[email protected]> for Cendio AB
; Copyright (C) 2009, 2016, 2020, 2024-2025, D. R. Commander.
; Copyright (C) 2023, Aliaksiej Kandracienka.
;
; Based on the x86 SIMD extension for IJG JPEG library
; Copyright (C) 1999-2006, MIYASAKA Masaru.
; For conditions of distribution and use, see copyright notice in jsimdext.inc
;
; This file should be assembled with NASM (Netwide Assembler) or Yasm.
;
; This file contains a slower but more accurate integer implementation of the
; forward DCT (Discrete Cosine Transform). The following code is based
; directly on the IJG's original jfdctint.c; see jfdctint.c for more details.
%include "jsimdext.inc"
%include "jdct.inc"
; --------------------------------------------------------------------------
%define CONST_BITS 13
%define PASS1_BITS 2
%define DESCALE_P1 (CONST_BITS - PASS1_BITS)
%define DESCALE_P2 (CONST_BITS + PASS1_BITS)
%if CONST_BITS == 13
F_0_298 equ 2446 ; FIX(0.298631336)
F_0_390 equ 3196 ; FIX(0.390180644)
F_0_541 equ 4433 ; FIX(0.541196100)
F_0_765 equ 6270 ; FIX(0.765366865)
F_0_899 equ 7373 ; FIX(0.899976223)
F_1_175 equ 9633 ; FIX(1.175875602)
F_1_501 equ 12299 ; FIX(1.501321110)
F_1_847 equ 15137 ; FIX(1.847759065)
F_1_961 equ 16069 ; FIX(1.961570560)
F_2_053 equ 16819 ; FIX(2.053119869)
F_2_562 equ 20995 ; FIX(2.562915447)
F_3_072 equ 25172 ; FIX(3.072711026)
%else
; NASM cannot do compile-time arithmetic on floating-point constants.
%define DESCALE(x, n) (((x) + (1 << ((n) - 1))) >> (n))
F_0_298 equ DESCALE( 320652955, 30 - CONST_BITS) ; FIX(0.298631336)
F_0_390 equ DESCALE( 418953276, 30 - CONST_BITS) ; FIX(0.390180644)
F_0_541 equ DESCALE( 581104887, 30 - CONST_BITS) ; FIX(0.541196100)
F_0_765 equ DESCALE( 821806413, 30 - CONST_BITS) ; FIX(0.765366865)
F_0_899 equ DESCALE( 966342111, 30 - CONST_BITS) ; FIX(0.899976223)
F_1_175 equ DESCALE(1262586813, 30 - CONST_BITS) ; FIX(1.175875602)
F_1_501 equ DESCALE(1612031267, 30 - CONST_BITS) ; FIX(1.501321110)
F_1_847 equ DESCALE(1984016188, 30 - CONST_BITS) ; FIX(1.847759065)
F_1_961 equ DESCALE(2106220350, 30 - CONST_BITS) ; FIX(1.961570560)
F_2_053 equ DESCALE(2204520673, 30 - CONST_BITS) ; FIX(2.053119869)
F_2_562 equ DESCALE(2751909506, 30 - CONST_BITS) ; FIX(2.562915447)
F_3_072 equ DESCALE(3299298341, 30 - CONST_BITS) ; FIX(3.072711026)
%endif
; --------------------------------------------------------------------------
SECTION SEG_CONST
ALIGNZ 32
GLOBAL_DATA(jconst_fdct_islow_sse2)
EXTN(jconst_fdct_islow_sse2):
PW_F130_F054 times 4 dw (F_0_541 + F_0_765), F_0_541
PW_F054_MF130 times 4 dw F_0_541, (F_0_541 - F_1_847)
PW_MF078_F117 times 4 dw (F_1_175 - F_1_961), F_1_175
PW_F117_F078 times 4 dw F_1_175, (F_1_175 - F_0_390)
PW_MF060_MF089 times 4 dw (F_0_298 - F_0_899), -F_0_899
PW_MF089_F060 times 4 dw -F_0_899, (F_1_501 - F_0_899)
PW_MF050_MF256 times 4 dw (F_2_053 - F_2_562), -F_2_562
PW_MF256_F050 times 4 dw -F_2_562, (F_3_072 - F_2_562)
PD_DESCALE_P1 times 4 dd 1 << (DESCALE_P1 - 1)
PD_DESCALE_P2 times 4 dd 1 << (DESCALE_P2 - 1)
PW_DESCALE_P2X times 8 dw 1 << (PASS1_BITS - 1)
ALIGNZ 32
; --------------------------------------------------------------------------
SECTION SEG_TEXT
BITS 64
; Perform the forward DCT on one block of samples.
;
; GLOBAL(void)
; jsimd_fdct_islow_sse2(DCTELEM *data)
;
; r10 = DCTELEM *data
%define wk(i) r15 - (WK_NUM - (i)) * SIZEOF_XMMWORD ; xmmword wk[WK_NUM]
%define WK_NUM 6
align 32
GLOBAL_FUNCTION(jsimd_fdct_islow_sse2)
EXTN(jsimd_fdct_islow_sse2):
ENDBR64
push rbp
mov rbp, rsp
push r15
and rsp, byte (-SIZEOF_XMMWORD) ; align to 128 bits
; Allocate stack space for wk array. r15 is used to access it.
mov r15, rsp
sub rsp, byte (SIZEOF_XMMWORD * WK_NUM)
COLLECT_ARGS 1
; ---- Pass 1: process rows.
mov rdx, r10 ; (DCTELEM *)
movdqa xmm0, XMMWORD [XMMBLOCK(0, 0, rdx, SIZEOF_DCTELEM)]
movdqa xmm1, XMMWORD [XMMBLOCK(1, 0, rdx, SIZEOF_DCTELEM)]
movdqa xmm2, XMMWORD [XMMBLOCK(2, 0, rdx, SIZEOF_DCTELEM)]
movdqa xmm3, XMMWORD [XMMBLOCK(3, 0, rdx, SIZEOF_DCTELEM)]
; xmm0 = (00 01 02 03 04 05 06 07)
; xmm2 = (20 21 22 23 24 25 26 27)
; xmm1 = (10 11 12 13 14 15 16 17)
; xmm3 = (30 31 32 33 34 35 36 37)
movdqa xmm4, xmm0 ; transpose coefficients(phase 1)
punpcklwd xmm0, xmm1 ; xmm0 = (00 10 01 11 02 12 03 13)
punpckhwd xmm4, xmm1 ; xmm4 = (04 14 05 15 06 16 07 17)
movdqa xmm5, xmm2 ; transpose coefficients(phase 1)
punpcklwd xmm2, xmm3 ; xmm2 = (20 30 21 31 22 32 23 33)
punpckhwd xmm5, xmm3 ; xmm5 = (24 34 25 35 26 36 27 37)
movdqa xmm6, XMMWORD [XMMBLOCK(4, 0, rdx, SIZEOF_DCTELEM)]
movdqa xmm7, XMMWORD [XMMBLOCK(5, 0, rdx, SIZEOF_DCTELEM)]
movdqa xmm1, XMMWORD [XMMBLOCK(6, 0, rdx, SIZEOF_DCTELEM)]
movdqa xmm3, XMMWORD [XMMBLOCK(7, 0, rdx, SIZEOF_DCTELEM)]
; xmm6 = ( 4 12 20 28 36 44 52 60)
; xmm1 = ( 6 14 22 30 38 46 54 62)
; xmm7 = ( 5 13 21 29 37 45 53 61)
; xmm3 = ( 7 15 23 31 39 47 55 63)
movdqa XMMWORD [wk(0)], xmm2 ; wk(0) = (20 30 21 31 22 32 23 33)
movdqa XMMWORD [wk(1)], xmm5 ; wk(1) = (24 34 25 35 26 36 27 37)
movdqa xmm2, xmm6 ; transpose coefficients(phase 1)
punpcklwd xmm6, xmm7 ; xmm6 = (40 50 41 51 42 52 43 53)
punpckhwd xmm2, xmm7 ; xmm2 = (44 54 45 55 46 56 47 57)
movdqa xmm5, xmm1 ; transpose coefficients(phase 1)
punpcklwd xmm1, xmm3 ; xmm1 = (60 70 61 71 62 72 63 73)
punpckhwd xmm5, xmm3 ; xmm5 = (64 74 65 75 66 76 67 77)
movdqa xmm7, xmm6 ; transpose coefficients(phase 2)
punpckldq xmm6, xmm1 ; xmm6 = (40 50 60 70 41 51 61 71)
punpckhdq xmm7, xmm1 ; xmm7 = (42 52 62 72 43 53 63 73)
movdqa xmm3, xmm2 ; transpose coefficients(phase 2)
punpckldq xmm2, xmm5 ; xmm2 = (44 54 64 74 45 55 65 75)
punpckhdq xmm3, xmm5 ; xmm3 = (46 56 66 76 47 57 67 77)
movdqa xmm1, XMMWORD [wk(0)] ; xmm1 = (20 30 21 31 22 32 23 33)
movdqa xmm5, XMMWORD [wk(1)] ; xmm5 = (24 34 25 35 26 36 27 37)
movdqa XMMWORD [wk(2)], xmm7 ; wk(2) = (42 52 62 72 43 53 63 73)
movdqa XMMWORD [wk(3)], xmm2 ; wk(3) = (44 54 64 74 45 55 65 75)
movdqa xmm7, xmm0 ; transpose coefficients(phase 2)
punpckldq xmm0, xmm1 ; xmm0 = (00 10 20 30 01 11 21 31)
punpckhdq xmm7, xmm1 ; xmm7 = (02 12 22 32 03 13 23 33)
movdqa xmm2, xmm4 ; transpose coefficients(phase 2)
punpckldq xmm4, xmm5 ; xmm4 = (04 14 24 34 05 15 25 35)
punpckhdq xmm2, xmm5 ; xmm2 = (06 16 26 36 07 17 27 37)
movdqa xmm1, xmm0 ; transpose coefficients(phase 3)
punpcklqdq xmm0, xmm6 ; xmm0 = (00 10 20 30 40 50 60 70) = data0
punpckhqdq xmm1, xmm6 ; xmm1 = (01 11 21 31 41 51 61 71) = data1
movdqa xmm5, xmm2 ; transpose coefficients(phase 3)
punpcklqdq xmm2, xmm3 ; xmm2 = (06 16 26 36 46 56 66 76) = data6
punpckhqdq xmm5, xmm3 ; xmm5 = (07 17 27 37 47 57 67 77) = data7
movdqa xmm6, xmm1
movdqa xmm3, xmm0
psubw xmm1, xmm2 ; xmm1 = data1 - data6 = tmp6
psubw xmm0, xmm5 ; xmm0 = data0 - data7 = tmp7
paddw xmm6, xmm2 ; xmm6 = data1 + data6 = tmp1
paddw xmm3, xmm5 ; xmm3 = data0 + data7 = tmp0
movdqa xmm2, XMMWORD [wk(2)] ; xmm2 = (42 52 62 72 43 53 63 73)
movdqa xmm5, XMMWORD [wk(3)] ; xmm5 = (44 54 64 74 45 55 65 75)
movdqa XMMWORD [wk(0)], xmm1 ; wk(0) = tmp6
movdqa XMMWORD [wk(1)], xmm0 ; wk(1) = tmp7
movdqa xmm1, xmm7 ; transpose coefficients(phase 3)
punpcklqdq xmm7, xmm2 ; xmm7 = (02 12 22 32 42 52 62 72) = data2
punpckhqdq xmm1, xmm2 ; xmm1 = (03 13 23 33 43 53 63 73) = data3
movdqa xmm0, xmm4 ; transpose coefficients(phase 3)
punpcklqdq xmm4, xmm5 ; xmm4 = (04 14 24 34 44 54 64 74) = data4
punpckhqdq xmm0, xmm5 ; xmm0 = (05 15 25 35 45 55 65 75) = data5
movdqa xmm2, xmm1
movdqa xmm5, xmm7
paddw xmm1, xmm4 ; xmm1 = data3 + data4 = tmp3
paddw xmm7, xmm0 ; xmm7 = data2 + data5 = tmp2
psubw xmm2, xmm4 ; xmm2 = data3 - data4 = tmp4
psubw xmm5, xmm0 ; xmm5 = data2 - data5 = tmp5
; -- Even part
movdqa xmm4, xmm3
movdqa xmm0, xmm6
paddw xmm3, xmm1 ; xmm3 = tmp10
paddw xmm6, xmm7 ; xmm6 = tmp11
psubw xmm4, xmm1 ; xmm4 = tmp13
psubw xmm0, xmm7 ; xmm0 = tmp12
movdqa xmm1, xmm3
paddw xmm3, xmm6 ; xmm3 = tmp10 + tmp11
psubw xmm1, xmm6 ; xmm1 = tmp10 - tmp11
psllw xmm3, PASS1_BITS ; xmm3 = data0
psllw xmm1, PASS1_BITS ; xmm1 = data4
movdqa XMMWORD [wk(2)], xmm3 ; wk(2) = data0
movdqa XMMWORD [wk(3)], xmm1 ; wk(3) = data4
; (Original)
; z1 = (tmp12 + tmp13) * 0.541196100;
; data2 = z1 + tmp13 * 0.765366865;
; data6 = z1 + tmp12 * -1.847759065;
;
; (This implementation)
; data2 = tmp13 * (0.541196100 + 0.765366865) + tmp12 * 0.541196100;
; data6 = tmp13 * 0.541196100 + tmp12 * (0.541196100 - 1.847759065);
movdqa xmm7, xmm4 ; xmm4 = tmp13
movdqa xmm6, xmm4
punpcklwd xmm7, xmm0 ; xmm0 = tmp12
punpckhwd xmm6, xmm0
movdqa xmm4, xmm7
movdqa xmm0, xmm6
pmaddwd xmm7, [rel PW_F130_F054] ; xmm7 = data2L
pmaddwd xmm6, [rel PW_F130_F054] ; xmm6 = data2H
pmaddwd xmm4, [rel PW_F054_MF130] ; xmm4 = data6L
pmaddwd xmm0, [rel PW_F054_MF130] ; xmm0 = data6H
paddd xmm7, [rel PD_DESCALE_P1]
paddd xmm6, [rel PD_DESCALE_P1]
psrad xmm7, DESCALE_P1
psrad xmm6, DESCALE_P1
paddd xmm4, [rel PD_DESCALE_P1]
paddd xmm0, [rel PD_DESCALE_P1]
psrad xmm4, DESCALE_P1
psrad xmm0, DESCALE_P1
packssdw xmm7, xmm6 ; xmm7 = data2
packssdw xmm4, xmm0 ; xmm4 = data6
movdqa XMMWORD [wk(4)], xmm7 ; wk(4) = data2
movdqa XMMWORD [wk(5)], xmm4 ; wk(5) = data6
; -- Odd part
movdqa xmm3, XMMWORD [wk(0)] ; xmm3 = tmp6
movdqa xmm1, XMMWORD [wk(1)] ; xmm1 = tmp7
movdqa xmm6, xmm2 ; xmm2 = tmp4
movdqa xmm0, xmm5 ; xmm5 = tmp5
paddw xmm6, xmm3 ; xmm6 = z3
paddw xmm0, xmm1 ; xmm0 = z4
; (Original)
; z5 = (z3 + z4) * 1.175875602;
; z3 = z3 * -1.961570560; z4 = z4 * -0.390180644;
; z3 += z5; z4 += z5;
;
; (This implementation)
; z3 = z3 * (1.175875602 - 1.961570560) + z4 * 1.175875602;
; z4 = z3 * 1.175875602 + z4 * (1.175875602 - 0.390180644);
movdqa xmm7, xmm6
movdqa xmm4, xmm6
punpcklwd xmm7, xmm0
punpckhwd xmm4, xmm0
movdqa xmm6, xmm7
movdqa xmm0, xmm4
pmaddwd xmm7, [rel PW_MF078_F117] ; xmm7 = z3L
pmaddwd xmm4, [rel PW_MF078_F117] ; xmm4 = z3H
pmaddwd xmm6, [rel PW_F117_F078] ; xmm6 = z4L
pmaddwd xmm0, [rel PW_F117_F078] ; xmm0 = z4H
movdqa XMMWORD [wk(0)], xmm7 ; wk(0) = z3L
movdqa XMMWORD [wk(1)], xmm4 ; wk(1) = z3H
; (Original)
; z1 = tmp4 + tmp7; z2 = tmp5 + tmp6;
; tmp4 = tmp4 * 0.298631336; tmp5 = tmp5 * 2.053119869;
; tmp6 = tmp6 * 3.072711026; tmp7 = tmp7 * 1.501321110;
; z1 = z1 * -0.899976223; z2 = z2 * -2.562915447;
; data7 = tmp4 + z1 + z3; data5 = tmp5 + z2 + z4;
; data3 = tmp6 + z2 + z3; data1 = tmp7 + z1 + z4;
;
; (This implementation)
; tmp4 = tmp4 * (0.298631336 - 0.899976223) + tmp7 * -0.899976223;
; tmp5 = tmp5 * (2.053119869 - 2.562915447) + tmp6 * -2.562915447;
; tmp6 = tmp5 * -2.562915447 + tmp6 * (3.072711026 - 2.562915447);
; tmp7 = tmp4 * -0.899976223 + tmp7 * (1.501321110 - 0.899976223);
; data7 = tmp4 + z3; data5 = tmp5 + z4;
; data3 = tmp6 + z3; data1 = tmp7 + z4;
movdqa xmm7, xmm2
movdqa xmm4, xmm2
punpcklwd xmm7, xmm1
punpckhwd xmm4, xmm1
movdqa xmm2, xmm7
movdqa xmm1, xmm4
pmaddwd xmm7, [rel PW_MF060_MF089] ; xmm7 = tmp4L
pmaddwd xmm4, [rel PW_MF060_MF089] ; xmm4 = tmp4H
pmaddwd xmm2, [rel PW_MF089_F060] ; xmm2 = tmp7L
pmaddwd xmm1, [rel PW_MF089_F060] ; xmm1 = tmp7H
paddd xmm7, XMMWORD [wk(0)] ; xmm7 = data7L
paddd xmm4, XMMWORD [wk(1)] ; xmm4 = data7H
paddd xmm2, xmm6 ; xmm2 = data1L
paddd xmm1, xmm0 ; xmm1 = data1H
paddd xmm7, [rel PD_DESCALE_P1]
paddd xmm4, [rel PD_DESCALE_P1]
psrad xmm7, DESCALE_P1
psrad xmm4, DESCALE_P1
paddd xmm2, [rel PD_DESCALE_P1]
paddd xmm1, [rel PD_DESCALE_P1]
psrad xmm2, DESCALE_P1
psrad xmm1, DESCALE_P1
packssdw xmm7, xmm4 ; xmm7 = data7
packssdw xmm2, xmm1 ; xmm2 = data1
movdqa xmm4, xmm5
movdqa xmm1, xmm5
punpcklwd xmm4, xmm3
punpckhwd xmm1, xmm3
movdqa xmm5, xmm4
movdqa xmm3, xmm1
pmaddwd xmm4, [rel PW_MF050_MF256] ; xmm4 = tmp5L
pmaddwd xmm1, [rel PW_MF050_MF256] ; xmm1 = tmp5H
pmaddwd xmm5, [rel PW_MF256_F050] ; xmm5 = tmp6L
pmaddwd xmm3, [rel PW_MF256_F050] ; xmm3 = tmp6H
paddd xmm4, xmm6 ; xmm4 = data5L
paddd xmm1, xmm0 ; xmm1 = data5H
paddd xmm5, XMMWORD [wk(0)] ; xmm5 = data3L
paddd xmm3, XMMWORD [wk(1)] ; xmm3 = data3H
paddd xmm4, [rel PD_DESCALE_P1]
paddd xmm1, [rel PD_DESCALE_P1]
psrad xmm4, DESCALE_P1
psrad xmm1, DESCALE_P1
paddd xmm5, [rel PD_DESCALE_P1]
paddd xmm3, [rel PD_DESCALE_P1]
psrad xmm5, DESCALE_P1
psrad xmm3, DESCALE_P1
packssdw xmm4, xmm1 ; xmm4 = data5
packssdw xmm5, xmm3 ; xmm5 = data3
; ---- Pass 2: process columns.
movdqa xmm6, XMMWORD [wk(2)] ; xmm6 = col0
movdqa xmm0, XMMWORD [wk(4)] ; xmm0 = col2
; xmm6 = (00 10 20 30 40 50 60 70)
; xmm0 = (02 12 22 32 42 52 62 72)
; xmm2 = (01 11 21 31 41 51 61 71)
; xmm5 = (03 13 23 33 43 53 63 73)
movdqa xmm1, xmm6 ; transpose coefficients(phase 1)
punpcklwd xmm6, xmm2 ; xmm6 = (00 01 10 11 20 21 30 31)
punpckhwd xmm1, xmm2 ; xmm1 = (40 41 50 51 60 61 70 71)
movdqa xmm3, xmm0 ; transpose coefficients(phase 1)
punpcklwd xmm0, xmm5 ; xmm0 = (02 03 12 13 22 23 32 33)
punpckhwd xmm3, xmm5 ; xmm3 = (42 43 52 53 62 63 72 73)
movdqa xmm2, XMMWORD [wk(3)] ; xmm2 = col4
movdqa xmm5, XMMWORD [wk(5)] ; xmm5 = col6
; xmm2 = (04 14 24 34 44 54 64 74)
; xmm5 = (06 16 26 36 46 56 66 76)
; xmm4 = (05 15 25 35 45 55 65 75)
; xmm7 = (07 17 27 37 47 57 67 77)
movdqa XMMWORD [wk(0)], xmm0 ; wk(0) = (02 03 12 13 22 23 32 33)
movdqa XMMWORD [wk(1)], xmm3 ; wk(1) = (42 43 52 53 62 63 72 73)
movdqa xmm0, xmm2 ; transpose coefficients(phase 1)
punpcklwd xmm2, xmm4 ; xmm2 = (04 05 14 15 24 25 34 35)
punpckhwd xmm0, xmm4 ; xmm0 = (44 45 54 55 64 65 74 75)
movdqa xmm3, xmm5 ; transpose coefficients(phase 1)
punpcklwd xmm5, xmm7 ; xmm5 = (06 07 16 17 26 27 36 37)
punpckhwd xmm3, xmm7 ; xmm3 = (46 47 56 57 66 67 76 77)
movdqa xmm4, xmm2 ; transpose coefficients(phase 2)
punpckldq xmm2, xmm5 ; xmm2 = (04 05 06 07 14 15 16 17)
punpckhdq xmm4, xmm5 ; xmm4 = (24 25 26 27 34 35 36 37)
movdqa xmm7, xmm0 ; transpose coefficients(phase 2)
punpckldq xmm0, xmm3 ; xmm0 = (44 45 46 47 54 55 56 57)
punpckhdq xmm7, xmm3 ; xmm7 = (64 65 66 67 74 75 76 77)
movdqa xmm5, XMMWORD [wk(0)] ; xmm5 = (02 03 12 13 22 23 32 33)
movdqa xmm3, XMMWORD [wk(1)] ; xmm3 = (42 43 52 53 62 63 72 73)
movdqa XMMWORD [wk(2)], xmm4 ; wk(2) = (24 25 26 27 34 35 36 37)
movdqa XMMWORD [wk(3)], xmm0 ; wk(3) = (44 45 46 47 54 55 56 57)
movdqa xmm4, xmm6 ; transpose coefficients(phase 2)
punpckldq xmm6, xmm5 ; xmm6 = (00 01 02 03 10 11 12 13)
punpckhdq xmm4, xmm5 ; xmm4 = (20 21 22 23 30 31 32 33)
movdqa xmm0, xmm1 ; transpose coefficients(phase 2)
punpckldq xmm1, xmm3 ; xmm1 = (40 41 42 43 50 51 52 53)
punpckhdq xmm0, xmm3 ; xmm0 = (60 61 62 63 70 71 72 73)
movdqa xmm5, xmm6 ; transpose coefficients(phase 3)
punpcklqdq xmm6, xmm2 ; xmm6 = (00 01 02 03 04 05 06 07) = data0
punpckhqdq xmm5, xmm2 ; xmm5 = (10 11 12 13 14 15 16 17) = data1
movdqa xmm3, xmm0 ; transpose coefficients(phase 3)
punpcklqdq xmm0, xmm7 ; xmm0 = (60 61 62 63 64 65 66 67) = data6
punpckhqdq xmm3, xmm7 ; xmm3 = (70 71 72 73 74 75 76 77) = data7
movdqa xmm2, xmm5
movdqa xmm7, xmm6
psubw xmm5, xmm0 ; xmm5 = data1 - data6 = tmp6
psubw xmm6, xmm3 ; xmm6 = data0 - data7 = tmp7
paddw xmm2, xmm0 ; xmm2 = data1 + data6 = tmp1
paddw xmm7, xmm3 ; xmm7 = data0 + data7 = tmp0
movdqa xmm0, XMMWORD [wk(2)] ; xmm0 = (24 25 26 27 34 35 36 37)
movdqa xmm3, XMMWORD [wk(3)] ; xmm3 = (44 45 46 47 54 55 56 57)
movdqa XMMWORD [wk(0)], xmm5 ; wk(0) = tmp6
movdqa XMMWORD [wk(1)], xmm6 ; wk(1) = tmp7
movdqa xmm5, xmm4 ; transpose coefficients(phase 3)
punpcklqdq xmm4, xmm0 ; xmm4 = (20 21 22 23 24 25 26 27) = data2
punpckhqdq xmm5, xmm0 ; xmm5 = (30 31 32 33 34 35 36 37) = data3
movdqa xmm6, xmm1 ; transpose coefficients(phase 3)
punpcklqdq xmm1, xmm3 ; xmm1 = (40 41 42 43 44 45 46 47) = data4
punpckhqdq xmm6, xmm3 ; xmm6 = (50 51 52 53 54 55 56 57) = data5
movdqa xmm0, xmm5
movdqa xmm3, xmm4
paddw xmm5, xmm1 ; xmm5 = data3 + data4 = tmp3
paddw xmm4, xmm6 ; xmm4 = data2 + data5 = tmp2
psubw xmm0, xmm1 ; xmm0 = data3 - data4 = tmp4
psubw xmm3, xmm6 ; xmm3 = data2 - data5 = tmp5
; -- Even part
movdqa xmm1, xmm7
movdqa xmm6, xmm2
paddw xmm7, xmm5 ; xmm7 = tmp10
paddw xmm2, xmm4 ; xmm2 = tmp11
psubw xmm1, xmm5 ; xmm1 = tmp13
psubw xmm6, xmm4 ; xmm6 = tmp12
movdqa xmm5, xmm7
paddw xmm7, xmm2 ; xmm7 = tmp10 + tmp11
psubw xmm5, xmm2 ; xmm5 = tmp10 - tmp11
paddw xmm7, [rel PW_DESCALE_P2X]
paddw xmm5, [rel PW_DESCALE_P2X]
psraw xmm7, PASS1_BITS ; xmm7 = data0
psraw xmm5, PASS1_BITS ; xmm5 = data4
movdqa XMMWORD [XMMBLOCK(0, 0, rdx, SIZEOF_DCTELEM)], xmm7
movdqa XMMWORD [XMMBLOCK(4, 0, rdx, SIZEOF_DCTELEM)], xmm5
; (Original)
; z1 = (tmp12 + tmp13) * 0.541196100;
; data2 = z1 + tmp13 * 0.765366865;
; data6 = z1 + tmp12 * -1.847759065;
;
; (This implementation)
; data2 = tmp13 * (0.541196100 + 0.765366865) + tmp12 * 0.541196100;
; data6 = tmp13 * 0.541196100 + tmp12 * (0.541196100 - 1.847759065);
movdqa xmm4, xmm1 ; xmm1 = tmp13
movdqa xmm2, xmm1
punpcklwd xmm4, xmm6 ; xmm6 = tmp12
punpckhwd xmm2, xmm6
movdqa xmm1, xmm4
movdqa xmm6, xmm2
pmaddwd xmm4, [rel PW_F130_F054] ; xmm4 = data2L
pmaddwd xmm2, [rel PW_F130_F054] ; xmm2 = data2H
pmaddwd xmm1, [rel PW_F054_MF130] ; xmm1 = data6L
pmaddwd xmm6, [rel PW_F054_MF130] ; xmm6 = data6H
paddd xmm4, [rel PD_DESCALE_P2]
paddd xmm2, [rel PD_DESCALE_P2]
psrad xmm4, DESCALE_P2
psrad xmm2, DESCALE_P2
paddd xmm1, [rel PD_DESCALE_P2]
paddd xmm6, [rel PD_DESCALE_P2]
psrad xmm1, DESCALE_P2
psrad xmm6, DESCALE_P2
packssdw xmm4, xmm2 ; xmm4 = data2
packssdw xmm1, xmm6 ; xmm1 = data6
movdqa XMMWORD [XMMBLOCK(2, 0, rdx, SIZEOF_DCTELEM)], xmm4
movdqa XMMWORD [XMMBLOCK(6, 0, rdx, SIZEOF_DCTELEM)], xmm1
; -- Odd part
movdqa xmm7, XMMWORD [wk(0)] ; xmm7 = tmp6
movdqa xmm5, XMMWORD [wk(1)] ; xmm5 = tmp7
movdqa xmm2, xmm0 ; xmm0 = tmp4
movdqa xmm6, xmm3 ; xmm3 = tmp5
paddw xmm2, xmm7 ; xmm2 = z3
paddw xmm6, xmm5 ; xmm6 = z4
; (Original)
; z5 = (z3 + z4) * 1.175875602;
; z3 = z3 * -1.961570560; z4 = z4 * -0.390180644;
; z3 += z5; z4 += z5;
;
; (This implementation)
; z3 = z3 * (1.175875602 - 1.961570560) + z4 * 1.175875602;
; z4 = z3 * 1.175875602 + z4 * (1.175875602 - 0.390180644);
movdqa xmm4, xmm2
movdqa xmm1, xmm2
punpcklwd xmm4, xmm6
punpckhwd xmm1, xmm6
movdqa xmm2, xmm4
movdqa xmm6, xmm1
pmaddwd xmm4, [rel PW_MF078_F117] ; xmm4 = z3L
pmaddwd xmm1, [rel PW_MF078_F117] ; xmm1 = z3H
pmaddwd xmm2, [rel PW_F117_F078] ; xmm2 = z4L
pmaddwd xmm6, [rel PW_F117_F078] ; xmm6 = z4H
movdqa XMMWORD [wk(0)], xmm4 ; wk(0) = z3L
movdqa XMMWORD [wk(1)], xmm1 ; wk(1) = z3H
; (Original)
; z1 = tmp4 + tmp7; z2 = tmp5 + tmp6;
; tmp4 = tmp4 * 0.298631336; tmp5 = tmp5 * 2.053119869;
; tmp6 = tmp6 * 3.072711026; tmp7 = tmp7 * 1.501321110;
; z1 = z1 * -0.899976223; z2 = z2 * -2.562915447;
; data7 = tmp4 + z1 + z3; data5 = tmp5 + z2 + z4;
; data3 = tmp6 + z2 + z3; data1 = tmp7 + z1 + z4;
;
; (This implementation)
; tmp4 = tmp4 * (0.298631336 - 0.899976223) + tmp7 * -0.899976223;
; tmp5 = tmp5 * (2.053119869 - 2.562915447) + tmp6 * -2.562915447;
; tmp6 = tmp5 * -2.562915447 + tmp6 * (3.072711026 - 2.562915447);
; tmp7 = tmp4 * -0.899976223 + tmp7 * (1.501321110 - 0.899976223);
; data7 = tmp4 + z3; data5 = tmp5 + z4;
; data3 = tmp6 + z3; data1 = tmp7 + z4;
movdqa xmm4, xmm0
movdqa xmm1, xmm0
punpcklwd xmm4, xmm5
punpckhwd xmm1, xmm5
movdqa xmm0, xmm4
movdqa xmm5, xmm1
pmaddwd xmm4, [rel PW_MF060_MF089] ; xmm4 = tmp4L
pmaddwd xmm1, [rel PW_MF060_MF089] ; xmm1 = tmp4H
pmaddwd xmm0, [rel PW_MF089_F060] ; xmm0 = tmp7L
pmaddwd xmm5, [rel PW_MF089_F060] ; xmm5 = tmp7H
paddd xmm4, XMMWORD [wk(0)] ; xmm4 = data7L
paddd xmm1, XMMWORD [wk(1)] ; xmm1 = data7H
paddd xmm0, xmm2 ; xmm0 = data1L
paddd xmm5, xmm6 ; xmm5 = data1H
paddd xmm4, [rel PD_DESCALE_P2]
paddd xmm1, [rel PD_DESCALE_P2]
psrad xmm4, DESCALE_P2
psrad xmm1, DESCALE_P2
paddd xmm0, [rel PD_DESCALE_P2]
paddd xmm5, [rel PD_DESCALE_P2]
psrad xmm0, DESCALE_P2
psrad xmm5, DESCALE_P2
packssdw xmm4, xmm1 ; xmm4 = data7
packssdw xmm0, xmm5 ; xmm0 = data1
movdqa XMMWORD [XMMBLOCK(7, 0, rdx, SIZEOF_DCTELEM)], xmm4
movdqa XMMWORD [XMMBLOCK(1, 0, rdx, SIZEOF_DCTELEM)], xmm0
movdqa xmm1, xmm3
movdqa xmm5, xmm3
punpcklwd xmm1, xmm7
punpckhwd xmm5, xmm7
movdqa xmm3, xmm1
movdqa xmm7, xmm5
pmaddwd xmm1, [rel PW_MF050_MF256] ; xmm1 = tmp5L
pmaddwd xmm5, [rel PW_MF050_MF256] ; xmm5 = tmp5H
pmaddwd xmm3, [rel PW_MF256_F050] ; xmm3 = tmp6L
pmaddwd xmm7, [rel PW_MF256_F050] ; xmm7 = tmp6H
paddd xmm1, xmm2 ; xmm1 = data5L
paddd xmm5, xmm6 ; xmm5 = data5H
paddd xmm3, XMMWORD [wk(0)] ; xmm3 = data3L
paddd xmm7, XMMWORD [wk(1)] ; xmm7 = data3H
paddd xmm1, [rel PD_DESCALE_P2]
paddd xmm5, [rel PD_DESCALE_P2]
psrad xmm1, DESCALE_P2
psrad xmm5, DESCALE_P2
paddd xmm3, [rel PD_DESCALE_P2]
paddd xmm7, [rel PD_DESCALE_P2]
psrad xmm3, DESCALE_P2
psrad xmm7, DESCALE_P2
packssdw xmm1, xmm5 ; xmm1 = data5
packssdw xmm3, xmm7 ; xmm3 = data3
movdqa XMMWORD [XMMBLOCK(5, 0, rdx, SIZEOF_DCTELEM)], xmm1
movdqa XMMWORD [XMMBLOCK(3, 0, rdx, SIZEOF_DCTELEM)], xmm3
UNCOLLECT_ARGS 1
lea rsp, [rbp - 8]
pop r15
pop rbp
ret
; For some reason, the OS X linker does not honor the request to align the
; segment unless we do this.
align 32
lib/libjpeg-turbo/simd/x86_64/jidctfst-sse2.asm
Executable
+508
View File
@@ -0,0 +1,508 @@
;
; Fast integer IDCT (64-bit SSE2)
;
; Copyright 2009 Pierre Ossman <[email protected]> for Cendio AB
; Copyright (C) 2009, 2016, 2024-2025, D. R. Commander.
; Copyright (C) 2018, Matthias Räncker.
; Copyright (C) 2023, Aliaksiej Kandracienka.
;
; Based on the x86 SIMD extension for IJG JPEG library
; Copyright (C) 1999-2006, MIYASAKA Masaru.
; For conditions of distribution and use, see copyright notice in jsimdext.inc
;
; This file should be assembled with NASM (Netwide Assembler) or Yasm.
;
; This file contains a fast, not so accurate integer implementation of the
; inverse DCT (Discrete Cosine Transform). The following code is based
; directly on the IJG's original jidctfst.c; see jidctfst.c for more details.
%include "jsimdext.inc"
%include "jdct.inc"
; --------------------------------------------------------------------------
%define CONST_BITS 8 ; 14 is also OK.
%define PASS1_BITS 2
%if IFAST_SCALE_BITS != PASS1_BITS
%error "'IFAST_SCALE_BITS' must be equal to 'PASS1_BITS'."
%endif
%if CONST_BITS == 8
F_1_082 equ 277 ; FIX(1.082392200)
F_1_414 equ 362 ; FIX(1.414213562)
F_1_847 equ 473 ; FIX(1.847759065)
F_2_613 equ 669 ; FIX(2.613125930)
F_1_613 equ (F_2_613 - 256) ; FIX(2.613125930) - FIX(1)
%else
; NASM cannot do compile-time arithmetic on floating-point constants.
%define DESCALE(x, n) (((x) + (1 << ((n) - 1))) >> (n))
F_1_082 equ DESCALE(1162209775, 30 - CONST_BITS) ; FIX(1.082392200)
F_1_414 equ DESCALE(1518500249, 30 - CONST_BITS) ; FIX(1.414213562)
F_1_847 equ DESCALE(1984016188, 30 - CONST_BITS) ; FIX(1.847759065)
F_2_613 equ DESCALE(2805822602, 30 - CONST_BITS) ; FIX(2.613125930)
F_1_613 equ (F_2_613 - (1 << CONST_BITS)) ; FIX(2.613125930) - FIX(1)
%endif
; --------------------------------------------------------------------------
SECTION SEG_CONST
; PRE_MULTIPLY_SCALE_BITS <= 2 (to avoid overflow)
; CONST_BITS + CONST_SHIFT + PRE_MULTIPLY_SCALE_BITS == 16 (for pmulhw)
%define PRE_MULTIPLY_SCALE_BITS 2
%define CONST_SHIFT (16 - PRE_MULTIPLY_SCALE_BITS - CONST_BITS)
ALIGNZ 32
GLOBAL_DATA(jconst_idct_ifast_sse2)
EXTN(jconst_idct_ifast_sse2):
PW_F1414 times 8 dw F_1_414 << CONST_SHIFT
PW_F1847 times 8 dw F_1_847 << CONST_SHIFT
PW_MF1613 times 8 dw -F_1_613 << CONST_SHIFT
PW_F1082 times 8 dw F_1_082 << CONST_SHIFT
PB_CENTERJSAMP times 16 db CENTERJSAMPLE
ALIGNZ 32
; --------------------------------------------------------------------------
SECTION SEG_TEXT
BITS 64
; Perform dequantization and inverse DCT on one block of coefficients.
;
; GLOBAL(void)
; jsimd_idct_ifast_sse2(void *dct_table, JCOEFPTR coef_block,
; JSAMPARRAY output_buf, JDIMENSION output_col)
;
; r10 = jpeg_component_info *compptr
; r11 = JCOEFPTR coef_block
; r12 = JSAMPARRAY output_buf
; r13d = JDIMENSION output_col
%define wk(i) r15 - (WK_NUM - (i)) * SIZEOF_XMMWORD
; xmmword wk[WK_NUM]
%define WK_NUM 2
align 32
GLOBAL_FUNCTION(jsimd_idct_ifast_sse2)
EXTN(jsimd_idct_ifast_sse2):
ENDBR64
push rbp
mov rbp, rsp
push r15
and rsp, byte (-SIZEOF_XMMWORD) ; align to 128 bits
; Allocate stack space for wk array. r15 is used to access it.
mov r15, rsp
sub rsp, byte (SIZEOF_XMMWORD * WK_NUM)
COLLECT_ARGS 4
; ---- Pass 1: process columns from input.
mov rdx, r10 ; quantptr
mov rsi, r11 ; inptr
%ifndef NO_ZERO_COLUMN_TEST_IFAST_SSE2
mov eax, dword [DWBLOCK(1, 0, rsi, SIZEOF_JCOEF)]
or eax, dword [DWBLOCK(2, 0, rsi, SIZEOF_JCOEF)]
jnz near .columnDCT
movdqa xmm0, XMMWORD [XMMBLOCK(1, 0, rsi, SIZEOF_JCOEF)]
movdqa xmm1, XMMWORD [XMMBLOCK(2, 0, rsi, SIZEOF_JCOEF)]
por xmm0, XMMWORD [XMMBLOCK(3, 0, rsi, SIZEOF_JCOEF)]
por xmm1, XMMWORD [XMMBLOCK(4, 0, rsi, SIZEOF_JCOEF)]
por xmm0, XMMWORD [XMMBLOCK(5, 0, rsi, SIZEOF_JCOEF)]
por xmm1, XMMWORD [XMMBLOCK(6, 0, rsi, SIZEOF_JCOEF)]
por xmm0, XMMWORD [XMMBLOCK(7, 0, rsi, SIZEOF_JCOEF)]
por xmm1, xmm0
packsswb xmm1, xmm1
packsswb xmm1, xmm1
movd eax, xmm1
test rax, rax
jnz short .columnDCT
; -- AC terms all zero
movdqa xmm0, XMMWORD [XMMBLOCK(0, 0, rsi, SIZEOF_JCOEF)]
pmullw xmm0, XMMWORD [XMMBLOCK(0, 0, rdx, SIZEOF_ISLOW_MULT_TYPE)]
movdqa xmm7, xmm0 ; xmm0 = in0 = (00 01 02 03 04 05 06 07)
punpcklwd xmm0, xmm0 ; xmm0 = (00 00 01 01 02 02 03 03)
punpckhwd xmm7, xmm7 ; xmm7 = (04 04 05 05 06 06 07 07)
pshufd xmm6, xmm0, 0x00 ; xmm6 = col0 = (00 00 00 00 00 00 00 00)
pshufd xmm2, xmm0, 0x55 ; xmm2 = col1 = (01 01 01 01 01 01 01 01)
pshufd xmm5, xmm0, 0xAA ; xmm5 = col2 = (02 02 02 02 02 02 02 02)
pshufd xmm0, xmm0, 0xFF ; xmm0 = col3 = (03 03 03 03 03 03 03 03)
pshufd xmm1, xmm7, 0x00 ; xmm1 = col4 = (04 04 04 04 04 04 04 04)
pshufd xmm4, xmm7, 0x55 ; xmm4 = col5 = (05 05 05 05 05 05 05 05)
pshufd xmm3, xmm7, 0xAA ; xmm3 = col6 = (06 06 06 06 06 06 06 06)
pshufd xmm7, xmm7, 0xFF ; xmm7 = col7 = (07 07 07 07 07 07 07 07)
movdqa XMMWORD [wk(0)], xmm2 ; wk(0) = col1
movdqa XMMWORD [wk(1)], xmm0 ; wk(1) = col3
jmp near .column_end
%endif
.columnDCT:
; -- Even part
movdqa xmm0, XMMWORD [XMMBLOCK(0, 0, rsi, SIZEOF_JCOEF)]
movdqa xmm1, XMMWORD [XMMBLOCK(2, 0, rsi, SIZEOF_JCOEF)]
pmullw xmm0, XMMWORD [XMMBLOCK(0, 0, rdx, SIZEOF_IFAST_MULT_TYPE)]
pmullw xmm1, XMMWORD [XMMBLOCK(2, 0, rdx, SIZEOF_IFAST_MULT_TYPE)]
movdqa xmm2, XMMWORD [XMMBLOCK(4, 0, rsi, SIZEOF_JCOEF)]
movdqa xmm3, XMMWORD [XMMBLOCK(6, 0, rsi, SIZEOF_JCOEF)]
pmullw xmm2, XMMWORD [XMMBLOCK(4, 0, rdx, SIZEOF_IFAST_MULT_TYPE)]
pmullw xmm3, XMMWORD [XMMBLOCK(6, 0, rdx, SIZEOF_IFAST_MULT_TYPE)]
movdqa xmm4, xmm0
movdqa xmm5, xmm1
psubw xmm0, xmm2 ; xmm0 = tmp11
psubw xmm1, xmm3
paddw xmm4, xmm2 ; xmm4 = tmp10
paddw xmm5, xmm3 ; xmm5 = tmp13
psllw xmm1, PRE_MULTIPLY_SCALE_BITS
pmulhw xmm1, [rel PW_F1414]
psubw xmm1, xmm5 ; xmm1 = tmp12
movdqa xmm6, xmm4
movdqa xmm7, xmm0
psubw xmm4, xmm5 ; xmm4 = tmp3
psubw xmm0, xmm1 ; xmm0 = tmp2
paddw xmm6, xmm5 ; xmm6 = tmp0
paddw xmm7, xmm1 ; xmm7 = tmp1
movdqa XMMWORD [wk(1)], xmm4 ; wk(1) = tmp3
movdqa XMMWORD [wk(0)], xmm0 ; wk(0) = tmp2
; -- Odd part
movdqa xmm2, XMMWORD [XMMBLOCK(1, 0, rsi, SIZEOF_JCOEF)]
movdqa xmm3, XMMWORD [XMMBLOCK(3, 0, rsi, SIZEOF_JCOEF)]
pmullw xmm2, XMMWORD [XMMBLOCK(1, 0, rdx, SIZEOF_IFAST_MULT_TYPE)]
pmullw xmm3, XMMWORD [XMMBLOCK(3, 0, rdx, SIZEOF_IFAST_MULT_TYPE)]
movdqa xmm5, XMMWORD [XMMBLOCK(5, 0, rsi, SIZEOF_JCOEF)]
movdqa xmm1, XMMWORD [XMMBLOCK(7, 0, rsi, SIZEOF_JCOEF)]
pmullw xmm5, XMMWORD [XMMBLOCK(5, 0, rdx, SIZEOF_IFAST_MULT_TYPE)]
pmullw xmm1, XMMWORD [XMMBLOCK(7, 0, rdx, SIZEOF_IFAST_MULT_TYPE)]
movdqa xmm4, xmm2
movdqa xmm0, xmm5
psubw xmm2, xmm1 ; xmm2 = z12
psubw xmm5, xmm3 ; xmm5 = z10
paddw xmm4, xmm1 ; xmm4 = z11
paddw xmm0, xmm3 ; xmm0 = z13
movdqa xmm1, xmm5 ; xmm1 = z10(unscaled)
psllw xmm2, PRE_MULTIPLY_SCALE_BITS
psllw xmm5, PRE_MULTIPLY_SCALE_BITS
movdqa xmm3, xmm4
psubw xmm4, xmm0
paddw xmm3, xmm0 ; xmm3 = tmp7
psllw xmm4, PRE_MULTIPLY_SCALE_BITS
pmulhw xmm4, [rel PW_F1414] ; xmm4 = tmp11
; To avoid overflow...
;
; (Original)
; tmp12 = -2.613125930 * z10 + z5;
;
; (This implementation)
; tmp12 = (-1.613125930 - 1) * z10 + z5;
; = -1.613125930 * z10 - z10 + z5;
movdqa xmm0, xmm5
paddw xmm5, xmm2
pmulhw xmm5, [rel PW_F1847] ; xmm5 = z5
pmulhw xmm0, [rel PW_MF1613]
pmulhw xmm2, [rel PW_F1082]
psubw xmm0, xmm1
psubw xmm2, xmm5 ; xmm2 = tmp10
paddw xmm0, xmm5 ; xmm0 = tmp12
; -- Final output stage
psubw xmm0, xmm3 ; xmm0 = tmp6
movdqa xmm1, xmm6
movdqa xmm5, xmm7
paddw xmm6, xmm3 ; xmm6 = data0 = (00 01 02 03 04 05 06 07)
paddw xmm7, xmm0 ; xmm7 = data1 = (10 11 12 13 14 15 16 17)
psubw xmm1, xmm3 ; xmm1 = data7 = (70 71 72 73 74 75 76 77)
psubw xmm5, xmm0 ; xmm5 = data6 = (60 61 62 63 64 65 66 67)
psubw xmm4, xmm0 ; xmm4 = tmp5
movdqa xmm3, xmm6 ; transpose coefficients(phase 1)
punpcklwd xmm6, xmm7 ; xmm6 = (00 10 01 11 02 12 03 13)
punpckhwd xmm3, xmm7 ; xmm3 = (04 14 05 15 06 16 07 17)
movdqa xmm0, xmm5 ; transpose coefficients(phase 1)
punpcklwd xmm5, xmm1 ; xmm5 = (60 70 61 71 62 72 63 73)
punpckhwd xmm0, xmm1 ; xmm0 = (64 74 65 75 66 76 67 77)
movdqa xmm7, XMMWORD [wk(0)] ; xmm7 = tmp2
movdqa xmm1, XMMWORD [wk(1)] ; xmm1 = tmp3
movdqa XMMWORD [wk(0)], xmm5 ; wk(0) = (60 70 61 71 62 72 63 73)
movdqa XMMWORD [wk(1)], xmm0 ; wk(1) = (64 74 65 75 66 76 67 77)
paddw xmm2, xmm4 ; xmm2 = tmp4
movdqa xmm5, xmm7
movdqa xmm0, xmm1
paddw xmm7, xmm4 ; xmm7 = data2 = (20 21 22 23 24 25 26 27)
paddw xmm1, xmm2 ; xmm1 = data4 = (40 41 42 43 44 45 46 47)
psubw xmm5, xmm4 ; xmm5 = data5 = (50 51 52 53 54 55 56 57)
psubw xmm0, xmm2 ; xmm0 = data3 = (30 31 32 33 34 35 36 37)
movdqa xmm4, xmm7 ; transpose coefficients(phase 1)
punpcklwd xmm7, xmm0 ; xmm7 = (20 30 21 31 22 32 23 33)
punpckhwd xmm4, xmm0 ; xmm4 = (24 34 25 35 26 36 27 37)
movdqa xmm2, xmm1 ; transpose coefficients(phase 1)
punpcklwd xmm1, xmm5 ; xmm1 = (40 50 41 51 42 52 43 53)
punpckhwd xmm2, xmm5 ; xmm2 = (44 54 45 55 46 56 47 57)
movdqa xmm0, xmm3 ; transpose coefficients(phase 2)
punpckldq xmm3, xmm4 ; xmm3 = (04 14 24 34 05 15 25 35)
punpckhdq xmm0, xmm4 ; xmm0 = (06 16 26 36 07 17 27 37)
movdqa xmm5, xmm6 ; transpose coefficients(phase 2)
punpckldq xmm6, xmm7 ; xmm6 = (00 10 20 30 01 11 21 31)
punpckhdq xmm5, xmm7 ; xmm5 = (02 12 22 32 03 13 23 33)
movdqa xmm4, XMMWORD [wk(0)] ; xmm4 = (60 70 61 71 62 72 63 73)
movdqa xmm7, XMMWORD [wk(1)] ; xmm7 = (64 74 65 75 66 76 67 77)
movdqa XMMWORD [wk(0)], xmm3 ; wk(0) = (04 14 24 34 05 15 25 35)
movdqa XMMWORD [wk(1)], xmm0 ; wk(1) = (06 16 26 36 07 17 27 37)
movdqa xmm3, xmm1 ; transpose coefficients(phase 2)
punpckldq xmm1, xmm4 ; xmm1 = (40 50 60 70 41 51 61 71)
punpckhdq xmm3, xmm4 ; xmm3 = (42 52 62 72 43 53 63 73)
movdqa xmm0, xmm2 ; transpose coefficients(phase 2)
punpckldq xmm2, xmm7 ; xmm2 = (44 54 64 74 45 55 65 75)
punpckhdq xmm0, xmm7 ; xmm0 = (46 56 66 76 47 57 67 77)
movdqa xmm4, xmm6 ; transpose coefficients(phase 3)
punpcklqdq xmm6, xmm1 ; xmm6 = col0 = (00 10 20 30 40 50 60 70)
punpckhqdq xmm4, xmm1 ; xmm4 = col1 = (01 11 21 31 41 51 61 71)
movdqa xmm7, xmm5 ; transpose coefficients(phase 3)
punpcklqdq xmm5, xmm3 ; xmm5 = col2 = (02 12 22 32 42 52 62 72)
punpckhqdq xmm7, xmm3 ; xmm7 = col3 = (03 13 23 33 43 53 63 73)
movdqa xmm1, XMMWORD [wk(0)] ; xmm1 = (04 14 24 34 05 15 25 35)
movdqa xmm3, XMMWORD [wk(1)] ; xmm3 = (06 16 26 36 07 17 27 37)
movdqa XMMWORD [wk(0)], xmm4 ; wk(0) = col1
movdqa XMMWORD [wk(1)], xmm7 ; wk(1) = col3
movdqa xmm4, xmm1 ; transpose coefficients(phase 3)
punpcklqdq xmm1, xmm2 ; xmm1 = col4 = (04 14 24 34 44 54 64 74)
punpckhqdq xmm4, xmm2 ; xmm4 = col5 = (05 15 25 35 45 55 65 75)
movdqa xmm7, xmm3 ; transpose coefficients(phase 3)
punpcklqdq xmm3, xmm0 ; xmm3 = col6 = (06 16 26 36 46 56 66 76)
punpckhqdq xmm7, xmm0 ; xmm7 = col7 = (07 17 27 37 47 57 67 77)
.column_end:
; -- Prefetch the next coefficient block
prefetchnta [rsi + DCTSIZE2 * SIZEOF_JCOEF + 0 * 32]
prefetchnta [rsi + DCTSIZE2 * SIZEOF_JCOEF + 1 * 32]
prefetchnta [rsi + DCTSIZE2 * SIZEOF_JCOEF + 2 * 32]
prefetchnta [rsi + DCTSIZE2 * SIZEOF_JCOEF + 3 * 32]
; ---- Pass 2: process rows from work array, store into output array.
mov rdi, r12 ; (JSAMPROW *)
mov eax, r13d
; -- Even part
; xmm6 = col0, xmm5 = col2, xmm1 = col4, xmm3 = col6
movdqa xmm2, xmm6
movdqa xmm0, xmm5
psubw xmm6, xmm1 ; xmm6 = tmp11
psubw xmm5, xmm3
paddw xmm2, xmm1 ; xmm2 = tmp10
paddw xmm0, xmm3 ; xmm0 = tmp13
psllw xmm5, PRE_MULTIPLY_SCALE_BITS
pmulhw xmm5, [rel PW_F1414]
psubw xmm5, xmm0 ; xmm5 = tmp12
movdqa xmm1, xmm2
movdqa xmm3, xmm6
psubw xmm2, xmm0 ; xmm2 = tmp3
psubw xmm6, xmm5 ; xmm6 = tmp2
paddw xmm1, xmm0 ; xmm1 = tmp0
paddw xmm3, xmm5 ; xmm3 = tmp1
movdqa xmm0, XMMWORD [wk(0)] ; xmm0 = col1
movdqa xmm5, XMMWORD [wk(1)] ; xmm5 = col3
movdqa XMMWORD [wk(0)], xmm2 ; wk(0) = tmp3
movdqa XMMWORD [wk(1)], xmm6 ; wk(1) = tmp2
; -- Odd part
; xmm0 = col1, xmm5 = col3, xmm4 = col5, xmm7 = col7
movdqa xmm2, xmm0
movdqa xmm6, xmm4
psubw xmm0, xmm7 ; xmm0 = z12
psubw xmm4, xmm5 ; xmm4 = z10
paddw xmm2, xmm7 ; xmm2 = z11
paddw xmm6, xmm5 ; xmm6 = z13
movdqa xmm7, xmm4 ; xmm7 = z10(unscaled)
psllw xmm0, PRE_MULTIPLY_SCALE_BITS
psllw xmm4, PRE_MULTIPLY_SCALE_BITS
movdqa xmm5, xmm2
psubw xmm2, xmm6
paddw xmm5, xmm6 ; xmm5 = tmp7
psllw xmm2, PRE_MULTIPLY_SCALE_BITS
pmulhw xmm2, [rel PW_F1414] ; xmm2 = tmp11
; To avoid overflow...
;
; (Original)
; tmp12 = -2.613125930 * z10 + z5;
;
; (This implementation)
; tmp12 = (-1.613125930 - 1) * z10 + z5;
; = -1.613125930 * z10 - z10 + z5;
movdqa xmm6, xmm4
paddw xmm4, xmm0
pmulhw xmm4, [rel PW_F1847] ; xmm4 = z5
pmulhw xmm6, [rel PW_MF1613]
pmulhw xmm0, [rel PW_F1082]
psubw xmm6, xmm7
psubw xmm0, xmm4 ; xmm0 = tmp10
paddw xmm6, xmm4 ; xmm6 = tmp12
; -- Final output stage
psubw xmm6, xmm5 ; xmm6 = tmp6
movdqa xmm7, xmm1
movdqa xmm4, xmm3
paddw xmm1, xmm5 ; xmm1 = data0 = (00 10 20 30 40 50 60 70)
paddw xmm3, xmm6 ; xmm3 = data1 = (01 11 21 31 41 51 61 71)
psraw xmm1, (PASS1_BITS + 3) ; descale
psraw xmm3, (PASS1_BITS + 3) ; descale
psubw xmm7, xmm5 ; xmm7 = data7 = (07 17 27 37 47 57 67 77)
psubw xmm4, xmm6 ; xmm4 = data6 = (06 16 26 36 46 56 66 76)
psraw xmm7, (PASS1_BITS + 3) ; descale
psraw xmm4, (PASS1_BITS + 3) ; descale
psubw xmm2, xmm6 ; xmm2 = tmp5
packsswb xmm1, xmm4
; xmm1 = (00 10 20 30 40 50 60 70 06 16 26 36 46 56 66 76)
packsswb xmm3, xmm7
; xmm3 = (01 11 21 31 41 51 61 71 07 17 27 37 47 57 67 77)
movdqa xmm5, XMMWORD [wk(1)] ; xmm5 = tmp2
movdqa xmm6, XMMWORD [wk(0)] ; xmm6 = tmp3
paddw xmm0, xmm2 ; xmm0 = tmp4
movdqa xmm4, xmm5
movdqa xmm7, xmm6
paddw xmm5, xmm2 ; xmm5 = data2 = (02 12 22 32 42 52 62 72)
paddw xmm6, xmm0 ; xmm6 = data4 = (04 14 24 34 44 54 64 74)
psraw xmm5, (PASS1_BITS + 3) ; descale
psraw xmm6, (PASS1_BITS + 3) ; descale
psubw xmm4, xmm2 ; xmm4 = data5 = (05 15 25 35 45 55 65 75)
psubw xmm7, xmm0 ; xmm7 = data3 = (03 13 23 33 43 53 63 73)
psraw xmm4, (PASS1_BITS + 3) ; descale
psraw xmm7, (PASS1_BITS + 3) ; descale
movdqa xmm2, [rel PB_CENTERJSAMP] ; xmm2 = [rel PB_CENTERJSAMP]
packsswb xmm5, xmm6
; xmm5 = (02 12 22 32 42 52 62 72 04 14 24 34 44 54 64 74)
packsswb xmm7, xmm4
; xmm7 = (03 13 23 33 43 53 63 73 05 15 25 35 45 55 65 75)
paddb xmm1, xmm2
paddb xmm3, xmm2
paddb xmm5, xmm2
paddb xmm7, xmm2
; transpose coefficients(phase 1)
movdqa xmm0, xmm1
punpcklbw xmm1, xmm3
; xmm1 = (00 01 10 11 20 21 30 31 40 41 50 51 60 61 70 71)
punpckhbw xmm0, xmm3
; xmm0 = (06 07 16 17 26 27 36 37 46 47 56 57 66 67 76 77)
movdqa xmm6, xmm5
punpcklbw xmm5, xmm7
; xmm5 = (02 03 12 13 22 23 32 33 42 43 52 53 62 63 72 73)
punpckhbw xmm6, xmm7
; xmm6 = (04 05 14 15 24 25 34 35 44 45 54 55 64 65 74 75)
; transpose coefficients(phase 2)
movdqa xmm4, xmm1
punpcklwd xmm1, xmm5
; xmm1 = (00 01 02 03 10 11 12 13 20 21 22 23 30 31 32 33)
punpckhwd xmm4, xmm5
; xmm4 = (40 41 42 43 50 51 52 53 60 61 62 63 70 71 72 73)
movdqa xmm2, xmm6
punpcklwd xmm6, xmm0
; xmm6 = (04 05 06 07 14 15 16 17 24 25 26 27 34 35 36 37)
punpckhwd xmm2, xmm0
; xmm2 = (44 45 46 47 54 55 56 57 64 65 66 67 74 75 76 77)
; transpose coefficients(phase 3)
movdqa xmm3, xmm1
punpckldq xmm1, xmm6
; xmm1 = (00 01 02 03 04 05 06 07 10 11 12 13 14 15 16 17)
punpckhdq xmm3, xmm6
; xmm3 = (20 21 22 23 24 25 26 27 30 31 32 33 34 35 36 37)
movdqa xmm7, xmm4
punpckldq xmm4, xmm2
; xmm4 = (40 41 42 43 44 45 46 47 50 51 52 53 54 55 56 57)
punpckhdq xmm7, xmm2
; xmm7 = (60 61 62 63 64 65 66 67 70 71 72 73 74 75 76 77)
pshufd xmm5, xmm1, 0x4E
; xmm5 = (10 11 12 13 14 15 16 17 00 01 02 03 04 05 06 07)
pshufd xmm0, xmm3, 0x4E
; xmm0 = (30 31 32 33 34 35 36 37 20 21 22 23 24 25 26 27)
pshufd xmm6, xmm4, 0x4E
; xmm6 = (50 51 52 53 54 55 56 57 40 41 42 43 44 45 46 47)
pshufd xmm2, xmm7, 0x4E
; xmm2 = (70 71 72 73 74 75 76 77 60 61 62 63 64 65 66 67)
mov rdxp, JSAMPROW [rdi + 0 * SIZEOF_JSAMPROW]
mov rsip, JSAMPROW [rdi + 2 * SIZEOF_JSAMPROW]
movq XMM_MMWORD [rdx + rax * SIZEOF_JSAMPLE], xmm1
movq XMM_MMWORD [rsi + rax * SIZEOF_JSAMPLE], xmm3
mov rdxp, JSAMPROW [rdi + 4 * SIZEOF_JSAMPROW]
mov rsip, JSAMPROW [rdi + 6 * SIZEOF_JSAMPROW]
movq XMM_MMWORD [rdx + rax * SIZEOF_JSAMPLE], xmm4
movq XMM_MMWORD [rsi + rax * SIZEOF_JSAMPLE], xmm7
mov rdxp, JSAMPROW [rdi + 1 * SIZEOF_JSAMPROW]
mov rsip, JSAMPROW [rdi + 3 * SIZEOF_JSAMPROW]
movq XMM_MMWORD [rdx + rax * SIZEOF_JSAMPLE], xmm5
movq XMM_MMWORD [rsi + rax * SIZEOF_JSAMPLE], xmm0
mov rdxp, JSAMPROW [rdi + 5 * SIZEOF_JSAMPROW]
mov rsip, JSAMPROW [rdi + 7 * SIZEOF_JSAMPROW]
movq XMM_MMWORD [rdx + rax * SIZEOF_JSAMPLE], xmm6
movq XMM_MMWORD [rsi + rax * SIZEOF_JSAMPLE], xmm2
UNCOLLECT_ARGS 4
lea rsp, [rbp - 8]
pop r15
pop rbp
ret
ret
; For some reason, the OS X linker does not honor the request to align the
; segment unless we do this.
align 32
lib/libjpeg-turbo/simd/x86_64/jidctint-avx2.asm
Executable
+426
View File
@@ -0,0 +1,426 @@
;
; Accurate integer IDCT (64-bit AVX2)
;
; Copyright 2009 Pierre Ossman <[email protected]> for Cendio AB
; Copyright (C) 2009, 2016, 2018, 2020, 2024-2025, D. R. Commander.
; Copyright (C) 2018, Matthias Räncker.
;
; Based on the x86 SIMD extension for IJG JPEG library
; Copyright (C) 1999-2006, MIYASAKA Masaru.
; For conditions of distribution and use, see copyright notice in jsimdext.inc
;
; This file should be assembled with NASM (Netwide Assembler) or Yasm.
;
; This file contains a slower but more accurate integer implementation of the
; inverse DCT (Discrete Cosine Transform). The following code is based
; directly on the IJG's original jidctint.c; see jidctint.c for more details.
%include "jsimdext.inc"
%include "jdct.inc"
; --------------------------------------------------------------------------
%define CONST_BITS 13
%define PASS1_BITS 2
%define DESCALE_P1 (CONST_BITS - PASS1_BITS)
%define DESCALE_P2 (CONST_BITS + PASS1_BITS + 3)
%if CONST_BITS == 13
F_0_298 equ 2446 ; FIX(0.298631336)
F_0_390 equ 3196 ; FIX(0.390180644)
F_0_541 equ 4433 ; FIX(0.541196100)
F_0_765 equ 6270 ; FIX(0.765366865)
F_0_899 equ 7373 ; FIX(0.899976223)
F_1_175 equ 9633 ; FIX(1.175875602)
F_1_501 equ 12299 ; FIX(1.501321110)
F_1_847 equ 15137 ; FIX(1.847759065)
F_1_961 equ 16069 ; FIX(1.961570560)
F_2_053 equ 16819 ; FIX(2.053119869)
F_2_562 equ 20995 ; FIX(2.562915447)
F_3_072 equ 25172 ; FIX(3.072711026)
%else
; NASM cannot do compile-time arithmetic on floating-point constants.
%define DESCALE(x, n) (((x) + (1 << ((n) - 1))) >> (n))
F_0_298 equ DESCALE( 320652955, 30 - CONST_BITS) ; FIX(0.298631336)
F_0_390 equ DESCALE( 418953276, 30 - CONST_BITS) ; FIX(0.390180644)
F_0_541 equ DESCALE( 581104887, 30 - CONST_BITS) ; FIX(0.541196100)
F_0_765 equ DESCALE( 821806413, 30 - CONST_BITS) ; FIX(0.765366865)
F_0_899 equ DESCALE( 966342111, 30 - CONST_BITS) ; FIX(0.899976223)
F_1_175 equ DESCALE(1262586813, 30 - CONST_BITS) ; FIX(1.175875602)
F_1_501 equ DESCALE(1612031267, 30 - CONST_BITS) ; FIX(1.501321110)
F_1_847 equ DESCALE(1984016188, 30 - CONST_BITS) ; FIX(1.847759065)
F_1_961 equ DESCALE(2106220350, 30 - CONST_BITS) ; FIX(1.961570560)
F_2_053 equ DESCALE(2204520673, 30 - CONST_BITS) ; FIX(2.053119869)
F_2_562 equ DESCALE(2751909506, 30 - CONST_BITS) ; FIX(2.562915447)
F_3_072 equ DESCALE(3299298341, 30 - CONST_BITS) ; FIX(3.072711026)
%endif
; --------------------------------------------------------------------------
; In-place 8x8x16-bit inverse matrix transpose using AVX2 instructions
; %1-%4: Input/output registers
; %5-%8: Temp registers
%macro DOTRANSPOSE 8
; %5 = (00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71)
; %6 = (03 13 23 33 43 53 63 73 02 12 22 32 42 52 62 72)
; %7 = (04 14 24 34 44 54 64 74 05 15 25 35 45 55 65 75)
; %8 = (07 17 27 37 47 57 67 77 06 16 26 36 46 56 66 76)
; transpose coefficients(phase 1)
vpermq %5, %1, 0xD8
; %5 = (00 10 20 30 01 11 21 31 40 50 60 70 41 51 61 71)
vpermq %6, %2, 0x72
; %6 = (02 12 22 32 03 13 23 33 42 52 62 72 43 53 63 73)
vpermq %7, %3, 0xD8
; %7 = (04 14 24 34 05 15 25 35 44 54 64 74 45 55 65 75)
vpermq %8, %4, 0x72
; %8 = (06 16 26 36 07 17 27 37 46 56 66 76 47 57 67 77)
; transpose coefficients(phase 2)
vpunpcklwd %1, %5, %6
; %1 = (00 02 10 12 20 22 30 32 40 42 50 52 60 62 70 72)
vpunpckhwd %2, %5, %6
; %2 = (01 03 11 13 21 23 31 33 41 43 51 53 61 63 71 73)
vpunpcklwd %3, %7, %8
; %3 = (04 06 14 16 24 26 34 36 44 46 54 56 64 66 74 76)
vpunpckhwd %4, %7, %8
; %4 = (05 07 15 17 25 27 35 37 45 47 55 57 65 67 75 77)
; transpose coefficients(phase 3)
vpunpcklwd %5, %1, %2
; %5 = (00 01 02 03 10 11 12 13 40 41 42 43 50 51 52 53)
vpunpcklwd %6, %3, %4
; %6 = (04 05 06 07 14 15 16 17 44 45 46 47 54 55 56 57)
vpunpckhwd %7, %1, %2
; %7 = (20 21 22 23 30 31 32 33 60 61 62 63 70 71 72 73)
vpunpckhwd %8, %3, %4
; %8 = (24 25 26 27 34 35 36 37 64 65 66 67 74 75 76 77)
; transpose coefficients(phase 4)
vpunpcklqdq %1, %5, %6
; %1 = (00 01 02 03 04 05 06 07 40 41 42 43 44 45 46 47)
vpunpckhqdq %2, %5, %6
; %2 = (10 11 12 13 14 15 16 17 50 51 52 53 54 55 56 57)
vpunpcklqdq %3, %7, %8
; %3 = (20 21 22 23 24 25 26 27 60 61 62 63 64 65 66 67)
vpunpckhqdq %4, %7, %8
; %4 = (30 31 32 33 34 35 36 37 70 71 72 73 74 75 76 77)
%endmacro
; --------------------------------------------------------------------------
; In-place 8x8x16-bit accurate integer inverse DCT using AVX2 instructions
; %1-%4: Input/output registers
; %5-%12: Temp registers
; %9: Pass (1 or 2)
%macro DODCT 13
; -- Even part
; (Original)
; z1 = (z2 + z3) * 0.541196100;
; tmp2 = z1 + z3 * -1.847759065;
; tmp3 = z1 + z2 * 0.765366865;
;
; (This implementation)
; tmp2 = z2 * 0.541196100 + z3 * (0.541196100 - 1.847759065);
; tmp3 = z2 * (0.541196100 + 0.765366865) + z3 * 0.541196100;
vperm2i128 %6, %3, %3, 0x01 ; %6 = in6_2
vpunpcklwd %5, %3, %6 ; %5 = in26_62L
vpunpckhwd %6, %3, %6 ; %6 = in26_62H
vpmaddwd %5, %5, [rel PW_F130_F054_MF130_F054] ; %5 = tmp3_2L
vpmaddwd %6, %6, [rel PW_F130_F054_MF130_F054] ; %6 = tmp3_2H
vperm2i128 %7, %1, %1, 0x01 ; %7 = in4_0
vpsignw %1, %1, [rel PW_1_NEG1]
vpaddw %7, %7, %1 ; %7 = (in0 + in4)_(in0 - in4)
vpxor %1, %1, %1
vpunpcklwd %8, %1, %7 ; %8 = tmp0_1L
vpunpckhwd %1, %1, %7 ; %1 = tmp0_1H
vpsrad %8, %8, (16 - CONST_BITS)
; vpsrad %8, 16 & vpslld %8, CONST_BITS
vpsrad %1, %1, (16 - CONST_BITS)
; vpsrad %1, 16 & vpslld %1, CONST_BITS
vpsubd %11, %8, %5 ; %11 = tmp0_1L - tmp3_2L = tmp13_12L
vpaddd %9, %8, %5 ; %9 = tmp0_1L + tmp3_2L = tmp10_11L
vpsubd %12, %1, %6 ; %12 = tmp0_1H - tmp3_2H = tmp13_12H
vpaddd %10, %1, %6 ; %10 = tmp0_1H + tmp3_2H = tmp10_11H
; -- Odd part
vpaddw %1, %4, %2 ; %1 = in7_5 + in3_1 = z3_4
; (Original)
; z5 = (z3 + z4) * 1.175875602;
; z3 = z3 * -1.961570560; z4 = z4 * -0.390180644;
; z3 += z5; z4 += z5;
;
; (This implementation)
; z3 = z3 * (1.175875602 - 1.961570560) + z4 * 1.175875602;
; z4 = z3 * 1.175875602 + z4 * (1.175875602 - 0.390180644);
vperm2i128 %8, %1, %1, 0x01 ; %8 = z4_3
vpunpcklwd %7, %1, %8 ; %7 = z34_43L
vpunpckhwd %8, %1, %8 ; %8 = z34_43H
vpmaddwd %7, %7, [rel PW_MF078_F117_F078_F117] ; %7 = z3_4L
vpmaddwd %8, %8, [rel PW_MF078_F117_F078_F117] ; %8 = z3_4H
; (Original)
; z1 = tmp0 + tmp3; z2 = tmp1 + tmp2;
; tmp0 = tmp0 * 0.298631336; tmp1 = tmp1 * 2.053119869;
; tmp2 = tmp2 * 3.072711026; tmp3 = tmp3 * 1.501321110;
; z1 = z1 * -0.899976223; z2 = z2 * -2.562915447;
; tmp0 += z1 + z3; tmp1 += z2 + z4;
; tmp2 += z2 + z3; tmp3 += z1 + z4;
;
; (This implementation)
; tmp0 = tmp0 * (0.298631336 - 0.899976223) + tmp3 * -0.899976223;
; tmp1 = tmp1 * (2.053119869 - 2.562915447) + tmp2 * -2.562915447;
; tmp2 = tmp1 * -2.562915447 + tmp2 * (3.072711026 - 2.562915447);
; tmp3 = tmp0 * -0.899976223 + tmp3 * (1.501321110 - 0.899976223);
; tmp0 += z3; tmp1 += z4;
; tmp2 += z3; tmp3 += z4;
vperm2i128 %2, %2, %2, 0x01 ; %2 = in1_3
vpunpcklwd %3, %4, %2 ; %3 = in71_53L
vpunpckhwd %4, %4, %2 ; %4 = in71_53H
vpmaddwd %5, %3, [rel PW_MF060_MF089_MF050_MF256] ; %5 = tmp0_1L
vpmaddwd %6, %4, [rel PW_MF060_MF089_MF050_MF256] ; %6 = tmp0_1H
vpaddd %5, %5, %7 ; %5 = tmp0_1L + z3_4L = tmp0_1L
vpaddd %6, %6, %8 ; %6 = tmp0_1H + z3_4H = tmp0_1H
vpmaddwd %3, %3, [rel PW_MF089_F060_MF256_F050] ; %3 = tmp3_2L
vpmaddwd %4, %4, [rel PW_MF089_F060_MF256_F050] ; %4 = tmp3_2H
vperm2i128 %7, %7, %7, 0x01 ; %7 = z4_3L
vperm2i128 %8, %8, %8, 0x01 ; %8 = z4_3H
vpaddd %7, %3, %7 ; %7 = tmp3_2L + z4_3L = tmp3_2L
vpaddd %8, %4, %8 ; %8 = tmp3_2H + z4_3H = tmp3_2H
; -- Final output stage
vpaddd %1, %9, %7 ; %1 = tmp10_11L + tmp3_2L = data0_1L
vpaddd %2, %10, %8 ; %2 = tmp10_11H + tmp3_2H = data0_1H
vpaddd %1, %1, [rel PD_DESCALE_P %+ %13]
vpaddd %2, %2, [rel PD_DESCALE_P %+ %13]
vpsrad %1, %1, DESCALE_P %+ %13
vpsrad %2, %2, DESCALE_P %+ %13
vpackssdw %1, %1, %2 ; %1 = data0_1
vpsubd %3, %9, %7 ; %3 = tmp10_11L - tmp3_2L = data7_6L
vpsubd %4, %10, %8 ; %4 = tmp10_11H - tmp3_2H = data7_6H
vpaddd %3, %3, [rel PD_DESCALE_P %+ %13]
vpaddd %4, %4, [rel PD_DESCALE_P %+ %13]
vpsrad %3, %3, DESCALE_P %+ %13
vpsrad %4, %4, DESCALE_P %+ %13
vpackssdw %4, %3, %4 ; %4 = data7_6
vpaddd %7, %11, %5 ; %7 = tmp13_12L + tmp0_1L = data3_2L
vpaddd %8, %12, %6 ; %8 = tmp13_12H + tmp0_1H = data3_2H
vpaddd %7, %7, [rel PD_DESCALE_P %+ %13]
vpaddd %8, %8, [rel PD_DESCALE_P %+ %13]
vpsrad %7, %7, DESCALE_P %+ %13
vpsrad %8, %8, DESCALE_P %+ %13
vpackssdw %2, %7, %8 ; %2 = data3_2
vpsubd %7, %11, %5 ; %7 = tmp13_12L - tmp0_1L = data4_5L
vpsubd %8, %12, %6 ; %8 = tmp13_12H - tmp0_1H = data4_5H
vpaddd %7, %7, [rel PD_DESCALE_P %+ %13]
vpaddd %8, %8, [rel PD_DESCALE_P %+ %13]
vpsrad %7, %7, DESCALE_P %+ %13
vpsrad %8, %8, DESCALE_P %+ %13
vpackssdw %3, %7, %8 ; %3 = data4_5
%endmacro
; --------------------------------------------------------------------------
SECTION SEG_CONST
ALIGNZ 32
GLOBAL_DATA(jconst_idct_islow_avx2)
EXTN(jconst_idct_islow_avx2):
PW_F130_F054_MF130_F054 times 4 dw (F_0_541 + F_0_765), F_0_541
times 4 dw (F_0_541 - F_1_847), F_0_541
PW_MF078_F117_F078_F117 times 4 dw (F_1_175 - F_1_961), F_1_175
times 4 dw (F_1_175 - F_0_390), F_1_175
PW_MF060_MF089_MF050_MF256 times 4 dw (F_0_298 - F_0_899), -F_0_899
times 4 dw (F_2_053 - F_2_562), -F_2_562
PW_MF089_F060_MF256_F050 times 4 dw -F_0_899, (F_1_501 - F_0_899)
times 4 dw -F_2_562, (F_3_072 - F_2_562)
PD_DESCALE_P1 times 8 dd 1 << (DESCALE_P1 - 1)
PD_DESCALE_P2 times 8 dd 1 << (DESCALE_P2 - 1)
PB_CENTERJSAMP times 32 db CENTERJSAMPLE
PW_1_NEG1 times 8 dw 1
times 8 dw -1
ALIGNZ 32
; --------------------------------------------------------------------------
SECTION SEG_TEXT
BITS 64
; Perform dequantization and inverse DCT on one block of coefficients.
;
; GLOBAL(void)
; jsimd_idct_islow_avx2(void *dct_table, JCOEFPTR coef_block,
; JSAMPARRAY output_buf, JDIMENSION output_col)
;
; r10 = jpeg_component_info *compptr
; r11 = JCOEFPTR coef_block
; r12 = JSAMPARRAY output_buf
; r13d = JDIMENSION output_col
align 32
GLOBAL_FUNCTION(jsimd_idct_islow_avx2)
EXTN(jsimd_idct_islow_avx2):
ENDBR64
push rbp
mov rbp, rsp ; rbp = aligned rbp
PUSH_XMM 4
COLLECT_ARGS 4
; ---- Pass 1: process columns.
%ifndef NO_ZERO_COLUMN_TEST_ISLOW_AVX2
mov eax, dword [DWBLOCK(1, 0, r11, SIZEOF_JCOEF)]
or eax, dword [DWBLOCK(2, 0, r11, SIZEOF_JCOEF)]
jnz near .columnDCT
movdqa xmm0, XMMWORD [XMMBLOCK(1, 0, r11, SIZEOF_JCOEF)]
movdqa xmm1, XMMWORD [XMMBLOCK(2, 0, r11, SIZEOF_JCOEF)]
vpor xmm0, xmm0, XMMWORD [XMMBLOCK(3, 0, r11, SIZEOF_JCOEF)]
vpor xmm1, xmm1, XMMWORD [XMMBLOCK(4, 0, r11, SIZEOF_JCOEF)]
vpor xmm0, xmm0, XMMWORD [XMMBLOCK(5, 0, r11, SIZEOF_JCOEF)]
vpor xmm1, xmm1, XMMWORD [XMMBLOCK(6, 0, r11, SIZEOF_JCOEF)]
vpor xmm0, xmm0, XMMWORD [XMMBLOCK(7, 0, r11, SIZEOF_JCOEF)]
vpor xmm1, xmm1, xmm0
vpacksswb xmm1, xmm1, xmm1
vpacksswb xmm1, xmm1, xmm1
movd eax, xmm1
test rax, rax
jnz short .columnDCT
; -- AC terms all zero
movdqa xmm5, XMMWORD [XMMBLOCK(0, 0, r11, SIZEOF_JCOEF)]
vpmullw xmm5, xmm5, XMMWORD [XMMBLOCK(0, 0, r10, SIZEOF_ISLOW_MULT_TYPE)]
vpsllw xmm5, xmm5, PASS1_BITS
vpunpcklwd xmm4, xmm5, xmm5 ; xmm4 = (00 00 01 01 02 02 03 03)
vpunpckhwd xmm5, xmm5, xmm5 ; xmm5 = (04 04 05 05 06 06 07 07)
vinserti128 ymm4, ymm4, xmm5, 1
vpshufd ymm0, ymm4, 0x00
; ymm0 = col0_4 = (00 00 00 00 00 00 00 00 04 04 04 04 04 04 04 04)
vpshufd ymm1, ymm4, 0x55
; ymm1 = col1_5 = (01 01 01 01 01 01 01 01 05 05 05 05 05 05 05 05)
vpshufd ymm2, ymm4, 0xAA
; ymm2 = col2_6 = (02 02 02 02 02 02 02 02 06 06 06 06 06 06 06 06)
vpshufd ymm3, ymm4, 0xFF
; ymm3 = col3_7 = (03 03 03 03 03 03 03 03 07 07 07 07 07 07 07 07)
jmp near .column_end
%endif
.columnDCT:
vmovdqu ymm4, YMMWORD [YMMBLOCK(0, 0, r11, SIZEOF_JCOEF)]
; ymm4 = in0_1
vmovdqu ymm5, YMMWORD [YMMBLOCK(2, 0, r11, SIZEOF_JCOEF)]
; ymm5 = in2_3
vmovdqu ymm6, YMMWORD [YMMBLOCK(4, 0, r11, SIZEOF_JCOEF)]
; ymm6 = in4_5
vmovdqu ymm7, YMMWORD [YMMBLOCK(6, 0, r11, SIZEOF_JCOEF)]
; ymm7 = in6_7
vpmullw ymm4, ymm4, YMMWORD [YMMBLOCK(0, 0, r10, SIZEOF_ISLOW_MULT_TYPE)]
vpmullw ymm5, ymm5, YMMWORD [YMMBLOCK(2, 0, r10, SIZEOF_ISLOW_MULT_TYPE)]
vpmullw ymm6, ymm6, YMMWORD [YMMBLOCK(4, 0, r10, SIZEOF_ISLOW_MULT_TYPE)]
vpmullw ymm7, ymm7, YMMWORD [YMMBLOCK(6, 0, r10, SIZEOF_ISLOW_MULT_TYPE)]
vperm2i128 ymm0, ymm4, ymm6, 0x20 ; ymm0 = in0_4
vperm2i128 ymm1, ymm5, ymm4, 0x31 ; ymm1 = in3_1
vperm2i128 ymm2, ymm5, ymm7, 0x20 ; ymm2 = in2_6
vperm2i128 ymm3, ymm7, ymm6, 0x31 ; ymm3 = in7_5
DODCT ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, ymm6, ymm7, ymm8, ymm9, ymm10, ymm11, 1
; ymm0 = data0_1, ymm1 = data3_2, ymm2 = data4_5, ymm3 = data7_6
DOTRANSPOSE ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, ymm6, ymm7
; ymm0 = data0_4, ymm1 = data1_5, ymm2 = data2_6, ymm3 = data3_7
.column_end:
; -- Prefetch the next coefficient block
prefetchnta [r11 + DCTSIZE2 * SIZEOF_JCOEF + 0 * 32]
prefetchnta [r11 + DCTSIZE2 * SIZEOF_JCOEF + 1 * 32]
prefetchnta [r11 + DCTSIZE2 * SIZEOF_JCOEF + 2 * 32]
prefetchnta [r11 + DCTSIZE2 * SIZEOF_JCOEF + 3 * 32]
; ---- Pass 2: process rows.
vperm2i128 ymm4, ymm3, ymm1, 0x31 ; ymm3 = in7_5
vperm2i128 ymm1, ymm3, ymm1, 0x20 ; ymm1 = in3_1
DODCT ymm0, ymm1, ymm2, ymm4, ymm3, ymm5, ymm6, ymm7, ymm8, ymm9, ymm10, ymm11, 2
; ymm0 = data0_1, ymm1 = data3_2, ymm2 = data4_5, ymm4 = data7_6
DOTRANSPOSE ymm0, ymm1, ymm2, ymm4, ymm3, ymm5, ymm6, ymm7
; ymm0 = data0_4, ymm1 = data1_5, ymm2 = data2_6, ymm4 = data3_7
vpacksswb ymm0, ymm0, ymm1 ; ymm0 = data01_45
vpacksswb ymm1, ymm2, ymm4 ; ymm1 = data23_67
vpaddb ymm0, ymm0, [rel PB_CENTERJSAMP]
vpaddb ymm1, ymm1, [rel PB_CENTERJSAMP]
vextracti128 xmm6, ymm1, 1 ; xmm3 = data67
vextracti128 xmm4, ymm0, 1 ; xmm2 = data45
vextracti128 xmm2, ymm1, 0 ; xmm1 = data23
vextracti128 xmm0, ymm0, 0 ; xmm0 = data01
vpshufd xmm1, xmm0, 0x4E
; xmm1 = (10 11 12 13 14 15 16 17 00 01 02 03 04 05 06 07)
vpshufd xmm3, xmm2, 0x4E
; xmm3 = (30 31 32 33 34 35 36 37 20 21 22 23 24 25 26 27)
vpshufd xmm5, xmm4, 0x4E
; xmm5 = (50 51 52 53 54 55 56 57 40 41 42 43 44 45 46 47)
vpshufd xmm7, xmm6, 0x4E
; xmm7 = (70 71 72 73 74 75 76 77 60 61 62 63 64 65 66 67)
vzeroupper
mov eax, r13d
mov rdxp, JSAMPROW [r12 + 0 * SIZEOF_JSAMPROW] ; (JSAMPLE *)
mov rsip, JSAMPROW [r12 + 1 * SIZEOF_JSAMPROW] ; (JSAMPLE *)
movq XMM_MMWORD [rdx + rax * SIZEOF_JSAMPLE], xmm0
movq XMM_MMWORD [rsi + rax * SIZEOF_JSAMPLE], xmm1
mov rdxp, JSAMPROW [r12 + 2 * SIZEOF_JSAMPROW] ; (JSAMPLE *)
mov rsip, JSAMPROW [r12 + 3 * SIZEOF_JSAMPROW] ; (JSAMPLE *)
movq XMM_MMWORD [rdx + rax * SIZEOF_JSAMPLE], xmm2
movq XMM_MMWORD [rsi + rax * SIZEOF_JSAMPLE], xmm3
mov rdxp, JSAMPROW [r12 + 4 * SIZEOF_JSAMPROW] ; (JSAMPLE *)
mov rsip, JSAMPROW [r12 + 5 * SIZEOF_JSAMPROW] ; (JSAMPLE *)
movq XMM_MMWORD [rdx + rax * SIZEOF_JSAMPLE], xmm4
movq XMM_MMWORD [rsi + rax * SIZEOF_JSAMPLE], xmm5
mov rdxp, JSAMPROW [r12 + 6 * SIZEOF_JSAMPROW] ; (JSAMPLE *)
mov rsip, JSAMPROW [r12 + 7 * SIZEOF_JSAMPROW] ; (JSAMPLE *)
movq XMM_MMWORD [rdx + rax * SIZEOF_JSAMPLE], xmm6
movq XMM_MMWORD [rsi + rax * SIZEOF_JSAMPLE], xmm7
UNCOLLECT_ARGS 4
POP_XMM 4
pop rbp
ret
; For some reason, the OS X linker does not honor the request to align the
; segment unless we do this.
align 32
lib/libjpeg-turbo/simd/x86_64/jidctint-sse2.asm
Executable
+876
View File
@@ -0,0 +1,876 @@
;
; Accurate integer IDCT (64-bit SSE2)
;
; Copyright 2009 Pierre Ossman <[email protected]> for Cendio AB
; Copyright (C) 2009, 2016, 2020, 2024-2025, D. R. Commander.
; Copyright (C) 2018, Matthias Räncker.
; Copyright (C) 2023, Aliaksiej Kandracienka.
;
; Based on the x86 SIMD extension for IJG JPEG library
; Copyright (C) 1999-2006, MIYASAKA Masaru.
; For conditions of distribution and use, see copyright notice in jsimdext.inc
;
; This file should be assembled with NASM (Netwide Assembler) or Yasm.
;
; This file contains a slower but more accurate integer implementation of the
; inverse DCT (Discrete Cosine Transform). The following code is based
; directly on the IJG's original jidctint.c; see jidctint.c for more details.
%include "jsimdext.inc"
%include "jdct.inc"
; --------------------------------------------------------------------------
%define CONST_BITS 13
%define PASS1_BITS 2
%define DESCALE_P1 (CONST_BITS - PASS1_BITS)
%define DESCALE_P2 (CONST_BITS + PASS1_BITS + 3)
%if CONST_BITS == 13
F_0_298 equ 2446 ; FIX(0.298631336)
F_0_390 equ 3196 ; FIX(0.390180644)
F_0_541 equ 4433 ; FIX(0.541196100)
F_0_765 equ 6270 ; FIX(0.765366865)
F_0_899 equ 7373 ; FIX(0.899976223)
F_1_175 equ 9633 ; FIX(1.175875602)
F_1_501 equ 12299 ; FIX(1.501321110)
F_1_847 equ 15137 ; FIX(1.847759065)
F_1_961 equ 16069 ; FIX(1.961570560)
F_2_053 equ 16819 ; FIX(2.053119869)
F_2_562 equ 20995 ; FIX(2.562915447)
F_3_072 equ 25172 ; FIX(3.072711026)
%else
; NASM cannot do compile-time arithmetic on floating-point constants.
%define DESCALE(x, n) (((x) + (1 << ((n) - 1))) >> (n))
F_0_298 equ DESCALE( 320652955, 30 - CONST_BITS) ; FIX(0.298631336)
F_0_390 equ DESCALE( 418953276, 30 - CONST_BITS) ; FIX(0.390180644)
F_0_541 equ DESCALE( 581104887, 30 - CONST_BITS) ; FIX(0.541196100)
F_0_765 equ DESCALE( 821806413, 30 - CONST_BITS) ; FIX(0.765366865)
F_0_899 equ DESCALE( 966342111, 30 - CONST_BITS) ; FIX(0.899976223)
F_1_175 equ DESCALE(1262586813, 30 - CONST_BITS) ; FIX(1.175875602)
F_1_501 equ DESCALE(1612031267, 30 - CONST_BITS) ; FIX(1.501321110)
F_1_847 equ DESCALE(1984016188, 30 - CONST_BITS) ; FIX(1.847759065)
F_1_961 equ DESCALE(2106220350, 30 - CONST_BITS) ; FIX(1.961570560)
F_2_053 equ DESCALE(2204520673, 30 - CONST_BITS) ; FIX(2.053119869)
F_2_562 equ DESCALE(2751909506, 30 - CONST_BITS) ; FIX(2.562915447)
F_3_072 equ DESCALE(3299298341, 30 - CONST_BITS) ; FIX(3.072711026)
%endif
; --------------------------------------------------------------------------
SECTION SEG_CONST
ALIGNZ 32
GLOBAL_DATA(jconst_idct_islow_sse2)
EXTN(jconst_idct_islow_sse2):
PW_F130_F054 times 4 dw (F_0_541 + F_0_765), F_0_541
PW_F054_MF130 times 4 dw F_0_541, (F_0_541 - F_1_847)
PW_MF078_F117 times 4 dw (F_1_175 - F_1_961), F_1_175
PW_F117_F078 times 4 dw F_1_175, (F_1_175 - F_0_390)
PW_MF060_MF089 times 4 dw (F_0_298 - F_0_899), -F_0_899
PW_MF089_F060 times 4 dw -F_0_899, (F_1_501 - F_0_899)
PW_MF050_MF256 times 4 dw (F_2_053 - F_2_562), -F_2_562
PW_MF256_F050 times 4 dw -F_2_562, (F_3_072 - F_2_562)
PD_DESCALE_P1 times 4 dd 1 << (DESCALE_P1 - 1)
PD_DESCALE_P2 times 4 dd 1 << (DESCALE_P2 - 1)
PB_CENTERJSAMP times 16 db CENTERJSAMPLE
ALIGNZ 32
; --------------------------------------------------------------------------
SECTION SEG_TEXT
BITS 64
; Perform dequantization and inverse DCT on one block of coefficients.
;
; GLOBAL(void)
; jsimd_idct_islow_sse2(void *dct_table, JCOEFPTR coef_block,
; JSAMPARRAY output_buf, JDIMENSION output_col)
;
; r10 = jpeg_component_info *compptr
; r11 = JCOEFPTR coef_block
; r12 = JSAMPARRAY output_buf
; r13d = JDIMENSION output_col
%define wk(i) r15 - (WK_NUM - (i)) * SIZEOF_XMMWORD
; xmmword wk[WK_NUM]
%define WK_NUM 12
align 32
GLOBAL_FUNCTION(jsimd_idct_islow_sse2)
EXTN(jsimd_idct_islow_sse2):
ENDBR64
push rbp
mov rbp, rsp
push r15
and rsp, byte (-SIZEOF_XMMWORD) ; align to 128 bits
; Allocate stack space for wk array. r15 is used to access it.
mov r15, rsp
sub rsp, (SIZEOF_XMMWORD * WK_NUM)
COLLECT_ARGS 4
; ---- Pass 1: process columns from input.
mov rdx, r10 ; quantptr
mov rsi, r11 ; inptr
%ifndef NO_ZERO_COLUMN_TEST_ISLOW_SSE2
mov eax, dword [DWBLOCK(1, 0, rsi, SIZEOF_JCOEF)]
or eax, dword [DWBLOCK(2, 0, rsi, SIZEOF_JCOEF)]
jnz near .columnDCT
movdqa xmm0, XMMWORD [XMMBLOCK(1, 0, rsi, SIZEOF_JCOEF)]
movdqa xmm1, XMMWORD [XMMBLOCK(2, 0, rsi, SIZEOF_JCOEF)]
por xmm0, XMMWORD [XMMBLOCK(3, 0, rsi, SIZEOF_JCOEF)]
por xmm1, XMMWORD [XMMBLOCK(4, 0, rsi, SIZEOF_JCOEF)]
por xmm0, XMMWORD [XMMBLOCK(5, 0, rsi, SIZEOF_JCOEF)]
por xmm1, XMMWORD [XMMBLOCK(6, 0, rsi, SIZEOF_JCOEF)]
por xmm0, XMMWORD [XMMBLOCK(7, 0, rsi, SIZEOF_JCOEF)]
por xmm1, xmm0
packsswb xmm1, xmm1
packsswb xmm1, xmm1
movd eax, xmm1
test rax, rax
jnz short .columnDCT
; -- AC terms all zero
movdqa xmm5, XMMWORD [XMMBLOCK(0, 0, rsi, SIZEOF_JCOEF)]
pmullw xmm5, XMMWORD [XMMBLOCK(0, 0, rdx, SIZEOF_ISLOW_MULT_TYPE)]
psllw xmm5, PASS1_BITS
movdqa xmm4, xmm5 ; xmm5 = in0 = (00 01 02 03 04 05 06 07)
punpcklwd xmm5, xmm5 ; xmm5 = (00 00 01 01 02 02 03 03)
punpckhwd xmm4, xmm4 ; xmm4 = (04 04 05 05 06 06 07 07)
pshufd xmm7, xmm5, 0x00 ; xmm7 = col0 = (00 00 00 00 00 00 00 00)
pshufd xmm6, xmm5, 0x55 ; xmm6 = col1 = (01 01 01 01 01 01 01 01)
pshufd xmm1, xmm5, 0xAA ; xmm1 = col2 = (02 02 02 02 02 02 02 02)
pshufd xmm5, xmm5, 0xFF ; xmm5 = col3 = (03 03 03 03 03 03 03 03)
pshufd xmm0, xmm4, 0x00 ; xmm0 = col4 = (04 04 04 04 04 04 04 04)
pshufd xmm3, xmm4, 0x55 ; xmm3 = col5 = (05 05 05 05 05 05 05 05)
pshufd xmm2, xmm4, 0xAA ; xmm2 = col6 = (06 06 06 06 06 06 06 06)
pshufd xmm4, xmm4, 0xFF ; xmm4 = col7 = (07 07 07 07 07 07 07 07)
movdqa XMMWORD [wk(8)], xmm6 ; wk(8) = col1
movdqa XMMWORD [wk(9)], xmm5 ; wk(9) = col3
movdqa XMMWORD [wk(10)], xmm3 ; wk(10) = col5
movdqa XMMWORD [wk(11)], xmm4 ; wk(11) = col7
jmp near .column_end
%endif
.columnDCT:
; -- Even part
movdqa xmm0, XMMWORD [XMMBLOCK(0, 0, rsi, SIZEOF_JCOEF)]
movdqa xmm1, XMMWORD [XMMBLOCK(2, 0, rsi, SIZEOF_JCOEF)]
pmullw xmm0, XMMWORD [XMMBLOCK(0, 0, rdx, SIZEOF_ISLOW_MULT_TYPE)]
pmullw xmm1, XMMWORD [XMMBLOCK(2, 0, rdx, SIZEOF_ISLOW_MULT_TYPE)]
movdqa xmm2, XMMWORD [XMMBLOCK(4, 0, rsi, SIZEOF_JCOEF)]
movdqa xmm3, XMMWORD [XMMBLOCK(6, 0, rsi, SIZEOF_JCOEF)]
pmullw xmm2, XMMWORD [XMMBLOCK(4, 0, rdx, SIZEOF_ISLOW_MULT_TYPE)]
pmullw xmm3, XMMWORD [XMMBLOCK(6, 0, rdx, SIZEOF_ISLOW_MULT_TYPE)]
; (Original)
; z1 = (z2 + z3) * 0.541196100;
; tmp2 = z1 + z3 * -1.847759065;
; tmp3 = z1 + z2 * 0.765366865;
;
; (This implementation)
; tmp2 = z2 * 0.541196100 + z3 * (0.541196100 - 1.847759065);
; tmp3 = z2 * (0.541196100 + 0.765366865) + z3 * 0.541196100;
movdqa xmm4, xmm1 ; xmm1 = in2 = z2
movdqa xmm5, xmm1
punpcklwd xmm4, xmm3 ; xmm3 = in6 = z3
punpckhwd xmm5, xmm3
movdqa xmm1, xmm4
movdqa xmm3, xmm5
pmaddwd xmm4, [rel PW_F130_F054] ; xmm4 = tmp3L
pmaddwd xmm5, [rel PW_F130_F054] ; xmm5 = tmp3H
pmaddwd xmm1, [rel PW_F054_MF130] ; xmm1 = tmp2L
pmaddwd xmm3, [rel PW_F054_MF130] ; xmm3 = tmp2H
movdqa xmm6, xmm0
paddw xmm0, xmm2 ; xmm0 = in0 + in4
psubw xmm6, xmm2 ; xmm6 = in0 - in4
pxor xmm7, xmm7
pxor xmm2, xmm2
punpcklwd xmm7, xmm0 ; xmm7 = tmp0L
punpckhwd xmm2, xmm0 ; xmm2 = tmp0H
psrad xmm7, (16 - CONST_BITS)
; psrad xmm7, 16 & pslld xmm7, CONST_BITS
psrad xmm2, (16 - CONST_BITS)
; psrad xmm2, 16 & pslld xmm2, CONST_BITS
movdqa xmm0, xmm7
paddd xmm7, xmm4 ; xmm7 = tmp10L
psubd xmm0, xmm4 ; xmm0 = tmp13L
movdqa xmm4, xmm2
paddd xmm2, xmm5 ; xmm2 = tmp10H
psubd xmm4, xmm5 ; xmm4 = tmp13H
movdqa XMMWORD [wk(0)], xmm7 ; wk(0) = tmp10L
movdqa XMMWORD [wk(1)], xmm2 ; wk(1) = tmp10H
movdqa XMMWORD [wk(2)], xmm0 ; wk(2) = tmp13L
movdqa XMMWORD [wk(3)], xmm4 ; wk(3) = tmp13H
pxor xmm5, xmm5
pxor xmm7, xmm7
punpcklwd xmm5, xmm6 ; xmm5 = tmp1L
punpckhwd xmm7, xmm6 ; xmm7 = tmp1H
psrad xmm5, (16 - CONST_BITS)
; psrad xmm5, 16 & pslld xmm5, CONST_BITS
psrad xmm7, (16 - CONST_BITS)
; psrad xmm7, 16 & pslld xmm7, CONST_BITS
movdqa xmm2, xmm5
paddd xmm5, xmm1 ; xmm5 = tmp11L
psubd xmm2, xmm1 ; xmm2 = tmp12L
movdqa xmm0, xmm7
paddd xmm7, xmm3 ; xmm7 = tmp11H
psubd xmm0, xmm3 ; xmm0 = tmp12H
movdqa XMMWORD [wk(4)], xmm5 ; wk(4) = tmp11L
movdqa XMMWORD [wk(5)], xmm7 ; wk(5) = tmp11H
movdqa XMMWORD [wk(6)], xmm2 ; wk(6) = tmp12L
movdqa XMMWORD [wk(7)], xmm0 ; wk(7) = tmp12H
; -- Odd part
movdqa xmm4, XMMWORD [XMMBLOCK(1, 0, rsi, SIZEOF_JCOEF)]
movdqa xmm6, XMMWORD [XMMBLOCK(3, 0, rsi, SIZEOF_JCOEF)]
pmullw xmm4, XMMWORD [XMMBLOCK(1, 0, rdx, SIZEOF_ISLOW_MULT_TYPE)]
pmullw xmm6, XMMWORD [XMMBLOCK(3, 0, rdx, SIZEOF_ISLOW_MULT_TYPE)]
movdqa xmm1, XMMWORD [XMMBLOCK(5, 0, rsi, SIZEOF_JCOEF)]
movdqa xmm3, XMMWORD [XMMBLOCK(7, 0, rsi, SIZEOF_JCOEF)]
pmullw xmm1, XMMWORD [XMMBLOCK(5, 0, rdx, SIZEOF_ISLOW_MULT_TYPE)]
pmullw xmm3, XMMWORD [XMMBLOCK(7, 0, rdx, SIZEOF_ISLOW_MULT_TYPE)]
movdqa xmm5, xmm6
movdqa xmm7, xmm4
paddw xmm5, xmm3 ; xmm5 = z3
paddw xmm7, xmm1 ; xmm7 = z4
; (Original)
; z5 = (z3 + z4) * 1.175875602;
; z3 = z3 * -1.961570560; z4 = z4 * -0.390180644;
; z3 += z5; z4 += z5;
;
; (This implementation)
; z3 = z3 * (1.175875602 - 1.961570560) + z4 * 1.175875602;
; z4 = z3 * 1.175875602 + z4 * (1.175875602 - 0.390180644);
movdqa xmm2, xmm5
movdqa xmm0, xmm5
punpcklwd xmm2, xmm7
punpckhwd xmm0, xmm7
movdqa xmm5, xmm2
movdqa xmm7, xmm0
pmaddwd xmm2, [rel PW_MF078_F117] ; xmm2 = z3L
pmaddwd xmm0, [rel PW_MF078_F117] ; xmm0 = z3H
pmaddwd xmm5, [rel PW_F117_F078] ; xmm5 = z4L
pmaddwd xmm7, [rel PW_F117_F078] ; xmm7 = z4H
movdqa XMMWORD [wk(10)], xmm2 ; wk(10) = z3L
movdqa XMMWORD [wk(11)], xmm0 ; wk(11) = z3H
; (Original)
; z1 = tmp0 + tmp3; z2 = tmp1 + tmp2;
; tmp0 = tmp0 * 0.298631336; tmp1 = tmp1 * 2.053119869;
; tmp2 = tmp2 * 3.072711026; tmp3 = tmp3 * 1.501321110;
; z1 = z1 * -0.899976223; z2 = z2 * -2.562915447;
; tmp0 += z1 + z3; tmp1 += z2 + z4;
; tmp2 += z2 + z3; tmp3 += z1 + z4;
;
; (This implementation)
; tmp0 = tmp0 * (0.298631336 - 0.899976223) + tmp3 * -0.899976223;
; tmp1 = tmp1 * (2.053119869 - 2.562915447) + tmp2 * -2.562915447;
; tmp2 = tmp1 * -2.562915447 + tmp2 * (3.072711026 - 2.562915447);
; tmp3 = tmp0 * -0.899976223 + tmp3 * (1.501321110 - 0.899976223);
; tmp0 += z3; tmp1 += z4;
; tmp2 += z3; tmp3 += z4;
movdqa xmm2, xmm3
movdqa xmm0, xmm3
punpcklwd xmm2, xmm4
punpckhwd xmm0, xmm4
movdqa xmm3, xmm2
movdqa xmm4, xmm0
pmaddwd xmm2, [rel PW_MF060_MF089] ; xmm2 = tmp0L
pmaddwd xmm0, [rel PW_MF060_MF089] ; xmm0 = tmp0H
pmaddwd xmm3, [rel PW_MF089_F060] ; xmm3 = tmp3L
pmaddwd xmm4, [rel PW_MF089_F060] ; xmm4 = tmp3H
paddd xmm2, XMMWORD [wk(10)] ; xmm2 = tmp0L
paddd xmm0, XMMWORD [wk(11)] ; xmm0 = tmp0H
paddd xmm3, xmm5 ; xmm3 = tmp3L
paddd xmm4, xmm7 ; xmm4 = tmp3H
movdqa XMMWORD [wk(8)], xmm2 ; wk(8) = tmp0L
movdqa XMMWORD [wk(9)], xmm0 ; wk(9) = tmp0H
movdqa xmm2, xmm1
movdqa xmm0, xmm1
punpcklwd xmm2, xmm6
punpckhwd xmm0, xmm6
movdqa xmm1, xmm2
movdqa xmm6, xmm0
pmaddwd xmm2, [rel PW_MF050_MF256] ; xmm2 = tmp1L
pmaddwd xmm0, [rel PW_MF050_MF256] ; xmm0 = tmp1H
pmaddwd xmm1, [rel PW_MF256_F050] ; xmm1 = tmp2L
pmaddwd xmm6, [rel PW_MF256_F050] ; xmm6 = tmp2H
paddd xmm2, xmm5 ; xmm2 = tmp1L
paddd xmm0, xmm7 ; xmm0 = tmp1H
paddd xmm1, XMMWORD [wk(10)] ; xmm1 = tmp2L
paddd xmm6, XMMWORD [wk(11)] ; xmm6 = tmp2H
movdqa XMMWORD [wk(10)], xmm2 ; wk(10) = tmp1L
movdqa XMMWORD [wk(11)], xmm0 ; wk(11) = tmp1H
; -- Final output stage
movdqa xmm5, XMMWORD [wk(0)] ; xmm5 = tmp10L
movdqa xmm7, XMMWORD [wk(1)] ; xmm7 = tmp10H
movdqa xmm2, xmm5
movdqa xmm0, xmm7
paddd xmm5, xmm3 ; xmm5 = data0L
paddd xmm7, xmm4 ; xmm7 = data0H
psubd xmm2, xmm3 ; xmm2 = data7L
psubd xmm0, xmm4 ; xmm0 = data7H
movdqa xmm3, [rel PD_DESCALE_P1] ; xmm3 = [rel PD_DESCALE_P1]
paddd xmm5, xmm3
paddd xmm7, xmm3
psrad xmm5, DESCALE_P1
psrad xmm7, DESCALE_P1
paddd xmm2, xmm3
paddd xmm0, xmm3
psrad xmm2, DESCALE_P1
psrad xmm0, DESCALE_P1
packssdw xmm5, xmm7 ; xmm5 = data0 = (00 01 02 03 04 05 06 07)
packssdw xmm2, xmm0 ; xmm2 = data7 = (70 71 72 73 74 75 76 77)
movdqa xmm4, XMMWORD [wk(4)] ; xmm4 = tmp11L
movdqa xmm3, XMMWORD [wk(5)] ; xmm3 = tmp11H
movdqa xmm7, xmm4
movdqa xmm0, xmm3
paddd xmm4, xmm1 ; xmm4 = data1L
paddd xmm3, xmm6 ; xmm3 = data1H
psubd xmm7, xmm1 ; xmm7 = data6L
psubd xmm0, xmm6 ; xmm0 = data6H
movdqa xmm1, [rel PD_DESCALE_P1] ; xmm1 = [rel PD_DESCALE_P1]
paddd xmm4, xmm1
paddd xmm3, xmm1
psrad xmm4, DESCALE_P1
psrad xmm3, DESCALE_P1
paddd xmm7, xmm1
paddd xmm0, xmm1
psrad xmm7, DESCALE_P1
psrad xmm0, DESCALE_P1
packssdw xmm4, xmm3 ; xmm4 = data1 = (10 11 12 13 14 15 16 17)
packssdw xmm7, xmm0 ; xmm7 = data6 = (60 61 62 63 64 65 66 67)
movdqa xmm6, xmm5 ; transpose coefficients(phase 1)
punpcklwd xmm5, xmm4 ; xmm5 = (00 10 01 11 02 12 03 13)
punpckhwd xmm6, xmm4 ; xmm6 = (04 14 05 15 06 16 07 17)
movdqa xmm1, xmm7 ; transpose coefficients(phase 1)
punpcklwd xmm7, xmm2 ; xmm7 = (60 70 61 71 62 72 63 73)
punpckhwd xmm1, xmm2 ; xmm1 = (64 74 65 75 66 76 67 77)
movdqa xmm3, XMMWORD [wk(6)] ; xmm3 = tmp12L
movdqa xmm0, XMMWORD [wk(7)] ; xmm0 = tmp12H
movdqa xmm4, XMMWORD [wk(10)] ; xmm4 = tmp1L
movdqa xmm2, XMMWORD [wk(11)] ; xmm2 = tmp1H
movdqa XMMWORD [wk(0)], xmm5 ; wk(0) = (00 10 01 11 02 12 03 13)
movdqa XMMWORD [wk(1)], xmm6 ; wk(1) = (04 14 05 15 06 16 07 17)
movdqa XMMWORD [wk(4)], xmm7 ; wk(4) = (60 70 61 71 62 72 63 73)
movdqa XMMWORD [wk(5)], xmm1 ; wk(5) = (64 74 65 75 66 76 67 77)
movdqa xmm5, xmm3
movdqa xmm6, xmm0
paddd xmm3, xmm4 ; xmm3 = data2L
paddd xmm0, xmm2 ; xmm0 = data2H
psubd xmm5, xmm4 ; xmm5 = data5L
psubd xmm6, xmm2 ; xmm6 = data5H
movdqa xmm7, [rel PD_DESCALE_P1] ; xmm7 = [rel PD_DESCALE_P1]
paddd xmm3, xmm7
paddd xmm0, xmm7
psrad xmm3, DESCALE_P1
psrad xmm0, DESCALE_P1
paddd xmm5, xmm7
paddd xmm6, xmm7
psrad xmm5, DESCALE_P1
psrad xmm6, DESCALE_P1
packssdw xmm3, xmm0 ; xmm3 = data2 = (20 21 22 23 24 25 26 27)
packssdw xmm5, xmm6 ; xmm5 = data5 = (50 51 52 53 54 55 56 57)
movdqa xmm1, XMMWORD [wk(2)] ; xmm1 = tmp13L
movdqa xmm4, XMMWORD [wk(3)] ; xmm4 = tmp13H
movdqa xmm2, XMMWORD [wk(8)] ; xmm2 = tmp0L
movdqa xmm7, XMMWORD [wk(9)] ; xmm7 = tmp0H
movdqa xmm0, xmm1
movdqa xmm6, xmm4
paddd xmm1, xmm2 ; xmm1 = data3L
paddd xmm4, xmm7 ; xmm4 = data3H
psubd xmm0, xmm2 ; xmm0 = data4L
psubd xmm6, xmm7 ; xmm6 = data4H
movdqa xmm2, [rel PD_DESCALE_P1] ; xmm2 = [rel PD_DESCALE_P1]
paddd xmm1, xmm2
paddd xmm4, xmm2
psrad xmm1, DESCALE_P1
psrad xmm4, DESCALE_P1
paddd xmm0, xmm2
paddd xmm6, xmm2
psrad xmm0, DESCALE_P1
psrad xmm6, DESCALE_P1
packssdw xmm1, xmm4 ; xmm1 = data3 = (30 31 32 33 34 35 36 37)
packssdw xmm0, xmm6 ; xmm0 = data4 = (40 41 42 43 44 45 46 47)
movdqa xmm7, XMMWORD [wk(0)] ; xmm7 = (00 10 01 11 02 12 03 13)
movdqa xmm2, XMMWORD [wk(1)] ; xmm2 = (04 14 05 15 06 16 07 17)
movdqa xmm4, xmm3 ; transpose coefficients(phase 1)
punpcklwd xmm3, xmm1 ; xmm3 = (20 30 21 31 22 32 23 33)
punpckhwd xmm4, xmm1 ; xmm4 = (24 34 25 35 26 36 27 37)
movdqa xmm6, xmm0 ; transpose coefficients(phase 1)
punpcklwd xmm0, xmm5 ; xmm0 = (40 50 41 51 42 52 43 53)
punpckhwd xmm6, xmm5 ; xmm6 = (44 54 45 55 46 56 47 57)
movdqa xmm1, xmm7 ; transpose coefficients(phase 2)
punpckldq xmm7, xmm3 ; xmm7 = (00 10 20 30 01 11 21 31)
punpckhdq xmm1, xmm3 ; xmm1 = (02 12 22 32 03 13 23 33)
movdqa xmm5, xmm2 ; transpose coefficients(phase 2)
punpckldq xmm2, xmm4 ; xmm2 = (04 14 24 34 05 15 25 35)
punpckhdq xmm5, xmm4 ; xmm5 = (06 16 26 36 07 17 27 37)
movdqa xmm3, XMMWORD [wk(4)] ; xmm3 = (60 70 61 71 62 72 63 73)
movdqa xmm4, XMMWORD [wk(5)] ; xmm4 = (64 74 65 75 66 76 67 77)
movdqa XMMWORD [wk(6)], xmm2 ; wk(6) = (04 14 24 34 05 15 25 35)
movdqa XMMWORD [wk(7)], xmm5 ; wk(7) = (06 16 26 36 07 17 27 37)
movdqa xmm2, xmm0 ; transpose coefficients(phase 2)
punpckldq xmm0, xmm3 ; xmm0 = (40 50 60 70 41 51 61 71)
punpckhdq xmm2, xmm3 ; xmm2 = (42 52 62 72 43 53 63 73)
movdqa xmm5, xmm6 ; transpose coefficients(phase 2)
punpckldq xmm6, xmm4 ; xmm6 = (44 54 64 74 45 55 65 75)
punpckhdq xmm5, xmm4 ; xmm5 = (46 56 66 76 47 57 67 77)
movdqa xmm3, xmm7 ; transpose coefficients(phase 3)
punpcklqdq xmm7, xmm0 ; xmm7 = col0 = (00 10 20 30 40 50 60 70)
punpckhqdq xmm3, xmm0 ; xmm3 = col1 = (01 11 21 31 41 51 61 71)
movdqa xmm4, xmm1 ; transpose coefficients(phase 3)
punpcklqdq xmm1, xmm2 ; xmm1 = col2 = (02 12 22 32 42 52 62 72)
punpckhqdq xmm4, xmm2 ; xmm4 = col3 = (03 13 23 33 43 53 63 73)
movdqa xmm0, XMMWORD [wk(6)] ; xmm0 = (04 14 24 34 05 15 25 35)
movdqa xmm2, XMMWORD [wk(7)] ; xmm2 = (06 16 26 36 07 17 27 37)
movdqa XMMWORD [wk(8)], xmm3 ; wk(8) = col1
movdqa XMMWORD [wk(9)], xmm4 ; wk(9) = col3
movdqa xmm3, xmm0 ; transpose coefficients(phase 3)
punpcklqdq xmm0, xmm6 ; xmm0 = col4 = (04 14 24 34 44 54 64 74)
punpckhqdq xmm3, xmm6 ; xmm3 = col5 = (05 15 25 35 45 55 65 75)
movdqa xmm4, xmm2 ; transpose coefficients(phase 3)
punpcklqdq xmm2, xmm5 ; xmm2 = col6 = (06 16 26 36 46 56 66 76)
punpckhqdq xmm4, xmm5 ; xmm4 = col7 = (07 17 27 37 47 57 67 77)
movdqa XMMWORD [wk(10)], xmm3 ; wk(10) = col5
movdqa XMMWORD [wk(11)], xmm4 ; wk(11) = col7
.column_end:
; -- Prefetch the next coefficient block
prefetchnta [rsi + DCTSIZE2 * SIZEOF_JCOEF + 0 * 32]
prefetchnta [rsi + DCTSIZE2 * SIZEOF_JCOEF + 1 * 32]
prefetchnta [rsi + DCTSIZE2 * SIZEOF_JCOEF + 2 * 32]
prefetchnta [rsi + DCTSIZE2 * SIZEOF_JCOEF + 3 * 32]
; ---- Pass 2: process rows from work array, store into output array.
mov rdi, r12 ; (JSAMPROW *)
mov eax, r13d
; -- Even part
; xmm7 = col0, xmm1 = col2, xmm0 = col4, xmm2 = col6
; (Original)
; z1 = (z2 + z3) * 0.541196100;
; tmp2 = z1 + z3 * -1.847759065;
; tmp3 = z1 + z2 * 0.765366865;
;
; (This implementation)
; tmp2 = z2 * 0.541196100 + z3 * (0.541196100 - 1.847759065);
; tmp3 = z2 * (0.541196100 + 0.765366865) + z3 * 0.541196100;
movdqa xmm6, xmm1 ; xmm1 = in2 = z2
movdqa xmm5, xmm1
punpcklwd xmm6, xmm2 ; xmm2 = in6 = z3
punpckhwd xmm5, xmm2
movdqa xmm1, xmm6
movdqa xmm2, xmm5
pmaddwd xmm6, [rel PW_F130_F054] ; xmm6 = tmp3L
pmaddwd xmm5, [rel PW_F130_F054] ; xmm5 = tmp3H
pmaddwd xmm1, [rel PW_F054_MF130] ; xmm1 = tmp2L
pmaddwd xmm2, [rel PW_F054_MF130] ; xmm2 = tmp2H
movdqa xmm3, xmm7
paddw xmm7, xmm0 ; xmm7 = in0 + in4
psubw xmm3, xmm0 ; xmm3 = in0 - in4
pxor xmm4, xmm4
pxor xmm0, xmm0
punpcklwd xmm4, xmm7 ; xmm4 = tmp0L
punpckhwd xmm0, xmm7 ; xmm0 = tmp0H
psrad xmm4, (16 - CONST_BITS)
; psrad xmm4, 16 & pslld xmm4, CONST_BITS
psrad xmm0, (16 - CONST_BITS)
; psrad xmm0, 16 & pslld xmm0, CONST_BITS
movdqa xmm7, xmm4
paddd xmm4, xmm6 ; xmm4 = tmp10L
psubd xmm7, xmm6 ; xmm7 = tmp13L
movdqa xmm6, xmm0
paddd xmm0, xmm5 ; xmm0 = tmp10H
psubd xmm6, xmm5 ; xmm6 = tmp13H
movdqa XMMWORD [wk(0)], xmm4 ; wk(0) = tmp10L
movdqa XMMWORD [wk(1)], xmm0 ; wk(1) = tmp10H
movdqa XMMWORD [wk(2)], xmm7 ; wk(2) = tmp13L
movdqa XMMWORD [wk(3)], xmm6 ; wk(3) = tmp13H
pxor xmm5, xmm5
pxor xmm4, xmm4
punpcklwd xmm5, xmm3 ; xmm5 = tmp1L
punpckhwd xmm4, xmm3 ; xmm4 = tmp1H
psrad xmm5, (16 - CONST_BITS)
; psrad xmm5, 16 & pslld xmm5, CONST_BITS
psrad xmm4, (16 - CONST_BITS)
; psrad xmm4, 16 & pslld xmm4, CONST_BITS
movdqa xmm0, xmm5
paddd xmm5, xmm1 ; xmm5 = tmp11L
psubd xmm0, xmm1 ; xmm0 = tmp12L
movdqa xmm7, xmm4
paddd xmm4, xmm2 ; xmm4 = tmp11H
psubd xmm7, xmm2 ; xmm7 = tmp12H
movdqa XMMWORD [wk(4)], xmm5 ; wk(4) = tmp11L
movdqa XMMWORD [wk(5)], xmm4 ; wk(5) = tmp11H
movdqa XMMWORD [wk(6)], xmm0 ; wk(6) = tmp12L
movdqa XMMWORD [wk(7)], xmm7 ; wk(7) = tmp12H
; -- Odd part
movdqa xmm6, XMMWORD [wk(9)] ; xmm6 = col3
movdqa xmm3, XMMWORD [wk(8)] ; xmm3 = col1
movdqa xmm1, XMMWORD [wk(11)] ; xmm1 = col7
movdqa xmm2, XMMWORD [wk(10)] ; xmm2 = col5
movdqa xmm5, xmm6
movdqa xmm4, xmm3
paddw xmm5, xmm1 ; xmm5 = z3
paddw xmm4, xmm2 ; xmm4 = z4
; (Original)
; z5 = (z3 + z4) * 1.175875602;
; z3 = z3 * -1.961570560; z4 = z4 * -0.390180644;
; z3 += z5; z4 += z5;
;
; (This implementation)
; z3 = z3 * (1.175875602 - 1.961570560) + z4 * 1.175875602;
; z4 = z3 * 1.175875602 + z4 * (1.175875602 - 0.390180644);
movdqa xmm0, xmm5
movdqa xmm7, xmm5
punpcklwd xmm0, xmm4
punpckhwd xmm7, xmm4
movdqa xmm5, xmm0
movdqa xmm4, xmm7
pmaddwd xmm0, [rel PW_MF078_F117] ; xmm0 = z3L
pmaddwd xmm7, [rel PW_MF078_F117] ; xmm7 = z3H
pmaddwd xmm5, [rel PW_F117_F078] ; xmm5 = z4L
pmaddwd xmm4, [rel PW_F117_F078] ; xmm4 = z4H
movdqa XMMWORD [wk(10)], xmm0 ; wk(10) = z3L
movdqa XMMWORD [wk(11)], xmm7 ; wk(11) = z3H
; (Original)
; z1 = tmp0 + tmp3; z2 = tmp1 + tmp2;
; tmp0 = tmp0 * 0.298631336; tmp1 = tmp1 * 2.053119869;
; tmp2 = tmp2 * 3.072711026; tmp3 = tmp3 * 1.501321110;
; z1 = z1 * -0.899976223; z2 = z2 * -2.562915447;
; tmp0 += z1 + z3; tmp1 += z2 + z4;
; tmp2 += z2 + z3; tmp3 += z1 + z4;
;
; (This implementation)
; tmp0 = tmp0 * (0.298631336 - 0.899976223) + tmp3 * -0.899976223;
; tmp1 = tmp1 * (2.053119869 - 2.562915447) + tmp2 * -2.562915447;
; tmp2 = tmp1 * -2.562915447 + tmp2 * (3.072711026 - 2.562915447);
; tmp3 = tmp0 * -0.899976223 + tmp3 * (1.501321110 - 0.899976223);
; tmp0 += z3; tmp1 += z4;
; tmp2 += z3; tmp3 += z4;
movdqa xmm0, xmm1
movdqa xmm7, xmm1
punpcklwd xmm0, xmm3
punpckhwd xmm7, xmm3
movdqa xmm1, xmm0
movdqa xmm3, xmm7
pmaddwd xmm0, [rel PW_MF060_MF089] ; xmm0 = tmp0L
pmaddwd xmm7, [rel PW_MF060_MF089] ; xmm7 = tmp0H
pmaddwd xmm1, [rel PW_MF089_F060] ; xmm1 = tmp3L
pmaddwd xmm3, [rel PW_MF089_F060] ; xmm3 = tmp3H
paddd xmm0, XMMWORD [wk(10)] ; xmm0 = tmp0L
paddd xmm7, XMMWORD [wk(11)] ; xmm7 = tmp0H
paddd xmm1, xmm5 ; xmm1 = tmp3L
paddd xmm3, xmm4 ; xmm3 = tmp3H
movdqa XMMWORD [wk(8)], xmm0 ; wk(8) = tmp0L
movdqa XMMWORD [wk(9)], xmm7 ; wk(9) = tmp0H
movdqa xmm0, xmm2
movdqa xmm7, xmm2
punpcklwd xmm0, xmm6
punpckhwd xmm7, xmm6
movdqa xmm2, xmm0
movdqa xmm6, xmm7
pmaddwd xmm0, [rel PW_MF050_MF256] ; xmm0 = tmp1L
pmaddwd xmm7, [rel PW_MF050_MF256] ; xmm7 = tmp1H
pmaddwd xmm2, [rel PW_MF256_F050] ; xmm2 = tmp2L
pmaddwd xmm6, [rel PW_MF256_F050] ; xmm6 = tmp2H
paddd xmm0, xmm5 ; xmm0 = tmp1L
paddd xmm7, xmm4 ; xmm7 = tmp1H
paddd xmm2, XMMWORD [wk(10)] ; xmm2 = tmp2L
paddd xmm6, XMMWORD [wk(11)] ; xmm6 = tmp2H
movdqa XMMWORD [wk(10)], xmm0 ; wk(10) = tmp1L
movdqa XMMWORD [wk(11)], xmm7 ; wk(11) = tmp1H
; -- Final output stage
movdqa xmm5, XMMWORD [wk(0)] ; xmm5 = tmp10L
movdqa xmm4, XMMWORD [wk(1)] ; xmm4 = tmp10H
movdqa xmm0, xmm5
movdqa xmm7, xmm4
paddd xmm5, xmm1 ; xmm5 = data0L
paddd xmm4, xmm3 ; xmm4 = data0H
psubd xmm0, xmm1 ; xmm0 = data7L
psubd xmm7, xmm3 ; xmm7 = data7H
movdqa xmm1, [rel PD_DESCALE_P2] ; xmm1 = [rel PD_DESCALE_P2]
paddd xmm5, xmm1
paddd xmm4, xmm1
psrad xmm5, DESCALE_P2
psrad xmm4, DESCALE_P2
paddd xmm0, xmm1
paddd xmm7, xmm1
psrad xmm0, DESCALE_P2
psrad xmm7, DESCALE_P2
packssdw xmm5, xmm4 ; xmm5 = data0 = (00 10 20 30 40 50 60 70)
packssdw xmm0, xmm7 ; xmm0 = data7 = (07 17 27 37 47 57 67 77)
movdqa xmm3, XMMWORD [wk(4)] ; xmm3 = tmp11L
movdqa xmm1, XMMWORD [wk(5)] ; xmm1 = tmp11H
movdqa xmm4, xmm3
movdqa xmm7, xmm1
paddd xmm3, xmm2 ; xmm3 = data1L
paddd xmm1, xmm6 ; xmm1 = data1H
psubd xmm4, xmm2 ; xmm4 = data6L
psubd xmm7, xmm6 ; xmm7 = data6H
movdqa xmm2, [rel PD_DESCALE_P2] ; xmm2 = [rel PD_DESCALE_P2]
paddd xmm3, xmm2
paddd xmm1, xmm2
psrad xmm3, DESCALE_P2
psrad xmm1, DESCALE_P2
paddd xmm4, xmm2
paddd xmm7, xmm2
psrad xmm4, DESCALE_P2
psrad xmm7, DESCALE_P2
packssdw xmm3, xmm1 ; xmm3 = data1 = (01 11 21 31 41 51 61 71)
packssdw xmm4, xmm7 ; xmm4 = data6 = (06 16 26 36 46 56 66 76)
packsswb xmm5, xmm4
; xmm5 = (00 10 20 30 40 50 60 70 06 16 26 36 46 56 66 76)
packsswb xmm3, xmm0
; xmm3 = (01 11 21 31 41 51 61 71 07 17 27 37 47 57 67 77)
movdqa xmm6, XMMWORD [wk(6)] ; xmm6 = tmp12L
movdqa xmm2, XMMWORD [wk(7)] ; xmm2 = tmp12H
movdqa xmm1, XMMWORD [wk(10)] ; xmm1 = tmp1L
movdqa xmm7, XMMWORD [wk(11)] ; xmm7 = tmp1H
movdqa XMMWORD [wk(0)], xmm5
; wk(0) = (00 10 20 30 40 50 60 70 06 16 26 36 46 56 66 76)
movdqa XMMWORD [wk(1)], xmm3
; wk(1) = (01 11 21 31 41 51 61 71 07 17 27 37 47 57 67 77)
movdqa xmm4, xmm6
movdqa xmm0, xmm2
paddd xmm6, xmm1 ; xmm6 = data2L
paddd xmm2, xmm7 ; xmm2 = data2H
psubd xmm4, xmm1 ; xmm4 = data5L
psubd xmm0, xmm7 ; xmm0 = data5H
movdqa xmm5, [rel PD_DESCALE_P2] ; xmm5 = [rel PD_DESCALE_P2]
paddd xmm6, xmm5
paddd xmm2, xmm5
psrad xmm6, DESCALE_P2
psrad xmm2, DESCALE_P2
paddd xmm4, xmm5
paddd xmm0, xmm5
psrad xmm4, DESCALE_P2
psrad xmm0, DESCALE_P2
packssdw xmm6, xmm2 ; xmm6 = data2 = (02 12 22 32 42 52 62 72)
packssdw xmm4, xmm0 ; xmm4 = data5 = (05 15 25 35 45 55 65 75)
movdqa xmm3, XMMWORD [wk(2)] ; xmm3 = tmp13L
movdqa xmm1, XMMWORD [wk(3)] ; xmm1 = tmp13H
movdqa xmm7, XMMWORD [wk(8)] ; xmm7 = tmp0L
movdqa xmm5, XMMWORD [wk(9)] ; xmm5 = tmp0H
movdqa xmm2, xmm3
movdqa xmm0, xmm1
paddd xmm3, xmm7 ; xmm3 = data3L
paddd xmm1, xmm5 ; xmm1 = data3H
psubd xmm2, xmm7 ; xmm2 = data4L
psubd xmm0, xmm5 ; xmm0 = data4H
movdqa xmm7, [rel PD_DESCALE_P2] ; xmm7 = [rel PD_DESCALE_P2]
paddd xmm3, xmm7
paddd xmm1, xmm7
psrad xmm3, DESCALE_P2
psrad xmm1, DESCALE_P2
paddd xmm2, xmm7
paddd xmm0, xmm7
psrad xmm2, DESCALE_P2
psrad xmm0, DESCALE_P2
movdqa xmm5, [rel PB_CENTERJSAMP] ; xmm5 = [rel PB_CENTERJSAMP]
packssdw xmm3, xmm1 ; xmm3 = data3 = (03 13 23 33 43 53 63 73)
packssdw xmm2, xmm0 ; xmm2 = data4 = (04 14 24 34 44 54 64 74)
movdqa xmm7, XMMWORD [wk(0)]
; xmm7 = (00 10 20 30 40 50 60 70 06 16 26 36 46 56 66 76)
movdqa xmm1, XMMWORD [wk(1)]
; xmm1 = (01 11 21 31 41 51 61 71 07 17 27 37 47 57 67 77)
packsswb xmm6, xmm2
; xmm6 = (02 12 22 32 42 52 62 72 04 14 24 34 44 54 64 74)
packsswb xmm3, xmm4
; xmm3 = (03 13 23 33 43 53 63 73 05 15 25 35 45 55 65 75)
paddb xmm7, xmm5
paddb xmm1, xmm5
paddb xmm6, xmm5
paddb xmm3, xmm5
; transpose coefficients(phase 1)
movdqa xmm0, xmm7
punpcklbw xmm7, xmm1
; xmm7 = (00 01 10 11 20 21 30 31 40 41 50 51 60 61 70 71)
punpckhbw xmm0, xmm1
; xmm0 = (06 07 16 17 26 27 36 37 46 47 56 57 66 67 76 77)
movdqa xmm2, xmm6
punpcklbw xmm6, xmm3
; xmm6 = (02 03 12 13 22 23 32 33 42 43 52 53 62 63 72 73)
punpckhbw xmm2, xmm3
; xmm2 = (04 05 14 15 24 25 34 35 44 45 54 55 64 65 74 75)
; transpose coefficients(phase 2)
movdqa xmm4, xmm7
punpcklwd xmm7, xmm6
; xmm7 = (00 01 02 03 10 11 12 13 20 21 22 23 30 31 32 33)
punpckhwd xmm4, xmm6
; xmm4 = (40 41 42 43 50 51 52 53 60 61 62 63 70 71 72 73)
movdqa xmm5, xmm2
punpcklwd xmm2, xmm0
; xmm2 = (04 05 06 07 14 15 16 17 24 25 26 27 34 35 36 37)
punpckhwd xmm5, xmm0
; xmm5 = (44 45 46 47 54 55 56 57 64 65 66 67 74 75 76 77)
; transpose coefficients(phase 3)
movdqa xmm1, xmm7
punpckldq xmm7, xmm2
; xmm7 = (00 01 02 03 04 05 06 07 10 11 12 13 14 15 16 17)
punpckhdq xmm1, xmm2
; xmm1 = (20 21 22 23 24 25 26 27 30 31 32 33 34 35 36 37)
movdqa xmm3, xmm4
punpckldq xmm4, xmm5
; xmm4 = (40 41 42 43 44 45 46 47 50 51 52 53 54 55 56 57)
punpckhdq xmm3, xmm5
; xmm3 = (60 61 62 63 64 65 66 67 70 71 72 73 74 75 76 77)
pshufd xmm6, xmm7, 0x4E
; xmm6 = (10 11 12 13 14 15 16 17 00 01 02 03 04 05 06 07)
pshufd xmm0, xmm1, 0x4E
; xmm0 = (30 31 32 33 34 35 36 37 20 21 22 23 24 25 26 27)
pshufd xmm2, xmm4, 0x4E
; xmm2 = (50 51 52 53 54 55 56 57 40 41 42 43 44 45 46 47)
pshufd xmm5, xmm3, 0x4E
; xmm5 = (70 71 72 73 74 75 76 77 60 61 62 63 64 65 66 67)
mov rdxp, JSAMPROW [rdi + 0 * SIZEOF_JSAMPROW]
mov rsip, JSAMPROW [rdi + 2 * SIZEOF_JSAMPROW]
movq XMM_MMWORD [rdx + rax * SIZEOF_JSAMPLE], xmm7
movq XMM_MMWORD [rsi + rax * SIZEOF_JSAMPLE], xmm1
mov rdxp, JSAMPROW [rdi + 4 * SIZEOF_JSAMPROW]
mov rsip, JSAMPROW [rdi + 6 * SIZEOF_JSAMPROW]
movq XMM_MMWORD [rdx + rax * SIZEOF_JSAMPLE], xmm4
movq XMM_MMWORD [rsi + rax * SIZEOF_JSAMPLE], xmm3
mov rdxp, JSAMPROW [rdi + 1 * SIZEOF_JSAMPROW]
mov rsip, JSAMPROW [rdi + 3 * SIZEOF_JSAMPROW]
movq XMM_MMWORD [rdx + rax * SIZEOF_JSAMPLE], xmm6
movq XMM_MMWORD [rsi + rax * SIZEOF_JSAMPLE], xmm0
mov rdxp, JSAMPROW [rdi + 5 * SIZEOF_JSAMPROW]
mov rsip, JSAMPROW [rdi + 7 * SIZEOF_JSAMPROW]
movq XMM_MMWORD [rdx + rax * SIZEOF_JSAMPLE], xmm2
movq XMM_MMWORD [rsi + rax * SIZEOF_JSAMPLE], xmm5
UNCOLLECT_ARGS 4
lea rsp, [rbp - 8]
pop r15
pop rbp
ret
; For some reason, the OS X linker does not honor the request to align the
; segment unless we do this.
align 32
lib/libjpeg-turbo/simd/x86_64/jidctred-sse2.asm
Executable
+577
View File
@@ -0,0 +1,577 @@
;
; Reduced-size IDCT (64-bit SSE2)
;
; Copyright 2009 Pierre Ossman <[email protected]> for Cendio AB
; Copyright (C) 2009, 2016, 2024-2025, D. R. Commander.
; Copyright (C) 2018, Matthias Räncker.
; Copyright (C) 2023, Aliaksiej Kandracienka.
;
; Based on the x86 SIMD extension for IJG JPEG library
; Copyright (C) 1999-2006, MIYASAKA Masaru.
; For conditions of distribution and use, see copyright notice in jsimdext.inc
;
; This file should be assembled with NASM (Netwide Assembler) or Yasm.
;
; This file contains inverse DCT routines that produce reduced-size output:
; either 4x4 or 2x2 pixels from an 8x8 DCT block. The following code is based
; directly on the IJG's original jidctred.c; see jidctred.c for more details.
%include "jsimdext.inc"
%include "jdct.inc"
; --------------------------------------------------------------------------
%define CONST_BITS 13
%define PASS1_BITS 2
%define DESCALE_P1_4 (CONST_BITS - PASS1_BITS + 1)
%define DESCALE_P2_4 (CONST_BITS + PASS1_BITS + 3 + 1)
%define DESCALE_P1_2 (CONST_BITS - PASS1_BITS + 2)
%define DESCALE_P2_2 (CONST_BITS + PASS1_BITS + 3 + 2)
%if CONST_BITS == 13
F_0_211 equ 1730 ; FIX(0.211164243)
F_0_509 equ 4176 ; FIX(0.509795579)
F_0_601 equ 4926 ; FIX(0.601344887)
F_0_720 equ 5906 ; FIX(0.720959822)
F_0_765 equ 6270 ; FIX(0.765366865)
F_0_850 equ 6967 ; FIX(0.850430095)
F_0_899 equ 7373 ; FIX(0.899976223)
F_1_061 equ 8697 ; FIX(1.061594337)
F_1_272 equ 10426 ; FIX(1.272758580)
F_1_451 equ 11893 ; FIX(1.451774981)
F_1_847 equ 15137 ; FIX(1.847759065)
F_2_172 equ 17799 ; FIX(2.172734803)
F_2_562 equ 20995 ; FIX(2.562915447)
F_3_624 equ 29692 ; FIX(3.624509785)
%else
; NASM cannot do compile-time arithmetic on floating-point constants.
%define DESCALE(x, n) (((x) + (1 << ((n) - 1))) >> (n))
F_0_211 equ DESCALE( 226735879, 30 - CONST_BITS) ; FIX(0.211164243)
F_0_509 equ DESCALE( 547388834, 30 - CONST_BITS) ; FIX(0.509795579)
F_0_601 equ DESCALE( 645689155, 30 - CONST_BITS) ; FIX(0.601344887)
F_0_720 equ DESCALE( 774124714, 30 - CONST_BITS) ; FIX(0.720959822)
F_0_765 equ DESCALE( 821806413, 30 - CONST_BITS) ; FIX(0.765366865)
F_0_850 equ DESCALE( 913142361, 30 - CONST_BITS) ; FIX(0.850430095)
F_0_899 equ DESCALE( 966342111, 30 - CONST_BITS) ; FIX(0.899976223)
F_1_061 equ DESCALE(1139878239, 30 - CONST_BITS) ; FIX(1.061594337)
F_1_272 equ DESCALE(1366614119, 30 - CONST_BITS) ; FIX(1.272758580)
F_1_451 equ DESCALE(1558831516, 30 - CONST_BITS) ; FIX(1.451774981)
F_1_847 equ DESCALE(1984016188, 30 - CONST_BITS) ; FIX(1.847759065)
F_2_172 equ DESCALE(2332956230, 30 - CONST_BITS) ; FIX(2.172734803)
F_2_562 equ DESCALE(2751909506, 30 - CONST_BITS) ; FIX(2.562915447)
F_3_624 equ DESCALE(3891787747, 30 - CONST_BITS) ; FIX(3.624509785)
%endif
; --------------------------------------------------------------------------
SECTION SEG_CONST
ALIGNZ 32
GLOBAL_DATA(jconst_idct_red_sse2)
EXTN(jconst_idct_red_sse2):
PW_F184_MF076 times 4 dw F_1_847, -F_0_765
PW_F256_F089 times 4 dw F_2_562, F_0_899
PW_F106_MF217 times 4 dw F_1_061, -F_2_172
PW_MF060_MF050 times 4 dw -F_0_601, -F_0_509
PW_F145_MF021 times 4 dw F_1_451, -F_0_211
PW_F362_MF127 times 4 dw F_3_624, -F_1_272
PW_F085_MF072 times 4 dw F_0_850, -F_0_720
PD_DESCALE_P1_4 times 4 dd 1 << (DESCALE_P1_4 - 1)
PD_DESCALE_P2_4 times 4 dd 1 << (DESCALE_P2_4 - 1)
PD_DESCALE_P1_2 times 4 dd 1 << (DESCALE_P1_2 - 1)
PD_DESCALE_P2_2 times 4 dd 1 << (DESCALE_P2_2 - 1)
PB_CENTERJSAMP times 16 db CENTERJSAMPLE
ALIGNZ 32
; --------------------------------------------------------------------------
SECTION SEG_TEXT
BITS 64
; Perform dequantization and inverse DCT on one block of coefficients,
; producing a reduced-size 4x4 output block.
;
; GLOBAL(void)
; jsimd_idct_4x4_sse2(void *dct_table, JCOEFPTR coef_block,
; JSAMPARRAY output_buf, JDIMENSION output_col)
;
; r10 = void *dct_table
; r11 = JCOEFPTR coef_block
; r12 = JSAMPARRAY output_buf
; r13d = JDIMENSION output_col
%define wk(i) r15 - (WK_NUM - (i)) * SIZEOF_XMMWORD ; xmmword wk[WK_NUM]
%define WK_NUM 2
align 32
GLOBAL_FUNCTION(jsimd_idct_4x4_sse2)
EXTN(jsimd_idct_4x4_sse2):
ENDBR64
push rbp
mov rbp, rsp
push r15
and rsp, byte (-SIZEOF_XMMWORD) ; align to 128 bits
; Allocate stack space for wk array. r15 is used to access it.
mov r15, rsp
sub rsp, byte (SIZEOF_XMMWORD * WK_NUM)
COLLECT_ARGS 4
; ---- Pass 1: process columns from input.
mov rdx, r10 ; quantptr
mov rsi, r11 ; inptr
%ifndef NO_ZERO_COLUMN_TEST_4X4_SSE2
mov eax, dword [DWBLOCK(1, 0, rsi, SIZEOF_JCOEF)]
or eax, dword [DWBLOCK(2, 0, rsi, SIZEOF_JCOEF)]
jnz short .columnDCT
movdqa xmm0, XMMWORD [XMMBLOCK(1, 0, rsi, SIZEOF_JCOEF)]
movdqa xmm1, XMMWORD [XMMBLOCK(2, 0, rsi, SIZEOF_JCOEF)]
por xmm0, XMMWORD [XMMBLOCK(3, 0, rsi, SIZEOF_JCOEF)]
por xmm1, XMMWORD [XMMBLOCK(5, 0, rsi, SIZEOF_JCOEF)]
por xmm0, XMMWORD [XMMBLOCK(6, 0, rsi, SIZEOF_JCOEF)]
por xmm1, XMMWORD [XMMBLOCK(7, 0, rsi, SIZEOF_JCOEF)]
por xmm0, xmm1
packsswb xmm0, xmm0
packsswb xmm0, xmm0
movd eax, xmm0
test rax, rax
jnz short .columnDCT
; -- AC terms all zero
movdqa xmm0, XMMWORD [XMMBLOCK(0, 0, rsi, SIZEOF_JCOEF)]
pmullw xmm0, XMMWORD [XMMBLOCK(0, 0, rdx, SIZEOF_ISLOW_MULT_TYPE)]
psllw xmm0, PASS1_BITS
movdqa xmm3, xmm0 ; xmm0 = in0 = (00 01 02 03 04 05 06 07)
punpcklwd xmm0, xmm0 ; xmm0 = (00 00 01 01 02 02 03 03)
punpckhwd xmm3, xmm3 ; xmm3 = (04 04 05 05 06 06 07 07)
pshufd xmm1, xmm0, 0x50
; xmm1 = [col0 col1] = (00 00 00 00 01 01 01 01)
pshufd xmm0, xmm0, 0xFA
; xmm0 = [col2 col3] = (02 02 02 02 03 03 03 03)
pshufd xmm6, xmm3, 0x50
; xmm6 = [col4 col5] = (04 04 04 04 05 05 05 05)
pshufd xmm3, xmm3, 0xFA
; xmm3 = [col6 col7] = (06 06 06 06 07 07 07 07)
jmp near .column_end
%endif
.columnDCT:
; -- Odd part
movdqa xmm0, XMMWORD [XMMBLOCK(1, 0, rsi, SIZEOF_JCOEF)]
movdqa xmm1, XMMWORD [XMMBLOCK(3, 0, rsi, SIZEOF_JCOEF)]
pmullw xmm0, XMMWORD [XMMBLOCK(1, 0, rdx, SIZEOF_ISLOW_MULT_TYPE)]
pmullw xmm1, XMMWORD [XMMBLOCK(3, 0, rdx, SIZEOF_ISLOW_MULT_TYPE)]
movdqa xmm2, XMMWORD [XMMBLOCK(5, 0, rsi, SIZEOF_JCOEF)]
movdqa xmm3, XMMWORD [XMMBLOCK(7, 0, rsi, SIZEOF_JCOEF)]
pmullw xmm2, XMMWORD [XMMBLOCK(5, 0, rdx, SIZEOF_ISLOW_MULT_TYPE)]
pmullw xmm3, XMMWORD [XMMBLOCK(7, 0, rdx, SIZEOF_ISLOW_MULT_TYPE)]
movdqa xmm4, xmm0
movdqa xmm5, xmm0
punpcklwd xmm4, xmm1
punpckhwd xmm5, xmm1
movdqa xmm0, xmm4
movdqa xmm1, xmm5
pmaddwd xmm4, [rel PW_F256_F089] ; xmm4 = (tmp2L)
pmaddwd xmm5, [rel PW_F256_F089] ; xmm5 = (tmp2H)
pmaddwd xmm0, [rel PW_F106_MF217] ; xmm0 = (tmp0L)
pmaddwd xmm1, [rel PW_F106_MF217] ; xmm1 = (tmp0H)
movdqa xmm6, xmm2
movdqa xmm7, xmm2
punpcklwd xmm6, xmm3
punpckhwd xmm7, xmm3
movdqa xmm2, xmm6
movdqa xmm3, xmm7
pmaddwd xmm6, [rel PW_MF060_MF050] ; xmm6 = (tmp2L)
pmaddwd xmm7, [rel PW_MF060_MF050] ; xmm7 = (tmp2H)
pmaddwd xmm2, [rel PW_F145_MF021] ; xmm2 = (tmp0L)
pmaddwd xmm3, [rel PW_F145_MF021] ; xmm3 = (tmp0H)
paddd xmm6, xmm4 ; xmm6 = tmp2L
paddd xmm7, xmm5 ; xmm7 = tmp2H
paddd xmm2, xmm0 ; xmm2 = tmp0L
paddd xmm3, xmm1 ; xmm3 = tmp0H
movdqa XMMWORD [wk(0)], xmm2 ; wk(0) = tmp0L
movdqa XMMWORD [wk(1)], xmm3 ; wk(1) = tmp0H
; -- Even part
movdqa xmm4, XMMWORD [XMMBLOCK(0, 0, rsi, SIZEOF_JCOEF)]
movdqa xmm5, XMMWORD [XMMBLOCK(2, 0, rsi, SIZEOF_JCOEF)]
movdqa xmm0, XMMWORD [XMMBLOCK(6, 0, rsi, SIZEOF_JCOEF)]
pmullw xmm4, XMMWORD [XMMBLOCK(0, 0, rdx, SIZEOF_ISLOW_MULT_TYPE)]
pmullw xmm5, XMMWORD [XMMBLOCK(2, 0, rdx, SIZEOF_ISLOW_MULT_TYPE)]
pmullw xmm0, XMMWORD [XMMBLOCK(6, 0, rdx, SIZEOF_ISLOW_MULT_TYPE)]
pxor xmm1, xmm1
pxor xmm2, xmm2
punpcklwd xmm1, xmm4 ; xmm1 = tmp0L
punpckhwd xmm2, xmm4 ; xmm2 = tmp0H
psrad xmm1, (16 - CONST_BITS - 1)
; psrad xmm1, 16 & pslld xmm1, CONST_BITS + 1
psrad xmm2, (16 - CONST_BITS - 1)
; psrad xmm2, 16 & pslld xmm2, CONST_BITS + 1
movdqa xmm3, xmm5 ; xmm5 = in2 = z2
punpcklwd xmm5, xmm0 ; xmm0 = in6 = z3
punpckhwd xmm3, xmm0
pmaddwd xmm5, [rel PW_F184_MF076] ; xmm5 = tmp2L
pmaddwd xmm3, [rel PW_F184_MF076] ; xmm3 = tmp2H
movdqa xmm4, xmm1
movdqa xmm0, xmm2
paddd xmm1, xmm5 ; xmm1 = tmp10L
paddd xmm2, xmm3 ; xmm2 = tmp10H
psubd xmm4, xmm5 ; xmm4 = tmp12L
psubd xmm0, xmm3 ; xmm0 = tmp12H
; -- Final output stage
movdqa xmm5, xmm1
movdqa xmm3, xmm2
paddd xmm1, xmm6 ; xmm1 = data0L
paddd xmm2, xmm7 ; xmm2 = data0H
psubd xmm5, xmm6 ; xmm5 = data3L
psubd xmm3, xmm7 ; xmm3 = data3H
movdqa xmm6, [rel PD_DESCALE_P1_4] ; xmm6 = [rel PD_DESCALE_P1_4]
paddd xmm1, xmm6
paddd xmm2, xmm6
psrad xmm1, DESCALE_P1_4
psrad xmm2, DESCALE_P1_4
paddd xmm5, xmm6
paddd xmm3, xmm6
psrad xmm5, DESCALE_P1_4
psrad xmm3, DESCALE_P1_4
packssdw xmm1, xmm2 ; xmm1 = data0 = (00 01 02 03 04 05 06 07)
packssdw xmm5, xmm3 ; xmm5 = data3 = (30 31 32 33 34 35 36 37)
movdqa xmm7, XMMWORD [wk(0)] ; xmm7 = tmp0L
movdqa xmm6, XMMWORD [wk(1)] ; xmm6 = tmp0H
movdqa xmm2, xmm4
movdqa xmm3, xmm0
paddd xmm4, xmm7 ; xmm4 = data1L
paddd xmm0, xmm6 ; xmm0 = data1H
psubd xmm2, xmm7 ; xmm2 = data2L
psubd xmm3, xmm6 ; xmm3 = data2H
movdqa xmm7, [rel PD_DESCALE_P1_4] ; xmm7 = [rel PD_DESCALE_P1_4]
paddd xmm4, xmm7
paddd xmm0, xmm7
psrad xmm4, DESCALE_P1_4
psrad xmm0, DESCALE_P1_4
paddd xmm2, xmm7
paddd xmm3, xmm7
psrad xmm2, DESCALE_P1_4
psrad xmm3, DESCALE_P1_4
packssdw xmm4, xmm0 ; xmm4 = data1 = (10 11 12 13 14 15 16 17)
packssdw xmm2, xmm3 ; xmm2 = data2 = (20 21 22 23 24 25 26 27)
movdqa xmm6, xmm1 ; transpose coefficients(phase 1)
punpcklwd xmm1, xmm4 ; xmm1 = (00 10 01 11 02 12 03 13)
punpckhwd xmm6, xmm4 ; xmm6 = (04 14 05 15 06 16 07 17)
movdqa xmm7, xmm2 ; transpose coefficients(phase 1)
punpcklwd xmm2, xmm5 ; xmm2 = (20 30 21 31 22 32 23 33)
punpckhwd xmm7, xmm5 ; xmm7 = (24 34 25 35 26 36 27 37)
movdqa xmm0, xmm1 ; transpose coefficients(phase 2)
punpckldq xmm1, xmm2 ; xmm1 = [col0 col1] = (00 10 20 30 01 11 21 31)
punpckhdq xmm0, xmm2 ; xmm0 = [col2 col3] = (02 12 22 32 03 13 23 33)
movdqa xmm3, xmm6 ; transpose coefficients(phase 2)
punpckldq xmm6, xmm7 ; xmm6 = [col4 col5] = (04 14 24 34 05 15 25 35)
punpckhdq xmm3, xmm7 ; xmm3 = [col6 col7] = (06 16 26 36 07 17 27 37)
.column_end:
; -- Prefetch the next coefficient block
prefetchnta [rsi + DCTSIZE2 * SIZEOF_JCOEF + 0 * 32]
prefetchnta [rsi + DCTSIZE2 * SIZEOF_JCOEF + 1 * 32]
prefetchnta [rsi + DCTSIZE2 * SIZEOF_JCOEF + 2 * 32]
prefetchnta [rsi + DCTSIZE2 * SIZEOF_JCOEF + 3 * 32]
; ---- Pass 2: process rows, store into output array.
mov rdi, r12 ; (JSAMPROW *)
mov eax, r13d
; -- Even part
pxor xmm4, xmm4
punpcklwd xmm4, xmm1 ; xmm4 = tmp0
psrad xmm4, (16 - CONST_BITS - 1)
; psrad xmm4, 16 & pslld xmm4, CONST_BITS + 1
; -- Odd part
punpckhwd xmm1, xmm0
punpckhwd xmm6, xmm3
movdqa xmm5, xmm1
movdqa xmm2, xmm6
pmaddwd xmm1, [rel PW_F256_F089] ; xmm1 = (tmp2)
pmaddwd xmm6, [rel PW_MF060_MF050] ; xmm6 = (tmp2)
pmaddwd xmm5, [rel PW_F106_MF217] ; xmm5 = (tmp0)
pmaddwd xmm2, [rel PW_F145_MF021] ; xmm2 = (tmp0)
paddd xmm6, xmm1 ; xmm6 = tmp2
paddd xmm2, xmm5 ; xmm2 = tmp0
; -- Even part
punpcklwd xmm0, xmm3
pmaddwd xmm0, [rel PW_F184_MF076] ; xmm0 = tmp2
movdqa xmm7, xmm4
paddd xmm4, xmm0 ; xmm4 = tmp10
psubd xmm7, xmm0 ; xmm7 = tmp12
; -- Final output stage
movdqa xmm1, [rel PD_DESCALE_P2_4] ; xmm1 = [rel PD_DESCALE_P2_4]
movdqa xmm5, xmm4
movdqa xmm3, xmm7
paddd xmm4, xmm6 ; xmm4 = data0 = (00 10 20 30)
paddd xmm7, xmm2 ; xmm7 = data1 = (01 11 21 31)
psubd xmm5, xmm6 ; xmm5 = data3 = (03 13 23 33)
psubd xmm3, xmm2 ; xmm3 = data2 = (02 12 22 32)
paddd xmm4, xmm1
paddd xmm7, xmm1
psrad xmm4, DESCALE_P2_4
psrad xmm7, DESCALE_P2_4
paddd xmm5, xmm1
paddd xmm3, xmm1
psrad xmm5, DESCALE_P2_4
psrad xmm3, DESCALE_P2_4
packssdw xmm4, xmm3 ; xmm4 = (00 10 20 30 02 12 22 32)
packssdw xmm7, xmm5 ; xmm7 = (01 11 21 31 03 13 23 33)
movdqa xmm0, xmm4 ; transpose coefficients(phase 1)
punpcklwd xmm4, xmm7 ; xmm4 = (00 01 10 11 20 21 30 31)
punpckhwd xmm0, xmm7 ; xmm0 = (02 03 12 13 22 23 32 33)
movdqa xmm6, xmm4 ; transpose coefficients(phase 2)
punpckldq xmm4, xmm0 ; xmm4 = (00 01 02 03 10 11 12 13)
punpckhdq xmm6, xmm0 ; xmm6 = (20 21 22 23 30 31 32 33)
packsswb xmm4, xmm6 ; xmm4 = (00 01 02 03 10 11 12 13 20 ..)
paddb xmm4, [rel PB_CENTERJSAMP]
pshufd xmm2, xmm4, 0x39 ; xmm2 = (10 11 12 13 20 21 22 23 30 ..)
pshufd xmm1, xmm4, 0x4E ; xmm1 = (20 21 22 23 30 31 32 33 00 ..)
pshufd xmm3, xmm4, 0x93 ; xmm3 = (30 31 32 33 00 01 02 03 10 ..)
mov rdxp, JSAMPROW [rdi + 0 * SIZEOF_JSAMPROW]
mov rsip, JSAMPROW [rdi + 1 * SIZEOF_JSAMPROW]
movd XMM_DWORD [rdx + rax * SIZEOF_JSAMPLE], xmm4
movd XMM_DWORD [rsi + rax * SIZEOF_JSAMPLE], xmm2
mov rdxp, JSAMPROW [rdi + 2 * SIZEOF_JSAMPROW]
mov rsip, JSAMPROW [rdi + 3 * SIZEOF_JSAMPROW]
movd XMM_DWORD [rdx + rax * SIZEOF_JSAMPLE], xmm1
movd XMM_DWORD [rsi + rax * SIZEOF_JSAMPLE], xmm3
UNCOLLECT_ARGS 4
lea rsp, [rbp - 8]
pop r15
pop rbp
ret
; --------------------------------------------------------------------------
;
; Perform dequantization and inverse DCT on one block of coefficients,
; producing a reduced-size 2x2 output block.
;
; GLOBAL(void)
; jsimd_idct_2x2_sse2(void *dct_table, JCOEFPTR coef_block,
; JSAMPARRAY output_buf, JDIMENSION output_col)
;
; r10 = void *dct_table
; r11 = JCOEFPTR coef_block
; r12 = JSAMPARRAY output_buf
; r13d = JDIMENSION output_col
align 32
GLOBAL_FUNCTION(jsimd_idct_2x2_sse2)
EXTN(jsimd_idct_2x2_sse2):
ENDBR64
push rbp
mov rbp, rsp
COLLECT_ARGS 4
push rbx
; ---- Pass 1: process columns from input.
mov rdx, r10 ; quantptr
mov rsi, r11 ; inptr
; | input: | result: |
; | 00 01 ** 03 ** 05 ** 07 | |
; | 10 11 ** 13 ** 15 ** 17 | |
; | ** ** ** ** ** ** ** ** | |
; | 30 31 ** 33 ** 35 ** 37 | A0 A1 A3 A5 A7 |
; | ** ** ** ** ** ** ** ** | B0 B1 B3 B5 B7 |
; | 50 51 ** 53 ** 55 ** 57 | |
; | ** ** ** ** ** ** ** ** | |
; | 70 71 ** 73 ** 75 ** 77 | |
; -- Odd part
movdqa xmm0, XMMWORD [XMMBLOCK(1, 0, rsi, SIZEOF_JCOEF)]
movdqa xmm1, XMMWORD [XMMBLOCK(3, 0, rsi, SIZEOF_JCOEF)]
pmullw xmm0, XMMWORD [XMMBLOCK(1, 0, rdx, SIZEOF_ISLOW_MULT_TYPE)]
pmullw xmm1, XMMWORD [XMMBLOCK(3, 0, rdx, SIZEOF_ISLOW_MULT_TYPE)]
movdqa xmm2, XMMWORD [XMMBLOCK(5, 0, rsi, SIZEOF_JCOEF)]
movdqa xmm3, XMMWORD [XMMBLOCK(7, 0, rsi, SIZEOF_JCOEF)]
pmullw xmm2, XMMWORD [XMMBLOCK(5, 0, rdx, SIZEOF_ISLOW_MULT_TYPE)]
pmullw xmm3, XMMWORD [XMMBLOCK(7, 0, rdx, SIZEOF_ISLOW_MULT_TYPE)]
; xmm0 = (10 11 ** 13 ** 15 ** 17)
; xmm1 = (30 31 ** 33 ** 35 ** 37)
; xmm2 = (50 51 ** 53 ** 55 ** 57)
; xmm3 = (70 71 ** 73 ** 75 ** 77)
pcmpeqd xmm7, xmm7
pslld xmm7, WORD_BIT ; xmm7 = { 0x0000 0xFFFF 0x0000 0xFFFF .. }
movdqa xmm4, xmm0 ; xmm4 = (10 11 ** 13 ** 15 ** 17)
movdqa xmm5, xmm2 ; xmm5 = (50 51 ** 53 ** 55 ** 57)
punpcklwd xmm4, xmm1 ; xmm4 = (10 30 11 31 ** ** 13 33)
punpcklwd xmm5, xmm3 ; xmm5 = (50 70 51 71 ** ** 53 73)
pmaddwd xmm4, [rel PW_F362_MF127]
pmaddwd xmm5, [rel PW_F085_MF072]
psrld xmm0, WORD_BIT ; xmm0 = (11 -- 13 -- 15 -- 17 --)
pand xmm1, xmm7 ; xmm1 = (-- 31 -- 33 -- 35 -- 37)
psrld xmm2, WORD_BIT ; xmm2 = (51 -- 53 -- 55 -- 57 --)
pand xmm3, xmm7 ; xmm3 = (-- 71 -- 73 -- 75 -- 77)
por xmm0, xmm1 ; xmm0 = (11 31 13 33 15 35 17 37)
por xmm2, xmm3 ; xmm2 = (51 71 53 73 55 75 57 77)
pmaddwd xmm0, [rel PW_F362_MF127]
pmaddwd xmm2, [rel PW_F085_MF072]
paddd xmm4, xmm5 ; xmm4 = tmp0[col0 col1 **** col3]
paddd xmm0, xmm2 ; xmm0 = tmp0[col1 col3 col5 col7]
; -- Even part
movdqa xmm6, XMMWORD [XMMBLOCK(0, 0, rsi, SIZEOF_JCOEF)]
pmullw xmm6, XMMWORD [XMMBLOCK(0, 0, rdx, SIZEOF_ISLOW_MULT_TYPE)]
; xmm6 = (00 01 ** 03 ** 05 ** 07)
movdqa xmm1, xmm6 ; xmm1 = (00 01 ** 03 ** 05 ** 07)
pslld xmm6, WORD_BIT ; xmm6 = (-- 00 -- ** -- ** -- **)
pand xmm1, xmm7 ; xmm1 = (-- 01 -- 03 -- 05 -- 07)
psrad xmm6, (WORD_BIT - CONST_BITS - 2)
; xmm6 = tmp10[col0 **** **** ****]
psrad xmm1, (WORD_BIT - CONST_BITS - 2)
; xmm1 = tmp10[col1 col3 col5 col7]
; -- Final output stage
movdqa xmm3, xmm6
movdqa xmm5, xmm1
paddd xmm6, xmm4 ; xmm6 = data0[col0 **** **** ****] = (A0 ** ** **)
paddd xmm1, xmm0 ; xmm1 = data0[col1 col3 col5 col7] = (A1 A3 A5 A7)
psubd xmm3, xmm4 ; xmm3 = data1[col0 **** **** ****] = (B0 ** ** **)
psubd xmm5, xmm0 ; xmm5 = data1[col1 col3 col5 col7] = (B1 B3 B5 B7)
movdqa xmm2, [rel PD_DESCALE_P1_2] ; xmm2 = [rel PD_DESCALE_P1_2]
punpckldq xmm6, xmm3 ; xmm6 = (A0 B0 ** **)
movdqa xmm7, xmm1
punpcklqdq xmm1, xmm5 ; xmm1 = (A1 A3 B1 B3)
punpckhqdq xmm7, xmm5 ; xmm7 = (A5 A7 B5 B7)
paddd xmm6, xmm2
psrad xmm6, DESCALE_P1_2
paddd xmm1, xmm2
paddd xmm7, xmm2
psrad xmm1, DESCALE_P1_2
psrad xmm7, DESCALE_P1_2
; -- Prefetch the next coefficient block
prefetchnta [rsi + DCTSIZE2 * SIZEOF_JCOEF + 0 * 32]
prefetchnta [rsi + DCTSIZE2 * SIZEOF_JCOEF + 1 * 32]
prefetchnta [rsi + DCTSIZE2 * SIZEOF_JCOEF + 2 * 32]
prefetchnta [rsi + DCTSIZE2 * SIZEOF_JCOEF + 3 * 32]
; ---- Pass 2: process rows, store into output array.
mov rdi, r12 ; (JSAMPROW *)
mov eax, r13d
; | input:| result:|
; | A0 B0 | |
; | A1 B1 | C0 C1 |
; | A3 B3 | D0 D1 |
; | A5 B5 | |
; | A7 B7 | |
; -- Odd part
packssdw xmm1, xmm1 ; xmm1 = (A1 A3 B1 B3 A1 A3 B1 B3)
packssdw xmm7, xmm7 ; xmm7 = (A5 A7 B5 B7 A5 A7 B5 B7)
pmaddwd xmm1, [rel PW_F362_MF127]
pmaddwd xmm7, [rel PW_F085_MF072]
paddd xmm1, xmm7 ; xmm1 = tmp0[row0 row1 row0 row1]
; -- Even part
pslld xmm6, (CONST_BITS + 2) ; xmm6 = tmp10[row0 row1 **** ****]
; -- Final output stage
movdqa xmm4, xmm6
paddd xmm6, xmm1 ; xmm6 = data0[row0 row1 **** ****] = (C0 C1 ** **)
psubd xmm4, xmm1 ; xmm4 = data1[row0 row1 **** ****] = (D0 D1 ** **)
punpckldq xmm6, xmm4 ; xmm6 = (C0 D0 C1 D1)
paddd xmm6, [rel PD_DESCALE_P2_2]
psrad xmm6, DESCALE_P2_2
packssdw xmm6, xmm6 ; xmm6 = (C0 D0 C1 D1 C0 D0 C1 D1)
packsswb xmm6, xmm6 ; xmm6 = (C0 D0 C1 D1 C0 D0 C1 D1 ..)
paddb xmm6, [rel PB_CENTERJSAMP]
pextrw ebx, xmm6, 0x00 ; ebx = (C0 D0 -- --)
pextrw ecx, xmm6, 0x01 ; ecx = (C1 D1 -- --)
mov rdxp, JSAMPROW [rdi + 0 * SIZEOF_JSAMPROW]
mov rsip, JSAMPROW [rdi + 1 * SIZEOF_JSAMPROW]
mov word [rdx + rax * SIZEOF_JSAMPLE], bx
mov word [rsi + rax * SIZEOF_JSAMPLE], cx
pop rbx
UNCOLLECT_ARGS 4
pop rbp
ret
; For some reason, the OS X linker does not honor the request to align the
; segment unless we do this.
align 32
lib/libjpeg-turbo/simd/x86_64/jquantf-sse2.asm
Executable
+149
View File
@@ -0,0 +1,149 @@
;
; Sample data conversion and quantization (64-bit SSE & SSE2)
;
; Copyright 2009 Pierre Ossman <[email protected]> for Cendio AB
; Copyright (C) 2009, 2016, 2024-2025, D. R. Commander.
; Copyright (C) 2018, Matthias Räncker.
;
; Based on the x86 SIMD extension for IJG JPEG library
; Copyright (C) 1999-2006, MIYASAKA Masaru.
; For conditions of distribution and use, see copyright notice in jsimdext.inc
;
; This file should be assembled with NASM (Netwide Assembler) or Yasm.
%include "jsimdext.inc"
%include "jdct.inc"
; --------------------------------------------------------------------------
SECTION SEG_TEXT
BITS 64
; Load data into workspace, applying unsigned->signed conversion
;
; GLOBAL(void)
; jsimd_convsamp_float_sse2(JSAMPARRAY sample_data, JDIMENSION start_col,
; FAST_FLOAT *workspace)
;
; r10 = JSAMPARRAY sample_data
; r11d = JDIMENSION start_col
; r12 = FAST_FLOAT *workspace
align 32
GLOBAL_FUNCTION(jsimd_convsamp_float_sse2)
EXTN(jsimd_convsamp_float_sse2):
ENDBR64
push rbp
mov rbp, rsp
COLLECT_ARGS 3
push rbx
pcmpeqw xmm7, xmm7
psllw xmm7, 7
packsswb xmm7, xmm7 ; xmm7 = PB_CENTERJSAMPLE (0x808080..)
mov rsi, r10
mov eax, r11d
mov rdi, r12
mov rcx, DCTSIZE / 2
.convloop:
mov rbxp, JSAMPROW [rsi + 0 * SIZEOF_JSAMPROW] ; (JSAMPLE *)
mov rdxp, JSAMPROW [rsi + 1 * SIZEOF_JSAMPROW] ; (JSAMPLE *)
movq xmm0, XMM_MMWORD [rbx + rax * SIZEOF_JSAMPLE]
movq xmm1, XMM_MMWORD [rdx + rax * SIZEOF_JSAMPLE]
psubb xmm0, xmm7 ; xmm0 = (01234567)
psubb xmm1, xmm7 ; xmm1 = (89abcdef)
punpcklbw xmm0, xmm0 ; xmm0 = (*0*1*2*3*4*5*6*7)
punpcklbw xmm1, xmm1 ; xmm1 = (*8*9*a*b*c*d*e*f)
punpcklwd xmm2, xmm0 ; xmm2 = (***0***1***2***3)
punpckhwd xmm0, xmm0 ; xmm0 = (***4***5***6***7)
punpcklwd xmm3, xmm1 ; xmm3 = (***8***9***a***b)
punpckhwd xmm1, xmm1 ; xmm1 = (***c***d***e***f)
psrad xmm2, (DWORD_BIT - BYTE_BIT) ; xmm2 = (0123)
psrad xmm0, (DWORD_BIT - BYTE_BIT) ; xmm0 = (4567)
cvtdq2ps xmm2, xmm2 ; xmm2 = (0123)
cvtdq2ps xmm0, xmm0 ; xmm0 = (4567)
psrad xmm3, (DWORD_BIT - BYTE_BIT) ; xmm3 = (89ab)
psrad xmm1, (DWORD_BIT - BYTE_BIT) ; xmm1 = (cdef)
cvtdq2ps xmm3, xmm3 ; xmm3 = (89ab)
cvtdq2ps xmm1, xmm1 ; xmm1 = (cdef)
movaps XMMWORD [XMMBLOCK(0, 0, rdi, SIZEOF_FAST_FLOAT)], xmm2
movaps XMMWORD [XMMBLOCK(0, 1, rdi, SIZEOF_FAST_FLOAT)], xmm0
movaps XMMWORD [XMMBLOCK(1, 0, rdi, SIZEOF_FAST_FLOAT)], xmm3
movaps XMMWORD [XMMBLOCK(1, 1, rdi, SIZEOF_FAST_FLOAT)], xmm1
add rsi, byte 2 * SIZEOF_JSAMPROW
add rdi, byte 2 * DCTSIZE * SIZEOF_FAST_FLOAT
dec rcx
jnz short .convloop
pop rbx
UNCOLLECT_ARGS 3
pop rbp
ret
; --------------------------------------------------------------------------
;
; Quantize/descale the coefficients, and store into coef_block
;
; GLOBAL(void)
; jsimd_quantize_float_sse2(JCOEFPTR coef_block, FAST_FLOAT *divisors,
; FAST_FLOAT *workspace)
;
; r10 = JCOEFPTR coef_block
; r11 = FAST_FLOAT *divisors
; r12 = FAST_FLOAT *workspace
align 32
GLOBAL_FUNCTION(jsimd_quantize_float_sse2)
EXTN(jsimd_quantize_float_sse2):
ENDBR64
push rbp
mov rbp, rsp
COLLECT_ARGS 3
mov rsi, r12
mov rdx, r11
mov rdi, r10
mov rax, DCTSIZE2 / 16
.quantloop:
movaps xmm0, XMMWORD [XMMBLOCK(0, 0, rsi, SIZEOF_FAST_FLOAT)]
movaps xmm1, XMMWORD [XMMBLOCK(0, 1, rsi, SIZEOF_FAST_FLOAT)]
mulps xmm0, XMMWORD [XMMBLOCK(0, 0, rdx, SIZEOF_FAST_FLOAT)]
mulps xmm1, XMMWORD [XMMBLOCK(0, 1, rdx, SIZEOF_FAST_FLOAT)]
movaps xmm2, XMMWORD [XMMBLOCK(1, 0, rsi, SIZEOF_FAST_FLOAT)]
movaps xmm3, XMMWORD [XMMBLOCK(1, 1, rsi, SIZEOF_FAST_FLOAT)]
mulps xmm2, XMMWORD [XMMBLOCK(1, 0, rdx, SIZEOF_FAST_FLOAT)]
mulps xmm3, XMMWORD [XMMBLOCK(1, 1, rdx, SIZEOF_FAST_FLOAT)]
cvtps2dq xmm0, xmm0
cvtps2dq xmm1, xmm1
cvtps2dq xmm2, xmm2
cvtps2dq xmm3, xmm3
packssdw xmm0, xmm1
packssdw xmm2, xmm3
movdqa XMMWORD [XMMBLOCK(0, 0, rdi, SIZEOF_JCOEF)], xmm0
movdqa XMMWORD [XMMBLOCK(1, 0, rdi, SIZEOF_JCOEF)], xmm2
add rsi, byte 16 * SIZEOF_FAST_FLOAT
add rdx, byte 16 * SIZEOF_FAST_FLOAT
add rdi, byte 16 * SIZEOF_JCOEF
dec rax
jnz short .quantloop
UNCOLLECT_ARGS 3
pop rbp
ret
; For some reason, the OS X linker does not honor the request to align the
; segment unless we do this.
align 32
lib/libjpeg-turbo/simd/x86_64/jquanti-sse2.asm
Executable
+187
View File
@@ -0,0 +1,187 @@
;
; Sample data conversion and quantization (64-bit SSE2)
;
; Copyright 2009 Pierre Ossman <[email protected]> for Cendio AB
; Copyright (C) 2009, 2016, 2024-2025, D. R. Commander.
; Copyright (C) 2018, Matthias Räncker.
;
; Based on the x86 SIMD extension for IJG JPEG library
; Copyright (C) 1999-2006, MIYASAKA Masaru.
; For conditions of distribution and use, see copyright notice in jsimdext.inc
;
; This file should be assembled with NASM (Netwide Assembler) or Yasm.
%include "jsimdext.inc"
%include "jdct.inc"
; --------------------------------------------------------------------------
SECTION SEG_TEXT
BITS 64
; Load data into workspace, applying unsigned->signed conversion
;
; GLOBAL(void)
; jsimd_convsamp_sse2(JSAMPARRAY sample_data, JDIMENSION start_col,
; DCTELEM *workspace)
;
; r10 = JSAMPARRAY sample_data
; r11d = JDIMENSION start_col
; r12 = DCTELEM *workspace
align 32
GLOBAL_FUNCTION(jsimd_convsamp_sse2)
EXTN(jsimd_convsamp_sse2):
ENDBR64
push rbp
mov rbp, rsp
COLLECT_ARGS 3
push rbx
pxor xmm6, xmm6 ; xmm6 = (all 0's)
pcmpeqw xmm7, xmm7
psllw xmm7, 7 ; xmm7 = { 0xFF80 0xFF80 0xFF80 0xFF80 .. }
mov rsi, r10
mov eax, r11d
mov rdi, r12
mov rcx, DCTSIZE / 4
.convloop:
mov rbxp, JSAMPROW [rsi + 0 * SIZEOF_JSAMPROW] ; (JSAMPLE *)
mov rdxp, JSAMPROW [rsi + 1 * SIZEOF_JSAMPROW] ; (JSAMPLE *)
movq xmm0, XMM_MMWORD [rbx + rax * SIZEOF_JSAMPLE]
; xmm0 = (01234567)
movq xmm1, XMM_MMWORD [rdx + rax * SIZEOF_JSAMPLE]
; xmm1 = (89abcdef)
mov rbxp, JSAMPROW [rsi + 2 * SIZEOF_JSAMPROW] ; (JSAMPLE *)
mov rdxp, JSAMPROW [rsi + 3 * SIZEOF_JSAMPROW] ; (JSAMPLE *)
movq xmm2, XMM_MMWORD [rbx + rax * SIZEOF_JSAMPLE]
; xmm2 = (ghijklmn)
movq xmm3, XMM_MMWORD [rdx + rax * SIZEOF_JSAMPLE]
; xmm3 = (opqrstuv)
punpcklbw xmm0, xmm6 ; xmm0 = (01234567)
punpcklbw xmm1, xmm6 ; xmm1 = (89abcdef)
paddw xmm0, xmm7
paddw xmm1, xmm7
punpcklbw xmm2, xmm6 ; xmm2 = (ghijklmn)
punpcklbw xmm3, xmm6 ; xmm3 = (opqrstuv)
paddw xmm2, xmm7
paddw xmm3, xmm7
movdqa XMMWORD [XMMBLOCK(0, 0, rdi, SIZEOF_DCTELEM)], xmm0
movdqa XMMWORD [XMMBLOCK(1, 0, rdi, SIZEOF_DCTELEM)], xmm1
movdqa XMMWORD [XMMBLOCK(2, 0, rdi, SIZEOF_DCTELEM)], xmm2
movdqa XMMWORD [XMMBLOCK(3, 0, rdi, SIZEOF_DCTELEM)], xmm3
add rsi, byte 4 * SIZEOF_JSAMPROW
add rdi, byte 4 * DCTSIZE * SIZEOF_DCTELEM
dec rcx
jnz short .convloop
pop rbx
UNCOLLECT_ARGS 3
pop rbp
ret
; --------------------------------------------------------------------------
;
; Quantize/descale the coefficients, and store into coef_block
;
; This implementation is based on an algorithm described in
; "Optimizing subroutines in assembly language:
; An optimization guide for x86 platforms" (https://agner.org/optimize).
;
; GLOBAL(void)
; jsimd_quantize_sse2(JCOEFPTR coef_block, DCTELEM *divisors,
; DCTELEM *workspace)
%define RECIPROCAL(m, n, b) \
XMMBLOCK(DCTSIZE * 0 + (m), (n), (b), SIZEOF_DCTELEM)
%define CORRECTION(m, n, b) \
XMMBLOCK(DCTSIZE * 1 + (m), (n), (b), SIZEOF_DCTELEM)
%define SCALE(m, n, b) \
XMMBLOCK(DCTSIZE * 2 + (m), (n), (b), SIZEOF_DCTELEM)
; r10 = JCOEFPTR coef_block
; r11 = DCTELEM *divisors
; r12 = DCTELEM *workspace
align 32
GLOBAL_FUNCTION(jsimd_quantize_sse2)
EXTN(jsimd_quantize_sse2):
ENDBR64
push rbp
mov rbp, rsp
COLLECT_ARGS 3
mov rsi, r12
mov rdx, r11
mov rdi, r10
mov rax, DCTSIZE2 / 32
.quantloop:
movdqa xmm4, XMMWORD [XMMBLOCK(0, 0, rsi, SIZEOF_DCTELEM)]
movdqa xmm5, XMMWORD [XMMBLOCK(1, 0, rsi, SIZEOF_DCTELEM)]
movdqa xmm6, XMMWORD [XMMBLOCK(2, 0, rsi, SIZEOF_DCTELEM)]
movdqa xmm7, XMMWORD [XMMBLOCK(3, 0, rsi, SIZEOF_DCTELEM)]
movdqa xmm0, xmm4
movdqa xmm1, xmm5
movdqa xmm2, xmm6
movdqa xmm3, xmm7
psraw xmm4, (WORD_BIT - 1)
psraw xmm5, (WORD_BIT - 1)
psraw xmm6, (WORD_BIT - 1)
psraw xmm7, (WORD_BIT - 1)
pxor xmm0, xmm4
pxor xmm1, xmm5
pxor xmm2, xmm6
pxor xmm3, xmm7
psubw xmm0, xmm4 ; if (xmm0 < 0) xmm0 = -xmm0;
psubw xmm1, xmm5 ; if (xmm1 < 0) xmm1 = -xmm1;
psubw xmm2, xmm6 ; if (xmm2 < 0) xmm2 = -xmm2;
psubw xmm3, xmm7 ; if (xmm3 < 0) xmm3 = -xmm3;
paddw xmm0, XMMWORD [CORRECTION(0, 0, rdx)]
; correction + roundfactor
paddw xmm1, XMMWORD [CORRECTION(1, 0, rdx)]
paddw xmm2, XMMWORD [CORRECTION(2, 0, rdx)]
paddw xmm3, XMMWORD [CORRECTION(3, 0, rdx)]
pmulhuw xmm0, XMMWORD [RECIPROCAL(0, 0, rdx)] ; reciprocal
pmulhuw xmm1, XMMWORD [RECIPROCAL(1, 0, rdx)]
pmulhuw xmm2, XMMWORD [RECIPROCAL(2, 0, rdx)]
pmulhuw xmm3, XMMWORD [RECIPROCAL(3, 0, rdx)]
pmulhuw xmm0, XMMWORD [SCALE(0, 0, rdx)] ; scale
pmulhuw xmm1, XMMWORD [SCALE(1, 0, rdx)]
pmulhuw xmm2, XMMWORD [SCALE(2, 0, rdx)]
pmulhuw xmm3, XMMWORD [SCALE(3, 0, rdx)]
pxor xmm0, xmm4
pxor xmm1, xmm5
pxor xmm2, xmm6
pxor xmm3, xmm7
psubw xmm0, xmm4
psubw xmm1, xmm5
psubw xmm2, xmm6
psubw xmm3, xmm7
movdqa XMMWORD [XMMBLOCK(0, 0, rdi, SIZEOF_DCTELEM)], xmm0
movdqa XMMWORD [XMMBLOCK(1, 0, rdi, SIZEOF_DCTELEM)], xmm1
movdqa XMMWORD [XMMBLOCK(2, 0, rdi, SIZEOF_DCTELEM)], xmm2
movdqa XMMWORD [XMMBLOCK(3, 0, rdi, SIZEOF_DCTELEM)], xmm3
add rsi, byte 32 * SIZEOF_DCTELEM
add rdx, byte 32 * SIZEOF_DCTELEM
add rdi, byte 32 * SIZEOF_JCOEF
dec rax
jnz near .quantloop
UNCOLLECT_ARGS 3
pop rbp
ret
; For some reason, the OS X linker does not honor the request to align the
; segment unless we do this.
align 32
lib/libjpeg-turbo/simd/x86_64/jsimd.c
Executable
+1108
View File
File diff suppressed because it is too large Load Diff