;
; yuvutils_neon.s
;

 GET hdr.ka_drawer

; neon yuv application of brightness, contrast and colour corrections.


  AREA |A$$code|, CODE, READONLY

; ka_yuv_correction_neon
; ----------------------
; r0 = uint8_t* dst
; r1 = uint8_t* src
; r2 = int size
; r3 = int16_t fac [0, 128] default 64
; on stack int16_t add
;
; assumes size is at least 16 bytes
; on exit dst[x] = fix(src[x])
;
  EXPORT ka_yuv_correction_neon
  ALIGN 32
ka_yuv_correction_neon

  vdup.16      q0, r3 ; fac
  ldr          r3, [sp, #0]
  vdup.16      q1, r3 ; add

  b            ka_y_next

ka_y_loop
  ; at leat 64 bytes
  vld1.8       {q2},[r1]!
  vld1.8       {q4},[r1]!
  vld1.8       {q6},[r1]!
  vld1.8       {q8},[r1]!

  vmovl.u8     q3, d5
  vmovl.u8     q2, d4
  vmul.u16     q3, q0
  vmul.u16     q2, q0
  vadd.s16     q2, q1
  vadd.s16     q3, q1
  vqrshrun.s16 d4, q2, #6
  vqrshrun.s16 d5, q3, #6
  vst1.8       {q2},[r0]!

  vmovl.u8     q5, d9
  vmovl.u8     q4, d8
  vmul.u16     q5, q0
  vmul.u16     q4, q0
  vadd.s16     q4, q1
  vadd.s16     q5, q1
  vqrshrun.s16 d8, q4, #6
  vqrshrun.s16 d9, q5, #6
  vst1.8       {q4},[r0]!

  vmovl.u8     q7, d13
  vmovl.u8     q6, d12
  vmul.u16     q7, q0
  vmul.u16     q6, q0
  vadd.s16     q6, q1
  vadd.s16     q7, q1
  vqrshrun.s16 d12, q6, #6
  vqrshrun.s16 d13, q7, #6
  vst1.8       {q6},[r0]!

  vmovl.u8     q9, d17
  vmovl.u8     q8, d16
  vmul.u16     q9, q0
  vmul.u16     q8, q0
  vadd.s16     q8, q1
  vadd.s16     q9, q1
  vqrshrun.s16 d16, q8, #6
  vqrshrun.s16 d17, q9, #6
  vst1.8       {q8},[r0]!

ka_y_next
  subs         r2, r2, #64
  bge          ka_y_loop

  add          r2, r2, #64
ka_y_32
  cmp          r2, #32
  blt          ka_y_16

  ; at leat 32 bytes
  vld1.8       {q2},[r1]!
  vld1.8       {q4},[r1]!

  vmovl.u8     q3, d5
  vmovl.u8     q2, d4
  vmul.u16     q3, q0
  vmul.u16     q2, q0
  vadd.s16     q2, q1
  vadd.s16     q3, q1
  vqrshrun.s16 d4, q2, #6
  vqrshrun.s16 d5, q3, #6
  vst1.8       {q2},[r0]!

  vmovl.u8     q5, d9
  vmovl.u8     q4, d8
  vmul.u16     q5, q0
  vmul.u16     q4, q0
  vadd.s16     q4, q1
  vadd.s16     q5, q1
  vqrshrun.s16 d8, q4, #6
  vqrshrun.s16 d9, q5, #6
  vst1.8       {q4},[r0]!

  sub          r2, r2, #32

ka_y_16
  cmp          r2, #16
  blt          ka_y_15

  ; at leat 16 bytes
  vld1.8       {q2},[r1]!

  vmovl.u8     q3, d5
  vmovl.u8     q2, d4
  vmul.u16     q3, q0
  vmul.u16     q2, q0
  vadd.s16     q2, q1
  vadd.s16     q3, q1
  vqrshrun.s16 d4, q2, #6
  vqrshrun.s16 d5, q3, #6
  vst1.8       {q2},[r0]!

  sub          r2, r2, #16

ka_y_15
  ; less than 16 bytes
  cmp          r2, #0
  ble          ka_y_0
  ; move back by 16 - remaining bytes
  rsb          r2, r2, #16
  sub          r0, r0, r2
  sub          r1, r1, r2
  ; at leat 16 bytes
  vld1.8       {q2},[r1]!

  vmovl.u8     q3, d5
  vmovl.u8     q2, d4
  vmul.u16     q3, q0
  vmul.u16     q2, q0
  vadd.s16     q2, q1
  vadd.s16     q3, q1
  vqrshrun.s16 d4, q2, #6
  vqrshrun.s16 d5, q3, #6
  vst1.8       {q2},[r0]!

ka_y_0
  mov          pc, r14

; ka_yuv_correcthalf_neon
; -----------------------
; r0 = uint8_t* dst
; r1 = uint8_t* src
; r2 = int size
; r3 = int16_t fac [0, 128] default 64
; on stack int16_t add
;
; assumes size is at least 16 bytes
; on exit dst[x] = fix((src[2x] + src[2x+1] +1)/2)
;
  EXPORT ka_yuv_correcthalf_neon
  ALIGN 32
ka_yuv_correcthalf_neon

  vdup.16      q0, r3 ; fac
  ldr          r3, [sp, #0]
  vdup.16      q1, r3 ; add

  b            ka_yh_next

ka_yh_loop
  ; at leat 64 bytes
  vld1.8       {q2},[r1]!
  vld1.8       {q3},[r1]!
  vld1.8       {q4},[r1]!
  vld1.8       {q5},[r1]!
  vld1.8       {q6},[r1]!
  vld1.8       {q7},[r1]!
  vld1.8       {q8},[r1]!
  vld1.8       {q9},[r1]!
  vuzp.u8      d4, d5
  vuzp.u8      d6, d7
  vuzp.u8      d8, d9
  vuzp.u8      d10, d11
  vuzp.u8      d12, d13
  vuzp.u8      d14, d15
  vuzp.u8      d16, d17
  vuzp.u8      d18, d19
  vrhadd.u8    d4, d5
  vrhadd.u8    d6, d7
  vrhadd.u8    d8, d9
  vrhadd.u8    d10, d11
  vrhadd.u8    d12, d13
  vrhadd.u8    d14, d15
  vrhadd.u8    d16, d17
  vrhadd.u8    d18, d19

  vmovl.u8     q3, d6
  vmovl.u8     q2, d4
  vmul.u16     q3, q0
  vmul.u16     q2, q0
  vadd.s16     q2, q1
  vadd.s16     q3, q1
  vqrshrun.s16 d4, q2, #6
  vqrshrun.s16 d5, q3, #6
  vst1.8       {q2},[r0]!

  vmovl.u8     q5, d10
  vmovl.u8     q4, d8
  vmul.u16     q5, q0
  vmul.u16     q4, q0
  vadd.s16     q4, q1
  vadd.s16     q5, q1
  vqrshrun.s16 d8, q4, #6
  vqrshrun.s16 d9, q5, #6
  vst1.8       {q4},[r0]!

  vmovl.u8     q7, d14
  vmovl.u8     q6, d12
  vmul.u16     q7, q0
  vmul.u16     q6, q0
  vadd.s16     q6, q1
  vadd.s16     q7, q1
  vqrshrun.s16 d12, q6, #6
  vqrshrun.s16 d13, q7, #6
  vst1.8       {q6},[r0]!

  vmovl.u8     q9, d18
  vmovl.u8     q8, d16
  vmul.u16     q9, q0
  vmul.u16     q8, q0
  vadd.s16     q8, q1
  vadd.s16     q9, q1
  vqrshrun.s16 d16, q8, #6
  vqrshrun.s16 d17, q9, #6
  vst1.8       {q8},[r0]!

ka_yh_next
  subs         r2, r2, #64
  bge          ka_yh_loop

  add          r2, r2, #64
ka_yh_32
  cmp          r2, #32
  blt          ka_yh_16

  ; at leat 32 bytes
  vld1.8       {q2},[r1]!
  vld1.8       {q3},[r1]!
  vld1.8       {q4},[r1]!
  vld1.8       {q5},[r1]!
  vuzp.u8      d4, d5
  vuzp.u8      d6, d7
  vuzp.u8      d8, d9
  vuzp.u8      d10, d11
  vrhadd.u8    d4, d5
  vrhadd.u8    d6, d7
  vrhadd.u8    d8, d9
  vrhadd.u8    d10, d11

  vmovl.u8     q3, d6
  vmovl.u8     q2, d4
  vmul.u16     q3, q0
  vmul.u16     q2, q0
  vadd.s16     q2, q1
  vadd.s16     q3, q1
  vqrshrun.s16 d4, q2, #6
  vqrshrun.s16 d5, q3, #6
  vst1.8       {q2},[r0]!

  vmovl.u8     q5, d10
  vmovl.u8     q4, d8
  vmul.u16     q5, q0
  vmul.u16     q4, q0
  vadd.s16     q4, q1
  vadd.s16     q5, q1
  vqrshrun.s16 d8, q4, #6
  vqrshrun.s16 d9, q5, #6
  vst1.8       {q4},[r0]!

  sub          r2, r2, #32

ka_yh_16
  cmp          r2, #16
  blt          ka_yh_15

  ; at leat 16 bytes
  vld1.8       {q2},[r1]!
  vld1.8       {q3},[r1]!
  vuzp.u8      d4, d5
  vuzp.u8      d6, d7
  vrhadd.u8    d4, d5
  vrhadd.u8    d6, d7

  vmovl.u8     q3, d6
  vmovl.u8     q2, d4
  vmul.u16     q3, q0
  vmul.u16     q2, q0
  vadd.s16     q2, q1
  vadd.s16     q3, q1
  vqrshrun.s16 d4, q2, #6
  vqrshrun.s16 d5, q3, #6
  vst1.8       {q2},[r0]!

  sub          r2, r2, #16

ka_yh_15
  ; less than 16 bytes
  cmp          r2, #0
  ble          ka_yh_0
  ; move back by 16 - remaining bytes
  rsb          r2, r2, #16
  sub          r0, r0, r2
  sub          r1, r1, r2, lsl #1
  ; at leat 16 bytes
  vld1.8       {q2},[r1]!
  vld1.8       {q3},[r1]!
  vuzp.u8      d4, d5
  vuzp.u8      d6, d7
  vrhadd.u8    d4, d5
  vrhadd.u8    d6, d7

  vmovl.u8     q3, d6
  vmovl.u8     q2, d4
  vmul.u16     q3, q0
  vmul.u16     q2, q0
  vadd.s16     q2, q1
  vadd.s16     q3, q1
  vqrshrun.s16 d4, q2, #6
  vqrshrun.s16 d5, q3, #6
  vst1.8       {q2},[r0]!

ka_yh_0
  mov          pc, r14

; ka_uv_correction_neon
; ---------------------
; r0 = uint16_t* dst (u + v)
; r1 = uint8_t* src1
; r2 = uint8_t* src2
; r3 = int size (in halwords)
; on stack int16_t fac [0, 128] default 64
; on stack int16_t add
;
; assumes size is at least 16 bytes (8 halwords)
; on exit dst[x].a = fix(src1[x])
; on exit dst[x].b = fix(src2[x])
;
  EXPORT ka_uv_correction_neon
  ALIGN 32
ka_uv_correction_neon

  vmov.32      d4[0], r3 ; save r3
  ldr          r3, [sp, #0]
  vdup.16      q0, r3 ; fac
  ldr          r3, [sp, #4]
  vdup.16      q1, r3 ; add
  vmov.32      r3, d4[0] ; restore r3
  mov          r3, r3, lsl #1 ; halfword to bytes

  b            ka_uv_next

ka_uv_loop
  ; at leat 64 bytes
  vld1.8       {d4},[r1]!
  vld1.8       {d5},[r2]!
  vld1.8       {d8},[r1]!
  vld1.8       {d9},[r2]!
  vld1.8       {d12},[r1]!
  vld1.8       {d13},[r2]!
  vld1.8       {d16},[r1]!
  vld1.8       {d17},[r2]!
  vzip.u8      d4, d5
  vzip.u8      d8, d9
  vzip.u8      d12, d13
  vzip.u8      d16, d17

  vmovl.u8     q3, d5
  vmovl.u8     q2, d4
  vmul.u16     q3, q0
  vmul.u16     q2, q0
  vadd.s16     q2, q1
  vadd.s16     q3, q1
  vqrshrun.s16 d4, q2, #6
  vqrshrun.s16 d5, q3, #6
  vst1.8       {q2},[r0]!

  vmovl.u8     q5, d9
  vmovl.u8     q4, d8
  vmul.u16     q5, q0
  vmul.u16     q4, q0
  vadd.s16     q4, q1
  vadd.s16     q5, q1
  vqrshrun.s16 d8, q4, #6
  vqrshrun.s16 d9, q5, #6
  vst1.8       {q4},[r0]!

  vmovl.u8     q7, d13
  vmovl.u8     q6, d12
  vmul.u16     q7, q0
  vmul.u16     q6, q0
  vadd.s16     q6, q1
  vadd.s16     q7, q1
  vqrshrun.s16 d12, q6, #6
  vqrshrun.s16 d13, q7, #6
  vst1.8       {q6},[r0]!

  vmovl.u8     q9, d17
  vmovl.u8     q8, d16
  vmul.u16     q9, q0
  vmul.u16     q8, q0
  vadd.s16     q8, q1
  vadd.s16     q9, q1
  vqrshrun.s16 d16, q8, #6
  vqrshrun.s16 d17, q9, #6
  vst1.8       {q8},[r0]!

ka_uv_next
  subs         r3, r3, #64
  bge          ka_uv_loop

  add          r3, r3, #64
ka_uv_32
  cmp          r3, #32
  blt          ka_uv_16

  ; at leat 32 bytes
  vld1.8       {d4},[r1]!
  vld1.8       {d5},[r2]!
  vld1.8       {d8},[r1]!
  vld1.8       {d9},[r2]!
  vzip.u8      d4, d5
  vzip.u8      d8, d9

  vmovl.u8     q3, d5
  vmovl.u8     q2, d4
  vmul.u16     q3, q0
  vmul.u16     q2, q0
  vadd.s16     q2, q1
  vadd.s16     q3, q1
  vqrshrun.s16 d4, q2, #6
  vqrshrun.s16 d5, q3, #6
  vst1.8       {q2},[r0]!

  vmovl.u8     q5, d9
  vmovl.u8     q4, d8
  vmul.u16     q5, q0
  vmul.u16     q4, q0
  vadd.s16     q4, q1
  vadd.s16     q5, q1
  vqrshrun.s16 d8, q4, #6
  vqrshrun.s16 d9, q5, #6
  vst1.8       {q4},[r0]!

  sub          r3, r3, #32

ka_uv_16
  cmp          r3, #16
  blt          ka_uv_15

  ; at leat 16 bytes
  vld1.8       {d4},[r1]!
  vld1.8       {d5},[r2]!
  vzip.u8      d4, d5

  vmovl.u8     q3, d5
  vmovl.u8     q2, d4
  vmul.u16     q3, q0
  vmul.u16     q2, q0
  vadd.s16     q2, q1
  vadd.s16     q3, q1
  vqrshrun.s16 d4, q2, #6
  vqrshrun.s16 d5, q3, #6
  vst1.8       {q2},[r0]!

  sub          r3, r3, #16

ka_uv_15
  ; less than 16 bytes
  cmp          r3, #0
  ble          ka_uv_0
  ; move back by 16 - remaining bytes
  rsb          r3, r3, #16
  sub          r0, r0, r3
  sub          r1, r1, r3, lsr #1
  sub          r2, r2, r3, lsr #1
  ; at leat 16 bytes
  vld1.8       {d4},[r1]!
  vld1.8       {d5},[r2]!
  vzip.u8      d4, d5

  vmovl.u8     q3, d5
  vmovl.u8     q2, d4
  vmul.u16     q3, q0
  vmul.u16     q2, q0
  vadd.s16     q2, q1
  vadd.s16     q3, q1
  vqrshrun.s16 d4, q2, #6
  vqrshrun.s16 d5, q3, #6
  vst1.8       {q2},[r0]!

ka_uv_0
  mov          pc, r14

; ka_uv_correcthalf_neon
; ----------------------
; r0 = uint16_t* dst (u + v)
; r1 = uint8_t* src1
; r2 = uint8_t* src2
; r3 = int size (in halwords)
; on stack int16_t fac [0, 128] default 64
; on stack int16_t add
;
; assumes size is at least 16 bytes (8 halwords)
; on exit dst[x].a = fix((src1[2x] + src1[2x+1] +1)/2)
; on exit dst[x].b = fix((src2[2x] + src2[2x+1] +1)/2)
;
  EXPORT ka_uv_correcthalf_neon
  ALIGN 32
ka_uv_correcthalf_neon

  vmov.32      d4[0], r3 ; save r3
  ldr          r3, [sp, #0]
  vdup.16      q0, r3 ; fac
  ldr          r3, [sp, #4]
  vdup.16      q1, r3 ; add
  vmov.32      r3, d4[0] ; restore r3
  mov          r3, r3, lsl #1 ; halfword to bytes

  b            ka_uvh_next

ka_uvh_loop
  ; at leat 64 bytes
  vld1.8       {q2},[r1]!
  vld1.8       {q3},[r2]!
  vld1.8       {q4},[r1]!
  vld1.8       {q5},[r2]!
  vld1.8       {q6},[r1]!
  vld1.8       {q7},[r2]!
  vld1.8       {q8},[r1]!
  vld1.8       {q9},[r2]!
  vuzp.u8      d4, d5
  vuzp.u8      d6, d7
  vuzp.u8      d8, d9
  vuzp.u8      d10, d11
  vuzp.u8      d12, d13
  vuzp.u8      d14, d15
  vuzp.u8      d16, d17
  vuzp.u8      d18, d19
  vrhadd.u8    d4, d5
  vrhadd.u8    d6, d7
  vrhadd.u8    d8, d9
  vrhadd.u8    d10, d11
  vrhadd.u8    d12, d13
  vrhadd.u8    d14, d15
  vrhadd.u8    d16, d17
  vrhadd.u8    d18, d19
  vzip.u8      d4, d6
  vzip.u8      d8, d10
  vzip.u8      d12, d14
  vzip.u8      d16, d18

  vmovl.u8     q3, d6
  vmovl.u8     q2, d4
  vmul.u16     q3, q0
  vmul.u16     q2, q0
  vadd.s16     q2, q1
  vadd.s16     q3, q1
  vqrshrun.s16 d4, q2, #6
  vqrshrun.s16 d5, q3, #6
  vst1.8       {q2},[r0]!

  vmovl.u8     q5, d10
  vmovl.u8     q4, d8
  vmul.u16     q5, q0
  vmul.u16     q4, q0
  vadd.s16     q4, q1
  vadd.s16     q5, q1
  vqrshrun.s16 d8, q4, #6
  vqrshrun.s16 d9, q5, #6
  vst1.8       {q4},[r0]!

  vmovl.u8     q7, d14
  vmovl.u8     q6, d12
  vmul.u16     q7, q0
  vmul.u16     q6, q0
  vadd.s16     q6, q1
  vadd.s16     q7, q1
  vqrshrun.s16 d12, q6, #6
  vqrshrun.s16 d13, q7, #6
  vst1.8       {q6},[r0]!

  vmovl.u8     q9, d18
  vmovl.u8     q8, d16
  vmul.u16     q9, q0
  vmul.u16     q8, q0
  vadd.s16     q8, q1
  vadd.s16     q9, q1
  vqrshrun.s16 d16, q8, #6
  vqrshrun.s16 d17, q9, #6
  vst1.8       {q8},[r0]!

ka_uvh_next
  subs         r3, r3, #64
  bge          ka_uvh_loop

  add          r3, r3, #64
ka_uvh_32
  cmp          r3, #32
  blt          ka_uvh_16

  ; at leat 32 bytes
  vld1.8       {q2},[r1]!
  vld1.8       {q3},[r2]!
  vld1.8       {q4},[r1]!
  vld1.8       {q5},[r2]!
  vuzp.u8      d4, d5
  vuzp.u8      d6, d7
  vuzp.u8      d8, d9
  vuzp.u8      d10, d11
  vrhadd.u8    d4, d5
  vrhadd.u8    d6, d7
  vrhadd.u8    d8, d9
  vrhadd.u8    d10, d11
  vzip.u8      d4, d6
  vzip.u8      d8, d10

  vmovl.u8     q3, d6
  vmovl.u8     q2, d4
  vmul.u16     q3, q0
  vmul.u16     q2, q0
  vadd.s16     q2, q1
  vadd.s16     q3, q1
  vqrshrun.s16 d4, q2, #6
  vqrshrun.s16 d5, q3, #6
  vst1.8       {q2},[r0]!

  vmovl.u8     q5, d10
  vmovl.u8     q4, d8
  vmul.u16     q5, q0
  vmul.u16     q4, q0
  vadd.s16     q4, q1
  vadd.s16     q5, q1
  vqrshrun.s16 d8, q4, #6
  vqrshrun.s16 d9, q5, #6
  vst1.8       {q4},[r0]!

  sub          r3, r3, #32

ka_uvh_16
  cmp          r3, #16
  blt          ka_uvh_15

  ; at leat 16 bytes
  vld1.8       {q2},[r1]!
  vld1.8       {q3},[r2]!
  vuzp.u8      d4, d5
  vuzp.u8      d6, d7
  vrhadd.u8    d4, d5
  vrhadd.u8    d6, d7
  vzip.u8      d4, d6

  vmovl.u8     q3, d6
  vmovl.u8     q2, d4
  vmul.u16     q3, q0
  vmul.u16     q2, q0
  vadd.s16     q2, q1
  vadd.s16     q3, q1
  vqrshrun.s16 d4, q2, #6
  vqrshrun.s16 d5, q3, #6
  vst1.8       {q2},[r0]!

  sub          r3, r3, #16

ka_uvh_15
  ; less than 16 bytes
  cmp          r3, #0
  ble          ka_uvh_0
  ; move back by 16 - remaining bytes
  rsb          r3, r3, #16
  sub          r0, r0, r3
  sub          r1, r1, r3, lsr #1
  sub          r2, r2, r3, lsr #1
  ; at leat 16 bytes
  vld1.8       {q2},[r1]!
  vld1.8       {q3},[r2]!
  vuzp.u8      d4, d5
  vuzp.u8      d6, d7
  vrhadd.u8    d4, d5
  vrhadd.u8    d6, d7
  vzip.u8      d4, d6

  vmovl.u8     q3, d6
  vmovl.u8     q2, d4
  vmul.u16     q3, q0
  vmul.u16     q2, q0
  vadd.s16     q2, q1
  vadd.s16     q3, q1
  vqrshrun.s16 d4, q2, #6
  vqrshrun.s16 d5, q3, #6
  vst1.8       {q2},[r0]!

ka_uvh_0
  mov          pc, r14

  ALIGN 32
  END
