OpenJPH
Open-source implementation of JPEG2000 Part-15
Loading...
Searching...
No Matches
ojph_colour_avx2.cpp
Go to the documentation of this file.
1//***************************************************************************/
2// This software is released under the 2-Clause BSD license, included
3// below.
4//
5// Copyright (c) 2019, Aous Naman
6// Copyright (c) 2019, Kakadu Software Pty Ltd, Australia
7// Copyright (c) 2019, The University of New South Wales, Australia
8//
9// Redistribution and use in source and binary forms, with or without
10// modification, are permitted provided that the following conditions are
11// met:
12//
13// 1. Redistributions of source code must retain the above copyright
14// notice, this list of conditions and the following disclaimer.
15//
16// 2. Redistributions in binary form must reproduce the above copyright
17// notice, this list of conditions and the following disclaimer in the
18// documentation and/or other materials provided with the distribution.
19//
20// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
21// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
22// TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
23// PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24// HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
26// TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
27// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
28// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
29// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
30// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31//***************************************************************************/
32// This file is part of the OpenJPH software implementation.
33// File: ojph_colour_avx2.cpp
34// Author: Aous Naman
35// Date: 11 October 2019
36//***************************************************************************/
37
38#include "ojph_arch.h"
39#if defined(OJPH_ARCH_I386) || defined(OJPH_ARCH_X86_64)
40
41#include <climits>
42#include <cmath>
43
44#include "ojph_defs.h"
45#include "ojph_mem.h"
46#include "ojph_colour.h"
47
48#include "ojph_params.h"
49#include "../codestream/ojph_params_local.h"
50
51#include <immintrin.h>
52
53namespace ojph {
54 namespace local {
55
57 // https://github.com/seung-lab/dijkstra3d/blob/master/libdivide.h
58 static inline
59 __m256i avx2_mm256_srai_epi64(__m256i a, int amt, __m256i m)
60 {
61 // note than m must be obtained using
62 // __m256i m = _mm256_set1_epi64x(1ULL << (63 - amt));
63 __m256i x = _mm256_srli_epi64(a, amt);
64 x = _mm256_xor_si256(x, m);
65 __m256i result = _mm256_sub_epi64(x, m);
66 return result;
67 }
68
70 void avx2_rev_convert(const line_buf *src_line,
71 const ui32 src_line_offset,
72 line_buf *dst_line,
73 const ui32 dst_line_offset,
74 si64 shift, ui32 width)
75 {
76 if (src_line->flags & line_buf::LFT_32BIT)
77 {
78 if (dst_line->flags & line_buf::LFT_32BIT)
79 {
80 const si32 *sp = src_line->i32 + src_line_offset;
81 si32 *dp = dst_line->i32 + dst_line_offset;
82 __m256i sh = _mm256_set1_epi32((si32)shift);
83 for (int i = (width + 7) >> 3; i > 0; --i, sp+=8, dp+=8)
84 {
85 __m256i s = _mm256_loadu_si256((__m256i*)sp);
86 s = _mm256_add_epi32(s, sh);
87 _mm256_storeu_si256((__m256i*)dp, s);
88 }
89 }
90 else
91 {
92 const si32 *sp = src_line->i32 + src_line_offset;
93 si64 *dp = dst_line->i64 + dst_line_offset;
94 __m256i sh = _mm256_set1_epi64x(shift);
95 for (int i = (width + 7) >> 3; i > 0; --i, sp+=8, dp+=8)
96 {
97 __m256i s, t;
98 s = _mm256_loadu_si256((__m256i*)sp);
99
100 t = _mm256_cvtepi32_epi64(_mm256_extracti128_si256(s, 0));
101 t = _mm256_add_epi64(t, sh);
102 _mm256_storeu_si256((__m256i*)dp, t);
103
104 t = _mm256_cvtepi32_epi64(_mm256_extracti128_si256(s, 1));
105 t = _mm256_add_epi64(t, sh);
106 _mm256_storeu_si256((__m256i*)dp + 1, t);
107 }
108 }
109 }
110 else
111 {
112 assert(src_line->flags | line_buf::LFT_64BIT);
113 assert(dst_line->flags | line_buf::LFT_32BIT);
114 const si64 *sp = src_line->i64 + src_line_offset;
115 si32 *dp = dst_line->i32 + dst_line_offset;
116 __m256i low_bits = _mm256_set_epi64x(0, (si64)ULLONG_MAX,
117 0, (si64)ULLONG_MAX);
118 __m256i sh = _mm256_set1_epi64x(shift);
119 for (int i = (width + 7) >> 3; i > 0; --i, sp+=8, dp+=8)
120 {
121 __m256i s, t;
122 s = _mm256_loadu_si256((__m256i*)sp);
123 s = _mm256_add_epi64(s, sh);
124
125 t = _mm256_shuffle_epi32(s, _MM_SHUFFLE(0, 0, 2, 0));
126 t = _mm256_and_si256(low_bits, t);
127
128 s = _mm256_loadu_si256((__m256i*)sp + 1);
129 s = _mm256_add_epi64(s, sh);
130
131 s = _mm256_shuffle_epi32(s, _MM_SHUFFLE(2, 0, 0, 0));
132 s = _mm256_andnot_si256(low_bits, s);
133
134 t = _mm256_or_si256(s, t);
135 t = _mm256_permute4x64_epi64(t, _MM_SHUFFLE(3, 1, 2, 0));
136 _mm256_storeu_si256((__m256i*)dp, t);
137 }
138 }
139 }
140
142 void avx2_rev_convert_nlt_type3(const line_buf *src_line,
143 const ui32 src_line_offset,
144 line_buf *dst_line,
145 const ui32 dst_line_offset,
146 si64 shift, ui32 width)
147 {
148 if (src_line->flags & line_buf::LFT_32BIT)
149 {
150 if (dst_line->flags & line_buf::LFT_32BIT)
151 {
152 const si32 *sp = src_line->i32 + src_line_offset;
153 si32 *dp = dst_line->i32 + dst_line_offset;
154 __m256i sh = _mm256_set1_epi32((si32)(-shift));
155 __m256i zero = _mm256_setzero_si256();
156 for (int i = (width + 7) >> 3; i > 0; --i, sp += 8, dp += 8)
157 {
158 __m256i s = _mm256_loadu_si256((__m256i*)sp);
159 __m256i c = _mm256_cmpgt_epi32(zero, s); // 0xFFFFFFFF for -ve val
160 __m256i v_m_sh = _mm256_sub_epi32(sh, s); // - shift - value
161 v_m_sh = _mm256_and_si256(c, v_m_sh); // keep only -shift-val
162 s = _mm256_andnot_si256(c, s); // keep only +ve or 0
163 s = _mm256_or_si256(s, v_m_sh); // combine
164 _mm256_storeu_si256((__m256i*)dp, s);
165 }
166 }
167 else
168 {
169 const si32 *sp = src_line->i32 + src_line_offset;
170 si64 *dp = dst_line->i64 + dst_line_offset;
171 __m256i sh = _mm256_set1_epi64x(-shift);
172 __m256i zero = _mm256_setzero_si256();
173 for (int i = (width + 7) >> 3; i > 0; --i, sp += 8, dp += 8)
174 {
175 __m256i s, t, u0, u1, c, v_m_sh;
176 s = _mm256_loadu_si256((__m256i*)sp);
177
178 t = _mm256_cmpgt_epi32(zero, s); // find -ve 32bit -1
179 u0 = _mm256_unpacklo_epi32(s, t); // correct 64bit data
180 c = _mm256_unpacklo_epi32(t, t); // 64bit -1 for -ve value
181
182 v_m_sh = _mm256_sub_epi64(sh, u0); // - shift - value
183 v_m_sh = _mm256_and_si256(c, v_m_sh); // keep only - shift - value
184 u0 = _mm256_andnot_si256(c, u0); // keep only +ve or 0
185 u0 = _mm256_or_si256(u0, v_m_sh); // combine
186
187 u1 = _mm256_unpackhi_epi32(s, t); // correct 64bit data
188 c = _mm256_unpackhi_epi32(t, t); // 64bit -1 for -ve value
189
190 v_m_sh = _mm256_sub_epi64(sh, u1); // - shift - value
191 v_m_sh = _mm256_and_si256(c, v_m_sh); // keep only - shift - value
192 u1 = _mm256_andnot_si256(c, u1); // keep only +ve or 0
193 u1 = _mm256_or_si256(u1, v_m_sh); // combine
194
195 t = _mm256_permute2x128_si256(u0, u1, (2 << 4) | 0);
196 _mm256_storeu_si256((__m256i*)dp, t);
197
198 t = _mm256_permute2x128_si256(u0, u1, (3 << 4) | 1);
199 _mm256_storeu_si256((__m256i*)dp + 1, t);
200 }
201 }
202 }
203 else
204 {
205 assert(src_line->flags | line_buf::LFT_64BIT);
206 assert(dst_line->flags | line_buf::LFT_32BIT);
207 const si64 *sp = src_line->i64 + src_line_offset;
208 si32 *dp = dst_line->i32 + dst_line_offset;
209 __m256i sh = _mm256_set1_epi64x(-shift);
210 __m256i zero = _mm256_setzero_si256();
211 __m256i half_mask = _mm256_set_epi64x(0, (si64)ULLONG_MAX,
212 0, (si64)ULLONG_MAX);
213 for (int i = (width + 7) >> 3; i > 0; --i, sp += 8, dp += 8)
214 {
215 // s for source, t for target, p for positive, n for negative,
216 // m for mask, and tm for temp
217 __m256i s, t, p, n, m, tm;
218 s = _mm256_loadu_si256((__m256i*)sp);
219
220 m = _mm256_cmpgt_epi64(zero, s); // 64b -1 for -ve value
221 tm = _mm256_sub_epi64(sh, s); // - shift - value
222 n = _mm256_and_si256(m, tm); // -ve
223 p = _mm256_andnot_si256(m, s); // +ve
224 tm = _mm256_or_si256(n, p);
225 tm = _mm256_shuffle_epi32(tm, _MM_SHUFFLE(0, 0, 2, 0));
226 t = _mm256_and_si256(half_mask, tm);
227
228 s = _mm256_loadu_si256((__m256i*)sp + 1);
229 m = _mm256_cmpgt_epi64(zero, s); // 64b -1 for -ve value
230 tm = _mm256_sub_epi64(sh, s); // - shift - value
231 n = _mm256_and_si256(m, tm); // -ve
232 p = _mm256_andnot_si256(m, s); // +ve
233 tm = _mm256_or_si256(n, p);
234 tm = _mm256_shuffle_epi32(tm, _MM_SHUFFLE(2, 0, 0, 0));
235 tm = _mm256_andnot_si256(half_mask, tm);
236
237 t = _mm256_or_si256(t, tm);
238 t = _mm256_permute4x64_epi64(t, _MM_SHUFFLE(3, 1, 2, 0));
239 _mm256_storeu_si256((__m256i*)dp, t);
240 }
241 }
242 }
243
245 static inline
246 __m256i ojph_mm256_max_ge_epi32(__m256i a, __m256i b, __m256 x, __m256 y)
247 {
248 // We must use _CMP_NLT_UQ or _CMP_GE_OQ, _CMP_GE_OS, or _CMP_NLT_US
249 // It is not clear to me which to use
250 __m256 ct = _mm256_cmp_ps(x, y, _CMP_NLT_UQ); // 0xFFFFFFFF for x >= y
251 __m256i c = _mm256_castps_si256(ct); // does not generate any code
252 __m256i d = _mm256_and_si256(c, a); // keep only a, where x >= y
253 __m256i e = _mm256_andnot_si256(c, b); // keep only b, where x < y
254 return _mm256_or_si256(d, e); // combine
255 }
256
258 static inline
259 __m256i ojph_mm256_min_lt_epi32(__m256i a, __m256i b, __m256 x, __m256 y)
260 {
261 // We must use _CMP_LT_OQ or _CMP_NGE_UQ, _CMP_LT_OS, or _CMP_NGE_US
262 // It is not clear to me which to use
263 __m256 ct = _mm256_cmp_ps(x, y, _CMP_NGE_UQ); // 0xFFFFFFFF for x < y
264 __m256i c = _mm256_castps_si256(ct); // does not generate any code
265 __m256i d = _mm256_and_si256(c, a); // keep only a, where x < y
266 __m256i e = _mm256_andnot_si256(c, b); // keep only b, where x >= y
267 return _mm256_or_si256(d, e); // combine
268 }
269
271 template<bool NLT_TYPE3>
272 static inline
273 void local_avx2_irv_convert_to_integer(const line_buf *src_line,
274 line_buf *dst_line, ui32 dst_line_offset,
275 ui32 bit_depth, bool is_signed, ui32 width)
276 {
277 assert((src_line->flags & line_buf::LFT_32BIT) &&
278 (src_line->flags & line_buf::LFT_INTEGER) == 0 &&
279 (dst_line->flags & line_buf::LFT_32BIT) &&
280 (dst_line->flags & line_buf::LFT_INTEGER));
281
282 assert(bit_depth <= 32);
283 const float* sp = src_line->f32;
284 si32* dp = dst_line->i32 + dst_line_offset;
285 // There is the possibility that converting to integer will
286 // exceed the dynamic range of 32bit integer; therefore, care must be
287 // exercised.
288 // We look if the floating point number is outside the half-closed
289 // interval [-0.5f, 0.5f). If so, we limit the resulting integer
290 // to the maximum/minimum that number supports.
291 si32 neg_limit = (si32)INT_MIN >> (32 - bit_depth);
292 __m256 mul = _mm256_set1_ps((float)(1ull << bit_depth));
293 __m256 fl_up_lim = _mm256_set1_ps(-(float)neg_limit); // val < upper
294 __m256 fl_low_lim = _mm256_set1_ps((float)neg_limit); // val >= lower
295 __m256i s32_up_lim = _mm256_set1_epi32(INT_MAX >> (32 - bit_depth));
296 __m256i s32_low_lim = _mm256_set1_epi32(INT_MIN >> (32 - bit_depth));
297
298 if (is_signed)
299 {
300 __m256i zero = _mm256_setzero_si256();
301 __m256i bias =
302 _mm256_set1_epi32(-(si32)((1ULL << (bit_depth - 1)) + 1));
303 for (int i = (int)width; i > 0; i -= 8, sp += 8, dp += 8) {
304 __m256 t = _mm256_loadu_ps(sp);
305 t = _mm256_mul_ps(t, mul);
306 __m256i u = _mm256_cvtps_epi32(t);
307 u = ojph_mm256_max_ge_epi32(u, s32_low_lim, t, fl_low_lim);
308 u = ojph_mm256_min_lt_epi32(u, s32_up_lim, t, fl_up_lim);
309 if (NLT_TYPE3)
310 {
311 __m256i c = _mm256_cmpgt_epi32(zero, u); // 0xFFFFFFFF for -ve val
312 __m256i neg = _mm256_sub_epi32(bias, u); // -bias -value
313 neg = _mm256_and_si256(c, neg); // keep only - bias - val
314 u = _mm256_andnot_si256(c, u); // keep only +ve or 0
315 u = _mm256_or_si256(neg, u); // combine
316 }
317 _mm256_storeu_si256((__m256i*)dp, u);
318 }
319 }
320 else
321 {
322 __m256i half = _mm256_set1_epi32((si32)(1ULL << (bit_depth - 1)));
323 for (int i = (int)width; i > 0; i -= 8, sp += 8, dp += 8) {
324 __m256 t = _mm256_loadu_ps(sp);
325 t = _mm256_mul_ps(t, mul);
326 __m256i u = _mm256_cvtps_epi32(t);
327 u = ojph_mm256_max_ge_epi32(u, s32_low_lim, t, fl_low_lim);
328 u = ojph_mm256_min_lt_epi32(u, s32_up_lim, t, fl_up_lim);
329 u = _mm256_add_epi32(u, half);
330 _mm256_storeu_si256((__m256i*)dp, u);
331 }
332 }
333 }
334
336 void avx2_irv_convert_to_integer(const line_buf *src_line,
337 line_buf *dst_line, ui32 dst_line_offset,
338 ui32 bit_depth, bool is_signed, ui32 width)
339 {
340 local_avx2_irv_convert_to_integer<false>(src_line, dst_line,
341 dst_line_offset, bit_depth, is_signed, width);
342 }
343
345 void avx2_irv_convert_to_integer_nlt_type3(const line_buf *src_line,
346 line_buf *dst_line, ui32 dst_line_offset,
347 ui32 bit_depth, bool is_signed, ui32 width)
348 {
349 local_avx2_irv_convert_to_integer<true>(src_line, dst_line,
350 dst_line_offset, bit_depth, is_signed, width);
351 }
352
354 template<int NLT_TYPE>
355 static inline
356 void local_avx2_irv_convert_to_integer_nlt2or4(const line_buf *src_line,
357 line_buf *dst_line, ui32 dst_line_offset,
358 ui32 bit_depth, bool is_signed, ui32 width, const nlt_rec* rec)
359 {
360 assert((src_line->flags & line_buf::LFT_32BIT) &&
361 (src_line->flags & line_buf::LFT_INTEGER) == 0 &&
362 (dst_line->flags & line_buf::LFT_32BIT) &&
363 (dst_line->flags & line_buf::LFT_INTEGER));
364 ojph_unused(bit_depth);
365 ojph_unused(is_signed);
366
367 assert(rec->get_bit_depth() <= 32);
368 const float* sp = src_line->f32;
369 si32* dp = dst_line->i32 + dst_line_offset;
370
371 __m256 mul = _mm256_set1_ps(rec->multiplier);
372 __m256 fd_min = _mm256_set1_ps(rec->fd_min);
373 __m256 fd_max = _mm256_set1_ps(rec->fd_max);
374 __m256 delta = _mm256_set1_ps(rec->delta);
375 __m256 inv_delta = _mm256_set1_ps(rec->inv_delta);
376 const float* lut = rec->dec_points;
377
378 __m256 half_ps = _mm256_set1_ps(0.5f);
379 __m256i one = _mm256_set1_epi32(1);
380
381 if (rec->is_signed())
382 {
383 __m256 half =
384 _mm256_set1_ps((float)(1ULL << (rec->get_bit_depth() - 1)));
385 __m256i bias =
386 _mm256_set1_epi32(-(si32)((1ULL << (rec->get_bit_depth() - 1)) + 1));
387 __m256i zero = _mm256_setzero_si256();
388 for (int i = (int)width; i > 0; i -= 8, sp += 8, dp += 8) {
389 __m256 t = _mm256_loadu_ps(sp);
390 t = _mm256_add_ps(t, half_ps); // convert to [0, 1]
391 t = _mm256_max_ps(t, fd_min);
392 t = _mm256_min_ps(t, fd_max);
393 __m256i k = _mm256_cvttps_epi32(
394 _mm256_mul_ps(_mm256_sub_ps(t, fd_min), inv_delta));
395 __m256 d_k = _mm256_add_ps(fd_min,
396 _mm256_mul_ps(_mm256_cvtepi32_ps(k), delta));
397 __m256 t_k = _mm256_i32gather_ps(lut, k, 4);
398 __m256 t_kp1 = _mm256_i32gather_ps(lut, _mm256_add_epi32(k, one), 4);
399 __m256 z = _mm256_add_ps(t_k,
400 _mm256_mul_ps(_mm256_mul_ps(_mm256_sub_ps(t, d_k), inv_delta),
401 _mm256_sub_ps(t_kp1, t_k)));
402 __m256i v =
403 _mm256_cvtps_epi32(_mm256_sub_ps(_mm256_mul_ps(z, mul), half));
404 if (NLT_TYPE == 4)
405 {
406 __m256i c = _mm256_cmpgt_epi32(zero, v); // 0xFFFFFFFF for -ve val
407 __m256i neg = _mm256_sub_epi32(bias, v); // - bias - value
408 neg = _mm256_and_si256(c, neg); // keep only - bias - val
409 v = _mm256_andnot_si256(c, v); // keep only +ve or 0
410 v = _mm256_or_si256(neg, v); // combine
411 }
412 _mm256_storeu_si256((__m256i*)dp, v);
413 }
414 }
415 else
416 {
417 for (int i = (int)width; i > 0; i -= 8, sp += 8, dp += 8) {
418 __m256 t = _mm256_loadu_ps(sp);
419 t = _mm256_add_ps(t, half_ps); // convert to [0, 1]
420 t = _mm256_max_ps(t, fd_min);
421 t = _mm256_min_ps(t, fd_max);
422 __m256i k = _mm256_cvttps_epi32(
423 _mm256_mul_ps(_mm256_sub_ps(t, fd_min), inv_delta));
424 __m256 d_k = _mm256_add_ps(fd_min,
425 _mm256_mul_ps(_mm256_cvtepi32_ps(k), delta));
426 __m256 t_k = _mm256_i32gather_ps(lut, k, 4);
427 __m256 t_kp1 =
428 _mm256_i32gather_ps(lut, _mm256_add_epi32(k, one), 4);
429 __m256 z = _mm256_add_ps(t_k,
430 _mm256_mul_ps(_mm256_mul_ps(_mm256_sub_ps(t, d_k), inv_delta),
431 _mm256_sub_ps(t_kp1, t_k)));
432 __m256i v = _mm256_cvtps_epi32(_mm256_mul_ps(z, mul));
433 _mm256_storeu_si256((__m256i*)dp, v);
434 }
435 }
436 }
437
439 void avx2_irv_convert_to_integer_nlt(const line_buf *src_line,
440 line_buf *dst_line, ui32 dst_line_offset,
441 ui32 bit_depth, bool is_signed, ui32 width, const nlt_rec* rec)
442 {
443 using nl = nlt_rec::nonlinearity;
444 if (rec->get_type() == nl::OJPH_NLT_LUT_STYLE_NLT)
445 local_avx2_irv_convert_to_integer_nlt2or4<2>(src_line, dst_line,
446 dst_line_offset, bit_depth, is_signed, width, rec);
447 else if (rec->get_type() == nl::OJPH_NLT_BINARY_COMPLEMENT_PLUS_LUT)
448 local_avx2_irv_convert_to_integer_nlt2or4<4>(src_line, dst_line,
449 dst_line_offset, bit_depth, is_signed, width, rec);
450 else
451 assert(0);
452 }
453
455 template<bool NLT_TYPE3>
456 static inline
457 void local_avx2_irv_convert_to_float(const line_buf *src_line,
458 ui32 src_line_offset, line_buf *dst_line,
459 ui32 bit_depth, bool is_signed, ui32 width)
460 {
461 assert((src_line->flags & line_buf::LFT_32BIT) &&
462 (src_line->flags & line_buf::LFT_INTEGER) &&
463 (dst_line->flags & line_buf::LFT_32BIT) &&
464 (dst_line->flags & line_buf::LFT_INTEGER) == 0);
465
466 assert(bit_depth <= 32);
467 __m256 mul = _mm256_set1_ps((float)(1.0 / (double)(1ULL << bit_depth)));
468
469 const si32* sp = src_line->i32 + src_line_offset;
470 float* dp = dst_line->f32;
471 if (is_signed)
472 {
473 __m256i zero = _mm256_setzero_si256();
474 __m256i bias =
475 _mm256_set1_epi32(-(si32)((1ULL << (bit_depth - 1)) + 1));
476 for (int i = (int)width; i > 0; i -= 8, sp += 8, dp += 8) {
477 __m256i t = _mm256_loadu_si256((__m256i*)sp);
478 if (NLT_TYPE3)
479 {
480 __m256i c = _mm256_cmpgt_epi32(zero, t); // 0xFFFFFFFF for -ve val
481 __m256i neg = _mm256_sub_epi32(bias, t); // - bias - value
482 neg = _mm256_and_si256(c, neg); // keep only - bias - val
483 c = _mm256_andnot_si256(c, t); // keep only +ve or 0
484 t = _mm256_or_si256(neg, c); // combine
485 }
486 __m256 v = _mm256_cvtepi32_ps(t);
487 v = _mm256_mul_ps(v, mul);
488 _mm256_storeu_ps(dp, v);
489 }
490 }
491 else
492 {
493 __m256i half = _mm256_set1_epi32((si32)(1ULL << (bit_depth - 1)));
494 for (int i = (int)width; i > 0; i -= 8, sp += 8, dp += 8) {
495 __m256i t = _mm256_loadu_si256((__m256i*)sp);
496 t = _mm256_sub_epi32(t, half);
497 __m256 v = _mm256_cvtepi32_ps(t);
498 v = _mm256_mul_ps(v, mul);
499 _mm256_storeu_ps(dp, v);
500 }
501 }
502 }
503
505 void avx2_irv_convert_to_float(const line_buf *src_line,
506 ui32 src_line_offset, line_buf *dst_line,
507 ui32 bit_depth, bool is_signed, ui32 width)
508 {
509 local_avx2_irv_convert_to_float<false>(src_line, src_line_offset,
510 dst_line, bit_depth, is_signed, width);
511 }
512
514 void avx2_irv_convert_to_float_nlt_type3(const line_buf *src_line,
515 ui32 src_line_offset, line_buf *dst_line,
516 ui32 bit_depth, bool is_signed, ui32 width)
517 {
518 local_avx2_irv_convert_to_float<true>(src_line, src_line_offset,
519 dst_line, bit_depth, is_signed, width);
520 }
521
523 template<int NLT_TYPE>
524 static inline
525 void local_avx2_irv_convert_to_float_nlt2or4(const line_buf *src_line,
526 ui32 src_line_offset, line_buf *dst_line,
527 ui32 bit_depth, bool is_signed, ui32 width, const nlt_rec* rec)
528 {
529 ojph_unused(is_signed);
530 ojph_unused(bit_depth);
531
532 assert((src_line->flags & line_buf::LFT_32BIT) &&
533 (src_line->flags & line_buf::LFT_INTEGER) &&
534 (dst_line->flags & line_buf::LFT_32BIT) &&
535 (dst_line->flags & line_buf::LFT_INTEGER) == 0);
536
537 assert(bit_depth <= 32);
538 __m256 mul = _mm256_set1_ps(rec->multiplier);
539 __m256 ft_min = _mm256_set1_ps(rec->ft_min);
540 __m256 ft_max = _mm256_set1_ps(rec->ft_max);
541 __m256 delta = _mm256_set1_ps(rec->delta);
542 __m256 inv_delta = _mm256_set1_ps(rec->inv_delta);
543 const float* lut = rec->approx_enc_points;
544
545 __m256 half_ps = _mm256_set1_ps(0.5f);
546 __m256i one = _mm256_set1_epi32(1);
547
548 const si32* sp = src_line->i32 + src_line_offset;
549 float* dp = dst_line->f32;
550 if (rec->is_signed())
551 {
552 __m256i bias =
553 _mm256_set1_epi32(-(si32)((1ULL << (rec->get_bit_depth() - 1)) + 1));
554 __m256i zero = _mm256_setzero_si256();
555 for (int i = (int)width; i > 0; i -= 8, sp += 8, dp += 8) {
556 __m256i v = _mm256_loadu_si256((__m256i*)sp);
557 if (NLT_TYPE == 4)
558 {
559 __m256i c = _mm256_cmpgt_epi32(zero, v); // 0xFFFFFFFF for -ve val
560 __m256i neg = _mm256_sub_epi32(bias, v); // - bias - value
561 neg = _mm256_and_si256(c, neg); // keep only - bias - val
562 v = _mm256_andnot_si256(c, v); // keep only +ve or 0
563 v = _mm256_or_si256(neg, v); // combine
564 }
565 __m256 t = _mm256_add_ps( // convert to [0, 1]
566 _mm256_mul_ps(_mm256_cvtepi32_ps(v), mul), half_ps);
567 t = _mm256_max_ps(t, ft_min);
568 t = _mm256_min_ps(t, ft_max);
569 __m256i k = _mm256_cvttps_epi32(
570 _mm256_mul_ps(_mm256_sub_ps(t, ft_min), inv_delta));
571 __m256 d_k = _mm256_add_ps(ft_min,
572 _mm256_mul_ps(_mm256_cvtepi32_ps(k), delta));
573 __m256 t_k = _mm256_i32gather_ps(lut, k, 4);
574 __m256 t_kp1 =
575 _mm256_i32gather_ps(lut, _mm256_add_epi32(k, one), 4);
576 __m256 y = _mm256_add_ps(t_k,
577 _mm256_mul_ps(_mm256_mul_ps(_mm256_sub_ps(t, d_k), inv_delta),
578 _mm256_sub_ps(t_kp1, t_k)));
579 _mm256_storeu_ps(dp, _mm256_sub_ps(y, half_ps));
580 }
581 }
582 else
583 {
584 for (int i = (int)width; i > 0; i -= 8, sp += 8, dp += 8) {
585 __m256i v = _mm256_loadu_si256((__m256i*)sp);
586 __m256 t = _mm256_mul_ps(_mm256_cvtepi32_ps(v), mul); // in [0, 1]
587 t = _mm256_max_ps(t, ft_min);
588 t = _mm256_min_ps(t, ft_max);
589 __m256i k = _mm256_cvttps_epi32(
590 _mm256_mul_ps(_mm256_sub_ps(t, ft_min), inv_delta));
591 __m256 d_k = _mm256_add_ps(ft_min,
592 _mm256_mul_ps(_mm256_cvtepi32_ps(k), delta));
593 __m256 t_k = _mm256_i32gather_ps(lut, k, 4);
594 __m256 t_kp1 =
595 _mm256_i32gather_ps(lut, _mm256_add_epi32(k, one), 4);
596 __m256 y = _mm256_add_ps(t_k,
597 _mm256_mul_ps(_mm256_mul_ps(_mm256_sub_ps(t, d_k), inv_delta),
598 _mm256_sub_ps(t_kp1, t_k)));
599 _mm256_storeu_ps(dp, _mm256_sub_ps(y, half_ps));
600 }
601 }
602 }
603
605 // The precise inverse of the look-up table, for a table whose derived
606 // version is fine enough that one step of it crosses at most one entry of
607 // the table that was supplied. The entry that a value falls in is found
608 // with the same arithmetic that the approximation above uses, and the
609 // entry after that is looked at as well, to catch the values that fall
610 // past the entry that was found.
612 template<int NLT_TYPE>
613 static inline
614 void local_avx2_irv_convert_to_float_nlt2or4_precise_1step(
615 const line_buf *src_line, ui32 src_line_offset, line_buf *dst_line,
616 ui32 bit_depth, bool is_signed, ui32 width, const nlt_rec* rec)
617 {
618 ojph_unused(is_signed);
619 ojph_unused(bit_depth);
620
621 assert((src_line->flags & line_buf::LFT_32BIT) &&
622 (src_line->flags & line_buf::LFT_INTEGER) &&
623 (dst_line->flags & line_buf::LFT_32BIT) &&
624 (dst_line->flags & line_buf::LFT_INTEGER) == 0);
625
626 assert(bit_depth <= 32);
627 __m256 mul = _mm256_set1_ps(rec->multiplier);
628 __m256 ft_min = _mm256_set1_ps(rec->ft_min);
629 __m256 ft_max = _mm256_set1_ps(rec->ft_max);
630 __m256 fd_min = _mm256_set1_ps(rec->fd_min);
631 __m256 delta = _mm256_set1_ps(rec->delta);
632 __m256 inv_delta = _mm256_set1_ps(rec->inv_delta);
633 const float* lut = rec->precise_dec_lut;
634 const ui16* indices = rec->precise_enc_dec_indices;
635
636 __m256 half_ps = _mm256_set1_ps(0.5f);
637 __m256 one_ps = _mm256_set1_ps(1.0f);
638 __m256 zero_ps = _mm256_setzero_ps();
639 __m256i one = _mm256_set1_epi32(1);
640 __m256i two = _mm256_set1_epi32(2);
641 __m256i low16 = _mm256_set1_epi32(0xFFFF);
642
643 const si32* sp = src_line->i32 + src_line_offset;
644 float* dp = dst_line->f32;
645 if (rec->is_signed())
646 {
647 __m256i bias =
648 _mm256_set1_epi32(-(si32)((1ULL << (rec->get_bit_depth() - 1)) + 1));
649 __m256i zero = _mm256_setzero_si256();
650 for (int i = (int)width; i > 0; i -= 8, sp += 8, dp += 8) {
651 __m256i v = _mm256_loadu_si256((__m256i*)sp);
652 if (NLT_TYPE == 4)
653 {
654 __m256i c = _mm256_cmpgt_epi32(zero, v); // 0xFFFFFFFF for -ve val
655 __m256i neg = _mm256_sub_epi32(bias, v); // - bias - value
656 neg = _mm256_and_si256(c, neg); // keep only - bias - val
657 v = _mm256_andnot_si256(c, v); // keep only +ve or 0
658 v = _mm256_or_si256(neg, v); // combine
659 }
660 __m256 t = _mm256_add_ps( // convert to [0, 1]
661 _mm256_mul_ps(_mm256_cvtepi32_ps(v), mul), half_ps);
662 t = _mm256_max_ps(t, ft_min);
663 t = _mm256_min_ps(t, ft_max);
664 __m256i kk = _mm256_cvttps_epi32(
665 _mm256_mul_ps(_mm256_sub_ps(t, ft_min), inv_delta));
666 // the indices are 16 bits wide, so gather 32 bits at a stride of
667 // two and keep the low half of each; the two entries that are read
668 // past the end of the indices are part of their allocation
669 __m256i k = _mm256_and_si256(
670 _mm256_i32gather_epi32((const int*)indices, kk, 2), low16);
671 __m256 y0 = _mm256_i32gather_ps(lut, k, 4);
672 __m256 y1 = _mm256_i32gather_ps(lut, _mm256_add_epi32(k, one), 4);
673 __m256 y2 = _mm256_i32gather_ps(lut, _mm256_add_epi32(k, two), 4);
674 // one step of search: a value that falls past the entry that was
675 // found belongs to the entry after it
676 __m256 adv = _mm256_cmp_ps(t, y1, _CMP_GT_OQ);
677 y0 = _mm256_blendv_ps(y0, y1, adv);
678 y1 = _mm256_blendv_ps(y1, y2, adv);
679 k = _mm256_add_epi32(k,
680 _mm256_and_si256(_mm256_castps_si256(adv), one));
681 // interpolate, but only over a segment that has a width; the last
682 // entry of the table is repeated, and a segment without a width
683 // contributes nothing
684 __m256 den = _mm256_sub_ps(y1, y0);
685 __m256 has = _mm256_cmp_ps(den, zero_ps, _CMP_GT_OQ);
686 den = _mm256_blendv_ps(one_ps, den, has);
687 __m256 y = _mm256_add_ps(
688 _mm256_add_ps(fd_min,
689 _mm256_mul_ps(_mm256_cvtepi32_ps(k), delta)),
690 _mm256_and_ps(_mm256_div_ps(
691 _mm256_mul_ps(_mm256_sub_ps(t, y0), delta), den), has));
692 _mm256_storeu_ps(dp, _mm256_sub_ps(y, half_ps));
693 }
694 }
695 else
696 {
697 for (int i = (int)width; i > 0; i -= 8, sp += 8, dp += 8) {
698 __m256i v = _mm256_loadu_si256((__m256i*)sp);
699 __m256 t = _mm256_mul_ps(_mm256_cvtepi32_ps(v), mul); // in [0, 1]
700 t = _mm256_max_ps(t, ft_min);
701 t = _mm256_min_ps(t, ft_max);
702 __m256i kk = _mm256_cvttps_epi32(
703 _mm256_mul_ps(_mm256_sub_ps(t, ft_min), inv_delta));
704 // the indices are 16 bits wide, so gather 32 bits at a stride of
705 // two and keep the low half of each; the two entries that are read
706 // past the end of the indices are part of their allocation
707 __m256i k = _mm256_and_si256(
708 _mm256_i32gather_epi32((const int*)indices, kk, 2), low16);
709 __m256 y0 = _mm256_i32gather_ps(lut, k, 4);
710 __m256 y1 = _mm256_i32gather_ps(lut, _mm256_add_epi32(k, one), 4);
711 __m256 y2 = _mm256_i32gather_ps(lut, _mm256_add_epi32(k, two), 4);
712 // one step of search: a value that falls past the entry that was
713 // found belongs to the entry after it
714 __m256 adv = _mm256_cmp_ps(t, y1, _CMP_GT_OQ);
715 y0 = _mm256_blendv_ps(y0, y1, adv);
716 y1 = _mm256_blendv_ps(y1, y2, adv);
717 k = _mm256_add_epi32(k,
718 _mm256_and_si256(_mm256_castps_si256(adv), one));
719 // interpolate, but only over a segment that has a width; the last
720 // entry of the table is repeated, and a segment without a width
721 // contributes nothing
722 __m256 den = _mm256_sub_ps(y1, y0);
723 __m256 has = _mm256_cmp_ps(den, zero_ps, _CMP_GT_OQ);
724 den = _mm256_blendv_ps(one_ps, den, has);
725 __m256 y = _mm256_add_ps(
726 _mm256_add_ps(fd_min,
727 _mm256_mul_ps(_mm256_cvtepi32_ps(k), delta)),
728 _mm256_and_ps(_mm256_div_ps(
729 _mm256_mul_ps(_mm256_sub_ps(t, y0), delta), den), has));
730 _mm256_storeu_ps(dp, _mm256_sub_ps(y, half_ps));
731 }
732 }
733 }
734
736 // The precise inverse of the look-up table, for a table whose derived
737 // version is coarse enough that one step of it can cross more than one
738 // entry of the table that was supplied. The entry that a value falls in
739 // is found with a fixed-depth binary search over the entries the step can
740 // reach; the depth is the worst case, ceil(log2(precise_max_steps + 1)),
741 // so every lane performs the same number of steps. This mirrors
742 // local_gen_irv_convert_to_float_nlt2or4_precise_search in ojph_colour.cpp,
743 // so keep the two in step when that one changes.
745 template<int NLT_TYPE>
746 static inline
747 void local_avx2_irv_convert_to_float_nlt2or4_precise_search(
748 const line_buf *src_line, ui32 src_line_offset, line_buf *dst_line,
749 ui32 bit_depth, bool is_signed, ui32 width, const nlt_rec* rec)
750 {
751 ojph_unused(is_signed);
752 ojph_unused(bit_depth);
753
754 assert((src_line->flags & line_buf::LFT_32BIT) &&
755 (src_line->flags & line_buf::LFT_INTEGER) &&
756 (dst_line->flags & line_buf::LFT_32BIT) &&
757 (dst_line->flags & line_buf::LFT_INTEGER) == 0);
758
759 assert(bit_depth <= 32);
760 assert(rec->precise_max_steps > 1);
761 __m256 mul = _mm256_set1_ps(rec->multiplier);
762 __m256 ft_min = _mm256_set1_ps(rec->ft_min);
763 __m256 ft_max = _mm256_set1_ps(rec->ft_max);
764 __m256 fd_min = _mm256_set1_ps(rec->fd_min);
765 __m256 delta = _mm256_set1_ps(rec->delta);
766 __m256 inv_delta = _mm256_set1_ps(rec->inv_delta);
767 const float* lut = rec->precise_dec_lut;
768 const ui16* indices = rec->precise_enc_dec_indices;
769 ui32 num_points = rec->num_points;
770
771 // the search interval is at most precise_max_steps + 1 entries wide;
772 // run the worst-case number of binary-search steps on every lane,
773 // which is ceil(log2(span))
774 ui32 span = rec->precise_max_steps + 1;
775 int steps = 32 - (int)count_leading_zeros(rec->precise_max_steps);
776
777 __m256 half_ps = _mm256_set1_ps(0.5f);
778 __m256 one_ps = _mm256_set1_ps(1.0f);
779 __m256 zero_ps = _mm256_setzero_ps();
780 __m256i low16 = _mm256_set1_epi32(0xFFFF);
781 __m256i span_epi32 = _mm256_set1_epi32((si32)span);
782 __m256i num_epi32 = _mm256_set1_epi32((si32)num_points);
783
784 const si32* sp = src_line->i32 + src_line_offset;
785 float* dp = dst_line->f32;
786 if (rec->is_signed())
787 {
788 __m256i bias =
789 _mm256_set1_epi32(-(si32)((1ULL << (rec->get_bit_depth() - 1)) + 1));
790 __m256i zero = _mm256_setzero_si256();
791 for (int i = (int)width; i > 0; i -= 8, sp += 8, dp += 8) {
792 __m256i v = _mm256_loadu_si256((__m256i*)sp);
793 if (NLT_TYPE == 4)
794 {
795 __m256i c = _mm256_cmpgt_epi32(zero, v); // 0xFFFFFFFF for -ve val
796 __m256i neg = _mm256_sub_epi32(bias, v); // - bias - value
797 neg = _mm256_and_si256(c, neg); // keep only - bias - val
798 v = _mm256_andnot_si256(c, v); // keep only +ve or 0
799 v = _mm256_or_si256(neg, v); // combine
800 }
801 __m256 t = _mm256_add_ps( // convert to [0, 1]
802 _mm256_mul_ps(_mm256_cvtepi32_ps(v), mul), half_ps);
803 t = _mm256_max_ps(t, ft_min);
804 t = _mm256_min_ps(t, ft_max);
805 __m256i kk = _mm256_cvttps_epi32(
806 _mm256_mul_ps(_mm256_sub_ps(t, ft_min), inv_delta));
807 // the indices are 16 bits wide, so gather 32 bits at a stride of
808 // two and keep the low half of each; the two entries that are read
809 // past the end of the indices are part of their allocation
810 __m256i lo = _mm256_and_si256(
811 _mm256_i32gather_epi32((const int*)indices, kk, 2), low16);
812 __m256i hi = _mm256_min_epi32(
813 _mm256_add_epi32(lo, span_epi32), num_epi32);
814
815 for (int step = 0; step < steps; ++step)
816 {
817 __m256i mid = _mm256_srli_epi32(_mm256_add_epi32(lo, hi), 1);
818 __m256 lmid = _mm256_i32gather_ps(lut, mid, 4);
819 __m256 ge = _mm256_cmp_ps(t, lmid, _CMP_GE_OQ);
820 __m256i g = _mm256_castps_si256(ge);
821 lo = _mm256_blendv_epi8(lo, mid, g);
822 hi = _mm256_blendv_epi8(mid, hi, g);
823 }
824
825 __m256 y0 = _mm256_i32gather_ps(lut, lo, 4);
826 __m256 y1 = _mm256_i32gather_ps(lut, hi, 4);
827 // interpolate, but only over a segment that has a width; the last
828 // entry of the table is repeated, and a segment without a width
829 // contributes nothing
830 __m256 den = _mm256_sub_ps(y1, y0);
831 __m256 has = _mm256_cmp_ps(den, zero_ps, _CMP_GT_OQ);
832 den = _mm256_blendv_ps(one_ps, den, has);
833 __m256 y = _mm256_add_ps(
834 _mm256_add_ps(fd_min,
835 _mm256_mul_ps(_mm256_cvtepi32_ps(lo), delta)),
836 _mm256_and_ps(_mm256_div_ps(
837 _mm256_mul_ps(_mm256_sub_ps(t, y0), delta), den), has));
838 _mm256_storeu_ps(dp, _mm256_sub_ps(y, half_ps));
839 }
840 }
841 else
842 {
843 for (int i = (int)width; i > 0; i -= 8, sp += 8, dp += 8) {
844 __m256i v = _mm256_loadu_si256((__m256i*)sp);
845 __m256 t = _mm256_mul_ps(_mm256_cvtepi32_ps(v), mul); // in [0, 1]
846 t = _mm256_max_ps(t, ft_min);
847 t = _mm256_min_ps(t, ft_max);
848 __m256i kk = _mm256_cvttps_epi32(
849 _mm256_mul_ps(_mm256_sub_ps(t, ft_min), inv_delta));
850 // the indices are 16 bits wide, so gather 32 bits at a stride of
851 // two and keep the low half of each; the two entries that are read
852 // past the end of the indices are part of their allocation
853 __m256i lo = _mm256_and_si256(
854 _mm256_i32gather_epi32((const int*)indices, kk, 2), low16);
855 __m256i hi = _mm256_min_epi32(
856 _mm256_add_epi32(lo, span_epi32), num_epi32);
857
858 for (int step = 0; step < steps; ++step)
859 {
860 __m256i mid = _mm256_srli_epi32(_mm256_add_epi32(lo, hi), 1);
861 __m256 lmid = _mm256_i32gather_ps(lut, mid, 4);
862 __m256 ge = _mm256_cmp_ps(t, lmid, _CMP_GE_OQ);
863 __m256i g = _mm256_castps_si256(ge);
864 lo = _mm256_blendv_epi8(lo, mid, g);
865 hi = _mm256_blendv_epi8(mid, hi, g);
866 }
867
868 __m256 y0 = _mm256_i32gather_ps(lut, lo, 4);
869 __m256 y1 = _mm256_i32gather_ps(lut, hi, 4);
870 __m256 den = _mm256_sub_ps(y1, y0);
871 __m256 has = _mm256_cmp_ps(den, zero_ps, _CMP_GT_OQ);
872 den = _mm256_blendv_ps(one_ps, den, has);
873 __m256 y = _mm256_add_ps(
874 _mm256_add_ps(fd_min,
875 _mm256_mul_ps(_mm256_cvtepi32_ps(lo), delta)),
876 _mm256_and_ps(_mm256_div_ps(
877 _mm256_mul_ps(_mm256_sub_ps(t, y0), delta), den), has));
878 _mm256_storeu_ps(dp, _mm256_sub_ps(y, half_ps));
879 }
880 }
881 }
882
884 void avx2_irv_convert_to_float_nlt(const line_buf *src_line,
885 ui32 src_line_offset, line_buf *dst_line,
886 ui32 bit_depth, bool is_signed, ui32 width, const nlt_rec* rec)
887 {
888 using nl = nlt_rec::nonlinearity;
889 if (rec->get_type() == nl::OJPH_NLT_LUT_STYLE_NLT)
890 {
891 if (rec->approx_encoding_nlt)
892 local_avx2_irv_convert_to_float_nlt2or4<2>(src_line,
893 src_line_offset, dst_line, bit_depth, is_signed, width, rec);
894 else if (rec->precise_max_steps == 1)
895 local_avx2_irv_convert_to_float_nlt2or4_precise_1step<2>(src_line,
896 src_line_offset, dst_line, bit_depth, is_signed, width, rec);
897 else
898 local_avx2_irv_convert_to_float_nlt2or4_precise_search<2>(src_line,
899 src_line_offset, dst_line, bit_depth, is_signed, width, rec);
900 }
901 else if (rec->get_type() == nl::OJPH_NLT_BINARY_COMPLEMENT_PLUS_LUT)
902 {
903 if (rec->approx_encoding_nlt)
904 local_avx2_irv_convert_to_float_nlt2or4<4>(src_line,
905 src_line_offset, dst_line, bit_depth, is_signed, width, rec);
906 else if (rec->precise_max_steps == 1)
907 local_avx2_irv_convert_to_float_nlt2or4_precise_1step<4>(src_line,
908 src_line_offset, dst_line, bit_depth, is_signed, width, rec);
909 else
910 local_avx2_irv_convert_to_float_nlt2or4_precise_search<4>(src_line,
911 src_line_offset, dst_line, bit_depth, is_signed, width, rec);
912 }
913 else
914 assert(0);
915 }
916
917
919 void avx2_rct_forward(const line_buf *r,
920 const line_buf *g,
921 const line_buf *b,
922 line_buf *y, line_buf *cb, line_buf *cr,
923 ui32 repeat)
924 {
925 assert((y->flags & line_buf::LFT_INTEGER) &&
926 (cb->flags & line_buf::LFT_INTEGER) &&
927 (cr->flags & line_buf::LFT_INTEGER) &&
928 (r->flags & line_buf::LFT_INTEGER) &&
929 (g->flags & line_buf::LFT_INTEGER) &&
930 (b->flags & line_buf::LFT_INTEGER));
931
932 if (y->flags & line_buf::LFT_32BIT)
933 {
934 assert((y->flags & line_buf::LFT_32BIT) &&
935 (cb->flags & line_buf::LFT_32BIT) &&
936 (cr->flags & line_buf::LFT_32BIT) &&
937 (r->flags & line_buf::LFT_32BIT) &&
938 (g->flags & line_buf::LFT_32BIT) &&
939 (b->flags & line_buf::LFT_32BIT));
940 const si32 *rp = r->i32, * gp = g->i32, * bp = b->i32;
941 si32 *yp = y->i32, * cbp = cb->i32, * crp = cr->i32;
942 for (int i = (repeat + 7) >> 3; i > 0; --i)
943 {
944 __m256i mr = _mm256_load_si256((__m256i*)rp);
945 __m256i mg = _mm256_load_si256((__m256i*)gp);
946 __m256i mb = _mm256_load_si256((__m256i*)bp);
947 __m256i t = _mm256_add_epi32(mr, mb);
948 t = _mm256_add_epi32(t, _mm256_slli_epi32(mg, 1));
949 _mm256_store_si256((__m256i*)yp, _mm256_srai_epi32(t, 2));
950 t = _mm256_sub_epi32(mb, mg);
951 _mm256_store_si256((__m256i*)cbp, t);
952 t = _mm256_sub_epi32(mr, mg);
953 _mm256_store_si256((__m256i*)crp, t);
954
955 rp += 8; gp += 8; bp += 8;
956 yp += 8; cbp += 8; crp += 8;
957 }
958 }
959 else
960 {
961 assert((y->flags & line_buf::LFT_64BIT) &&
962 (cb->flags & line_buf::LFT_64BIT) &&
963 (cr->flags & line_buf::LFT_64BIT) &&
964 (r->flags & line_buf::LFT_32BIT) &&
965 (g->flags & line_buf::LFT_32BIT) &&
966 (b->flags & line_buf::LFT_32BIT));
967 __m256i v2 = _mm256_set1_epi64x(1ULL << (63 - 2));
968 const si32 *rp = r->i32, *gp = g->i32, *bp = b->i32;
969 si64 *yp = y->i64, *cbp = cb->i64, *crp = cr->i64;
970 for (int i = (repeat + 7) >> 3; i > 0; --i)
971 {
972 __m256i mr32 = _mm256_load_si256((__m256i*)rp);
973 __m256i mg32 = _mm256_load_si256((__m256i*)gp);
974 __m256i mb32 = _mm256_load_si256((__m256i*)bp);
975 __m256i mr, mg, mb, t;
976 mr = _mm256_cvtepi32_epi64(_mm256_extracti128_si256(mr32, 0));
977 mg = _mm256_cvtepi32_epi64(_mm256_extracti128_si256(mg32, 0));
978 mb = _mm256_cvtepi32_epi64(_mm256_extracti128_si256(mb32, 0));
979
980 t = _mm256_add_epi64(mr, mb);
981 t = _mm256_add_epi64(t, _mm256_slli_epi64(mg, 1));
982 _mm256_store_si256((__m256i*)yp, avx2_mm256_srai_epi64(t, 2, v2));
983 t = _mm256_sub_epi64(mb, mg);
984 _mm256_store_si256((__m256i*)cbp, t);
985 t = _mm256_sub_epi64(mr, mg);
986 _mm256_store_si256((__m256i*)crp, t);
987
988 yp += 4; cbp += 4; crp += 4;
989
990 mr = _mm256_cvtepi32_epi64(_mm256_extracti128_si256(mr32, 1));
991 mg = _mm256_cvtepi32_epi64(_mm256_extracti128_si256(mg32, 1));
992 mb = _mm256_cvtepi32_epi64(_mm256_extracti128_si256(mb32, 1));
993
994 t = _mm256_add_epi64(mr, mb);
995 t = _mm256_add_epi64(t, _mm256_slli_epi64(mg, 1));
996 _mm256_store_si256((__m256i*)yp, avx2_mm256_srai_epi64(t, 2, v2));
997 t = _mm256_sub_epi64(mb, mg);
998 _mm256_store_si256((__m256i*)cbp, t);
999 t = _mm256_sub_epi64(mr, mg);
1000 _mm256_store_si256((__m256i*)crp, t);
1001
1002 rp += 8; gp += 8; bp += 8;
1003 yp += 4; cbp += 4; crp += 4;
1004 }
1005 }
1006 }
1007
1009 void avx2_rct_backward(const line_buf *y,
1010 const line_buf *cb,
1011 const line_buf *cr,
1012 line_buf *r, line_buf *g, line_buf *b,
1013 ui32 repeat)
1014 {
1015 assert((y->flags & line_buf::LFT_INTEGER) &&
1016 (cb->flags & line_buf::LFT_INTEGER) &&
1017 (cr->flags & line_buf::LFT_INTEGER) &&
1018 (r->flags & line_buf::LFT_INTEGER) &&
1019 (g->flags & line_buf::LFT_INTEGER) &&
1020 (b->flags & line_buf::LFT_INTEGER));
1021
1022 if (y->flags & line_buf::LFT_32BIT)
1023 {
1024 assert((y->flags & line_buf::LFT_32BIT) &&
1025 (cb->flags & line_buf::LFT_32BIT) &&
1026 (cr->flags & line_buf::LFT_32BIT) &&
1027 (r->flags & line_buf::LFT_32BIT) &&
1028 (g->flags & line_buf::LFT_32BIT) &&
1029 (b->flags & line_buf::LFT_32BIT));
1030 const si32 *yp = y->i32, *cbp = cb->i32, *crp = cr->i32;
1031 si32 *rp = r->i32, *gp = g->i32, *bp = b->i32;
1032 for (int i = (repeat + 7) >> 3; i > 0; --i)
1033 {
1034 __m256i my = _mm256_load_si256((__m256i*)yp);
1035 __m256i mcb = _mm256_load_si256((__m256i*)cbp);
1036 __m256i mcr = _mm256_load_si256((__m256i*)crp);
1037
1038 __m256i t = _mm256_add_epi32(mcb, mcr);
1039 t = _mm256_sub_epi32(my, _mm256_srai_epi32(t, 2));
1040 _mm256_store_si256((__m256i*)gp, t);
1041 __m256i u = _mm256_add_epi32(mcb, t);
1042 _mm256_store_si256((__m256i*)bp, u);
1043 u = _mm256_add_epi32(mcr, t);
1044 _mm256_store_si256((__m256i*)rp, u);
1045
1046 yp += 8; cbp += 8; crp += 8;
1047 rp += 8; gp += 8; bp += 8;
1048 }
1049 }
1050 else
1051 {
1052 assert((y->flags & line_buf::LFT_64BIT) &&
1053 (cb->flags & line_buf::LFT_64BIT) &&
1054 (cr->flags & line_buf::LFT_64BIT) &&
1055 (r->flags & line_buf::LFT_32BIT) &&
1056 (g->flags & line_buf::LFT_32BIT) &&
1057 (b->flags & line_buf::LFT_32BIT));
1058 __m256i v2 = _mm256_set1_epi64x(1ULL << (63 - 2));
1059 __m256i low_bits = _mm256_set_epi64x(0, (si64)ULLONG_MAX,
1060 0, (si64)ULLONG_MAX);
1061 const si64 *yp = y->i64, *cbp = cb->i64, *crp = cr->i64;
1062 si32 *rp = r->i32, *gp = g->i32, *bp = b->i32;
1063 for (int i = (repeat + 7) >> 3; i > 0; --i)
1064 {
1065 __m256i my, mcb, mcr, tr, tg, tb;
1066 my = _mm256_load_si256((__m256i*)yp);
1067 mcb = _mm256_load_si256((__m256i*)cbp);
1068 mcr = _mm256_load_si256((__m256i*)crp);
1069
1070 tg = _mm256_add_epi64(mcb, mcr);
1071 tg = _mm256_sub_epi64(my, avx2_mm256_srai_epi64(tg, 2, v2));
1072 tb = _mm256_add_epi64(mcb, tg);
1073 tr = _mm256_add_epi64(mcr, tg);
1074
1075 __m256i mr, mg, mb;
1076 mr = _mm256_shuffle_epi32(tr, _MM_SHUFFLE(0, 0, 2, 0));
1077 mr = _mm256_and_si256(low_bits, mr);
1078 mg = _mm256_shuffle_epi32(tg, _MM_SHUFFLE(0, 0, 2, 0));
1079 mg = _mm256_and_si256(low_bits, mg);
1080 mb = _mm256_shuffle_epi32(tb, _MM_SHUFFLE(0, 0, 2, 0));
1081 mb = _mm256_and_si256(low_bits, mb);
1082
1083 yp += 4; cbp += 4; crp += 4;
1084
1085 my = _mm256_load_si256((__m256i*)yp);
1086 mcb = _mm256_load_si256((__m256i*)cbp);
1087 mcr = _mm256_load_si256((__m256i*)crp);
1088
1089 tg = _mm256_add_epi64(mcb, mcr);
1090 tg = _mm256_sub_epi64(my, avx2_mm256_srai_epi64(tg, 2, v2));
1091 tb = _mm256_add_epi64(mcb, tg);
1092 tr = _mm256_add_epi64(mcr, tg);
1093
1094 tr = _mm256_shuffle_epi32(tr, _MM_SHUFFLE(2, 0, 0, 0));
1095 tr = _mm256_andnot_si256(low_bits, tr);
1096 mr = _mm256_or_si256(mr, tr);
1097 mr = _mm256_permute4x64_epi64(mr, _MM_SHUFFLE(3, 1, 2, 0));
1098
1099 tg = _mm256_shuffle_epi32(tg, _MM_SHUFFLE(2, 0, 0, 0));
1100 tg = _mm256_andnot_si256(low_bits, tg);
1101 mg = _mm256_or_si256(mg, tg);
1102 mg = _mm256_permute4x64_epi64(mg, _MM_SHUFFLE(3, 1, 2, 0));
1103
1104 tb = _mm256_shuffle_epi32(tb, _MM_SHUFFLE(2, 0, 0, 0));
1105 tb = _mm256_andnot_si256(low_bits, tb);
1106 mb = _mm256_or_si256(mb, tb);
1107 mb = _mm256_permute4x64_epi64(mb, _MM_SHUFFLE(3, 1, 2, 0));
1108
1109 _mm256_store_si256((__m256i*)rp, mr);
1110 _mm256_store_si256((__m256i*)gp, mg);
1111 _mm256_store_si256((__m256i*)bp, mb);
1112
1113 yp += 4; cbp += 4; crp += 4;
1114 rp += 8; gp += 8; bp += 8;
1115 }
1116 }
1117 }
1118
1119 }
1120}
1121
1122#endif
ojph::line_buf::LFT_INTEGER
@ LFT_INTEGER
Definition ojph_mem.h:170
ojph::line_buf::LFT_32BIT
@ LFT_32BIT
Definition ojph_mem.h:168
ojph::line_buf::LFT_64BIT
@ LFT_64BIT
Definition ojph_mem.h:169
ojph::local::avx2_rct_forward
void avx2_rct_forward(const line_buf *r, const line_buf *g, const line_buf *b, line_buf *y, line_buf *cb, line_buf *cr, ui32 repeat)
ojph::local::avx2_rct_backward
void avx2_rct_backward(const line_buf *y, const line_buf *cb, const line_buf *cr, line_buf *r, line_buf *g, line_buf *b, ui32 repeat)
ojph::local::avx2_rev_convert
void avx2_rev_convert(const line_buf *src_line, const ui32 src_line_offset, line_buf *dst_line, const ui32 dst_line_offset, si64 shift, ui32 width)
ojph::local::avx2_irv_convert_to_float
void avx2_irv_convert_to_float(const line_buf *src_line, ui32 src_line_offset, line_buf *dst_line, ui32 bit_depth, bool is_signed, ui32 width)
ojph::local::avx2_irv_convert_to_integer_nlt
void avx2_irv_convert_to_integer_nlt(const line_buf *src_line, line_buf *dst_line, ui32 dst_line_offset, ui32 bit_depth, bool is_signed, ui32 width, const nlt_rec *rec)
ojph::local::avx2_rev_convert_nlt_type3
void avx2_rev_convert_nlt_type3(const line_buf *src_line, const ui32 src_line_offset, line_buf *dst_line, const ui32 dst_line_offset, si64 shift, ui32 width)
ojph::local::avx2_irv_convert_to_float_nlt
void avx2_irv_convert_to_float_nlt(const line_buf *src_line, ui32 src_line_offset, line_buf *dst_line, ui32 bit_depth, bool is_signed, ui32 width, const nlt_rec *rec)
ojph::local::avx2_irv_convert_to_integer
void avx2_irv_convert_to_integer(const line_buf *src_line, line_buf *dst_line, ui32 dst_line_offset, ui32 bit_depth, bool is_signed, ui32 width)
ojph::local::avx2_irv_convert_to_float_nlt_type3
void avx2_irv_convert_to_float_nlt_type3(const line_buf *src_line, ui32 src_line_offset, line_buf *dst_line, ui32 bit_depth, bool is_signed, ui32 width)
ojph::local::avx2_irv_convert_to_integer_nlt_type3
void avx2_irv_convert_to_integer_nlt_type3(const line_buf *src_line, line_buf *dst_line, ui32 dst_line_offset, ui32 bit_depth, bool is_signed, ui32 width)
ojph::si64
int64_t si64
Definition ojph_defs.h:57
ojph::ui16
uint16_t ui16
Definition ojph_defs.h:52
ojph::si32
int32_t si32
Definition ojph_defs.h:55
ojph::ui32
uint32_t ui32
Definition ojph_defs.h:54
ojph_arch.h
ojph_colour.h
ojph_defs.h
ojph_unused
#define ojph_unused(x)
Definition ojph_defs.h:78
ojph_mem.h
ojph_params.h
ojph_params_local.h
ojph::local::nlt_rec
Definition ojph_params_local.h:852
ojph::local::nlt_rec::nonlinearity
ojph::param_nlt::nonlinearity nonlinearity
Definition ojph_params_local.h:853