xf86-video-intel: 4 commits - src/sna/blt.c src/sna/sna_display.c src/sna/sna_dri2.c
Chris Wilson
ickle at kemper.freedesktop.org
Fri Jul 29 16:43:30 UTC 2016
src/sna/blt.c | 1258 +++++++++++++++++++++++---------------------------
src/sna/sna_display.c | 15
src/sna/sna_dri2.c | 6
3 files changed, 610 insertions(+), 669 deletions(-)
New commits:
commit 49daf5df124b5ae6c7508e934768c292f4143040
Author: Chris Wilson <chris at chris-wilson.co.uk>
Date: Thu Jul 28 18:33:41 2016 +0100
sna: Use GCC pragma to enable SSE2 blt routines
Rather than use per-function attributes, if we set the target for the
block using a pragma we can compile the SSE2 routines on 32bit ISA as
well.
Signed-off-by: Chris Wilson <chris at chris-wilson.co.uk>
diff --git a/src/sna/blt.c b/src/sna/blt.c
index ab7bd22..cb90437 100644
--- a/src/sna/blt.c
+++ b/src/sna/blt.c
@@ -32,89 +32,21 @@
#include "sna.h"
#include <pixman.h>
-#if __x86_64__
-#define USE_SSE2 1
-#endif
-
-#if USE_SSE2
+#if defined(sse2)
+#pragma GCC push_options
+#pragma GCC target("sse2,inline-all-stringops,fpmath=sse")
+#pragma GCC optimize("Ofast")
#include <xmmintrin.h>
#if __x86_64__
#define have_sse2() 1
#else
-enum {
- MMX = 0x1,
- MMX_EXTENSIONS = 0x2,
- SSE = 0x6,
- SSE2 = 0x8,
- CMOV = 0x10
-};
-
-#ifdef __GNUC__
-static unsigned int
-detect_cpu_features(void)
-{
- unsigned int features;
- unsigned int result = 0;
-
- char vendor[13];
- vendor[0] = 0;
- vendor[12] = 0;
-
- asm (
- "pushf\n"
- "pop %%eax\n"
- "mov %%eax, %%ecx\n"
- "xor $0x00200000, %%eax\n"
- "push %%eax\n"
- "popf\n"
- "pushf\n"
- "pop %%eax\n"
- "mov $0x0, %%edx\n"
- "xor %%ecx, %%eax\n"
- "jz 1f\n"
-
- "mov $0x00000000, %%eax\n"
- "push %%ebx\n"
- "cpuid\n"
- "mov %%ebx, %%eax\n"
- "pop %%ebx\n"
- "mov %%eax, %1\n"
- "mov %%edx, %2\n"
- "mov %%ecx, %3\n"
- "mov $0x00000001, %%eax\n"
- "push %%ebx\n"
- "cpuid\n"
- "pop %%ebx\n"
- "1:\n"
- "mov %%edx, %0\n"
- : "=r" (result), "=m" (vendor[0]), "=m" (vendor[4]), "=m" (vendor[8])
- :: "%eax", "%ecx", "%edx");
-
- features = 0;
- if (result) {
- /* result now contains the standard feature bits */
- if (result & (1 << 15))
- features |= CMOV;
- if (result & (1 << 23))
- features |= MMX;
- if (result & (1 << 25))
- features |= SSE;
- if (result & (1 << 26))
- features |= SSE2;
- }
- return features;
-}
-#else
-static unsigned int detect_cpu_features(void) { return 0; }
-#endif
-
static bool have_sse2(void)
{
static int sse2_present = -1;
if (sse2_present == -1)
- sse2_present = detect_cpu_features() & SSE2;
+ sse2_present = sna_cpu_detect() & SSE2;
return sse2_present;
}
@@ -149,97 +81,152 @@ xmm_save_128u(__m128i *dst, __m128i data)
{
_mm_storeu_si128(dst, data);
}
-#endif
-fast void
-memcpy_blt(const void *src, void *dst, int bpp,
- int32_t src_stride, int32_t dst_stride,
- int16_t src_x, int16_t src_y,
- int16_t dst_x, int16_t dst_y,
- uint16_t width, uint16_t height)
+static force_inline void
+to_sse128xN(uint8_t *dst, const uint8_t *src, int bytes)
{
- const uint8_t *src_bytes;
- uint8_t *dst_bytes;
- int byte_width;
-
- assert(src);
- assert(dst);
- assert(width && height);
- assert(bpp >= 8);
- assert(width*bpp <= 8*src_stride);
- assert(width*bpp <= 8*dst_stride);
+ int i;
- DBG(("%s: src=(%d, %d), dst=(%d, %d), size=%dx%d, pitch=%d/%d\n",
- __FUNCTION__, src_x, src_y, dst_x, dst_y, width, height, src_stride, dst_stride));
+ for (i = 0; i < bytes / 128; i++) {
+ __m128i xmm0, xmm1, xmm2, xmm3;
+ __m128i xmm4, xmm5, xmm6, xmm7;
- bpp /= 8;
+ xmm0 = xmm_load_128u((const __m128i*)src + 0);
+ xmm1 = xmm_load_128u((const __m128i*)src + 1);
+ xmm2 = xmm_load_128u((const __m128i*)src + 2);
+ xmm3 = xmm_load_128u((const __m128i*)src + 3);
+ xmm4 = xmm_load_128u((const __m128i*)src + 4);
+ xmm5 = xmm_load_128u((const __m128i*)src + 5);
+ xmm6 = xmm_load_128u((const __m128i*)src + 6);
+ xmm7 = xmm_load_128u((const __m128i*)src + 7);
- src_bytes = (const uint8_t *)src + src_stride * src_y + src_x * bpp;
- dst_bytes = (uint8_t *)dst + dst_stride * dst_y + dst_x * bpp;
+ xmm_save_128((__m128i*)dst + 0, xmm0);
+ xmm_save_128((__m128i*)dst + 1, xmm1);
+ xmm_save_128((__m128i*)dst + 2, xmm2);
+ xmm_save_128((__m128i*)dst + 3, xmm3);
+ xmm_save_128((__m128i*)dst + 4, xmm4);
+ xmm_save_128((__m128i*)dst + 5, xmm5);
+ xmm_save_128((__m128i*)dst + 6, xmm6);
+ xmm_save_128((__m128i*)dst + 7, xmm7);
- byte_width = width * bpp;
- if (byte_width == src_stride && byte_width == dst_stride) {
- byte_width *= height;
- height = 1;
+ dst += 128;
+ src += 128;
}
+}
- switch (byte_width) {
- case 1:
- do {
- *dst_bytes = *src_bytes;
- src_bytes += src_stride;
- dst_bytes += dst_stride;
- } while (--height);
- break;
+static force_inline void
+to_sse64(uint8_t *dst, const uint8_t *src)
+{
+ __m128i xmm1, xmm2, xmm3, xmm4;
- case 2:
- do {
- *(uint16_t *)dst_bytes = *(const uint16_t *)src_bytes;
- src_bytes += src_stride;
- dst_bytes += dst_stride;
- } while (--height);
- break;
+ xmm1 = xmm_load_128u((const __m128i*)src + 0);
+ xmm2 = xmm_load_128u((const __m128i*)src + 1);
+ xmm3 = xmm_load_128u((const __m128i*)src + 2);
+ xmm4 = xmm_load_128u((const __m128i*)src + 3);
- case 4:
- do {
- *(uint32_t *)dst_bytes = *(const uint32_t *)src_bytes;
- src_bytes += src_stride;
- dst_bytes += dst_stride;
- } while (--height);
- break;
+ xmm_save_128((__m128i*)dst + 0, xmm1);
+ xmm_save_128((__m128i*)dst + 1, xmm2);
+ xmm_save_128((__m128i*)dst + 2, xmm3);
+ xmm_save_128((__m128i*)dst + 3, xmm4);
+}
- case 8:
- do {
- *(uint64_t *)dst_bytes = *(const uint64_t *)src_bytes;
- src_bytes += src_stride;
- dst_bytes += dst_stride;
- } while (--height);
- break;
- case 16:
- do {
- ((uint64_t *)dst_bytes)[0] = ((const uint64_t *)src_bytes)[0];
- ((uint64_t *)dst_bytes)[1] = ((const uint64_t *)src_bytes)[1];
- src_bytes += src_stride;
- dst_bytes += dst_stride;
- } while (--height);
- break;
+static force_inline void
+to_sse32(uint8_t *dst, const uint8_t *src)
+{
+ __m128i xmm1, xmm2;
- default:
- do {
- memcpy(dst_bytes, src_bytes, byte_width);
- src_bytes += src_stride;
- dst_bytes += dst_stride;
- } while (--height);
- break;
+ xmm1 = xmm_load_128u((const __m128i*)src + 0);
+ xmm2 = xmm_load_128u((const __m128i*)src + 1);
+
+ xmm_save_128((__m128i*)dst + 0, xmm1);
+ xmm_save_128((__m128i*)dst + 1, xmm2);
+}
+
+static force_inline void
+to_sse16(uint8_t *dst, const uint8_t *src)
+{
+ xmm_save_128((__m128i*)dst, xmm_load_128u((const __m128i*)src));
+}
+
+static void to_memcpy(uint8_t *dst, const uint8_t *src, unsigned len)
+{
+ assert(len);
+ if ((uintptr_t)dst & 15) {
+ if (len <= 16 - ((uintptr_t)dst & 15)) {
+ memcpy(dst, src, len);
+ return;
+ }
+
+ if ((uintptr_t)dst & 1) {
+ assert(len >= 1);
+ *dst++ = *src++;
+ len--;
+ }
+ if ((uintptr_t)dst & 2) {
+ assert(((uintptr_t)dst & 1) == 0);
+ assert(len >= 2);
+ *(uint16_t *)dst = *(const uint16_t *)src;
+ dst += 2;
+ src += 2;
+ len -= 2;
+ }
+ if ((uintptr_t)dst & 4) {
+ assert(((uintptr_t)dst & 3) == 0);
+ assert(len >= 4);
+ *(uint32_t *)dst = *(const uint32_t *)src;
+ dst += 4;
+ src += 4;
+ len -= 4;
+ }
+ if ((uintptr_t)dst & 8) {
+ assert(((uintptr_t)dst & 7) == 0);
+ assert(len >= 8);
+ *(uint64_t *)dst = *(const uint64_t *)src;
+ dst += 8;
+ src += 8;
+ len -= 8;
+ }
+ }
+
+ assert(((uintptr_t)dst & 15) == 0);
+ while (len >= 64) {
+ to_sse64(dst, src);
+ dst += 64;
+ src += 64;
+ len -= 64;
+ }
+ if (len == 0)
+ return;
+
+ if (len & 32) {
+ to_sse32(dst, src);
+ dst += 32;
+ src += 32;
+ }
+ if (len & 16) {
+ to_sse16(dst, src);
+ dst += 16;
+ src += 16;
}
+ if (len & 8) {
+ *(uint64_t *)dst = *(uint64_t *)src;
+ dst += 8;
+ src += 8;
+ }
+ if (len & 4) {
+ *(uint32_t *)dst = *(uint32_t *)src;
+ dst += 4;
+ src += 4;
+ }
+ memcpy(dst, src, len & 3);
}
-static fast_memcpy void
-memcpy_to_tiled_x__swizzle_0(const void *src, void *dst, int bpp,
- int32_t src_stride, int32_t dst_stride,
- int16_t src_x, int16_t src_y,
- int16_t dst_x, int16_t dst_y,
- uint16_t width, uint16_t height)
+static void
+memcpy_to_tiled_x__swizzle_0__sse2(const void *src, void *dst, int bpp,
+ int32_t src_stride, int32_t dst_stride,
+ int16_t src_x, int16_t src_y,
+ int16_t dst_x, int16_t dst_y,
+ uint16_t width, uint16_t height)
{
const unsigned tile_width = 512;
const unsigned tile_height = 8;
@@ -250,189 +237,112 @@ memcpy_to_tiled_x__swizzle_0(const void *src, void *dst, int bpp,
const unsigned tile_shift = ffs(tile_pixels) - 1;
const unsigned tile_mask = tile_pixels - 1;
+ unsigned offset_x, length_x;
+
DBG(("%s(bpp=%d): src=(%d, %d), dst=(%d, %d), size=%dx%d, pitch=%d/%d\n",
__FUNCTION__, bpp, src_x, src_y, dst_x, dst_y, width, height, src_stride, dst_stride));
assert(src != dst);
if (src_x | src_y)
src = (const uint8_t *)src + src_y * src_stride + src_x * cpp;
- assert(src_stride >= width * cpp);
- src_stride -= width * cpp;
+ width *= cpp;
+ assert(src_stride >= width);
+
+ if (dst_x & tile_mask) {
+ offset_x = (dst_x & tile_mask) * cpp;
+ length_x = min(tile_width - offset_x, width);
+ } else
+ length_x = 0;
+ dst = (uint8_t *)dst + (dst_x >> tile_shift) * tile_size;
while (height--) {
- unsigned w = width * cpp;
+ unsigned w = width;
+ const uint8_t *src_row = src;
uint8_t *tile_row = dst;
+ src = (const uint8_t *)src + src_stride;
+
tile_row += dst_y / tile_height * dst_stride * tile_height;
tile_row += (dst_y & (tile_height-1)) * tile_width;
- if (dst_x) {
- tile_row += (dst_x >> tile_shift) * tile_size;
- if (dst_x & tile_mask) {
- const unsigned x = (dst_x & tile_mask) * cpp;
- const unsigned len = min(tile_width - x, w);
- memcpy(assume_misaligned(tile_row + x, tile_width, x),
- src, len);
+ dst_y++;
- tile_row += tile_size;
- src = (const uint8_t *)src + len;
- w -= len;
- }
+ if (length_x) {
+ to_memcpy(tile_row + offset_x, src_row, length_x);
+
+ tile_row += tile_size;
+ src_row = (const uint8_t *)src_row + length_x;
+ w -= length_x;
}
while (w >= tile_width) {
- memcpy(assume_aligned(tile_row, tile_width),
- src, tile_width);
+ assert(((uintptr_t)tile_row & (tile_width - 1)) == 0);
+ to_sse128xN(assume_aligned(tile_row, tile_width),
+ src_row, tile_width);
tile_row += tile_size;
- src = (const uint8_t *)src + tile_width;
+ src_row = (const uint8_t *)src_row + tile_width;
w -= tile_width;
}
- memcpy(assume_aligned(tile_row, tile_width), src, w);
- src = (const uint8_t *)src + src_stride + w;
- dst_y++;
- }
-}
-
-static fast_memcpy void
-memcpy_from_tiled_x__swizzle_0(const void *src, void *dst, int bpp,
- int32_t src_stride, int32_t dst_stride,
- int16_t src_x, int16_t src_y,
- int16_t dst_x, int16_t dst_y,
- uint16_t width, uint16_t height)
-{
- const unsigned tile_width = 512;
- const unsigned tile_height = 8;
- const unsigned tile_size = 4096;
-
- const unsigned cpp = bpp / 8;
- const unsigned tile_pixels = tile_width / cpp;
- const unsigned tile_shift = ffs(tile_pixels) - 1;
- const unsigned tile_mask = tile_pixels - 1;
-
- DBG(("%s(bpp=%d): src=(%d, %d), dst=(%d, %d), size=%dx%d, pitch=%d/%d\n",
- __FUNCTION__, bpp, src_x, src_y, dst_x, dst_y, width, height, src_stride, dst_stride));
- assert(src != dst);
-
- if (dst_x | dst_y)
- dst = (uint8_t *)dst + dst_y * dst_stride + dst_x * cpp;
- assert(dst_stride >= width * cpp);
- dst_stride -= width * cpp;
-
- while (height--) {
- unsigned w = width * cpp;
- const uint8_t *tile_row = src;
-
- tile_row += src_y / tile_height * src_stride * tile_height;
- tile_row += (src_y & (tile_height-1)) * tile_width;
- if (src_x) {
- tile_row += (src_x >> tile_shift) * tile_size;
- if (src_x & tile_mask) {
- const unsigned x = (src_x & tile_mask) * cpp;
- const unsigned len = min(tile_width - x, w);
- memcpy(dst, assume_misaligned(tile_row + x, tile_width, x), len);
-
- tile_row += tile_size;
- dst = (uint8_t *)dst + len;
- w -= len;
- }
- }
- while (w >= tile_width) {
- memcpy(dst,
- assume_aligned(tile_row, tile_width),
- tile_width);
-
- tile_row += tile_size;
- dst = (uint8_t *)dst + tile_width;
- w -= tile_width;
+ if (w) {
+ assert(((uintptr_t)tile_row & (tile_width - 1)) == 0);
+ to_memcpy(assume_aligned(tile_row, tile_width),
+ src_row, w);
}
- memcpy(dst, assume_aligned(tile_row, tile_width), w);
- dst = (uint8_t *)dst + dst_stride + w;
- src_y++;
}
}
-static fast_memcpy void
-memcpy_between_tiled_x__swizzle_0(const void *src, void *dst, int bpp,
- int32_t src_stride, int32_t dst_stride,
- int16_t src_x, int16_t src_y,
- int16_t dst_x, int16_t dst_y,
- uint16_t width, uint16_t height)
+static force_inline void
+from_sse128xNu(uint8_t *dst, const uint8_t *src, int bytes)
{
- const unsigned tile_width = 512;
- const unsigned tile_height = 8;
- const unsigned tile_size = 4096;
-
- const unsigned cpp = bpp / 8;
- const unsigned tile_pixels = tile_width / cpp;
- const unsigned tile_shift = ffs(tile_pixels) - 1;
- const unsigned tile_mask = tile_pixels - 1;
-
- DBG(("%s(bpp=%d): src=(%d, %d), dst=(%d, %d), size=%dx%d, pitch=%d/%d\n",
- __FUNCTION__, bpp, src_x, src_y, dst_x, dst_y, width, height, src_stride, dst_stride));
- assert(src != dst);
- assert((dst_x & tile_mask) == (src_x & tile_mask));
-
- while (height--) {
- unsigned w = width * cpp;
- uint8_t *dst_row = dst;
- const uint8_t *src_row = src;
-
- dst_row += dst_y / tile_height * dst_stride * tile_height;
- dst_row += (dst_y & (tile_height-1)) * tile_width;
- if (dst_x)
- dst_row += (dst_x >> tile_shift) * tile_size;
- dst_y++;
+ int i;
- src_row += src_y / tile_height * src_stride * tile_height;
- src_row += (src_y & (tile_height-1)) * tile_width;
- if (src_x)
- src_row += (src_x >> tile_shift) * tile_size;
- src_y++;
+ assert(((uintptr_t)src & 15) == 0);
- if (dst_x & tile_mask) {
- const unsigned x = (dst_x & tile_mask) * cpp;
- const unsigned len = min(tile_width - x, w);
+ for (i = 0; i < bytes / 128; i++) {
+ __m128i xmm0, xmm1, xmm2, xmm3;
+ __m128i xmm4, xmm5, xmm6, xmm7;
- memcpy(assume_misaligned(dst_row + x, tile_width, x),
- assume_misaligned(src_row + x, tile_width, x),
- len);
+ xmm0 = xmm_load_128((const __m128i*)src + 0);
+ xmm1 = xmm_load_128((const __m128i*)src + 1);
+ xmm2 = xmm_load_128((const __m128i*)src + 2);
+ xmm3 = xmm_load_128((const __m128i*)src + 3);
+ xmm4 = xmm_load_128((const __m128i*)src + 4);
+ xmm5 = xmm_load_128((const __m128i*)src + 5);
+ xmm6 = xmm_load_128((const __m128i*)src + 6);
+ xmm7 = xmm_load_128((const __m128i*)src + 7);
- dst_row += tile_size;
- src_row += tile_size;
- w -= len;
- }
+ xmm_save_128u((__m128i*)dst + 0, xmm0);
+ xmm_save_128u((__m128i*)dst + 1, xmm1);
+ xmm_save_128u((__m128i*)dst + 2, xmm2);
+ xmm_save_128u((__m128i*)dst + 3, xmm3);
+ xmm_save_128u((__m128i*)dst + 4, xmm4);
+ xmm_save_128u((__m128i*)dst + 5, xmm5);
+ xmm_save_128u((__m128i*)dst + 6, xmm6);
+ xmm_save_128u((__m128i*)dst + 7, xmm7);
- while (w >= tile_width) {
- memcpy(assume_aligned(dst_row, tile_width),
- assume_aligned(src_row, tile_width),
- tile_width);
- dst_row += tile_size;
- src_row += tile_size;
- w -= tile_width;
- }
- memcpy(assume_aligned(dst_row, tile_width),
- assume_aligned(src_row, tile_width),
- w);
+ dst += 128;
+ src += 128;
}
}
-#if defined(sse2) && defined(__x86_64__)
-
-sse2 static force_inline void
-to_sse128xN(uint8_t *dst, const uint8_t *src, int bytes)
+static force_inline void
+from_sse128xNa(uint8_t *dst, const uint8_t *src, int bytes)
{
int i;
+ assert(((uintptr_t)dst & 15) == 0);
+ assert(((uintptr_t)src & 15) == 0);
+
for (i = 0; i < bytes / 128; i++) {
__m128i xmm0, xmm1, xmm2, xmm3;
__m128i xmm4, xmm5, xmm6, xmm7;
- xmm0 = xmm_load_128u((const __m128i*)src + 0);
- xmm1 = xmm_load_128u((const __m128i*)src + 1);
- xmm2 = xmm_load_128u((const __m128i*)src + 2);
- xmm3 = xmm_load_128u((const __m128i*)src + 3);
- xmm4 = xmm_load_128u((const __m128i*)src + 4);
- xmm5 = xmm_load_128u((const __m128i*)src + 5);
- xmm6 = xmm_load_128u((const __m128i*)src + 6);
- xmm7 = xmm_load_128u((const __m128i*)src + 7);
+ xmm0 = xmm_load_128((const __m128i*)src + 0);
+ xmm1 = xmm_load_128((const __m128i*)src + 1);
+ xmm2 = xmm_load_128((const __m128i*)src + 2);
+ xmm3 = xmm_load_128((const __m128i*)src + 3);
+ xmm4 = xmm_load_128((const __m128i*)src + 4);
+ xmm5 = xmm_load_128((const __m128i*)src + 5);
+ xmm6 = xmm_load_128((const __m128i*)src + 6);
+ xmm7 = xmm_load_128((const __m128i*)src + 7);
xmm_save_128((__m128i*)dst + 0, xmm0);
xmm_save_128((__m128i*)dst + 1, xmm1);
@@ -448,15 +358,36 @@ to_sse128xN(uint8_t *dst, const uint8_t *src, int bytes)
}
}
-sse2 static force_inline void
-to_sse64(uint8_t *dst, const uint8_t *src)
+static force_inline void
+from_sse64u(uint8_t *dst, const uint8_t *src)
{
__m128i xmm1, xmm2, xmm3, xmm4;
- xmm1 = xmm_load_128u((const __m128i*)src + 0);
- xmm2 = xmm_load_128u((const __m128i*)src + 1);
- xmm3 = xmm_load_128u((const __m128i*)src + 2);
- xmm4 = xmm_load_128u((const __m128i*)src + 3);
+ assert(((uintptr_t)src & 15) == 0);
+
+ xmm1 = xmm_load_128((const __m128i*)src + 0);
+ xmm2 = xmm_load_128((const __m128i*)src + 1);
+ xmm3 = xmm_load_128((const __m128i*)src + 2);
+ xmm4 = xmm_load_128((const __m128i*)src + 3);
+
+ xmm_save_128u((__m128i*)dst + 0, xmm1);
+ xmm_save_128u((__m128i*)dst + 1, xmm2);
+ xmm_save_128u((__m128i*)dst + 2, xmm3);
+ xmm_save_128u((__m128i*)dst + 3, xmm4);
+}
+
+static force_inline void
+from_sse64a(uint8_t *dst, const uint8_t *src)
+{
+ __m128i xmm1, xmm2, xmm3, xmm4;
+
+ assert(((uintptr_t)dst & 15) == 0);
+ assert(((uintptr_t)src & 15) == 0);
+
+ xmm1 = xmm_load_128((const __m128i*)src + 0);
+ xmm2 = xmm_load_128((const __m128i*)src + 1);
+ xmm3 = xmm_load_128((const __m128i*)src + 2);
+ xmm4 = xmm_load_128((const __m128i*)src + 3);
xmm_save_128((__m128i*)dst + 0, xmm1);
xmm_save_128((__m128i*)dst + 1, xmm2);
@@ -464,103 +395,56 @@ to_sse64(uint8_t *dst, const uint8_t *src)
xmm_save_128((__m128i*)dst + 3, xmm4);
}
-sse2 static force_inline void
-to_sse32(uint8_t *dst, const uint8_t *src)
+static force_inline void
+from_sse32u(uint8_t *dst, const uint8_t *src)
{
__m128i xmm1, xmm2;
- xmm1 = xmm_load_128u((const __m128i*)src + 0);
- xmm2 = xmm_load_128u((const __m128i*)src + 1);
+ xmm1 = xmm_load_128((const __m128i*)src + 0);
+ xmm2 = xmm_load_128((const __m128i*)src + 1);
+
+ xmm_save_128u((__m128i*)dst + 0, xmm1);
+ xmm_save_128u((__m128i*)dst + 1, xmm2);
+}
+
+static force_inline void
+from_sse32a(uint8_t *dst, const uint8_t *src)
+{
+ __m128i xmm1, xmm2;
+
+ assert(((uintptr_t)dst & 15) == 0);
+ assert(((uintptr_t)src & 15) == 0);
+
+ xmm1 = xmm_load_128((const __m128i*)src + 0);
+ xmm2 = xmm_load_128((const __m128i*)src + 1);
xmm_save_128((__m128i*)dst + 0, xmm1);
xmm_save_128((__m128i*)dst + 1, xmm2);
}
-sse2 static force_inline void
-to_sse16(uint8_t *dst, const uint8_t *src)
+static force_inline void
+from_sse16u(uint8_t *dst, const uint8_t *src)
{
- xmm_save_128((__m128i*)dst, xmm_load_128u((const __m128i*)src));
+ assert(((uintptr_t)src & 15) == 0);
+
+ xmm_save_128u((__m128i*)dst, xmm_load_128((const __m128i*)src));
}
-sse2 static void to_memcpy(uint8_t *dst, const uint8_t *src, unsigned len)
+static force_inline void
+from_sse16a(uint8_t *dst, const uint8_t *src)
{
- assert(len);
- if ((uintptr_t)dst & 15) {
- if (len <= 16 - ((uintptr_t)dst & 15)) {
- memcpy(dst, src, len);
- return;
- }
+ assert(((uintptr_t)dst & 15) == 0);
+ assert(((uintptr_t)src & 15) == 0);
- if ((uintptr_t)dst & 1) {
- assert(len >= 1);
- *dst++ = *src++;
- len--;
- }
- if ((uintptr_t)dst & 2) {
- assert(((uintptr_t)dst & 1) == 0);
- assert(len >= 2);
- *(uint16_t *)dst = *(const uint16_t *)src;
- dst += 2;
- src += 2;
- len -= 2;
- }
- if ((uintptr_t)dst & 4) {
- assert(((uintptr_t)dst & 3) == 0);
- assert(len >= 4);
- *(uint32_t *)dst = *(const uint32_t *)src;
- dst += 4;
- src += 4;
- len -= 4;
- }
- if ((uintptr_t)dst & 8) {
- assert(((uintptr_t)dst & 7) == 0);
- assert(len >= 8);
- *(uint64_t *)dst = *(const uint64_t *)src;
- dst += 8;
- src += 8;
- len -= 8;
- }
- }
-
- assert(((uintptr_t)dst & 15) == 0);
- while (len >= 64) {
- to_sse64(dst, src);
- dst += 64;
- src += 64;
- len -= 64;
- }
- if (len == 0)
- return;
-
- if (len & 32) {
- to_sse32(dst, src);
- dst += 32;
- src += 32;
- }
- if (len & 16) {
- to_sse16(dst, src);
- dst += 16;
- src += 16;
- }
- if (len & 8) {
- *(uint64_t *)dst = *(uint64_t *)src;
- dst += 8;
- src += 8;
- }
- if (len & 4) {
- *(uint32_t *)dst = *(uint32_t *)src;
- dst += 4;
- src += 4;
- }
- memcpy(dst, src, len & 3);
+ xmm_save_128((__m128i*)dst, xmm_load_128((const __m128i*)src));
}
-sse2 static fast_memcpy void
-memcpy_to_tiled_x__swizzle_0__sse2(const void *src, void *dst, int bpp,
- int32_t src_stride, int32_t dst_stride,
- int16_t src_x, int16_t src_y,
- int16_t dst_x, int16_t dst_y,
- uint16_t width, uint16_t height)
+static void
+memcpy_from_tiled_x__swizzle_0__sse2(const void *src, void *dst, int bpp,
+ int32_t src_stride, int32_t dst_stride,
+ int16_t src_x, int16_t src_y,
+ int16_t dst_x, int16_t dst_y,
+ uint16_t width, uint16_t height)
{
const unsigned tile_width = 512;
const unsigned tile_height = 8;
@@ -571,214 +455,331 @@ memcpy_to_tiled_x__swizzle_0__sse2(const void *src, void *dst, int bpp,
const unsigned tile_shift = ffs(tile_pixels) - 1;
const unsigned tile_mask = tile_pixels - 1;
- unsigned offset_x, length_x;
+ unsigned length_x, offset_x;
DBG(("%s(bpp=%d): src=(%d, %d), dst=(%d, %d), size=%dx%d, pitch=%d/%d\n",
__FUNCTION__, bpp, src_x, src_y, dst_x, dst_y, width, height, src_stride, dst_stride));
assert(src != dst);
- if (src_x | src_y)
- src = (const uint8_t *)src + src_y * src_stride + src_x * cpp;
+ if (dst_x | dst_y)
+ dst = (uint8_t *)dst + dst_y * dst_stride + dst_x * cpp;
width *= cpp;
- assert(src_stride >= width);
-
- if (dst_x & tile_mask) {
- offset_x = (dst_x & tile_mask) * cpp;
+ assert(dst_stride >= width);
+ if (src_x & tile_mask) {
+ offset_x = (src_x & tile_mask) * cpp;
length_x = min(tile_width - offset_x, width);
- } else
- length_x = 0;
- dst = (uint8_t *)dst + (dst_x >> tile_shift) * tile_size;
+ dst_stride -= width;
+ dst_stride += (width - length_x) & 15;
+ } else {
+ offset_x = 0;
+ dst_stride -= width & ~15;
+ }
+ assert(dst_stride >= 0);
+ src = (const uint8_t *)src + (src_x >> tile_shift) * tile_size;
while (height--) {
unsigned w = width;
- const uint8_t *src_row = src;
- uint8_t *tile_row = dst;
-
- src = (const uint8_t *)src + src_stride;
-
- tile_row += dst_y / tile_height * dst_stride * tile_height;
- tile_row += (dst_y & (tile_height-1)) * tile_width;
- dst_y++;
+ const uint8_t *tile_row = src;
- if (length_x) {
- to_memcpy(tile_row + offset_x, src_row, length_x);
+ tile_row += src_y / tile_height * src_stride * tile_height;
+ tile_row += (src_y & (tile_height-1)) * tile_width;
+ src_y++;
+ if (offset_x) {
+ memcpy(dst, tile_row + offset_x, length_x);
tile_row += tile_size;
- src_row = (const uint8_t *)src_row + length_x;
+ dst = (uint8_t *)dst + length_x;
w -= length_x;
}
- while (w >= tile_width) {
- assert(((uintptr_t)tile_row & (tile_width - 1)) == 0);
- to_sse128xN(assume_aligned(tile_row, tile_width),
- src_row, tile_width);
- tile_row += tile_size;
- src_row = (const uint8_t *)src_row + tile_width;
- w -= tile_width;
- }
- if (w) {
- assert(((uintptr_t)tile_row & (tile_width - 1)) == 0);
- to_memcpy(assume_aligned(tile_row, tile_width),
- src_row, w);
+
+ if ((uintptr_t)dst & 15) {
+ while (w >= tile_width) {
+ from_sse128xNu(dst,
+ assume_aligned(tile_row, tile_width),
+ tile_width);
+ tile_row += tile_size;
+ dst = (uint8_t *)dst + tile_width;
+ w -= tile_width;
+ }
+ while (w >= 64) {
+ from_sse64u(dst, tile_row);
+ tile_row += 64;
+ dst = (uint8_t *)dst + 64;
+ w -= 64;
+ }
+ if (w & 32) {
+ from_sse32u(dst, tile_row);
+ tile_row += 32;
+ dst = (uint8_t *)dst + 32;
+ }
+ if (w & 16) {
+ from_sse16u(dst, tile_row);
+ tile_row += 16;
+ dst = (uint8_t *)dst + 16;
+ }
+ memcpy(dst, assume_aligned(tile_row, 16), w & 15);
+ } else {
+ while (w >= tile_width) {
+ from_sse128xNa(assume_aligned(dst, 16),
+ assume_aligned(tile_row, tile_width),
+ tile_width);
+ tile_row += tile_size;
+ dst = (uint8_t *)dst + tile_width;
+ w -= tile_width;
+ }
+ while (w >= 64) {
+ from_sse64a(dst, tile_row);
+ tile_row += 64;
+ dst = (uint8_t *)dst + 64;
+ w -= 64;
+ }
+ if (w & 32) {
+ from_sse32a(dst, tile_row);
+ tile_row += 32;
+ dst = (uint8_t *)dst + 32;
+ }
+ if (w & 16) {
+ from_sse16a(dst, tile_row);
+ tile_row += 16;
+ dst = (uint8_t *)dst + 16;
+ }
+ memcpy(assume_aligned(dst, 16),
+ assume_aligned(tile_row, 16),
+ w & 15);
}
+ dst = (uint8_t *)dst + dst_stride;
}
}
-sse2 static force_inline void
-from_sse128xNu(uint8_t *dst, const uint8_t *src, int bytes)
+static void
+memcpy_between_tiled_x__swizzle_0__sse2(const void *src, void *dst, int bpp,
+ int32_t src_stride, int32_t dst_stride,
+ int16_t src_x, int16_t src_y,
+ int16_t dst_x, int16_t dst_y,
+ uint16_t width, uint16_t height)
{
- int i;
-
- assert(((uintptr_t)src & 15) == 0);
+ const unsigned tile_width = 512;
+ const unsigned tile_height = 8;
+ const unsigned tile_size = 4096;
- for (i = 0; i < bytes / 128; i++) {
- __m128i xmm0, xmm1, xmm2, xmm3;
- __m128i xmm4, xmm5, xmm6, xmm7;
+ const unsigned cpp = bpp / 8;
+ const unsigned tile_pixels = tile_width / cpp;
+ const unsigned tile_shift = ffs(tile_pixels) - 1;
+ const unsigned tile_mask = tile_pixels - 1;
- xmm0 = xmm_load_128((const __m128i*)src + 0);
- xmm1 = xmm_load_128((const __m128i*)src + 1);
- xmm2 = xmm_load_128((const __m128i*)src + 2);
- xmm3 = xmm_load_128((const __m128i*)src + 3);
- xmm4 = xmm_load_128((const __m128i*)src + 4);
- xmm5 = xmm_load_128((const __m128i*)src + 5);
- xmm6 = xmm_load_128((const __m128i*)src + 6);
- xmm7 = xmm_load_128((const __m128i*)src + 7);
+ unsigned ox, lx;
- xmm_save_128u((__m128i*)dst + 0, xmm0);
- xmm_save_128u((__m128i*)dst + 1, xmm1);
- xmm_save_128u((__m128i*)dst + 2, xmm2);
- xmm_save_128u((__m128i*)dst + 3, xmm3);
- xmm_save_128u((__m128i*)dst + 4, xmm4);
- xmm_save_128u((__m128i*)dst + 5, xmm5);
- xmm_save_128u((__m128i*)dst + 6, xmm6);
- xmm_save_128u((__m128i*)dst + 7, xmm7);
+ DBG(("%s(bpp=%d): src=(%d, %d), dst=(%d, %d), size=%dx%d, pitch=%d/%d\n",
+ __FUNCTION__, bpp, src_x, src_y, dst_x, dst_y, width, height, src_stride, dst_stride));
+ assert(src != dst);
- dst += 128;
- src += 128;
- }
-}
+ width *= cpp;
+ dst_stride *= tile_height;
+ src_stride *= tile_height;
-sse2 static force_inline void
-from_sse128xNa(uint8_t *dst, const uint8_t *src, int bytes)
-{
- int i;
+ assert((dst_x & tile_mask) == (src_x & tile_mask));
+ if (dst_x & tile_mask) {
+ ox = (dst_x & tile_mask) * cpp;
+ lx = min(tile_width - ox, width);
+ assert(lx != 0);
+ } else
+ lx = 0;
- assert(((uintptr_t)dst & 15) == 0);
- assert(((uintptr_t)src & 15) == 0);
+ if (dst_x)
+ dst = (uint8_t *)dst + (dst_x >> tile_shift) * tile_size;
+ if (src_x)
+ src = (const uint8_t *)src + (src_x >> tile_shift) * tile_size;
- for (i = 0; i < bytes / 128; i++) {
- __m128i xmm0, xmm1, xmm2, xmm3;
- __m128i xmm4, xmm5, xmm6, xmm7;
+ while (height--) {
+ const uint8_t *src_row;
+ uint8_t *dst_row;
+ unsigned w = width;
- xmm0 = xmm_load_128((const __m128i*)src + 0);
- xmm1 = xmm_load_128((const __m128i*)src + 1);
- xmm2 = xmm_load_128((const __m128i*)src + 2);
- xmm3 = xmm_load_128((const __m128i*)src + 3);
- xmm4 = xmm_load_128((const __m128i*)src + 4);
- xmm5 = xmm_load_128((const __m128i*)src + 5);
- xmm6 = xmm_load_128((const __m128i*)src + 6);
- xmm7 = xmm_load_128((const __m128i*)src + 7);
+ dst_row = dst;
+ dst_row += dst_y / tile_height * dst_stride;
+ dst_row += (dst_y & (tile_height-1)) * tile_width;
+ dst_y++;
- xmm_save_128((__m128i*)dst + 0, xmm0);
- xmm_save_128((__m128i*)dst + 1, xmm1);
- xmm_save_128((__m128i*)dst + 2, xmm2);
- xmm_save_128((__m128i*)dst + 3, xmm3);
- xmm_save_128((__m128i*)dst + 4, xmm4);
- xmm_save_128((__m128i*)dst + 5, xmm5);
- xmm_save_128((__m128i*)dst + 6, xmm6);
- xmm_save_128((__m128i*)dst + 7, xmm7);
+ src_row = src;
+ src_row += src_y / tile_height * src_stride;
+ src_row += (src_y & (tile_height-1)) * tile_width;
+ src_y++;
- dst += 128;
- src += 128;
+ if (lx) {
+ to_memcpy(dst_row + ox, src_row + ox, lx);
+ dst_row += tile_size;
+ src_row += tile_size;
+ w -= lx;
+ }
+ while (w >= tile_width) {
+ assert(((uintptr_t)dst_row & (tile_width - 1)) == 0);
+ assert(((uintptr_t)src_row & (tile_width - 1)) == 0);
+ to_sse128xN(assume_aligned(dst_row, tile_width),
+ assume_aligned(src_row, tile_width),
+ tile_width);
+ dst_row += tile_size;
+ src_row += tile_size;
+ w -= tile_width;
+ }
+ if (w) {
+ assert(((uintptr_t)dst_row & (tile_width - 1)) == 0);
+ assert(((uintptr_t)src_row & (tile_width - 1)) == 0);
+ to_memcpy(assume_aligned(dst_row, tile_width),
+ assume_aligned(src_row, tile_width),
+ w);
+ }
}
}
-sse2 static force_inline void
-from_sse64u(uint8_t *dst, const uint8_t *src)
-{
- __m128i xmm1, xmm2, xmm3, xmm4;
+#pragma GCC push_options
+#endif
- assert(((uintptr_t)src & 15) == 0);
+fast void
+memcpy_blt(const void *src, void *dst, int bpp,
+ int32_t src_stride, int32_t dst_stride,
+ int16_t src_x, int16_t src_y,
+ int16_t dst_x, int16_t dst_y,
+ uint16_t width, uint16_t height)
+{
+ const uint8_t *src_bytes;
+ uint8_t *dst_bytes;
+ int byte_width;
- xmm1 = xmm_load_128((const __m128i*)src + 0);
- xmm2 = xmm_load_128((const __m128i*)src + 1);
- xmm3 = xmm_load_128((const __m128i*)src + 2);
- xmm4 = xmm_load_128((const __m128i*)src + 3);
+ assert(src);
+ assert(dst);
+ assert(width && height);
+ assert(bpp >= 8);
+ assert(width*bpp <= 8*src_stride);
+ assert(width*bpp <= 8*dst_stride);
- xmm_save_128u((__m128i*)dst + 0, xmm1);
- xmm_save_128u((__m128i*)dst + 1, xmm2);
- xmm_save_128u((__m128i*)dst + 2, xmm3);
- xmm_save_128u((__m128i*)dst + 3, xmm4);
-}
+ DBG(("%s: src=(%d, %d), dst=(%d, %d), size=%dx%d, pitch=%d/%d\n",
+ __FUNCTION__, src_x, src_y, dst_x, dst_y, width, height, src_stride, dst_stride));
-sse2 static force_inline void
-from_sse64a(uint8_t *dst, const uint8_t *src)
-{
- __m128i xmm1, xmm2, xmm3, xmm4;
+ bpp /= 8;
- assert(((uintptr_t)dst & 15) == 0);
- assert(((uintptr_t)src & 15) == 0);
+ src_bytes = (const uint8_t *)src + src_stride * src_y + src_x * bpp;
+ dst_bytes = (uint8_t *)dst + dst_stride * dst_y + dst_x * bpp;
- xmm1 = xmm_load_128((const __m128i*)src + 0);
- xmm2 = xmm_load_128((const __m128i*)src + 1);
- xmm3 = xmm_load_128((const __m128i*)src + 2);
- xmm4 = xmm_load_128((const __m128i*)src + 3);
+ byte_width = width * bpp;
+ if (byte_width == src_stride && byte_width == dst_stride) {
+ byte_width *= height;
+ height = 1;
+ }
- xmm_save_128((__m128i*)dst + 0, xmm1);
- xmm_save_128((__m128i*)dst + 1, xmm2);
- xmm_save_128((__m128i*)dst + 2, xmm3);
- xmm_save_128((__m128i*)dst + 3, xmm4);
-}
+ switch (byte_width) {
+ case 1:
+ do {
+ *dst_bytes = *src_bytes;
+ src_bytes += src_stride;
+ dst_bytes += dst_stride;
+ } while (--height);
+ break;
-sse2 static force_inline void
-from_sse32u(uint8_t *dst, const uint8_t *src)
-{
- __m128i xmm1, xmm2;
+ case 2:
+ do {
+ *(uint16_t *)dst_bytes = *(const uint16_t *)src_bytes;
+ src_bytes += src_stride;
+ dst_bytes += dst_stride;
+ } while (--height);
+ break;
- xmm1 = xmm_load_128((const __m128i*)src + 0);
- xmm2 = xmm_load_128((const __m128i*)src + 1);
+ case 4:
+ do {
+ *(uint32_t *)dst_bytes = *(const uint32_t *)src_bytes;
+ src_bytes += src_stride;
+ dst_bytes += dst_stride;
+ } while (--height);
+ break;
- xmm_save_128u((__m128i*)dst + 0, xmm1);
- xmm_save_128u((__m128i*)dst + 1, xmm2);
-}
+ case 8:
+ do {
+ *(uint64_t *)dst_bytes = *(const uint64_t *)src_bytes;
+ src_bytes += src_stride;
+ dst_bytes += dst_stride;
+ } while (--height);
+ break;
+ case 16:
+ do {
+ ((uint64_t *)dst_bytes)[0] = ((const uint64_t *)src_bytes)[0];
+ ((uint64_t *)dst_bytes)[1] = ((const uint64_t *)src_bytes)[1];
+ src_bytes += src_stride;
+ dst_bytes += dst_stride;
+ } while (--height);
+ break;
-sse2 static force_inline void
-from_sse32a(uint8_t *dst, const uint8_t *src)
-{
- __m128i xmm1, xmm2;
+ default:
+ do {
+ memcpy(dst_bytes, src_bytes, byte_width);
+ src_bytes += src_stride;
+ dst_bytes += dst_stride;
+ } while (--height);
+ break;
+ }
+}
- assert(((uintptr_t)dst & 15) == 0);
- assert(((uintptr_t)src & 15) == 0);
+static fast_memcpy void
+memcpy_to_tiled_x__swizzle_0(const void *src, void *dst, int bpp,
+ int32_t src_stride, int32_t dst_stride,
+ int16_t src_x, int16_t src_y,
+ int16_t dst_x, int16_t dst_y,
+ uint16_t width, uint16_t height)
+{
+ const unsigned tile_width = 512;
+ const unsigned tile_height = 8;
+ const unsigned tile_size = 4096;
- xmm1 = xmm_load_128((const __m128i*)src + 0);
- xmm2 = xmm_load_128((const __m128i*)src + 1);
+ const unsigned cpp = bpp / 8;
+ const unsigned tile_pixels = tile_width / cpp;
+ const unsigned tile_shift = ffs(tile_pixels) - 1;
+ const unsigned tile_mask = tile_pixels - 1;
- xmm_save_128((__m128i*)dst + 0, xmm1);
- xmm_save_128((__m128i*)dst + 1, xmm2);
-}
+ DBG(("%s(bpp=%d): src=(%d, %d), dst=(%d, %d), size=%dx%d, pitch=%d/%d\n",
+ __FUNCTION__, bpp, src_x, src_y, dst_x, dst_y, width, height, src_stride, dst_stride));
+ assert(src != dst);
-sse2 static force_inline void
-from_sse16u(uint8_t *dst, const uint8_t *src)
-{
- assert(((uintptr_t)src & 15) == 0);
+ if (src_x | src_y)
+ src = (const uint8_t *)src + src_y * src_stride + src_x * cpp;
+ assert(src_stride >= width * cpp);
+ src_stride -= width * cpp;
- xmm_save_128u((__m128i*)dst, xmm_load_128((const __m128i*)src));
-}
+ while (height--) {
+ unsigned w = width * cpp;
+ uint8_t *tile_row = dst;
-sse2 static force_inline void
-from_sse16a(uint8_t *dst, const uint8_t *src)
-{
- assert(((uintptr_t)dst & 15) == 0);
- assert(((uintptr_t)src & 15) == 0);
+ tile_row += dst_y / tile_height * dst_stride * tile_height;
+ tile_row += (dst_y & (tile_height-1)) * tile_width;
+ if (dst_x) {
+ tile_row += (dst_x >> tile_shift) * tile_size;
+ if (dst_x & tile_mask) {
+ const unsigned x = (dst_x & tile_mask) * cpp;
+ const unsigned len = min(tile_width - x, w);
+ memcpy(assume_misaligned(tile_row + x, tile_width, x),
+ src, len);
- xmm_save_128((__m128i*)dst, xmm_load_128((const __m128i*)src));
+ tile_row += tile_size;
+ src = (const uint8_t *)src + len;
+ w -= len;
+ }
+ }
+ while (w >= tile_width) {
+ memcpy(assume_aligned(tile_row, tile_width),
+ src, tile_width);
+ tile_row += tile_size;
+ src = (const uint8_t *)src + tile_width;
+ w -= tile_width;
+ }
+ memcpy(assume_aligned(tile_row, tile_width), src, w);
+ src = (const uint8_t *)src + src_stride + w;
+ dst_y++;
+ }
}
-sse2 static fast_memcpy void
-memcpy_from_tiled_x__swizzle_0__sse2(const void *src, void *dst, int bpp,
- int32_t src_stride, int32_t dst_stride,
- int16_t src_x, int16_t src_y,
- int16_t dst_x, int16_t dst_y,
- uint16_t width, uint16_t height)
+static fast_memcpy void
+memcpy_from_tiled_x__swizzle_0(const void *src, void *dst, int bpp,
+ int32_t src_stride, int32_t dst_stride,
+ int16_t src_x, int16_t src_y,
+ int16_t dst_x, int16_t dst_y,
+ uint16_t width, uint16_t height)
{
const unsigned tile_width = 512;
const unsigned tile_height = 8;
@@ -789,108 +790,54 @@ memcpy_from_tiled_x__swizzle_0__sse2(const void *src, void *dst, int bpp,
const unsigned tile_shift = ffs(tile_pixels) - 1;
const unsigned tile_mask = tile_pixels - 1;
- unsigned length_x, offset_x;
-
DBG(("%s(bpp=%d): src=(%d, %d), dst=(%d, %d), size=%dx%d, pitch=%d/%d\n",
__FUNCTION__, bpp, src_x, src_y, dst_x, dst_y, width, height, src_stride, dst_stride));
assert(src != dst);
if (dst_x | dst_y)
dst = (uint8_t *)dst + dst_y * dst_stride + dst_x * cpp;
- width *= cpp;
- assert(dst_stride >= width);
- if (src_x & tile_mask) {
- offset_x = (src_x & tile_mask) * cpp;
- length_x = min(tile_width - offset_x, width);
- dst_stride -= width;
- dst_stride += (width - length_x) & 15;
- } else {
- offset_x = 0;
- dst_stride -= width & ~15;
- }
- assert(dst_stride >= 0);
- src = (const uint8_t *)src + (src_x >> tile_shift) * tile_size;
+ assert(dst_stride >= width * cpp);
+ dst_stride -= width * cpp;
while (height--) {
- unsigned w = width;
+ unsigned w = width * cpp;
const uint8_t *tile_row = src;
tile_row += src_y / tile_height * src_stride * tile_height;
tile_row += (src_y & (tile_height-1)) * tile_width;
- src_y++;
-
- if (offset_x) {
- memcpy(dst, tile_row + offset_x, length_x);
- tile_row += tile_size;
- dst = (uint8_t *)dst + length_x;
- w -= length_x;
- }
+ if (src_x) {
+ tile_row += (src_x >> tile_shift) * tile_size;
+ if (src_x & tile_mask) {
+ const unsigned x = (src_x & tile_mask) * cpp;
+ const unsigned len = min(tile_width - x, w);
+ memcpy(dst, assume_misaligned(tile_row + x, tile_width, x), len);
- if ((uintptr_t)dst & 15) {
- while (w >= tile_width) {
- from_sse128xNu(dst,
- assume_aligned(tile_row, tile_width),
- tile_width);
- tile_row += tile_size;
- dst = (uint8_t *)dst + tile_width;
- w -= tile_width;
- }
- while (w >= 64) {
- from_sse64u(dst, tile_row);
- tile_row += 64;
- dst = (uint8_t *)dst + 64;
- w -= 64;
- }
- if (w & 32) {
- from_sse32u(dst, tile_row);
- tile_row += 32;
- dst = (uint8_t *)dst + 32;
- }
- if (w & 16) {
- from_sse16u(dst, tile_row);
- tile_row += 16;
- dst = (uint8_t *)dst + 16;
- }
- memcpy(dst, assume_aligned(tile_row, 16), w & 15);
- } else {
- while (w >= tile_width) {
- from_sse128xNa(assume_aligned(dst, 16),
- assume_aligned(tile_row, tile_width),
- tile_width);
tile_row += tile_size;
- dst = (uint8_t *)dst + tile_width;
- w -= tile_width;
- }
- while (w >= 64) {
- from_sse64a(dst, tile_row);
- tile_row += 64;
- dst = (uint8_t *)dst + 64;
- w -= 64;
- }
- if (w & 32) {
- from_sse32a(dst, tile_row);
- tile_row += 32;
- dst = (uint8_t *)dst + 32;
- }
- if (w & 16) {
- from_sse16a(dst, tile_row);
- tile_row += 16;
- dst = (uint8_t *)dst + 16;
+ dst = (uint8_t *)dst + len;
+ w -= len;
}
- memcpy(assume_aligned(dst, 16),
- assume_aligned(tile_row, 16),
- w & 15);
}
- dst = (uint8_t *)dst + dst_stride;
+ while (w >= tile_width) {
+ memcpy(dst,
+ assume_aligned(tile_row, tile_width),
+ tile_width);
+
+ tile_row += tile_size;
+ dst = (uint8_t *)dst + tile_width;
+ w -= tile_width;
+ }
+ memcpy(dst, assume_aligned(tile_row, tile_width), w);
+ dst = (uint8_t *)dst + dst_stride + w;
+ src_y++;
}
}
-sse2 static fast_memcpy void
-memcpy_between_tiled_x__swizzle_0__sse2(const void *src, void *dst, int bpp,
- int32_t src_stride, int32_t dst_stride,
- int16_t src_x, int16_t src_y,
- int16_t dst_x, int16_t dst_y,
- uint16_t width, uint16_t height)
+static fast_memcpy void
+memcpy_between_tiled_x__swizzle_0(const void *src, void *dst, int bpp,
+ int32_t src_stride, int32_t dst_stride,
+ int16_t src_x, int16_t src_y,
+ int16_t dst_x, int16_t dst_y,
+ uint16_t width, uint16_t height)
{
const unsigned tile_width = 512;
const unsigned tile_height = 8;
@@ -901,72 +848,55 @@ memcpy_between_tiled_x__swizzle_0__sse2(const void *src, void *dst, int bpp,
const unsigned tile_shift = ffs(tile_pixels) - 1;
const unsigned tile_mask = tile_pixels - 1;
- unsigned ox, lx;
-
DBG(("%s(bpp=%d): src=(%d, %d), dst=(%d, %d), size=%dx%d, pitch=%d/%d\n",
__FUNCTION__, bpp, src_x, src_y, dst_x, dst_y, width, height, src_stride, dst_stride));
assert(src != dst);
-
- width *= cpp;
- dst_stride *= tile_height;
- src_stride *= tile_height;
-
assert((dst_x & tile_mask) == (src_x & tile_mask));
- if (dst_x & tile_mask) {
- ox = (dst_x & tile_mask) * cpp;
- lx = min(tile_width - ox, width);
- assert(lx != 0);
- } else
- lx = 0;
-
- if (dst_x)
- dst = (uint8_t *)dst + (dst_x >> tile_shift) * tile_size;
- if (src_x)
- src = (const uint8_t *)src + (src_x >> tile_shift) * tile_size;
while (height--) {
- const uint8_t *src_row;
- uint8_t *dst_row;
- unsigned w = width;
+ unsigned w = width * cpp;
+ uint8_t *dst_row = dst;
+ const uint8_t *src_row = src;
- dst_row = dst;
- dst_row += dst_y / tile_height * dst_stride;
+ dst_row += dst_y / tile_height * dst_stride * tile_height;
dst_row += (dst_y & (tile_height-1)) * tile_width;
+ if (dst_x)
+ dst_row += (dst_x >> tile_shift) * tile_size;
dst_y++;
- src_row = src;
- src_row += src_y / tile_height * src_stride;
+ src_row += src_y / tile_height * src_stride * tile_height;
src_row += (src_y & (tile_height-1)) * tile_width;
+ if (src_x)
+ src_row += (src_x >> tile_shift) * tile_size;
src_y++;
- if (lx) {
- to_memcpy(dst_row + ox, src_row + ox, lx);
+ if (dst_x & tile_mask) {
+ const unsigned x = (dst_x & tile_mask) * cpp;
+ const unsigned len = min(tile_width - x, w);
+
+ memcpy(assume_misaligned(dst_row + x, tile_width, x),
+ assume_misaligned(src_row + x, tile_width, x),
+ len);
+
dst_row += tile_size;
src_row += tile_size;
- w -= lx;
+ w -= len;
}
+
while (w >= tile_width) {
- assert(((uintptr_t)dst_row & (tile_width - 1)) == 0);
- assert(((uintptr_t)src_row & (tile_width - 1)) == 0);
- to_sse128xN(assume_aligned(dst_row, tile_width),
- assume_aligned(src_row, tile_width),
- tile_width);
+ memcpy(assume_aligned(dst_row, tile_width),
+ assume_aligned(src_row, tile_width),
+ tile_width);
dst_row += tile_size;
src_row += tile_size;
w -= tile_width;
}
- if (w) {
- assert(((uintptr_t)dst_row & (tile_width - 1)) == 0);
- assert(((uintptr_t)src_row & (tile_width - 1)) == 0);
- to_memcpy(assume_aligned(dst_row, tile_width),
- assume_aligned(src_row, tile_width),
- w);
- }
+ memcpy(assume_aligned(dst_row, tile_width),
+ assume_aligned(src_row, tile_width),
+ w);
}
}
-#endif
-
#define memcpy_to_tiled_x(swizzle) \
fast_memcpy static void \
memcpy_to_tiled_x__##swizzle (const void *src, void *dst, int bpp, \
@@ -1241,7 +1171,7 @@ void choose_memcpy_tiled_x(struct kgem *kgem, int swizzling, unsigned cpu)
break;
case I915_BIT_6_SWIZZLE_NONE:
DBG(("%s: no swizzling\n", __FUNCTION__));
-#if defined(sse2) && defined(__x86_64__)
+#if defined(sse2)
if (cpu & SSE2) {
kgem->memcpy_to_tiled_x = memcpy_to_tiled_x__swizzle_0__sse2;
kgem->memcpy_from_tiled_x = memcpy_from_tiled_x__swizzle_0__sse2;
@@ -1498,7 +1428,7 @@ memcpy_xor(const void *src, void *dst, int bpp,
height = 1;
}
-#if USE_SSE2
+#if defined(sse2) && __x86_64__
if (have_sse2()) {
do {
uint32_t *d = (uint32_t *)dst_bytes;
commit 24f613cae4147e0e1e770ee22932b6e2fb7064a2
Author: Chris Wilson <chris at chris-wilson.co.uk>
Date: Thu Jul 28 22:42:28 2016 +0100
sna: Hide the hardware cursor when switching back after an error
If the kernel complains when trying to move the cursor, we disable the
hwcursor and switch to swcursor. However, if the kernel is still showing
the hwcursor we end up with multiple visible cursors. Attempt to disable
the hwcursor (that may or may not work) as we switch over.
Signed-off-by: Chris Wilson <chris at chris-wilson.co.uk>
diff --git a/src/sna/sna_display.c b/src/sna/sna_display.c
index b3eb19f..f10e340 100644
--- a/src/sna/sna_display.c
+++ b/src/sna/sna_display.c
@@ -5842,6 +5842,8 @@ static void __restore_swcursor(ScrnInfoPtr scrn)
static void restore_swcursor(struct sna *sna)
{
+ sna->cursor.info->HideCursor(sna->scrn);
+
/* XXX Force the cursor to be restored (avoiding recursion) */
FreeCursor(sna->cursor.ref, None);
sna->cursor.ref = NULL;
commit 8bfac0f21ab817c7ccd654a0571bb07e6c7b77d5
Author: Chris Wilson <chris at chris-wilson.co.uk>
Date: Thu Jul 28 19:43:01 2016 +0100
sna/dri2: Only force the TearFree/swcursor hack when using TearFree
Signed-off-by: Chris Wilson <chris at chris-wilson.co.uk>
diff --git a/src/sna/sna_dri2.c b/src/sna/sna_dri2.c
index 4ec0cca..2266791 100644
--- a/src/sna/sna_dri2.c
+++ b/src/sna/sna_dri2.c
@@ -1006,7 +1006,7 @@ static void set_bo(PixmapPtr pixmap, struct kgem_bo *bo)
* causing recursion and mayhem.
*/
DBG(("%s: marking whole pixmap as damaged\n", __FUNCTION__));
- sna->ignore_copy_area = true;
+ sna->ignore_copy_area = sna->flags & SNA_TEAR_FREE;
DamageRegionAppend(&pixmap->drawable, ®ion);
}
@@ -1358,7 +1358,7 @@ __sna_dri2_copy_region(struct sna *sna, DrawablePtr draw, RegionPtr region,
}
if (APPLY_DAMAGE || flags & DRI2_DAMAGE) {
DBG(("%s: marking region as damaged\n", __FUNCTION__));
- sna->ignore_copy_area = true;
+ sna->ignore_copy_area = sna->flags & SNA_TEAR_FREE;
DamageRegionAppend(&pixmap->drawable, region);
}
@@ -2416,7 +2416,7 @@ static void sna_dri2_xchg_crtc(struct sna *sna, DrawablePtr draw, xf86CrtcPtr cr
if (APPLY_DAMAGE) {
DBG(("%s: marking drawable as damaged\n", __FUNCTION__));
- sna->ignore_copy_area = true;
+ sna->ignore_copy_area = sna->flags & SNA_TEAR_FREE;
DamageRegionAppend(&win->drawable, &win->clipList);
}
sna_shadow_set_crtc(sna, crtc, get_private(back)->bo);
commit ab57363a1c0b970a220a59a53562b46c817cd7ea
Author: Chris Wilson <chris at chris-wilson.co.uk>
Date: Thu Jul 28 19:42:25 2016 +0100
sna: Enable kernel fail logging for cursors
Log the kernel messages when we fail to set a cursor.
Signed-off-by: Chris Wilson <chris at chris-wilson.co.uk>
diff --git a/src/sna/sna_display.c b/src/sna/sna_display.c
index ef29a91..b3eb19f 100644
--- a/src/sna/sna_display.c
+++ b/src/sna/sna_display.c
@@ -1303,6 +1303,9 @@ bool sna_crtc_set_sprite_rotation(xf86CrtcPtr crtc,
}
#if HAS_DEBUG_FULL
+#if !HAS_DEBUG_FULL
+#define LogF ErrorF
+#endif
struct kmsg {
int fd;
int saved_loglevel;
@@ -1399,8 +1402,8 @@ sna_crtc_apply(xf86CrtcPtr crtc)
return EINVAL;
}
- sigio = sigio_block();
kmsg_open(&kmsg);
+ sigio = sigio_block();
assert(sna->mode.num_real_output < ARRAY_SIZE(output_ids));
sna_crtc_disable_cursor(sna, sna_crtc);
@@ -1500,8 +1503,8 @@ sna_crtc_apply(xf86CrtcPtr crtc)
sna_crtc_force_outputs_on(crtc);
unblock:
- kmsg_close(&kmsg, ret);
sigio_unblock(sigio);
+ kmsg_close(&kmsg, ret);
return ret;
}
@@ -5853,12 +5856,14 @@ sna_show_cursors(ScrnInfoPtr scrn)
{
xf86CrtcConfigPtr xf86_config = XF86_CRTC_CONFIG_PTR(scrn);
struct sna *sna = to_sna(scrn);
+ struct kmsg kmsg;
int sigio, c;
DBG(("%s: cursor?=%d\n", __FUNCTION__, sna->cursor.ref != NULL));
if (sna->cursor.ref == NULL)
return;
+ kmsg_open(&kmsg);
sigio = sigio_block();
for (c = 0; c < sna->mode.num_real_crtc; c++) {
xf86CrtcPtr crtc = xf86_config->crtc[c];
@@ -5910,6 +5915,7 @@ sna_show_cursors(ScrnInfoPtr scrn)
}
sigio_unblock(sigio);
sna->cursor.active = true;
+ kmsg_close(&kmsg, sna->cursor.disable);
if (unlikely(sna->cursor.disable))
restore_swcursor(sna);
@@ -6032,6 +6038,7 @@ sna_set_cursor_position(ScrnInfoPtr scrn, int x, int y)
{
xf86CrtcConfigPtr xf86_config = XF86_CRTC_CONFIG_PTR(scrn);
struct sna *sna = to_sna(scrn);
+ struct kmsg kmsg;
int sigio, c;
__DBG(("%s(%d, %d), cursor? %d\n", __FUNCTION__,
@@ -6039,6 +6046,7 @@ sna_set_cursor_position(ScrnInfoPtr scrn, int x, int y)
if (sna->cursor.ref == NULL)
return;
+ kmsg_open(&kmsg);
sigio = sigio_block();
sna->cursor.last_x = x;
sna->cursor.last_y = y;
@@ -6145,6 +6153,7 @@ disable:
}
}
sigio_unblock(sigio);
+ kmsg_close(&kmsg, sna->cursor.disable);
if (unlikely(sna->cursor.disable))
restore_swcursor(sna);
More information about the xorg-commit
mailing list