[RFC xserver v6 09/14] modesetting: Create scanout buffers using supported modifiers

[Louis-Francis Ratté-Boulianne]

Use most optimal buffer format (e.g. tiled/compressed) available
for scanout.

v2: Don't use multi-plane modifier to create scanout buffer

v3: Add flag to retrieve modifiers set from enabled CRTCs only

v4: Fix uses when GBM/EGL driver doesn't support modifiers

Signed-off-by: Louis-Francis Ratté-Boulianne <lfrb at collabora.com>
Reviewed-by: Daniel Stone <daniels at collabora.com>
---
 hw/xfree86/drivers/modesetting/drmmode_display.c | 77 ++++++++++++++++++++++++
 1 file changed, 77 insertions(+)

diff --git a/hw/xfree86/drivers/modesetting/drmmode_display.c b/hw/xfree86/drivers/modesetting/drmmode_display.c
index 4a60f93b2..7f0d1d2ea 100644
--- a/hw/xfree86/drivers/modesetting/drmmode_display.c
+++ b/hw/xfree86/drivers/modesetting/drmmode_display.c
@@ -73,6 +73,66 @@ modifiers_ptr(struct drm_format_modifier_blob *blob)
 
 #endif
 
+static uint32_t
+get_modifiers_set(ScrnInfoPtr scrn, uint32_t format, uint64_t **modifiers,
+                  Bool enabled_crtc_only, Bool exclude_multiplane)
+{
+    xf86CrtcConfigPtr xf86_config = XF86_CRTC_CONFIG_PTR(scrn);
+    modesettingPtr ms = modesettingPTR(scrn);
+    drmmode_ptr drmmode = &ms->drmmode;
+    int c, i, j, k, count_modifiers = 0;
+    uint64_t *tmp, *ret = NULL;
+
+    *modifiers = NULL;
+    for (c = 0; c < xf86_config->num_crtc; c++) {
+        xf86CrtcPtr crtc = xf86_config->crtc[c];
+        drmmode_crtc_private_ptr drmmode_crtc = crtc->driver_private;
+
+        if (enabled_crtc_only && !crtc->enabled)
+            continue;
+
+        for (i = 0; i < drmmode_crtc->num_formats; i++) {
+            drmmode_format_ptr iter = &drmmode_crtc->formats[i];
+
+            if (iter->format != format)
+                continue;
+
+            for (j = 0; j < iter->num_modifiers; j++) {
+                Bool found = FALSE;
+
+		/* Don't choose multi-plane formats for our screen pixmap.
+		 * These will get used with frontbuffer rendering, which will
+		 * lead to worse-than-tearing with multi-plane formats, as the
+		 * primary and auxiliary planes go out of sync. */
+		if (exclude_multiplane &&
+                    gbm_device_get_format_modifier_plane_count(drmmode->gbm,
+                                                               format,
+                                                               iter->modifiers[j]) > 1) {
+                    continue;
+                }
+
+                for (k = 0; k < count_modifiers; k++) {
+                    if (iter->modifiers[j] == ret[k])
+                        found = TRUE;
+                }
+                if (!found) {
+                    count_modifiers++;
+                    tmp = realloc(ret, count_modifiers * sizeof(uint64_t));
+                    if (!tmp) {
+                        free(ret);
+                        return 0;
+                    }
+                    ret = tmp;
+                    ret[count_modifiers - 1] = iter->modifiers[j];
+                }
+            }
+        }
+    }
+
+    *modifiers = ret;
+    return count_modifiers;
+}
+
 static Bool
 drmmode_zaphod_string_matches(ScrnInfoPtr scrn, const char *s, char *output_name)
 {
@@ -598,6 +658,23 @@ drmmode_create_bo(drmmode_ptr drmmode, drmmode_bo *bo,
 
 #ifdef GLAMOR_HAS_GBM
     if (drmmode->glamor) {
+#ifdef GBM_BO_WITH_MODIFIERS
+        uint32_t num_modifiers;
+        uint64_t *modifiers = NULL;
+
+        num_modifiers = get_modifiers_set(drmmode->scrn, DRM_FORMAT_ARGB8888,
+                                          &modifiers, FALSE, TRUE);
+        if (num_modifiers > 0 &&
+            !(num_modifiers == 1 && modifiers[0] == DRM_FORMAT_MOD_INVALID)) {
+            bo->gbm = gbm_bo_create_with_modifiers(drmmode->gbm, width, height,
+                                                   GBM_FORMAT_ARGB8888,
+                                                   modifiers, num_modifiers);
+            free(modifiers);
+            if (bo->gbm)
+                return TRUE;
+        }
+#endif
+
         bo->gbm = gbm_bo_create(drmmode->gbm, width, height,
                                 GBM_FORMAT_ARGB8888,
                                 GBM_BO_USE_RENDERING | GBM_BO_USE_SCANOUT);
-- 
2.14.3