[PATCH 4/5] drm/bochs: fix DRM_FORMAT_* handling for big endian machines.

From: Gerd Hoffmann
Date: Mon Sep 03 2018 - 06:58:11 EST


Use DRM_FORMAT_HOST_XRGB8888, so we are using the correct format code
on bigendian machines. Also add DRIVER_PREFER_HOST_BYTE_ORDER driver
feature flag so drm_mode_addfb() asks for the correct format code.

Create our own plane and use drm_crtc_init_with_planes() instead of
depending on the default created by drm_crtc_init(). That way the plane
format list is correct on bigendian machines.

With this patch applied both ADDFB and ADDFB2 ioctls work correctly in
the bochs-drm.ko driver on big endian machines. Without the patch only
ADDFB (which still seems to be used by the majority of userspace) works
correctly.

Signed-off-by: Gerd Hoffmann <kraxel@xxxxxxxxxx>
---
drivers/gpu/drm/bochs/bochs_drv.c | 3 ++-
drivers/gpu/drm/bochs/bochs_fbdev.c | 5 ++---
drivers/gpu/drm/bochs/bochs_kms.c | 33 ++++++++++++++++++++++++++++++++-
drivers/gpu/drm/bochs/bochs_mm.c | 2 +-
4 files changed, 37 insertions(+), 6 deletions(-)

diff --git a/drivers/gpu/drm/bochs/bochs_drv.c b/drivers/gpu/drm/bochs/bochs_drv.c
index 7b20318483..fa84a3c841 100644
--- a/drivers/gpu/drm/bochs/bochs_drv.c
+++ b/drivers/gpu/drm/bochs/bochs_drv.c
@@ -81,7 +81,8 @@ static const struct file_operations bochs_fops = {
};

static struct drm_driver bochs_driver = {
- .driver_features = DRIVER_GEM | DRIVER_MODESET,
+ .driver_features = (DRIVER_GEM | DRIVER_MODESET |
+ DRIVER_PREFER_HOST_BYTE_ORDER),
.load = bochs_load,
.unload = bochs_unload,
.fops = &bochs_fops,
diff --git a/drivers/gpu/drm/bochs/bochs_fbdev.c b/drivers/gpu/drm/bochs/bochs_fbdev.c
index 14eb8d0d5a..bf728790fa 100644
--- a/drivers/gpu/drm/bochs/bochs_fbdev.c
+++ b/drivers/gpu/drm/bochs/bochs_fbdev.c
@@ -64,9 +64,8 @@ static int bochsfb_create(struct drm_fb_helper *helper,

mode_cmd.width = sizes->surface_width;
mode_cmd.height = sizes->surface_height;
- mode_cmd.pitches[0] = mode_cmd.width * ((sizes->surface_bpp + 7) / 8);
- mode_cmd.pixel_format = drm_mode_legacy_fb_format(sizes->surface_bpp,
- sizes->surface_depth);
+ mode_cmd.pitches[0] = sizes->surface_width * 4;
+ mode_cmd.pixel_format = DRM_FORMAT_HOST_XRGB8888;
size = mode_cmd.pitches[0] * mode_cmd.height;

/* alloc, pin & map bo */
diff --git a/drivers/gpu/drm/bochs/bochs_kms.c b/drivers/gpu/drm/bochs/bochs_kms.c
index ca5a9afdd5..2662cdcf2d 100644
--- a/drivers/gpu/drm/bochs/bochs_kms.c
+++ b/drivers/gpu/drm/bochs/bochs_kms.c
@@ -129,12 +129,43 @@ static const struct drm_crtc_helper_funcs bochs_helper_funcs = {
.commit = bochs_crtc_commit,
};

+static const uint32_t bochs_formats[] = {
+ DRM_FORMAT_HOST_XRGB8888,
+};
+
+static struct drm_plane *bochs_primary_plane(struct drm_device *dev)
+{
+ struct drm_plane *primary;
+ int ret;
+
+ primary = kzalloc(sizeof(*primary), GFP_KERNEL);
+ if (primary == NULL) {
+ DRM_DEBUG_KMS("Failed to allocate primary plane\n");
+ return NULL;
+ }
+
+ ret = drm_universal_plane_init(dev, primary, 0,
+ &drm_primary_helper_funcs,
+ bochs_formats,
+ ARRAY_SIZE(bochs_formats),
+ NULL,
+ DRM_PLANE_TYPE_PRIMARY, NULL);
+ if (ret) {
+ kfree(primary);
+ primary = NULL;
+ }
+
+ return primary;
+}
+
static void bochs_crtc_init(struct drm_device *dev)
{
struct bochs_device *bochs = dev->dev_private;
struct drm_crtc *crtc = &bochs->crtc;
+ struct drm_plane *primary = bochs_primary_plane(dev);

- drm_crtc_init(dev, crtc, &bochs_crtc_funcs);
+ drm_crtc_init_with_planes(dev, crtc, primary, NULL,
+ &bochs_crtc_funcs, NULL);
drm_crtc_helper_add(crtc, &bochs_helper_funcs);
}

diff --git a/drivers/gpu/drm/bochs/bochs_mm.c b/drivers/gpu/drm/bochs/bochs_mm.c
index c9c7097030..fdf151fbdb 100644
--- a/drivers/gpu/drm/bochs/bochs_mm.c
+++ b/drivers/gpu/drm/bochs/bochs_mm.c
@@ -506,7 +506,7 @@ bochs_user_framebuffer_create(struct drm_device *dev,
(mode_cmd->pixel_format >> 16) & 0xff,
(mode_cmd->pixel_format >> 24) & 0xff);

- if (mode_cmd->pixel_format != DRM_FORMAT_XRGB8888)
+ if (mode_cmd->pixel_format != DRM_FORMAT_HOST_XRGB8888)
return ERR_PTR(-ENOENT);

obj = drm_gem_object_lookup(filp, mode_cmd->handles[0]);
--
2.9.3