fix: don't assume 0x0000..0xFFFF as KMS plane alpha range

xilinx drivers don't follow this convention and use a smaller range (0..255).

Author: ardera

src/modesetting.c

@@ -757,11 +757,16 @@ static int fetch_plane(int drm_fd, uint32_t plane_id, struct drm_plane *plane_ou
         } else if (streq(info->name, "alpha")) {
             has_alpha = true;
             assert(info->flags == DRM_MODE_PROP_RANGE);
-            assert(info->values[0] == 0);
-            assert(info->values[1] == 0xFFFF);
-            assert(props->prop_values[j] <= 0xFFFF);
-
-            committed_alpha = (uint16_t) props->prop_values[j];
+            assert(info->values[0] <= 0xFFFF);
+            assert(info->values[1] <= 0xFFFF);
+            assert(info->values[0] <= info->values[1]);
+            plane_out->min_alpha = (uint16_t) info->values[0];
+            plane_out->max_alpha = (uint16_t) info->values[1];
+
+            uint64_t value = props->prop_values[j];
+            assert(plane_out->min_alpha <= value);
+            assert(value <= plane_out->max_alpha);
+            committed_alpha = (uint16_t) value;
         } else if (streq(info->name, "pixel blend mode")) {
             has_blend_mode = true;
             assert(info->flags == DRM_MODE_PROP_ENUM);
@@ -2568,7 +2573,7 @@ int kms_req_builder_push_fb_layer(
 
         if (index == 0) {
             if (plane->has_alpha) {
-                drmModeAtomicAddProperty(builder->req, plane_id, plane->ids.alpha, DRM_BLEND_ALPHA_OPAQUE);
+                drmModeAtomicAddProperty(builder->req, plane_id, plane->ids.alpha, plane->max_alpha);
             }
 
             if (plane->has_blend_mode && plane->supported_blend_modes[kNone_DrmBlendMode]) {

src/modesetting.h

@@ -215,8 +215,6 @@
 
 #define DECLARE_PROP_ID_AS_UINT32(prop_name, prop_var_name) uint32_t prop_var_name;
 
-#define DRM_BLEND_ALPHA_OPAQUE 0xFFFF
-
 enum drm_blend_mode {
     kPremultiplied_DrmBlendMode,
     kCoverage_DrmBlendMode,
@@ -502,6 +500,13 @@ struct drm_plane {
     /// @brief Whether this plane has a mutable alpha property we can set.
     bool has_alpha;
 
+    /// @brief The minimum and maximum alpha values.
+    ///
+    /// Only valid if @ref has_alpha is true.
+    ///
+    /// This should be 0x0000..0xFFFF, but the xilinx driver uses different values.
+    uint16_t min_alpha, max_alpha;
+
     /// @brief Whether this plane has a mutable pixel blend mode we can set.
     bool has_blend_mode;
 
@@ -750,15 +755,15 @@ DECLARE_REF_OPS(kms_req_builder);
 
 /**
  * @brief Gets the @ref drmdev associated with this KMS request builder.
- * 
+ *
  * @param builder The KMS request builder.
  * @returns The drmdev associated with this KMS request builder.
  */
 struct drmdev *kms_req_builder_get_drmdev(struct kms_req_builder *builder);
 
 /**
  * @brief Gets the CRTC associated with this KMS request builder.
- * 
+ *
  * @param builder The KMS request builder.
  * @returns The CRTC associated with this KMS request builder.
  */
@@ -768,7 +773,7 @@ struct drm_crtc *kms_req_builder_get_crtc(struct kms_req_builder *builder);
  * @brief Adds a property to the KMS request that will set the given video mode
  * on this CRTC on commit, regardless of whether the currently committed output
  * mode is the same.
- * 
+ *
  * @param builder The KMS request builder.
  * @param mode The output mode to set (on @ref kms_req_commit)
  * @returns Zero if successful, positive errno-style error on failure.
@@ -779,7 +784,7 @@ int kms_req_builder_set_mode(struct kms_req_builder *builder, const drmModeModeI
  * @brief Adds a property to the KMS request that will unset the configured
  * output mode for this CRTC on commit, regardless of whether the currently
  * committed output mdoe is already unset.
- * 
+ *
  * @param builder The KMS request builder.
  * @returns Zero if successful, positive errno-style error on failure.
  */
@@ -788,7 +793,7 @@ int kms_req_builder_unset_mode(struct kms_req_builder *builder);
 /**
  * @brief Adds a property to the KMS request that will change the connector
  * that this CRTC is displaying content on to @param connector_id.
- * 
+ *
  * @param builder The KMS request builder.
  * @param connector_id The connector that this CRTC should display contents on.
  * @returns Zero if successful, EINVAL if the @param connector_id is invalid.
@@ -799,11 +804,11 @@ int kms_req_builder_set_connector(struct kms_req_builder *builder, uint32_t conn
  * @brief True if the next layer pushed using @ref kms_req_builder_push_fb_layer
  * should be opaque, i.e. use a framebuffer which has a pixel format that has no
  * alpha channel.
- * 
+ *
  * This is true for the bottom-most layer. There are some display controllers
  * that don't support non-opaque pixel formats for the bottom-most (primary)
  * plane. So ignoring this might lead to an EINVAL on commit.
- * 
+ *
  * @param builder The KMS request builder.
  * @returns True if the next layer should preferably be opaque, false if there's
  *          no preference.
@@ -812,30 +817,30 @@ bool kms_req_builder_prefer_next_layer_opaque(struct kms_req_builder *builder);
 
 /**
  * @brief Adds a new framebuffer (display) layer on top of the last layer.
- * 
+ *
  * If this is the first layer, the framebuffer should cover the entire screen
  * (CRTC).
- * 
+ *
  * To allow the use of explicit fencing, specify an in_fence_fd in @param layer
  * and a @param deferred_release_callback.
- * 
+ *
  * If explicit fencing is supported:
  *   - the in_fence_fd should be a DRM syncobj fd that signals
  *     when the GPU has finished rendering to the framebuffer and is ready
  *     to be scanned out.
  *   - @param deferred_release_callback will be called
  *     with a DRM syncobj fd that is signaled once the framebuffer is no longer
  *     being displayed on screen (and can be rendered into again)
- * 
+ *
  * If explicit fencing is not supported:
  *   - the in_fence_fd in @param layer will be closed by this procedure.
  *   - @param deferred_release_callback will NOT be called and
  *     @param release_callback will be called instead.
- * 
+ *
  * Explicit fencing is supported: When atomic modesetting is being used and
  * the driver supports it. (Driver has IN_FENCE_FD plane and OUT_FENCE_PTR crtc
  * properties)
- * 
+ *
  * @param builder          The KMS request builder.
  * @param layer            The exact details (src pos, output pos, rotation,
  *                         framebuffer) of the layer that should be shown on
@@ -878,7 +883,7 @@ int kms_req_builder_push_fb_layer(
 /**
  * @brief Push a "fake" layer that just keeps one zpos free, incase something
  * other than KMS wants to display contents there. (e.g. omxplayer)
- * 
+ *
  * @param builder The KMS request builder.
  * @param zpos_out Filled with the zpos that won't be occupied by the request
  *                 builder.
@@ -889,7 +894,7 @@ int kms_req_builder_push_zpos_placeholder_layer(struct kms_req_builder *builder,
 /**
  * @brief A KMS request (atomic or legacy modesetting) that can be committed to
  * change the state of a single CRTC.
- * 
+ *
  * Only way to construct this is by building a KMS request using
  * @ref kms_req_builder and then calling @ref kms_req_builder_build.
  */
@@ -900,7 +905,7 @@ DECLARE_REF_OPS(kms_req);
 /**
  * @brief Build the KMS request builder into an actual, immutable KMS request
  * that can be committed. Internally this doesn't do much at all.
- * 
+ *
  * @param builder The KMS request builder that should be built.
  * @returns KMS request that can be committed using @ref kms_req_commit_blocking
  *          or @ref kms_req_commit_nonblocking.