output: fix output transform compositions
This change ensures that wlr_output_transform_compose correctly composes
transforms when the first transform includes a rotation and the second
transform includes a flip.
@@ -1118,7 +1118,15 @@ enum wl_output_transform wlr_output_transform_invert(
enum wl_output_transform wlr_output_transform_compose(
enum wl_output_transform tr_a, enum wl_output_transform tr_b) {
uint32_t flipped = (tr_a ^ tr_b) & WL_OUTPUT_TRANSFORM_FLIPPED;
- uint32_t rotated =
- (tr_a + tr_b) & (WL_OUTPUT_TRANSFORM_90 | WL_OUTPUT_TRANSFORM_180);
+ uint32_t rotation_mask = WL_OUTPUT_TRANSFORM_90 | WL_OUTPUT_TRANSFORM_180;
+ uint32_t rotated;
+ if (tr_b & WL_OUTPUT_TRANSFORM_FLIPPED) {
+ // When a rotation of k degrees is followed by a flip, the
+ // equivalent transform is a flip followed by a rotation of
+ // -k degrees.
+ rotated = (tr_b - tr_a) & rotation_mask;
+ } else {
+ rotated = (tr_a + tr_b) & rotation_mask;
+ }
return flipped | rotated;
}
