rules: implement unredir window rules option

To replace wintypes::redir_ignore as well as
unredir-if-possible-exclude.

Signed-off-by: Yuxuan Shui <[email protected]>

src/config.h

@@ -136,6 +136,29 @@ struct included_config_file {
 	struct list_node siblings;
 };
 
+enum window_unredir_option {
+	/// This window should trigger unredirection if it meets certain conditions, and
+	/// it should terminate unredirection otherwise. Termination of unredir is always
+	/// suppressed if there is another window triggering unredirection, this is the
+	/// same for `WINDOW_UNREDIR_TERMINATE` as well.
+	///
+	/// This is the default choice for windows.
+	WINDOW_UNREDIR_WHEN_POSSIBLE_ELSE_TERMINATE,
+	/// This window should trigger unredirection if it meets certain conditions.
+	/// Otherwise it should have no effect on the compositor's redirection status.
+	WINDOW_UNREDIR_WHEN_POSSIBLE,
+	/// This window should always take the compositor out of unredirection, and never
+	/// trigger unredirection.
+	WINDOW_UNREDIR_TERMINATE,
+	/// This window should not cause either redirection or unredirection.
+	WINDOW_UNREDIR_PASSIVE,
+	/// This window always trigger unredirection
+	WINDOW_UNREDIR_FORCE,
+
+	/// Sentinel value
+	WINDOW_UNREDIR_INVALID,
+};
+
 struct window_maybe_options {
 	/// Radius of rounded window corners, -1 means not set.
 	int corner_radius;
@@ -164,7 +187,7 @@ struct window_maybe_options {
 	/// Whether the window is painted.
 	enum tristate paint;
 	/// Whether this window should be considered for unredirect-if-possible.
-	enum tristate unredir_ignore;
+	enum window_unredir_option unredir;
 };
 
 // Make sure `window_options` has no implicit padding.
@@ -176,16 +199,16 @@ struct window_options {
 	double dim;
 	const struct shader_info *shader;
 	unsigned int corner_radius;
+	enum window_unredir_option unredir;
 	bool transparent_clipping;
 	bool shadow;
 	bool invert_color;
 	bool blur_background;
 	bool fade;
 	bool clip_shadow_above;
 	bool paint;
-	bool unredir_ignore;
 
-	char padding[4];
+	char padding[1];
 };
 #pragma GCC diagnostic pop
 

src/config_libconfig.c

@@ -529,6 +529,42 @@ void generate_fading_config(struct options *opt) {
 	dynarr_extend_from(opt->all_scripts, scripts, number_of_scripts);
 }
 
+static enum window_unredir_option parse_unredir_option(config_setting_t *setting) {
+	if (config_setting_type(setting) == CONFIG_TYPE_BOOL) {
+		auto bval = config_setting_get_bool(setting);
+		return bval ? WINDOW_UNREDIR_WHEN_POSSIBLE_ELSE_TERMINATE
+		            : WINDOW_UNREDIR_TERMINATE;
+	}
+	const char *sval = config_setting_get_string(setting);
+	if (!sval) {
+		log_error("Invalid value for \"unredir\" at line %d. It must be a "
+		          "boolean or a string.",
+		          config_setting_source_line(setting));
+		return WINDOW_UNREDIR_INVALID;
+	}
+	if (strcmp(sval, "yes") == 0 || strcmp(sval, "true") == 0 ||
+	    strcmp(sval, "default") == 0 || strcmp(sval, "when-possible-else-terminate")) {
+		return WINDOW_UNREDIR_WHEN_POSSIBLE_ELSE_TERMINATE;
+	}
+	if (strcmp(sval, "preferred") == 0 || strcmp(sval, "when-possible") == 0) {
+		return WINDOW_UNREDIR_WHEN_POSSIBLE;
+	}
+	if (strcmp(sval, "no") == 0 || strcmp(sval, "false") == 0 ||
+	    strcmp(sval, "terminate") == 0) {
+		return WINDOW_UNREDIR_TERMINATE;
+	}
+	if (strcmp(sval, "passive") == 0) {
+		return WINDOW_UNREDIR_PASSIVE;
+	}
+	if (strcmp(sval, "force") == 0) {
+		return WINDOW_UNREDIR_FORCE;
+	}
+	log_error("Invalid string value for \"unredir\" at line %d. It must be one of "
+	          "\"preferred\", \"passive\", or \"force\". Got \"%s\".",
+	          config_setting_source_line(setting), sval);
+	return WINDOW_UNREDIR_INVALID;
+}
+
 static const struct {
 	const char *name;
 	ptrdiff_t offset;
@@ -585,6 +621,11 @@ static c2_lptr_t *parse_rule(config_setting_t *setting) {
 	if (config_setting_lookup_int(setting, "corner-radius", &ival)) {
 		wopts->corner_radius = ival;
 	}
+
+	auto unredir_setting = config_setting_lookup(setting, "unredir-if-possible");
+	if (unredir_setting) {
+		wopts->unredir = parse_unredir_option(unredir_setting);
+	}
 	return rule;
 }
 

src/picom.c

@@ -898,30 +898,29 @@ static bool paint_preprocess(session_t *ps, bool *animation, struct win **out_bo
 		// paint, but this is typically unnecessary, may cause flickering when
 		// fading is enabled, and could create inconsistency when the wallpaper
 		// is not correctly set.
-		if (ps->o.unredir_if_possible && is_highest) {
-			if (w->mode == WMODE_SOLID && !ps->o.force_win_blend &&
-			    w->is_fullscreen && !window_options.unredir_ignore) {
-				unredir_possible = true;
-			}
+		if (ps->o.unredir_if_possible && is_highest && w->mode == WMODE_SOLID &&
+		    !ps->o.force_win_blend && w->is_fullscreen &&
+		    (window_options.unredir == WINDOW_UNREDIR_WHEN_POSSIBLE ||
+		     window_options.unredir == WINDOW_UNREDIR_WHEN_POSSIBLE_ELSE_TERMINATE)) {
+			unredir_possible = true;
 		}
 
-		// Unredirect screen if some window is requesting compositor bypass, even
-		// if that window is not on the top.
-		if (ps->o.unredir_if_possible && win_is_bypassing_compositor(ps, w) &&
-		    !window_options.unredir_ignore) {
-			// Here we deviate from EWMH a bit. EWMH says we must not
-			// unredirect the screen if the window requesting bypassing would
-			// look different after unredirecting. Instead we always follow
-			// the request.
+		// Unredirect screen if some window is forcing unredir, even when they are
+		// not on the top.
+		if (ps->o.unredir_if_possible && window_options.unredir == WINDOW_UNREDIR_FORCE) {
 			unredir_possible = true;
 		}
 
 		w->prev_trans = bottom;
 		bottom = w;
 
-		// If the screen is not redirected and the window has redir_ignore set,
-		// this window should not cause the screen to become redirected
-		if (!(ps->o.wintype_option[w->window_type].redir_ignore && !ps->redirected)) {
+		// If the screen is not redirected check if the window's unredir setting
+		// allows unredirection to be terminated.
+		if (ps->redirected || window_options.unredir == WINDOW_UNREDIR_TERMINATE ||
+		    window_options.unredir == WINDOW_UNREDIR_WHEN_POSSIBLE_ELSE_TERMINATE) {
+			// Setting is_highest to false will stop all windows stacked below
+			// from triggering unredirection. But if `unredir_possible` is
+			// already set, this will not prevent unredirection.
 			is_highest = false;
 		}
 
@@ -948,8 +947,11 @@ static bool paint_preprocess(session_t *ps, bool *animation, struct win **out_bo
 	if (ps->o.redirected_force != UNSET) {
 		unredir_possible = !ps->o.redirected_force;
 	} else if (ps->o.unredir_if_possible && is_highest && !ps->redirected) {
-		// If there's no window to paint, and the screen isn't redirected,
-		// don't redirect it.
+		// `is_highest` being true means there's no window with a unredir setting
+		// that allows unredirection to be terminated. So if screen is not
+		// redirected, keep it that way.
+		//
+		// (might not be the best naming.)
 		unredir_possible = true;
 	}
 	if (unredir_possible) {
@@ -1994,7 +1996,7 @@ static struct window_options win_options_from_config(const struct options *opts)
 	    .invert_color = false,
 	    .paint = true,
 	    .clip_shadow_above = false,
-	    .unredir_ignore = false,
+	    .unredir = WINDOW_UNREDIR_WHEN_POSSIBLE_ELSE_TERMINATE,
 	    .opacity = 1,
 	};
 }

src/wm/win.c

@@ -1047,7 +1047,7 @@ void win_on_factor_change(session_t *ps, struct win *w) {
 		win_update_opacity_rule(ps, w);
 		w->opacity = win_calc_opacity_target(ps, w, focused);
 		w->options.paint = TRI_UNKNOWN;
-		w->options.unredir_ignore = TRI_UNKNOWN;
+		w->options.unredir = WINDOW_UNREDIR_INVALID;
 		w->options.fade = TRI_UNKNOWN;
 		w->options.transparent_clipping = TRI_UNKNOWN;
 		if (w->a.map_state == XCB_MAP_STATE_VIEWABLE &&
@@ -1056,8 +1056,21 @@ void win_on_factor_change(session_t *ps, struct win *w) {
 		}
 		if (w->a.map_state == XCB_MAP_STATE_VIEWABLE &&
 		    c2_match(ps->c2_state, w, ps->o.unredir_if_possible_blacklist, NULL)) {
-			w->options.unredir_ignore = TRI_TRUE;
+			if (ps->o.wintype_option[w->window_type].redir_ignore) {
+				w->options.unredir = WINDOW_UNREDIR_PASSIVE;
+			} else {
+				w->options.unredir = WINDOW_UNREDIR_TERMINATE;
+			}
+		} else if (win_is_bypassing_compositor(ps, w)) {
+			// Here we deviate from EWMH a bit. EWMH says we must not
+			// unredirect the screen if the window requesting bypassing would
+			// look different after unredirecting. Instead we always follow
+			// the request.
+			w->options.unredir = WINDOW_UNREDIR_FORCE;
+		} else if (ps->o.wintype_option[w->window_type].redir_ignore) {
+			w->options.unredir = WINDOW_UNREDIR_WHEN_POSSIBLE;
 		}
+
 		if (c2_match(ps->c2_state, w, ps->o.fade_blacklist, NULL)) {
 			w->options.fade = TRI_FALSE;
 		}
@@ -1076,6 +1089,12 @@ void win_on_factor_change(session_t *ps, struct win *w) {
 		if (safe_isnan(w->options.opacity) && w->has_opacity_prop) {
 			w->options.opacity = ((double)w->opacity_prop) / OPAQUE;
 		}
+		if (w->options.unredir == WINDOW_UNREDIR_INVALID &&
+		    win_is_bypassing_compositor(ps, w)) {
+			// If `unredir` is not set by a rule, we follow the bypassing
+			// compositor property.
+			w->options.unredir = WINDOW_UNREDIR_FORCE;
+		}
 		w->opacity = win_options(w).opacity;
 	}
 

src/wm/win.h

@@ -307,7 +307,7 @@ static const struct window_maybe_options WIN_MAYBE_OPTIONS_DEFAULT = {
     .opacity = NAN,
     .shader = NULL,
     .corner_radius = -1,
-    .unredir_ignore = TRI_UNKNOWN,
+    .unredir = WINDOW_UNREDIR_INVALID,
 };
 
 /// Combine two window options. The `upper` value has higher priority, the `lower` value
@@ -316,7 +316,7 @@ static const struct window_maybe_options WIN_MAYBE_OPTIONS_DEFAULT = {
 static inline struct window_maybe_options __attribute__((always_inline))
 win_maybe_options_fold(struct window_maybe_options upper, struct window_maybe_options lower) {
 	return (struct window_maybe_options){
-	    .unredir_ignore = tri_or(upper.unredir_ignore, lower.unredir_ignore),
+	    .unredir = upper.unredir == WINDOW_UNREDIR_INVALID ? lower.unredir : upper.unredir,
 	    .blur_background = tri_or(upper.blur_background, lower.blur_background),
 	    .clip_shadow_above = tri_or(upper.clip_shadow_above, lower.clip_shadow_above),
 	    .shadow = tri_or(upper.shadow, lower.shadow),
@@ -334,8 +334,9 @@ win_maybe_options_fold(struct window_maybe_options upper, struct window_maybe_op
 /// values that are not set in the `window_maybe_options`.
 static inline struct window_options __attribute__((always_inline))
 win_maybe_options_or(struct window_maybe_options maybe, struct window_options def) {
+	assert(def.unredir != WINDOW_UNREDIR_INVALID);
 	return (struct window_options){
-	    .unredir_ignore = tri_or_bool(maybe.unredir_ignore, def.unredir_ignore),
+	    .unredir = maybe.unredir == WINDOW_UNREDIR_INVALID ? def.unredir : maybe.unredir,
 	    .blur_background = tri_or_bool(maybe.blur_background, def.blur_background),
 	    .clip_shadow_above = tri_or_bool(maybe.clip_shadow_above, def.clip_shadow_above),
 	    .shadow = tri_or_bool(maybe.shadow, def.shadow),