diff --git a/include/sway/config.h b/include/sway/config.h
index d9f561571d..13f576b00f 100644
--- a/include/sway/config.h
+++ b/include/sway/config.h
@@ -296,14 +296,6 @@ struct output_config {
 	char *background_fallback;
 };
 
-/**
- * An output config pre-matched to an output
- */
-struct matched_output_config {
-	struct sway_output *output;
-	struct output_config *config;
-};
-
 /**
  * Stores size of gaps for each side
  */
@@ -693,13 +685,10 @@ const char *sway_output_scale_filter_to_string(enum scale_filter_mode scale_filt
 
 struct output_config *new_output_config(const char *name);
 
-bool apply_output_configs(struct matched_output_config *configs,
-		size_t configs_len, bool test_only, bool degrade_to_off);
-
-void apply_all_output_configs(void);
+bool apply_output_configs(struct output_config **ocs, size_t ocs_len,
+		bool test_only, bool degrade_to_off);
 
-void sort_output_configs_by_priority(struct matched_output_config *configs,
-		size_t configs_len);
+void apply_stored_output_configs(void);
 
 /**
  * store_output_config stores a new output config. An output may be matched by
diff --git a/sway/commands/output.c b/sway/commands/output.c
index 9478e0badd..3f65b909c0 100644
--- a/sway/commands/output.c
+++ b/sway/commands/output.c
@@ -111,7 +111,7 @@ struct cmd_results *cmd_output(int argc, char **argv) {
 	// entire config and before the deferred commands so that an auto generated
 	// workspace name is not given to re-enabled outputs.
 	if (!config->reloading && !config->validating) {
-		apply_all_output_configs();
+		apply_stored_output_configs();
 		if (background) {
 			if (!spawn_swaybg()) {
 				return cmd_results_new(CMD_FAILURE,
diff --git a/sway/config.c b/sway/config.c
index 5058efcc03..5fc414a142 100644
--- a/sway/config.c
+++ b/sway/config.c
@@ -533,7 +533,7 @@ bool load_main_config(const char *file, bool is_active, bool validating) {
 		}
 		sway_switch_retrigger_bindings_for_all();
 
-		apply_all_output_configs();
+		apply_stored_output_configs();
 		spawn_swaybg();
 
 		config->reloading = false;
diff --git a/sway/config/output.c b/sway/config/output.c
index cc6aedb778..4a6ba1555c 100644
--- a/sway/config/output.c
+++ b/sway/config/output.c
@@ -25,13 +25,6 @@
 #include <wlr/backend/drm.h>
 #endif
 
-int output_name_cmp(const void *item, const void *data) {
-	const struct output_config *output = item;
-	const char *name = data;
-
-	return strcmp(output->name, name);
-}
-
 void output_get_identifier(char *identifier, size_t len,
 		struct sway_output *output) {
 	struct wlr_output *wlr_output = output->wlr_output;
@@ -591,69 +584,52 @@ static bool finalize_output_config(struct output_config *oc, struct sway_output
 	return true;
 }
 
-static void default_output_config(struct output_config *oc,
-		struct wlr_output *wlr_output) {
-	oc->enabled = 1;
-	oc->power = 1;
-	struct wlr_output_mode *mode = wlr_output_preferred_mode(wlr_output);
-	if (mode != NULL) {
-		oc->width = mode->width;
-		oc->height = mode->height;
-		oc->refresh_rate = mode->refresh / 1000.f;
-	}
-	oc->x = oc->y = -1;
-	oc->scale = 0; // auto
-	oc->scale_filter = SCALE_FILTER_DEFAULT;
-	struct sway_output *output = wlr_output->data;
-	oc->subpixel = output->detected_subpixel;
-	oc->transform = WL_OUTPUT_TRANSFORM_NORMAL;
-	oc->max_render_time = 0;
-	oc->allow_tearing = 0;
-}
-
-// find_output_config returns a merged output_config containing all stored
-// configuration that applies to the specified output.
-struct output_config *find_output_config(struct sway_output *sway_output) {
+// find_output_config_from_list returns a merged output_config containing all
+// stored configuration that applies to the specified output.
+static struct output_config *find_output_config_from_list(
+		struct output_config **configs, size_t configs_len,
+		struct sway_output *sway_output) {
 	const char *name = sway_output->wlr_output->name;
-	struct output_config *oc = NULL;
-
 	struct output_config *result = new_output_config(name);
-	if (config->reloading) {
-		default_output_config(result, sway_output->wlr_output);
+	if (result == NULL) {
+		return NULL;
 	}
 
+	// Set output defaults for the "base" configuration
+	result->enabled = 1;
+	result->power = 1;
+	result->scale = 0; // auto
+	result->subpixel = sway_output->detected_subpixel;
+	result->transform = WL_OUTPUT_TRANSFORM_NORMAL;
+	result->max_render_time = 0;
+	result->allow_tearing = 0;
+
 	char id[128];
 	output_get_identifier(id, sizeof(id), sway_output);
 
-	int i;
-	bool match = false;
-	if ((i = list_seq_find(config->output_configs, output_name_cmp, "*")) >= 0) {
-		match = true;
-		oc = config->output_configs->items[i];
-		merge_output_config(result, oc);
-	}
-	if ((i = list_seq_find(config->output_configs, output_name_cmp, name)) >= 0) {
-		match = true;
-		oc = config->output_configs->items[i];
-		merge_output_config(result, oc);
-	}
-	if ((i = list_seq_find(config->output_configs, output_name_cmp, id)) >= 0) {
-		match = true;
-		oc = config->output_configs->items[i];
-		merge_output_config(result, oc);
-	}
-
-	if (!match && !config->reloading) {
-		// No name, identifier, or wildcard config. Since we are not
-		// reloading with defaults, the output config will be empty, so
-		// just return NULL
-		free_output_config(result);
-		return NULL;
+	// We take a new config and merge on top, in order, the wildcard config,
+	// output config by name, and output config by identifier to form the final
+	// config. If there are multiple matches, they are merged in order.
+	struct output_config *oc = NULL;
+	const char *names[] = {"*", name, id, NULL};
+	for (const char **name = &names[0]; *name; name++) {
+		for (size_t idx = 0; idx < configs_len; idx++) {
+			oc = configs[idx];
+			if (strcmp(oc->name, *name) == 0) {
+				merge_output_config(result, oc);
+			}
+		}
 	}
 
 	return result;
 }
 
+struct output_config *find_output_config(struct sway_output *sway_output) {
+	return find_output_config_from_list(
+			(struct output_config **)config->output_configs->items,
+			config->output_configs->length, sway_output);
+}
+
 static bool config_has_manual_mode(struct output_config *oc) {
 	if (!oc) {
 		return false;
@@ -666,6 +642,14 @@ static bool config_has_manual_mode(struct output_config *oc) {
 	return false;
 }
 
+/**
+ * An output config pre-matched to an output
+ */
+struct matched_output_config {
+	struct sway_output *output;
+	struct output_config *config;
+};
+
 struct search_context {
 	struct wlr_output_swapchain_manager *swapchain_mgr;
 	struct wlr_backend_output_state *states;
@@ -884,12 +868,12 @@ static int compare_matched_output_config_priority(const void *a, const void *b)
 	return 0;
 }
 
-void sort_output_configs_by_priority(struct matched_output_config *configs,
-		size_t configs_len) {
+static void sort_output_configs_by_priority(
+		struct matched_output_config *configs, size_t configs_len) {
 	qsort(configs, configs_len, sizeof(*configs), compare_matched_output_config_priority);
 }
 
-bool apply_output_configs(struct matched_output_config *configs,
+static bool apply_resolved_output_configs(struct matched_output_config *configs,
 		size_t configs_len, bool test_only, bool degrade_to_off) {
 	struct wlr_backend_output_state *states = calloc(configs_len, sizeof(*states));
 	if (!states) {
@@ -997,11 +981,12 @@ bool apply_output_configs(struct matched_output_config *configs,
 	return ok;
 }
 
-void apply_all_output_configs(void) {
+bool apply_output_configs(struct output_config **ocs, size_t ocs_len,
+		bool test_only, bool degrade_to_off) {
 	size_t configs_len = wl_list_length(&root->all_outputs);
 	struct matched_output_config *configs = calloc(configs_len, sizeof(*configs));
 	if (!configs) {
-		return;
+		return false;
 	}
 
 	int config_idx = 0;
@@ -1014,16 +999,22 @@ void apply_all_output_configs(void) {
 
 		struct matched_output_config *config = &configs[config_idx++];
 		config->output = sway_output;
-		config->config = find_output_config(sway_output);
+		config->config = find_output_config_from_list(ocs, ocs_len, sway_output);
 	}
 
 	sort_output_configs_by_priority(configs, configs_len);
-	apply_output_configs(configs, configs_len, false, true);
+	bool ok = apply_resolved_output_configs(configs, configs_len, test_only, degrade_to_off);
 	for (size_t idx = 0; idx < configs_len; idx++) {
 		struct matched_output_config *cfg = &configs[idx];
 		free_output_config(cfg->config);
 	}
 	free(configs);
+	return ok;
+}
+
+void apply_stored_output_configs(void) {
+	apply_output_configs((struct output_config **)config->output_configs->items,
+			config->output_configs->length, false, true);
 }
 
 void free_output_config(struct output_config *oc) {
diff --git a/sway/desktop/output.c b/sway/desktop/output.c
index 8711a24811..0b0d5291ee 100644
--- a/sway/desktop/output.c
+++ b/sway/desktop/output.c
@@ -386,7 +386,7 @@ static int timer_modeset_handle(void *data) {
 	wl_event_source_remove(server->delayed_modeset);
 	server->delayed_modeset = NULL;
 
-	apply_all_output_configs();
+	apply_stored_output_configs();
 	return 0;
 }
 
@@ -544,9 +544,8 @@ void handle_new_output(struct wl_listener *listener, void *data) {
 }
 
 static struct output_config *output_config_for_config_head(
-		struct wlr_output_configuration_head_v1 *config_head,
-		struct sway_output *output) {
-	struct output_config *oc = new_output_config(output->wlr_output->name);
+		struct wlr_output_configuration_head_v1 *config_head) {
+	struct output_config *oc = new_output_config(config_head->state.output->name);
 	oc->enabled = config_head->state.enabled;
 	if (!oc->enabled) {
 		return oc;
@@ -572,67 +571,48 @@ static struct output_config *output_config_for_config_head(
 }
 
 static void output_manager_apply(struct sway_server *server,
-		struct wlr_output_configuration_v1 *config, bool test_only) {
-	size_t configs_len = wl_list_length(&root->all_outputs);
-	struct matched_output_config *configs = calloc(configs_len, sizeof(*configs));
+		struct wlr_output_configuration_v1 *cfg, bool test_only) {
+	bool ok = false;
+	size_t configs_len = config->output_configs->length + wl_list_length(&cfg->heads);
+	struct output_config **configs = calloc(configs_len, sizeof(*configs));
 	if (!configs) {
-		return;
+		goto done;
 	}
-
-	int config_idx = 0;
-	struct sway_output *sway_output;
-	wl_list_for_each(sway_output, &root->all_outputs, link) {
-		if (sway_output == root->fallback_output) {
-			configs_len--;
-			continue;
-		}
-
-		struct matched_output_config *cfg = &configs[config_idx++];
-		cfg->output = sway_output;
-
-		struct wlr_output_configuration_head_v1 *config_head;
-		wl_list_for_each(config_head, &config->heads, link) {
-			if (config_head->state.output == sway_output->wlr_output) {
-				cfg->config = output_config_for_config_head(config_head, sway_output);
-				break;
-			}
-		}
-		if (!cfg->config) {
-			cfg->config = find_output_config(sway_output);
-		}
+	size_t start_new_configs = config->output_configs->length;
+	for (size_t idx = 0; idx < start_new_configs; idx++) {
+		configs[idx] = config->output_configs->items[idx];
 	}
 
-	sort_output_configs_by_priority(configs, configs_len);
-	bool ok = apply_output_configs(configs, configs_len, test_only, false);
-	for (size_t idx = 0; idx < configs_len; idx++) {
-		struct matched_output_config *cfg = &configs[idx];
+	size_t config_idx = start_new_configs;
+	struct wlr_output_configuration_head_v1 *config_head;
+	wl_list_for_each(config_head, &cfg->heads, link) {
+		// Generate the configuration and store it as a temporary
+		// config. We keep a record of it so we can remove it later.
+		struct output_config *oc = output_config_for_config_head(config_head);
+		configs[config_idx++] = oc;
+	}
 
-		// Only store new configs for successful non-test commits. Old configs,
-		// test-only and failed commits just get freed.
-		bool store_config = false;
+	// Try to commit without degrade to off enabled. Note that this will fail
+	// if any output configured for enablement fails to be enabled, even if it
+	// was not part of the config heads we were asked to configure.
+	ok = apply_output_configs(configs, configs_len, test_only, false);
+	for (size_t idx = start_new_configs; idx < configs_len; idx++) {
+		struct output_config *cfg = configs[idx];
 		if (!test_only && ok) {
-			struct wlr_output_configuration_head_v1 *config_head;
-			wl_list_for_each(config_head, &config->heads, link) {
-				if (config_head->state.output == cfg->output->wlr_output) {
-					store_config = true;
-					break;
-				}
-			}
-		}
-		if (store_config) {
-			store_output_config(cfg->config);
+			store_output_config(cfg);
 		} else {
-			free_output_config(cfg->config);
+			free_output_config(cfg);
 		}
 	}
 	free(configs);
 
+done:
 	if (ok) {
-		wlr_output_configuration_v1_send_succeeded(config);
+		wlr_output_configuration_v1_send_succeeded(cfg);
 	} else {
-		wlr_output_configuration_v1_send_failed(config);
+		wlr_output_configuration_v1_send_failed(cfg);
 	}
-	wlr_output_configuration_v1_destroy(config);
+	wlr_output_configuration_v1_destroy(cfg);
 }
 
 void handle_output_manager_apply(struct wl_listener *listener, void *data) {