B: forcibly lock resource when creating scope
more details in the included comment block, but tl;dr we need to
explicitly lock the relevant resource before deleting any existing
resource_config_scopes for that resource, since otherwise we
(implicitly) acquire locks in a cycle when concurrently setting
pipelines
@@ -262,12 +262,43 @@ func findOrCreateResourceConfigScope(
return nil, err
}
} else if uniqueResource != nil {
+ // This `SELECT ... FOR UPDATE` on the resource is just to avoid a
+ // deadlock, which occurs when concurrently setting a pipeline and
+ // running FindOrCreateScope on the resource that's being updated in
+ // the pipeline. Specifically, it happens with the following "DELETE
+ // FROM resource_config_scopes" query - this deletes the old resource
+ // config scope, which in turn triggers an "ON DELETE SET NULL" in the
+ // resource. However, there's some implicit lock that's acquired when
+ // setting the pipeline on the resource_config_scope, and without this
+ // dummy query, the locks are acquired in a bad order wrt one another:
+ //
+ // DELETE FROM resource_config_scopes:
+ // 1. Lock resource_config_scopes
+ // 2. Lock resource
+ //
+ // INSERT INTO resources (occurs when setting the pipeline):
+ // 1. Lock resource
+ // 2. Lock resource_config_scope
+ //
+ // Thus, forcing the DELETE FROM resource_config_scopes query to
+ // acquire a lock on the affected resource fixes this order (first
+ // resource, then resource_config_scope) to avoid a cycle.
+ _, err := psql.Select("1").
+ From("resources").
+ Where(sq.Eq{
+ "id": uniqueResource.ID(),
+ }).
+ Suffix("FOR UPDATE").
+ RunWith(tx).
+ Exec()
+ if err != nil {
+ return nil, err
+ }
+
// delete outdated scopes for resource
- _, err := psql.Delete("resource_config_scopes").
- Where(sq.And{
- sq.Eq{
+ _, err = psql.Delete("resource_config_scopes").
+ Where(sq.Eq{
"resource_id": resource.ID(),
- },
}).
RunWith(tx).
Exec()
