[MINOR BC] Fix pending tenant pull race conditions (#1463)

> Minor breaking change: clearing pending_since no longer fires Eloquent
events, PullingPendingTenant is now fired at a different point in the
lifecycle and does not guarantee the tenant will actually be pulled.

`pullPendingFromPool` had a race condition when user A attempted to pull
a tenant at the same time as user B. Both could end up grabbing the same
tenant, and the result was unexpected, e.g. one of them ending up with
no pending tenant pulled at all even though there was a pending tenant
in the pool.

instead of selecting a pending tenant and updating the same model, we
now run `update()` conditionally -- it clears `pending_since` _only_ if
the tenant is still pending, and we check the affected row count. Only
one process can get a row back, the other gets 0 and retries with the
next pending candidate in the pool. The loop always terminates since
every lost claim means the pool shrank by one. Eventually it's empty and
we create a new tenant (or return null).

The claim and the attribute update happen in a single transaction now,
so if updating `$attributes` fails, the claim rolls back and the tenant
stays in the pool.

Added a regression test that simulates a concurrent "steal"
synchronously via a PullingPendingTenant listener. Fails with the old
code, passes with the HasPending changes.

Very minor BC:
- Clearing `pending_since` no longer fires model updating/updated events
(since the update goes through query builder). `PendingTenantPulled`
still fires the same as before and is the listener you'd want to use
anyway.
- `PullingPendingTenant` now fires before the claim (and outside the
transaction), so it can fire more than once with concurrent pulls (e.g.
when a tenant gets claimed by someone else). `PendingTenantPulled` is
still the one that fires exactly once for the actually pulled tenant.

---------

Co-authored-by: Samuel Stancl <samuel@archte.ch>

src/Database/Concerns/HasPending.php

@@ -100,27 +100,51 @@ trait HasPending
      */
     public static function pullPendingFromPool(bool $firstOrCreate = false, array $attributes = []): ?Tenant
     {
-        $tenant = DB::transaction(function () use ($attributes): ?Tenant {
+        // Attempt pulling a pending tenant.
-            /** @var (Model&Tenant)|null $tenant */
+        // The loop handles the case where a single tenant is being pulled by multiple processes at the same time.
-            $tenant = static::onlyPending()->first();
+        // If a tenant was pulled by a concurrent process, try pulling the next one in the pool.
+        while (true) {
+            /** @var (Model&Tenant)|null $pullCandidate */
+            $pullCandidate = static::onlyPending()->first();
 
-            if ($tenant !== null) {
+            if ($pullCandidate === null) {
-                event(new PullingPendingTenant($tenant));
+                return $firstOrCreate ? static::create($attributes) : null;
-                $tenant->update(array_merge($attributes, [
-                    'pending_since' => null,
-                ]));
             }
 
+            // Fired before the claim, so it can fire once per attempt, including for a candidate
+            // that ends up being claimed by a different process (in which case the loop retries).
+            // PendingTenantPulled (below) fires exactly once, for the actually pulled tenant.
+            event(new PullingPendingTenant($pullCandidate));
+
+            $tenant = DB::transaction(function () use ($pullCandidate, $attributes): ?Tenant {
+                $tenantWasPulled = static::onlyPending()
+                    ->whereKey($pullCandidate->getKey())
+                    ->update([$pullCandidate->getColumnForQuery('pending_since') => null]) > 0;
+
+                if (! $tenantWasPulled) {
+                    return null;
+                }
+
+                // The tenant's pending_since was just cleared, and a PullingPendingTenant listener
+                // may have made changes to the tenant, so re-fetch it to make sure it's up to date.
+                /** @var Model&Tenant $pulledTenant */
+                $pulledTenant = static::findOrFail($pullCandidate->getKey());
+
+                if (! empty($attributes)) {
+                    $pulledTenant->update($attributes);
+                }
+
+                return $pulledTenant;
+            });
+
+            if ($tenant === null) {
+                // If another pull claimed this tenant first, try claiming the next one
+                continue;
+            }
+
+            event(new PendingTenantPulled($tenant));
+
             return $tenant;
-        });
-
-        if ($tenant === null) {
-            return $firstOrCreate ? static::create($attributes) : null;
         }
-
-        // Only triggered if a tenant that was pulled from the pool is returned
-        event(new PendingTenantPulled($tenant));
-
-        return $tenant;
     }
 }

tests/PendingTenantsTest.php

@@ -5,6 +5,7 @@ declare(strict_types=1);
 use Illuminate\Database\QueryException;
 use Illuminate\Database\Schema\Blueprint;
 use Illuminate\Support\Facades\Artisan;
+use Illuminate\Support\Facades\DB;
 use Illuminate\Support\Facades\Event;
 use Illuminate\Support\Facades\Schema;
 use Illuminate\Support\Str;
@@ -126,6 +127,52 @@ test('a new tenant gets created while pulling a pending tenant if the pending po
     expect(Tenant::withPending()->get()->count())->toBe(1); // All tenants
 });
 
+test('pulling a pending tenant retries when the tenant is claimed concurrently', function () {
+    Tenant::createPending();
+    Tenant::createPending();
+
+    $stolenId = null;
+
+    Event::listen(PullingPendingTenant::class, function (PullingPendingTenant $event) use (&$stolenId) {
+        if ($stolenId !== null) {
+            return;
+        }
+
+        $stolenId = $event->tenant->id;
+
+        // Steal the tenant like a concurrent process would
+        Tenant::onlyPending()
+            ->whereKey($event->tenant->id)
+            ->update([$event->tenant->getColumnForQuery('pending_since') => null]);
+    });
+
+    $pulled = Tenant::pullPendingFromPool();
+
+    expect($pulled)->not()->toBeNull();
+    expect($pulled->id)->not()->toBe($stolenId); // Stolen tenant was skipped, the next one was claimed by the pull
+    expect(Tenant::onlyPending()->count())->toBe(0); // Both tenants claimed
+});
+
+test('the pull is rolled back and the tenant stays in the pool if setting attributes fails', function () {
+    // Pulling a tenant and setting its attributes happen in one transaction,
+    // so if setting the attributes fails, the whole pull rolls back and the tenant stays in the pool.
+    Schema::table('tenants', function (Blueprint $table) {
+        $table->string('slug')->nullable()->unique();
+    });
+
+    Tenant::$extraCustomColumns = ['slug'];
+
+    Tenant::create(['slug' => 'taken']);
+    Tenant::createPending();
+
+    // During the pull, set slug to 'taken', which is already used by another tenant to make the attribute update throw
+    expect(fn () => Tenant::pullPendingFromPool(false, ['slug' => 'taken']))
+        ->toThrow(QueryException::class);
+
+    // The pull rolled back, so the tenant is still pending
+    expect(Tenant::onlyPending()->count())->toBe(1);
+});
+
 test('withoutPending chained with where clauses returns correct results', function () {
     $tenant = Tenant::create();
     $pendingTenant = Tenant::createPending();