Drive circuit breaker state transitions via internal/clock
The Open->HalfOpen promotion used time.Now/time.Since directly, forcing tests to use real time.Sleep and diverging from the project's clock convention. Add an unexported withClock option (default clock.System) and replace the real sleeps in tests with mock-clock Advance, making the transitions deterministic and the package faster.
This commit is contained in:
@@ -72,7 +72,7 @@ func (b *Breaker) State() State {
|
|||||||
// stateLocked returns the effective state, promoting Open → HalfOpen when the
|
// stateLocked returns the effective state, promoting Open → HalfOpen when the
|
||||||
// open duration has elapsed. Caller must hold b.mu.
|
// open duration has elapsed. Caller must hold b.mu.
|
||||||
func (b *Breaker) stateLocked() State {
|
func (b *Breaker) stateLocked() State {
|
||||||
if b.state == StateOpen && time.Since(b.openedAt) >= b.opts.openDuration {
|
if b.state == StateOpen && b.opts.clk.Since(b.openedAt) >= b.opts.openDuration {
|
||||||
b.state = StateHalfOpen
|
b.state = StateHalfOpen
|
||||||
b.halfOpenCur = 0
|
b.halfOpenCur = 0
|
||||||
}
|
}
|
||||||
@@ -142,7 +142,7 @@ func (b *Breaker) record(success bool) {
|
|||||||
// tripLocked transitions to the Open state and records the timestamp.
|
// tripLocked transitions to the Open state and records the timestamp.
|
||||||
func (b *Breaker) tripLocked() {
|
func (b *Breaker) tripLocked() {
|
||||||
b.state = StateOpen
|
b.state = StateOpen
|
||||||
b.openedAt = time.Now()
|
b.openedAt = b.opts.clk.Now()
|
||||||
b.halfOpenCur = 0
|
b.halfOpenCur = 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.codelab.vc/pkg/httpx/internal/clock"
|
||||||
"git.codelab.vc/pkg/httpx/middleware"
|
"git.codelab.vc/pkg/httpx/middleware"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -80,9 +81,11 @@ func TestBreaker_OpenRejectsRequests(t *testing.T) {
|
|||||||
|
|
||||||
func TestBreaker_TransitionsToHalfOpenAfterDuration(t *testing.T) {
|
func TestBreaker_TransitionsToHalfOpenAfterDuration(t *testing.T) {
|
||||||
const openDuration = 50 * time.Millisecond
|
const openDuration = 50 * time.Millisecond
|
||||||
|
clk := clock.Mock(time.Now())
|
||||||
b := NewBreaker(
|
b := NewBreaker(
|
||||||
WithFailureThreshold(1),
|
WithFailureThreshold(1),
|
||||||
WithOpenDuration(openDuration),
|
WithOpenDuration(openDuration),
|
||||||
|
withClock(clk),
|
||||||
)
|
)
|
||||||
|
|
||||||
// Trip the breaker.
|
// Trip the breaker.
|
||||||
@@ -96,8 +99,8 @@ func TestBreaker_TransitionsToHalfOpenAfterDuration(t *testing.T) {
|
|||||||
t.Fatalf("state = %v, want %v", s, StateOpen)
|
t.Fatalf("state = %v, want %v", s, StateOpen)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Wait for the open duration to elapse.
|
// Advance past the open duration.
|
||||||
time.Sleep(openDuration + 10*time.Millisecond)
|
clk.Advance(openDuration + time.Millisecond)
|
||||||
|
|
||||||
if s := b.State(); s != StateHalfOpen {
|
if s := b.State(); s != StateHalfOpen {
|
||||||
t.Fatalf("state = %v, want %v", s, StateHalfOpen)
|
t.Fatalf("state = %v, want %v", s, StateHalfOpen)
|
||||||
@@ -106,9 +109,11 @@ func TestBreaker_TransitionsToHalfOpenAfterDuration(t *testing.T) {
|
|||||||
|
|
||||||
func TestBreaker_HalfOpenToClosedOnSuccess(t *testing.T) {
|
func TestBreaker_HalfOpenToClosedOnSuccess(t *testing.T) {
|
||||||
const openDuration = 50 * time.Millisecond
|
const openDuration = 50 * time.Millisecond
|
||||||
|
clk := clock.Mock(time.Now())
|
||||||
b := NewBreaker(
|
b := NewBreaker(
|
||||||
WithFailureThreshold(1),
|
WithFailureThreshold(1),
|
||||||
WithOpenDuration(openDuration),
|
WithOpenDuration(openDuration),
|
||||||
|
withClock(clk),
|
||||||
)
|
)
|
||||||
|
|
||||||
// Trip the breaker.
|
// Trip the breaker.
|
||||||
@@ -118,8 +123,8 @@ func TestBreaker_HalfOpenToClosedOnSuccess(t *testing.T) {
|
|||||||
}
|
}
|
||||||
done(false)
|
done(false)
|
||||||
|
|
||||||
// Wait for half-open.
|
// Advance into half-open.
|
||||||
time.Sleep(openDuration + 10*time.Millisecond)
|
clk.Advance(openDuration + time.Millisecond)
|
||||||
|
|
||||||
// A successful request in half-open should close the breaker.
|
// A successful request in half-open should close the breaker.
|
||||||
done, err = b.Allow()
|
done, err = b.Allow()
|
||||||
@@ -135,9 +140,11 @@ func TestBreaker_HalfOpenToClosedOnSuccess(t *testing.T) {
|
|||||||
|
|
||||||
func TestBreaker_HalfOpenToOpenOnFailure(t *testing.T) {
|
func TestBreaker_HalfOpenToOpenOnFailure(t *testing.T) {
|
||||||
const openDuration = 50 * time.Millisecond
|
const openDuration = 50 * time.Millisecond
|
||||||
|
clk := clock.Mock(time.Now())
|
||||||
b := NewBreaker(
|
b := NewBreaker(
|
||||||
WithFailureThreshold(1),
|
WithFailureThreshold(1),
|
||||||
WithOpenDuration(openDuration),
|
WithOpenDuration(openDuration),
|
||||||
|
withClock(clk),
|
||||||
)
|
)
|
||||||
|
|
||||||
// Trip the breaker.
|
// Trip the breaker.
|
||||||
@@ -147,8 +154,8 @@ func TestBreaker_HalfOpenToOpenOnFailure(t *testing.T) {
|
|||||||
}
|
}
|
||||||
done(false)
|
done(false)
|
||||||
|
|
||||||
// Wait for half-open.
|
// Advance into half-open.
|
||||||
time.Sleep(openDuration + 10*time.Millisecond)
|
clk.Advance(openDuration + time.Millisecond)
|
||||||
|
|
||||||
// A failed request in half-open should re-open the breaker.
|
// A failed request in half-open should re-open the breaker.
|
||||||
done, err = b.Allow()
|
done, err = b.Allow()
|
||||||
|
|||||||
@@ -1,11 +1,16 @@
|
|||||||
package circuitbreaker
|
package circuitbreaker
|
||||||
|
|
||||||
import "time"
|
import (
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.codelab.vc/pkg/httpx/internal/clock"
|
||||||
|
)
|
||||||
|
|
||||||
type options struct {
|
type options struct {
|
||||||
failureThreshold int // consecutive failures to trip
|
failureThreshold int // consecutive failures to trip
|
||||||
openDuration time.Duration // how long to stay open before half-open
|
openDuration time.Duration // how long to stay open before half-open
|
||||||
halfOpenMax int // max concurrent requests in half-open
|
halfOpenMax int // max concurrent requests in half-open
|
||||||
|
clk clock.Clock // time source (real by default)
|
||||||
}
|
}
|
||||||
|
|
||||||
func defaults() options {
|
func defaults() options {
|
||||||
@@ -13,12 +18,23 @@ func defaults() options {
|
|||||||
failureThreshold: 5,
|
failureThreshold: 5,
|
||||||
openDuration: 30 * time.Second,
|
openDuration: 30 * time.Second,
|
||||||
halfOpenMax: 1,
|
halfOpenMax: 1,
|
||||||
|
clk: clock.System(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Option configures a Breaker.
|
// Option configures a Breaker.
|
||||||
type Option func(*options)
|
type Option func(*options)
|
||||||
|
|
||||||
|
// withClock sets the clock used for state-transition timing. Unexported; for
|
||||||
|
// deterministic tests.
|
||||||
|
func withClock(c clock.Clock) Option {
|
||||||
|
return func(o *options) {
|
||||||
|
if c != nil {
|
||||||
|
o.clk = c
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// WithFailureThreshold sets the number of consecutive failures required to
|
// WithFailureThreshold sets the number of consecutive failures required to
|
||||||
// trip the breaker from Closed to Open. Default is 5.
|
// trip the breaker from Closed to Open. Default is 5.
|
||||||
func WithFailureThreshold(n int) Option {
|
func WithFailureThreshold(n int) Option {
|
||||||
|
|||||||
Reference in New Issue
Block a user