Circuit breakers are designed to fail open, meaning they’ll stop traffic to a service that’s misbehaving. But what happens when that circuit breaker opens?
Here’s a service user-service that depends on profile-service.
package main
import (
"fmt"
"log"
"net/http"
"time"
"github.com/sony/gobreaker"
)
var profileService *gobreaker.CircuitBreaker
func init() {
profileService = gobreaker.NewCircuitBreaker(gobreaker.Settings{
Name: "profile-service",
// After 5 consecutive failures, open the circuit.
MaxRequests: 5,
// Reset after 1 minute.
Interval: time.Minute,
// If the service is healthy, return to closed.
ReadyToTrip: func(counts gobreaker.Counts) bool {
return counts.ConsecutiveFailures >= 5
},
})
}
func getUserProfile(userID string) (string, error) {
// Try to get profile data from profile-service.
// If the circuit breaker is open, this will immediately return an error.
_, err := profileService.Execute(func() (interface{}, error) {
resp, err := http.Get(fmt.Sprintf("http://localhost:8081/profiles/%s", userID))
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("profile service returned status %d", resp.StatusCode)
}
// In a real app, you'd read the body here.
return "profile data", nil
})
if err != nil {
return "", fmt.Errorf("failed to get profile for user %s: %w", userID, err)
}
return "profile data", nil
}
func main() {
http.HandleFunc("/users/profile", func(w http.ResponseWriter, r *http.Request) {
userID := r.URL.Query().Get("id")
if userID == "" {
http.Error(w, "Missing user ID", http.StatusBadRequest)
return
}
profile, err := getUserProfile(userID)
if err != nil {
// This is where we need a fallback strategy.
log.Printf("Error getting profile: %v", err)
http.Error(w, "Could not retrieve profile information", http.StatusInternalServerError)
return
}
fmt.Fprintf(w, "User Profile: %s", profile)
})
log.Println("Starting user-service on :8080")
log.Fatal(http.ListenAndServe(":8080", nil))
}
Imagine profile-service on localhost:8081 is down.
# Start a dummy profile service that immediately fails
go run - "github.com/go-chi/http-proxy" --target http://localhost:9999 # A non-existent service
Now, when user-service tries to call profile-service, the http.Get will fail. After 5 such failures, profileService circuit breaker will trip.
curl "http://localhost:8080/users/profile?id=123"
# ... (after 5 failures, this will start returning an error)
curl "http://localhost:8080/users/profile?id=123"
# Output: Error: Could not retrieve profile information
The circuit breaker is open. It’s protecting user-service from hammering a broken profile-service. But what should user-service do now? It can’t get the real profile.
The most common fallback is to serve stale or cached data. If you have a cache (like Redis or Memcached) holding profile information, user-service can attempt to fetch from there first. If the cache hit is successful, you serve that. If the cache also fails or is empty, then you try the downstream service. When the downstream service fails (and the circuit breaker opens), you fall back to the cache.
// Add this to your imports
import "github.com/go-redis/redis/v8"
import "context"
// Add a Redis client
var redisClient *redis.Client
func init() {
// ... existing circuit breaker init ...
redisClient = redis.NewClient(&redis.Options{
Addr: "localhost:6379", // Replace with your Redis address
})
}
func getCachedProfile(ctx context.Context, userID string) (string, error) {
val, err := redisClient.Get(ctx, fmt.Sprintf("profile:%s", userID)).Result()
if err == redis.Nil {
return "", fmt.Errorf("cache miss")
} else if err != nil {
return "", fmt.Errorf("redis error: %w", err)
}
return val, nil
}
func setCachedProfile(ctx context.Context, userID string, data string) error {
return redisClient.Set(ctx, fmt.Sprintf("profile:%s", userID), data, time.Minute*5).Err() // Cache for 5 minutes
}
func getUserProfileWithFallback(ctx context.Context, userID string) (string, error) {
// 1. Try to get profile from the primary service (profile-service)
profileData, err := profileService.Execute(func() (interface{}, error) {
resp, err := http.Get(fmt.Sprintf("http://localhost:8081/profiles/%s", userID))
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("profile service returned status %d", resp.StatusCode)
}
// In a real app, you'd read the body here.
return "profile data from live service", nil
})
if err == nil {
// Successfully got live data, cache it and return.
go setCachedProfile(ctx, userID, profileData.(string)) // Cache in background
return profileData.(string), nil
}
// 2. If primary service failed, try the cache.
log.Printf("Primary service failed, attempting cache fallback: %v", err)
cachedData, cacheErr := getCachedProfile(ctx, userID)
if cacheErr == nil {
log.Printf("Successfully retrieved profile from cache for user %s", userID)
return cachedData, nil // Return cached data
}
// 3. If both failed, return an error.
log.Printf("Cache fallback also failed for user %s: %v", userID, cacheErr)
return "", fmt.Errorf("failed to get profile for user %s: primary service error (%v), cache error (%v)", userID, err, cacheErr)
}
func main() {
http.HandleFunc("/users/profile", func(w http.ResponseWriter, r *http.Request) {
userID := r.URL.Query().Get("id")
if userID == "" {
http.Error(w, "Missing user ID", http.StatusBadRequest)
return
}
profile, err := getUserProfileWithFallback(r.Context(), userID) // Use the new function
if err != nil {
log.Printf("Error getting profile: %v", err)
http.Error(w, "Could not retrieve profile information", http.StatusInternalServerError)
return
}
fmt.Fprintf(w, "User Profile: %s", profile)
})
log.Println("Starting user-service on :8080")
log.Fatal(http.ListenAndServe(":8080", nil))
}
Now, if profile-service is down, user-service will first try profile-service. When that fails and the circuit breaker opens, it will then try Redis. If Redis has data, it serves that.
Another fallback is to return a default or simplified response. For example, if the user profile isn’t available, user-service could return a generic "User Profile Unavailable" message, or just the user’s basic information (like their username, if that’s available from a different, more reliable source), without the detailed profile.
func getUserProfileWithDefault(ctx context.Context, userID string) (string, error) {
// Try to get profile data from profile-service.
profileData, err := profileService.Execute(func() (interface{}, error) {
resp, err := http.Get(fmt.Sprintf("http://localhost:8081/profiles/%s", userID))
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("profile service returned status %d", resp.StatusCode)
}
// In a real app, you'd read the body here.
return "profile data", nil
})
if err == nil {
// Successfully got live data.
return profileData.(string), nil
}
// If it failed, return a default response.
log.Printf("Primary service failed, returning default response: %v", err)
return fmt.Sprintf("Default profile data for user %s (details unavailable)", userID), nil
}
The gobreaker library’s Execute function returns an error immediately if the circuit is open. This allows you to wrap that call in a try-catch (or if err != nil in Go) and implement your fallback logic. The key is that the circuit breaker itself doesn’t implement the fallback; it just signals when to fall back by returning an error. You, as the developer, decide what that fallback looks like.
The most insidious fallback is doing nothing, or more accurately, letting the error propagate up without a specific strategy. This often results in the client seeing a generic server error (500 Internal Server Error) and losing the context of what could have been shown. The circuit breaker is there to prevent cascading failures, but a smart fallback strategy is what keeps your application functional and user-friendly during partial outages.
When the circuit breaker eventually resets and the downstream service becomes available again, your fallback logic should naturally stop being invoked, and the primary service will be used again. The transition back to normal operation should be as seamless as possible.
The next step after implementing fallbacks is to consider how to trigger alerts when a circuit breaker opens, so you’re aware of the degraded state even if the fallback is working perfectly.