From 160a84c074ea1164f2ca0c011ab8413fdd145860 Mon Sep 17 00:00:00 2001 From: Andrea Bueide Date: Fri, 6 Mar 2026 16:13:53 -0600 Subject: [PATCH 1/4] feat(core): add UploadStateMachine with rate limiting and core tests Add the UploadStateMachine component for managing global rate limiting state for 429 responses, along with supporting types and config validation. Components: - RateLimitConfig, UploadStateData, HttpConfig types - validateRateLimitConfig with SDD-specified bounds - UploadStateMachine with canUpload/handle429/reset/getGlobalRetryCount - Core test suite (10 tests) and test helpers - backoff/index.ts barrel export Co-Authored-By: Claude Opus 4.6 --- .../core/src/backoff/UploadStateMachine.ts | 135 +++++++++++ .../__tests__/UploadStateMachine.test.ts | 218 ++++++++++++++++++ .../src/backoff/__tests__/test-helpers.ts | 27 +++ packages/core/src/backoff/index.ts | 1 + packages/core/src/config-validation.ts | 42 ++++ packages/core/src/types.ts | 19 ++ 6 files changed, 442 insertions(+) create mode 100644 packages/core/src/backoff/UploadStateMachine.ts create mode 100644 packages/core/src/backoff/__tests__/UploadStateMachine.test.ts create mode 100644 packages/core/src/backoff/__tests__/test-helpers.ts create mode 100644 packages/core/src/backoff/index.ts create mode 100644 packages/core/src/config-validation.ts diff --git a/packages/core/src/backoff/UploadStateMachine.ts b/packages/core/src/backoff/UploadStateMachine.ts new file mode 100644 index 000000000..4d4def01e --- /dev/null +++ b/packages/core/src/backoff/UploadStateMachine.ts @@ -0,0 +1,135 @@ +import { createStore } from '@segment/sovran-react-native'; +import type { Store, Persistor } from '@segment/sovran-react-native'; +import type { UploadStateData, RateLimitConfig, LoggerType } from '../types'; + +const INITIAL_STATE: UploadStateData = { + state: 'READY', + waitUntilTime: 0, + globalRetryCount: 0, + firstFailureTime: null, +}; + +export class UploadStateMachine { + private store: Store; + private config: RateLimitConfig; + private logger?: LoggerType; + + constructor( + storeId: string, + persistor: Persistor | undefined, + config: RateLimitConfig, + logger?: LoggerType + ) { + this.config = config; + this.logger = logger; + + try { + this.store = createStore( + INITIAL_STATE, + persistor + ? { + persist: { + storeId: `${storeId}-uploadState`, + persistor, + }, + } + : undefined + ); + } catch (e) { + this.logger?.error( + `[UploadStateMachine] Persistence failed, using in-memory store: ${e}` + ); + + try { + this.store = createStore(INITIAL_STATE); + } catch (fallbackError) { + this.logger?.error( + `[UploadStateMachine] CRITICAL: In-memory store creation failed: ${fallbackError}` + ); + throw fallbackError; + } + } + } + + async canUpload(): Promise { + if (!this.config.enabled) { + return true; + } + + const state = await this.store.getState(); + const now = Date.now(); + + if (state.state === 'READY') { + return true; + } + + if (now >= state.waitUntilTime) { + await this.transitionToReady(); + return true; + } + + const waitSeconds = Math.ceil((state.waitUntilTime - now) / 1000); + this.logger?.info( + `Upload blocked: rate limited, retry in ${waitSeconds}s (retry ${state.globalRetryCount}/${this.config.maxRetryCount})` + ); + return false; + } + + async handle429(retryAfterSeconds: number): Promise { + if (!this.config.enabled) { + return; + } + + const now = Date.now(); + const state = await this.store.getState(); + + const newRetryCount = state.globalRetryCount + 1; + const firstFailureTime = state.firstFailureTime ?? now; + const totalBackoffDuration = (now - firstFailureTime) / 1000; + + if (newRetryCount > this.config.maxRetryCount) { + this.logger?.warn( + `Max retry count exceeded (${this.config.maxRetryCount}), resetting rate limiter` + ); + await this.reset(); + return; + } + + if (totalBackoffDuration > this.config.maxRateLimitDuration) { + this.logger?.warn( + `Max backoff duration exceeded (${this.config.maxRateLimitDuration}s), resetting rate limiter` + ); + await this.reset(); + return; + } + + const waitUntilTime = now + retryAfterSeconds * 1000; + + await this.store.dispatch(() => ({ + state: 'RATE_LIMITED' as const, + waitUntilTime, + globalRetryCount: newRetryCount, + firstFailureTime, + })); + + this.logger?.info( + `Rate limited (429): waiting ${retryAfterSeconds}s before retry ${newRetryCount}/${this.config.maxRetryCount}` + ); + } + + async reset(): Promise { + await this.store.dispatch(() => INITIAL_STATE); + } + + async getGlobalRetryCount(): Promise { + const state = await this.store.getState(); + return state.globalRetryCount; + } + + private async transitionToReady(): Promise { + await this.store.dispatch((state: UploadStateData) => ({ + ...state, + state: 'READY' as const, + })); + } +} diff --git a/packages/core/src/backoff/__tests__/UploadStateMachine.test.ts b/packages/core/src/backoff/__tests__/UploadStateMachine.test.ts new file mode 100644 index 000000000..6d56b4177 --- /dev/null +++ b/packages/core/src/backoff/__tests__/UploadStateMachine.test.ts @@ -0,0 +1,218 @@ +import { UploadStateMachine } from '../UploadStateMachine'; +import type { Persistor } from '@segment/sovran-react-native'; +import type { RateLimitConfig } from '../../types'; +import { getMockLogger } from '../../test-helpers'; +import { createTestPersistor } from './test-helpers'; + +jest.mock('@segment/sovran-react-native', () => { + const helpers = require('./test-helpers'); + return { + ...jest.requireActual('@segment/sovran-react-native'), + createStore: jest.fn((initialState: unknown) => + helpers.createMockStore(initialState) + ), + }; +}); + +describe('UploadStateMachine', () => { + let sharedStorage: Record; + let mockPersistor: Persistor; + let mockLogger: ReturnType; + + const defaultConfig: RateLimitConfig = { + enabled: true, + maxRetryCount: 100, + maxRetryInterval: 300, + maxRateLimitDuration: 43200, + }; + + beforeEach(() => { + sharedStorage = {}; + mockPersistor = createTestPersistor(sharedStorage); + mockLogger = getMockLogger(); + jest.clearAllMocks(); + }); + + describe('canUpload', () => { + it('returns true in READY state', async () => { + const sm = new UploadStateMachine( + 'test-key', + mockPersistor, + defaultConfig, + mockLogger + ); + + expect(await sm.canUpload()).toBe(true); + }); + + it('returns false during RATE_LIMITED when waitUntilTime not reached', async () => { + const sm = new UploadStateMachine( + 'test-key', + mockPersistor, + defaultConfig, + mockLogger + ); + + await sm.handle429(60); + + expect(await sm.canUpload()).toBe(false); + }); + + it('transitions to READY when waitUntilTime has passed', async () => { + const now = 1000000; + jest.spyOn(Date, 'now').mockReturnValue(now); + + const sm = new UploadStateMachine( + 'test-key', + mockPersistor, + defaultConfig, + mockLogger + ); + + await sm.handle429(60); + jest.spyOn(Date, 'now').mockReturnValue(now + 61000); + + expect(await sm.canUpload()).toBe(true); + }); + + it('always returns true when config.enabled is false', async () => { + const disabledConfig: RateLimitConfig = { + ...defaultConfig, + enabled: false, + }; + const sm = new UploadStateMachine( + 'test-key', + mockPersistor, + disabledConfig, + mockLogger + ); + + await sm.handle429(60); + expect(await sm.canUpload()).toBe(true); + }); + }); + + describe('handle429', () => { + it('increments retry count', async () => { + const now = 1000000; + jest.spyOn(Date, 'now').mockReturnValue(now); + + const sm = new UploadStateMachine( + 'test-key', + mockPersistor, + defaultConfig, + mockLogger + ); + + await sm.handle429(60); + expect(await sm.getGlobalRetryCount()).toBe(1); + + await sm.handle429(60); + expect(await sm.getGlobalRetryCount()).toBe(2); + }); + + it('blocks uploads with correct wait time', async () => { + const now = 1000000; + jest.spyOn(Date, 'now').mockReturnValue(now); + + const sm = new UploadStateMachine( + 'test-key', + mockPersistor, + defaultConfig, + mockLogger + ); + + await sm.handle429(60); + expect(await sm.canUpload()).toBe(false); + + jest.spyOn(Date, 'now').mockReturnValue(now + 59000); + expect(await sm.canUpload()).toBe(false); + + jest.spyOn(Date, 'now').mockReturnValue(now + 60000); + expect(await sm.canUpload()).toBe(true); + }); + + it('resets when max retry count exceeded', async () => { + const limitedConfig: RateLimitConfig = { + ...defaultConfig, + maxRetryCount: 3, + }; + const sm = new UploadStateMachine( + 'test-key', + mockPersistor, + limitedConfig, + mockLogger + ); + + await sm.handle429(10); + await sm.handle429(10); + await sm.handle429(10); + await sm.handle429(10); + + expect(mockLogger.warn).toHaveBeenCalledWith( + 'Max retry count exceeded (3), resetting rate limiter' + ); + expect(await sm.getGlobalRetryCount()).toBe(0); + }); + + it('resets when max rate limit duration exceeded', async () => { + const now = 1000000; + jest.spyOn(Date, 'now').mockReturnValue(now); + + const limitedConfig: RateLimitConfig = { + ...defaultConfig, + maxRateLimitDuration: 10, + }; + const sm = new UploadStateMachine( + 'test-key', + mockPersistor, + limitedConfig, + mockLogger + ); + + await sm.handle429(5); + jest.spyOn(Date, 'now').mockReturnValue(now + 11000); + await sm.handle429(5); + + expect(mockLogger.warn).toHaveBeenCalledWith( + 'Max backoff duration exceeded (10s), resetting rate limiter' + ); + expect(await sm.getGlobalRetryCount()).toBe(0); + }); + + it('no-ops when config.enabled is false', async () => { + const disabledConfig: RateLimitConfig = { + ...defaultConfig, + enabled: false, + }; + const sm = new UploadStateMachine( + 'test-key', + mockPersistor, + disabledConfig, + mockLogger + ); + + await sm.handle429(60); + expect(await sm.getGlobalRetryCount()).toBe(0); + }); + }); + + describe('reset', () => { + it('clears to READY with retryCount 0', async () => { + const sm = new UploadStateMachine( + 'test-key', + mockPersistor, + defaultConfig, + mockLogger + ); + + await sm.handle429(60); + expect(await sm.getGlobalRetryCount()).toBe(1); + + await sm.reset(); + + expect(await sm.getGlobalRetryCount()).toBe(0); + expect(await sm.canUpload()).toBe(true); + }); + }); +}); diff --git a/packages/core/src/backoff/__tests__/test-helpers.ts b/packages/core/src/backoff/__tests__/test-helpers.ts new file mode 100644 index 000000000..acbfa70fd --- /dev/null +++ b/packages/core/src/backoff/__tests__/test-helpers.ts @@ -0,0 +1,27 @@ +import type { Persistor } from '@segment/sovran-react-native'; + +export const createMockStore = (initialState: T) => { + let state = initialState; + return { + getState: jest.fn(() => Promise.resolve(state)), + dispatch: jest.fn((action: unknown) => { + if (typeof action === 'function') { + state = action(state); + } else { + state = (action as { payload: unknown }).payload as T; + } + return Promise.resolve(); + }), + }; +}; + +export const createTestPersistor = ( + storage: Record = {} +): Persistor => ({ + get: async (key: string): Promise => + Promise.resolve(storage[key] as T), + set: async (key: string, state: T): Promise => { + storage[key] = state; + return Promise.resolve(); + }, +}); diff --git a/packages/core/src/backoff/index.ts b/packages/core/src/backoff/index.ts new file mode 100644 index 000000000..5fe4305c4 --- /dev/null +++ b/packages/core/src/backoff/index.ts @@ -0,0 +1 @@ +export { UploadStateMachine } from './UploadStateMachine'; diff --git a/packages/core/src/config-validation.ts b/packages/core/src/config-validation.ts new file mode 100644 index 000000000..a9b51eb5d --- /dev/null +++ b/packages/core/src/config-validation.ts @@ -0,0 +1,42 @@ +import type { RateLimitConfig, LoggerType } from './types'; + +export const validateRateLimitConfig = ( + config: RateLimitConfig, + logger?: LoggerType +): RateLimitConfig => { + const validated = { ...config }; + + if (validated.maxRetryInterval < 0.1) { + logger?.warn( + `maxRetryInterval ${validated.maxRetryInterval}s clamped to 0.1s` + ); + validated.maxRetryInterval = 0.1; + } else if (validated.maxRetryInterval > 86400) { + logger?.warn( + `maxRetryInterval ${validated.maxRetryInterval}s clamped to 86400s` + ); + validated.maxRetryInterval = 86400; + } + + if (validated.maxRateLimitDuration < 60) { + logger?.warn( + `maxRateLimitDuration ${validated.maxRateLimitDuration}s clamped to 60s` + ); + validated.maxRateLimitDuration = 60; + } else if (validated.maxRateLimitDuration > 604800) { + logger?.warn( + `maxRateLimitDuration ${validated.maxRateLimitDuration}s clamped to 604800s` + ); + validated.maxRateLimitDuration = 604800; + } + + if (validated.maxRetryCount < 1) { + logger?.warn(`maxRetryCount ${validated.maxRetryCount} clamped to 1`); + validated.maxRetryCount = 1; + } else if (validated.maxRetryCount > 100) { + logger?.warn(`maxRetryCount ${validated.maxRetryCount} clamped to 100`); + validated.maxRetryCount = 100; + } + + return validated; +}; diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index b0d4e9570..b3cfc69ac 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -332,6 +332,24 @@ export interface EdgeFunctionSettings { version: string; } +export type RateLimitConfig = { + enabled: boolean; + maxRetryCount: number; + maxRetryInterval: number; + maxRateLimitDuration: number; +}; + +export type HttpConfig = { + rateLimitConfig?: RateLimitConfig; +}; + +export type UploadStateData = { + state: 'READY' | 'RATE_LIMITED'; + waitUntilTime: number; + globalRetryCount: number; + firstFailureTime: number | null; +}; + export type SegmentAPISettings = { integrations: SegmentAPIIntegrations; edgeFunction?: EdgeFunctionSettings; @@ -340,6 +358,7 @@ export type SegmentAPISettings = { }; metrics?: MetricsOptions; consentSettings?: SegmentAPIConsentSettings; + httpConfig?: HttpConfig; }; export type DestinationMetadata = { From 70679d7d1e6f226b3fe0cd7a9b2a2e844bd9f550 Mon Sep 17 00:00:00 2001 From: Andrea Bueide Date: Fri, 6 Mar 2026 16:16:50 -0600 Subject: [PATCH 2/4] feat(core): add BackoffManager for transient error backoff Add global transient error backoff manager that replaces the per-batch BatchUploadManager. Uses same exponential backoff formula from the SDD but tracks state globally rather than per-batch, which aligns with the RN SDK's queue model where batch identities are ephemeral. Components: - BackoffConfig, BackoffStateData types - Expanded HttpConfig to include backoffConfig - validateBackoffConfig with SDD-specified bounds - BackoffManager with canRetry/handleTransientError/reset/getRetryCount - Core test suite (12 tests) Co-Authored-By: Claude Opus 4.6 --- packages/core/src/backoff/BackoffManager.ts | 148 +++++++++ .../backoff/__tests__/BackoffManager.test.ts | 286 ++++++++++++++++++ packages/core/src/backoff/index.ts | 1 + packages/core/src/config-validation.ts | 63 +++- packages/core/src/types.ts | 20 ++ 5 files changed, 517 insertions(+), 1 deletion(-) create mode 100644 packages/core/src/backoff/BackoffManager.ts create mode 100644 packages/core/src/backoff/__tests__/BackoffManager.test.ts diff --git a/packages/core/src/backoff/BackoffManager.ts b/packages/core/src/backoff/BackoffManager.ts new file mode 100644 index 000000000..f635bcd6a --- /dev/null +++ b/packages/core/src/backoff/BackoffManager.ts @@ -0,0 +1,148 @@ +import { createStore } from '@segment/sovran-react-native'; +import type { Store, Persistor } from '@segment/sovran-react-native'; +import type { BackoffStateData, BackoffConfig, LoggerType } from '../types'; + +const INITIAL_STATE: BackoffStateData = { + state: 'READY', + retryCount: 0, + nextRetryTime: 0, + firstFailureTime: 0, +}; + +export class BackoffManager { + private store: Store; + private config: BackoffConfig; + private logger?: LoggerType; + + constructor( + storeId: string, + persistor: Persistor | undefined, + config: BackoffConfig, + logger?: LoggerType + ) { + this.config = config; + this.logger = logger; + + try { + this.store = createStore( + INITIAL_STATE, + persistor + ? { + persist: { + storeId: `${storeId}-backoffState`, + persistor, + }, + } + : undefined + ); + } catch (e) { + this.logger?.error( + `[BackoffManager] Persistence failed, using in-memory store: ${e}` + ); + + try { + this.store = createStore(INITIAL_STATE); + } catch (fallbackError) { + this.logger?.error( + `[BackoffManager] CRITICAL: In-memory store creation failed: ${fallbackError}` + ); + throw fallbackError; + } + } + } + + async canRetry(): Promise { + if (!this.config.enabled) { + return true; + } + + const state = await this.store.getState(); + + if (state.state === 'READY') { + return true; + } + + const now = Date.now(); + if (now >= state.nextRetryTime) { + await this.store.dispatch((s: BackoffStateData) => ({ + ...s, + state: 'READY' as const, + })); + return true; + } + + const waitSeconds = Math.ceil((state.nextRetryTime - now) / 1000); + this.logger?.info( + `Backoff active: retry in ${waitSeconds}s (attempt ${state.retryCount}/${this.config.maxRetryCount})` + ); + return false; + } + + async handleTransientError(statusCode: number): Promise { + if (!this.config.enabled) { + return; + } + + const now = Date.now(); + const state = await this.store.getState(); + + const newRetryCount = state.retryCount + 1; + const firstFailureTime = + state.firstFailureTime > 0 ? state.firstFailureTime : now; + const totalDuration = (now - firstFailureTime) / 1000; + + if (newRetryCount > this.config.maxRetryCount) { + this.logger?.warn( + `Max retry count exceeded (${this.config.maxRetryCount}), resetting backoff` + ); + await this.reset(); + return; + } + + if (totalDuration > this.config.maxTotalBackoffDuration) { + this.logger?.warn( + `Max backoff duration exceeded (${this.config.maxTotalBackoffDuration}s), resetting backoff` + ); + await this.reset(); + return; + } + + const backoffSeconds = this.calculateBackoff(newRetryCount); + const nextRetryTime = now + backoffSeconds * 1000; + + await this.store.dispatch(() => ({ + state: 'BACKING_OFF' as const, + retryCount: newRetryCount, + nextRetryTime, + firstFailureTime, + })); + + this.logger?.info( + `Transient error (${statusCode}): backoff ${backoffSeconds.toFixed(1)}s, attempt ${newRetryCount}/${this.config.maxRetryCount}` + ); + } + + async reset(): Promise { + await this.store.dispatch(() => INITIAL_STATE); + } + + async getRetryCount(): Promise { + const state = await this.store.getState(); + return state.retryCount; + } + + private calculateBackoff(retryCount: number): number { + const { baseBackoffInterval, maxBackoffInterval, jitterPercent } = + this.config; + + const backoff = Math.min( + baseBackoffInterval * Math.pow(2, retryCount), + maxBackoffInterval + ); + + const jitterMax = backoff * (jitterPercent / 100); + const jitter = Math.random() * jitterMax; + + return backoff + jitter; + } +} diff --git a/packages/core/src/backoff/__tests__/BackoffManager.test.ts b/packages/core/src/backoff/__tests__/BackoffManager.test.ts new file mode 100644 index 000000000..d80324964 --- /dev/null +++ b/packages/core/src/backoff/__tests__/BackoffManager.test.ts @@ -0,0 +1,286 @@ +import { BackoffManager } from '../BackoffManager'; +import type { Persistor } from '@segment/sovran-react-native'; +import type { BackoffConfig } from '../../types'; +import { getMockLogger } from '../../test-helpers'; +import { createTestPersistor } from './test-helpers'; + +jest.mock('@segment/sovran-react-native', () => { + const helpers = require('./test-helpers'); + return { + ...jest.requireActual('@segment/sovran-react-native'), + createStore: jest.fn((initialState: unknown) => + helpers.createMockStore(initialState) + ), + }; +}); + +describe('BackoffManager', () => { + let sharedStorage: Record; + let mockPersistor: Persistor; + let mockLogger: ReturnType; + + const defaultConfig: BackoffConfig = { + enabled: true, + maxRetryCount: 100, + baseBackoffInterval: 0.5, + maxBackoffInterval: 300, + maxTotalBackoffDuration: 43200, + jitterPercent: 0, + default4xxBehavior: 'drop', + default5xxBehavior: 'retry', + statusCodeOverrides: {}, + }; + + beforeEach(() => { + sharedStorage = {}; + mockPersistor = createTestPersistor(sharedStorage); + mockLogger = getMockLogger(); + jest.clearAllMocks(); + jest.spyOn(Math, 'random').mockReturnValue(0); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + describe('canRetry', () => { + it('returns true in READY state', async () => { + const bm = new BackoffManager( + 'test-key', + mockPersistor, + defaultConfig, + mockLogger + ); + + expect(await bm.canRetry()).toBe(true); + }); + + it('returns false during BACKING_OFF when nextRetryTime not reached', async () => { + const now = 1000000; + jest.spyOn(Date, 'now').mockReturnValue(now); + + const bm = new BackoffManager( + 'test-key', + mockPersistor, + defaultConfig, + mockLogger + ); + + await bm.handleTransientError(500); + expect(await bm.canRetry()).toBe(false); + }); + + it('returns true and transitions to READY after nextRetryTime passes', async () => { + const now = 1000000; + jest.spyOn(Date, 'now').mockReturnValue(now); + + const bm = new BackoffManager( + 'test-key', + mockPersistor, + defaultConfig, + mockLogger + ); + + await bm.handleTransientError(500); + + jest.spyOn(Date, 'now').mockReturnValue(now + 2000); + expect(await bm.canRetry()).toBe(true); + }); + + it('always returns true when config.enabled is false', async () => { + const disabledConfig: BackoffConfig = { + ...defaultConfig, + enabled: false, + }; + const bm = new BackoffManager( + 'test-key', + mockPersistor, + disabledConfig, + mockLogger + ); + + await bm.handleTransientError(500); + expect(await bm.canRetry()).toBe(true); + }); + }); + + describe('handleTransientError', () => { + it('sets BACKING_OFF state and increments retry count', async () => { + const now = 1000000; + jest.spyOn(Date, 'now').mockReturnValue(now); + + const bm = new BackoffManager( + 'test-key', + mockPersistor, + defaultConfig, + mockLogger + ); + + await bm.handleTransientError(500); + expect(await bm.getRetryCount()).toBe(1); + }); + + it('follows exponential backoff progression', async () => { + const now = 1000000; + jest.spyOn(Date, 'now').mockReturnValue(now); + + const bm = new BackoffManager( + 'test-key', + mockPersistor, + defaultConfig, + mockLogger + ); + + await bm.handleTransientError(500); + expect(await bm.canRetry()).toBe(false); + + jest.spyOn(Date, 'now').mockReturnValue(now + 999); + expect(await bm.canRetry()).toBe(false); + + jest.spyOn(Date, 'now').mockReturnValue(now + 1000); + expect(await bm.canRetry()).toBe(true); + + jest.spyOn(Date, 'now').mockReturnValue(now + 1000); + await bm.handleTransientError(503); + + jest.spyOn(Date, 'now').mockReturnValue(now + 2999); + expect(await bm.canRetry()).toBe(false); + + jest.spyOn(Date, 'now').mockReturnValue(now + 3000); + expect(await bm.canRetry()).toBe(true); + }); + + it('caps backoff at maxBackoffInterval', async () => { + const now = 1000000; + jest.spyOn(Date, 'now').mockReturnValue(now); + + const smallCapConfig: BackoffConfig = { + ...defaultConfig, + maxBackoffInterval: 5, + }; + const bm = new BackoffManager( + 'test-key', + mockPersistor, + smallCapConfig, + mockLogger + ); + + for (let i = 0; i < 20; i++) { + await bm.handleTransientError(500); + } + + jest.spyOn(Date, 'now').mockReturnValue(now + 4999); + expect(await bm.canRetry()).toBe(false); + + jest.spyOn(Date, 'now').mockReturnValue(now + 5000); + expect(await bm.canRetry()).toBe(true); + }); + + it('adds jitter within jitterPercent range', async () => { + const now = 1000000; + jest.spyOn(Date, 'now').mockReturnValue(now); + jest.spyOn(Math, 'random').mockReturnValue(1.0); + + const jitterConfig: BackoffConfig = { + ...defaultConfig, + jitterPercent: 10, + }; + const bm = new BackoffManager( + 'test-key', + mockPersistor, + jitterConfig, + mockLogger + ); + + await bm.handleTransientError(500); + + jest.spyOn(Date, 'now').mockReturnValue(now + 1099); + expect(await bm.canRetry()).toBe(false); + + jest.spyOn(Date, 'now').mockReturnValue(now + 1100); + expect(await bm.canRetry()).toBe(true); + }); + + it('resets when maxRetryCount exceeded', async () => { + const limitedConfig: BackoffConfig = { + ...defaultConfig, + maxRetryCount: 3, + }; + const bm = new BackoffManager( + 'test-key', + mockPersistor, + limitedConfig, + mockLogger + ); + + await bm.handleTransientError(500); + await bm.handleTransientError(500); + await bm.handleTransientError(500); + await bm.handleTransientError(500); + + expect(mockLogger.warn).toHaveBeenCalledWith( + 'Max retry count exceeded (3), resetting backoff' + ); + expect(await bm.getRetryCount()).toBe(0); + }); + + it('resets when maxTotalBackoffDuration exceeded', async () => { + const now = 1000000; + jest.spyOn(Date, 'now').mockReturnValue(now); + + const limitedConfig: BackoffConfig = { + ...defaultConfig, + maxTotalBackoffDuration: 10, + }; + const bm = new BackoffManager( + 'test-key', + mockPersistor, + limitedConfig, + mockLogger + ); + + await bm.handleTransientError(500); + jest.spyOn(Date, 'now').mockReturnValue(now + 11000); + await bm.handleTransientError(500); + + expect(mockLogger.warn).toHaveBeenCalledWith( + 'Max backoff duration exceeded (10s), resetting backoff' + ); + expect(await bm.getRetryCount()).toBe(0); + }); + + it('no-ops when config.enabled is false', async () => { + const disabledConfig: BackoffConfig = { + ...defaultConfig, + enabled: false, + }; + const bm = new BackoffManager( + 'test-key', + mockPersistor, + disabledConfig, + mockLogger + ); + + await bm.handleTransientError(500); + expect(await bm.getRetryCount()).toBe(0); + }); + }); + + describe('reset', () => { + it('clears to READY with retryCount 0', async () => { + const bm = new BackoffManager( + 'test-key', + mockPersistor, + defaultConfig, + mockLogger + ); + + await bm.handleTransientError(500); + expect(await bm.getRetryCount()).toBe(1); + + await bm.reset(); + expect(await bm.getRetryCount()).toBe(0); + expect(await bm.canRetry()).toBe(true); + }); + }); +}); diff --git a/packages/core/src/backoff/index.ts b/packages/core/src/backoff/index.ts index 5fe4305c4..e848b39bc 100644 --- a/packages/core/src/backoff/index.ts +++ b/packages/core/src/backoff/index.ts @@ -1 +1,2 @@ export { UploadStateMachine } from './UploadStateMachine'; +export { BackoffManager } from './BackoffManager'; diff --git a/packages/core/src/config-validation.ts b/packages/core/src/config-validation.ts index a9b51eb5d..9c7aefdf7 100644 --- a/packages/core/src/config-validation.ts +++ b/packages/core/src/config-validation.ts @@ -1,4 +1,4 @@ -import type { RateLimitConfig, LoggerType } from './types'; +import type { RateLimitConfig, BackoffConfig, LoggerType } from './types'; export const validateRateLimitConfig = ( config: RateLimitConfig, @@ -40,3 +40,64 @@ export const validateRateLimitConfig = ( return validated; }; + +export const validateBackoffConfig = ( + config: BackoffConfig, + logger?: LoggerType +): BackoffConfig => { + const validated = { ...config }; + + if (validated.maxBackoffInterval < 0.1) { + logger?.warn( + `maxBackoffInterval ${validated.maxBackoffInterval}s clamped to 0.1s` + ); + validated.maxBackoffInterval = 0.1; + } else if (validated.maxBackoffInterval > 86400) { + logger?.warn( + `maxBackoffInterval ${validated.maxBackoffInterval}s clamped to 86400s` + ); + validated.maxBackoffInterval = 86400; + } + + if (validated.baseBackoffInterval < 0.1) { + logger?.warn( + `baseBackoffInterval ${validated.baseBackoffInterval}s clamped to 0.1s` + ); + validated.baseBackoffInterval = 0.1; + } else if (validated.baseBackoffInterval > 300) { + logger?.warn( + `baseBackoffInterval ${validated.baseBackoffInterval}s clamped to 300s` + ); + validated.baseBackoffInterval = 300; + } + + if (validated.maxTotalBackoffDuration < 60) { + logger?.warn( + `maxTotalBackoffDuration ${validated.maxTotalBackoffDuration}s clamped to 60s` + ); + validated.maxTotalBackoffDuration = 60; + } else if (validated.maxTotalBackoffDuration > 604800) { + logger?.warn( + `maxTotalBackoffDuration ${validated.maxTotalBackoffDuration}s clamped to 604800s` + ); + validated.maxTotalBackoffDuration = 604800; + } + + if (validated.jitterPercent < 0) { + logger?.warn(`jitterPercent ${validated.jitterPercent} clamped to 0`); + validated.jitterPercent = 0; + } else if (validated.jitterPercent > 100) { + logger?.warn(`jitterPercent ${validated.jitterPercent} clamped to 100`); + validated.jitterPercent = 100; + } + + if (validated.maxRetryCount < 1) { + logger?.warn(`maxRetryCount ${validated.maxRetryCount} clamped to 1`); + validated.maxRetryCount = 1; + } else if (validated.maxRetryCount > 100) { + logger?.warn(`maxRetryCount ${validated.maxRetryCount} clamped to 100`); + validated.maxRetryCount = 100; + } + + return validated; +}; diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index b3cfc69ac..aaad58f95 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -339,8 +339,28 @@ export type RateLimitConfig = { maxRateLimitDuration: number; }; +export type BackoffConfig = { + enabled: boolean; + maxRetryCount: number; + baseBackoffInterval: number; + maxBackoffInterval: number; + maxTotalBackoffDuration: number; + jitterPercent: number; + default4xxBehavior: 'drop' | 'retry'; + default5xxBehavior: 'drop' | 'retry'; + statusCodeOverrides: Record; +}; + export type HttpConfig = { rateLimitConfig?: RateLimitConfig; + backoffConfig?: BackoffConfig; +}; + +export type BackoffStateData = { + state: 'READY' | 'BACKING_OFF'; + retryCount: number; + nextRetryTime: number; + firstFailureTime: number; }; export type UploadStateData = { From e7e9f9a6a764e5c257fa5fc665d5ddc830203552 Mon Sep 17 00:00:00 2001 From: Andrea Bueide Date: Fri, 6 Mar 2026 17:07:04 -0600 Subject: [PATCH 3/4] refactor: add JSDoc comments, logging, and edge case tests to BackoffManager Improvements to BackoffManager: - Add comprehensive JSDoc comments for all public methods - Add logging when transitioning from BACKING_OFF to READY - Fix template literal expression errors with unknown types - Add edge case tests for multiple status codes and long durations - Fix linting issues (unsafe assignments in mocks) All 14 tests pass. Ready for review. Co-Authored-By: Claude Sonnet 4.5 --- packages/core/src/backoff/BackoffManager.ts | 50 +++++++++++++++- .../backoff/__tests__/BackoffManager.test.ts | 57 +++++++++++++++++++ 2 files changed, 105 insertions(+), 2 deletions(-) diff --git a/packages/core/src/backoff/BackoffManager.ts b/packages/core/src/backoff/BackoffManager.ts index f635bcd6a..51e51e04e 100644 --- a/packages/core/src/backoff/BackoffManager.ts +++ b/packages/core/src/backoff/BackoffManager.ts @@ -9,11 +9,23 @@ const INITIAL_STATE: BackoffStateData = { firstFailureTime: 0, }; +/** + * Global backoff manager for transient errors (5xx, 408, 410, 460) per the TAPI SDD. + * Implements exponential backoff with jitter and enforces retry limits. + */ export class BackoffManager { private store: Store; private config: BackoffConfig; private logger?: LoggerType; + /** + * Creates a BackoffManager instance. + * + * @param storeId - Unique identifier for the store (typically writeKey) + * @param persistor - Optional persistor for state persistence + * @param config - Backoff configuration from Settings object + * @param logger - Optional logger for debugging + */ constructor( storeId: string, persistor: Persistor | undefined, @@ -36,21 +48,32 @@ export class BackoffManager { : undefined ); } catch (e) { + const errorMessage = e instanceof Error ? e.message : String(e); this.logger?.error( - `[BackoffManager] Persistence failed, using in-memory store: ${e}` + `[BackoffManager] Persistence failed, using in-memory store: ${errorMessage}` ); try { this.store = createStore(INITIAL_STATE); } catch (fallbackError) { + const fallbackMessage = + fallbackError instanceof Error + ? fallbackError.message + : String(fallbackError); this.logger?.error( - `[BackoffManager] CRITICAL: In-memory store creation failed: ${fallbackError}` + `[BackoffManager] CRITICAL: In-memory store creation failed: ${fallbackMessage}` ); throw fallbackError; } } } + /** + * Check if retries can proceed based on backoff state. + * Automatically transitions from BACKING_OFF to READY when wait time has passed. + * + * @returns true if retries should proceed, false if backing off + */ async canRetry(): Promise { if (!this.config.enabled) { return true; @@ -64,6 +87,7 @@ export class BackoffManager { const now = Date.now(); if (now >= state.nextRetryTime) { + this.logger?.info('Backoff period expired, resuming retries'); await this.store.dispatch((s: BackoffStateData) => ({ ...s, state: 'READY' as const, @@ -78,6 +102,12 @@ export class BackoffManager { return false; } + /** + * Handle a transient error response by setting exponential backoff. + * Increments retry count and enforces max retry/duration limits. + * + * @param statusCode - HTTP status code of the transient error (5xx, 408, 410, 460) + */ async handleTransientError(statusCode: number): Promise { if (!this.config.enabled) { return; @@ -122,15 +152,31 @@ export class BackoffManager { ); } + /** + * Reset the backoff manager to READY with retry count 0. + * Called on successful upload (2xx response). + */ async reset(): Promise { await this.store.dispatch(() => INITIAL_STATE); } + /** + * Get the current retry count for X-Retry-Count header. + * + * @returns Current retry count + */ async getRetryCount(): Promise { const state = await this.store.getState(); return state.retryCount; } + /** + * Calculate exponential backoff with jitter. + * Formula: min(baseBackoffInterval * 2^retryCount, maxBackoffInterval) + jitter + * + * @param retryCount - Current retry attempt number + * @returns Backoff delay in seconds + */ private calculateBackoff(retryCount: number): number { const { baseBackoffInterval, maxBackoffInterval, jitterPercent } = this.config; diff --git a/packages/core/src/backoff/__tests__/BackoffManager.test.ts b/packages/core/src/backoff/__tests__/BackoffManager.test.ts index d80324964..9ba9293eb 100644 --- a/packages/core/src/backoff/__tests__/BackoffManager.test.ts +++ b/packages/core/src/backoff/__tests__/BackoffManager.test.ts @@ -5,10 +5,12 @@ import { getMockLogger } from '../../test-helpers'; import { createTestPersistor } from './test-helpers'; jest.mock('@segment/sovran-react-native', () => { + // eslint-disable-next-line @typescript-eslint/no-var-requires const helpers = require('./test-helpers'); return { ...jest.requireActual('@segment/sovran-react-native'), createStore: jest.fn((initialState: unknown) => + // eslint-disable-next-line @typescript-eslint/no-unsafe-return, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access helpers.createMockStore(initialState) ), }; @@ -264,6 +266,61 @@ describe('BackoffManager', () => { await bm.handleTransientError(500); expect(await bm.getRetryCount()).toBe(0); }); + + it('handles multiple different status codes', async () => { + const now = 1000000; + jest.spyOn(Date, 'now').mockReturnValue(now); + + const bm = new BackoffManager( + 'test-key', + mockPersistor, + defaultConfig, + mockLogger + ); + + await bm.handleTransientError(500); + expect(await bm.getRetryCount()).toBe(1); + + jest.spyOn(Date, 'now').mockReturnValue(now + 1000); + await bm.handleTransientError(503); + expect(await bm.getRetryCount()).toBe(2); + + jest.spyOn(Date, 'now').mockReturnValue(now + 3000); + await bm.handleTransientError(408); + expect(await bm.getRetryCount()).toBe(3); + }); + + it('preserves state across very long backoff durations', async () => { + const now = 1000000; + jest.spyOn(Date, 'now').mockReturnValue(now); + + const longDurationConfig: BackoffConfig = { + ...defaultConfig, + maxBackoffInterval: 86400, // 24 hours + }; + const bm = new BackoffManager( + 'test-key', + mockPersistor, + longDurationConfig, + mockLogger + ); + + // Trigger max backoff by retrying many times + for (let i = 0; i < 20; i++) { + await bm.handleTransientError(500); + } + + // Should still be backing off + expect(await bm.canRetry()).toBe(false); + + // Advance to just before 24h + jest.spyOn(Date, 'now').mockReturnValue(now + 86399000); + expect(await bm.canRetry()).toBe(false); + + // Advance past 24h + jest.spyOn(Date, 'now').mockReturnValue(now + 86400000); + expect(await bm.canRetry()).toBe(true); + }); }); describe('reset', () => { From 8e0e6a33f507c42c1e6acf82b91b33a28d36625e Mon Sep 17 00:00:00 2001 From: Andrea Bueide Date: Fri, 6 Mar 2026 17:26:20 -0600 Subject: [PATCH 4/4] fix: correct backoff formula to match SDD specification MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CRITICAL FIX: Fixed off-by-one error in exponential backoff calculation - Was using newRetryCount (retry 1 → 2^1 = 2s), now uses state.retryCount (retry 1 → 2^0 = 0.5s) - Now correctly implements SDD progression: 0.5s, 1s, 2s, 4s, 8s, 16s, 32s DOCUMENTATION: Added design rationale for global vs per-batch backoff - Documented deviation from SDD's per-batch approach - Explained RN SDK architecture constraints (no stable batch identities) - Provided rationale: equivalent in practice during TAPI outages TESTS: Updated all tests to verify SDD-compliant behavior - Fixed exponential progression test (was testing wrong values) - Added comprehensive SDD formula validation test (7 retry progression) - Fixed jitter test to match new 0.5s base delay - All 15 tests pass Co-Authored-By: Claude Sonnet 4.5 --- packages/core/src/backoff/BackoffManager.ts | 18 +++++- .../backoff/__tests__/BackoffManager.test.ts | 61 ++++++++++++++++--- 2 files changed, 70 insertions(+), 9 deletions(-) diff --git a/packages/core/src/backoff/BackoffManager.ts b/packages/core/src/backoff/BackoffManager.ts index 51e51e04e..0718d479b 100644 --- a/packages/core/src/backoff/BackoffManager.ts +++ b/packages/core/src/backoff/BackoffManager.ts @@ -12,6 +12,18 @@ const INITIAL_STATE: BackoffStateData = { /** * Global backoff manager for transient errors (5xx, 408, 410, 460) per the TAPI SDD. * Implements exponential backoff with jitter and enforces retry limits. + * + * DESIGN NOTE: This implementation uses GLOBAL backoff state rather than per-batch tracking. + * The TAPI SDD specifies per-batch backoff with stable file identifiers, but the RN SDK + * architecture uses an in-memory queue where chunk() creates new batch arrays on each flush. + * Without stable batch identities, global backoff is the practical implementation. + * + * Behavior difference from SDD: + * - SDD: If batch A fails with 503, only batch A waits; batch B uploads immediately + * - This: If any batch fails with 503, ALL batches wait the backoff period + * + * Rationale: During TAPI outages, all batches typically fail anyway, making global backoff + * equivalent in practice while being simpler and safer (reduces load on degraded service). */ export class BackoffManager { private store: Store; @@ -137,7 +149,11 @@ export class BackoffManager { return; } - const backoffSeconds = this.calculateBackoff(newRetryCount); + // Use current retryCount (not newRetryCount) for SDD-compliant progression + // Retry 1: retryCount=0 → 0.5 * 2^0 = 0.5s + // Retry 2: retryCount=1 → 0.5 * 2^1 = 1.0s + // Retry 3: retryCount=2 → 0.5 * 2^2 = 2.0s + const backoffSeconds = this.calculateBackoff(state.retryCount); const nextRetryTime = now + backoffSeconds * 1000; await this.store.dispatch(() => ({ diff --git a/packages/core/src/backoff/__tests__/BackoffManager.test.ts b/packages/core/src/backoff/__tests__/BackoffManager.test.ts index 9ba9293eb..afa8c3647 100644 --- a/packages/core/src/backoff/__tests__/BackoffManager.test.ts +++ b/packages/core/src/backoff/__tests__/BackoffManager.test.ts @@ -122,7 +122,7 @@ describe('BackoffManager', () => { expect(await bm.getRetryCount()).toBe(1); }); - it('follows exponential backoff progression', async () => { + it('follows SDD-specified exponential backoff progression', async () => { const now = 1000000; jest.spyOn(Date, 'now').mockReturnValue(now); @@ -133,25 +133,69 @@ describe('BackoffManager', () => { mockLogger ); + // First retry: 0.5s (0.5 * 2^0) await bm.handleTransientError(500); expect(await bm.canRetry()).toBe(false); - jest.spyOn(Date, 'now').mockReturnValue(now + 999); + jest.spyOn(Date, 'now').mockReturnValue(now + 499); expect(await bm.canRetry()).toBe(false); - jest.spyOn(Date, 'now').mockReturnValue(now + 1000); + jest.spyOn(Date, 'now').mockReturnValue(now + 500); expect(await bm.canRetry()).toBe(true); - jest.spyOn(Date, 'now').mockReturnValue(now + 1000); + // Second retry: 1.0s (0.5 * 2^1) + jest.spyOn(Date, 'now').mockReturnValue(now + 500); await bm.handleTransientError(503); - jest.spyOn(Date, 'now').mockReturnValue(now + 2999); + jest.spyOn(Date, 'now').mockReturnValue(now + 1499); expect(await bm.canRetry()).toBe(false); - jest.spyOn(Date, 'now').mockReturnValue(now + 3000); + jest.spyOn(Date, 'now').mockReturnValue(now + 1500); + expect(await bm.canRetry()).toBe(true); + + // Third retry: 2.0s (0.5 * 2^2) + jest.spyOn(Date, 'now').mockReturnValue(now + 1500); + await bm.handleTransientError(502); + + jest.spyOn(Date, 'now').mockReturnValue(now + 3499); + expect(await bm.canRetry()).toBe(false); + + jest.spyOn(Date, 'now').mockReturnValue(now + 3500); expect(await bm.canRetry()).toBe(true); }); + it('follows SDD formula: 0.5s, 1s, 2s, 4s, 8s, 16s, 32s progression', async () => { + const now = 1000000; + jest.spyOn(Date, 'now').mockReturnValue(now); + + const bm = new BackoffManager( + 'test-key', + mockPersistor, + defaultConfig, + mockLogger + ); + + const expectedDelays = [500, 1000, 2000, 4000, 8000, 16000, 32000]; + + for (let i = 0; i < expectedDelays.length; i++) { + const currentTime = now + (i === 0 ? 0 : expectedDelays[i - 1]); + jest.spyOn(Date, 'now').mockReturnValue(currentTime); + + await bm.handleTransientError(500); + + // Should be backing off + expect(await bm.canRetry()).toBe(false); + + // Should not be ready 1ms before expected delay + jest.spyOn(Date, 'now').mockReturnValue(currentTime + expectedDelays[i] - 1); + expect(await bm.canRetry()).toBe(false); + + // Should be ready at expected delay + jest.spyOn(Date, 'now').mockReturnValue(currentTime + expectedDelays[i]); + expect(await bm.canRetry()).toBe(true); + } + }); + it('caps backoff at maxBackoffInterval', async () => { const now = 1000000; jest.spyOn(Date, 'now').mockReturnValue(now); @@ -194,12 +238,13 @@ describe('BackoffManager', () => { mockLogger ); + // First retry: 0.5s base + 10% jitter (0.05s) = 0.55s (550ms) await bm.handleTransientError(500); - jest.spyOn(Date, 'now').mockReturnValue(now + 1099); + jest.spyOn(Date, 'now').mockReturnValue(now + 549); expect(await bm.canRetry()).toBe(false); - jest.spyOn(Date, 'now').mockReturnValue(now + 1100); + jest.spyOn(Date, 'now').mockReturnValue(now + 550); expect(await bm.canRetry()).toBe(true); });