Skip to content

Cache the routing decision, not the colour it resolves to #9

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
partouf merged 2 commits into from
Sep 22, 2026
Merged

Cache the routing decision, not the colour it resolves to #9

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d6f19bb
Cache the routing decision, not the colour it resolves to
partouf Sep 22, 2026
4276858
Keep the last known colour when SSM fails
partouf Sep 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
139 changes: 69 additions & 70 deletions src/services/routing.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,19 @@ let activeColorCache = {
TTL: 30000, // 30 seconds TTL
};

// In-memory cache for routing lookups
const routingCache = new Map<string, RoutingInfo>();
// In-memory cache of routing decisions. Deliberately holds only what the routing table
// says -- never a value derived from the active colour, which changes on every deploy.
// The colour is applied in resolveRouting() on each request, from activeColorCache, so a
// missed /admin/clear-cache costs at most that cache's TTL instead of lasting forever.
const routingCache = new Map<string, RoutingDecision>();

interface RoutingDecision {
type: 'url' | 'queue';
environment: string;
targetUrl?: string;
// Absent for queue routing means the environment's own coloured queue.
queueName?: string;
}

export interface RoutingInfo {
type: 'url' | 'queue';
Expand Down Expand Up @@ -78,7 +89,16 @@ async function getActiveColor(): Promise<string> {
logger.info(`Active color from SSM: ${color}`);
return color;
} catch (error) {
logger.warn('Failed to get active color from SSM, defaulting to blue:', error);
// Every request resolves its colour through here, so an SSM problem must not
// repoint the whole environment at blue. Keep serving the last known colour, and
// hold it for another TTL rather than re-asking SSM on every compile.
const lastKnownColor = activeColorCache.color;
if (lastKnownColor) {
logger.warn('Failed to get active color from SSM, keeping last known value:', error);
activeColorCache = {...activeColorCache, timestamp: now};
return lastKnownColor;
}
logger.warn('Failed to get active color from SSM and none is cached, defaulting to blue:', error);
return 'blue';
}
}
Expand Down Expand Up @@ -122,10 +142,44 @@ function buildQueueUrl(queueName: string, activeColor: string): string {
return baseUrl + fifoQueueName;
}

async function resolveRouting(decision: RoutingDecision): Promise<RoutingInfo> {
if (decision.type === 'url') {
return {type: 'url', target: decision.targetUrl || '', environment: decision.environment};
}

const queueUrl = decision.queueName
? buildQueueUrl(decision.queueName, await getActiveColor())
: await getColoredQueueUrl();
return {type: 'queue', target: queueUrl, environment: decision.environment};
}

function getCompilerRoutingTableName(): string {
return process.env.COMPILER_ROUTING_TABLE || 'CompilerRouting';
}

function decisionFromItem(compilerId: string, item: Record<string, any> | undefined): RoutingDecision {
const environment = item ? item.environment?.S || '' : 'unknown';
const targetUrl = item?.routingType?.S === 'url' ? item.targetUrl?.S : undefined;

if (targetUrl) {
logger.info(`Compiler ${compilerId} routed to URL: ${targetUrl}`);
return {type: 'url', targetUrl, environment};
}

const queueName = item?.routingType?.S === 'url' ? undefined : item?.queueName?.S;
if (queueName) {
logger.info(`Compiler ${compilerId} routed to queue: ${queueName}`);
return {type: 'queue', queueName, environment};
}

if (item) {
logger.info(`Compiler ${compilerId} routed to colored queue (no queueName in DynamoDB)`);
} else {
logger.info(`No routing found for compiler ${compilerId}, using colored queue`);
}
return {type: 'queue', environment};
}

export async function lookupCompilerRouting(compilerId: string): Promise<RoutingInfo> {
try {
// Create composite key with environment prefix for isolation
Expand All @@ -134,10 +188,10 @@ export async function lookupCompilerRouting(compilerId: string): Promise<Routing

// Check cache first
const cacheKey = compositeKey;
const cachedEntry = routingCache.get(cacheKey);
if (cachedEntry) {
const cachedDecision = routingCache.get(cacheKey);
if (cachedDecision) {
logger.debug(`Routing cache hit for compiler: ${compilerId}`);
return cachedEntry;
return resolveRouting(cachedDecision);
}

// Look up compiler in DynamoDB routing table using composite key
Expand Down Expand Up @@ -176,67 +230,10 @@ export async function lookupCompilerRouting(compilerId: string): Promise<Routing
}
}

if (item) {
const routingType = item.routingType?.S || 'queue';

if (routingType === 'url') {
const targetUrl = item.targetUrl?.S || '';
if (targetUrl) {
const result: RoutingInfo = {
type: 'url',
target: targetUrl,
environment: item.environment?.S || '',
};
// Cache the result
routingCache.set(cacheKey, result);
logger.info(`Compiler ${compilerId} routed to URL: ${targetUrl}`);
logger.debug(`Routing lookup complete for compiler: ${compilerId}`);
return result;
}
} else {
// Queue routing - use queueName from DynamoDB to build full queue URL
const queueName = item.queueName?.S;
if (queueName) {
const activeColor = await getActiveColor();
const queueUrl = buildQueueUrl(queueName, activeColor);
const result: RoutingInfo = {
type: 'queue',
target: queueUrl,
environment: item.environment?.S || '',
};
// Cache the result
routingCache.set(cacheKey, result);
logger.info(`Compiler ${compilerId} routed to queue: ${queueName} (${queueUrl})`);
logger.debug(`Routing lookup complete for compiler: ${compilerId}`);
return result;
}
// Fallback to colored queue if no queueName specified
const queueUrl = await getColoredQueueUrl();
const result: RoutingInfo = {
type: 'queue',
target: queueUrl,
environment: item.environment?.S || '',
};
// Cache the result
routingCache.set(cacheKey, result);
logger.info(`Compiler ${compilerId} routed to colored queue (no queueName in DynamoDB)`);
logger.debug(`Routing lookup complete for compiler: ${compilerId}`);
return result;
}
}

// No routing found, use colored queue
logger.info(`No routing found for compiler ${compilerId}, using colored queue`);
const queueUrl = await getColoredQueueUrl();
const result: RoutingInfo = {
type: 'queue',
target: queueUrl,
environment: 'unknown',
};
// Cache the result
routingCache.set(cacheKey, result);
logger.debug(`Routing lookup complete for compiler: ${compilerId}, using colored queue`);
return result;
const decision = decisionFromItem(compilerId, item);
routingCache.set(cacheKey, decision);
logger.debug(`Routing lookup complete for compiler: ${compilerId}`);
return resolveRouting(decision);
} catch (error) {
// On any error, fall back to colored queue
logger.warn(`Failed to lookup routing for compiler ${compilerId}:`, error);
Expand All @@ -251,9 +248,11 @@ export async function lookupCompilerRouting(compilerId: string): Promise<Routing

/**
* Clears all routing and color caches.
* This is called via the /admin/clear-cache endpoint during blue-green deployments
* to ensure the router immediately picks up the new active color without waiting
* for the 30-second cache TTL to expire.
* Called via /admin/clear-cache at the end of a blue-green deployment, so the router
* picks up the new active color and any routing-table changes immediately rather than
* waiting for the active color cache's 30-second TTL. Only the routing-table half needs
* this: routing decisions are cached without an expiry, since nothing else invalidates
* them, while the color they are resolved with expires on its own.
*/
export function clearRoutingCaches(): void {
// Clear active color cache
Expand Down
136 changes: 136 additions & 0 deletions test/unit/routing.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -253,6 +253,142 @@ describe('Routing Service - S3 Overflow', () => {
});
});

describe('colour resolution', () => {
beforeEach(() => {
resetAllMocks();
clearRoutingCaches();
process.env.ENVIRONMENT_NAME = 'test';
});

afterEach(() => {
vi.useRealTimers();
});

it('should follow a colour switch on a cached compiler once the colour TTL expires', async () => {
vi.useFakeTimers();
mockSSM.on(GetParameterCommand).resolves({Parameter: {Value: 'blue'}});
mockDynamoDB.on(GetItemCommand).resolves({
Item: {
compilerId: {S: 'test#gcc-12'},
routingType: {S: 'queue'},
queueName: {S: 'test-compilation-queue'},
},
});

const before = await lookupCompilerRouting('gcc-12');
expect(before.target).toContain('test-compilation-queue-blue.fifo');

// The deploy switches colour, and its /admin/clear-cache never arrives.
mockSSM.on(GetParameterCommand).resolves({Parameter: {Value: 'green'}});
const lookupsBefore = mockDynamoDB.commandCalls(GetItemCommand).length;
vi.advanceTimersByTime(31_000);

const after = await lookupCompilerRouting('gcc-12');

expect(after.target).toContain('test-compilation-queue-green.fifo');
// Still a routing cache hit: only the colour was re-read, not the routing row.
expect(mockDynamoDB.commandCalls(GetItemCommand).length).toBe(lookupsBefore);
});

it('should keep serving a cached compiler from the colour cache within the TTL', async () => {
mockSSM.on(GetParameterCommand).resolves({Parameter: {Value: 'blue'}});
mockDynamoDB.on(GetItemCommand).resolves({
Item: {
compilerId: {S: 'test#gcc-12'},
routingType: {S: 'queue'},
queueName: {S: 'test-compilation-queue'},
},
});

await lookupCompilerRouting('gcc-12');
const ssmCallsAfterFirst = mockSSM.commandCalls(GetParameterCommand).length;
await lookupCompilerRouting('gcc-12');

expect(mockSSM.commandCalls(GetParameterCommand).length).toBe(ssmCallsAfterFirst);
});

it('should keep the last known colour when SSM fails', async () => {
vi.useFakeTimers();
mockSSM.on(GetParameterCommand).resolves({Parameter: {Value: 'green'}});
mockDynamoDB.on(GetItemCommand).resolves({
Item: {
compilerId: {S: 'test#gcc-12'},
routingType: {S: 'queue'},
queueName: {S: 'test-compilation-queue'},
},
});

await lookupCompilerRouting('gcc-12');

// Resolving the colour is on the request path now, so an SSM outage must not
// repoint the whole environment at blue.
mockSSM.on(GetParameterCommand).rejects(new Error('ThrottlingException'));
vi.advanceTimersByTime(31_000);

const during = await lookupCompilerRouting('gcc-12');

expect(during.target).toContain('test-compilation-queue-green.fifo');
});

it('should hold the last known colour for a TTL rather than re-asking SSM per request', async () => {
vi.useFakeTimers();
mockSSM.on(GetParameterCommand).resolves({Parameter: {Value: 'green'}});
mockDynamoDB.on(GetItemCommand).resolves({
Item: {
compilerId: {S: 'test#gcc-12'},
routingType: {S: 'queue'},
queueName: {S: 'test-compilation-queue'},
},
});

await lookupCompilerRouting('gcc-12');
mockSSM.on(GetParameterCommand).rejects(new Error('ThrottlingException'));
vi.advanceTimersByTime(31_000);
await lookupCompilerRouting('gcc-12');
const callsAfterFailure = mockSSM.commandCalls(GetParameterCommand).length;

await lookupCompilerRouting('gcc-12');
await lookupCompilerRouting('gcc-12');

expect(mockSSM.commandCalls(GetParameterCommand).length).toBe(callsAfterFailure);
});

it('should fall back to blue when SSM fails and no colour is cached', async () => {
mockSSM.on(GetParameterCommand).rejects(new Error('AccessDenied'));
mockDynamoDB.on(GetItemCommand).resolves({
Item: {
compilerId: {S: 'test#gcc-12'},
routingType: {S: 'queue'},
queueName: {S: 'test-compilation-queue'},
},
});

const result = await lookupCompilerRouting('gcc-12');

expect(result.target).toContain('test-compilation-queue-blue.fifo');
});

it('should not resolve a colour for a URL-routed compiler', async () => {
mockSSM.on(GetParameterCommand).resolves({Parameter: {Value: 'blue'}});
mockDynamoDB.on(GetItemCommand).resolves({
Item: {
compilerId: {S: 'test#winprod-msvc'},
routingType: {S: 'url'},
targetUrl: {S: 'https://winprod.compiler-explorer.com'},
},
});

const result = await lookupCompilerRouting('winprod-msvc');

expect(result).toEqual({
type: 'url',
target: 'https://winprod.compiler-explorer.com',
environment: '',
});
expect(mockSSM.commandCalls(GetParameterCommand)).toHaveLength(0);
});
});

describe('clearRoutingCaches', () => {
beforeEach(() => {
resetAllMocks();
Expand Down
Loading