fix: long timeranges for task queries truncated

.changeset/silver-forks-rhyme.md

@@ -0,0 +1,5 @@
+---
+"@hyperdx/api": patch
+---
+
+fix: set a max size for alert timeranges

packages/api/src/tasks/__tests__/util.test.ts

@@ -1,4 +1,5 @@
 import {
+ calcAlertDateRange,
  escapeJsonString,
  roundDownTo,
  roundDownToXMinutes,
@@ -335,4 +336,192 @@ describe('util', () => {
  expect(escapeJsonString('foo\u0000bar')).toBe('foo\\u0000bar');
  });
  });
+
+ describe('calcAlertDateRange', () => {
+ const now = Date.now();
+ const oneMinuteMs = 60 * 1000;
+ const oneHourMs = 60 * oneMinuteMs;
+
+ it('should return unchanged dates when range is within limits', () => {
+ const startTime = now - 10 * oneMinuteMs; // 10 minutes ago
+ const endTime = now;
+ const windowSizeInMins = 5;
+
+ const [start, end] = calcAlertDateRange(
+ startTime,
+ endTime,
+ windowSizeInMins,
+ );
+
+ expect(start.getTime()).toBe(startTime);
+ expect(end.getTime()).toBe(endTime);
+ });
+
+ it('should truncate start time when too many windows (> 50)', () => {
+ const windowSizeInMins = 1;
+ const maxWindows = 50;
+ const tooManyWindowsMs =
+ (maxWindows + 10) * windowSizeInMins * oneMinuteMs; // 60 minutes
+ const startTime = now - tooManyWindowsMs;
+ const endTime = now;
+
+ const [start, end] = calcAlertDateRange(
+ startTime,
+ endTime,
+ windowSizeInMins,
+ );
+
+ // Should truncate to exactly 50 windows
+ const expectedStartTime =
+ endTime - maxWindows * windowSizeInMins * oneMinuteMs;
+ expect(start.getTime()).toBe(expectedStartTime);
+ expect(end.getTime()).toBe(endTime);
+ });
+
+ it('should truncate start time when time range exceeds 6 hours for short windows (< 15 mins)', () => {
+ const windowSizeInMins = 10;
+ const maxLookbackTime = 6 * oneHourMs; // 6 hours for windows < 15 minutes
+ const tooLongRangeMs = maxLookbackTime + oneHourMs; // 7 hours
+ const startTime = now - tooLongRangeMs;
+ const endTime = now;
+
+ const [start, end] = calcAlertDateRange(
+ startTime,
+ endTime,
+ windowSizeInMins,
+ );
+
+ const expectedStartTime = endTime - maxLookbackTime;
+ expect(start.getTime()).toBeGreaterThan(startTime);
+ expect(start.getTime()).toBe(expectedStartTime);
+ expect(end.getTime()).toBe(endTime);
+ });
+
+ it('should truncate start time when time range exceeds 24 hours for long windows (>= 15 mins)', () => {
+ const windowSizeInMins = 30;
+ const maxLookbackTime = 24 * oneHourMs; // 24 hours for windows >= 15 minutes
+ const tooLongRangeMs = maxLookbackTime + 2 * oneHourMs; // 26 hours
+ const startTime = now - tooLongRangeMs;
+ const endTime = now;
+
+ const [start, end] = calcAlertDateRange(
+ startTime,
+ endTime,
+ windowSizeInMins,
+ );
+
+ const expectedStartTime = endTime - maxLookbackTime;
+ expect(start.getTime()).toBe(expectedStartTime);
+ expect(end.getTime()).toBe(endTime);
+ });
+
+ it('should apply the more restrictive truncation when both limits are exceeded', () => {
+ const windowSizeInMins = 1;
+ const maxWindows = 50;
+ const maxLookbackTime = 6 * oneHourMs; // 6 hours for 1-minute windows
+
+ // Create a range that exceeds both limits
+ const excessiveRangeMs = Math.max(
+ (maxWindows + 100) * windowSizeInMins * oneMinuteMs, // 150 windows
+ maxLookbackTime + 2 * oneHourMs, // 8 hours
+ );
+ const startTime = now - excessiveRangeMs;
+ const endTime = now;
+
+ const [start, end] = calcAlertDateRange(
+ startTime,
+ endTime,
+ windowSizeInMins,
+ );
+
+ // Should use the more restrictive limit (maxWindows in this case)
+ const expectedStartTime =
+ endTime - maxWindows * windowSizeInMins * oneMinuteMs;
+ expect(start.getTime()).toBe(expectedStartTime);
+ expect(end.getTime()).toBe(endTime);
+ });
+
+ it('should handle very large window sizes correctly', () => {
+ const windowSizeInMins = 120; // 2 hours
+ const maxLookbackTime = 24 * oneHourMs; // 24 hours for large windows
+ const normalRange = 12 * oneHourMs; // 12 hours - within limit
+ const startTime = now - normalRange;
+ const endTime = now;
+
+ const [start, end] = calcAlertDateRange(
+ startTime,
+ endTime,
+ windowSizeInMins,
+ );
+
+ // Should remain unchanged since within limits
+ expect(start.getTime()).toBe(startTime);
+ expect(end.getTime()).toBe(endTime);
+ });
+
+ it('should handle exactly 50 windows without truncation', () => {
+ const windowSizeInMins = 5;
+ const maxWindows = 50;
+ const exactlyMaxWindowsMs = maxWindows * windowSizeInMins * oneMinuteMs; // 250 minutes
+ const startTime = now - exactlyMaxWindowsMs;
+ const endTime = now;
+
+ const [start, end] = calcAlertDateRange(
+ startTime,
+ endTime,
+ windowSizeInMins,
+ );
+
+ // Should remain unchanged since exactly at the limit
+ expect(start.getTime()).toBe(startTime);
+ expect(end.getTime()).toBe(endTime);
+ });
+
+ it('should handle fractional windows correctly', () => {
+ const windowSizeInMins = 7;
+ const partialWindowsMs = 7.5 * windowSizeInMins * oneMinuteMs; // 7.5 windows
+ const startTime = now - partialWindowsMs;
+ const endTime = now;
+
+ const [start, end] = calcAlertDateRange(
+ startTime,
+ endTime,
+ windowSizeInMins,
+ );
+
+ // Should remain unchanged since well within limits
+ expect(start.getTime()).toBe(startTime);
+ expect(end.getTime()).toBe(endTime);
+ });
+
+ it('should handle zero time range', () => {
+ const startTime = now;
+ const endTime = now;
+ const windowSizeInMins = 5;
+
+ const [start, end] = calcAlertDateRange(
+ startTime,
+ endTime,
+ windowSizeInMins,
+ );
+
+ expect(start.getTime()).toBe(startTime);
+ expect(end.getTime()).toBe(endTime);
+ });
+
+ it('should return Date objects', () => {
+ const startTime = now - 10 * oneMinuteMs;
+ const endTime = now;
+ const windowSizeInMins = 5;
+
+ const [start, end] = calcAlertDateRange(
+ startTime,
+ endTime,
+ windowSizeInMins,
+ );
+
+ expect(start).toBeInstanceOf(Date);
+ expect(end).toBeInstanceOf(Date);
+ });
+ });
 });

packages/api/src/tasks/checkAlerts.ts

@@ -36,7 +36,11 @@ import {
  renderAlertTemplate,
 } from '@/tasks/template';
 import { CheckAlertsTaskArgs, HdxTask } from '@/tasks/types';
-import { roundDownToXMinutes, unflattenObject } from '@/tasks/util';
+import {
+ calcAlertDateRange,
+ roundDownToXMinutes,
+ unflattenObject,
+} from '@/tasks/util';
 import logger from '@/utils/logger';
 
 import { tasksTracer } from './tracer';
@@ -171,18 +175,22 @@ export const processAlert = async (
  );
  return;
  }
- const checkStartTime = previous
- ? previous.createdAt
- : fns.subMinutes(nowInMinsRoundDown, windowSizeInMins);
- const checkEndTime = nowInMinsRoundDown;
+ const dateRange = calcAlertDateRange(
+ (previous
+ ? previous.createdAt
+ : fns.subMinutes(nowInMinsRoundDown, windowSizeInMins)
+ ).getTime(),
+ nowInMinsRoundDown.getTime(),
+ windowSizeInMins,
+ );
 
  let chartConfig: ChartConfigWithOptDateRange | undefined;
  if (details.taskType === AlertTaskType.SAVED_SEARCH) {
  const savedSearch = details.savedSearch;
  chartConfig = {
  connection: connectionId,
  displayType: DisplayType.Line,
- dateRange: [checkStartTime, checkEndTime],
+ dateRange,
  dateRangeStartInclusive: true,
  dateRangeEndInclusive: false,
  from: source.from,
@@ -206,7 +214,7 @@ export const processAlert = async (
  if (tile.config.displayType === DisplayType.Line) {
  chartConfig = {
  connection: connectionId,
- dateRange: [checkStartTime, checkEndTime],
+ dateRange,
  dateRangeStartInclusive: true,
  dateRangeEndInclusive: false,
  displayType: tile.config.displayType,
@@ -252,9 +260,10 @@ export const processAlert = async (
  logger.info(
  {
  alertId: alert.id,
+ chartConfig,
  checksData,
- checkStartTime,
- checkEndTime,
+ checkStartTime: dateRange[0],
+ checkEndTime: dateRange[1],
  },
  `Received alert metric [${alert.source} source]`,
  );

packages/api/src/tasks/util.ts

@@ -1,5 +1,7 @@
 import { set } from 'lodash';
 
+import logger from '@/utils/logger';
+
 // transfer keys of attributes with dot into nested object
 // ex: { 'a.b': 'c', 'd.e.f': 'g' } -> { a: { b: 'c' }, d: { e: { f: 'g' } } }
 export const unflattenObject = (
@@ -38,3 +40,46 @@ export const roundDownToXMinutes = (x: number) => roundDownTo(1000 * 60 * x);
 export const escapeJsonString = (str: string) => {
  return JSON.stringify(str).slice(1, -1);
 };
+
+const MAX_NUM_WINDOWS = 50;
+const maxLookbackTime = (windowSizeInMins: number) =>
+ 3600_000 * (windowSizeInMins < 15 ? 6 : 24);
+export function calcAlertDateRange(
+ _startTime: number,
+ _endTime: number,
+ windowSizeInMins: number,
+): [Date, Date] {
+ let startTime = _startTime;
+ const endTime = _endTime;
+ const numWindows = (endTime - startTime) / 60_000 / windowSizeInMins;
+ // Truncate if too many windows are present
+ if (numWindows > MAX_NUM_WINDOWS) {
+ startTime = endTime - MAX_NUM_WINDOWS * 1000 * 60 * windowSizeInMins;
+ logger.info(
+ {
+ requestedStartTime: _startTime,
+ startTime,
+ endTime,
+ windowSizeInMins,
+ numWindows,
+ },
+ 'startTime truncated due to too many windows',
+ );
+ }
+ // Truncate if time range is over threshold
+ const MAX_LOOKBACK_TIME = maxLookbackTime(windowSizeInMins);
+ if (endTime - startTime > MAX_LOOKBACK_TIME) {
+ startTime = endTime - MAX_LOOKBACK_TIME;
+ logger.info(
+ {
+ requestedStartTime: _startTime,
+ startTime,
+ endTime,
+ windowSizeInMins,
+ numWindows,
+ },
+ 'startTime truncated due to long lookback time',
+ );
+ }
+ return [new Date(startTime), new Date(endTime)];
+}