ESQL: Enhanced DATE_TRUNC with arbitrary intervals

docs/changelog/120302.yaml

@@ -0,0 +1,6 @@
+pr: 120302
+summary: "ESQL: Support for arbitrary month and year intervals in `BUCKET`"
+area: ES|QL
+type: enhancement
+issues:
+ - 120094

server/src/main/java/org/elasticsearch/common/Rounding.java

@@ -59,8 +59,8 @@ public enum DateTimeUnit {
  WEEK_OF_WEEKYEAR((byte) 1, "week", IsoFields.WEEK_OF_WEEK_BASED_YEAR, true, TimeUnit.DAYS.toMillis(7)) {
  private final long extraLocalOffsetLookup = TimeUnit.DAYS.toMillis(7);
 
- long roundFloor(long utcMillis) {
- return DateUtils.roundWeekOfWeekYear(utcMillis);
+ long roundFloor(long utcMillis, int multiplier) {
+ return DateUtils.roundWeekIntervalOfWeekYear(utcMillis, multiplier);
  }
 
  @Override
@@ -71,8 +71,8 @@ long extraLocalOffsetLookup() {
  YEAR_OF_CENTURY((byte) 2, "year", ChronoField.YEAR_OF_ERA, false, 12) {
  private final long extraLocalOffsetLookup = TimeUnit.DAYS.toMillis(366);
 
- long roundFloor(long utcMillis) {
- return DateUtils.roundYear(utcMillis);
+ long roundFloor(long utcMillis, int multiplier) {
+ return DateUtils.roundYearInterval(utcMillis, multiplier);
  }
 
  long extraLocalOffsetLookup() {
@@ -82,8 +82,11 @@ long extraLocalOffsetLookup() {
  QUARTER_OF_YEAR((byte) 3, "quarter", IsoFields.QUARTER_OF_YEAR, false, 3) {
  private final long extraLocalOffsetLookup = TimeUnit.DAYS.toMillis(92);
 
- long roundFloor(long utcMillis) {
- return DateUtils.roundQuarterOfYear(utcMillis);
+ long roundFloor(long utcMillis, int multiplier) {
+ if (multiplier == 1) {
+ return DateUtils.roundQuarterOfYear(utcMillis);
+ }
+ return DateUtils.roundIntervalMonthOfYear(utcMillis, multiplier * 3);
  }
 
  long extraLocalOffsetLookup() {
@@ -93,26 +96,29 @@ long extraLocalOffsetLookup() {
  MONTH_OF_YEAR((byte) 4, "month", ChronoField.MONTH_OF_YEAR, false, 1) {
  private final long extraLocalOffsetLookup = TimeUnit.DAYS.toMillis(31);
 
- long roundFloor(long utcMillis) {
- return DateUtils.roundMonthOfYear(utcMillis);
+ long roundFloor(long utcMillis, int multiplier) {
+ if (multiplier == 1) {
+ return DateUtils.roundMonthOfYear(utcMillis);
+ }
+ return DateUtils.roundIntervalMonthOfYear(utcMillis, multiplier);
  }
 
  long extraLocalOffsetLookup() {
  return extraLocalOffsetLookup;
  }
  },
  DAY_OF_MONTH((byte) 5, "day", ChronoField.DAY_OF_MONTH, true, ChronoField.DAY_OF_MONTH.getBaseUnit().getDuration().toMillis()) {
- long roundFloor(long utcMillis) {
- return DateUtils.roundFloor(utcMillis, this.ratio);
+ long roundFloor(long utcMillis, int multiplier) {
+ return DateUtils.roundFloor(utcMillis, this.ratio * multiplier);
  }
 
  long extraLocalOffsetLookup() {
  return ratio;
  }
  },
  HOUR_OF_DAY((byte) 6, "hour", ChronoField.HOUR_OF_DAY, true, ChronoField.HOUR_OF_DAY.getBaseUnit().getDuration().toMillis()) {
- long roundFloor(long utcMillis) {
- return DateUtils.roundFloor(utcMillis, ratio);
+ long roundFloor(long utcMillis, int multiplier) {
+ return DateUtils.roundFloor(utcMillis, ratio * multiplier);
  }
 
  long extraLocalOffsetLookup() {
@@ -126,8 +132,8 @@ long extraLocalOffsetLookup() {
  true,
  ChronoField.MINUTE_OF_HOUR.getBaseUnit().getDuration().toMillis()
  ) {
- long roundFloor(long utcMillis) {
- return DateUtils.roundFloor(utcMillis, ratio);
+ long roundFloor(long utcMillis, int multiplier) {
+ return DateUtils.roundFloor(utcMillis, ratio * multiplier);
  }
 
  long extraLocalOffsetLookup() {
@@ -141,8 +147,8 @@ long extraLocalOffsetLookup() {
  true,
  ChronoField.SECOND_OF_MINUTE.getBaseUnit().getDuration().toMillis()
  ) {
- long roundFloor(long utcMillis) {
- return DateUtils.roundFloor(utcMillis, ratio);
+ long roundFloor(long utcMillis, int multiplier) {
+ return DateUtils.roundFloor(utcMillis, ratio * multiplier);
  }
 
  long extraLocalOffsetLookup() {
@@ -171,10 +177,11 @@ long extraLocalOffsetLookup() {
  * This rounds down the supplied milliseconds since the epoch down to the next unit. In order to retain performance this method
  * should be as fast as possible and not try to convert dates to java-time objects if possible
  *
- * @param utcMillis the milliseconds since the epoch
- * @return the rounded down milliseconds since the epoch
+ * @param utcMillis the milliseconds since the epoch
+ * @param multiplier the factor by which the unit is multiplied
+ * @return the rounded down milliseconds since the epoch
  */
- abstract long roundFloor(long utcMillis);
+ abstract long roundFloor(long utcMillis, int multiplier);
 
  /**
  * When looking up {@link LocalTimeOffset} go this many milliseconds
@@ -329,17 +336,24 @@ public static class Builder {
 
  private final DateTimeUnit unit;
  private final long interval;
+ private final int multiplier;
 
  private ZoneId timeZone = ZoneOffset.UTC;
  private long offset = 0;
 
  public Builder(DateTimeUnit unit) {
+ this(unit, 1);
+ }
+
+ public Builder(DateTimeUnit unit, int multiplier) {
  this.unit = unit;
+ this.multiplier = multiplier;
  this.interval = -1;
  }
 
  public Builder(TimeValue interval) {
  this.unit = null;
+ this.multiplier = -1;
  if (interval.millis() < 1) throw new IllegalArgumentException("Zero or negative time interval not supported");
  this.interval = interval.millis();
  }
@@ -365,7 +379,7 @@ public Builder offset(long offset) {
  public Rounding build() {
  Rounding rounding;
  if (unit != null) {
- rounding = new TimeUnitRounding(unit, timeZone);
+ rounding = new TimeUnitRounding(unit, multiplier, timeZone);
  } else {
  rounding = new TimeIntervalRounding(interval, timeZone);
  }
@@ -422,11 +436,17 @@ static class TimeUnitRounding extends Rounding {
  private final DateTimeUnit unit;
  private final ZoneId timeZone;
  private final boolean unitRoundsToMidnight;
+ private final int multiplier;
 
  TimeUnitRounding(DateTimeUnit unit, ZoneId timeZone) {
+ this(unit, 1, timeZone);
+ }
+
+ TimeUnitRounding(DateTimeUnit unit, int multiplier, ZoneId timeZone) {
  this.unit = unit;
  this.timeZone = timeZone;
  this.unitRoundsToMidnight = this.unit.field.getBaseUnit().getDuration().toMillis() > 3600000L;
+ this.multiplier = multiplier;
  }
 
  TimeUnitRounding(StreamInput in) throws IOException {
@@ -660,7 +680,7 @@ private class FixedToMidnightRounding extends TimeUnitPreparedRounding {
 
  @Override
  public long round(long utcMillis) {
- return offset.localToUtcInThisOffset(unit.roundFloor(offset.utcToLocalTime(utcMillis)));
+ return offset.localToUtcInThisOffset(unit.roundFloor(offset.utcToLocalTime(utcMillis), multiplier));
  }
 
  @Override
@@ -686,7 +706,7 @@ private class FixedNotToMidnightRounding extends TimeUnitPreparedRounding {
 
  @Override
  public long round(long utcMillis) {
- return offset.localToUtcInThisOffset(unit.roundFloor(offset.utcToLocalTime(utcMillis)));
+ return offset.localToUtcInThisOffset(unit.roundFloor(offset.utcToLocalTime(utcMillis), multiplier));
  }
 
  @Override
@@ -710,7 +730,7 @@ private class ToMidnightRounding extends TimeUnitPreparedRounding implements Loc
  @Override
  public long round(long utcMillis) {
  LocalTimeOffset offset = lookup.lookup(utcMillis);
- return offset.localToUtc(unit.roundFloor(offset.utcToLocalTime(utcMillis)), this);
+ return offset.localToUtc(unit.roundFloor(offset.utcToLocalTime(utcMillis), multiplier), this);
  }
 
  @Override
@@ -764,14 +784,14 @@ private class NotToMidnightRounding extends AbstractNotToMidnightRounding implem
  @Override
  public long round(long utcMillis) {
  LocalTimeOffset offset = lookup.lookup(utcMillis);
- long roundedLocalMillis = unit.roundFloor(offset.utcToLocalTime(utcMillis));
+ long roundedLocalMillis = unit.roundFloor(offset.utcToLocalTime(utcMillis), multiplier);
  return offset.localToUtc(roundedLocalMillis, this);
  }
 
  @Override
  public long inGap(long localMillis, Gap gap) {
  // Round from just before the start of the gap
- return gap.previous().localToUtc(unit.roundFloor(gap.firstMissingLocalTime() - 1), this);
+ return gap.previous().localToUtc(unit.roundFloor(gap.firstMissingLocalTime() - 1, multiplier), this);
  }
 
  @Override

server/src/main/java/org/elasticsearch/common/time/DateUtils.java

@@ -325,8 +325,8 @@ public static int compareNanosToMillis(long nanos, long millis) {
  }
 
  /**
- * Rounds the given utc milliseconds sicne the epoch down to the next unit millis
- *
+ * Rounds the given utc milliseconds since the epoch down to the next unit millis
+ * <p>
  * Note: This does not check for correctness of the result, as this only works with units smaller or equal than a day
  * In order to ensure the performance of this methods, there are no guards or checks in it
  *
@@ -366,6 +366,22 @@ public static long roundMonthOfYear(final long utcMillis) {
  return DateUtils.of(year, month);
  }
 
+ /**
+ * Round down to the beginning of the nearest multiple of the specified month interval based on the year
+ * @param utcMillis the milliseconds since the epoch
+ * @param monthInterval the interval in months to round down to
+ * @return The milliseconds since the epoch rounded down to the beginning of the nearest multiple of the specified month interval based on the year
+ */
+ public static long roundIntervalMonthOfYear(final long utcMillis, final int monthInterval) {
+ if (monthInterval <= 0) {
+ throw new IllegalArgumentException("month interval [" + monthInterval + "] must be positive");
+ }
+ int year = getYear(utcMillis);
+ int month = getMonthOfYear(utcMillis, year);
+ int firstMonthOfInterval = (((year * 12 + month - 1) / monthInterval) * monthInterval) + 1;
+ return DateUtils.of(firstMonthOfInterval / 12, firstMonthOfInterval % 12);
+ }
+
  /**
  * Round down to the beginning of the year of the specified time
  * @param utcMillis the milliseconds since the epoch
@@ -376,13 +392,42 @@ public static long roundYear(final long utcMillis) {
  return utcMillisAtStartOfYear(year);
  }
 
+ /**
+ * Round down to the beginning of the nearest multiple of the specified year interval
+ * @param utcMillis the milliseconds since the epoch
+ * @param yearInterval the interval in years to round down to
+ * @return The milliseconds since the epoch rounded down to the beginning of the nearest multiple of the specified year interval
+ */
+ public static long roundYearInterval(final long utcMillis, final int yearInterval) {
+ if (yearInterval <= 0) {
+ throw new IllegalArgumentException("year interval [" + yearInterval + "] must be positive");
+ }
+ int year = getYear(utcMillis);
+ return utcMillisAtStartOfYear((year - 1) / yearInterval * yearInterval + 1);
+ }
+
  /**
  * Round down to the beginning of the week based on week year of the specified time
  * @param utcMillis the milliseconds since the epoch
  * @return The milliseconds since the epoch rounded down to the beginning of the week based on week year
  */
  public static long roundWeekOfWeekYear(final long utcMillis) {
- return roundFloor(utcMillis + 3 * 86400 * 1000L, 604800000) - 3 * 86400 * 1000L;
+ return roundWeekIntervalOfWeekYear(utcMillis, 1);
+ }
+
+ /**
+ * Round down to the beginning of the nearest multiple of the specified week interval based on week year
+ * <p>
+ * Consider Sun Dec 29 1969 00:00:00.000 as the start of the first week.
+ * @param utcMillis the milliseconds since the epoch
+ * @param weekInterval the interval in weeks to round down to
+ * @return The milliseconds since the epoch rounded down to the beginning of the nearest multiple of the specified week interval based on week year
+ */
+ public static long roundWeekIntervalOfWeekYear(final long utcMillis, final int weekInterval) {
+ if (weekInterval <= 0) {
+ throw new IllegalArgumentException("week interval [" + weekInterval + "] must be positive");
+ }
+ return roundFloor(utcMillis + 3 * 86400 * 1000L, 604800000L * weekInterval) - 3 * 86400 * 1000L;
  }
 
  /**

server/src/test/java/org/elasticsearch/common/time/DateUtilsTests.java

@@ -228,6 +228,14 @@ public void testRoundMonthOfYear() {
  assertThat(DateUtils.roundMonthOfYear(1), is(0L));
  long dec1969 = LocalDate.of(1969, 12, 1).atStartOfDay().toInstant(ZoneOffset.UTC).toEpochMilli();
  assertThat(DateUtils.roundMonthOfYear(-1), is(dec1969));
+
+ IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> DateUtils.roundIntervalMonthOfYear(0, -1));
+ assertThat(exc.getMessage(), is("month interval [-1] must be positive"));
+ assertThat(DateUtils.roundIntervalMonthOfYear(1, 5), is(0L));
+ long epochMilli = LocalDate.of(1969, 7, 1).atStartOfDay().toInstant(ZoneOffset.UTC).toEpochMilli();
+ assertThat(DateUtils.roundIntervalMonthOfYear(-1, 13), is(epochMilli));
+ epochMilli = LocalDate.of(2024, 10, 1).atStartOfDay().toInstant(ZoneOffset.UTC).toEpochMilli();
+ assertThat(DateUtils.roundIntervalMonthOfYear(1737378896000L, 7), is(epochMilli));
  }
 
  public void testRoundYear() {
@@ -237,9 +245,39 @@ public void testRoundYear() {
  assertThat(DateUtils.roundYear(-1), is(startOf1969));
  long endOf1970 = ZonedDateTime.of(1970, 12, 31, 23, 59, 59, 999_999_999, ZoneOffset.UTC).toInstant().toEpochMilli();
  assertThat(DateUtils.roundYear(endOf1970), is(0L));
- // test with some leapyear
+ // test with some leap year
  long endOf1996 = ZonedDateTime.of(1996, 12, 31, 23, 59, 59, 999_999_999, ZoneOffset.UTC).toInstant().toEpochMilli();
  long startOf1996 = Year.of(1996).atDay(1).atStartOfDay().toInstant(ZoneOffset.UTC).toEpochMilli();
  assertThat(DateUtils.roundYear(endOf1996), is(startOf1996));
+
+ IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> DateUtils.roundYearInterval(0, -1));
+ assertThat(exc.getMessage(), is("year interval [-1] must be positive"));
+ assertThat(DateUtils.roundYearInterval(0, 2), is(startOf1969));
+ long startOf1968 = Year.of(1968).atDay(1).atStartOfDay().toInstant(ZoneOffset.UTC).toEpochMilli();
+ assertThat(DateUtils.roundYearInterval(0, 7), is(startOf1968));
+ long startOf1966 = Year.of(1966).atDay(1).atStartOfDay().toInstant(ZoneOffset.UTC).toEpochMilli();
+ assertThat(DateUtils.roundYearInterval(1, 5), is(startOf1966));
+ long startOf1961 = Year.of(1961).atDay(1).atStartOfDay().toInstant(ZoneOffset.UTC).toEpochMilli();
+ assertThat(DateUtils.roundYearInterval(-1, 10), is(startOf1961));
+ long startOf1992 = Year.of(1992).atDay(1).atStartOfDay().toInstant(ZoneOffset.UTC).toEpochMilli();
+ assertThat(DateUtils.roundYearInterval(endOf1996, 11), is(startOf1992));
+ }
+
+ public void testRoundWeek() {
+ long epochMilli = Year.of(1969).atMonth(12).atDay(29).atStartOfDay().toInstant(ZoneOffset.UTC).toEpochMilli();
+ assertThat(DateUtils.roundWeekOfWeekYear(0), is(epochMilli));
+ assertThat(DateUtils.roundWeekOfWeekYear(1), is(epochMilli));
+ assertThat(DateUtils.roundWeekOfWeekYear(-1), is(epochMilli));
+
+ IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> DateUtils.roundWeekIntervalOfWeekYear(0, -1));
+ assertThat(exc.getMessage(), is("week interval [-1] must be positive"));
+ assertThat(DateUtils.roundWeekIntervalOfWeekYear(0, 3), is(epochMilli));
+ assertThat(DateUtils.roundWeekIntervalOfWeekYear(1, 3), is(epochMilli));
+ assertThat(DateUtils.roundWeekIntervalOfWeekYear(-1, 2), is(epochMilli));
+
+ epochMilli = Year.of(2025).atMonth(1).atDay(20).atStartOfDay().toInstant(ZoneOffset.UTC).toEpochMilli();
+ assertThat(DateUtils.roundWeekOfWeekYear(1737378896000L), is(epochMilli));
+ epochMilli = Year.of(2025).atMonth(1).atDay(13).atStartOfDay().toInstant(ZoneOffset.UTC).toEpochMilli();
+ assertThat(DateUtils.roundWeekIntervalOfWeekYear(1737378896000L, 4), is(epochMilli));
  }
 }

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java

@@ -190,14 +190,14 @@ private static Rounding.Prepared createRounding(final Period period, final ZoneI
  rounding = new Rounding.Builder(Rounding.DateTimeUnit.WEEK_OF_WEEKYEAR);
  } else if (period.getDays() > 1) {
  rounding = new Rounding.Builder(new TimeValue(period.getDays(), TimeUnit.DAYS));
- } else if (period.getMonths() == 1) {
- rounding = new Rounding.Builder(Rounding.DateTimeUnit.MONTH_OF_YEAR);
  } else if (period.getMonths() == 3) {
- // java.time.Period does not have a QUATERLY period, so a period of 3 months
+ // java.time.Period does not have a QUARTERLY period, so a period of 3 months
  // returns a quarterly rounding
  rounding = new Rounding.Builder(Rounding.DateTimeUnit.QUARTER_OF_YEAR);
- } else if (period.getYears() == 1) {
- rounding = new Rounding.Builder(Rounding.DateTimeUnit.YEAR_OF_CENTURY);
+ } else if (period.getMonths() > 0) {
+ rounding = new Rounding.Builder(Rounding.DateTimeUnit.MONTH_OF_YEAR, period.getMonths());
+ } else if (period.getYears() > 0) {
+ rounding = new Rounding.Builder(Rounding.DateTimeUnit.YEAR_OF_CENTURY, period.getYears());
  } else {
  throw new IllegalArgumentException("Time interval is not supported");
  }

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncRoundingTests.java

@@ -82,11 +82,14 @@ public void testCreateRoundingPeriod() {
  rounding = createRounding(Period.ofMonths(3));
  assertEquals(1, rounding.roundingSize(Rounding.DateTimeUnit.QUARTER_OF_YEAR), 0d);
 
+ rounding = createRounding(Period.ofMonths(5));
+ assertEquals(1, rounding.roundingSize(Rounding.DateTimeUnit.MONTH_OF_YEAR), 0d);
+
  rounding = createRounding(Period.ofYears(1));
  assertEquals(1, rounding.roundingSize(Rounding.DateTimeUnit.YEAR_OF_CENTURY), 0d);
 
- e = expectThrows(IllegalArgumentException.class, () -> createRounding(Period.ofYears(3)));
- assertThat(e.getMessage(), containsString("Time interval is not supported"));
+ rounding = createRounding(Period.ofYears(3));
+ assertEquals(1, rounding.roundingSize(Rounding.DateTimeUnit.YEAR_OF_CENTURY), 0d);
  }
 
  public void testCreateRoundingNullInterval() {