Feat: SELECT APPROXIMATE ~ WITHIN n PERCENT ERROR

The best synopsis is choosed according to the analyzed column statistics Issue: #50

Author: taewhi

traindb-core/src/main/codegen/templates/Parser.jj

@@ -2768,18 +2768,31 @@ SqlNode WindowRange() :
  */
 void ApproxConstraintOpt(List<SqlNode> hints) :
 {
- SqlNumericLiteral timeConstraint;
+ SqlNumericLiteral approxConstraint;
  List<SqlNode> list = new ArrayList<SqlNode>();
  SqlParserPos pos;
 }
 {
- [ <WITHIN> ] timeConstraint = UnsignedNumericLiteral() [ <SECONDS> ] {
- list.add(timeConstraint);
- pos = getPos();
- hints.add(new SqlHint(pos, new SqlIdentifier("approx_time", pos),
- SqlNodeList.of(pos, list), SqlHint.HintOptionFormat.LITERAL_LIST));
- return;
- }
+ [ <WITHIN> ] approxConstraint = UnsignedNumericLiteral()
+ [
+ (
+ <SECONDS> {
+ list.add(approxConstraint);
+ pos = getPos();
+ hints.add(new SqlHint(pos, new SqlIdentifier("approx_time", pos),
+ SqlNodeList.of(pos, list), SqlHint.HintOptionFormat.LITERAL_LIST));
+ return;
+ }
+ |
+ <PERCENT> <ERROR> {
+ list.add(approxConstraint);
+ pos = getPos();
+ hints.add(new SqlHint(pos, new SqlIdentifier("approx_error", pos),
+ SqlNodeList.of(pos, list), SqlHint.HintOptionFormat.LITERAL_LIST));
+ return;
+ }
+ )
+ ]
 |
  {
  return;

traindb-core/src/main/java/traindb/planner/TrainDBPlanner.java

@@ -49,6 +49,9 @@
 import org.apache.calcite.tools.RelBuilder;
 import org.apache.calcite.util.Holder;
 import org.checkerframework.checker.nullness.qual.Nullable;
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
 import traindb.adapter.jdbc.JdbcConvention;
 import traindb.adapter.jdbc.JdbcTableScan;
 import traindb.adapter.jdbc.TrainDBJdbcTable;
@@ -177,7 +180,7 @@ public RelOptTable getSynopsisTable(MSynopsis synopsis, RelOptTable baseTable) {
  }
 
  public MSynopsis getBestSynopsis(Collection<MSynopsis> synopses, TableScan scan,
- List<RelHint> hints) {
+ List<RelHint> hints, List<String> requiredColumnNames) {
  Collection<MSynopsis> hintSynopses = new ArrayList<>();
  for (RelHint hint : hints) {
  if (hint.hintName.equals("synopsis")) {
@@ -196,21 +199,45 @@ public MSynopsis getBestSynopsis(Collection<MSynopsis> synopses, TableScan scan,
 
  Collection<MSynopsis> filteredSynopses = new HashSet<>();
  for (RelHint hint : hints) {
- if (!hint.hintName.equals("approx_time")) {
- continue;
- }
- List<String> hintExecTime = hint.listOptions;
- for (String str : hintExecTime) {
- try {
- double hintTime = Double.valueOf(str);
- for (MSynopsis syn : synopses) {
- if (caqpExecutionTimePolicy.check(syn, hintTime)) {
- filteredSynopses.add(syn);
+ try {
+ if (hint.hintName.equals("approx_time")) {
+ List<String> hintExecTime = hint.listOptions;
+ for (String str : hintExecTime) {
+ double hintTime = Double.valueOf(str);
+ for (MSynopsis syn : synopses) {
+ if (caqpExecutionTimePolicy.check(syn, hintTime)) {
+ filteredSynopses.add(syn);
+ }
+ }
+ }
+ } else if (hint.hintName.equals("approx_error")) {
+ List<String> hintErrors = hint.listOptions;
+ for (String s : hintErrors) {
+ double hintError = Double.valueOf(s) /* percent */ * 0.01;
+ for (MSynopsis syn : synopses) {
+ String synopsisStatistics = syn.getSynopsisStatistics();
+ if (synopsisStatistics == null || synopsisStatistics.isEmpty()) {
+ continue;
+ }
+ JSONParser parser = new JSONParser();
+ JSONArray jsonColumnStats = (JSONArray) parser.parse(synopsisStatistics);
+ double score = 0;
+ for (int i = 0; i < jsonColumnStats.size(); i++) {
+ JSONObject colstat = (JSONObject) jsonColumnStats.get(i);
+ String columnName = (String) colstat.get("Column");
+ if (requiredColumnNames.contains(columnName)) {
+ score += (Double) colstat.get("Quality Score");
+ }
+ }
+ double errorEstimate = 1.0 - (score / requiredColumnNames.size());
+ if (errorEstimate > hintError) {
+ filteredSynopses.add(syn);
+ }
  }
  }
- } catch (Exception e) {
- // ignore
  }
+ } catch (Exception e) {
+ // ignore
  }
  }
  if (filteredSynopses.size() < synopses.size()) {

traindb-core/src/main/java/traindb/planner/rules/ApproxAggregateSynopsisAggregateScanRule.java

@@ -76,7 +76,8 @@ public void onMatch(RelOptRuleCall call) {
  return;
  }
 
- MSynopsis bestSynopsis = planner.getBestSynopsis(candidateSynopses, scan, aggregate.getHints());
+ MSynopsis bestSynopsis = planner.getBestSynopsis(
+ candidateSynopses, scan, aggregate.getHints(), requiredColumnNames);
 
  List<Integer> targets = new ArrayList<>();
  for (int i = 0; i < inputColumns.size(); i++) {

traindb-core/src/main/java/traindb/planner/rules/ApproxAggregateSynopsisFilterScanRule.java

@@ -101,8 +101,8 @@ public void onMatch(RelOptRuleCall call) {
  return;
  }
 
- MSynopsis bestSynopsis =
- planner.getBestSynopsis(candidateSynopses, scan, aggregate.getHints());
+ MSynopsis bestSynopsis = planner.getBestSynopsis(
+ candidateSynopses, scan, aggregate.getHints(), requiredColumnNames);
 
  List<Integer> targets = new ArrayList<>();
  for (int i = 0; i < inputColumns.size(); i++) {

traindb-core/src/main/java/traindb/planner/rules/ApproxAggregateSynopsisProjectScanRule.java

@@ -87,8 +87,8 @@ public void onMatch(RelOptRuleCall call) {
  return;
  }
 
- MSynopsis bestSynopsis =
- planner.getBestSynopsis(candidateSynopses, scan, aggregate.getHints());
+ MSynopsis bestSynopsis = planner.getBestSynopsis(
+ candidateSynopses, scan, aggregate.getHints(), requiredColumnNames);
 
  final List<String> synopsisColumns = bestSynopsis.getColumnNames();
 

traindb-core/src/main/java/traindb/planner/rules/ApproxAggregateSynopsisRule.java

@@ -165,8 +165,8 @@ public void onMatch(RelOptRuleCall call) {
  continue;
  }
 
- MSynopsis bestSynopsis =
- planner.getBestSynopsis(candidateSynopses, scan, aggregate.getHints());
+ MSynopsis bestSynopsis = planner.getBestSynopsis(
+ candidateSynopses, scan, aggregate.getHints(), requiredColumnNames);
  RelOptTableImpl synopsisTable =
  (RelOptTableImpl) planner.getSynopsisTable(bestSynopsis, scan.getTable());
  TableScan newScan = new JdbcTableScan(scan.getCluster(), scan.getHints(), synopsisTable,

traindb-core/src/main/java/traindb/sql/calcite/TrainDBHintStrategyTable.java

@@ -29,6 +29,7 @@ static HintStrategyTable createHintStrategies(HintStrategyTable.Builder builder)
  .hintStrategy("approximate", HintPredicates.AGGREGATE)
  .hintStrategy("synopsis", HintPredicates.AGGREGATE)
  .hintStrategy("approx_time", HintPredicates.AGGREGATE)
+ .hintStrategy("approx_error", HintPredicates.AGGREGATE)
  .build();
  }
 }