Skip to content

Commit 279b3d4

Browse files
jogroganjogrogan
authored
Add missing rules for LISTAGG & UNNEST (#118)
* Add missing rules for LISTAGG & UNNEST * attempt to fix test flakiness
1 parent 3f1a571 commit 279b3d4

File tree

3 files changed

+152
-31
lines changed

3 files changed

+152
-31
lines changed

‎Makefile

+4-4
Original file line numberDiff line numberDiff line change
@@ -108,17 +108,17 @@ integration-tests: deploy-dev-environment
108108
kubectl port-forward -n kafka svc/one-kafka-external-bootstrap 9092 & echo $$! > port-forward.pid
109109
kubectl port-forward -n flink svc/flink-sql-gateway 8083 & echo $$! > port-forward-2.pid
110110
kubectl port-forward -n flink svc/basic-session-deployment-rest 8081 & echo $$! > port-forward-3.pid
111-
./gradlew intTest || kill `cat port-forward.pid port-forward-2.pid, port-forward-3.pid`
111+
./gradlew intTest --no-parallel || kill `cat port-forward.pid port-forward-2.pid, port-forward-3.pid`
112112
kill `cat port-forward.pid`
113113
kill `cat port-forward-2.pid`
114114
kill `cat port-forward-3.pid`
115115

116116
# kind cluster used in github workflow needs to have different routing set up, avoiding the need to forward kafka ports
117117
integration-tests-kind: deploy-dev-environment
118118
kubectl wait kafka.kafka.strimzi.io/one --for=condition=Ready --timeout=10m -n kafka
119-
kubectl wait kafkatopic.kafka.strimzi.io/kafka-database-existing-topic-1 --for=condition=Ready --timeout=10m
120-
kubectl wait kafkatopic.kafka.strimzi.io/kafka-database-existing-topic-2 --for=condition=Ready --timeout=10m
121-
./gradlew intTest -i
119+
kubectl wait kafkatopic.kafka.strimzi.io/kafka-database-existing-topic-1 --for=condition=Ready --timeout=10m
120+
kubectl wait kafkatopic.kafka.strimzi.io/kafka-database-existing-topic-2 --for=condition=Ready --timeout=10m
121+
./gradlew intTest -i --no-parallel
122122

123123
generate-models:
124124
./generate-models.sh

‎hoptimator-k8s/src/test/resources/k8s-ddl-function.id

+65-22
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
!set outputformat mysql
22
!use k8s
33

4-
create or replace view ADS."case" AS SELECT CASE WHEN "FIRST_NAME" = 'Bob' THEN ARRAY['a'] ELSE ARRAY['b'] END || CASE WHEN "FIRST_NAME" = 'Alice' THEN ARRAY['c'] ELSE ARRAY['d'] END AS arr from profile.members;
4+
create or replace materialized view ADS."case" AS SELECT CASE WHEN "FIRST_NAME" = 'Bob' THEN ARRAY['a'] ELSE ARRAY['b'] END || CASE WHEN "FIRST_NAME" = 'Alice' THEN ARRAY['c'] ELSE ARRAY['d'] END AS arr from profile.members;
55
(0 rows modified)
66

77
!update
@@ -18,24 +18,41 @@ select * from ADS."case";
1818

1919
!ok
2020

21-
create or replace view ADS."json" AS SELECT JSON_VALUE('{"a": 1}', '$.a') AS json from profile.members;
21+
create or replace materialized view ADS."createJson" AS SELECT JSON_OBJECT('name':"FIRST_NAME", 'type':4) AS json from profile.members;
2222
(0 rows modified)
2323

2424
!update
2525

26-
select * from ADS."json";
27-
+------+
28-
| JSON |
29-
+------+
30-
| 1 |
31-
| 1 |
32-
| 1 |
33-
+------+
26+
select * from ADS."createJson";
27+
+-----------------------------+
28+
| JSON |
29+
+-----------------------------+
30+
| {"name":"Alice","type":4} |
31+
| {"name":"Bob","type":4} |
32+
| {"name":"Charlie","type":4} |
33+
+-----------------------------+
3434
(3 rows)
3535

3636
!ok
3737

38-
create or replace view ADS."regex" AS SELECT REGEXP_REPLACE("FIRST_NAME", '(B)ob', '$1ill') AS name from profile.members;
38+
create or replace materialized view ADS."extractJson" AS SELECT JSON_VALUE("JSON", '$.name') AS name from ads."createJson";
39+
(0 rows modified)
40+
41+
!update
42+
43+
select * from ADS."extractJson";
44+
+---------+
45+
| NAME |
46+
+---------+
47+
| Alice |
48+
| Bob |
49+
| Charlie |
50+
+---------+
51+
(3 rows)
52+
53+
!ok
54+
55+
create or replace materialized view ADS."regex" AS SELECT REGEXP_REPLACE("FIRST_NAME", '(B)ob', '$1ill') AS name from profile.members;
3956
(0 rows modified)
4057

4158
!update
@@ -52,7 +69,7 @@ select * from ads."regex";
5269

5370
!ok
5471

55-
create or replace view ADS."concat" AS SELECT CONCAT('_', "FIRST_NAME", '_') AS name from profile.members;
72+
create or replace materialized view ADS."concat" AS SELECT CONCAT('_', "FIRST_NAME", '_') AS name from profile.members;
5673
(0 rows modified)
5774

5875
!update
@@ -69,8 +86,7 @@ select * from ads."concat";
6986

7087
!ok
7188

72-
73-
create or replace view ADS."listagg" AS SELECT LISTAGG("FIRST_NAME") AS agg FROM profile.members;
89+
create or replace materialized view ADS."listagg" AS SELECT LISTAGG("FIRST_NAME") AS agg FROM profile.members;
7490
(0 rows modified)
7591

7692
!update
@@ -85,7 +101,24 @@ select * from ads."listagg";
85101

86102
!ok
87103

88-
create or replace view ADS."unnested" AS SELECT * FROM UNNEST(ARRAY(SELECT "FIRST_NAME" FROM profile.members)) AS name;
104+
create or replace materialized view ADS."arr" AS SELECT ARRAY["FIRST_NAME"] AS arr FROM profile.members;
105+
(0 rows modified)
106+
107+
!update
108+
109+
select * from ADS."arr";
110+
+-----------+
111+
| ARR |
112+
+-----------+
113+
| [Alice] |
114+
| [Bob] |
115+
| [Charlie] |
116+
+-----------+
117+
(3 rows)
118+
119+
!ok
120+
121+
create or replace materialized view ADS."unnested" AS SELECT * FROM UNNEST(SELECT ARR FROM ADS."arr") AS name;
89122
(0 rows modified)
90123

91124
!update
@@ -102,32 +135,42 @@ select * from ADS."unnested";
102135

103136
!ok
104137

105-
drop view ads."case";
138+
drop materialized view ads."unnested";
106139
(0 rows modified)
107140

108141
!update
109142

110-
drop view ads."json";
143+
drop materialized view ads."arr";
111144
(0 rows modified)
112145

113146
!update
114147

115-
drop view ads."regex";
148+
drop materialized view ads."listagg";
116149
(0 rows modified)
117150

118151
!update
119152

120-
drop view ads."concat";
153+
drop materialized view ads."concat";
121154
(0 rows modified)
122155

123156
!update
124157

125-
drop view ads."listagg";
158+
drop materialized view ads."regex";
126159
(0 rows modified)
127160

128161
!update
129162

130-
drop view ads."unnested";
163+
drop materialized view ads."extractJson";
131164
(0 rows modified)
132165

133-
!update
166+
!update
167+
168+
drop materialized view ads."createJson";
169+
(0 rows modified)
170+
171+
!update
172+
173+
drop materialized view ads."case";
174+
(0 rows modified)
175+
176+
!update

‎hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/PipelineRules.java

+83-5
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818
import org.apache.calcite.prepare.Prepare;
1919
import org.apache.calcite.rel.RelNode;
2020
import org.apache.calcite.rel.convert.ConverterRule;
21+
import org.apache.calcite.rel.core.Aggregate;
22+
import org.apache.calcite.rel.core.AggregateCall;
2123
import org.apache.calcite.rel.core.Calc;
2224
import org.apache.calcite.rel.core.CorrelationId;
2325
import org.apache.calcite.rel.core.Filter;
@@ -26,6 +28,9 @@
2628
import org.apache.calcite.rel.core.Project;
2729
import org.apache.calcite.rel.core.TableModify;
2830
import org.apache.calcite.rel.core.TableScan;
31+
import org.apache.calcite.rel.core.Uncollect;
32+
import org.apache.calcite.rel.hint.RelHint;
33+
import org.apache.calcite.rel.logical.LogicalAggregate;
2934
import org.apache.calcite.rel.logical.LogicalCalc;
3035
import org.apache.calcite.rel.logical.LogicalFilter;
3136
import org.apache.calcite.rel.logical.LogicalJoin;
@@ -38,6 +43,7 @@
3843
import org.apache.calcite.rex.RexProgram;
3944
import org.apache.calcite.schema.Table;
4045
import org.apache.calcite.sql.type.SqlTypeFactoryImpl;
46+
import org.apache.calcite.util.ImmutableBitSet;
4147

4248
import com.google.common.collect.ImmutableSet;
4349

@@ -51,7 +57,7 @@ private PipelineRules() {
5157

5258
public static Collection<RelOptRule> rules() {
5359
return Arrays.asList(PipelineFilterRule.INSTANCE, PipelineProjectRule.INSTANCE, PipelineJoinRule.INSTANCE,
54-
PipelineCalcRule.INSTANCE);
60+
PipelineCalcRule.INSTANCE, PipelineAggregateRule.INSTANCE, PipelineUncollectRule.INSTANCE);
5561
}
5662

5763
public static class PipelineTableScanRule extends ConverterRule {
@@ -72,9 +78,8 @@ protected PipelineTableScanRule(Config config, String database) {
7278
@Override
7379
public RelNode convert(RelNode rel) {
7480
TableScan scan = (TableScan) rel;
75-
RelOptTable table = scan.getTable();
7681
RelTraitSet traitSet = scan.getTraitSet().replace(PipelineRel.CONVENTION);
77-
return new PipelineTableScan(rel.getCluster(), traitSet, database, table);
82+
return new PipelineTableScan(rel.getCluster(), traitSet, database, scan.getTable());
7883
}
7984
}
8085

@@ -115,9 +120,8 @@ protected PipelineTableModifyRule(Config config, String database) {
115120
@Override
116121
public RelNode convert(RelNode rel) {
117122
TableModify mod = (TableModify) rel;
118-
RelOptTable table = mod.getTable();
119123
RelTraitSet traitSet = mod.getTraitSet().replace(PipelineRel.CONVENTION);
120-
return new PipelineTableModify(database, rel.getCluster(), traitSet, table, mod.getCatalogReader(),
124+
return new PipelineTableModify(database, rel.getCluster(), traitSet, mod.getTable(), mod.getCatalogReader(),
121125
convert(mod.getInput(), traitSet), mod.getOperation(), mod.getUpdateColumnList(),
122126
mod.getSourceExpressionList(), mod.isFlattened());
123127
}
@@ -307,6 +311,80 @@ public void implement(Implementor implementor) {
307311
}
308312
}
309313

314+
static class PipelineAggregateRule extends ConverterRule {
315+
static final PipelineAggregateRule INSTANCE =
316+
Config.INSTANCE.withConversion(LogicalAggregate.class, Convention.NONE, PipelineRel.CONVENTION, "PipelineAggregateRule")
317+
.withRuleFactory(PipelineAggregateRule::new)
318+
.as(Config.class)
319+
.toRule(PipelineAggregateRule.class);
320+
321+
protected PipelineAggregateRule(Config config) {
322+
super(config);
323+
}
324+
325+
@Override
326+
public RelNode convert(RelNode rel) {
327+
Aggregate agg = (Aggregate) rel;
328+
RelTraitSet traitSet = agg.getTraitSet().replace(PipelineRel.CONVENTION);
329+
return new PipelineAggregate(rel.getCluster(), traitSet, agg.getHints(), convert(agg.getInput(), PipelineRel.CONVENTION),
330+
agg.getGroupSet(), agg.getGroupSets(), agg.getAggCallList());
331+
}
332+
}
333+
334+
static class PipelineAggregate extends Aggregate implements PipelineRel {
335+
336+
PipelineAggregate(RelOptCluster cluster, RelTraitSet traitSet, List<RelHint> hints, RelNode input,
337+
ImmutableBitSet groupSet, List<ImmutableBitSet> groupSets, List<AggregateCall> aggCalls) {
338+
super(cluster, traitSet, hints, input, groupSet, groupSets, aggCalls);
339+
}
340+
341+
@Override
342+
public PipelineAggregate copy(RelTraitSet traitSet, RelNode input, ImmutableBitSet groupSet,
343+
List<ImmutableBitSet> groupSets, List<AggregateCall> aggCalls) {
344+
return new PipelineAggregate(getCluster(), traitSet, Collections.emptyList(), input, groupSet, groupSets, aggCalls);
345+
}
346+
347+
@Override
348+
public void implement(Implementor implementor) {
349+
}
350+
}
351+
352+
static class PipelineUncollectRule extends ConverterRule {
353+
static final PipelineUncollectRule INSTANCE =
354+
Config.INSTANCE.withConversion(Uncollect.class, Convention.NONE, PipelineRel.CONVENTION, "PipelineUncollectRule")
355+
.withRuleFactory(PipelineUncollectRule::new)
356+
.as(Config.class)
357+
.toRule(PipelineUncollectRule.class);
358+
359+
protected PipelineUncollectRule(Config config) {
360+
super(config);
361+
}
362+
363+
@Override
364+
public RelNode convert(RelNode rel) {
365+
Uncollect uncollect = (Uncollect) rel;
366+
RelTraitSet traitSet = uncollect.getTraitSet().replace(PipelineRel.CONVENTION);
367+
return new PipelineUncollect(rel.getCluster(), traitSet, convert(uncollect.getInput(), PipelineRel.CONVENTION),
368+
uncollect.withOrdinality, Collections.emptyList());
369+
}
370+
}
371+
372+
static class PipelineUncollect extends Uncollect implements PipelineRel {
373+
PipelineUncollect(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, boolean withOrdinality,
374+
List<String> itemAliases) {
375+
super(cluster, traitSet, input, withOrdinality, itemAliases);
376+
}
377+
378+
@Override
379+
public PipelineUncollect copy(RelTraitSet traitSet, RelNode input) {
380+
return new PipelineUncollect(getCluster(), traitSet, input, withOrdinality, Collections.emptyList());
381+
}
382+
383+
@Override
384+
public void implement(Implementor implementor) {
385+
}
386+
}
387+
310388
static Table findTable(CalciteSchema schema, List<String> qualifiedName) {
311389
if (qualifiedName.size() == 0) {
312390
throw new IllegalArgumentException("Empty qualified name.");

0 commit comments

Comments
 (0)