fix(vrl): fix if statement type definitions (#12954)

Author: fuchsnj

lib/vrl/compiler/src/compiler.rs

@@ -237,7 +237,7 @@ impl<'a> Compiler<'a> {
         // the block, and merge back into it any mutations that happened to
         // state the snapshot was already tracking. Then, revert the compiler
         // local state to the updated snapshot.
-        self.local = local_snapshot.merge_mutations(self.local.clone());
+        self.local = local_snapshot.apply_child_scope(self.local.clone());
 
         block
     }
@@ -266,6 +266,8 @@ impl<'a> Compiler<'a> {
         node: Node<ast::IfStatement>,
         external: &mut ExternalEnv,
     ) -> IfStatement {
+        use crate::type_def::{Details, TypeDef};
+
         let ast::IfStatement {
             predicate,
             consequent,
@@ -280,13 +282,54 @@ impl<'a> Compiler<'a> {
             }
         };
 
+        let original_locals = self.local.clone();
+        let original_external = external.target().cloned();
+
         let consequent = self.compile_block(consequent, external);
-        let alternative = alternative.map(|block| self.compile_block(block, external));
 
-        IfStatement {
-            predicate,
-            consequent,
-            alternative,
+        match alternative {
+            Some(block) => {
+                let consequent_locals = self.local.clone();
+                let consequent_external = external.target().cloned();
+
+                self.local = original_locals;
+
+                let else_block = self.compile_block(block, external);
+
+                // assignments must be the result of either the if or else block, but not the original value
+                self.local = self.local.clone().merge(consequent_locals);
+                if let Some(details) = Details::merge_optional(
+                    consequent_external,
+                    external.target().cloned(),
+                    TypeDef::any(),
+                ) {
+                    external.update_target(details);
+                }
+
+                IfStatement {
+                    predicate,
+                    consequent,
+                    alternative: Some(else_block),
+                }
+            }
+            None => {
+                // assignments must be the result of either the if block or the original value
+                self.local = self.local.clone().merge(original_locals);
+
+                if let Some(details) = Details::merge_optional(
+                    original_external,
+                    external.target().cloned(),
+                    TypeDef::any(),
+                ) {
+                    external.update_target(details);
+                }
+
+                IfStatement {
+                    predicate,
+                    consequent,
+                    alternative: None,
+                }
+            }
         }
     }
 

lib/vrl/compiler/src/expression/if_statement.rs

@@ -49,7 +49,7 @@ impl Expression for IfStatement {
         let type_def = self.consequent.type_def(state);
 
         match &self.alternative {
-            None => type_def,
+            None => type_def.add_null(),
             Some(alternative) => type_def.merge_deep(alternative.type_def(state)),
         }
     }

lib/vrl/compiler/src/state.rs

@@ -30,16 +30,29 @@ impl LocalEnv {
         self.bindings.remove(ident)
     }
 
-    /// Merge state present in both `self` and `other`.
-    pub(crate) fn merge_mutations(mut self, other: Self) -> Self {
-        for (ident, other_details) in other.bindings.into_iter() {
+    /// Any state the child scope modified that was part of the parent is copied to the parent scope
+    pub(crate) fn apply_child_scope(mut self, child: Self) -> Self {
+        for (ident, child_details) in child.bindings {
             if let Some(self_details) = self.bindings.get_mut(&ident) {
-                *self_details = other_details;
+                *self_details = child_details;
             }
         }
 
         self
     }
+
+    /// Merges two local envs together. This is useful in cases such as if statements
+    /// where different LocalEnv's can be created, and the result is decided at runtime.
+    /// The compile-time type must be the union of the options.
+    #[cfg(feature = "expr-if_statement")]
+    pub(crate) fn merge(mut self, other: Self) -> Self {
+        for (ident, other_details) in other.bindings {
+            if let Some(self_details) = self.bindings.get_mut(&ident) {
+                *self_details = self_details.clone().merge(other_details);
+            }
+        }
+        self
+    }
 }
 
 /// A lexical scope within the program.

lib/vrl/compiler/src/type_def.rs

@@ -350,21 +350,7 @@ impl TypeDef {
 
     pub fn merge(&mut self, other: Self, strategy: merge::Strategy) {
         self.fallible |= other.fallible;
-
-        // NOTE: technically we shouldn't do this, but to keep backward compatibility with the
-        // "old" `Kind` implementation, we consider a type that is marked as "any" equal to the old
-        // implementation's `unknown`, which, when merging, would discard that type.
-        //
-        // see: https://github.com/vectordotdev/vector/blob/18415050b60b08197e8135b7659390256995e844/lib/vrl/compiler/src/type_def.rs#L428-L429
-        if !self.is_any() && other.is_any() {
-            // keep `self`'s `kind` definition
-        } else if self.is_any() && !other.is_any() {
-            // overwrite `self`'s `kind` definition with `others`'s
-            self.kind = other.kind;
-        } else {
-            // merge the two `kind`s
-            self.kind.merge(other.kind, strategy);
-        }
+        self.kind.merge(other.kind, strategy);
     }
 
     pub fn merge_overwrite(mut self, other: Self) -> Self {
@@ -399,3 +385,78 @@ pub(crate) struct Details {
     pub(crate) type_def: TypeDef,
     pub(crate) value: Option<Value>,
 }
+
+impl Details {
+    /// Returns the union of 2 possible states
+    #[cfg(any(test, feature = "expr-if_statement"))]
+    pub(crate) fn merge(self, other: Self) -> Self {
+        Self {
+            type_def: self.type_def.merge_deep(other.type_def),
+            value: if self.value == other.value {
+                self.value
+            } else {
+                None
+            },
+        }
+    }
+
+    /// Returns the union of 2 possible states
+    #[cfg(any(feature = "expr-if_statement"))]
+    pub(crate) fn merge_optional(
+        a: Option<Self>,
+        b: Option<Self>,
+        none_type: TypeDef,
+    ) -> Option<Self> {
+        match (a, b) {
+            (Some(a), Some(b)) => Some(a.merge(b)),
+            (Some(_), None) | (None, Some(_)) => Some(Details {
+                type_def: none_type,
+                value: None,
+            }),
+            (None, None) => None,
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn merge_details_same_literal() {
+        let a = Details {
+            type_def: TypeDef::integer(),
+            value: Some(Value::from(5)),
+        };
+        let b = Details {
+            type_def: TypeDef::float(),
+            value: Some(Value::from(5)),
+        };
+        assert_eq!(
+            a.merge(b),
+            Details {
+                type_def: TypeDef::integer().add_float(),
+                value: Some(Value::from(5))
+            }
+        )
+    }
+
+    #[test]
+    fn merge_details_different_literal() {
+        let a = Details {
+            type_def: TypeDef::any(),
+            value: Some(Value::from(5)),
+        };
+        let b = Details {
+            type_def: TypeDef::object(Collection::empty()),
+            value: Some(Value::from(6)),
+        };
+        assert_eq!(
+            a.merge(b),
+            Details {
+                type_def: TypeDef::any(),
+                value: None
+            }
+        )
+    }
+}

lib/vrl/stdlib/src/append.rs

@@ -76,7 +76,8 @@ impl Expression for AppendFn {
     fn type_def(&self, state: (&state::LocalEnv, &state::ExternalEnv)) -> TypeDef {
         self.value
             .type_def(state)
-            .merge_append(self.items.type_def(state))
+            .restrict_array()
+            .merge_append(self.items.type_def(state).restrict_array())
     }
 }
 

lib/vrl/stdlib/src/merge.rs

@@ -89,7 +89,8 @@ impl Expression for MergeFn {
     fn type_def(&self, state: (&state::LocalEnv, &state::ExternalEnv)) -> TypeDef {
         self.to
             .type_def(state)
-            .merge_shallow(self.from.type_def(state))
+            .restrict_object()
+            .merge_shallow(self.from.type_def(state).restrict_object())
     }
 }
 

lib/vrl/stdlib/src/push.rs

@@ -84,8 +84,11 @@ impl Expression for PushFn {
             0.into(),
             self.item.type_def(state).into(),
         )]));
-
-        self.value.type_def(state).merge_append(item).infallible()
+        self.value
+            .type_def(state)
+            .restrict_array()
+            .merge_append(item)
+            .infallible()
     }
 }
 

lib/vrl/tests/tests/expressions/if_statement/if_else_external_assignment.vrl

@@ -0,0 +1,19 @@
+# result: true
+
+.a = 5
+if true {
+  .a = 0.0
+} else {
+  .a = "string"
+}
+assert!(.a == 0.0)
+assert!(type_def(.a) == {"float": true, "bytes": true})
+
+.a = 5
+if false {
+  .a = 0.0
+} else {
+  .a = "string"
+}
+assert!(.a == "string")
+assert!(type_def(.a) == {"float": true, "bytes": true})

lib/vrl/tests/tests/expressions/if_statement/if_else_local_assignment.vrl

@@ -0,0 +1,19 @@
+# result: true
+
+a = 5
+if true {
+  a = 0.0
+} else {
+  a = "string"
+}
+assert!(a == 0.0)
+assert!(type_def(a) == {"float": true, "bytes": true})
+
+a = 5
+if false {
+  a = 0.0
+} else {
+  a = "string"
+}
+assert!(a == "string")
+assert!(type_def(a) == {"float": true, "bytes": true})

lib/vrl/tests/tests/expressions/if_statement/if_expression.vrl

@@ -0,0 +1,14 @@
+# result: true
+
+result = if true {
+  "string"
+}
+assert!(result == "string")
+assert!(type_def(result) == {"bytes": true, "null": true})
+
+
+result = if false {
+  "string"
+}
+assert!(result == null)
+assert!(type_def(result) == {"bytes": true, "null": true})

lib/vrl/tests/tests/expressions/if_statement/if_external_assignment.vrl

@@ -0,0 +1,15 @@
+# result: true
+
+.a = 5
+if false {
+  .a = "string"
+}
+assert!(.a == 5)
+assert!(type_def(.a) == {"integer": true, "bytes": true})
+
+.a = 5
+if true {
+  .a = "string"
+}
+assert!(.a == "string")
+assert!(type_def(.a) == {"integer": true, "bytes": true})

lib/vrl/tests/tests/expressions/if_statement/if_local_assignment.vrl

@@ -0,0 +1,15 @@
+# result: true
+
+a = 5
+if false {
+  a = "string"
+}
+assert!(a == 5)
+assert!(type_def(a) == {"integer": true, "bytes": true})
+
+a = 5
+if true {
+  a = "string"
+}
+assert!(a == "string")
+assert!(type_def(a) == {"integer": true, "bytes": true})

lib/vrl/tests/tests/expressions/if_statement/if_new_external_assignment.vrl

@@ -0,0 +1,7 @@
+# result: {"any": true}
+
+assert!(type_def(.a) == {"any": true})
+if false {
+  .a = "string"
+}
+type_def(.a)

lib/vrl/tests/tests/expressions/if_statement/if_null.vrl

@@ -1,5 +1,7 @@
-# result: null
+# result: true
 
-if false {
+result = if false {
     "yes"
 }
+assert!(result == null)
+assert!(type_def(result) == {"bytes": true, "null": true})

lib/vrl/tests/tests/expressions/if_statement/if_resolves.vrl

@@ -1,5 +1,7 @@
-# result: "yes"
+# result: true
 
-if true {
+result = if true {
     "yes"
 }
+assert!(result == "yes")
+assert!(type_def(result) == {"bytes": true, "null": true})

lib/vrl/tests/tests/issues/12948_conditional_type_def_regression.vrl

@@ -0,0 +1,7 @@
+# result: null
+
+if .platform == "Apache2" || .platform == "Nginx" {
+  apache2 = del(.apache2)
+  if is_null(apache2) { apache2 = {} }
+  .http = merge(object!(apache2), {})
+}

scripts/test-vrl.sh

@@ -10,5 +10,5 @@ set -euo pipefail
 (
   cd "$(dirname "${BASH_SOURCE[0]}")/../lib/vrl/tests"
 
-  cargo run -- --runtime=ast && cargo run -- --runtime=vm
+  cargo run -- --runtime=ast
 )