Implementing JEP-18: Lexical scope

src/Lexer.php

@@ -20,6 +20,7 @@ class Lexer
     const T_RBRACKET = 'rbracket';
     const T_FLATTEN = 'flatten';
     const T_IDENTIFIER = 'identifier';
+    const T_VARIABLE = 'variable';
     const T_NUMBER = 'number';
     const T_QUOTED_IDENTIFIER = 'quoted_identifier';
     const T_UNKNOWN = 'unknown';
@@ -31,6 +32,7 @@ class Lexer
     const T_LITERAL = 'literal';
     const T_EOF = 'eof';
     const T_COMPARATOR = 'comparator';
+    const T_ASSIGN = 'assign';
 
     const STATE_IDENTIFIER = 0;
     const STATE_NUMBER = 1;
@@ -46,6 +48,7 @@ class Lexer
     const STATE_EQ = 11;
     const STATE_NOT = 12;
     const STATE_AND = 13;
+    const STATE_VARIABLE = 14;
 
     /** @var array We know what token we are consuming based on each char */
     private static $transitionTable = [
@@ -84,6 +87,7 @@ class Lexer
         ')'  => self::STATE_SINGLE_CHAR,
         '{'  => self::STATE_SINGLE_CHAR,
         '}'  => self::STATE_SINGLE_CHAR,
+        '$'  => self::STATE_VARIABLE,
         '_'  => self::STATE_IDENTIFIER,
         'A'  => self::STATE_IDENTIFIER,
         'B'  => self::STATE_IDENTIFIER,
@@ -223,18 +227,31 @@ public function tokenize($input)
             } elseif ($state === self::STATE_IDENTIFIER) {
 
                 // Consume identifiers
-                $start = key($chars);
-                $buffer = '';
-                do {
-                    $buffer .= $current;
-                    $current = next($chars);
-                } while ($current !== false && isset($this->validIdentifier[$current]));
                 $tokens[] = [
                     'type'  => self::T_IDENTIFIER,
-                    'value' => $buffer,
-                    'pos'   => $start
+                    'pos'   => key($chars),
+                    'value' => $this->consumeIdentifier($chars)
                 ];
 
+            } elseif ($state === self::STATE_VARIABLE) {
+
+                // Consume variable reference
+                $start = key($chars);
+                $actual = next($chars);
+                if (self::$transitionTable[$actual] === self::STATE_IDENTIFIER) {
+                    $tokens[] = [
+                        'type'  => self::T_VARIABLE,
+                        'pos'   => $start,
+                        'value' => $this->consumeIdentifier($chars)
+                    ];
+                } else {
+                    $tokens[] = [
+                        'type'  => self::T_UNKNOWN,
+                        'pos'   => $start,
+                        'value' => $current
+                    ];
+                }
+
             } elseif ($state === self::STATE_WHITESPACE) {
 
                 // Skip whitespace
@@ -317,7 +334,7 @@ public function tokenize($input)
             } elseif ($state === self::STATE_EQ) {
 
                 // Consume equals
-                $tokens[] = $this->matchOr($chars, '=', '=', self::T_COMPARATOR, self::T_UNKNOWN);
+                $tokens[] = $this->matchOr($chars, '=', '=', self::T_COMPARATOR, self::T_ASSIGN);
 
             } elseif ($state == self::STATE_AND) {
 
@@ -417,6 +434,27 @@ private function inside(array &$chars, $delim, $type)
         return ['type' => $type, 'value' => $buffer, 'pos' => $position];
     }
 
+    /**
+     * Consumes input until any character is found that is invalid in an identifier.
+     *
+     * It is assumed the first character in the input has already been recognized as
+     * a valid first character for an identifier.
+     *
+     * @param &array $chars Reference to the input to be consumed
+     *
+     * @return string Returns the consumed identifier
+     */
+    private function consumeIdentifier(array &$chars): string {
+        $current = current($chars);
+        $buffer = '';
+        do {
+            $buffer .= $current;
+            $current = next($chars);
+        } while ($current !== false && isset($this->validIdentifier[$current]));
+
+        return $buffer;
+    }
+
     /**
      * Parses a JSON token or sets the token type to "unknown" on error.
      *

src/Parser.php

@@ -22,6 +22,7 @@ class Parser
         T::T_EOF               => 0,
         T::T_QUOTED_IDENTIFIER => 0,
         T::T_IDENTIFIER        => 0,
+        T::T_VARIABLE          => 0,
         T::T_RBRACKET          => 0,
         T::T_RPAREN            => 0,
         T::T_COMMA             => 0,
@@ -105,6 +106,11 @@ private function expr($rbp = 0)
     private function nud_identifier()
     {
         $token = $this->token;
+
+        if ($token['value'] === 'let' && $this->lookahead() === T::T_VARIABLE) {
+            return $this->parseLetExpression();
+        }
+
         $this->next();
         return ['type' => 'field', 'value' => $token['value']];
     }
@@ -117,6 +123,14 @@ private function nud_quoted_identifier()
         return ['type' => 'field', 'value' => $token['value']];
     }
 
+    private function nud_variable()
+    {
+        $token = $this->token;
+
+        $this->next();
+        return $token;
+    }
+
     private function nud_current()
     {
         $this->next();
@@ -461,6 +475,42 @@ private function parseMultiSelectList()
         return ['type' => 'multi_select_list', 'children' => $nodes];
     }
 
+    private function parseLetExpression()
+    {
+        static $validVariable = [ T::T_VARIABLE => true ];
+        static $validAssign = [ T::T_ASSIGN => true ];
+
+        $bindings = [];
+
+        do {
+            $this->next($validVariable);
+            $variable = $this->token['value'];
+
+            $this->next($validAssign);
+            $this->next();
+
+            $bindings[] = [
+                'type'     => 'variable_binding',
+                'value'    => $variable,
+                'children' => [ $this->expr() ]
+            ];
+        } while ($this->token['type'] === T::T_COMMA);
+
+        if ($this->token['type'] !== T::T_IDENTIFIER || $this->token['value'] !== 'in') {
+            $this->syntax("Expected ',' or 'in'");
+        }
+
+        $this->next();
+
+        return [
+            'type' => 'let',
+            'children' => [
+                [ 'type' => 'bindings', 'children' => $bindings ],
+                $this->expr()
+            ]
+        ];
+    }
+
     private function syntax($msg)
     {
         return new SyntaxErrorException($msg, $this->token, $this->expression);

src/TreeCompiler.php

@@ -26,7 +26,7 @@ public function visit(array $ast, $fnName, $expr)
             ->write('use JmesPath\\FnDispatcher as Fd;')
             ->write('use JmesPath\\Utils;')
             ->write('')
-            ->write('function %s(Ti $interpreter, $value) {', $fnName)
+            ->write('function %s(Ti $interpreter, $value, array $bindings = []) {', $fnName)
             ->indent()
                 ->dispatch($ast)
                 ->write('')
@@ -409,6 +409,56 @@ private function visit_comparator(array $node)
         return $this;
     }
 
+    private function visit_let(array $node) {
+        return $this
+            ->write('$value = (function() use ($value, $bindings) {')
+            ->indent()
+            ->write('$newBindings = [];')
+            ->dispatch($node['children'][0])
+            ->write('$bindings = array_merge($bindings, $newBindings);')
+            ->dispatch($node['children'][1])
+            ->write('return $value;')
+            ->outdent()
+            ->write('})();');
+    }
+
+    private function visit_bindings(array $node) {
+        $value = $this->makeVar('prev');
+        $this
+            ->write('if ($value !== null) {')
+            ->indent()
+            ->write('%s = $value;', $value);
+
+        $first = true;
+        foreach ($node['children'] as $child) {
+            if (!$first) {
+                $this->write('$value = %s;', $value);
+            }
+            $first = false;
+            $this->dispatch($child);
+        }
+
+        return $this
+            ->write('$value = %s;', $value)
+            ->outdent()
+            ->write('}');
+    }
+
+    private function visit_variable_binding(array $node) {
+        return $this->dispatch($node['children'][0])
+            ->write("\$newBindings['{$node['value']}'] = \$value;");
+    }
+
+    private function visit_variable(array $node) {
+        return $this
+            ->write("if (!array_key_exists('{$node['value']}', \$bindings)) {")
+            ->indent()
+            ->write('throw new \\RuntimeException("Undefined variable: \\${$node[\'value\']}");')
+            ->outdent()
+            ->write('}')
+            ->write("\$value = \$bindings['{$node['value']}'];");
+    }
+
     /** @internal */
     public function __call($method, $args)
     {