Namespaces

  • Nette
    • Application
      • Diagnostics
      • Responses
      • Routers
      • UI
    • Caching
      • Storages
    • ComponentModel
    • Config
      • Adapters
      • Extensions
    • Database
      • Diagnostics
      • Drivers
      • Reflection
      • Table
    • DI
      • Diagnostics
    • Diagnostics
    • Forms
      • Controls
      • Rendering
    • Http
    • Iterators
    • Latte
      • Macros
    • Loaders
    • Localization
    • Mail
    • Reflection
    • Security
      • Diagnostics
    • Templating
    • Utils
      • PhpGenerator
  • NetteModule
  • None
  • PHP

Classes

  • Arrays
  • Finder
  • Html
  • Json
  • LimitedScope
  • MimeTypeDetector
  • Neon
  • NeonEntity
  • Paginator
  • Strings
  • Tokenizer
  • Validators

Exceptions

  • AssertionException
  • JsonException
  • NeonException
  • RegexpException
  • TokenizerException
  • Overview
  • Namespace
  • Class
  • Tree
  • Deprecated
  1: <?php
  2: 
  3: /**
  4:  * This file is part of the Nette Framework (http://nette.org)
  5:  *
  6:  * Copyright (c) 2004 David Grudl (http://davidgrudl.com)
  7:  *
  8:  * For the full copyright and license information, please view
  9:  * the file license.txt that was distributed with this source code.
 10:  */
 11: 
 12: namespace Nette\Utils;
 13: 
 14: use Nette;
 15: 
 16: 
 17: 
 18: /**
 19:  * Simple lexical analyser.
 20:  *
 21:  * @author     David Grudl
 22:  */
 23: class Tokenizer extends Nette\Object
 24: {
 25:     /** @var array */
 26:     public $tokens;
 27: 
 28:     /** @var int */
 29:     public $position = 0;
 30: 
 31:     /** @var array */
 32:     public $ignored = array();
 33: 
 34:     /** @var string */
 35:     private $input;
 36: 
 37:     /** @var string */
 38:     private $re;
 39: 
 40:     /** @var array */
 41:     private $types;
 42: 
 43:     /** @var array|string */
 44:     public $current;
 45: 
 46: 
 47: 
 48:     /**
 49:      * @param  array of [(int) symbol type => pattern]
 50:      * @param  string  regular expression flag
 51:      */
 52:     public function __construct(array $patterns, $flags = '')
 53:     {
 54:         $this->re = '~(' . implode(')|(', $patterns) . ')~A' . $flags;
 55:         $keys = array_keys($patterns);
 56:         $this->types = $keys === range(0, count($patterns) - 1) ? FALSE : $keys;
 57:     }
 58: 
 59: 
 60: 
 61:     /**
 62:      * Tokenize string.
 63:      * @param  string
 64:      * @return array
 65:      */
 66:     public function tokenize($input)
 67:     {
 68:         $this->input = $input;
 69:         if ($this->types) {
 70:             $this->tokens = Strings::matchAll($input, $this->re);
 71:             $len = 0;
 72:             $count = count($this->types);
 73:             $line = 1;
 74:             foreach ($this->tokens as & $match) {
 75:                 $type = NULL;
 76:                 for ($i = 1; $i <= $count; $i++) {
 77:                     if (!isset($match[$i])) {
 78:                         break;
 79:                     } elseif ($match[$i] != NULL) {
 80:                         $type = $this->types[$i - 1]; break;
 81:                     }
 82:                 }
 83:                 $match = self::createToken($match[0], $type, $line);
 84:                 $len += strlen($match['value']);
 85:                 $line += substr_count($match['value'], "\n");
 86:             }
 87:             if ($len !== strlen($input)) {
 88:                 $errorOffset = $len;
 89:             }
 90: 
 91:         } else {
 92:             $this->tokens = Strings::split($input, $this->re, PREG_SPLIT_NO_EMPTY);
 93:             if ($this->tokens && !Strings::match(end($this->tokens), $this->re)) {
 94:                 $tmp = Strings::split($this->input, $this->re, PREG_SPLIT_NO_EMPTY | PREG_SPLIT_OFFSET_CAPTURE);
 95:                 list(, $errorOffset) = end($tmp);
 96:             }
 97:         }
 98: 
 99:         if (isset($errorOffset)) {
100:             $line = $errorOffset ? substr_count($this->input, "\n", 0, $errorOffset) + 1 : 1;
101:             $col = $errorOffset - strrpos(substr($this->input, 0, $errorOffset), "\n") + 1;
102:             $token = str_replace("\n", '\n', substr($input, $errorOffset, 10));
103:             throw new TokenizerException("Unexpected '$token' on line $line, column $col.");
104:         }
105:         return $this->tokens;
106:     }
107: 
108: 
109: 
110:     public static function createToken($value, $type = NULL, $line = NULL)
111:     {
112:         return array('value' => $value, 'type' => $type, 'line' => $line);
113:     }
114: 
115: 
116: 
117:     /**
118:      * Returns position of token in input string.
119:      * @param  int token number
120:      * @return array [offset, line, column]
121:      */
122:     public function getOffset($i)
123:     {
124:         $tokens = Strings::split($this->input, $this->re, PREG_SPLIT_NO_EMPTY | PREG_SPLIT_OFFSET_CAPTURE);
125:         $offset = isset($tokens[$i]) ? $tokens[$i][1] : strlen($this->input);
126:         return array(
127:             $offset,
128:             ($offset ? substr_count($this->input, "\n", 0, $offset) + 1 : 1),
129:             $offset - strrpos(substr($this->input, 0, $offset), "\n"),
130:         );
131:     }
132: 
133: 
134: 
135:     /**
136:      * Returns next token as string.
137:      * @param  desired token
138:      * @return string
139:      */
140:     public function fetch()
141:     {
142:         $args = func_get_args();
143:         return $this->scan($args, TRUE);
144:     }
145: 
146: 
147: 
148:     /**
149:      * Returns next token.
150:      * @param  desired token
151:      * @return array|string
152:      */
153:     public function fetchToken()
154:     {
155:         $args = func_get_args();
156:         return $this->scan($args, TRUE) === FALSE ? FALSE : $this->current;
157:     }
158: 
159: 
160: 
161:     /**
162:      * Returns concatenation of all next tokens.
163:      * @param  desired token
164:      * @return string
165:      */
166:     public function fetchAll()
167:     {
168:         $args = func_get_args();
169:         return $this->scan($args, FALSE);
170:     }
171: 
172: 
173: 
174:     /**
175:      * Returns concatenation of all next tokens until it sees a token with the given value.
176:      * @param  tokens
177:      * @return string
178:      */
179:     public function fetchUntil($arg)
180:     {
181:         $args = func_get_args();
182:         return $this->scan($args, FALSE, TRUE, TRUE);
183:     }
184: 
185: 
186: 
187:     /**
188:      * Checks the next token.
189:      * @param  token
190:      * @return string
191:      */
192:     public function isNext($arg)
193:     {
194:         $args = func_get_args();
195:         return (bool) $this->scan($args, TRUE, FALSE);
196:     }
197: 
198: 
199: 
200:     /**
201:      * Checks the previous token.
202:      * @param  token
203:      * @return string
204:      */
205:     public function isPrev($arg)
206:     {
207:         $args = func_get_args();
208:         return (bool) $this->scan($args, TRUE, FALSE, FALSE, TRUE);
209:     }
210: 
211: 
212: 
213:     /**
214:      * Checks existence of next token.
215:      * @return bool
216:      */
217:     public function hasNext()
218:     {
219:         return isset($this->tokens[$this->position]);
220:     }
221: 
222: 
223: 
224:     /**
225:      * Checks existence of previous token.
226:      * @return bool
227:      */
228:     public function hasPrev()
229:     {
230:         return $this->position > 1;
231:     }
232: 
233: 
234: 
235:     /**
236:      * Checks the current token.
237:      * @param  token
238:      * @return string
239:      */
240:     public function isCurrent($arg)
241:     {
242:         $args = func_get_args();
243:         if (is_array($this->current)) {
244:             return in_array($this->current['value'], $args, TRUE)
245:                 || in_array($this->current['type'], $args, TRUE);
246:         } else {
247:             return in_array($this->current, $args, TRUE);
248:         }
249:     }
250: 
251: 
252: 
253:     public function reset()
254:     {
255:         $this->position = 0;
256:         $this->current = NULL;
257:     }
258: 
259: 
260: 
261:     /**
262:      * Looks for (first) (not) wanted tokens.
263:      * @param  int token number
264:      * @return array
265:      */
266:     private function scan($wanted, $first, $advance = TRUE, $neg = FALSE, $prev = FALSE)
267:     {
268:         $res = FALSE;
269:         $pos = $this->position + ($prev ? -2 : 0);
270:         while (isset($this->tokens[$pos])) {
271:             $token = $this->tokens[$pos];
272:             $pos += $prev ? -1 : 1;
273:             $value = is_array($token) ? $token['value'] : $token;
274:             $type = is_array($token) ? $token['type'] : $token;
275:             if (!$wanted || (in_array($value, $wanted, TRUE) || in_array($type, $wanted, TRUE)) ^ $neg) {
276:                 if ($advance) {
277:                     $this->position = $pos;
278:                     $this->current = $token;
279:                 }
280:                 $res .= $value;
281:                 if ($first) {
282:                     break;
283:                 }
284: 
285:             } elseif ($neg || !in_array($type, $this->ignored, TRUE)) {
286:                 break;
287:             }
288:         }
289:         return $res;
290:     }
291: 
292: }
293: 
294: 
295: 
296: /**
297:  * The exception that indicates tokenizer error.
298:  */
299: class TokenizerException extends \Exception
300: {
301: }
302: 
Nette Framework 2.0.8 API API documentation generated by ApiGen 2.8.0