Namespaces

  • Latte
    • Loaders
    • Macros
    • Runtime
  • Nette
    • Application
      • Responses
      • Routers
      • UI
    • Bridges
      • ApplicationDI
      • ApplicationLatte
      • ApplicationTracy
      • CacheDI
      • CacheLatte
      • DatabaseDI
      • DatabaseTracy
      • DITracy
      • FormsDI
      • FormsLatte
      • Framework
      • HttpDI
      • HttpTracy
      • MailDI
      • ReflectionDI
      • SecurityDI
      • SecurityTracy
    • Caching
      • Storages
    • ComponentModel
    • Database
      • Conventions
      • Drivers
      • Reflection
      • Table
    • DI
      • Config
        • Adapters
      • Extensions
    • Forms
      • Controls
      • Rendering
    • Http
    • Iterators
    • Loaders
    • Localization
    • Mail
    • Neon
    • PhpGenerator
    • Reflection
    • Security
    • Utils
  • none
  • Tracy
    • Bridges
      • Nette

Classes

  • ArrayHash
  • ArrayList
  • Arrays
  • Callback
  • DateTime
  • FileSystem
  • Finder
  • Html
  • Image
  • Json
  • LimitedScope
  • MimeTypeDetector
  • ObjectMixin
  • Paginator
  • Random
  • Strings
  • TokenIterator
  • Tokenizer
  • Validators

Interfaces

  • IHtmlString

Exceptions

  • AssertionException
  • ImageException
  • JsonException
  • RegexpException
  • TokenizerException
  • UnknownImageFileException
  • Overview
  • Namespace
  • Class
  • Tree
  • Deprecated
 1: <?php
 2: 
 3: /**
 4:  * This file is part of the Nette Framework (https://nette.org)
 5:  * Copyright (c) 2004 David Grudl (https://davidgrudl.com)
 6:  */
 7: 
 8: namespace Nette\Utils;
 9: 
10: 
11: /**
12:  * Simple lexical analyser.
13:  */
14: class Tokenizer
15: {
16:     const VALUE = 0,
17:         OFFSET = 1,
18:         TYPE = 2;
19: 
20:     /** @var string */
21:     private $re;
22: 
23:     /** @var array|false */
24:     private $types;
25: 
26: 
27:     /**
28:      * @param  array of [(int|string) token type => (string) pattern]
29:      * @param  string  regular expression flags
30:      */
31:     public function __construct(array $patterns, $flags = '')
32:     {
33:         $this->re = '~(' . implode(')|(', $patterns) . ')~A' . $flags;
34:         $keys = array_keys($patterns);
35:         $this->types = $keys === range(0, count($patterns) - 1) ? false : $keys;
36:     }
37: 
38: 
39:     /**
40:      * Tokenizes string.
41:      * @param  string
42:      * @return array
43:      * @throws TokenizerException
44:      */
45:     public function tokenize($input)
46:     {
47:         if ($this->types) {
48:             preg_match_all($this->re, $input, $tokens, PREG_SET_ORDER);
49:             $len = 0;
50:             $count = count($this->types);
51:             foreach ($tokens as &$match) {
52:                 $type = null;
53:                 for ($i = 1; $i <= $count; $i++) {
54:                     if (!isset($match[$i])) {
55:                         break;
56:                     } elseif ($match[$i] != null) {
57:                         $type = $this->types[$i - 1];
58:                         break;
59:                     }
60:                 }
61:                 $match = array(self::VALUE => $match[0], self::OFFSET => $len, self::TYPE => $type);
62:                 $len += strlen($match[self::VALUE]);
63:             }
64:             if ($len !== strlen($input)) {
65:                 $errorOffset = $len;
66:             }
67: 
68:         } else {
69:             $tokens = preg_split($this->re, $input, -1, PREG_SPLIT_NO_EMPTY | PREG_SPLIT_OFFSET_CAPTURE | PREG_SPLIT_DELIM_CAPTURE);
70:             $last = end($tokens);
71:             if ($tokens && !preg_match($this->re, $last[0])) {
72:                 $errorOffset = $last[1];
73:             }
74:         }
75: 
76:         if (isset($errorOffset)) {
77:             list($line, $col) = $this->getCoordinates($input, $errorOffset);
78:             $token = str_replace("\n", '\n', substr($input, $errorOffset, 10));
79:             throw new TokenizerException("Unexpected '$token' on line $line, column $col.");
80:         }
81:         return $tokens;
82:     }
83: 
84: 
85:     /**
86:      * Returns position of token in input string.
87:      * @param  string
88:      * @param  int
89:      * @return array of [line, column]
90:      */
91:     public static function getCoordinates($text, $offset)
92:     {
93:         $text = substr($text, 0, $offset);
94:         return array(substr_count($text, "\n") + 1, $offset - strrpos("\n" . $text, "\n") + 1);
95:     }
96: }
97: 
Nette 2.3-20201001 API API documentation generated by ApiGen 2.8.0