Tokenizer.php 112 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429
  1. <?php
  2. /*
  3. Copyright 2007 Jeroen van der Meer <http://jero.net/>
  4. Copyright 2008 Edward Z. Yang <http://htmlpurifier.org/>
  5. Copyright 2009 Geoffrey Sneddon <http://gsnedders.com/>
  6. Permission is hereby granted, free of charge, to any person obtaining a
  7. copy of this software and associated documentation files (the
  8. "Software"), to deal in the Software without restriction, including
  9. without limitation the rights to use, copy, modify, merge, publish,
  10. distribute, sublicense, and/or sell copies of the Software, and to
  11. permit persons to whom the Software is furnished to do so, subject to
  12. the following conditions:
  13. The above copyright notice and this permission notice shall be included
  14. in all copies or substantial portions of the Software.
  15. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  16. OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  17. MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  18. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
  19. CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  20. TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  21. SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  22. */
  23. // Some conventions:
  24. // /* */ indicates verbatim text from the HTML 5 specification
  25. // // indicates regular comments
  26. // all flags are in hyphenated form
  27. class HTML5_Tokenizer {
  28. /**
  29. * Points to an InputStream object.
  30. */
  31. protected $stream;
  32. /**
  33. * Tree builder that the tokenizer emits token to.
  34. */
  35. private $tree;
  36. /**
  37. * Current content model we are parsing as.
  38. */
  39. protected $content_model;
  40. /**
  41. * Current token that is being built, but not yet emitted. Also
  42. * is the last token emitted, if applicable.
  43. */
  44. protected $token;
  45. // These are constants describing the content model
  46. const PCDATA = 0;
  47. const RCDATA = 1;
  48. const CDATA = 2;
  49. const PLAINTEXT = 3;
  50. // These are constants describing tokens
  51. // XXX should probably be moved somewhere else, probably the
  52. // HTML5 class.
  53. const DOCTYPE = 0;
  54. const STARTTAG = 1;
  55. const ENDTAG = 2;
  56. const COMMENT = 3;
  57. const CHARACTER = 4;
  58. const SPACECHARACTER = 5;
  59. const EOF = 6;
  60. const PARSEERROR = 7;
  61. // These are constants representing bunches of characters.
  62. const ALPHA = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz';
  63. const UPPER_ALPHA = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ';
  64. const LOWER_ALPHA = 'abcdefghijklmnopqrstuvwxyz';
  65. const DIGIT = '0123456789';
  66. const HEX = '0123456789ABCDEFabcdef';
  67. const WHITESPACE = "\t\n\x0c ";
  68. /**
  69. * @param $data Data to parse
  70. */
  71. public function __construct($data, $builder = null) {
  72. $this->stream = new HTML5_InputStream($data);
  73. if (!$builder) $this->tree = new HTML5_TreeBuilder;
  74. else $this->tree = $builder;
  75. $this->content_model = self::PCDATA;
  76. }
  77. public function parseFragment($context = null) {
  78. $this->tree->setupContext($context);
  79. if ($this->tree->content_model) {
  80. $this->content_model = $this->tree->content_model;
  81. $this->tree->content_model = null;
  82. }
  83. $this->parse();
  84. }
  85. // XXX maybe convert this into an iterator? regardless, this function
  86. // and the save function should go into a Parser facade of some sort
  87. /**
  88. * Performs the actual parsing of the document.
  89. */
  90. public function parse() {
  91. // Current state
  92. $state = 'data';
  93. // This is used to avoid having to have look-behind in the data state.
  94. $lastFourChars = '';
  95. /**
  96. * Escape flag as specified by the HTML5 specification: "used to
  97. * control the behavior of the tokeniser. It is either true or
  98. * false, and initially must be set to the false state."
  99. */
  100. $escape = false;
  101. //echo "\n\n";
  102. while($state !== null) {
  103. /*echo $state . ' ';
  104. switch ($this->content_model) {
  105. case self::PCDATA: echo 'PCDATA'; break;
  106. case self::RCDATA: echo 'RCDATA'; break;
  107. case self::CDATA: echo 'CDATA'; break;
  108. case self::PLAINTEXT: echo 'PLAINTEXT'; break;
  109. }
  110. if ($escape) echo " escape";
  111. echo "\n";*/
  112. switch($state) {
  113. case 'data':
  114. /* Consume the next input character */
  115. $char = $this->stream->char();
  116. $lastFourChars .= $char;
  117. if (strlen($lastFourChars) > 4) $lastFourChars = substr($lastFourChars, -4);
  118. // see below for meaning
  119. $hyp_cond =
  120. !$escape &&
  121. (
  122. $this->content_model === self::RCDATA ||
  123. $this->content_model === self::CDATA
  124. );
  125. $amp_cond =
  126. !$escape &&
  127. (
  128. $this->content_model === self::PCDATA ||
  129. $this->content_model === self::RCDATA
  130. );
  131. $lt_cond =
  132. $this->content_model === self::PCDATA ||
  133. (
  134. (
  135. $this->content_model === self::RCDATA ||
  136. $this->content_model === self::CDATA
  137. ) &&
  138. !$escape
  139. );
  140. $gt_cond =
  141. $escape &&
  142. (
  143. $this->content_model === self::RCDATA ||
  144. $this->content_model === self::CDATA
  145. );
  146. if($char === '&' && $amp_cond) {
  147. /* U+0026 AMPERSAND (&)
  148. When the content model flag is set to one of the PCDATA or RCDATA
  149. states and the escape flag is false: switch to the
  150. character reference data state. Otherwise: treat it as per
  151. the "anything else" entry below. */
  152. $state = 'character reference data';
  153. } elseif(
  154. $char === '-' &&
  155. $hyp_cond &&
  156. $lastFourChars === '<!--'
  157. ) {
  158. /*
  159. U+002D HYPHEN-MINUS (-)
  160. If the content model flag is set to either the RCDATA state or
  161. the CDATA state, and the escape flag is false, and there are at
  162. least three characters before this one in the input stream, and the
  163. last four characters in the input stream, including this one, are
  164. U+003C LESS-THAN SIGN, U+0021 EXCLAMATION MARK, U+002D HYPHEN-MINUS,
  165. and U+002D HYPHEN-MINUS ("<!--"), then set the escape flag to true. */
  166. $escape = true;
  167. /* In any case, emit the input character as a character token. Stay
  168. in the data state. */
  169. $this->emitToken(array(
  170. 'type' => self::CHARACTER,
  171. 'data' => '-'
  172. ));
  173. // We do the "any case" part as part of "anything else".
  174. /* U+003C LESS-THAN SIGN (<) */
  175. } elseif($char === '<' && $lt_cond) {
  176. /* When the content model flag is set to the PCDATA state: switch
  177. to the tag open state.
  178. When the content model flag is set to either the RCDATA state or
  179. the CDATA state and the escape flag is false: switch to the tag
  180. open state.
  181. Otherwise: treat it as per the "anything else" entry below. */
  182. $state = 'tag open';
  183. /* U+003E GREATER-THAN SIGN (>) */
  184. } elseif(
  185. $char === '>' &&
  186. $gt_cond &&
  187. substr($lastFourChars, 1) === '-->'
  188. ) {
  189. /* If the content model flag is set to either the RCDATA state or
  190. the CDATA state, and the escape flag is true, and the last three
  191. characters in the input stream including this one are U+002D
  192. HYPHEN-MINUS, U+002D HYPHEN-MINUS, U+003E GREATER-THAN SIGN ("-->"),
  193. set the escape flag to false. */
  194. $escape = false;
  195. /* In any case, emit the input character as a character token.
  196. Stay in the data state. */
  197. $this->emitToken(array(
  198. 'type' => self::CHARACTER,
  199. 'data' => '>'
  200. ));
  201. // We do the "any case" part as part of "anything else".
  202. } elseif($char === false) {
  203. /* EOF
  204. Emit an end-of-file token. */
  205. $state = null;
  206. $this->tree->emitToken(array(
  207. 'type' => self::EOF
  208. ));
  209. } elseif($char === "\t" || $char === "\n" || $char === "\x0c" || $char === ' ') {
  210. // Directly after emitting a token you switch back to the "data
  211. // state". At that point spaceCharacters are important so they are
  212. // emitted separately.
  213. $chars = $this->stream->charsWhile(self::WHITESPACE);
  214. $this->emitToken(array(
  215. 'type' => self::SPACECHARACTER,
  216. 'data' => $char . $chars
  217. ));
  218. $lastFourChars .= $chars;
  219. if (strlen($lastFourChars) > 4) $lastFourChars = substr($lastFourChars, -4);
  220. } else {
  221. /* Anything else
  222. THIS IS AN OPTIMIZATION: Get as many character that
  223. otherwise would also be treated as a character token and emit it
  224. as a single character token. Stay in the data state. */
  225. $mask = '';
  226. if ($hyp_cond) $mask .= '-';
  227. if ($amp_cond) $mask .= '&';
  228. if ($lt_cond) $mask .= '<';
  229. if ($gt_cond) $mask .= '>';
  230. if ($mask === '') {
  231. $chars = $this->stream->remainingChars();
  232. } else {
  233. $chars = $this->stream->charsUntil($mask);
  234. }
  235. $this->emitToken(array(
  236. 'type' => self::CHARACTER,
  237. 'data' => $char . $chars
  238. ));
  239. $lastFourChars .= $chars;
  240. if (strlen($lastFourChars) > 4) $lastFourChars = substr($lastFourChars, -4);
  241. $state = 'data';
  242. }
  243. break;
  244. case 'character reference data':
  245. /* (This cannot happen if the content model flag
  246. is set to the CDATA state.) */
  247. /* Attempt to consume a character reference, with no
  248. additional allowed character. */
  249. $entity = $this->consumeCharacterReference();
  250. /* If nothing is returned, emit a U+0026 AMPERSAND
  251. character token. Otherwise, emit the character token that
  252. was returned. */
  253. // This is all done when consuming the character reference.
  254. $this->emitToken(array(
  255. 'type' => self::CHARACTER,
  256. 'data' => $entity
  257. ));
  258. /* Finally, switch to the data state. */
  259. $state = 'data';
  260. break;
  261. case 'tag open':
  262. $char = $this->stream->char();
  263. switch($this->content_model) {
  264. case self::RCDATA:
  265. case self::CDATA:
  266. /* Consume the next input character. If it is a
  267. U+002F SOLIDUS (/) character, switch to the close
  268. tag open state. Otherwise, emit a U+003C LESS-THAN
  269. SIGN character token and reconsume the current input
  270. character in the data state. */
  271. // We consumed above.
  272. if($char === '/') {
  273. $state = 'close tag open';
  274. } else {
  275. $this->emitToken(array(
  276. 'type' => self::CHARACTER,
  277. 'data' => '<'
  278. ));
  279. $this->stream->unget();
  280. $state = 'data';
  281. }
  282. break;
  283. case self::PCDATA:
  284. /* If the content model flag is set to the PCDATA state
  285. Consume the next input character: */
  286. // We consumed above.
  287. if($char === '!') {
  288. /* U+0021 EXCLAMATION MARK (!)
  289. Switch to the markup declaration open state. */
  290. $state = 'markup declaration open';
  291. } elseif($char === '/') {
  292. /* U+002F SOLIDUS (/)
  293. Switch to the close tag open state. */
  294. $state = 'close tag open';
  295. } elseif('A' <= $char && $char <= 'Z') {
  296. /* U+0041 LATIN LETTER A through to U+005A LATIN LETTER Z
  297. Create a new start tag token, set its tag name to the lowercase
  298. version of the input character (add 0x0020 to the character's code
  299. point), then switch to the tag name state. (Don't emit the token
  300. yet; further details will be filled in before it is emitted.) */
  301. $this->token = array(
  302. 'name' => strtolower($char),
  303. 'type' => self::STARTTAG,
  304. 'attr' => array()
  305. );
  306. $state = 'tag name';
  307. } elseif('a' <= $char && $char <= 'z') {
  308. /* U+0061 LATIN SMALL LETTER A through to U+007A LATIN SMALL LETTER Z
  309. Create a new start tag token, set its tag name to the input
  310. character, then switch to the tag name state. (Don't emit
  311. the token yet; further details will be filled in before it
  312. is emitted.) */
  313. $this->token = array(
  314. 'name' => $char,
  315. 'type' => self::STARTTAG,
  316. 'attr' => array()
  317. );
  318. $state = 'tag name';
  319. } elseif($char === '>') {
  320. /* U+003E GREATER-THAN SIGN (>)
  321. Parse error. Emit a U+003C LESS-THAN SIGN character token and a
  322. U+003E GREATER-THAN SIGN character token. Switch to the data state. */
  323. $this->emitToken(array(
  324. 'type' => self::PARSEERROR,
  325. 'data' => 'expected-tag-name-but-got-right-bracket'
  326. ));
  327. $this->emitToken(array(
  328. 'type' => self::CHARACTER,
  329. 'data' => '<>'
  330. ));
  331. $state = 'data';
  332. } elseif($char === '?') {
  333. /* U+003F QUESTION MARK (?)
  334. Parse error. Switch to the bogus comment state. */
  335. $this->emitToken(array(
  336. 'type' => self::PARSEERROR,
  337. 'data' => 'expected-tag-name-but-got-question-mark'
  338. ));
  339. $this->token = array(
  340. 'data' => '?',
  341. 'type' => self::COMMENT
  342. );
  343. $state = 'bogus comment';
  344. } else {
  345. /* Anything else
  346. Parse error. Emit a U+003C LESS-THAN SIGN character token and
  347. reconsume the current input character in the data state. */
  348. $this->emitToken(array(
  349. 'type' => self::PARSEERROR,
  350. 'data' => 'expected-tag-name'
  351. ));
  352. $this->emitToken(array(
  353. 'type' => self::CHARACTER,
  354. 'data' => '<'
  355. ));
  356. $state = 'data';
  357. $this->stream->unget();
  358. }
  359. break;
  360. }
  361. break;
  362. case 'close tag open':
  363. if (
  364. $this->content_model === self::RCDATA ||
  365. $this->content_model === self::CDATA
  366. ) {
  367. /* If the content model flag is set to the RCDATA or CDATA
  368. states... */
  369. $name = strtolower($this->stream->charsWhile(self::ALPHA));
  370. $following = $this->stream->char();
  371. $this->stream->unget();
  372. if (
  373. !$this->token ||
  374. $this->token['name'] !== $name ||
  375. $this->token['name'] === $name && !in_array($following, array("\x09", "\x0A", "\x0C", "\x20", "\x3E", "\x2F", false))
  376. ) {
  377. /* if no start tag token has ever been emitted by this instance
  378. of the tokenizer (fragment case), or, if the next few
  379. characters do not match the tag name of the last start tag
  380. token emitted (compared in an ASCII case-insensitive manner),
  381. or if they do but they are not immediately followed by one of
  382. the following characters:
  383. * U+0009 CHARACTER TABULATION
  384. * U+000A LINE FEED (LF)
  385. * U+000C FORM FEED (FF)
  386. * U+0020 SPACE
  387. * U+003E GREATER-THAN SIGN (>)
  388. * U+002F SOLIDUS (/)
  389. * EOF
  390. ...then emit a U+003C LESS-THAN SIGN character token, a
  391. U+002F SOLIDUS character token, and switch to the data
  392. state to process the next input character. */
  393. // XXX: Probably ought to replace in_array with $following === x ||...
  394. // We also need to emit $name now we've consumed that, as we
  395. // know it'll just be emitted as a character token.
  396. $this->emitToken(array(
  397. 'type' => self::CHARACTER,
  398. 'data' => '</' . $name
  399. ));
  400. $state = 'data';
  401. } else {
  402. // This matches what would happen if we actually did the
  403. // otherwise below (but we can't because we've consumed too
  404. // much).
  405. // Start the end tag token with the name we already have.
  406. $this->token = array(
  407. 'name' => $name,
  408. 'type' => self::ENDTAG
  409. );
  410. // Change to tag name state.
  411. $state = 'tag name';
  412. }
  413. } elseif ($this->content_model === self::PCDATA) {
  414. /* Otherwise, if the content model flag is set to the PCDATA
  415. state [...]: */
  416. $char = $this->stream->char();
  417. if ('A' <= $char && $char <= 'Z') {
  418. /* U+0041 LATIN LETTER A through to U+005A LATIN LETTER Z
  419. Create a new end tag token, set its tag name to the lowercase version
  420. of the input character (add 0x0020 to the character's code point), then
  421. switch to the tag name state. (Don't emit the token yet; further details
  422. will be filled in before it is emitted.) */
  423. $this->token = array(
  424. 'name' => strtolower($char),
  425. 'type' => self::ENDTAG
  426. );
  427. $state = 'tag name';
  428. } elseif ('a' <= $char && $char <= 'z') {
  429. /* U+0061 LATIN SMALL LETTER A through to U+007A LATIN SMALL LETTER Z
  430. Create a new end tag token, set its tag name to the
  431. input character, then switch to the tag name state.
  432. (Don't emit the token yet; further details will be
  433. filled in before it is emitted.) */
  434. $this->token = array(
  435. 'name' => $char,
  436. 'type' => self::ENDTAG
  437. );
  438. $state = 'tag name';
  439. } elseif($char === '>') {
  440. /* U+003E GREATER-THAN SIGN (>)
  441. Parse error. Switch to the data state. */
  442. $this->emitToken(array(
  443. 'type' => self::PARSEERROR,
  444. 'data' => 'expected-closing-tag-but-got-right-bracket'
  445. ));
  446. $state = 'data';
  447. } elseif($char === false) {
  448. /* EOF
  449. Parse error. Emit a U+003C LESS-THAN SIGN character token and a U+002F
  450. SOLIDUS character token. Reconsume the EOF character in the data state. */
  451. $this->emitToken(array(
  452. 'type' => self::PARSEERROR,
  453. 'data' => 'expected-closing-tag-but-got-eof'
  454. ));
  455. $this->emitToken(array(
  456. 'type' => self::CHARACTER,
  457. 'data' => '</'
  458. ));
  459. $this->stream->unget();
  460. $state = 'data';
  461. } else {
  462. /* Parse error. Switch to the bogus comment state. */
  463. $this->emitToken(array(
  464. 'type' => self::PARSEERROR,
  465. 'data' => 'expected-closing-tag-but-got-char'
  466. ));
  467. $this->token = array(
  468. 'data' => $char,
  469. 'type' => self::COMMENT
  470. );
  471. $state = 'bogus comment';
  472. }
  473. }
  474. break;
  475. case 'tag name':
  476. /* Consume the next input character: */
  477. $char = $this->stream->char();
  478. if($char === "\t" || $char === "\n" || $char === "\x0c" || $char === ' ') {
  479. /* U+0009 CHARACTER TABULATION
  480. U+000A LINE FEED (LF)
  481. U+000C FORM FEED (FF)
  482. U+0020 SPACE
  483. Switch to the before attribute name state. */
  484. $state = 'before attribute name';
  485. } elseif($char === '/') {
  486. /* U+002F SOLIDUS (/)
  487. Switch to the self-closing start tag state. */
  488. $state = 'self-closing start tag';
  489. } elseif($char === '>') {
  490. /* U+003E GREATER-THAN SIGN (>)
  491. Emit the current tag token. Switch to the data state. */
  492. $this->emitToken($this->token);
  493. $state = 'data';
  494. } elseif('A' <= $char && $char <= 'Z') {
  495. /* U+0041 LATIN CAPITAL LETTER A through to U+005A LATIN CAPITAL LETTER Z
  496. Append the lowercase version of the current input
  497. character (add 0x0020 to the character's code point) to
  498. the current tag token's tag name. Stay in the tag name state. */
  499. $chars = $this->stream->charsWhile(self::UPPER_ALPHA);
  500. $this->token['name'] .= strtolower($char . $chars);
  501. $state = 'tag name';
  502. } elseif($char === false) {
  503. /* EOF
  504. Parse error. Reconsume the EOF character in the data state. */
  505. $this->emitToken(array(
  506. 'type' => self::PARSEERROR,
  507. 'data' => 'eof-in-tag-name'
  508. ));
  509. $this->stream->unget();
  510. $state = 'data';
  511. } else {
  512. /* Anything else
  513. Append the current input character to the current tag token's tag name.
  514. Stay in the tag name state. */
  515. $chars = $this->stream->charsUntil("\t\n\x0C />" . self::UPPER_ALPHA);
  516. $this->token['name'] .= $char . $chars;
  517. $state = 'tag name';
  518. }
  519. break;
  520. case 'before attribute name':
  521. /* Consume the next input character: */
  522. $char = $this->stream->char();
  523. // this conditional is optimized, check bottom
  524. if($char === "\t" || $char === "\n" || $char === "\x0c" || $char === ' ') {
  525. /* U+0009 CHARACTER TABULATION
  526. U+000A LINE FEED (LF)
  527. U+000C FORM FEED (FF)
  528. U+0020 SPACE
  529. Stay in the before attribute name state. */
  530. $state = 'before attribute name';
  531. } elseif($char === '/') {
  532. /* U+002F SOLIDUS (/)
  533. Switch to the self-closing start tag state. */
  534. $state = 'self-closing start tag';
  535. } elseif($char === '>') {
  536. /* U+003E GREATER-THAN SIGN (>)
  537. Emit the current tag token. Switch to the data state. */
  538. $this->emitToken($this->token);
  539. $state = 'data';
  540. } elseif('A' <= $char && $char <= 'Z') {
  541. /* U+0041 LATIN CAPITAL LETTER A through to U+005A LATIN CAPITAL LETTER Z
  542. Start a new attribute in the current tag token. Set that
  543. attribute's name to the lowercase version of the current
  544. input character (add 0x0020 to the character's code
  545. point), and its value to the empty string. Switch to the
  546. attribute name state.*/
  547. $this->token['attr'][] = array(
  548. 'name' => strtolower($char),
  549. 'value' => ''
  550. );
  551. $state = 'attribute name';
  552. } elseif($char === false) {
  553. /* EOF
  554. Parse error. Reconsume the EOF character in the data state. */
  555. $this->emitToken(array(
  556. 'type' => self::PARSEERROR,
  557. 'data' => 'expected-attribute-name-but-got-eof'
  558. ));
  559. $this->stream->unget();
  560. $state = 'data';
  561. } else {
  562. /* U+0022 QUOTATION MARK (")
  563. U+0027 APOSTROPHE (')
  564. U+003C LESS-THAN SIGN (<)
  565. U+003D EQUALS SIGN (=)
  566. Parse error. Treat it as per the "anything else" entry
  567. below. */
  568. if($char === '"' || $char === "'" || $char === '<' || $char === '=') {
  569. $this->emitToken(array(
  570. 'type' => self::PARSEERROR,
  571. 'data' => 'invalid-character-in-attribute-name'
  572. ));
  573. }
  574. /* Anything else
  575. Start a new attribute in the current tag token. Set that attribute's
  576. name to the current input character, and its value to the empty string.
  577. Switch to the attribute name state. */
  578. $this->token['attr'][] = array(
  579. 'name' => $char,
  580. 'value' => ''
  581. );
  582. $state = 'attribute name';
  583. }
  584. break;
  585. case 'attribute name':
  586. // Consume the next input character:
  587. $char = $this->stream->char();
  588. // this conditional is optimized, check bottom
  589. if($char === "\t" || $char === "\n" || $char === "\x0c" || $char === ' ') {
  590. /* U+0009 CHARACTER TABULATION
  591. U+000A LINE FEED (LF)
  592. U+000C FORM FEED (FF)
  593. U+0020 SPACE
  594. Switch to the after attribute name state. */
  595. $state = 'after attribute name';
  596. } elseif($char === '/') {
  597. /* U+002F SOLIDUS (/)
  598. Switch to the self-closing start tag state. */
  599. $state = 'self-closing start tag';
  600. } elseif($char === '=') {
  601. /* U+003D EQUALS SIGN (=)
  602. Switch to the before attribute value state. */
  603. $state = 'before attribute value';
  604. } elseif($char === '>') {
  605. /* U+003E GREATER-THAN SIGN (>)
  606. Emit the current tag token. Switch to the data state. */
  607. $this->emitToken($this->token);
  608. $state = 'data';
  609. } elseif('A' <= $char && $char <= 'Z') {
  610. /* U+0041 LATIN CAPITAL LETTER A through to U+005A LATIN CAPITAL LETTER Z
  611. Append the lowercase version of the current input
  612. character (add 0x0020 to the character's code point) to
  613. the current attribute's name. Stay in the attribute name
  614. state. */
  615. $chars = $this->stream->charsWhile(self::UPPER_ALPHA);
  616. $last = count($this->token['attr']) - 1;
  617. $this->token['attr'][$last]['name'] .= strtolower($char . $chars);
  618. $state = 'attribute name';
  619. } elseif($char === false) {
  620. /* EOF
  621. Parse error. Reconsume the EOF character in the data state. */
  622. $this->emitToken(array(
  623. 'type' => self::PARSEERROR,
  624. 'data' => 'eof-in-attribute-name'
  625. ));
  626. $this->stream->unget();
  627. $state = 'data';
  628. } else {
  629. /* U+0022 QUOTATION MARK (")
  630. U+0027 APOSTROPHE (')
  631. U+003C LESS-THAN SIGN (<)
  632. Parse error. Treat it as per the "anything else"
  633. entry below. */
  634. if($char === '"' || $char === "'" || $char === '<') {
  635. $this->emitToken(array(
  636. 'type' => self::PARSEERROR,
  637. 'data' => 'invalid-character-in-attribute-name'
  638. ));
  639. }
  640. /* Anything else
  641. Append the current input character to the current attribute's name.
  642. Stay in the attribute name state. */
  643. $chars = $this->stream->charsUntil("\t\n\x0C /=>\"'" . self::UPPER_ALPHA);
  644. $last = count($this->token['attr']) - 1;
  645. $this->token['attr'][$last]['name'] .= $char . $chars;
  646. $state = 'attribute name';
  647. }
  648. /* When the user agent leaves the attribute name state
  649. (and before emitting the tag token, if appropriate), the
  650. complete attribute's name must be compared to the other
  651. attributes on the same token; if there is already an
  652. attribute on the token with the exact same name, then this
  653. is a parse error and the new attribute must be dropped, along
  654. with the value that gets associated with it (if any). */
  655. // this might be implemented in the emitToken method
  656. break;
  657. case 'after attribute name':
  658. // Consume the next input character:
  659. $char = $this->stream->char();
  660. // this is an optimized conditional, check the bottom
  661. if($char === "\t" || $char === "\n" || $char === "\x0c" || $char === ' ') {
  662. /* U+0009 CHARACTER TABULATION
  663. U+000A LINE FEED (LF)
  664. U+000C FORM FEED (FF)
  665. U+0020 SPACE
  666. Stay in the after attribute name state. */
  667. $state = 'after attribute name';
  668. } elseif($char === '/') {
  669. /* U+002F SOLIDUS (/)
  670. Switch to the self-closing start tag state. */
  671. $state = 'self-closing start tag';
  672. } elseif($char === '=') {
  673. /* U+003D EQUALS SIGN (=)
  674. Switch to the before attribute value state. */
  675. $state = 'before attribute value';
  676. } elseif($char === '>') {
  677. /* U+003E GREATER-THAN SIGN (>)
  678. Emit the current tag token. Switch to the data state. */
  679. $this->emitToken($this->token);
  680. $state = 'data';
  681. } elseif('A' <= $char && $char <= 'Z') {
  682. /* U+0041 LATIN CAPITAL LETTER A through to U+005A LATIN CAPITAL LETTER Z
  683. Start a new attribute in the current tag token. Set that
  684. attribute's name to the lowercase version of the current
  685. input character (add 0x0020 to the character's code
  686. point), and its value to the empty string. Switch to the
  687. attribute name state. */
  688. $this->token['attr'][] = array(
  689. 'name' => strtolower($char),
  690. 'value' => ''
  691. );
  692. $state = 'attribute name';
  693. } elseif($char === false) {
  694. /* EOF
  695. Parse error. Reconsume the EOF character in the data state. */
  696. $this->emitToken(array(
  697. 'type' => self::PARSEERROR,
  698. 'data' => 'expected-end-of-tag-but-got-eof'
  699. ));
  700. $this->stream->unget();
  701. $state = 'data';
  702. } else {
  703. /* U+0022 QUOTATION MARK (")
  704. U+0027 APOSTROPHE (')
  705. U+003C LESS-THAN SIGN(<)
  706. Parse error. Treat it as per the "anything else"
  707. entry below. */
  708. if($char === '"' || $char === "'" || $char === "<") {
  709. $this->emitToken(array(
  710. 'type' => self::PARSEERROR,
  711. 'data' => 'invalid-character-after-attribute-name'
  712. ));
  713. }
  714. /* Anything else
  715. Start a new attribute in the current tag token. Set that attribute's
  716. name to the current input character, and its value to the empty string.
  717. Switch to the attribute name state. */
  718. $this->token['attr'][] = array(
  719. 'name' => $char,
  720. 'value' => ''
  721. );
  722. $state = 'attribute name';
  723. }
  724. break;
  725. case 'before attribute value':
  726. // Consume the next input character:
  727. $char = $this->stream->char();
  728. // this is an optimized conditional
  729. if($char === "\t" || $char === "\n" || $char === "\x0c" || $char === ' ') {
  730. /* U+0009 CHARACTER TABULATION
  731. U+000A LINE FEED (LF)
  732. U+000C FORM FEED (FF)
  733. U+0020 SPACE
  734. Stay in the before attribute value state. */
  735. $state = 'before attribute value';
  736. } elseif($char === '"') {
  737. /* U+0022 QUOTATION MARK (")
  738. Switch to the attribute value (double-quoted) state. */
  739. $state = 'attribute value (double-quoted)';
  740. } elseif($char === '&') {
  741. /* U+0026 AMPERSAND (&)
  742. Switch to the attribute value (unquoted) state and reconsume
  743. this input character. */
  744. $this->stream->unget();
  745. $state = 'attribute value (unquoted)';
  746. } elseif($char === '\'') {
  747. /* U+0027 APOSTROPHE (')
  748. Switch to the attribute value (single-quoted) state. */
  749. $state = 'attribute value (single-quoted)';
  750. } elseif($char === '>') {
  751. /* U+003E GREATER-THAN SIGN (>)
  752. Parse error. Emit the current tag token. Switch to the data state. */
  753. $this->emitToken(array(
  754. 'type' => self::PARSEERROR,
  755. 'data' => 'expected-attribute-value-but-got-right-bracket'
  756. ));
  757. $this->emitToken($this->token);
  758. $state = 'data';
  759. } elseif($char === false) {
  760. /* EOF
  761. Parse error. Reconsume the EOF character in the data state. */
  762. $this->emitToken(array(
  763. 'type' => self::PARSEERROR,
  764. 'data' => 'expected-attribute-value-but-got-eof'
  765. ));
  766. $this->stream->unget();
  767. $state = 'data';
  768. } else {
  769. /* U+003D EQUALS SIGN (=)
  770. * U+003C LESS-THAN SIGN (<)
  771. Parse error. Treat it as per the "anything else" entry below. */
  772. if($char === '=' || $char === '<') {
  773. $this->emitToken(array(
  774. 'type' => self::PARSEERROR,
  775. 'data' => 'equals-in-unquoted-attribute-value'
  776. ));
  777. }
  778. /* Anything else
  779. Append the current input character to the current attribute's value.
  780. Switch to the attribute value (unquoted) state. */
  781. $last = count($this->token['attr']) - 1;
  782. $this->token['attr'][$last]['value'] .= $char;
  783. $state = 'attribute value (unquoted)';
  784. }
  785. break;
  786. case 'attribute value (double-quoted)':
  787. // Consume the next input character:
  788. $char = $this->stream->char();
  789. if($char === '"') {
  790. /* U+0022 QUOTATION MARK (")
  791. Switch to the after attribute value (quoted) state. */
  792. $state = 'after attribute value (quoted)';
  793. } elseif($char === '&') {
  794. /* U+0026 AMPERSAND (&)
  795. Switch to the character reference in attribute value
  796. state, with the additional allowed character
  797. being U+0022 QUOTATION MARK ("). */
  798. $this->characterReferenceInAttributeValue('"');
  799. } elseif($char === false) {
  800. /* EOF
  801. Parse error. Reconsume the EOF character in the data state. */
  802. $this->emitToken(array(
  803. 'type' => self::PARSEERROR,
  804. 'data' => 'eof-in-attribute-value-double-quote'
  805. ));
  806. $this->stream->unget();
  807. $state = 'data';
  808. } else {
  809. /* Anything else
  810. Append the current input character to the current attribute's value.
  811. Stay in the attribute value (double-quoted) state. */
  812. $chars = $this->stream->charsUntil('"&');
  813. $last = count($this->token['attr']) - 1;
  814. $this->token['attr'][$last]['value'] .= $char . $chars;
  815. $state = 'attribute value (double-quoted)';
  816. }
  817. break;
  818. case 'attribute value (single-quoted)':
  819. // Consume the next input character:
  820. $char = $this->stream->char();
  821. if($char === "'") {
  822. /* U+0022 QUOTATION MARK (')
  823. Switch to the after attribute value state. */
  824. $state = 'after attribute value (quoted)';
  825. } elseif($char === '&') {
  826. /* U+0026 AMPERSAND (&)
  827. Switch to the entity in attribute value state. */
  828. $this->characterReferenceInAttributeValue("'");
  829. } elseif($char === false) {
  830. /* EOF
  831. Parse error. Reconsume the EOF character in the data state. */
  832. $this->emitToken(array(
  833. 'type' => self::PARSEERROR,
  834. 'data' => 'eof-in-attribute-value-single-quote'
  835. ));
  836. $this->stream->unget();
  837. $state = 'data';
  838. } else {
  839. /* Anything else
  840. Append the current input character to the current attribute's value.
  841. Stay in the attribute value (single-quoted) state. */
  842. $chars = $this->stream->charsUntil("'&");
  843. $last = count($this->token['attr']) - 1;
  844. $this->token['attr'][$last]['value'] .= $char . $chars;
  845. $state = 'attribute value (single-quoted)';
  846. }
  847. break;
  848. case 'attribute value (unquoted)':
  849. // Consume the next input character:
  850. $char = $this->stream->char();
  851. if($char === "\t" || $char === "\n" || $char === "\x0c" || $char === ' ') {
  852. /* U+0009 CHARACTER TABULATION
  853. U+000A LINE FEED (LF)
  854. U+000C FORM FEED (FF)
  855. U+0020 SPACE
  856. Switch to the before attribute name state. */
  857. $state = 'before attribute name';
  858. } elseif($char === '&') {
  859. /* U+0026 AMPERSAND (&)
  860. Switch to the entity in attribute value state, with the
  861. additional allowed character being U+003E
  862. GREATER-THAN SIGN (>). */
  863. $this->characterReferenceInAttributeValue('>');
  864. } elseif($char === '>') {
  865. /* U+003E GREATER-THAN SIGN (>)
  866. Emit the current tag token. Switch to the data state. */
  867. $this->emitToken($this->token);
  868. $state = 'data';
  869. } elseif ($char === false) {
  870. /* EOF
  871. Parse error. Reconsume the EOF character in the data state. */
  872. $this->emitToken(array(
  873. 'type' => self::PARSEERROR,
  874. 'data' => 'eof-in-attribute-value-no-quotes'
  875. ));
  876. $this->stream->unget();
  877. $state = 'data';
  878. } else {
  879. /* U+0022 QUOTATION MARK (")
  880. U+0027 APOSTROPHE (')
  881. U+003C LESS-THAN SIGN (<)
  882. U+003D EQUALS SIGN (=)
  883. Parse error. Treat it as per the "anything else"
  884. entry below. */
  885. if($char === '"' || $char === "'" || $char === '=' || $char == '<') {
  886. $this->emitToken(array(
  887. 'type' => self::PARSEERROR,
  888. 'data' => 'unexpected-character-in-unquoted-attribute-value'
  889. ));
  890. }
  891. /* Anything else
  892. Append the current input character to the current attribute's value.
  893. Stay in the attribute value (unquoted) state. */
  894. $chars = $this->stream->charsUntil("\t\n\x0c &>\"'=");
  895. $last = count($this->token['attr']) - 1;
  896. $this->token['attr'][$last]['value'] .= $char . $chars;
  897. $state = 'attribute value (unquoted)';
  898. }
  899. break;
  900. case 'after attribute value (quoted)':
  901. /* Consume the next input character: */
  902. $char = $this->stream->char();
  903. if($char === "\t" || $char === "\n" || $char === "\x0c" || $char === ' ') {
  904. /* U+0009 CHARACTER TABULATION
  905. U+000A LINE FEED (LF)
  906. U+000C FORM FEED (FF)
  907. U+0020 SPACE
  908. Switch to the before attribute name state. */
  909. $state = 'before attribute name';
  910. } elseif ($char === '/') {
  911. /* U+002F SOLIDUS (/)
  912. Switch to the self-closing start tag state. */
  913. $state = 'self-closing start tag';
  914. } elseif ($char === '>') {
  915. /* U+003E GREATER-THAN SIGN (>)
  916. Emit the current tag token. Switch to the data state. */
  917. $this->emitToken($this->token);
  918. $state = 'data';
  919. } elseif ($char === false) {
  920. /* EOF
  921. Parse error. Reconsume the EOF character in the data state. */
  922. $this->emitToken(array(
  923. 'type' => self::PARSEERROR,
  924. 'data' => 'unexpected-EOF-after-attribute-value'
  925. ));
  926. $this->stream->unget();
  927. $state = 'data';
  928. } else {
  929. /* Anything else
  930. Parse error. Reconsume the character in the before attribute
  931. name state. */
  932. $this->emitToken(array(
  933. 'type' => self::PARSEERROR,
  934. 'data' => 'unexpected-character-after-attribute-value'
  935. ));
  936. $this->stream->unget();
  937. $state = 'before attribute name';
  938. }
  939. break;
  940. case 'self-closing start tag':
  941. /* Consume the next input character: */
  942. $char = $this->stream->char();
  943. if ($char === '>') {
  944. /* U+003E GREATER-THAN SIGN (>)
  945. Set the self-closing flag of the current tag token.
  946. Emit the current tag token. Switch to the data state. */
  947. // not sure if this is the name we want
  948. $this->token['self-closing'] = true;
  949. $this->emitToken($this->token);
  950. $state = 'data';
  951. } elseif ($char === false) {
  952. /* EOF
  953. Parse error. Reconsume the EOF character in the data state. */
  954. $this->emitToken(array(
  955. 'type' => self::PARSEERROR,
  956. 'data' => 'unexpected-eof-after-self-closing'
  957. ));
  958. $this->stream->unget();
  959. $state = 'data';
  960. } else {
  961. /* Anything else
  962. Parse error. Reconsume the character in the before attribute name state. */
  963. $this->emitToken(array(
  964. 'type' => self::PARSEERROR,
  965. 'data' => 'unexpected-character-after-self-closing'
  966. ));
  967. $this->stream->unget();
  968. $state = 'before attribute name';
  969. }
  970. break;
  971. case 'bogus comment':
  972. /* (This can only happen if the content model flag is set to the PCDATA state.) */
  973. /* Consume every character up to the first U+003E GREATER-THAN SIGN
  974. character (>) or the end of the file (EOF), whichever comes first. Emit
  975. a comment token whose data is the concatenation of all the characters
  976. starting from and including the character that caused the state machine
  977. to switch into the bogus comment state, up to and including the last
  978. consumed character before the U+003E character, if any, or up to the
  979. end of the file otherwise. (If the comment was started by the end of
  980. the file (EOF), the token is empty.) */
  981. $this->token['data'] .= (string) $this->stream->charsUntil('>');
  982. $this->stream->char();
  983. $this->emitToken($this->token);
  984. /* Switch to the data state. */
  985. $state = 'data';
  986. break;
  987. case 'markup declaration open':
  988. // Consume for below
  989. $hyphens = $this->stream->charsWhile('-', 2);
  990. if ($hyphens === '-') {
  991. $this->stream->unget();
  992. }
  993. if ($hyphens !== '--') {
  994. $alpha = $this->stream->charsWhile(self::ALPHA, 7);
  995. }
  996. /* If the next two characters are both U+002D HYPHEN-MINUS (-)
  997. characters, consume those two characters, create a comment token whose
  998. data is the empty string, and switch to the comment state. */
  999. if($hyphens === '--') {
  1000. $state = 'comment start';
  1001. $this->token = array(
  1002. 'data' => '',
  1003. 'type' => self::COMMENT
  1004. );
  1005. /* Otherwise if the next seven characters are a case-insensitive match
  1006. for the word "DOCTYPE", then consume those characters and switch to the
  1007. DOCTYPE state. */
  1008. } elseif(strtoupper($alpha) === 'DOCTYPE') {
  1009. $state = 'DOCTYPE';
  1010. // XXX not implemented
  1011. /* Otherwise, if the insertion mode is "in foreign content"
  1012. and the current node is not an element in the HTML namespace
  1013. and the next seven characters are an ASCII case-sensitive
  1014. match for the string "[CDATA[" (the five uppercase letters
  1015. "CDATA" with a U+005B LEFT SQUARE BRACKET character before
  1016. and after), then consume those characters and switch to the
  1017. CDATA section state (which is unrelated to the content model
  1018. flag's CDATA state). */
  1019. /* Otherwise, is is a parse error. Switch to the bogus comment state.
  1020. The next character that is consumed, if any, is the first character
  1021. that will be in the comment. */
  1022. } else {
  1023. $this->emitToken(array(
  1024. 'type' => self::PARSEERROR,
  1025. 'data' => 'expected-dashes-or-doctype'
  1026. ));
  1027. $this->token = array(
  1028. 'data' => (string) $alpha,
  1029. 'type' => self::COMMENT
  1030. );
  1031. $state = 'bogus comment';
  1032. }
  1033. break;
  1034. case 'comment start':
  1035. /* Consume the next input character: */
  1036. $char = $this->stream->char();
  1037. if ($char === '-') {
  1038. /* U+002D HYPHEN-MINUS (-)
  1039. Switch to the comment start dash state. */
  1040. $state = 'comment start dash';
  1041. } elseif ($char === '>') {
  1042. /* U+003E GREATER-THAN SIGN (>)
  1043. Parse error. Emit the comment token. Switch to the
  1044. data state. */
  1045. $this->emitToken(array(
  1046. 'type' => self::PARSEERROR,
  1047. 'data' => 'incorrect-comment'
  1048. ));
  1049. $this->emitToken($this->token);
  1050. $state = 'data';
  1051. } elseif ($char === false) {
  1052. /* EOF
  1053. Parse error. Emit the comment token. Reconsume the
  1054. EOF character in the data state. */
  1055. $this->emitToken(array(
  1056. 'type' => self::PARSEERROR,
  1057. 'data' => 'eof-in-comment'
  1058. ));
  1059. $this->emitToken($this->token);
  1060. $this->stream->unget();
  1061. $state = 'data';
  1062. } else {
  1063. /* Anything else
  1064. Append the input character to the comment token's
  1065. data. Switch to the comment state. */
  1066. $this->token['data'] .= $char;
  1067. $state = 'comment';
  1068. }
  1069. break;
  1070. case 'comment start dash':
  1071. /* Consume the next input character: */
  1072. $char = $this->stream->char();
  1073. if ($char === '-') {
  1074. /* U+002D HYPHEN-MINUS (-)
  1075. Switch to the comment end state */
  1076. $state = 'comment end';
  1077. } elseif ($char === '>') {
  1078. /* U+003E GREATER-THAN SIGN (>)
  1079. Parse error. Emit the comment token. Switch to the
  1080. data state. */
  1081. $this->emitToken(array(
  1082. 'type' => self::PARSEERROR,
  1083. 'data' => 'incorrect-comment'
  1084. ));
  1085. $this->emitToken($this->token);
  1086. $state = 'data';
  1087. } elseif ($char === false) {
  1088. /* Parse error. Emit the comment token. Reconsume the
  1089. EOF character in the data state. */
  1090. $this->emitToken(array(
  1091. 'type' => self::PARSEERROR,
  1092. 'data' => 'eof-in-comment'
  1093. ));
  1094. $this->emitToken($this->token);
  1095. $this->stream->unget();
  1096. $state = 'data';
  1097. } else {
  1098. $this->token['data'] .= '-' . $char;
  1099. $state = 'comment';
  1100. }
  1101. break;
  1102. case 'comment':
  1103. /* Consume the next input character: */
  1104. $char = $this->stream->char();
  1105. if($char === '-') {
  1106. /* U+002D HYPHEN-MINUS (-)
  1107. Switch to the comment end dash state */
  1108. $state = 'comment end dash';
  1109. } elseif($char === false) {
  1110. /* EOF
  1111. Parse error. Emit the comment token. Reconsume the EOF character
  1112. in the data state. */
  1113. $this->emitToken(array(
  1114. 'type' => self::PARSEERROR,
  1115. 'data' => 'eof-in-comment'
  1116. ));
  1117. $this->emitToken($this->token);
  1118. $this->stream->unget();
  1119. $state = 'data';
  1120. } else {
  1121. /* Anything else
  1122. Append the input character to the comment token's data. Stay in
  1123. the comment state. */
  1124. $chars = $this->stream->charsUntil('-');
  1125. $this->token['data'] .= $char . $chars;
  1126. }
  1127. break;
  1128. case 'comment end dash':
  1129. /* Consume the next input character: */
  1130. $char = $this->stream->char();
  1131. if($char === '-') {
  1132. /* U+002D HYPHEN-MINUS (-)
  1133. Switch to the comment end state */
  1134. $state = 'comment end';
  1135. } elseif($char === false) {
  1136. /* EOF
  1137. Parse error. Emit the comment token. Reconsume the EOF character
  1138. in the data state. */
  1139. $this->emitToken(array(
  1140. 'type' => self::PARSEERROR,
  1141. 'data' => 'eof-in-comment-end-dash'
  1142. ));
  1143. $this->emitToken($this->token);
  1144. $this->stream->unget();
  1145. $state = 'data';
  1146. } else {
  1147. /* Anything else
  1148. Append a U+002D HYPHEN-MINUS (-) character and the input
  1149. character to the comment token's data. Switch to the comment state. */
  1150. $this->token['data'] .= '-'.$char;
  1151. $state = 'comment';
  1152. }
  1153. break;
  1154. case 'comment end':
  1155. /* Consume the next input character: */
  1156. $char = $this->stream->char();
  1157. if($char === '>') {
  1158. /* U+003E GREATER-THAN SIGN (>)
  1159. Emit the comment token. Switch to the data state. */
  1160. $this->emitToken($this->token);
  1161. $state = 'data';
  1162. } elseif($char === '-') {
  1163. /* U+002D HYPHEN-MINUS (-)
  1164. Parse error. Append a U+002D HYPHEN-MINUS (-) character
  1165. to the comment token's data. Stay in the comment end
  1166. state. */
  1167. $this->emitToken(array(
  1168. 'type' => self::PARSEERROR,
  1169. 'data' => 'unexpected-dash-after-double-dash-in-comment'
  1170. ));
  1171. $this->token['data'] .= '-';
  1172. } elseif($char === "\t" || $char === "\n" || $char === "\x0a" || $char === ' ') {
  1173. $this->emitToken(array(
  1174. 'type' => self::PARSEERROR,
  1175. 'data' => 'unexpected-space-after-double-dash-in-comment'
  1176. ));
  1177. $this->token['data'] .= '--' . $char;
  1178. $state = 'comment end space';
  1179. } elseif($char === '!') {
  1180. $this->emitToken(array(
  1181. 'type' => self::PARSEERROR,
  1182. 'data' => 'unexpected-bang-after-double-dash-in-comment'
  1183. ));
  1184. $state = 'comment end bang';
  1185. } elseif($char === false) {
  1186. /* EOF
  1187. Parse error. Emit the comment token. Reconsume the
  1188. EOF character in the data state. */
  1189. $this->emitToken(array(
  1190. 'type' => self::PARSEERROR,
  1191. 'data' => 'eof-in-comment-double-dash'
  1192. ));
  1193. $this->emitToken($this->token);
  1194. $this->stream->unget();
  1195. $state = 'data';
  1196. } else {
  1197. /* Anything else
  1198. Parse error. Append two U+002D HYPHEN-MINUS (-)
  1199. characters and the input character to the comment token's
  1200. data. Switch to the comment state. */
  1201. $this->emitToken(array(
  1202. 'type' => self::PARSEERROR,
  1203. 'data' => 'unexpected-char-in-comment'
  1204. ));
  1205. $this->token['data'] .= '--'.$char;
  1206. $state = 'comment';
  1207. }
  1208. break;
  1209. case 'comment end bang':
  1210. $char = $this->stream->char();
  1211. if ($char === '>') {
  1212. $this->emitToken($this->token);
  1213. $state = 'data';
  1214. } elseif ($char === "-") {
  1215. $this->token['data'] .= '--!';
  1216. $state = 'comment end dash';
  1217. } elseif ($char === false) {
  1218. $this->emitToken(array(
  1219. 'type' => self::PARSEERROR,
  1220. 'data' => 'eof-in-comment-end-bang'
  1221. ));
  1222. $this->emitToken($this->token);
  1223. $this->stream->unget();
  1224. $state = 'data';
  1225. } else {
  1226. $this->token['data'] .= '--!' . $char;
  1227. $state = 'comment';
  1228. }
  1229. break;
  1230. case 'comment end space':
  1231. $char = $this->stream->char();
  1232. if ($char === '>') {
  1233. $this->emitToken($this->token);
  1234. $state = 'data';
  1235. } elseif ($char === '-') {
  1236. $state = 'comment end dash';
  1237. } elseif ($char === "\t" || $char === "\n" || $char === "\x0c" || $char === ' ') {
  1238. $this->token['data'] .= $char;
  1239. } elseif ($char === false) {
  1240. $this->emitToken(array(
  1241. 'type' => self::PARSEERROR,
  1242. 'data' => 'unexpected-eof-in-comment-end-space',
  1243. ));
  1244. $this->emitToken($this->token);
  1245. $this->stream->unget();
  1246. $state = 'data';
  1247. } else {
  1248. $this->token['data'] .= $char;
  1249. $state = 'comment';
  1250. }
  1251. break;
  1252. case 'DOCTYPE':
  1253. /* Consume the next input character: */
  1254. $char = $this->stream->char();
  1255. if($char === "\t" || $char === "\n" || $char === "\x0c" || $char === ' ') {
  1256. /* U+0009 CHARACTER TABULATION
  1257. U+000A LINE FEED (LF)
  1258. U+000C FORM FEED (FF)
  1259. U+0020 SPACE
  1260. Switch to the before DOCTYPE name state. */
  1261. $state = 'before DOCTYPE name';
  1262. } elseif($char === false) {
  1263. /* EOF
  1264. Parse error. Create a new DOCTYPE token. Set its
  1265. force-quirks flag to on. Emit the token. Reconsume the
  1266. EOF character in the data state. */
  1267. $this->emitToken(array(
  1268. 'type' => self::PARSEERROR,
  1269. 'data' => 'need-space-after-doctype-but-got-eof'
  1270. ));
  1271. $this->emitToken(array(
  1272. 'name' => '',
  1273. 'type' => self::DOCTYPE,
  1274. 'force-quirks' => true,
  1275. 'error' => true
  1276. ));
  1277. $this->stream->unget();
  1278. $state = 'data';
  1279. } else {
  1280. /* Anything else
  1281. Parse error. Reconsume the current character in the
  1282. before DOCTYPE name state. */
  1283. $this->emitToken(array(
  1284. 'type' => self::PARSEERROR,
  1285. 'data' => 'need-space-after-doctype'
  1286. ));
  1287. $this->stream->unget();
  1288. $state = 'before DOCTYPE name';
  1289. }
  1290. break;
  1291. case 'before DOCTYPE name':
  1292. /* Consume the next input character: */
  1293. $char = $this->stream->char();
  1294. if($char === "\t" || $char === "\n" || $char === "\x0c" || $char === ' ') {
  1295. /* U+0009 CHARACTER TABULATION
  1296. U+000A LINE FEED (LF)
  1297. U+000C FORM FEED (FF)
  1298. U+0020 SPACE
  1299. Stay in the before DOCTYPE name state. */
  1300. } elseif($char === '>') {
  1301. /* U+003E GREATER-THAN SIGN (>)
  1302. Parse error. Create a new DOCTYPE token. Set its
  1303. force-quirks flag to on. Emit the token. Switch to the
  1304. data state. */
  1305. $this->emitToken(array(
  1306. 'type' => self::PARSEERROR,
  1307. 'data' => 'expected-doctype-name-but-got-right-bracket'
  1308. ));
  1309. $this->emitToken(array(
  1310. 'name' => '',
  1311. 'type' => self::DOCTYPE,
  1312. 'force-quirks' => true,
  1313. 'error' => true
  1314. ));
  1315. $state = 'data';
  1316. } elseif('A' <= $char && $char <= 'Z') {
  1317. /* U+0041 LATIN CAPITAL LETTER A through to U+005A LATIN CAPITAL LETTER Z
  1318. Create a new DOCTYPE token. Set the token's name to the
  1319. lowercase version of the input character (add 0x0020 to
  1320. the character's code point). Switch to the DOCTYPE name
  1321. state. */
  1322. $this->token = array(
  1323. 'name' => strtolower($char),
  1324. 'type' => self::DOCTYPE,
  1325. 'error' => true
  1326. );
  1327. $state = 'DOCTYPE name';
  1328. } elseif($char === false) {
  1329. /* EOF
  1330. Parse error. Create a new DOCTYPE token. Set its
  1331. force-quirks flag to on. Emit the token. Reconsume the
  1332. EOF character in the data state. */
  1333. $this->emitToken(array(
  1334. 'type' => self::PARSEERROR,
  1335. 'data' => 'expected-doctype-name-but-got-eof'
  1336. ));
  1337. $this->emitToken(array(
  1338. 'name' => '',
  1339. 'type' => self::DOCTYPE,
  1340. 'force-quirks' => true,
  1341. 'error' => true
  1342. ));
  1343. $this->stream->unget();
  1344. $state = 'data';
  1345. } else {
  1346. /* Anything else
  1347. Create a new DOCTYPE token. Set the token's name to the
  1348. current input character. Switch to the DOCTYPE name state. */
  1349. $this->token = array(
  1350. 'name' => $char,
  1351. 'type' => self::DOCTYPE,
  1352. 'error' => true
  1353. );
  1354. $state = 'DOCTYPE name';
  1355. }
  1356. break;
  1357. case 'DOCTYPE name':
  1358. /* Consume the next input character: */
  1359. $char = $this->stream->char();
  1360. if($char === "\t" || $char === "\n" || $char === "\x0c" || $char === ' ') {
  1361. /* U+0009 CHARACTER TABULATION
  1362. U+000A LINE FEED (LF)
  1363. U+000C FORM FEED (FF)
  1364. U+0020 SPACE
  1365. Switch to the after DOCTYPE name state. */
  1366. $state = 'after DOCTYPE name';
  1367. } elseif($char === '>') {
  1368. /* U+003E GREATER-THAN SIGN (>)
  1369. Emit the current DOCTYPE token. Switch to the data state. */
  1370. $this->emitToken($this->token);
  1371. $state = 'data';
  1372. } elseif('A' <= $char && $char <= 'Z') {
  1373. /* U+0041 LATIN CAPITAL LETTER A through to U+005A LATIN CAPITAL LETTER Z
  1374. Append the lowercase version of the input character
  1375. (add 0x0020 to the character's code point) to the current
  1376. DOCTYPE token's name. Stay in the DOCTYPE name state. */
  1377. $this->token['name'] .= strtolower($char);
  1378. } elseif($char === false) {
  1379. /* EOF
  1380. Parse error. Set the DOCTYPE token's force-quirks flag
  1381. to on. Emit that DOCTYPE token. Reconsume the EOF
  1382. character in the data state. */
  1383. $this->emitToken(array(
  1384. 'type' => self::PARSEERROR,
  1385. 'data' => 'eof-in-doctype-name'
  1386. ));
  1387. $this->token['force-quirks'] = true;
  1388. $this->emitToken($this->token);
  1389. $this->stream->unget();
  1390. $state = 'data';
  1391. } else {
  1392. /* Anything else
  1393. Append the current input character to the current
  1394. DOCTYPE token's name. Stay in the DOCTYPE name state. */
  1395. $this->token['name'] .= $char;
  1396. }
  1397. // XXX this is probably some sort of quirks mode designation,
  1398. // check tree-builder to be sure. In general 'error' needs
  1399. // to be specc'ified, this probably means removing it at the end
  1400. $this->token['error'] = ($this->token['name'] === 'HTML')
  1401. ? false
  1402. : true;
  1403. break;
  1404. case 'after DOCTYPE name':
  1405. /* Consume the next input character: */
  1406. $char = $this->stream->char();
  1407. if($char === "\t" || $char === "\n" || $char === "\x0c" || $char === ' ') {
  1408. /* U+0009 CHARACTER TABULATION
  1409. U+000A LINE FEED (LF)
  1410. U+000C FORM FEED (FF)
  1411. U+0020 SPACE
  1412. Stay in the after DOCTYPE name state. */
  1413. } elseif($char === '>') {
  1414. /* U+003E GREATER-THAN SIGN (>)
  1415. Emit the current DOCTYPE token. Switch to the data state. */
  1416. $this->emitToken($this->token);
  1417. $state = 'data';
  1418. } elseif($char === false) {
  1419. /* EOF
  1420. Parse error. Set the DOCTYPE token's force-quirks flag
  1421. to on. Emit that DOCTYPE token. Reconsume the EOF
  1422. character in the data state. */
  1423. $this->emitToken(array(
  1424. 'type' => self::PARSEERROR,
  1425. 'data' => 'eof-in-doctype'
  1426. ));
  1427. $this->token['force-quirks'] = true;
  1428. $this->emitToken($this->token);
  1429. $this->stream->unget();
  1430. $state = 'data';
  1431. } else {
  1432. /* Anything else */
  1433. $nextSix = strtoupper($char . $this->stream->charsWhile(self::ALPHA, 5));
  1434. if ($nextSix === 'PUBLIC') {
  1435. /* If the next six characters are an ASCII
  1436. case-insensitive match for the word "PUBLIC", then
  1437. consume those characters and switch to the before
  1438. DOCTYPE public identifier state. */
  1439. $state = 'before DOCTYPE public identifier';
  1440. } elseif ($nextSix === 'SYSTEM') {
  1441. /* Otherwise, if the next six characters are an ASCII
  1442. case-insensitive match for the word "SYSTEM", then
  1443. consume those characters and switch to the before
  1444. DOCTYPE system identifier state. */
  1445. $state = 'before DOCTYPE system identifier';
  1446. } else {
  1447. /* Otherwise, this is the parse error. Set the DOCTYPE
  1448. token's force-quirks flag to on. Switch to the bogus
  1449. DOCTYPE state. */
  1450. $this->emitToken(array(
  1451. 'type' => self::PARSEERROR,
  1452. 'data' => 'expected-space-or-right-bracket-in-doctype'
  1453. ));
  1454. $this->token['force-quirks'] = true;
  1455. $this->token['error'] = true;
  1456. $state = 'bogus DOCTYPE';
  1457. }
  1458. }
  1459. break;
  1460. case 'before DOCTYPE public identifier':
  1461. /* Consume the next input character: */
  1462. $char = $this->stream->char();
  1463. if($char === "\t" || $char === "\n" || $char === "\x0c" || $char === ' ') {
  1464. /* U+0009 CHARACTER TABULATION
  1465. U+000A LINE FEED (LF)
  1466. U+000C FORM FEED (FF)
  1467. U+0020 SPACE
  1468. Stay in the before DOCTYPE public identifier state. */
  1469. } elseif ($char === '"') {
  1470. /* U+0022 QUOTATION MARK (")
  1471. Set the DOCTYPE token's public identifier to the empty
  1472. string (not missing), then switch to the DOCTYPE public
  1473. identifier (double-quoted) state. */
  1474. $this->token['public'] = '';
  1475. $state = 'DOCTYPE public identifier (double-quoted)';
  1476. } elseif ($char === "'") {
  1477. /* U+0027 APOSTROPHE (')
  1478. Set the DOCTYPE token's public identifier to the empty
  1479. string (not missing), then switch to the DOCTYPE public
  1480. identifier (single-quoted) state. */
  1481. $this->token['public'] = '';
  1482. $state = 'DOCTYPE public identifier (single-quoted)';
  1483. } elseif ($char === '>') {
  1484. /* Parse error. Set the DOCTYPE token's force-quirks flag
  1485. to on. Emit that DOCTYPE token. Switch to the data state. */
  1486. $this->emitToken(array(
  1487. 'type' => self::PARSEERROR,
  1488. 'data' => 'unexpected-end-of-doctype'
  1489. ));
  1490. $this->token['force-quirks'] = true;
  1491. $this->emitToken($this->token);
  1492. $state = 'data';
  1493. } elseif ($char === false) {
  1494. /* Parse error. Set the DOCTYPE token's force-quirks
  1495. flag to on. Emit that DOCTYPE token. Reconsume the EOF
  1496. character in the data state. */
  1497. $this->emitToken(array(
  1498. 'type' => self::PARSEERROR,
  1499. 'data' => 'eof-in-doctype'
  1500. ));
  1501. $this->token['force-quirks'] = true;
  1502. $this->emitToken($this->token);
  1503. $this->stream->unget();
  1504. $state = 'data';
  1505. } else {
  1506. /* Parse error. Set the DOCTYPE token's force-quirks flag
  1507. to on. Switch to the bogus DOCTYPE state. */
  1508. $this->emitToken(array(
  1509. 'type' => self::PARSEERROR,
  1510. 'data' => 'unexpected-char-in-doctype'
  1511. ));
  1512. $this->token['force-quirks'] = true;
  1513. $state = 'bogus DOCTYPE';
  1514. }
  1515. break;
  1516. case 'DOCTYPE public identifier (double-quoted)':
  1517. /* Consume the next input character: */
  1518. $char = $this->stream->char();
  1519. if ($char === '"') {
  1520. /* U+0022 QUOTATION MARK (")
  1521. Switch to the after DOCTYPE public identifier state. */
  1522. $state = 'after DOCTYPE public identifier';
  1523. } elseif ($char === '>') {
  1524. /* U+003E GREATER-THAN SIGN (>)
  1525. Parse error. Set the DOCTYPE token's force-quirks flag
  1526. to on. Emit that DOCTYPE token. Switch to the data state. */
  1527. $this->emitToken(array(
  1528. 'type' => self::PARSEERROR,
  1529. 'data' => 'unexpected-end-of-doctype'
  1530. ));
  1531. $this->token['force-quirks'] = true;
  1532. $this->emitToken($this->token);
  1533. $state = 'data';
  1534. } elseif ($char === false) {
  1535. /* EOF
  1536. Parse error. Set the DOCTYPE token's force-quirks flag
  1537. to on. Emit that DOCTYPE token. Reconsume the EOF
  1538. character in the data state. */
  1539. $this->emitToken(array(
  1540. 'type' => self::PARSEERROR,
  1541. 'data' => 'eof-in-doctype'
  1542. ));
  1543. $this->token['force-quirks'] = true;
  1544. $this->emitToken($this->token);
  1545. $this->stream->unget();
  1546. $state = 'data';
  1547. } else {
  1548. /* Anything else
  1549. Append the current input character to the current
  1550. DOCTYPE token's public identifier. Stay in the DOCTYPE
  1551. public identifier (double-quoted) state. */
  1552. $this->token['public'] .= $char;
  1553. }
  1554. break;
  1555. case 'DOCTYPE public identifier (single-quoted)':
  1556. /* Consume the next input character: */
  1557. $char = $this->stream->char();
  1558. if ($char === "'") {
  1559. /* U+0027 APOSTROPHE (')
  1560. Switch to the after DOCTYPE public identifier state. */
  1561. $state = 'after DOCTYPE public identifier';
  1562. } elseif ($char === '>') {
  1563. /* U+003E GREATER-THAN SIGN (>)
  1564. Parse error. Set the DOCTYPE token's force-quirks flag
  1565. to on. Emit that DOCTYPE token. Switch to the data state. */
  1566. $this->emitToken(array(
  1567. 'type' => self::PARSEERROR,
  1568. 'data' => 'unexpected-end-of-doctype'
  1569. ));
  1570. $this->token['force-quirks'] = true;
  1571. $this->emitToken($this->token);
  1572. $state = 'data';
  1573. } elseif ($char === false) {
  1574. /* EOF
  1575. Parse error. Set the DOCTYPE token's force-quirks flag
  1576. to on. Emit that DOCTYPE token. Reconsume the EOF
  1577. character in the data state. */
  1578. $this->emitToken(array(
  1579. 'type' => self::PARSEERROR,
  1580. 'data' => 'eof-in-doctype'
  1581. ));
  1582. $this->token['force-quirks'] = true;
  1583. $this->emitToken($this->token);
  1584. $this->stream->unget();
  1585. $state = 'data';
  1586. } else {
  1587. /* Anything else
  1588. Append the current input character to the current
  1589. DOCTYPE token's public identifier. Stay in the DOCTYPE
  1590. public identifier (double-quoted) state. */
  1591. $this->token['public'] .= $char;
  1592. }
  1593. break;
  1594. case 'after DOCTYPE public identifier':
  1595. /* Consume the next input character: */
  1596. $char = $this->stream->char();
  1597. if($char === "\t" || $char === "\n" || $char === "\x0c" || $char === ' ') {
  1598. /* U+0009 CHARACTER TABULATION
  1599. U+000A LINE FEED (LF)
  1600. U+000C FORM FEED (FF)
  1601. U+0020 SPACE
  1602. Stay in the after DOCTYPE public identifier state. */
  1603. } elseif ($char === '"') {
  1604. /* U+0022 QUOTATION MARK (")
  1605. Set the DOCTYPE token's system identifier to the
  1606. empty string (not missing), then switch to the DOCTYPE
  1607. system identifier (double-quoted) state. */
  1608. $this->token['system'] = '';
  1609. $state = 'DOCTYPE system identifier (double-quoted)';
  1610. } elseif ($char === "'") {
  1611. /* U+0027 APOSTROPHE (')
  1612. Set the DOCTYPE token's system identifier to the
  1613. empty string (not missing), then switch to the DOCTYPE
  1614. system identifier (single-quoted) state. */
  1615. $this->token['system'] = '';
  1616. $state = 'DOCTYPE system identifier (single-quoted)';
  1617. } elseif ($char === '>') {
  1618. /* U+003E GREATER-THAN SIGN (>)
  1619. Emit the current DOCTYPE token. Switch to the data state. */
  1620. $this->emitToken($this->token);
  1621. $state = 'data';
  1622. } elseif ($char === false) {
  1623. /* Parse error. Set the DOCTYPE token's force-quirks
  1624. flag to on. Emit that DOCTYPE token. Reconsume the EOF
  1625. character in the data state. */
  1626. $this->emitToken(array(
  1627. 'type' => self::PARSEERROR,
  1628. 'data' => 'eof-in-doctype'
  1629. ));
  1630. $this->token['force-quirks'] = true;
  1631. $this->emitToken($this->token);
  1632. $this->stream->unget();
  1633. $state = 'data';
  1634. } else {
  1635. /* Anything else
  1636. Parse error. Set the DOCTYPE token's force-quirks flag
  1637. to on. Switch to the bogus DOCTYPE state. */
  1638. $this->emitToken(array(
  1639. 'type' => self::PARSEERROR,
  1640. 'data' => 'unexpected-char-in-doctype'
  1641. ));
  1642. $this->token['force-quirks'] = true;
  1643. $state = 'bogus DOCTYPE';
  1644. }
  1645. break;
  1646. case 'before DOCTYPE system identifier':
  1647. /* Consume the next input character: */
  1648. $char = $this->stream->char();
  1649. if($char === "\t" || $char === "\n" || $char === "\x0c" || $char === ' ') {
  1650. /* U+0009 CHARACTER TABULATION
  1651. U+000A LINE FEED (LF)
  1652. U+000C FORM FEED (FF)
  1653. U+0020 SPACE
  1654. Stay in the before DOCTYPE system identifier state. */
  1655. } elseif ($char === '"') {
  1656. /* U+0022 QUOTATION MARK (")
  1657. Set the DOCTYPE token's system identifier to the empty
  1658. string (not missing), then switch to the DOCTYPE system
  1659. identifier (double-quoted) state. */
  1660. $this->token['system'] = '';
  1661. $state = 'DOCTYPE system identifier (double-quoted)';
  1662. } elseif ($char === "'") {
  1663. /* U+0027 APOSTROPHE (')
  1664. Set the DOCTYPE token's system identifier to the empty
  1665. string (not missing), then switch to the DOCTYPE system
  1666. identifier (single-quoted) state. */
  1667. $this->token['system'] = '';
  1668. $state = 'DOCTYPE system identifier (single-quoted)';
  1669. } elseif ($char === '>') {
  1670. /* Parse error. Set the DOCTYPE token's force-quirks flag
  1671. to on. Emit that DOCTYPE token. Switch to the data state. */
  1672. $this->emitToken(array(
  1673. 'type' => self::PARSEERROR,
  1674. 'data' => 'unexpected-char-in-doctype'
  1675. ));
  1676. $this->token['force-quirks'] = true;
  1677. $this->emitToken($this->token);
  1678. $state = 'data';
  1679. } elseif ($char === false) {
  1680. /* Parse error. Set the DOCTYPE token's force-quirks
  1681. flag to on. Emit that DOCTYPE token. Reconsume the EOF
  1682. character in the data state. */
  1683. $this->emitToken(array(
  1684. 'type' => self::PARSEERROR,
  1685. 'data' => 'eof-in-doctype'
  1686. ));
  1687. $this->token['force-quirks'] = true;
  1688. $this->emitToken($this->token);
  1689. $this->stream->unget();
  1690. $state = 'data';
  1691. } else {
  1692. /* Parse error. Set the DOCTYPE token's force-quirks flag
  1693. to on. Switch to the bogus DOCTYPE state. */
  1694. $this->emitToken(array(
  1695. 'type' => self::PARSEERROR,
  1696. 'data' => 'unexpected-char-in-doctype'
  1697. ));
  1698. $this->token['force-quirks'] = true;
  1699. $state = 'bogus DOCTYPE';
  1700. }
  1701. break;
  1702. case 'DOCTYPE system identifier (double-quoted)':
  1703. /* Consume the next input character: */
  1704. $char = $this->stream->char();
  1705. if ($char === '"') {
  1706. /* U+0022 QUOTATION MARK (")
  1707. Switch to the after DOCTYPE system identifier state. */
  1708. $state = 'after DOCTYPE system identifier';
  1709. } elseif ($char === '>') {
  1710. /* U+003E GREATER-THAN SIGN (>)
  1711. Parse error. Set the DOCTYPE token's force-quirks flag
  1712. to on. Emit that DOCTYPE token. Switch to the data state. */
  1713. $this->emitToken(array(
  1714. 'type' => self::PARSEERROR,
  1715. 'data' => 'unexpected-end-of-doctype'
  1716. ));
  1717. $this->token['force-quirks'] = true;
  1718. $this->emitToken($this->token);
  1719. $state = 'data';
  1720. } elseif ($char === false) {
  1721. /* EOF
  1722. Parse error. Set the DOCTYPE token's force-quirks flag
  1723. to on. Emit that DOCTYPE token. Reconsume the EOF
  1724. character in the data state. */
  1725. $this->emitToken(array(
  1726. 'type' => self::PARSEERROR,
  1727. 'data' => 'eof-in-doctype'
  1728. ));
  1729. $this->token['force-quirks'] = true;
  1730. $this->emitToken($this->token);
  1731. $this->stream->unget();
  1732. $state = 'data';
  1733. } else {
  1734. /* Anything else
  1735. Append the current input character to the current
  1736. DOCTYPE token's system identifier. Stay in the DOCTYPE
  1737. system identifier (double-quoted) state. */
  1738. $this->token['system'] .= $char;
  1739. }
  1740. break;
  1741. case 'DOCTYPE system identifier (single-quoted)':
  1742. /* Consume the next input character: */
  1743. $char = $this->stream->char();
  1744. if ($char === "'") {
  1745. /* U+0027 APOSTROPHE (')
  1746. Switch to the after DOCTYPE system identifier state. */
  1747. $state = 'after DOCTYPE system identifier';
  1748. } elseif ($char === '>') {
  1749. /* U+003E GREATER-THAN SIGN (>)
  1750. Parse error. Set the DOCTYPE token's force-quirks flag
  1751. to on. Emit that DOCTYPE token. Switch to the data state. */
  1752. $this->emitToken(array(
  1753. 'type' => self::PARSEERROR,
  1754. 'data' => 'unexpected-end-of-doctype'
  1755. ));
  1756. $this->token['force-quirks'] = true;
  1757. $this->emitToken($this->token);
  1758. $state = 'data';
  1759. } elseif ($char === false) {
  1760. /* EOF
  1761. Parse error. Set the DOCTYPE token's force-quirks flag
  1762. to on. Emit that DOCTYPE token. Reconsume the EOF
  1763. character in the data state. */
  1764. $this->emitToken(array(
  1765. 'type' => self::PARSEERROR,
  1766. 'data' => 'eof-in-doctype'
  1767. ));
  1768. $this->token['force-quirks'] = true;
  1769. $this->emitToken($this->token);
  1770. $this->stream->unget();
  1771. $state = 'data';
  1772. } else {
  1773. /* Anything else
  1774. Append the current input character to the current
  1775. DOCTYPE token's system identifier. Stay in the DOCTYPE
  1776. system identifier (double-quoted) state. */
  1777. $this->token['system'] .= $char;
  1778. }
  1779. break;
  1780. case 'after DOCTYPE system identifier':
  1781. /* Consume the next input character: */
  1782. $char = $this->stream->char();
  1783. if($char === "\t" || $char === "\n" || $char === "\x0c" || $char === ' ') {
  1784. /* U+0009 CHARACTER TABULATION
  1785. U+000A LINE FEED (LF)
  1786. U+000C FORM FEED (FF)
  1787. U+0020 SPACE
  1788. Stay in the after DOCTYPE system identifier state. */
  1789. } elseif ($char === '>') {
  1790. /* U+003E GREATER-THAN SIGN (>)
  1791. Emit the current DOCTYPE token. Switch to the data state. */
  1792. $this->emitToken($this->token);
  1793. $state = 'data';
  1794. } elseif ($char === false) {
  1795. /* Parse error. Set the DOCTYPE token's force-quirks
  1796. flag to on. Emit that DOCTYPE token. Reconsume the EOF
  1797. character in the data state. */
  1798. $this->emitToken(array(
  1799. 'type' => self::PARSEERROR,
  1800. 'data' => 'eof-in-doctype'
  1801. ));
  1802. $this->token['force-quirks'] = true;
  1803. $this->emitToken($this->token);
  1804. $this->stream->unget();
  1805. $state = 'data';
  1806. } else {
  1807. /* Anything else
  1808. Parse error. Switch to the bogus DOCTYPE state.
  1809. (This does not set the DOCTYPE token's force-quirks
  1810. flag to on.) */
  1811. $this->emitToken(array(
  1812. 'type' => self::PARSEERROR,
  1813. 'data' => 'unexpected-char-in-doctype'
  1814. ));
  1815. $state = 'bogus DOCTYPE';
  1816. }
  1817. break;
  1818. case 'bogus DOCTYPE':
  1819. /* Consume the next input character: */
  1820. $char = $this->stream->char();
  1821. if ($char === '>') {
  1822. /* U+003E GREATER-THAN SIGN (>)
  1823. Emit the DOCTYPE token. Switch to the data state. */
  1824. $this->emitToken($this->token);
  1825. $state = 'data';
  1826. } elseif($char === false) {
  1827. /* EOF
  1828. Emit the DOCTYPE token. Reconsume the EOF character in
  1829. the data state. */
  1830. $this->emitToken($this->token);
  1831. $this->stream->unget();
  1832. $state = 'data';
  1833. } else {
  1834. /* Anything else
  1835. Stay in the bogus DOCTYPE state. */
  1836. }
  1837. break;
  1838. // case 'cdataSection':
  1839. }
  1840. }
  1841. }
  1842. /**
  1843. * Returns a serialized representation of the tree.
  1844. */
  1845. public function save() {
  1846. return $this->tree->save();
  1847. }
  1848. /**
  1849. * @return HTML5_TreeBuilder The tree
  1850. */
  1851. public function getTree() {
  1852. return $this->tree;
  1853. }
  1854. /**
  1855. * Returns the input stream.
  1856. */
  1857. public function stream() {
  1858. return $this->stream;
  1859. }
  1860. private function consumeCharacterReference($allowed = false, $inattr = false) {
  1861. // This goes quite far against spec, and is far closer to the Python
  1862. // impl., mainly because we don't do the large unconsuming the spec
  1863. // requires.
  1864. // All consumed characters.
  1865. $chars = $this->stream->char();
  1866. /* This section defines how to consume a character
  1867. reference. This definition is used when parsing character
  1868. references in text and in attributes.
  1869. The behavior depends on the identity of the next character
  1870. (the one immediately after the U+0026 AMPERSAND character): */
  1871. if (
  1872. $chars[0] === "\x09" ||
  1873. $chars[0] === "\x0A" ||
  1874. $chars[0] === "\x0C" ||
  1875. $chars[0] === "\x20" ||
  1876. $chars[0] === '<' ||
  1877. $chars[0] === '&' ||
  1878. $chars === false ||
  1879. $chars[0] === $allowed
  1880. ) {
  1881. /* U+0009 CHARACTER TABULATION
  1882. U+000A LINE FEED (LF)
  1883. U+000C FORM FEED (FF)
  1884. U+0020 SPACE
  1885. U+003C LESS-THAN SIGN
  1886. U+0026 AMPERSAND
  1887. EOF
  1888. The additional allowed character, if there is one
  1889. Not a character reference. No characters are consumed,
  1890. and nothing is returned. (This is not an error, either.) */
  1891. // We already consumed, so unconsume.
  1892. $this->stream->unget();
  1893. return '&';
  1894. } elseif ($chars[0] === '#') {
  1895. /* Consume the U+0023 NUMBER SIGN. */
  1896. // Um, yeah, we already did that.
  1897. /* The behavior further depends on the character after
  1898. the U+0023 NUMBER SIGN: */
  1899. $chars .= $this->stream->char();
  1900. if (isset($chars[1]) && ($chars[1] === 'x' || $chars[1] === 'X')) {
  1901. /* U+0078 LATIN SMALL LETTER X
  1902. U+0058 LATIN CAPITAL LETTER X */
  1903. /* Consume the X. */
  1904. // Um, yeah, we already did that.
  1905. /* Follow the steps below, but using the range of
  1906. characters U+0030 DIGIT ZERO through to U+0039 DIGIT
  1907. NINE, U+0061 LATIN SMALL LETTER A through to U+0066
  1908. LATIN SMALL LETTER F, and U+0041 LATIN CAPITAL LETTER
  1909. A, through to U+0046 LATIN CAPITAL LETTER F (in other
  1910. words, 0123456789, ABCDEF, abcdef). */
  1911. $char_class = self::HEX;
  1912. /* When it comes to interpreting the
  1913. number, interpret it as a hexadecimal number. */
  1914. $hex = true;
  1915. } else {
  1916. /* Anything else */
  1917. // Unconsume because we shouldn't have consumed this.
  1918. $chars = $chars[0];
  1919. $this->stream->unget();
  1920. /* Follow the steps below, but using the range of
  1921. characters U+0030 DIGIT ZERO through to U+0039 DIGIT
  1922. NINE (i.e. just 0123456789). */
  1923. $char_class = self::DIGIT;
  1924. /* When it comes to interpreting the number,
  1925. interpret it as a decimal number. */
  1926. $hex = false;
  1927. }
  1928. /* Consume as many characters as match the range of characters given above. */
  1929. $consumed = $this->stream->charsWhile($char_class);
  1930. if ($consumed === '' || $consumed === false) {
  1931. /* If no characters match the range, then don't consume
  1932. any characters (and unconsume the U+0023 NUMBER SIGN
  1933. character and, if appropriate, the X character). This
  1934. is a parse error; nothing is returned. */
  1935. $this->emitToken(array(
  1936. 'type' => self::PARSEERROR,
  1937. 'data' => 'expected-numeric-entity'
  1938. ));
  1939. return '&' . $chars;
  1940. } else {
  1941. /* Otherwise, if the next character is a U+003B SEMICOLON,
  1942. consume that too. If it isn't, there is a parse error. */
  1943. if ($this->stream->char() !== ';') {
  1944. $this->stream->unget();
  1945. $this->emitToken(array(
  1946. 'type' => self::PARSEERROR,
  1947. 'data' => 'numeric-entity-without-semicolon'
  1948. ));
  1949. }
  1950. /* If one or more characters match the range, then take
  1951. them all and interpret the string of characters as a number
  1952. (either hexadecimal or decimal as appropriate). */
  1953. $codepoint = $hex ? hexdec($consumed) : (int) $consumed;
  1954. /* If that number is one of the numbers in the first column
  1955. of the following table, then this is a parse error. Find the
  1956. row with that number in the first column, and return a
  1957. character token for the Unicode character given in the
  1958. second column of that row. */
  1959. $new_codepoint = HTML5_Data::getRealCodepoint($codepoint);
  1960. if ($new_codepoint) {
  1961. $this->emitToken(array(
  1962. 'type' => self::PARSEERROR,
  1963. 'data' => 'illegal-windows-1252-entity'
  1964. ));
  1965. return HTML5_Data::utf8chr($new_codepoint);
  1966. } else {
  1967. /* Otherwise, if the number is greater than 0x10FFFF, then
  1968. * this is a parse error. Return a U+FFFD REPLACEMENT
  1969. * CHARACTER. */
  1970. if ($codepoint > 0x10FFFF) {
  1971. $this->emitToken(array(
  1972. 'type' => self::PARSEERROR,
  1973. 'data' => 'overlong-character-entity' // XXX probably not correct
  1974. ));
  1975. return "\xEF\xBF\xBD";
  1976. }
  1977. /* Otherwise, return a character token for the Unicode
  1978. * character whose code point is that number. If the
  1979. * number is in the range 0x0001 to 0x0008, 0x000E to
  1980. * 0x001F, 0x007F to 0x009F, 0xD800 to 0xDFFF, 0xFDD0 to
  1981. * 0xFDEF, or is one of 0x000B, 0xFFFE, 0xFFFF, 0x1FFFE,
  1982. * 0x1FFFF, 0x2FFFE, 0x2FFFF, 0x3FFFE, 0x3FFFF, 0x4FFFE,
  1983. * 0x4FFFF, 0x5FFFE, 0x5FFFF, 0x6FFFE, 0x6FFFF, 0x7FFFE,
  1984. * 0x7FFFF, 0x8FFFE, 0x8FFFF, 0x9FFFE, 0x9FFFF, 0xAFFFE,
  1985. * 0xAFFFF, 0xBFFFE, 0xBFFFF, 0xCFFFE, 0xCFFFF, 0xDFFFE,
  1986. * 0xDFFFF, 0xEFFFE, 0xEFFFF, 0xFFFFE, 0xFFFFF, 0x10FFFE,
  1987. * or 0x10FFFF, then this is a parse error. */
  1988. // && has higher precedence than ||
  1989. if (
  1990. $codepoint >= 0x0000 && $codepoint <= 0x0008 ||
  1991. $codepoint === 0x000B ||
  1992. $codepoint >= 0x000E && $codepoint <= 0x001F ||
  1993. $codepoint >= 0x007F && $codepoint <= 0x009F ||
  1994. $codepoint >= 0xD800 && $codepoint <= 0xDFFF ||
  1995. $codepoint >= 0xFDD0 && $codepoint <= 0xFDEF ||
  1996. ($codepoint & 0xFFFE) === 0xFFFE ||
  1997. $codepoint == 0x10FFFF || $codepoint == 0x10FFFE
  1998. ) {
  1999. $this->emitToken(array(
  2000. 'type' => self::PARSEERROR,
  2001. 'data' => 'illegal-codepoint-for-numeric-entity'
  2002. ));
  2003. }
  2004. return HTML5_Data::utf8chr($codepoint);
  2005. }
  2006. }
  2007. } else {
  2008. /* Anything else */
  2009. /* Consume the maximum number of characters possible,
  2010. with the consumed characters matching one of the
  2011. identifiers in the first column of the named character
  2012. references table (in a case-sensitive manner). */
  2013. // What we actually do here is consume as much as we can while it
  2014. // matches the start of one of the identifiers in the first column.
  2015. $refs = HTML5_Data::getNamedCharacterReferences();
  2016. // Get the longest string which is the start of an identifier
  2017. // ($chars) as well as the longest identifier which matches ($id)
  2018. // and its codepoint ($codepoint).
  2019. $codepoint = false;
  2020. $char = $chars;
  2021. while ($char !== false && isset($refs[$char])) {
  2022. $refs = $refs[$char];
  2023. if (isset($refs['codepoint'])) {
  2024. $id = $chars;
  2025. $codepoint = $refs['codepoint'];
  2026. }
  2027. $chars .= $char = $this->stream->char();
  2028. }
  2029. // Unconsume the one character we just took which caused the while
  2030. // statement to fail. This could be anything and could cause state
  2031. // changes (as if it matches the while loop it must be
  2032. // alphanumeric so we can just concat it to whatever we get later).
  2033. $this->stream->unget();
  2034. if ($char !== false) {
  2035. $chars = substr($chars, 0, -1);
  2036. }
  2037. /* If no match can be made, then this is a parse error.
  2038. No characters are consumed, and nothing is returned. */
  2039. if (!$codepoint) {
  2040. $this->emitToken(array(
  2041. 'type' => self::PARSEERROR,
  2042. 'data' => 'expected-named-entity'
  2043. ));
  2044. return '&' . $chars;
  2045. }
  2046. /* If the last character matched is not a U+003B SEMICOLON
  2047. (;), there is a parse error. */
  2048. $semicolon = true;
  2049. if (substr($id, -1) !== ';') {
  2050. $this->emitToken(array(
  2051. 'type' => self::PARSEERROR,
  2052. 'data' => 'named-entity-without-semicolon'
  2053. ));
  2054. $semicolon = false;
  2055. }
  2056. /* If the character reference is being consumed as part of
  2057. an attribute, and the last character matched is not a
  2058. U+003B SEMICOLON (;), and the next character is in the
  2059. range U+0030 DIGIT ZERO to U+0039 DIGIT NINE, U+0041
  2060. LATIN CAPITAL LETTER A to U+005A LATIN CAPITAL LETTER Z,
  2061. or U+0061 LATIN SMALL LETTER A to U+007A LATIN SMALL LETTER Z,
  2062. then, for historical reasons, all the characters that were
  2063. matched after the U+0026 AMPERSAND (&) must be unconsumed,
  2064. and nothing is returned. */
  2065. if ($inattr && !$semicolon) {
  2066. // The next character is either the next character in $chars or in the stream.
  2067. if (strlen($chars) > strlen($id)) {
  2068. $next = substr($chars, strlen($id), 1);
  2069. } else {
  2070. $next = $this->stream->char();
  2071. $this->stream->unget();
  2072. }
  2073. if (
  2074. '0' <= $next && $next <= '9' ||
  2075. 'A' <= $next && $next <= 'Z' ||
  2076. 'a' <= $next && $next <= 'z'
  2077. ) {
  2078. return '&' . $chars;
  2079. }
  2080. }
  2081. /* Otherwise, return a character token for the character
  2082. corresponding to the character reference name (as given
  2083. by the second column of the named character references table). */
  2084. return HTML5_Data::utf8chr($codepoint) . substr($chars, strlen($id));
  2085. }
  2086. }
  2087. private function characterReferenceInAttributeValue($allowed = false) {
  2088. /* Attempt to consume a character reference. */
  2089. $entity = $this->consumeCharacterReference($allowed, true);
  2090. /* If nothing is returned, append a U+0026 AMPERSAND
  2091. character to the current attribute's value.
  2092. Otherwise, append the returned character token to the
  2093. current attribute's value. */
  2094. $char = (!$entity)
  2095. ? '&'
  2096. : $entity;
  2097. $last = count($this->token['attr']) - 1;
  2098. $this->token['attr'][$last]['value'] .= $char;
  2099. /* Finally, switch back to the attribute value state that you
  2100. were in when were switched into this state. */
  2101. }
  2102. /**
  2103. * Emits a token, passing it on to the tree builder.
  2104. */
  2105. protected function emitToken($token, $checkStream = true, $dry = false) {
  2106. if ($checkStream) {
  2107. // Emit errors from input stream.
  2108. while ($this->stream->errors) {
  2109. $this->emitToken(array_shift($this->stream->errors), false);
  2110. }
  2111. }
  2112. if($token['type'] === self::ENDTAG && !empty($token['attr'])) {
  2113. for ($i = 0; $i < count($token['attr']); $i++) {
  2114. $this->emitToken(array(
  2115. 'type' => self::PARSEERROR,
  2116. 'data' => 'attributes-in-end-tag'
  2117. ));
  2118. }
  2119. }
  2120. if($token['type'] === self::ENDTAG && !empty($token['self-closing'])) {
  2121. $this->emitToken(array(
  2122. 'type' => self::PARSEERROR,
  2123. 'data' => 'self-closing-flag-on-end-tag',
  2124. ));
  2125. }
  2126. if($token['type'] === self::STARTTAG) {
  2127. // This could be changed to actually pass the tree-builder a hash
  2128. $hash = array();
  2129. foreach ($token['attr'] as $keypair) {
  2130. if (isset($hash[$keypair['name']])) {
  2131. $this->emitToken(array(
  2132. 'type' => self::PARSEERROR,
  2133. 'data' => 'duplicate-attribute',
  2134. ));
  2135. } else {
  2136. $hash[$keypair['name']] = $keypair['value'];
  2137. }
  2138. }
  2139. }
  2140. if(!$dry) {
  2141. // the current structure of attributes is not a terribly good one
  2142. $this->tree->emitToken($token);
  2143. }
  2144. if(!$dry && is_int($this->tree->content_model)) {
  2145. $this->content_model = $this->tree->content_model;
  2146. $this->tree->content_model = null;
  2147. } elseif($token['type'] === self::ENDTAG) {
  2148. $this->content_model = self::PCDATA;
  2149. }
  2150. }
  2151. }