first commit

2023-07-12 16:31:19 +01:00
commit 3424b1927b
23306 changed files with 2217776 additions and 0 deletions
@@ -0,0 +1,52 @@
+var test = require('tap').test
+var tok = require('../')
+
+test('tokenize', function (t) {
+
+  function e (input, output, invalid) {
+    t.deepEqual(tok(input, true), output)
+    if(invalid)
+      t.throws(function  () {
+        tok(input)
+      })
+  }
+
+  e('function () { }', 
+    ['function', ' ', '(', ')', ' ', '{', ' ', '}'])
+
+  e('"hello"', 
+    ['"hello"'])
+
+  e("'hello'", 
+    ["'hello'"])
+
+  e('"   "', 
+    ['"   "'])
+
+  e('" \\"  "', 
+    ['" \\"  "'])
+
+  e('" \\\n  "', 
+    ['" \\\n  "'])
+
+  //regrettably, js doesn't have multiline strings.
+
+  e('" \n  "', 
+    ['"',' \n  ', '"'], true)
+
+  e("'   '", 
+    ["'   '"])
+
+  e("' \\'  '", 
+    ["' \\'  '"])
+
+  e("' \n  '", 
+    ["'"," \n  ", "'"], true)
+
+  t.end()
+})
+
+
+
+
+