Add lexer test runner and whitespace characters tests

Whitespace characters for the tests are U+000B, U+000C, U+000D, U+0085, U+2028, U+2029

compiler/testData/lexer/kdoc/codeBlocks.kt

@@ -0,0 +1,20 @@
+/**
+ * @foo This is text.
+ *    This is indented code block
+ *    Second line of code block
+ * @foo This is text.
+ *	Tabulated code block
+ *	Tabulated second line
+ * @foo This is text.
+ * ``` kotlin
+ * @foo This is code block.
+ * ```
+ * @foo This is text.
+ * ~~~
+ * @foo This is code block
+ * With multiple lines.
+ * ~~~
+ * @foo This is yet more text.
+ * ```
+ * @foo This is an unclosed code block.
+ */

compiler/testData/lexer/kdoc/codeBlocks.txt

@@ -0,0 +1,75 @@
+KDOC_START ('/**')
+WHITE_SPACE ('\n ')
+KDOC_LEADING_ASTERISK ('*')
+KDOC_TEXT (' ')
+KDOC_TAG_NAME ('@foo')
+WHITE_SPACE (' ')
+KDOC_TEXT ('This is text.')
+WHITE_SPACE ('\n ')
+KDOC_LEADING_ASTERISK ('*')
+KDOC_CODE_BLOCK_TEXT ('    This is indented code block')
+WHITE_SPACE ('\n ')
+KDOC_LEADING_ASTERISK ('*')
+KDOC_CODE_BLOCK_TEXT ('    Second line of code block')
+WHITE_SPACE ('\n ')
+KDOC_LEADING_ASTERISK ('*')
+KDOC_TEXT (' ')
+KDOC_TAG_NAME ('@foo')
+WHITE_SPACE (' ')
+KDOC_TEXT ('This is text.')
+WHITE_SPACE ('\n ')
+KDOC_LEADING_ASTERISK ('*')
+KDOC_CODE_BLOCK_TEXT ('	Tabulated code block')
+WHITE_SPACE ('\n ')
+KDOC_LEADING_ASTERISK ('*')
+KDOC_CODE_BLOCK_TEXT ('	Tabulated second line')
+WHITE_SPACE ('\n ')
+KDOC_LEADING_ASTERISK ('*')
+KDOC_TEXT (' ')
+KDOC_TAG_NAME ('@foo')
+WHITE_SPACE (' ')
+KDOC_TEXT ('This is text.')
+WHITE_SPACE ('\n ')
+KDOC_LEADING_ASTERISK ('*')
+KDOC_TEXT (' ``` kotlin')
+WHITE_SPACE ('\n ')
+KDOC_LEADING_ASTERISK ('*')
+KDOC_CODE_BLOCK_TEXT (' @foo This is code block.')
+WHITE_SPACE ('\n ')
+KDOC_LEADING_ASTERISK ('*')
+KDOC_CODE_BLOCK_TEXT (' ')
+KDOC_TEXT ('```')
+WHITE_SPACE ('\n ')
+KDOC_LEADING_ASTERISK ('*')
+KDOC_TEXT (' ')
+KDOC_TAG_NAME ('@foo')
+WHITE_SPACE (' ')
+KDOC_TEXT ('This is text.')
+WHITE_SPACE ('\n ')
+KDOC_LEADING_ASTERISK ('*')
+KDOC_TEXT (' ~~~')
+WHITE_SPACE ('\n ')
+KDOC_LEADING_ASTERISK ('*')
+KDOC_CODE_BLOCK_TEXT (' @foo This is code block')
+WHITE_SPACE ('\n ')
+KDOC_LEADING_ASTERISK ('*')
+KDOC_CODE_BLOCK_TEXT (' With multiple lines.')
+WHITE_SPACE ('\n ')
+KDOC_LEADING_ASTERISK ('*')
+KDOC_CODE_BLOCK_TEXT (' ')
+KDOC_TEXT ('~~~')
+WHITE_SPACE ('\n ')
+KDOC_LEADING_ASTERISK ('*')
+KDOC_TEXT (' ')
+KDOC_TAG_NAME ('@foo')
+WHITE_SPACE (' ')
+KDOC_TEXT ('This is yet more text.')
+WHITE_SPACE ('\n ')
+KDOC_LEADING_ASTERISK ('*')
+KDOC_TEXT (' ```')
+WHITE_SPACE ('\n ')
+KDOC_LEADING_ASTERISK ('*')
+KDOC_CODE_BLOCK_TEXT (' @foo This is an unclosed code block.')
+WHITE_SPACE ('\n ')
+KDOC_TEXT ('*/')
+WHITE_SPACE ('\n')

compiler/testData/lexer/kdoc/codeBlocksWithVerticalTabs.kt

@@ -0,0 +1,20 @@
+/**
+ * @foo Thisis text.
+ *    This isindented code block
+ *    Second lineof code block
+ * @foo Thisistext.
+ *	Tabulated code block
+ *	Tabulated secondline
+ * @foo Thisis text.
+ * ``` kotlin
+ * @foo This iscode block.
+ * ```
+ * @foo Thisis text.
+ * ~~~
+ * @foo This is code block
+ * With multiplelines.
+ * ~~~
+ * @foo Thisis yet moretext.
+ * ```
+ * @foo Thisis an unclosed codeblock.
+ */

compiler/testData/lexer/kdoc/codeBlocksWithVerticalTabs.txt

@@ -0,0 +1,74 @@
+KDOC_START ('/**')
+WHITE_SPACE ('\n ')
+KDOC_LEADING_ASTERISK ('*')
+KDOC_TEXT (' ')
+KDOC_TAG_NAME ('@foo')
+WHITE_SPACE (' ')
+KDOC_TEXT ('Thisis text.')
+WHITE_SPACE ('\n ')
+KDOC_LEADING_ASTERISK ('*')
+KDOC_CODE_BLOCK_TEXT ('    This isindented code block')
+WHITE_SPACE ('\n ')
+KDOC_LEADING_ASTERISK ('*')
+KDOC_CODE_BLOCK_TEXT ('    Second lineof code block')
+WHITE_SPACE ('\n ')
+KDOC_LEADING_ASTERISK ('*')
+KDOC_TEXT (' ')
+KDOC_TAG_NAME ('@foo')
+WHITE_SPACE (' ')
+KDOC_TEXT ('Thisistext.')
+WHITE_SPACE ('\n ')
+KDOC_LEADING_ASTERISK ('*')
+KDOC_CODE_BLOCK_TEXT ('	Tabulated code block')
+WHITE_SPACE ('\n ')
+KDOC_LEADING_ASTERISK ('*')
+KDOC_CODE_BLOCK_TEXT ('	Tabulated secondline')
+WHITE_SPACE ('\n ')
+KDOC_LEADING_ASTERISK ('*')
+KDOC_TEXT (' ')
+KDOC_TAG_NAME ('@foo')
+WHITE_SPACE (' ')
+KDOC_TEXT ('Thisis text.')
+WHITE_SPACE ('\n ')
+KDOC_LEADING_ASTERISK ('*')
+KDOC_TEXT (' ``` kotlin')
+WHITE_SPACE ('\n ')
+KDOC_LEADING_ASTERISK ('*')
+KDOC_CODE_BLOCK_TEXT (' @foo This iscode block.')
+WHITE_SPACE ('\n ')
+KDOC_LEADING_ASTERISK ('*')
+KDOC_CODE_BLOCK_TEXT (' ')
+KDOC_TEXT ('```')
+WHITE_SPACE ('\n ')
+KDOC_LEADING_ASTERISK ('*')
+KDOC_TEXT (' ')
+KDOC_TAG_NAME ('@foo')
+WHITE_SPACE (' ')
+KDOC_TEXT ('Thisis text.')
+WHITE_SPACE ('\n ')
+KDOC_LEADING_ASTERISK ('*')
+KDOC_TEXT (' ~~~')
+WHITE_SPACE ('\n ')
+KDOC_LEADING_ASTERISK ('*')
+KDOC_CODE_BLOCK_TEXT (' @foo This is code block')
+WHITE_SPACE ('\n ')
+KDOC_LEADING_ASTERISK ('*')
+KDOC_CODE_BLOCK_TEXT (' With multiplelines.')
+WHITE_SPACE ('\n ')
+KDOC_LEADING_ASTERISK ('*')
+KDOC_CODE_BLOCK_TEXT (' ')
+KDOC_TEXT ('~~~')
+WHITE_SPACE ('\n ')
+KDOC_LEADING_ASTERISK ('*')
+KDOC_TEXT (' ')
+KDOC_TAG_NAME ('@foo')
+WHITE_SPACE (' ')
+KDOC_TEXT ('Thisis yet moretext.')
+WHITE_SPACE ('\n ')
+KDOC_LEADING_ASTERISK ('*')
+KDOC_TEXT (' ```')
+WHITE_SPACE ('\n ')
+KDOC_LEADING_ASTERISK ('*')
+KDOC_CODE_BLOCK_TEXT (' @foo Thisis an unclosed codeblock.')
+WHITE_SPACE ('\n ')
+KDOC_END ('*/')