preserve leading indentation in KDoc code blocks

#KT-9933 Fixed

compiler/frontend/src/org/jetbrains/kotlin/kdoc/lexer/KDoc.flex

@@ -178,33 +178,20 @@ CODE_FENCE_END=("```" | "~~~")
         yybegin(CONTENTS);
         return KDocTokens.TEXT;
     }
-
-    {WHITE_SPACE_CHAR}+ {
-        if (yytextContainLineBreaks()) {
-            yybegin(CODE_BLOCK_LINE_BEGINNING);
-            return TokenType.WHITE_SPACE;
-        }
-        return KDocTokens.TEXT;
-    }
-
-    . {
-        yybegin(CODE_BLOCK);
-        return KDocTokens.TEXT;
-    }
 }
 
 <CODE_BLOCK_LINE_BEGINNING, CODE_BLOCK_CONTENTS_BEGINNING, CODE_BLOCK> {
-
     {WHITE_SPACE_CHAR}+ {
         if (yytextContainLineBreaks()) {
             yybegin(CODE_BLOCK_LINE_BEGINNING);
             return TokenType.WHITE_SPACE;
         }
-        return KDocTokens.TEXT;
+        return KDocTokens.CODE_BLOCK_TEXT;
     }
+
     . {
         yybegin(CODE_BLOCK);
-        return KDocTokens.TEXT;
+        return KDocTokens.CODE_BLOCK_TEXT;
     }
 }
 

compiler/frontend/src/org/jetbrains/kotlin/kdoc/lexer/KDocLexer.java

@@ -28,7 +28,7 @@ public class KDocLexer extends MergingLexerAdapter {
                 new FlexAdapter(
                         new _KDocLexer((Reader) null)
                 ),
-                TokenSet.create(KDocTokens.TEXT)
+                TokenSet.create(KDocTokens.TEXT, KDocTokens.CODE_BLOCK_TEXT)
         );
     }
 }

compiler/frontend/src/org/jetbrains/kotlin/kdoc/lexer/KDocTokens.java

@@ -55,6 +55,7 @@ public interface KDocTokens {
     KDocToken LEADING_ASTERISK      = new KDocToken("KDOC_LEADING_ASTERISK");
 
     KDocToken TEXT                  = new KDocToken("KDOC_TEXT");
+    KDocToken CODE_BLOCK_TEXT       = new KDocToken("KDOC_CODE_BLOCK_TEXT");
 
     KDocToken TAG_NAME              = new KDocToken("KDOC_TAG_NAME");
     ILazyParseableElementType MARKDOWN_LINK = new ILazyParseableElementType("KDOC_MARKDOWN_LINK", KotlinLanguage.INSTANCE) {
@@ -67,6 +68,6 @@ public interface KDocTokens {
     KDocToken MARKDOWN_ESCAPED_CHAR = new KDocToken("KDOC_MARKDOWN_ESCAPED_CHAR");
     KDocToken MARKDOWN_INLINE_LINK = new KDocToken("KDOC_MARKDOWN_INLINE_LINK");
 
-    TokenSet KDOC_HIGHLIGHT_TOKENS = TokenSet.create(START, END, LEADING_ASTERISK, TEXT, MARKDOWN_LINK, MARKDOWN_ESCAPED_CHAR, MARKDOWN_INLINE_LINK);
+    TokenSet KDOC_HIGHLIGHT_TOKENS = TokenSet.create(START, END, LEADING_ASTERISK, TEXT, CODE_BLOCK_TEXT, MARKDOWN_LINK, MARKDOWN_ESCAPED_CHAR, MARKDOWN_INLINE_LINK);
-    TokenSet CONTENT_TOKENS = TokenSet.create(TEXT, TAG_NAME, MARKDOWN_LINK, MARKDOWN_ESCAPED_CHAR, MARKDOWN_INLINE_LINK);
+    TokenSet CONTENT_TOKENS = TokenSet.create(TEXT, CODE_BLOCK_TEXT, TAG_NAME, MARKDOWN_LINK, MARKDOWN_ESCAPED_CHAR, MARKDOWN_INLINE_LINK);
 }

compiler/frontend/src/org/jetbrains/kotlin/kdoc/lexer/_KDocLexer.java

@@ -1,4 +1,4 @@
-/* The following code was generated by JFlex 1.4.3 on 4/5/16 7:39 PM */
+/* The following code was generated by JFlex 1.4.3 on 4/28/16 3:03 PM */
 
 package org.jetbrains.kotlin.kdoc.lexer;
 
@@ -13,7 +13,7 @@ import org.jetbrains.kotlin.kdoc.parser.KDocKnownTag;
 /**
  * This class is a scanner generated by 
  * <a href="http://www.jflex.de/">JFlex</a> 1.4.3
- * on 4/5/16 7:39 PM from the specification file
+ * on 4/28/16 3:03 PM from the specification file
  * <tt>/Users/yole/jetbrains/kotlin/compiler/frontend/src/org/jetbrains/kotlin/kdoc/lexer/KDoc.flex</tt>
  */
 class _KDocLexer implements FlexLexer {
@@ -608,14 +608,6 @@ class _KDocLexer implements FlexLexer {
             return KDocTokens.TEXT;
           }
         case 26: break;
-        case 9: 
-          { if (yytextContainLineBreaks()) {
-            yybegin(CODE_BLOCK_LINE_BEGINNING);
-            return TokenType.WHITE_SPACE;
-        }
-        return KDocTokens.TEXT;
-          }
-        case 27: break;
         case 20: 
           // lookahead expression with fixed base length
           zzMarkedPos = zzStartRead + 3;
@@ -623,37 +615,40 @@ class _KDocLexer implements FlexLexer {
         yybegin(CONTENTS);
         return KDocTokens.TEXT;
           }
-        case 28: break;
+        case 27: break;
         case 19: 
           { yybegin(CONTENTS);
         return KDocTokens.MARKDOWN_INLINE_LINK;
           }
-        case 29: break;
+        case 28: break;
         case 5: 
           { yybegin(CONTENTS);
         return KDocTokens.TEXT;
           }
-        case 30: break;
+        case 29: break;
         case 12: 
           { yybegin(CONTENTS);
                    return KDocTokens.MARKDOWN_ESCAPED_CHAR;
           }
-        case 31: break;
+        case 30: break;
         case 16: 
           { yybegin(TAG_TEXT_BEGINNING);
                   return KDocTokens.MARKDOWN_LINK;
           }
-        case 32: break;
+        case 31: break;
         case 4: 
           { yybegin(CONTENTS_BEGINNING);
                                             return KDocTokens.LEADING_ASTERISK;
           }
-        case 33: break;
+        case 32: break;
-        case 8: 
+        case 9: 
-          { yybegin(CODE_BLOCK);
+          { if (yytextContainLineBreaks()) {
-        return KDocTokens.TEXT;
+            yybegin(CODE_BLOCK_LINE_BEGINNING);
+            return TokenType.WHITE_SPACE;
+        }
+        return KDocTokens.CODE_BLOCK_TEXT;
           }
-        case 34: break;
+        case 33: break;
         case 3: 
           { if (yytextContainLineBreaks()) {
             yybegin(LINE_BEGINNING);
@@ -663,35 +658,40 @@ class _KDocLexer implements FlexLexer {
             return KDocTokens.TEXT;  // internal white space
         }
           }
-        case 35: break;
+        case 34: break;
         case 18: 
           // lookahead expression with fixed lookahead length
           yypushback(1);
           { yybegin(CONTENTS);
                   return KDocTokens.MARKDOWN_LINK;
           }
-        case 36: break;
+        case 35: break;
         case 17: 
           { yybegin(CONTENTS);
                   return KDocTokens.MARKDOWN_LINK;
           }
-        case 37: break;
+        case 36: break;
         case 11: 
           { if (isLastToken()) return KDocTokens.END;
                                             else return KDocTokens.TEXT;
           }
-        case 38: break;
+        case 37: break;
         case 7: 
           { yybegin(TAG_TEXT_BEGINNING);
         return KDocTokens.MARKDOWN_LINK;
           }
-        case 39: break;
+        case 38: break;
         case 6: 
           { if (yytextContainLineBreaks()) {
             yybegin(LINE_BEGINNING);
         }
         return TokenType.WHITE_SPACE;
           }
+        case 39: break;
+        case 8: 
+          { yybegin(CODE_BLOCK);
+        return KDocTokens.CODE_BLOCK_TEXT;
+          }
         case 40: break;
         default:
           if (zzInput == YYEOF && zzStartRead == zzCurrentPos) {

compiler/frontend/src/org/jetbrains/kotlin/kdoc/psi/impl/KDocTag.kt

@@ -77,18 +77,42 @@ open class KDocTag(node: ASTNode) : KDocElementImpl(node) {
      */
     open fun getContent(): String {
         val builder = StringBuilder()
+        val codeBlockBuilder = StringBuilder()
+        var targetBuilder = builder
 
         var contentStarted = false
         var afterAsterisk = false
 
+        fun startCodeBlock() {
+            targetBuilder = codeBlockBuilder
+        }
+
+        fun flushCodeBlock() {
+            if (targetBuilder == codeBlockBuilder) {
+                builder.append(trimCommonIndent(codeBlockBuilder))
+                codeBlockBuilder.setLength(0)
+                targetBuilder = builder
+            }
+        }
+
         var children = childrenAfterTagName()
         if (hasSubject(children)) {
             children = children.drop(1)
         }
         for (node in children) {
             val type = node.elementType
+            if (type == KDocTokens.CODE_BLOCK_TEXT) {
+                startCodeBlock()
+            }
+            else if (KDocTokens.CONTENT_TOKENS.contains(type)) {
+                flushCodeBlock()
+            }
+
             if (KDocTokens.CONTENT_TOKENS.contains(type)) {
-                builder.append(if (!contentStarted || afterAsterisk) node.text.trimStart() else node.text)
+                targetBuilder.append(if (!contentStarted || (afterAsterisk && targetBuilder == builder))
+                                         node.text.trimStart()
+                                     else
+                                         node.text)
                 contentStarted = true
                 afterAsterisk = false
             }
@@ -96,13 +120,23 @@ open class KDocTag(node: ASTNode) : KDocElementImpl(node) {
                 afterAsterisk = true
             }
             if (type == TokenType.WHITE_SPACE && contentStarted) {
-                builder.append("\n".repeat(StringUtil.countNewLines(node.text)))
+                targetBuilder.append("\n".repeat(StringUtil.countNewLines(node.text)))
             }
             if (type == KDocElementTypes.KDOC_TAG) {
                 break
             }
         }
 
+        flushCodeBlock()
+
         return builder.toString().trimEnd(' ', '\t')
     }
+
+    private fun trimCommonIndent(builder: StringBuilder): String {
+        val lines = builder.toString().split('\n')
+        val minIndent = lines.filter { it.trim().isNotEmpty() }.map { it.calcIndent() }.min() ?: 0
+        return lines.map { it.drop(minIndent) }.joinToString("\n")
+    }
+
+    fun String.calcIndent() = indexOfFirst { !it.isWhitespace() }
 }

compiler/testData/kdoc/lexer/codeBlocks.txt

@@ -10,10 +10,11 @@ KDOC_LEADING_ASTERISK ('*')
 KDOC_TEXT (' ``` kotlin')
 WHITE_SPACE ('\n ')
 KDOC_LEADING_ASTERISK ('*')
-KDOC_TEXT (' @foo This is code block.')
+KDOC_CODE_BLOCK_TEXT (' @foo This is code block.')
 WHITE_SPACE ('\n ')
 KDOC_LEADING_ASTERISK ('*')
-KDOC_TEXT (' ```')
+KDOC_CODE_BLOCK_TEXT (' ')
+KDOC_TEXT ('```')
 WHITE_SPACE ('\n ')
 KDOC_LEADING_ASTERISK ('*')
 KDOC_TEXT (' ')
@@ -25,13 +26,14 @@ KDOC_LEADING_ASTERISK ('*')
 KDOC_TEXT (' ~~~')
 WHITE_SPACE ('\n ')
 KDOC_LEADING_ASTERISK ('*')
-KDOC_TEXT (' @foo This is code block')
+KDOC_CODE_BLOCK_TEXT (' @foo This is code block')
 WHITE_SPACE ('\n ')
 KDOC_LEADING_ASTERISK ('*')
-KDOC_TEXT (' With multiple lines.')
+KDOC_CODE_BLOCK_TEXT (' With multiple lines.')
 WHITE_SPACE ('\n ')
 KDOC_LEADING_ASTERISK ('*')
-KDOC_TEXT (' ~~~')
+KDOC_CODE_BLOCK_TEXT (' ')
+KDOC_TEXT ('~~~')
 WHITE_SPACE ('\n ')
 KDOC_LEADING_ASTERISK ('*')
 KDOC_TEXT (' ')
@@ -43,6 +45,6 @@ KDOC_LEADING_ASTERISK ('*')
 KDOC_TEXT (' ```')
 WHITE_SPACE ('\n ')
 KDOC_LEADING_ASTERISK ('*')
-KDOC_TEXT (' @foo This is an unclosed code block.')
+KDOC_CODE_BLOCK_TEXT (' @foo This is an unclosed code block.')
 WHITE_SPACE ('\n ')
 KDOC_END ('*/')

compiler/testData/psi/kdoc/AtTags.txt

@@ -19,10 +19,11 @@ JetFile: AtTags.kt
         PsiElement(KDOC_TEXT)(' ```')
         PsiWhiteSpace('\n ')
         PsiElement(KDOC_LEADING_ASTERISK)('*')
-        PsiElement(KDOC_TEXT)(' @notATag and some description')
+        PsiElement(KDOC_CODE_BLOCK_TEXT)(' @notATag and some description')
         PsiWhiteSpace('\n ')
         PsiElement(KDOC_LEADING_ASTERISK)('*')
-        PsiElement(KDOC_TEXT)(' ```')
+        PsiElement(KDOC_CODE_BLOCK_TEXT)(' ')
+        PsiElement(KDOC_TEXT)('```')
         PsiWhiteSpace('\n ')
         PsiElement(KDOC_LEADING_ASTERISK)('*')
         PsiElement(KDOC_TEXT)(' @')

compiler/tests-common/org/jetbrains/kotlin/test/InTextDirectivesUtils.java

@@ -20,6 +20,7 @@ import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 import com.intellij.openapi.util.text.StringUtil;
 import com.intellij.util.ArrayUtil;
+import kotlin.text.StringsKt;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import org.junit.Assert;
@@ -104,6 +105,11 @@ public final class InTextDirectivesUtils {
 
     @NotNull
     public static List<String> findLinesWithPrefixesRemoved(String fileText, String... prefixes) {
+        return findLinesWithPrefixesRemoved(fileText, true, prefixes);
+    }
+
+    @NotNull
+    public static List<String> findLinesWithPrefixesRemoved(String fileText, boolean trim, String... prefixes) {
         List<String> result = new ArrayList<String>();
         List<String> cleanedPrefixes = cleanDirectivesFromComments(Arrays.asList(prefixes));
 
@@ -115,7 +121,7 @@ public final class InTextDirectivesUtils {
                     if (noPrefixLine.isEmpty() ||
                             Character.isWhitespace(noPrefixLine.charAt(0)) ||
                             Character.isWhitespace(prefix.charAt(prefix.length() - 1))) {
-                        result.add(noPrefixLine.trim());
+                        result.add(trim ? noPrefixLine.trim() : StringUtil.trimTrailing(StringsKt.drop(noPrefixLine, 1)));
                         break;
                     } else {
                         throw new AssertionError(

idea/testData/editor/quickDoc/OnMethodUsageWithCodeBlock.kt

@@ -3,7 +3,9 @@
  *
  * ```
  * Code block
- * Second line
+ *     Second line
+ *
+ * Third line
  * ```
  */
 fun testMethod() {
@@ -20,5 +22,7 @@ fun test() {
 //INFO: <b>public</b> <b>fun</b> testMethod(): Unit <i>defined in</i> root package<p>Some documentation.</p>
 //INFO: <pre><code>
 //INFO: Code block
-//INFO: Second line
+//INFO:     Second line
+//INFO:
+//INFO: Third line
 //INFO: </code><pre>

idea/tests/org/jetbrains/kotlin/idea/editor/quickDoc/AbstractQuickDocProviderTest.java

@@ -52,7 +52,7 @@ public abstract class AbstractQuickDocProviderTest extends KotlinLightCodeInsigh
 
         File testDataFile = new File(path);
         String textData = FileUtil.loadFile(testDataFile, true);
-        List<String> directives = InTextDirectivesUtils.findLinesWithPrefixesRemoved(textData, "INFO:");
+        List<String> directives = InTextDirectivesUtils.findLinesWithPrefixesRemoved(textData, false, "INFO:");
 
         if (directives.isEmpty()) {
             throw new FileComparisonFailure(