Install jansi only when colors are enabled, add test

Also minor cleanup. Remove the comment about the issue jansi#35 because
although the issue is fixed, the behavior is correct right now: we
enable colors by default iff stderr is a TTY (and the platform is not
Windows), and to determine that we need to call `CLibrary.isatty`.

 #KT-55784

compiler/cli/src/org/jetbrains/kotlin/cli/common/CLITool.kt

@@ -58,8 +58,8 @@ abstract class CLITool<A : CommonToolArguments> {
         val collector = PrintingMessageCollector(errStream, messageRenderer, arguments.verbose)
 
         try {
-            if (PlainTextMessageRenderer.COLOR_ENABLED) {
-                AnsiConsole.systemInstall()
+            if (messageRenderer is PlainTextMessageRenderer) {
+                messageRenderer.enableColorsIfNeeded()
             }
 
             errStream.print(messageRenderer.renderPreamble())
@@ -80,8 +80,8 @@ abstract class CLITool<A : CommonToolArguments> {
         } finally {
             errStream.print(messageRenderer.renderConclusion())
 
-            if (PlainTextMessageRenderer.COLOR_ENABLED) {
-                AnsiConsole.systemUninstall()
+            if (messageRenderer is PlainTextMessageRenderer) {
+                messageRenderer.disableColorsIfNeeded()
             }
         }
     }

compiler/cli/src/org/jetbrains/kotlin/cli/common/messages/PlainTextMessageRenderer.java

@@ -18,6 +18,7 @@ package org.jetbrains.kotlin.cli.common.messages;
 
 import kotlin.text.StringsKt;
 import org.fusesource.jansi.Ansi;
+import org.fusesource.jansi.AnsiConsole;
 import org.fusesource.jansi.internal.CLibrary;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
@@ -31,21 +32,19 @@ import java.util.Set;
 import static org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.*;
 
 public abstract class PlainTextMessageRenderer implements MessageRenderer {
-    public static final boolean COLOR_ENABLED;
+    private static final boolean IS_STDERR_A_TTY;
 
     static {
-        boolean colorEnabled = false;
+        boolean isStderrATty = false;
         // TODO: investigate why ANSI escape codes on Windows only work in REPL for some reason
         if (!PropertiesKt.isWindows() && "true".equals(CompilerSystemProperties.KOTLIN_COLORS_ENABLED_PROPERTY.getValue())) {
             try {
-                // AnsiConsole doesn't check isatty() for stderr (see https://github.com/fusesource/jansi/pull/35).
-                colorEnabled = CLibrary.isatty(CLibrary.STDERR_FILENO) != 0;
+                isStderrATty = CLibrary.isatty(CLibrary.STDERR_FILENO) != 0;
             }
-            catch (UnsatisfiedLinkError e) {
-                colorEnabled = false;
+            catch (UnsatisfiedLinkError ignored) {
             }
         }
-        COLOR_ENABLED = colorEnabled;
+        IS_STDERR_A_TTY = isStderrATty;
     }
 
     private static final String LINE_SEPARATOR = System.lineSeparator();
@@ -55,11 +54,10 @@ public abstract class PlainTextMessageRenderer implements MessageRenderer {
     private final boolean colorEnabled;
 
     public PlainTextMessageRenderer() {
-        this(COLOR_ENABLED);
+        this(IS_STDERR_A_TTY);
     }
 
-    // This constructor is not used in this project
-    // but it can be useful in a compilation server to still be able to generate colored output
+    // This constructor can be used in a compilation server to still be able to generate colored output, even if stderr is not a TTY.
     @SuppressWarnings("WeakerAccess")
     public PlainTextMessageRenderer(boolean colorEnabled) {
         this.colorEnabled = colorEnabled;
@@ -180,4 +178,16 @@ public abstract class PlainTextMessageRenderer implements MessageRenderer {
     public String renderConclusion() {
         return "";
     }
+
+    public void enableColorsIfNeeded() {
+        if (colorEnabled) {
+            AnsiConsole.systemInstall();
+        }
+    }
+
+    public void disableColorsIfNeeded() {
+        if (colorEnabled) {
+            AnsiConsole.systemUninstall();
+        }
+    }
 }

compiler/tests/org/jetbrains/kotlin/integration/ColorsTest.kt

@@ -0,0 +1,94 @@
+/*
+ * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.integration
+
+import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys
+import org.jetbrains.kotlin.cli.common.config.addKotlinSourceRoot
+import org.jetbrains.kotlin.cli.common.isWindows
+import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSourceLocation
+import org.jetbrains.kotlin.cli.common.messages.MessageRenderer
+import org.jetbrains.kotlin.cli.common.messages.PlainTextMessageRenderer
+import org.jetbrains.kotlin.cli.common.messages.PrintingMessageCollector
+import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles
+import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
+import org.jetbrains.kotlin.cli.jvm.compiler.KotlinToJVMBytecodeCompiler
+import org.jetbrains.kotlin.config.JVMConfigurationKeys
+import org.jetbrains.kotlin.test.ConfigurationKind
+import org.jetbrains.kotlin.test.KotlinTestUtils
+import org.jetbrains.kotlin.test.TestCaseWithTmpdir
+import org.jetbrains.kotlin.test.TestJdkKind
+import java.io.ByteArrayOutputStream
+import java.io.File
+import java.io.PrintStream
+
+// This test checks that the compiler outputs ANSI escape codes enabling colors, bold text, etc when outputting warnings/errors.
+// By default, the compiler does so on non-Windows platforms only if the output is a terminal (isatty returns 1 for stderr),
+// but you can also pass a custom instance of PlainTextMessageRenderer and override that parameter.
+class ColorsTest : TestCaseWithTmpdir() {
+    fun testColorsDisabledByDefault() {
+        doTest(MessageRenderer.WITHOUT_PATHS, false)
+    }
+
+    fun testColorsDisabledWithDefaultConstructor() {
+        doTest(CustomRenderer(), false)
+    }
+
+    fun testColorsEnabledCustom() {
+        doTest(CustomRenderer(true), true)
+    }
+
+    fun testColorsDisabledCustom() {
+        doTest(CustomRenderer(false), false)
+    }
+
+    private fun doTest(renderer: MessageRenderer, colorsShouldBeEnabled: Boolean) {
+        // Colors are currently disabled on Windows.
+        if (isWindows) return
+
+        // Create a source file which yields exactly one error when being compiled.
+        File(tmpdir, "source.kt").writeText("val result: String = 42")
+
+        val log = ByteArrayOutputStream()
+
+        val configuration = KotlinTestUtils.newConfiguration(ConfigurationKind.ALL, TestJdkKind.FULL_JDK).apply {
+            put(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY, PrintingMessageCollector(PrintStream(log), renderer, false))
+            addKotlinSourceRoot(tmpdir.absolutePath)
+            put(JVMConfigurationKeys.OUTPUT_DIRECTORY, tmpdir)
+        }
+
+        val environment = KotlinCoreEnvironment.createForTests(testRootDisposable, configuration, EnvironmentConfigFiles.JVM_CONFIG_FILES)
+
+        // Compilation should return false, because there's one error.
+        assertFalse(KotlinToJVMBytecodeCompiler.compileBunchOfSources(environment))
+
+        val firstBytes = log.toByteArray().take(7)
+        val logStartsWithColors = firstBytes.joinToString(" ") { it.toString(16) } == "1b 5b 31 3b 33 31 6d"
+        val logStartsWithWordError = firstBytes == "error: ".map { it.code.toByte() }
+
+        when {
+            logStartsWithColors -> if (!colorsShouldBeEnabled) {
+                fail("There should be no colors in the compiler log, but it seems that there are.")
+            }
+            logStartsWithWordError -> if (colorsShouldBeEnabled) {
+                fail("There should be colors in the compiler log, but there aren't any.")
+            }
+            else -> {
+                fail("The compiler log starts with something unexpected. Possibly the test needs to be updated.")
+            }
+        }
+    }
+
+    private class CustomRenderer : PlainTextMessageRenderer {
+        constructor() : super()
+        constructor(colorsShouldBeEnabled: Boolean) : super(colorsShouldBeEnabled)
+
+        override fun getName(): String = "Test"
+
+        // Do not output paths, so that the log will start with the word "error", so that we can investigate just the first few bytes
+        // of the log and see if it's the color enabling ANSI codes, or the word "error".
+        override fun getPath(location: CompilerMessageSourceLocation): String? = null
+    }
+}