Skip to content

Commit

Permalink
improvement: monitor build server liveness
Browse files Browse the repository at this point in the history
  • Loading branch information
kasiaMarek authored and tgodzik committed Aug 3, 2023
1 parent e798ea3 commit 3639477
Show file tree
Hide file tree
Showing 10 changed files with 289 additions and 5 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,7 @@ class BuildServerConnection private (
if (isShuttingDown.compareAndSet(false, true)) {
conn.server.buildShutdown().get(2, TimeUnit.SECONDS)
conn.server.onBuildExit()
conn.livenessMonitor.shutdown()
scribe.info("Shut down connection with build server.")
// Cancel pending compilations on our side, this is not needed for Bloop.
cancel()
Expand Down Expand Up @@ -394,6 +395,11 @@ class BuildServerConnection private (
CancelTokens.future(_ => actionFuture)
}

def isBuildServerResponsive: Future[Boolean] = {
val original = connection
original.map(_.livenessMonitor.isBuildServerResponsive)
}

}

object BuildServerConnection {
Expand Down Expand Up @@ -422,13 +428,15 @@ object BuildServerConnection {
def setupServer(): Future[LauncherConnection] = {
connect().map { case conn @ SocketConnection(_, output, input, _, _) =>
val tracePrinter = Trace.setupTracePrinter("BSP", workspace)
val requestMonitor = new RequestMonitor
val launcher = new Launcher.Builder[MetalsBuildServer]()
.traceMessages(tracePrinter.orNull)
.setOutput(output)
.setInput(input)
.setLocalService(localClient)
.setRemoteInterface(classOf[MetalsBuildServer])
.setExecutorService(ec)
.wrapMessages(requestMonitor.wrapper(_))
.create()
val listening = launcher.startListening()
val server = launcher.getRemoteProxy
Expand All @@ -451,6 +459,14 @@ object BuildServerConnection {
stopListening,
result.getVersion(),
result.getCapabilities(),
new ServerLivenessMonitor(
requestMonitor,
server,
languageClient,
result.getDisplayName(),
config.metalsToIdleTime,
config.pingInterval,
),
)
}
}
Expand Down Expand Up @@ -541,6 +557,7 @@ object BuildServerConnection {
cancelServer: Cancelable,
version: String,
capabilities: BuildServerCapabilities,
livenessMonitor: ServerLivenessMonitor,
) {

def cancelables: List[Cancelable] =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -238,7 +238,7 @@ object Messages {
)

def moreInfo: String =
" Select 'More information' to learn how to fix this problem.."
" Select 'More information' to learn how to fix this problem."

def allProjectsMisconfigured: String =
"Navigation will not work for this build due to mis-configuration." + moreInfo
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
package scala.meta.internal.metals

import scala.concurrent.duration.Duration
import scala.util.Try

import scala.meta.internal.metals.Configs._
import scala.meta.internal.pc.PresentationCompilerConfigImpl
import scala.meta.pc.PresentationCompilerConfig.OverrideDefFormat
Expand Down Expand Up @@ -40,6 +43,8 @@ import scala.meta.pc.PresentationCompilerConfig.OverrideDefFormat
* @param macOsMaxWatchRoots The maximum number of root directories to watch on MacOS.
* @param maxLogFileSize The maximum size of the log file before it gets backed up and truncated.
* @param maxLogBackups The maximum number of backup log files.
* @param metalsToIdleTime The time that needs to pass with no action to consider metals as idle.
* @param pingInterval Interval in which we ping the build server.
*/
final case class MetalsServerConfig(
globSyntax: GlobSyntaxConfig = GlobSyntaxConfig.default,
Expand Down Expand Up @@ -102,6 +107,14 @@ final case class MetalsServerConfig(
.withFilter(_.forall(Character.isDigit(_)))
.map(_.toInt)
.getOrElse(10),
metalsToIdleTime: Duration =
Option(System.getProperty("metals.server-to-idle-time"))
.flatMap(opt => Try(Duration(opt)).toOption)
.getOrElse(Duration("10m")),
pingInterval: Duration =
Option(System.getProperty("metals.build-server-ping-interval"))
.flatMap(opt => Try(Duration(opt)).toOption)
.getOrElse(Duration("1m")),
) {
override def toString: String =
List[String](
Expand All @@ -122,6 +135,8 @@ final case class MetalsServerConfig(
s"loglevel=${loglevel}",
s"max-logfile-size=${maxLogFileSize}",
s"max-log-backup=${maxLogBackups}",
s"server-to-idle-time=${metalsToIdleTime}",
s"build-server-ping-interval=${pingInterval}",
).mkString("MetalsServerConfig(\n ", ",\n ", "\n)")
}
object MetalsServerConfig {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
package scala.meta.internal.metals

import java.util.concurrent.Executors
import java.util.concurrent.ScheduledExecutorService
import java.util.concurrent.ScheduledFuture
import java.util.concurrent.TimeUnit

import scala.concurrent.ExecutionContext
import scala.concurrent.duration.Duration

import scala.meta.internal.metals.MetalsEnrichments._

import org.eclipse.lsp4j.MessageActionItem
import org.eclipse.lsp4j.MessageType
import org.eclipse.lsp4j.ShowMessageRequestParams
import org.eclipse.lsp4j.jsonrpc.MessageConsumer
import org.eclipse.lsp4j.jsonrpc.messages.Message
import org.eclipse.lsp4j.jsonrpc.messages.NotificationMessage
import org.eclipse.lsp4j.jsonrpc.messages.RequestMessage
import org.eclipse.lsp4j.jsonrpc.messages.ResponseMessage
import org.eclipse.lsp4j.services.LanguageClient

class RequestMonitor {
@volatile private var lastOutgoing_ : Option[Long] = None
@volatile private var lastIncoming_ : Option[Long] = None

val wrapper: MessageConsumer => MessageConsumer = consumer =>
new MessageConsumer {
def consume(message: Message): Unit = {
message match {
// we don't count the `buildTargets` request, since it's the one used for pinging
case m: RequestMessage if m.getMethod() != "workspace/buildTargets" =>
outgoingMessage()
case _: ResponseMessage => incomingMessage()
case _: NotificationMessage => incomingMessage()
case _ =>
}
consumer.consume(message)
}

}

private def outgoingMessage() = lastOutgoing_ = now
private def incomingMessage(): Unit = lastIncoming_ = now
private def now = Some(System.currentTimeMillis())

def lastOutgoing: Option[Long] = lastOutgoing_
def lastIncoming: Option[Long] = lastIncoming_
}

class ServerLivenessMonitor(
requestMonitor: RequestMonitor,
server: MetalsBuildServer,
languageClient: LanguageClient,
serverName: String,
metalsIdleInterval: Duration,
pingInterval: Duration,
)(implicit ex: ExecutionContext) {
@volatile private var isDismissed = false
@volatile private var isServerResponsive = true
val scheduler: ScheduledExecutorService = Executors.newScheduledThreadPool(1)
val runnable: Runnable = new Runnable {
def run(): Unit = {
def now = System.currentTimeMillis()
def lastIncoming =
requestMonitor.lastIncoming
.map(now - _)
.getOrElse(pingInterval.toMillis)
def notResponding = lastIncoming > (pingInterval.toMillis * 2)
def metalsIsIdle =
requestMonitor.lastOutgoing.exists(lastOutgoing =>
(now - lastOutgoing) > metalsIdleInterval.toMillis
)
if (!metalsIsIdle) {
if (notResponding) {
isServerResponsive = false
if (!isDismissed) {
languageClient
.showMessageRequest(
ServerLivenessMonitor.ServerNotResponding
.params(pingInterval, serverName)
)
.asScala
.map {
case ServerLivenessMonitor.ServerNotResponding.dismiss =>
isDismissed = true
case _ =>
}
}
} else {
isServerResponsive = true
}
server.workspaceBuildTargets()
}
}
}

val scheduled: ScheduledFuture[_ <: Object] =
scheduler.scheduleAtFixedRate(
runnable,
pingInterval.toMillis,
pingInterval.toMillis,
TimeUnit.MILLISECONDS,
)

def isBuildServerResponsive: Boolean = isServerResponsive

def shutdown(): Unit = {
scheduled.cancel(true)
scheduler.shutdown()
}
}

object ServerLivenessMonitor {
object ServerNotResponding {
def message(pingInterval: Duration, serverName: String): String =
s"The build server has not responded in over $pingInterval. You may want to restart $serverName build server."

def params(
pingInterval: Duration,
serverName: String,
): ShowMessageRequestParams = {
val params = new ShowMessageRequestParams()
params.setMessage(message(pingInterval, serverName))
params.setActions(List(dismiss, ok).asJava)
params.setType(MessageType.Warning)
params
}
val dismiss = new MessageActionItem("Dismiss")
val ok = new MessageActionItem("OK")
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,10 @@ import java.nio.charset.StandardCharsets
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicBoolean

import scala.concurrent.Await
import scala.concurrent.ExecutionContext
import scala.concurrent.duration.Duration
import scala.util.Try

import scala.meta.internal.bsp.BspResolvedResult
import scala.meta.internal.bsp.BspSession
Expand Down Expand Up @@ -184,6 +187,7 @@ final class Doctor(
buildToolHeading,
buildServerHeading,
importBuildHeading,
isServerResponsive,
)
if (targetIds.isEmpty) {
DoctorFolderResults(
Expand Down Expand Up @@ -278,6 +282,10 @@ final class Doctor(
)
}

isServerResponsive.withFilter(!_).foreach { _ =>
html.element("p")(_.text(buildServerNotResponsive))
}

val (message, explicitChoice) = selectedBuildServerMessage()

if (explicitChoice) {
Expand Down Expand Up @@ -434,6 +442,12 @@ final class Doctor(
)
}

private def isServerResponsive: Option[Boolean] =
currentBuildServer().flatMap { conn =>
val isResponsiveFuture = conn.main.isBuildServerResponsive
Try(Await.result(isResponsiveFuture, Duration("1s"))).toOption
}

private def extractScalaTargetInfo(
scalaTarget: ScalaTarget,
javaTarget: Option[JavaTarget],
Expand Down Expand Up @@ -513,6 +527,8 @@ final class Doctor(
s"Make sure the workspace directory '$workspace' matches the root of your build."
private val noBuildTargetRecTwo =
"Try removing the directories .metals/ and .bloop/, then restart metals And import the build again."
private val buildServerNotResponsive =
"Build server is not responding."
}

case class DoctorVisibilityDidChangeParams(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,7 @@ final case class DoctorFolderHeader(
buildTool: Option[String],
buildServer: String,
importBuildStatus: Option[String],
isBuildServerResponsive: Option[Boolean],
) {
def toJson: Obj = {
val base =
Expand All @@ -145,6 +146,9 @@ final case class DoctorFolderHeader(

buildTool.foreach { bt => base.update("buildTool", bt) }
importBuildStatus.foreach { ibs => base.update("importBuildStatus", ibs) }
isBuildServerResponsive.foreach { ibsr =>
base.update("isBuildServerResponsive", ibsr)
}
base
}
}
18 changes: 14 additions & 4 deletions tests/unit/src/main/scala/bill/Bill.scala
Original file line number Diff line number Diff line change
Expand Up @@ -69,11 +69,10 @@ import org.eclipse.lsp4j.jsonrpc.Launcher
* - no incremental compilation, every compilation is a clean compile.
*/
object Bill {

val logName = ".bill-metals.log"

class Server() extends BuildServer with ScalaBuildServer {

private var sleepBeforePingResponse: Option[Duration] = None
val languages: util.List[String] = Collections.singletonList("scala")
var client: BuildClient = _
override def onConnectWithClient(server: BuildClient): Unit =
Expand Down Expand Up @@ -191,6 +190,9 @@ object Bill {
}
override def workspaceBuildTargets()
: CompletableFuture[WorkspaceBuildTargetsResult] = {
sleepBeforePingResponse.foreach(duration =>
Thread.sleep(duration.toMillis)
)
CompletableFuture.completedFuture {
new WorkspaceBuildTargetsResult(Collections.singletonList(target))
}
Expand Down Expand Up @@ -374,10 +376,18 @@ object Bill {
}
override def buildTargetScalaTestClasses(
params: ScalaTestClassesParams
): CompletableFuture[ScalaTestClassesResult] = ???
): CompletableFuture[ScalaTestClassesResult] =
Future.successful(new ScalaTestClassesResult(List.empty.asJava)).asJava
override def buildTargetScalaMainClasses(
params: ScalaMainClassesParams
): CompletableFuture[ScalaMainClassesResult] = ???
): CompletableFuture[ScalaMainClassesResult] = {
params.getTargets().asScala.toList match {
case List(break, time) if break.getUri == "break" =>
sleepBeforePingResponse = Try(Duration(time.getUri)).toOption
case _ =>
}
Future.successful(new ScalaMainClassesResult(List.empty.asJava)).asJava
}

override def buildTargetDependencyModules(
params: DependencyModulesParams
Expand Down
9 changes: 9 additions & 0 deletions tests/unit/src/main/scala/tests/BaseLspSuite.scala
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import java.util.concurrent.Executors
import scala.concurrent.ExecutionContext
import scala.concurrent.ExecutionContextExecutorService
import scala.concurrent.Future
import scala.concurrent.duration.Duration
import scala.util.control.NonFatal

import scala.meta.internal.io.PathIO
Expand All @@ -17,6 +18,7 @@ import scala.meta.internal.metals.InitializationOptions
import scala.meta.internal.metals.MetalsServerConfig
import scala.meta.internal.metals.MtagsResolver
import scala.meta.internal.metals.RecursivelyDelete
import scala.meta.internal.metals.ServerLivenessMonitor
import scala.meta.internal.metals.SlowTaskConfig
import scala.meta.internal.metals.Time
import scala.meta.internal.metals.UserConfiguration
Expand Down Expand Up @@ -223,3 +225,10 @@ abstract class BaseLspSuite(
}
}
}

object ServerLivenessTestData {
val serverName = "Bill"
val pingInterval: Duration = Duration("3s")
def serverNotRespondingMessage: String =
ServerLivenessMonitor.ServerNotResponding.message(pingInterval, serverName)
}
Loading

0 comments on commit 3639477

Please sign in to comment.