improvement: add jdk indexing

metals/src/main/resources/shared-db/migration/V1__Jar_type_hierarchy.sql

@@ -0,0 +1,18 @@
+-- Indexed jars, the MD5 digest of path, modified time and size as key
+create table indexed_jar(
+  id int auto_increment unique,
+  md5 varchar primary key,
+  type_hierarchy_indexed bit
+);
+
+-- Type hierarchy information, e.g. symbol: "a/MyException#", extended_name: "Exception"
+create table type_hierarchy(
+  symbol varchar not null,
+  parent_name varchar not null,
+  path varchar not null,
+  jar int,
+  is_resolved bit,
+  foreign key (jar) references indexed_jar (id) on delete cascade
+);
+
+create index type_hierarchy_jar on type_hierarchy(jar);

metals/src/main/scala/scala/meta/internal/metals/H2ConnectionProvider.scala

@@ -0,0 +1,194 @@
+package scala.meta.internal.metals
+
+import java.nio.file.Files
+import java.sql.Connection
+import java.sql.DriverManager
+import java.sql.SQLException
+import java.util.Properties
+import java.util.concurrent.atomic.AtomicReference
+
+import scala.util.Try
+import scala.util.control.NonFatal
+
+import scala.meta.internal.metals.MetalsEnrichments._
+import scala.meta.internal.metals.Tables.ConnectionState
+import scala.meta.internal.pc.InterruptException
+import scala.meta.io.AbsolutePath
+
+import org.flywaydb.core.Flyway
+import org.flywaydb.core.api.FlywayException
+import org.h2.mvstore.DataUtils
+import org.h2.mvstore.MVStoreException
+import org.h2.tools.Upgrade
+
+abstract class H2ConnectionProvider(
+    directory: => AbsolutePath,
+    name: String,
+    migrations: String,
+) extends Cancelable {
+
+  protected val ref: AtomicReference[ConnectionState] =
+    new AtomicReference(ConnectionState.Empty)
+  private val user = "sa"
+
+  protected def connection: Connection = connect()
+
+  protected def optDirectory: Option[AbsolutePath] = Try(directory).toOption
+  protected def databasePath: Option[AbsolutePath] =
+    optDirectory.map(_.resolve("metals.h2.db"))
+
+  def connect(): Connection = {
+    ref.get() match {
+      case empty @ ConnectionState.Empty =>
+        if (ref.compareAndSet(empty, ConnectionState.InProgress)) {
+          val conn = tryAutoServer()
+          ref.set(ConnectionState.Connected(conn))
+          conn
+        } else
+          connect()
+      case Tables.ConnectionState.InProgress =>
+        Thread.sleep(100)
+        connect()
+      case Tables.ConnectionState.Connected(conn) =>
+        conn
+    }
+  }
+
+  // The try/catch dodge-ball court in these methods is not glamorous, I'm sure it can be refactored for more
+  // readability and extensibility but it seems to get the job done for now. The most important goals are:
+  // 1. Never fail to establish a connection, even if that means using an in-memory database with degraded UX.
+  // 2. Log helpful error message with actionable advice on how to fix the problem.
+  private def tryAutoServer(): Connection = {
+    try persistentConnection(isAutoServer = true)
+    catch {
+      case NonFatal(e) =>
+        val message =
+          s"unable to setup persistent H2 database with AUTO_SERVER=true, falling back to AUTO_SERVER=false."
+        e match {
+          case InterruptException() =>
+            scribe.info(message)
+          case _ =>
+            scribe.error(e)
+        }
+        tryNoAutoServer()
+    }
+  }
+
+  protected def tryNoAutoServer(): Connection = {
+    try {
+      persistentConnection(isAutoServer = false)
+    } catch {
+      case NonFatal(e) =>
+        scribe.error(e)
+        inMemoryConnection()
+    }
+  }
+
+  protected def inMemoryConnection(): Connection = {
+    tryUrl(s"jdbc:h2:mem:${name};DB_CLOSE_DELAY=-1")
+  }
+
+  protected def persistentConnection(isAutoServer: Boolean): Connection = {
+    val autoServer =
+      if (isAutoServer) ";AUTO_SERVER=TRUE"
+      else ""
+    val dbfile = directory.resolve("metals")
+    // from "h2" % "2.0.206" the only option is the MVStore, which uses `metals.mv.db` file
+    val oldDbfile = directory.resolve("metals.h2.db")
+    if (oldDbfile.exists) {
+      scribe.info(s"Deleting old database format $oldDbfile")
+      oldDbfile.delete()
+    }
+    Files.createDirectories(dbfile.toNIO.getParent)
+    System.setProperty(
+      "h2.bindAddress",
+      System.getProperty("h2.bindAddress", "127.0.0.1"),
+    )
+    val url = s"jdbc:h2:file:$dbfile$autoServer"
+    upgradeIfNeeded(url)
+    tryUrl(url)
+  }
+
+  private def tryUrl(url: String): Connection = {
+    val flyway =
+      Flyway.configure
+        .dataSource(url, user, null)
+        .locations(s"classpath:$migrations")
+        .cleanDisabled(false)
+        .load()
+    migrateOrRestart(flyway)
+    DriverManager.getConnection(url, user, null)
+  }
+
+  /**
+   * Between h2 "2.1.x" and "2.2.x" write/read formats in MVStore changed
+   * (https://github.com/h2database/h2database/pull/3834)
+   */
+  private def upgradeIfNeeded(url: String): Unit = {
+    val oldVersion = 214
+    val formatVersionChangedMessage =
+      "The write format 2 is smaller than the supported format 3"
+    try {
+      DriverManager.getConnection(url, user, null).close()
+    } catch {
+      case e: SQLException if e.getErrorCode() == 90048 =>
+        e.getCause() match {
+          case e: MVStoreException
+              if e.getErrorCode() == DataUtils.ERROR_UNSUPPORTED_FORMAT &&
+                e.getMessage().startsWith(formatVersionChangedMessage) =>
+            val info: Properties = new Properties()
+            info.put("user", user)
+            try {
+              val didUpgrade = Upgrade.upgrade(url, info, oldVersion)
+              if (didUpgrade) scribe.info(s"Upgraded H2 database.")
+              else deleteDatabase()
+            } catch {
+              case NonFatal(_) => deleteDatabase()
+            }
+
+          case e => throw e
+        }
+    }
+  }
+
+  private def migrateOrRestart(
+      flyway: Flyway
+  ): Unit = {
+    try {
+      flyway.migrate()
+    } catch {
+      case _: FlywayException =>
+        scribe.warn(s"resetting database: $databasePath")
+        flyway.clean()
+        flyway.migrate()
+    }
+  }
+
+  private def deleteDatabase() =
+    optDirectory.foreach { directory =>
+      val dbFile = directory.resolve("metals.mv.db")
+      if (dbFile.exists) {
+        scribe.warn(
+          s"Deleting old database, due to failed database upgrade. Non-default build tool and build server choices will be lost."
+        )
+        dbFile.delete()
+      }
+    }
+
+  def databaseExists(): Boolean =
+    databasePath.exists(_.exists)
+
+  def cancel(): Unit = {
+    ref.get() match {
+      case v @ ConnectionState.Connected(conn) =>
+        if (ref.compareAndSet(v, ConnectionState.Empty)) {
+          conn.close()
+        }
+      case ConnectionState.InProgress =>
+        Thread.sleep(100)
+        cancel()
+      case _ =>
+    }
+  }
+
+}

metals/src/main/scala/scala/meta/internal/metals/Indexer.scala

@@ -15,6 +15,7 @@ import scala.concurrent.Future
 import scala.concurrent.Promise
 import scala.util.control.NonFatal
 
+import scala.meta.Dialect
 import scala.meta.dialects._
 import scala.meta.inputs.Input
 import scala.meta.internal.bsp.BspSession
@@ -78,6 +79,7 @@ final case class Indexer(
     workspaceFolder: AbsolutePath,
     implementationProvider: ImplementationProvider,
     resetService: () => Unit,
+    sharedIndices: SqlSharedIndices,
 )(implicit rc: ReportContext) {
 
   private implicit def ec: ExecutionContextExecutorService = executionContext
@@ -497,7 +499,18 @@ final case class Indexer(
       case Right(zip) =>
         scribe.debug(s"Indexing JDK sources from $zip")
         usedJars += zip
-        definitionIndex.addJDKSources(zip)
+        val dialect = ScalaVersions.dialectForDependencyJar(zip.filename)
+        sharedIndices.jvmTypeHierarchy.getTypeHierarchy(zip) match {
+          case Some(overrides) =>
+            definitionIndex.addIndexedSourceJar(zip, Nil, dialect)
+            implementationProvider.addTypeHierarchyElements(overrides)
+          case None =>
+            val (_, overrides) = indexJar(zip, dialect)
+            sharedIndices.jvmTypeHierarchy.addTypeHierarchyInfo(
+              zip,
+              overrides,
+            )
+        }
       case Left(notFound) =>
         val candidates = notFound.candidates.mkString(", ")
         scribe.warn(
@@ -507,9 +520,9 @@ final case class Indexer(
     for {
       item <- dependencySources.getItems.asScala
     } {
-      jdkSources.foreach(source =>
+      jdkSources.foreach { source =>
         data.addDependencySource(source, item.getTarget)
-      )
+      }
     }
     usedJars.toSet
   }
@@ -612,38 +625,37 @@ final case class Indexer(
    */
   private def addSourceJarSymbols(path: AbsolutePath): Unit = {
     val dialect = ScalaVersions.dialectForDependencyJar(path.filename)
-    def indexJar() = {
-      val indexResult = definitionIndex.addSourceJar(path, dialect)
-      val toplevels = indexResult.flatMap {
-        case IndexingResult(path, toplevels, _) =>
-          toplevels.map((_, path))
-      }
-      val overrides = indexResult.flatMap {
-        case IndexingResult(path, _, list) =>
-          list.flatMap { case (symbol, overridden) =>
-            overridden.map((path, symbol, _))
-          }
-      }
-      implementationProvider.addTypeHierarchyElements(overrides)
-      (toplevels, overrides)
-    }
-
     tables.jarSymbols.getTopLevels(path) match {
       case Some(toplevels) =>
         tables.jarSymbols.getTypeHierarchy(path) match {
           case Some(overrides) =>
             definitionIndex.addIndexedSourceJar(path, toplevels, dialect)
             implementationProvider.addTypeHierarchyElements(overrides)
           case None =>
-            val (_, overrides) = indexJar()
+            val (_, overrides) = indexJar(path, dialect)
             tables.jarSymbols.addTypeHierarchyInfo(path, overrides)
         }
       case None =>
-        val (toplevels, overrides) = indexJar()
+        val (toplevels, overrides) = indexJar(path, dialect)
         tables.jarSymbols.putJarIndexingInfo(path, toplevels, overrides)
     }
   }
 
+  private def indexJar(path: AbsolutePath, dialect: Dialect) = {
+    val indexResult = definitionIndex.addSourceJar(path, dialect)
+    val toplevels = indexResult.flatMap {
+      case IndexingResult(path, toplevels, _) =>
+        toplevels.map((_, path))
+    }
+    val overrides = indexResult.flatMap { case IndexingResult(path, _, list) =>
+      list.flatMap { case (symbol, overridden) =>
+        overridden.map((path, symbol, _))
+      }
+    }
+    implementationProvider.addTypeHierarchyElements(overrides)
+    (toplevels, overrides)
+  }
+
   def reindexWorkspaceSources(
       paths: Seq[AbsolutePath]
   ): Unit = {