Skip to content

Commit

Permalink
improvement: add jdk indexing
Browse files Browse the repository at this point in the history
  • Loading branch information
kasiaMarek committed Jun 5, 2024
1 parent be9ad2e commit 0dea7f6
Show file tree
Hide file tree
Showing 14 changed files with 444 additions and 310 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
-- Indexed jars, the MD5 digest of path, modified time and size as key
create table indexed_jar(
id int auto_increment unique,
md5 varchar primary key,
type_hierarchy_indexed bit
);

-- Type hierarchy information, e.g. symbol: "a/MyException#", extended_name: "Exception"
create table type_hierarchy(
symbol varchar not null,
parent_name varchar not null,
path varchar not null,
jar int,
is_resolved bit,
foreign key (jar) references indexed_jar (id) on delete cascade
);

create index type_hierarchy_jar on type_hierarchy(jar);
194 changes: 194 additions & 0 deletions metals/src/main/scala/scala/meta/internal/metals/H2Connection.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,194 @@
package scala.meta.internal.metals

import java.nio.file.Files
import java.sql.Connection
import java.sql.DriverManager
import java.sql.SQLException
import java.util.Properties
import java.util.concurrent.atomic.AtomicReference

import scala.util.Try
import scala.util.control.NonFatal

import scala.meta.internal.metals.MetalsEnrichments._
import scala.meta.internal.metals.Tables.ConnectionState
import scala.meta.internal.pc.InterruptException
import scala.meta.io.AbsolutePath

import org.flywaydb.core.Flyway
import org.flywaydb.core.api.FlywayException
import org.h2.mvstore.DataUtils
import org.h2.mvstore.MVStoreException
import org.h2.tools.Upgrade

abstract class H2ConnectionProvider(
directory: () => AbsolutePath,
name: String,
migrations: String,
) extends Cancelable {

protected val ref: AtomicReference[ConnectionState] =
new AtomicReference(ConnectionState.Empty)
private val user = "sa"

protected def connection: Connection = connect()

protected def optDirectory: Option[AbsolutePath] = Try(directory()).toOption
protected def databasePath: Option[AbsolutePath] =
optDirectory.map(_.resolve("metals.h2.db"))

def connect(): Connection = {
ref.get() match {
case empty @ ConnectionState.Empty =>
if (ref.compareAndSet(empty, ConnectionState.InProgress)) {
val conn = tryAutoServer()
ref.set(ConnectionState.Connected(conn))
conn
} else
connect()
case Tables.ConnectionState.InProgress =>
Thread.sleep(100)
connect()
case Tables.ConnectionState.Connected(conn) =>
conn
}
}

// The try/catch dodge-ball court in these methods is not glamorous, I'm sure it can be refactored for more
// readability and extensibility but it seems to get the job done for now. The most important goals are:
// 1. Never fail to establish a connection, even if that means using an in-memory database with degraded UX.
// 2. Log helpful error message with actionable advice on how to fix the problem.
private def tryAutoServer(): Connection = {
try persistentConnection(isAutoServer = true)
catch {
case NonFatal(e) =>
val message =
s"unable to setup persistent H2 database with AUTO_SERVER=true, falling back to AUTO_SERVER=false."
e match {
case InterruptException() =>
scribe.info(message)
case _ =>
scribe.error(e)
}
tryNoAutoServer()
}
}

protected def tryNoAutoServer(): Connection = {
try {
persistentConnection(isAutoServer = false)
} catch {
case NonFatal(e) =>
scribe.error(e)
inMemoryConnection()
}
}

protected def inMemoryConnection(): Connection = {
tryUrl(s"jdbc:h2:mem:${name};DB_CLOSE_DELAY=-1")
}

protected def persistentConnection(isAutoServer: Boolean): Connection = {
val autoServer =
if (isAutoServer) ";AUTO_SERVER=TRUE"
else ""
val dbfile = directory().resolve("metals")
// from "h2" % "2.0.206" the only option is the MVStore, which uses `metals.mv.db` file
val oldDbfile = directory().resolve("metals.h2.db")
if (oldDbfile.exists) {
scribe.info(s"Deleting old database format $oldDbfile")
oldDbfile.delete()
}
Files.createDirectories(dbfile.toNIO.getParent)
System.setProperty(
"h2.bindAddress",
System.getProperty("h2.bindAddress", "127.0.0.1"),
)
val url = s"jdbc:h2:file:$dbfile$autoServer"
upgradeIfNeeded(url)
tryUrl(url)
}

private def tryUrl(url: String): Connection = {
val flyway =
Flyway.configure
.dataSource(url, user, null)
.locations(s"classpath:$migrations")
.cleanDisabled(false)
.load()
migrateOrRestart(flyway)
DriverManager.getConnection(url, user, null)
}

/**
* Between h2 "2.1.x" and "2.2.x" write/read formats in MVStore changed
* (https://github.com/h2database/h2database/pull/3834)
*/
private def upgradeIfNeeded(url: String): Unit = {
val oldVersion = 214
val formatVersionChangedMessage =
"The write format 2 is smaller than the supported format 3"
try {
DriverManager.getConnection(url, user, null).close()
} catch {
case e: SQLException if e.getErrorCode() == 90048 =>
e.getCause() match {
case e: MVStoreException
if e.getErrorCode() == DataUtils.ERROR_UNSUPPORTED_FORMAT &&
e.getMessage().startsWith(formatVersionChangedMessage) =>
val info: Properties = new Properties()
info.put("user", user)
try {
val didUpgrade = Upgrade.upgrade(url, info, oldVersion)
if (didUpgrade) scribe.info(s"Upgraded H2 database.")
else deleteDatabase()
} catch {
case NonFatal(_) => deleteDatabase()
}

case e => throw e
}
}
}

private def migrateOrRestart(
flyway: Flyway
): Unit = {
try {
flyway.migrate()
} catch {
case _: FlywayException =>
scribe.warn(s"resetting database: $databasePath")
flyway.clean()
flyway.migrate()
}
}

private def deleteDatabase() =
optDirectory.foreach { directory =>
val dbFile = directory.resolve("metals.mv.db")
if (dbFile.exists) {
scribe.warn(
s"Deleting old database, due to failed database upgrade. Non-default build tool and build server choices will be lost."
)
dbFile.delete()
}
}

def databaseExists(): Boolean =
databasePath.exists(_.exists)

def cancel(): Unit = {
ref.get() match {
case v @ ConnectionState.Connected(conn) =>
if (ref.compareAndSet(v, ConnectionState.Empty)) {
conn.close()
}
case ConnectionState.InProgress =>
Thread.sleep(100)
cancel()
case _ =>
}
}

}
54 changes: 33 additions & 21 deletions metals/src/main/scala/scala/meta/internal/metals/Indexer.scala
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import scala.concurrent.Future
import scala.concurrent.Promise
import scala.util.control.NonFatal

import scala.meta.Dialect
import scala.meta.dialects._
import scala.meta.inputs.Input
import scala.meta.internal.bsp.BspSession
Expand Down Expand Up @@ -78,6 +79,7 @@ final case class Indexer(
workspaceFolder: AbsolutePath,
implementationProvider: ImplementationProvider,
resetService: () => Unit,
sharedIndices: SqlSharedIndices,
)(implicit rc: ReportContext) {

private implicit def ec: ExecutionContextExecutorService = executionContext
Expand Down Expand Up @@ -494,7 +496,18 @@ final case class Indexer(
case Right(zip) =>
scribe.debug(s"Indexing JDK sources from $zip")
usedJars += zip
definitionIndex.addJDKSources(zip)
val dialect = ScalaVersions.dialectForDependencyJar(zip.filename)
sharedIndices.jvmTypeHierarchy.getTypeHierarchy(zip) match {
case Some(overrides) =>
definitionIndex.addIndexedSourceJar(zip, Nil, dialect)
implementationProvider.addTypeHierarchyElements(overrides)
case None =>
val (_, overrides) = indexJar(zip, dialect)
sharedIndices.jvmTypeHierarchy.addTypeHierarchyInfo(
zip,
overrides,
)
}
case Left(notFound) =>
val candidates = notFound.candidates.mkString(", ")
scribe.warn(
Expand All @@ -504,9 +517,9 @@ final case class Indexer(
for {
item <- dependencySources.getItems.asScala
} {
jdkSources.foreach(source =>
jdkSources.foreach { source =>
data.addDependencySource(source, item.getTarget)
)
}
}
usedJars.toSet
}
Expand Down Expand Up @@ -609,38 +622,37 @@ final case class Indexer(
*/
private def addSourceJarSymbols(path: AbsolutePath): Unit = {
val dialect = ScalaVersions.dialectForDependencyJar(path.filename)
def indexJar() = {
val indexResult = definitionIndex.addSourceJar(path, dialect)
val toplevels = indexResult.flatMap {
case IndexingResult(path, toplevels, _) =>
toplevels.map((_, path))
}
val overrides = indexResult.flatMap {
case IndexingResult(path, _, list) =>
list.flatMap { case (symbol, overridden) =>
overridden.map((path, symbol, _))
}
}
implementationProvider.addTypeHierarchyElements(overrides)
(toplevels, overrides)
}

tables.jarSymbols.getTopLevels(path) match {
case Some(toplevels) =>
tables.jarSymbols.getTypeHierarchy(path) match {
case Some(overrides) =>
definitionIndex.addIndexedSourceJar(path, toplevels, dialect)
implementationProvider.addTypeHierarchyElements(overrides)
case None =>
val (_, overrides) = indexJar()
val (_, overrides) = indexJar(path, dialect)
tables.jarSymbols.addTypeHierarchyInfo(path, overrides)
}
case None =>
val (toplevels, overrides) = indexJar()
val (toplevels, overrides) = indexJar(path, dialect)
tables.jarSymbols.putJarIndexingInfo(path, toplevels, overrides)
}
}

private def indexJar(path: AbsolutePath, dialect: Dialect) = {
val indexResult = definitionIndex.addSourceJar(path, dialect)
val toplevels = indexResult.flatMap {
case IndexingResult(path, toplevels, _) =>
toplevels.map((_, path))
}
val overrides = indexResult.flatMap { case IndexingResult(path, _, list) =>
list.flatMap { case (symbol, overridden) =>
overridden.map((path, symbol, _))
}
}
implementationProvider.addTypeHierarchyElements(overrides)
(toplevels, overrides)
}

def reindexWorkspaceSources(
paths: Seq[AbsolutePath]
): Unit = {
Expand Down
Loading

0 comments on commit 0dea7f6

Please sign in to comment.