MySQL

build.gradle

@@ -11,7 +11,9 @@ plugins {
 }
 
 configurations {
-    testImplementation.extendsFrom testShadow
+	jdbcDrivers
+	compileOnly.extendsFrom jdbcDrivers
+	testImplementation.extendsFrom testShadow, shadow, jdbcDrivers
 }
 
 allprojects {
@@ -31,6 +33,11 @@ allprojects {
 
 dependencies {
 	shadow group: 'org.bstats', name: 'bstats-bukkit', version: '3.2.1'
+	// Provided by Paper. Compile and test against the versions in Paper 1.21.4.
+	jdbcDrivers('com.mysql:mysql-connector-j:9.1.0') {
+		exclude group: 'com.google.protobuf', module: 'protobuf-java' // X DevAPI is unused
+	}
+	jdbcDrivers 'org.xerial:sqlite-jdbc:3.47.0.0'
 
 	implementation group: 'io.papermc.paper', name: 'paper-api', version: '26.2.build.+'
 	implementation group: 'com.google.code.findbugs', name: 'findbugs', version: '3.0.1'
@@ -96,6 +103,21 @@ test {
 	exclude '**/*'
 }
 
+tasks.register('variableStorageTest', Test) {
+	description = 'Runs variable storage unit tests without a Minecraft or MySQL server.'
+	testClassesDirs = sourceSets.test.output.classesDirs
+	classpath = sourceSets.test.runtimeClasspath
+	include 'ch/njol/skript/variables/*Test.class'
+}
+
+// Run separately from quickTest: release resource tasks share the main output directory.
+tasks.register('jdbcPackagingCheck', JavaExec) {
+	dependsOn jar, compileTestJava
+	classpath = sourceSets.test.output + configurations.jdbcDrivers
+	mainClass = 'ch.njol.skript.variables.JdbcPackagingCheck'
+	args tasks.jar.archiveFile.get().asFile.absolutePath
+}
+
 tasks.register('sourceJar', Jar) {
 	from sourceSets.main.allJava
 	archiveClassifier = 'sources'
@@ -110,6 +132,9 @@ tasks.withType(ShadowJar).configureEach {
 		include(dependency('org.bstats:bstats-base'))
 	}
 	relocate 'org.bstats', 'ch.njol.skript.bstats'
+	exclude 'META-INF/services/java.sql.Driver'
+	exclude 'META-INF/*.SF', 'META-INF/*.RSA', 'META-INF/*.DSA'
+	exclude 'module-info.class', 'META-INF/versions/**/module-info.class'
 	manifest {
 		attributes(
 			'Name': 'ch/njol/skript',

src/main/java/ch/njol/skript/variables/JdbcDatabase.java

@@ -0,0 +1,93 @@
+package ch.njol.skript.variables;
+
+import ch.njol.skript.Skript;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+/**
+	- Handles the JDBC connection used by {@link SQLStorage} and its adapters.
+
+	- {@link ConnectionFactory} handles the database-specific connection setup,
+	so {@link SQLiteStorage} can use the connection
+	and statement code without needing another database plugin.
+
+	- This class opens connections, prepares statements, and cleans up resources.
+	- It does not handle serializers, variable routing, commits, or retries.
+	- Those are handled by the storage class using it.
+
+	- Statements and results returned by this class must be closed by the caller.
+
+	- This class does not handle synchronization itself. {@link SQLStorage} handles
+	access using its database lock. {@link PooledMySQLStorage} works differently
+	and uses {@link MySQLConnectionPool} because its worker has its own connection
+	and recovery setup.
+*/
+
+public final class JdbcDatabase implements AutoCloseable {
+
+	@FunctionalInterface
+	public interface ConnectionFactory {
+		Connection open() throws SQLException;
+	}
+
+	private final ConnectionFactory factory;
+	private Connection connection;
+
+	public JdbcDatabase(ConnectionFactory factory) {
+		this.factory = factory;
+	}
+
+	public boolean open() {
+		try {
+			if (connection != null && !connection.isClosed())
+				return true;
+			connection = factory.open();
+			return true;
+		} catch (SQLException e) {
+			return false;
+		}
+	}
+
+	public Connection getConnection() throws SQLException {
+		if (connection == null || connection.isClosed())
+			throw new SQLException("Database connection is closed");
+		return connection;
+	}
+
+	public PreparedStatement prepare(String sql) throws SQLException {
+		PreparedStatement statement = getConnection().prepareStatement(sql);
+		statement.setQueryTimeout(10);
+		return statement;
+	}
+
+	public ResultSet query(String sql) throws SQLException {
+		PreparedStatement statement = prepare(sql);
+		try {
+			if (statement.execute()) {
+				statement.closeOnCompletion();
+				return statement.getResultSet();
+			}
+			statement.close();
+			return null;
+		} catch (SQLException e) {
+			statement.close();
+			throw e;
+		}
+	}
+
+	@Override
+	public void close() {
+		if (connection == null)
+			return;
+		try {
+			connection.close();
+		} catch (SQLException e) {
+			Skript.warning("Could not close a variable database connection");
+		} finally {
+			connection = null;
+		}
+	}
+}

src/main/java/ch/njol/skript/variables/MySQLConnectionPool.java

@@ -0,0 +1,84 @@
+package ch.njol.skript.variables;
+
+import javax.sql.ConnectionPoolDataSource;
+import javax.sql.PooledConnection;
+import java.sql.Connection;
+import java.sql.SQLException;
+
+/**
+	- Manages the connection used by {@link PooledMySQLStorage}.
+
+	- The pool gives us a connection we can reuse and replaces it if it breaks,
+	which can happen pretty often with MySQL.
+
+	- Everything is done one at a time. The backend only has one database worker,
+	so we don't need multiple connections handling transactions at the same time.
+
+	- The pool is initialized before the worker starts using it. Callers close their
+	connection handles after they're done, and the backend closes the pool when
+	it shuts down. This class does not handle concurrent access itself.
+
+	- {@link MySQLStorage} exposes this backend through the databases configuration.
+	The pool applies the driver, encoding, timeouts and TLS settings.
+*/
+
+final class MySQLConnectionPool implements AutoCloseable {
+
+	private final ConnectionPoolDataSource source;
+	private PooledConnection pooled;
+
+	MySQLConnectionPool(ConnectionPoolDataSource source) {
+		this.source = source;
+	}
+
+	MySQLConnectionPool(String host, int port, String database,
+			String user, String password, String sslMode) throws SQLException {
+		source = dataSource(host, port, database, user, password, sslMode);
+	}
+
+	static com.mysql.cj.jdbc.MysqlConnectionPoolDataSource dataSource(String host, int port,
+			String database, String user, String password, String sslMode) throws SQLException {
+		var source = new com.mysql.cj.jdbc.MysqlConnectionPoolDataSource();
+		source.setServerName(host);
+		source.setPortNumber(port);
+		source.setDatabaseName(database);
+		source.setUser(user);
+		source.setPassword(password);
+		source.setConnectTimeout(5000);
+		source.setSocketTimeout(5000);
+		source.setSslMode(sslMode);
+		source.setCharacterEncoding("UTF-8");
+		// Paper owns the shared driver; Skript only closes its own connections.
+		return source;
+	}
+
+	Connection acquire() throws SQLException {
+		if (pooled == null)
+			pooled = source.getPooledConnection();
+		try {
+			Connection connection = pooled.getConnection();
+			if (!connection.isValid(5)) {
+				connection.close();
+				throw new SQLException("MySQL connection validation failed");
+			}
+			return connection;
+		} catch (SQLException e) {
+			invalidate();
+			throw e;
+		}
+	}
+
+	void invalidate() {
+		if (pooled != null) {
+			try {
+				pooled.close();
+			} catch (SQLException ignored) {}
+			pooled = null;
+		}
+	}
+
+	@Override
+	public void close() {
+		invalidate();
+	}
+}

src/main/java/ch/njol/skript/variables/MySQLJournal.java

@@ -0,0 +1,128 @@
+package ch.njol.skript.variables;
+
+import java.io.*;
+import java.nio.channels.FileChannel;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.*;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.zip.CRC32;
+import java.util.zip.CheckedInputStream;
+import java.util.zip.CheckedOutputStream;
+
+/**
+	- Stores a copy of changes that still need to be saved to MySQL.
+
+	- Before sending changes to MySQL, {@link PooledMySQLStorage} saves them here.
+	The copy is removed only after the database transaction succeeds, so the changes
+	can be tried again if the connection fails or something goes wrong after the commit.
+
+	- The snapshot contains the changes, which database they belong to, and a checksum
+	to make sure the data is not corrupted. The database identity also stops us from
+	replaying the changes into the wrong database.
+
+	- This is only for recovering changes that haven't been fully saved yet. It is not
+	a database backup, migration tool, or another way of serializing values.
+
+	- On startup, the snapshot is loaded and applied on top of the data read from MySQL.
+	After that, the backend worker handles the journal. If MySQL is still unavailable
+	when the server shuts down normally, the pending changes are kept for later.
+
+	- The journal is not thread-safe. Changes that only existed in memory when the
+	server suddenly crashed cannot be recovered.
+*/
+
+final class MySQLJournal {
+
+	private static final int MAGIC = 0x534B4D31;
+	private final Path path;
+	private final String target;
+
+	MySQLJournal(Path path, String target) {
+		this.path = path;
+		this.target = target;
+	}
+
+	Map<String, SerializedVariable> read() throws IOException {
+		Map<String, SerializedVariable> pending = new LinkedHashMap<>();
+		if (!Files.exists(path))
+			return pending;
+		CRC32 checksum = new CRC32();
+		try (DataInputStream in = new DataInputStream(
+				new CheckedInputStream(new BufferedInputStream(Files.newInputStream(path)), checksum))) {
+			if (in.readInt() != MAGIC)
+				throw new IOException("Unknown MySQL recovery format");
+			String savedTarget = readString(in);
+			int count = in.readInt();
+			if (count < 0)
+				throw new IOException("Invalid MySQL recovery record count");
+			if (count > 0 && !target.equals(savedTarget))
+				throw new IOException("MySQL recovery file belongs to a different database");
+			for (int i = 0; i < count; i++) {
+				String name = readString(in);
+				SerializedVariable.Value value = null;
+				if (in.readBoolean())
+					value = new SerializedVariable.Value(readString(in), readBytes(in));
+				pending.put(name, new SerializedVariable(name, value));
+			}
+			long actualChecksum = checksum.getValue();
+			if (in.readLong() != actualChecksum)
+				throw new IOException("MySQL recovery checksum mismatch");
+			if (in.read() != -1)
+				throw new IOException("Trailing data in MySQL recovery file");
+		}
+		return pending;
+	}
+
+	void write(Map<String, SerializedVariable> pending) throws IOException {
+		Path temporary = path.resolveSibling(path.getFileName() + ".tmp");
+		CRC32 checksum = new CRC32();
+		try (FileOutputStream file = new FileOutputStream(temporary.toFile());
+				DataOutputStream out = new DataOutputStream(
+						new CheckedOutputStream(new BufferedOutputStream(file), checksum))) {
+			out.writeInt(MAGIC);
+			writeString(out, target);
+			out.writeInt(pending.size());
+			for (SerializedVariable variable : pending.values()) {
+				writeString(out, variable.name);
+				out.writeBoolean(variable.value != null);
+				if (variable.value != null) {
+					writeString(out, variable.value.type);
+					out.writeInt(variable.value.data.length);
+					out.write(variable.value.data);
+				}
+			}
+			out.writeLong(checksum.getValue());
+			out.flush();
+			file.getFD().sync();
+		}
+		// Do not replace the last valid snapshot on filesystems without atomic rename.
+		Files.move(temporary, path, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
+		try (FileChannel directory = FileChannel.open(path.toAbsolutePath().getParent(), StandardOpenOption.READ)) {
+			directory.force(true);
+		} catch (IOException | UnsupportedOperationException ignored) {
+			// Directory fsync is not supported on all platforms.
+		}
+	}
+
+	private static void writeString(DataOutputStream out, String value) throws IOException {
+		byte[] bytes = value.getBytes(StandardCharsets.UTF_8);
+		out.writeInt(bytes.length);
+		out.write(bytes);
+	}
+
+	private static String readString(DataInputStream in) throws IOException {
+		return new String(readBytes(in), StandardCharsets.UTF_8);
+	}
+
+	private static byte[] readBytes(DataInputStream in) throws IOException {
+		int length = in.readInt();
+		if (length < 0)
+			throw new IOException("Invalid MySQL recovery value length");
+		// Stream instead of allocating an untrusted length before checking for truncation.
+		byte[] bytes = in.readNBytes(length);
+		if (bytes.length != length)
+			throw new EOFException("Truncated MySQL recovery value");
+		return bytes;
+	}
+}