Changed cached OsuFile loading.
Instead of being loaded individually from the database, OsuFiles are now loaded in batch by traversing the database. In most cases, this should cause a ~10% loading speed improvement. Defaults to the previous approach if few OsuFiles are being loaded. Other changes: - Don't drop indexes upon batch insertion if few entries are being inserted. - Trying to increase fetch size with setFetchSize(100). - Added more parser statuses. Signed-off-by: Jeffrey Han <itdelatrisu@gmail.com>
This commit is contained in:
Jeffrey Han
parent
c6041d8cba
commit
40a800eb76
|
|
@ -31,6 +31,7 @@ import java.util.ArrayList;
|
|||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.newdawn.slick.Color;
|
||||
|
|
@ -55,8 +56,11 @@ public class OsuParser {
|
|||
/** The total number of directories to parse. */
|
||||
private static int totalDirectories = -1;
|
||||
|
||||
/** Whether or not the database is currently being updated. */
|
||||
private static boolean updatingDatabase = false;
|
||||
/** Parser statuses. */
|
||||
public enum Status { NONE, PARSING, CACHE, INSERTING };
|
||||
|
||||
/** The current status. */
|
||||
private static Status status = Status.NONE;
|
||||
|
||||
// This class should not be instantiated.
|
||||
private OsuParser() {}
|
||||
|
|
@ -85,14 +89,19 @@ public class OsuParser {
|
|||
return null;
|
||||
|
||||
// progress tracking
|
||||
status = Status.PARSING;
|
||||
currentDirectoryIndex = 0;
|
||||
totalDirectories = dirs.length;
|
||||
|
||||
// get last modified map from database
|
||||
Map<String, Long> map = OsuDB.getLastModifiedMap();
|
||||
|
||||
// OsuFile lists
|
||||
List<ArrayList<OsuFile>> allOsuFiles = new LinkedList<ArrayList<OsuFile>>();
|
||||
List<OsuFile> cachedOsuFiles = new LinkedList<OsuFile>(); // loaded from database
|
||||
List<OsuFile> parsedOsuFiles = new LinkedList<OsuFile>(); // loaded from parser
|
||||
|
||||
// parse directories
|
||||
LinkedList<OsuFile> parsedOsuFiles = new LinkedList<OsuFile>();
|
||||
OsuGroupNode lastNode = null;
|
||||
for (File dir : dirs) {
|
||||
currentDirectoryIndex++;
|
||||
|
|
@ -120,7 +129,10 @@ public class OsuParser {
|
|||
// check last modified times
|
||||
long lastModified = map.get(path);
|
||||
if (lastModified == file.lastModified()) {
|
||||
osuFiles.add(OsuDB.getOsuFile(dir, file));
|
||||
// add to cached beatmap list
|
||||
OsuFile osu = new OsuFile(file);
|
||||
osuFiles.add(osu);
|
||||
cachedOsuFiles.add(osu);
|
||||
continue;
|
||||
} else
|
||||
OsuDB.delete(dir.getName(), file.getName());
|
||||
|
|
@ -130,13 +142,17 @@ public class OsuParser {
|
|||
// Change boolean to 'true' to parse them immediately.
|
||||
OsuFile osu = parseFile(file, dir, osuFiles, false);
|
||||
|
||||
if (osu != null)
|
||||
// add to parsed beatmap list
|
||||
if (osu != null) {
|
||||
osuFiles.add(osu);
|
||||
parsedOsuFiles.add(osu);
|
||||
}
|
||||
if (!osuFiles.isEmpty()) { // add entry if non-empty
|
||||
}
|
||||
|
||||
// add group entry if non-empty
|
||||
if (!osuFiles.isEmpty()) {
|
||||
osuFiles.trimToSize();
|
||||
Collections.sort(osuFiles);
|
||||
lastNode = OsuGroupList.get().addSongGroup(osuFiles);
|
||||
allOsuFiles.add(osuFiles);
|
||||
}
|
||||
|
||||
// stop parsing files (interrupted)
|
||||
|
|
@ -144,16 +160,28 @@ public class OsuParser {
|
|||
break;
|
||||
}
|
||||
|
||||
// load cached entries from database
|
||||
if (!cachedOsuFiles.isEmpty()) {
|
||||
status = Status.CACHE;
|
||||
OsuDB.load(cachedOsuFiles);
|
||||
}
|
||||
|
||||
// add group entries to OsuGroupList
|
||||
for (ArrayList<OsuFile> osuFiles : allOsuFiles) {
|
||||
Collections.sort(osuFiles);
|
||||
lastNode = OsuGroupList.get().addSongGroup(osuFiles);
|
||||
}
|
||||
|
||||
// clear string DB
|
||||
stringdb = new HashMap<String, String>();
|
||||
|
||||
// add entries to database
|
||||
// add beatmap entries to database
|
||||
if (!parsedOsuFiles.isEmpty()) {
|
||||
updatingDatabase = true;
|
||||
status = Status.INSERTING;
|
||||
OsuDB.insert(parsedOsuFiles);
|
||||
updatingDatabase = false;
|
||||
}
|
||||
|
||||
status = Status.NONE;
|
||||
currentFile = null;
|
||||
currentDirectoryIndex = -1;
|
||||
totalDirectories = -1;
|
||||
|
|
@ -197,7 +225,8 @@ public class OsuParser {
|
|||
if (!osuFiles.isEmpty()) {
|
||||
// if possible, reuse the same File object from another OsuFile in the group
|
||||
File groupAudioFileName = osuFiles.get(0).audioFilename;
|
||||
if (tokens[1].equalsIgnoreCase(groupAudioFileName.getName()))
|
||||
if (groupAudioFileName != null &&
|
||||
tokens[1].equalsIgnoreCase(groupAudioFileName.getName()))
|
||||
audioFileName = groupAudioFileName;
|
||||
}
|
||||
if (!audioFileName.isFile()) {
|
||||
|
|
@ -550,10 +579,6 @@ public class OsuParser {
|
|||
if (parseObjects)
|
||||
parseHitObjects(osu);
|
||||
|
||||
// add OsuFile to song group
|
||||
if (osuFiles != null)
|
||||
osuFiles.add(osu);
|
||||
|
||||
return osu;
|
||||
}
|
||||
|
||||
|
|
@ -666,10 +691,10 @@ public class OsuParser {
|
|||
* Returns the name of the current file being parsed, or null if none.
|
||||
*/
|
||||
public static String getCurrentFileName() {
|
||||
if (updatingDatabase)
|
||||
return "";
|
||||
else
|
||||
if (status == Status.PARSING)
|
||||
return (currentFile != null) ? currentFile.getName() : null;
|
||||
else
|
||||
return (status == Status.NONE) ? null : "";
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -684,9 +709,9 @@ public class OsuParser {
|
|||
}
|
||||
|
||||
/**
|
||||
* Returns whether or not the beatmap database is currently being updated.
|
||||
* Returns the current parser status.
|
||||
*/
|
||||
public static boolean isUpdatingDatabase() { return updatingDatabase; }
|
||||
public static Status getStatus() { return status; }
|
||||
|
||||
/**
|
||||
* Returns the String object in the database for the given String.
|
||||
|
|
|
|||
|
|
@ -409,7 +409,8 @@ public class UI {
|
|||
text = "Unpacking new beatmaps...";
|
||||
progress = OszUnpacker.getUnpackerProgress();
|
||||
} else if ((file = OsuParser.getCurrentFileName()) != null) {
|
||||
text = (OsuParser.isUpdatingDatabase()) ? "Updating database..." : "Loading beatmaps...";
|
||||
text = (OsuParser.getStatus() == OsuParser.Status.INSERTING) ?
|
||||
"Updating database..." : "Loading beatmaps...";
|
||||
progress = OsuParser.getParserProgress();
|
||||
} else if ((file = SoundController.getCurrentFileName()) != null) {
|
||||
text = "Loading sounds...";
|
||||
|
|
|
|||
|
|
@ -43,11 +43,17 @@ public class OsuDB {
|
|||
*/
|
||||
private static final String DATABASE_VERSION = "2014-03-04";
|
||||
|
||||
/** Minimum batch size to invoke batch loading. */
|
||||
private static final int LOAD_BATCH_MIN = 100;
|
||||
|
||||
/** Minimum batch size to invoke batch insertion. */
|
||||
private static final int INSERT_BATCH_MIN = 100;
|
||||
|
||||
/** Database connection. */
|
||||
private static Connection connection;
|
||||
|
||||
/** Query statements. */
|
||||
private static PreparedStatement insertStmt, selectStmt, lastModStmt, deleteMapStmt, deleteGroupStmt;
|
||||
private static PreparedStatement insertStmt, selectStmt, selectAllStmt, lastModStmt, deleteMapStmt, deleteGroupStmt;
|
||||
|
||||
// This class should not be instantiated.
|
||||
private OsuDB() {}
|
||||
|
|
@ -75,6 +81,7 @@ public class OsuDB {
|
|||
"?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
|
||||
);
|
||||
lastModStmt = connection.prepareStatement("SELECT dir, file, lastModified FROM beatmaps");
|
||||
selectAllStmt = connection.prepareStatement("SELECT * FROM beatmaps");
|
||||
selectStmt = connection.prepareStatement("SELECT * FROM beatmaps WHERE dir = ? AND file = ?");
|
||||
deleteMapStmt = connection.prepareStatement("DELETE FROM beatmaps WHERE dir = ? AND file = ?");
|
||||
deleteGroupStmt = connection.prepareStatement("DELETE FROM beatmaps WHERE dir = ?");
|
||||
|
|
@ -191,8 +198,11 @@ public class OsuDB {
|
|||
connection.setAutoCommit(false);
|
||||
|
||||
// drop indexes
|
||||
boolean recreateIndexes = (batch.size() >= INSERT_BATCH_MIN);
|
||||
if (recreateIndexes) {
|
||||
String sql = "DROP INDEX IF EXISTS idx";
|
||||
stmt.executeUpdate(sql);
|
||||
}
|
||||
|
||||
// batch insert
|
||||
for (OsuFile osu : batch) {
|
||||
|
|
@ -202,8 +212,10 @@ public class OsuDB {
|
|||
insertStmt.executeBatch();
|
||||
|
||||
// re-create indexes
|
||||
sql = "CREATE INDEX idx ON beatmaps (dir, file)";
|
||||
if (recreateIndexes) {
|
||||
String sql = "CREATE INDEX idx ON beatmaps (dir, file)";
|
||||
stmt.executeUpdate(sql);
|
||||
}
|
||||
|
||||
// restore previous auto-commit mode
|
||||
connection.commit();
|
||||
|
|
@ -263,21 +275,85 @@ public class OsuDB {
|
|||
}
|
||||
|
||||
/**
|
||||
* Returns an OsuFile from the database, or null if any error occurred.
|
||||
* @param dir the directory
|
||||
* @param file the file
|
||||
* @return the OsuFile with all fields filled, or null if any error occurred
|
||||
* Loads OsuFile fields from the database.
|
||||
* @param osu the OsuFile object
|
||||
*/
|
||||
public static OsuFile getOsuFile(File dir, File file) {
|
||||
public static void load(OsuFile osu) {
|
||||
if (connection == null)
|
||||
return null;
|
||||
return;
|
||||
|
||||
try {
|
||||
OsuFile osu = new OsuFile(file);
|
||||
selectStmt.setString(1, dir.getName());
|
||||
selectStmt.setString(2, file.getName());
|
||||
selectStmt.setString(1, osu.getFile().getParentFile().getName());
|
||||
selectStmt.setString(2, osu.getFile().getName());
|
||||
ResultSet rs = selectStmt.executeQuery();
|
||||
if (rs.next())
|
||||
setOsuFileFields(rs, osu);
|
||||
rs.close();
|
||||
} catch (SQLException e) {
|
||||
ErrorHandler.error("Failed to load OsuFile from database.", e, true);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads OsuFile fields from the database in a batch.
|
||||
* @param batch a list of OsuFile objects
|
||||
*/
|
||||
public static void load(List<OsuFile> batch) {
|
||||
if (connection == null)
|
||||
return;
|
||||
|
||||
// batch size too small
|
||||
int size = batch.size();
|
||||
if (size < LOAD_BATCH_MIN) {
|
||||
for (OsuFile osu : batch)
|
||||
load(osu);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
// create map
|
||||
HashMap<String, HashMap<String, OsuFile>> map = new HashMap<String, HashMap<String, OsuFile>>();
|
||||
for (OsuFile osu : batch) {
|
||||
String parent = osu.getFile().getParentFile().getName();
|
||||
String name = osu.getFile().getName();
|
||||
HashMap<String, OsuFile> m = map.get(parent);
|
||||
if (m == null) {
|
||||
m = new HashMap<String, OsuFile>();
|
||||
map.put(parent, m);
|
||||
}
|
||||
m.put(name, osu);
|
||||
}
|
||||
|
||||
// iterate through database to load OsuFiles
|
||||
int count = 0;
|
||||
selectAllStmt.setFetchSize(100);
|
||||
ResultSet rs = selectAllStmt.executeQuery();
|
||||
while (rs.next()) {
|
||||
String parent = rs.getString(1);
|
||||
HashMap<String, OsuFile> m = map.get(parent);
|
||||
if (m != null) {
|
||||
String name = rs.getString(2);
|
||||
OsuFile osu = m.get(name);
|
||||
if (osu != null) {
|
||||
setOsuFileFields(rs, osu);
|
||||
if (++count >= size)
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
rs.close();
|
||||
} catch (SQLException e) {
|
||||
ErrorHandler.error("Failed to load OsuFiles from database.", e, true);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets all OsuFile fields using a given result set.
|
||||
* @param rs the result set containing the fields
|
||||
* @param osu the OsuFile
|
||||
* @throws SQLException
|
||||
*/
|
||||
private static void setOsuFileFields(ResultSet rs, OsuFile osu) throws SQLException {
|
||||
osu.beatmapID = rs.getInt(4);
|
||||
osu.beatmapSetID = rs.getInt(5);
|
||||
osu.title = OsuParser.getDBString(rs.getString(6));
|
||||
|
|
@ -300,7 +376,7 @@ public class OsuDB {
|
|||
osu.bpmMin = rs.getInt(23);
|
||||
osu.bpmMax = rs.getInt(24);
|
||||
osu.endTime = rs.getInt(25);
|
||||
osu.audioFilename = new File(dir, OsuParser.getDBString(rs.getString(26)));
|
||||
osu.audioFilename = new File(osu.getFile().getParentFile(), OsuParser.getDBString(rs.getString(26)));
|
||||
osu.audioLeadIn = rs.getInt(27);
|
||||
osu.previewTime = rs.getInt(28);
|
||||
osu.countdown = rs.getByte(29);
|
||||
|
|
@ -315,13 +391,6 @@ public class OsuDB {
|
|||
osu.breaksFromString(rs.getString(38));
|
||||
osu.comboFromString(rs.getString(39));
|
||||
}
|
||||
rs.close();
|
||||
return osu;
|
||||
} catch (SQLException e) {
|
||||
ErrorHandler.error("Failed to get OsuFile from database.", e, true);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a map of file paths ({dir}/{file}) to last modified times, or
|
||||
|
|
@ -334,6 +403,7 @@ public class OsuDB {
|
|||
try {
|
||||
Map<String, Long> map = new HashMap<String, Long>();
|
||||
ResultSet rs = lastModStmt.executeQuery();
|
||||
lastModStmt.setFetchSize(100);
|
||||
while (rs.next()) {
|
||||
String path = String.format("%s/%s", rs.getString(1), rs.getString(2));
|
||||
long lastModified = rs.getLong(3);
|
||||
|
|
@ -392,6 +462,7 @@ public class OsuDB {
|
|||
insertStmt.close();
|
||||
lastModStmt.close();
|
||||
selectStmt.close();
|
||||
selectAllStmt.close();
|
||||
deleteMapStmt.close();
|
||||
deleteGroupStmt.close();
|
||||
connection.close();
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user