Captain-Chaos
/
WorldPainter
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Introduce new save format with individually loadable regions for higher distributed export performance 

Some code changes for more granular exporting of Java platform worlds
master
Captain Chaos 2022-08-25 10:14:07 +02:00
parent 644c712563
commit fccab11107
5 changed files with 243 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
WorldPainter/WPCore/src/main/java/org/pepsoft/worldpainter/Dimension.java
View File

@ -35,13 +35,12 @@ import javax.vecmath.Point3i;
import java.awt.*;
import java.beans.PropertyChangeListener;
import java.beans.PropertyChangeSupport;
import java.io.File;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.Serializable;
import java.io.*;
import java.util.List;
import java.util.*;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import static java.util.Collections.emptySet;
import static java.util.stream.Collectors.toSet;
@ -99,6 +98,10 @@ public class Dimension extends InstanceKeeper implements TileProvider, Serializa
return world;
}
public void setWorld(World2 world) {
this.world = world;
}
public Anchor getAnchor() {
return anchor;
}
@ -1740,6 +1743,59 @@ public class Dimension extends InstanceKeeper implements TileProvider, Serializa
return mostPrevalentBiome;
}
public synchronized void save(ZipOutputStream out) throws IOException {
setEventsInhibited(true);
try {
// First serialise everything but the tiles to a separate file
final String path = anchor + "/";
out.putNextEntry(new ZipEntry(path + "dim-data.bin"));
try {
final Map<Point, Tile> savedTiles = tiles;
final World2 savedWorld = world;
try {
tiles = null;
world = null;
final ObjectOutputStream dataout = new ObjectOutputStream(out);
dataout.writeObject(this);
dataout.flush();
} finally {
tiles = savedTiles;
world = savedWorld;
}
} finally {
out.closeEntry();
}
// Then serialise the tiles, grouped by region
final int regionX1 = lowestX >> 2, regionX2 = highestX >> 2, regionY1 = lowestY >> 2, regionY2 = highestY >> 2;
for (int regionX = regionX1; regionX <= regionX2; regionX++) {
for (int regionY = regionY1; regionY <= regionY2; regionY++) {
final List<Tile> tileList = new ArrayList<>();
for (int tileX = 0; tileX < 4; tileX++) {
for (int tileY = 0; tileY < 4; tileY++) {
final Tile tile = tiles.get(new Point((regionX << 2) | tileX, (regionY << 2) | tileY));
if (tile != null) {
tile.prepareForSaving();
tileList.add(tile);
}
}
}
out.putNextEntry(new ZipEntry(path + "region-data-" + regionX + "," + regionY + ".bin"));
try {
final ObjectOutputStream dataout = new ObjectOutputStream(out);
dataout.writeObject(tileList);
dataout.flush();
} finally {
out.closeEntry();
}
}
}
} finally {
setEventsInhibited(false);
}
}
// Tile.Listener
@Override
@ -1857,6 +1913,9 @@ public class Dimension extends InstanceKeeper implements TileProvider, Serializa
rememberedChangeNo = -1;
biomeHistogramRef = ThreadLocal.withInitial(() -> new int[255]);
if (tiles == null) {
tiles = new HashMap<>();
}
for (Tile tile: tiles.values()) {
tile.addListener(this);
Set<Seed> seeds = tile.getSeeds();
@ -2003,11 +2062,11 @@ public class Dimension extends InstanceKeeper implements TileProvider, Serializa
}
}
private final World2 world;
private World2 world;
private final long seed;
@Deprecated
private int dim = 0;
final Map<Point, Tile> tiles = new HashMap<>();
Map<Point, Tile> tiles = new HashMap<>();
private final TileFactory tileFactory;
private int lowestX = Integer.MAX_VALUE, highestX = Integer.MIN_VALUE, lowestY = Integer.MAX_VALUE, highestY = Integer.MIN_VALUE;
private Terrain subsurfaceMaterial = Terrain.STONE_MIX;
@ -2356,6 +2415,20 @@ public class Dimension extends InstanceKeeper implements TileProvider, Serializa
return 31 * (31 * (31 * dim + role.hashCode()) + (invert ? 1 : 0)) + layer;
}
/**
* Parse a string previously produced by {@link #toString()} into a new {@code Anchor} instance.
*/
public static Anchor fromString(String str) {
final String[] parts = str.split(" ");
final int dim = Integer.parseInt(parts[0]);
final Role role = Role.valueOf(parts[1]);
final boolean invert = (parts.length > 2) && parts[2].equals("CEILING");
final int layer = invert
? ((parts.length > 3) ? Integer.parseInt(parts[3]) : 0)
: ((parts.length > 2) ? Integer.parseInt(parts[2]) : 0);
return new Anchor(dim, role, invert, layer);
}
/**
* The game dimension to which this anchor refers. See {@link Constants#DIM_NORMAL},
* {@link Constants#DIM_NETHER} and {@link Constants#DIM_END} for predefined values. Note that they don't

87
WorldPainter/WPCore/src/main/java/org/pepsoft/worldpainter/Tile.java
View File

@ -1346,6 +1346,50 @@ public class Tile extends InstanceKeeper implements Serializable, UndoListener,
}
}
synchronized void prepareForSaving() {
// Make sure all buffers are current, otherwise we may save out of date
// data to disk
ensureAllReadable();
// Take the opportunity to save memory and disk space by throwing away "empty" layer buffers. Since this is
// functionally a null operation there is no need to notify listeners, make the buffer writable or otherwise
// notify the undo manager
for (Iterator<Map.Entry<Layer, BitSet>> i = bitLayerData.entrySet().iterator(); i.hasNext(); ) {
final Map.Entry<Layer, BitSet> entry = i.next();
if (entry.getValue().isEmpty()) {
i.remove();
cachedLayers = null;
}
}
layerLoop:
for (Iterator<Map.Entry<Layer, byte[]>> i = layerData.entrySet().iterator(); i.hasNext(); ) {
Map.Entry<Layer, byte[]> entry = i.next();
final Layer layer = entry.getKey();
final byte[] buffer = entry.getValue();
if (layer.getDataSize() == NIBBLE) {
final byte defaultByte = (byte) (layer.getDefaultValue() << 4 | layer.getDefaultValue());
for (byte bufferByte: buffer) {
if (bufferByte != defaultByte) {
continue layerLoop;
}
}
// If we reach here all bytes were default bytes
i.remove();
cachedLayers = null;
} else if (layer.getDataSize() == BYTE) {
final byte defaultByte = (byte) layer.getDefaultValue();
for (byte bufferByte: buffer) {
if (bufferByte != defaultByte) {
continue layerLoop;
}
}
// If we reach here all bytes were default bytes
i.remove();
cachedLayers = null;
}
}
}
private boolean getBitPerBlockLayerValue(BitSet bitSet, int x, int y) {
return bitSet.get(x | (y << TILE_SIZE_BITS));
}
@ -1565,48 +1609,7 @@ public class Tile extends InstanceKeeper implements Serializable, UndoListener,
}
private synchronized void writeObject(ObjectOutputStream out) throws IOException {
// Make sure all buffers are current, otherwise we may save out of date
// data to disk
ensureAllReadable();
// Take the opportunity to save memory and disk space by throwing away "empty" layer buffers. Since this is
// functionally a null operation there is no need to notify listeners, make the buffer writable or otherwise
// notify the undo manager
for (Iterator<Map.Entry<Layer, BitSet>> i = bitLayerData.entrySet().iterator(); i.hasNext(); ) {
final Map.Entry<Layer, BitSet> entry = i.next();
if (entry.getValue().isEmpty()) {
i.remove();
cachedLayers = null;
}
}
layerLoop:
for (Iterator<Map.Entry<Layer, byte[]>> i = layerData.entrySet().iterator(); i.hasNext(); ) {
Map.Entry<Layer, byte[]> entry = i.next();
final Layer layer = entry.getKey();
final byte[] buffer = entry.getValue();
if (layer.getDataSize() == NIBBLE) {
final byte defaultByte = (byte) (layer.getDefaultValue() << 4 | layer.getDefaultValue());
for (byte bufferByte: buffer) {
if (bufferByte != defaultByte) {
continue layerLoop;
}
}
// If we reach here all bytes were default bytes
i.remove();
cachedLayers = null;
} else if (layer.getDataSize() == BYTE) {
final byte defaultByte = (byte) layer.getDefaultValue();
for (byte bufferByte: buffer) {
if (bufferByte != defaultByte) {
continue layerLoop;
}
}
// If we reach here all bytes were default bytes
i.remove();
cachedLayers = null;
}
}
prepareForSaving();
out.defaultWriteObject();
}

35
WorldPainter/WPCore/src/main/java/org/pepsoft/worldpainter/World2.java
View File

@ -24,6 +24,8 @@ import java.beans.PropertyChangeSupport;
import java.io.*;
import java.util.List;
import java.util.*;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import static java.util.stream.Collectors.toSet;
import static org.pepsoft.minecraft.Material.WOOL_MAGENTA;
@ -541,6 +543,29 @@ public class World2 extends InstanceKeeper implements Serializable, Cloneable {
return MemoryUtils.getSize(this, new HashSet<>(Arrays.asList(UndoManager.class, Dimension.Listener.class, PropertyChangeSupport.class, Layer.class, Terrain.class)));
}
public synchronized void save(ZipOutputStream out) throws IOException {
// First serialise everything but the dimensions to a separate file
out.putNextEntry(new ZipEntry("world-data.bin"));
try {
final Map<Anchor, Dimension> savedDimensions = dimensionsByAnchor;
try {
dimensionsByAnchor = null;
final ObjectOutputStream dataout = new ObjectOutputStream(out);
dataout.writeObject(this);
dataout.flush();
} finally {
dimensionsByAnchor = savedDimensions;
}
} finally {
out.closeEntry();
}
// Then serialise the dimensions individually
for (Dimension dimension: dimensionsByAnchor.values()) {
dimension.save(out);
}
}
/**
* Get the set of warnings generated during loading, if any.
*
@ -563,6 +588,10 @@ public class World2 extends InstanceKeeper implements Serializable, Cloneable {
in.defaultReadObject();
propertyChangeSupport = new PropertyChangeSupport(this);
if (dimensionsByAnchor == null) {
dimensionsByAnchor = new HashMap<>();
}
// Legacy maps
if (wpVersion < 1) {
if (maxheight == 0) {
@ -759,7 +788,6 @@ public class World2 extends InstanceKeeper implements Serializable, Cloneable {
generatorOptions = null;
}
if (wpVersion < 10) {
dimensionsByAnchor = new HashMap<>();
dimensions.values().forEach(dimension -> dimensionsByAnchor.put(dimension.getAnchor(), dimension));
dimensions = null;
}
@ -859,6 +887,11 @@ public class World2 extends InstanceKeeper implements Serializable, Cloneable {
*/
public static final String METADATA_KEY_PLUGINS = "org.pepsoft.worldpainter.plugins";
/**
* A string containing the name of the world.
*/
public static final String METADATA_KEY_NAME = "name";
private static final int CURRENT_WP_VERSION = 10;
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(World2.class);

69
WorldPainter/WPCore/src/main/java/org/pepsoft/worldpainter/WorldIO.java
View File

@ -1,5 +1,7 @@
package org.pepsoft.worldpainter;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.jetbrains.annotations.NotNull;
import org.pepsoft.minecraft.Direction;
import org.pepsoft.minecraft.SeededGenerator;
import org.pepsoft.minecraft.SuperflatGenerator;
@ -17,10 +19,10 @@ import org.pepsoft.worldpainter.vo.EventVO;
import java.io.*;
import java.util.*;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import java.util.zip.ZipException;
import java.util.zip.*;
import static com.fasterxml.jackson.core.JsonGenerator.Feature.AUTO_CLOSE_TARGET;
import static com.fasterxml.jackson.databind.SerializationFeature.WRITE_DATES_AS_TIMESTAMPS;
import static org.pepsoft.minecraft.Material.*;
import static org.pepsoft.worldpainter.DefaultPlugin.JAVA_MCREGION;
import static org.pepsoft.worldpainter.Dimension.Anchor.NORMAL_DETAIL;
@ -62,24 +64,38 @@ public class WorldIO {
*/
public void save(OutputStream out) throws IOException {
try (ObjectOutputStream wrappedOut = new ObjectOutputStream(new GZIPOutputStream(out))) {
Map<String, Object> metadata = new HashMap<>();
metadata.put(World2.METADATA_KEY_WP_VERSION, Version.VERSION);
metadata.put(World2.METADATA_KEY_WP_BUILD, Version.BUILD);
metadata.put(World2.METADATA_KEY_TIMESTAMP, new Date());
if (WPPluginManager.getInstance() != null) {
List<String[]> pluginArray = new ArrayList<>();
WPPluginManager.getInstance().getAllPlugins().stream()
.filter(plugin -> ! plugin.getClass().getName().startsWith("org.pepsoft.worldpainter"))
.forEach(plugin -> pluginArray.add(new String[]{plugin.getName(), plugin.getVersion()}));
if (! pluginArray.isEmpty()) {
metadata.put(World2.METADATA_KEY_PLUGINS, pluginArray.toArray(new String[pluginArray.size()][]));
}
}
wrappedOut.writeObject(metadata);
wrappedOut.writeObject(getMetadata());
wrappedOut.writeObject(world);
}
}
/**
* Save the world to a binary stream, such that it can later be loaded using {@link #load(InputStream)}. The stream
* is closed before returning.
*
* <p>This version uses a format that compresses every region separately, for faster access to individual regions
* without having to laod the entire world.
*
* @param out The stream to which to save the world.
* @throws IOException If an I/O error occurred saving the world.
*/
// TODO this saves multiple copies of layers, etc.! Either solve that on loading, or else use this only for
// exporting
public void saveCompartmentalised(OutputStream out) throws IOException {
try (ZipOutputStream wrappedOut = new ZipOutputStream(out)) {
final ObjectMapper objectMapper = new ObjectMapper()
.disable(AUTO_CLOSE_TARGET)
.disable(WRITE_DATES_AS_TIMESTAMPS);
wrappedOut.putNextEntry(new ZipEntry("metadata.json"));
try {
objectMapper.writeValue(wrappedOut, getMetadata());
} finally {
wrappedOut.closeEntry();
}
world.save(wrappedOut);
}
}
/**
* Load a world from a binary stream to which it was previously saved by
* {@link #save(OutputStream)}, or by a previous version of WorldPainter.
@ -124,6 +140,25 @@ public class WorldIO {
}
}
@NotNull
private Map<String, Object> getMetadata() {
final Map<String, Object> metadata = new HashMap<>();
metadata.put(World2.METADATA_KEY_NAME, world.getName());
metadata.put(World2.METADATA_KEY_WP_VERSION, Version.VERSION);
metadata.put(World2.METADATA_KEY_WP_BUILD, Version.BUILD);
metadata.put(World2.METADATA_KEY_TIMESTAMP, new Date());
if (WPPluginManager.getInstance() != null) {
final List<String[]> pluginArray = new ArrayList<>();
WPPluginManager.getInstance().getAllPlugins().stream()
.filter(plugin -> ! plugin.getClass().getName().startsWith("org.pepsoft.worldpainter"))
.forEach(plugin -> pluginArray.add(new String[]{plugin.getName(), plugin.getVersion()}));
if (! pluginArray.isEmpty()) {
metadata.put(World2.METADATA_KEY_PLUGINS, pluginArray.toArray(new String[pluginArray.size()][]));
}
}
return metadata;
}
private World2 migrate(Object object) {
if (object instanceof World) {
World oldWorld = (World) object;

61
WorldPainter/WPCore/src/main/java/org/pepsoft/worldpainter/exporting/JavaWorldExporter.java
View File

@ -57,33 +57,7 @@ public class JavaWorldExporter extends AbstractWorldExporter { // TODO can this
}
@SuppressWarnings("ConstantConditions") // Clarity
@Override
public Map<Integer, ChunkFactory.Stats> export(File baseDir, String name, File backupDir, ProgressReceiver progressReceiver) throws IOException, ProgressReceiver.OperationCancelled {
// Sanity checks
final Set<Point> selectedTiles = world.getTilesToExport();
final Set<Integer> selectedDimensions = world.getDimensionsToExport();
if ((selectedTiles != null) && ((selectedDimensions == null) || (selectedDimensions.size() != 1))) {
throw new IllegalArgumentException("If a tile selection is active then exactly one dimension must be selected");
}
// Backup existing level
File worldDir = new File(baseDir, FileUtils.sanitiseName(name));
logger.info("Exporting world " + world.getName() + " to map at " + worldDir + " in " + platform.displayName + " format");
if (worldDir.isDirectory()) {
if (backupDir != null) {
logger.info("Directory already exists; backing up to " + backupDir);
if (!worldDir.renameTo(backupDir)) {
throw new FileInUseException("Could not move " + worldDir + " to " + backupDir);
}
} else {
throw new IllegalStateException("Directory already exists and no backup directory specified");
}
}
// Record start of export
long start = System.currentTimeMillis();
// Export dimensions
protected JavaLevel createWorld(File worldDir, String name) throws IOException {
Dimension dim0 = world.getDimension(NORMAL_DETAIL);
JavaLevel level = JavaLevel.create(platform, world.getMaxHeight());
level.setSeed(dim0.getMinecraftSeed());
@ -208,6 +182,37 @@ public class JavaWorldExporter extends AbstractWorldExporter { // TODO can this
// Save the level.dat file. This will also create a session.lock file, hopefully kicking out any Minecraft
// instances which may have the map open:
level.save(worldDir);
return level;
}
@Override
public Map<Integer, ChunkFactory.Stats> export(File baseDir, String name, File backupDir, ProgressReceiver progressReceiver) throws IOException, ProgressReceiver.OperationCancelled {
// Sanity checks
final Set<Point> selectedTiles = world.getTilesToExport();
final Set<Integer> selectedDimensions = world.getDimensionsToExport();
if ((selectedTiles != null) && ((selectedDimensions == null) || (selectedDimensions.size() != 1))) {
throw new IllegalArgumentException("If a tile selection is active then exactly one dimension must be selected");
}
// Backup existing level
File worldDir = new File(baseDir, FileUtils.sanitiseName(name));
logger.info("Exporting world " + world.getName() + " to map at " + worldDir + " in " + platform.displayName + " format");
if (worldDir.isDirectory()) {
if (backupDir != null) {
logger.info("Directory already exists; backing up to " + backupDir);
if (!worldDir.renameTo(backupDir)) {
throw new FileInUseException("Could not move " + worldDir + " to " + backupDir);
}
} else {
throw new IllegalStateException("Directory already exists and no backup directory specified");
}
}
// Record start of export
long start = System.currentTimeMillis();
// Create the level.dat file
final JavaLevel level = createWorld(worldDir, name);
// Lock the level.dat file, to keep Minecraft out until we are done
File levelDatFile = new File(worldDir, "level.dat");
@ -264,7 +269,7 @@ public class JavaWorldExporter extends AbstractWorldExporter { // TODO can this
event.setAttribute(ATTRIBUTE_KEY_MAP_FEATURES, world.isMapFeatures());
event.setAttribute(ATTRIBUTE_KEY_GAME_TYPE_NAME, world.getGameType().name());
event.setAttribute(ATTRIBUTE_KEY_ALLOW_CHEATS, world.isAllowCheats());
event.setAttribute(ATTRIBUTE_KEY_GENERATOR, dim0.getGenerator().getType().name());
event.setAttribute(ATTRIBUTE_KEY_GENERATOR, world.getDimension(NORMAL_DETAIL).getGenerator().getType().name());
Dimension dimension = world.getDimension(NORMAL_DETAIL);
event.setAttribute(ATTRIBUTE_KEY_TILES, dimension.getTileCount());
logLayers(dimension, event, "");