diff --git a/WorldPainter/WPCore/src/main/java/org/pepsoft/worldpainter/Dimension.java b/WorldPainter/WPCore/src/main/java/org/pepsoft/worldpainter/Dimension.java index 40680fe6..c22dd5b9 100644 --- a/WorldPainter/WPCore/src/main/java/org/pepsoft/worldpainter/Dimension.java +++ b/WorldPainter/WPCore/src/main/java/org/pepsoft/worldpainter/Dimension.java @@ -35,13 +35,12 @@ import javax.vecmath.Point3i; import java.awt.*; import java.beans.PropertyChangeListener; import java.beans.PropertyChangeSupport; -import java.io.File; -import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.Serializable; +import java.io.*; import java.util.List; import java.util.*; import java.util.stream.Collectors; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; import static java.util.Collections.emptySet; import static java.util.stream.Collectors.toSet; @@ -99,6 +98,10 @@ public class Dimension extends InstanceKeeper implements TileProvider, Serializa return world; } + public void setWorld(World2 world) { + this.world = world; + } + public Anchor getAnchor() { return anchor; } @@ -1740,6 +1743,59 @@ public class Dimension extends InstanceKeeper implements TileProvider, Serializa return mostPrevalentBiome; } + public synchronized void save(ZipOutputStream out) throws IOException { + setEventsInhibited(true); + try { + // First serialise everything but the tiles to a separate file + final String path = anchor + "/"; + out.putNextEntry(new ZipEntry(path + "dim-data.bin")); + try { + final Map savedTiles = tiles; + final World2 savedWorld = world; + try { + tiles = null; + world = null; + final ObjectOutputStream dataout = new ObjectOutputStream(out); + dataout.writeObject(this); + dataout.flush(); + } finally { + tiles = savedTiles; + world = savedWorld; + } + } finally { + out.closeEntry(); + } + + // Then serialise the tiles, grouped by region + final int regionX1 = lowestX >> 2, regionX2 = highestX >> 2, regionY1 = lowestY >> 2, regionY2 = highestY >> 2; + for (int regionX = regionX1; regionX <= regionX2; regionX++) { + for (int regionY = regionY1; regionY <= regionY2; regionY++) { + final List tileList = new ArrayList<>(); + for (int tileX = 0; tileX < 4; tileX++) { + for (int tileY = 0; tileY < 4; tileY++) { + final Tile tile = tiles.get(new Point((regionX << 2) | tileX, (regionY << 2) | tileY)); + if (tile != null) { + tile.prepareForSaving(); + tileList.add(tile); + } + } + } + + out.putNextEntry(new ZipEntry(path + "region-data-" + regionX + "," + regionY + ".bin")); + try { + final ObjectOutputStream dataout = new ObjectOutputStream(out); + dataout.writeObject(tileList); + dataout.flush(); + } finally { + out.closeEntry(); + } + } + } + } finally { + setEventsInhibited(false); + } + } + // Tile.Listener @Override @@ -1857,6 +1913,9 @@ public class Dimension extends InstanceKeeper implements TileProvider, Serializa rememberedChangeNo = -1; biomeHistogramRef = ThreadLocal.withInitial(() -> new int[255]); + if (tiles == null) { + tiles = new HashMap<>(); + } for (Tile tile: tiles.values()) { tile.addListener(this); Set seeds = tile.getSeeds(); @@ -2003,11 +2062,11 @@ public class Dimension extends InstanceKeeper implements TileProvider, Serializa } } - private final World2 world; + private World2 world; private final long seed; @Deprecated private int dim = 0; - final Map tiles = new HashMap<>(); + Map tiles = new HashMap<>(); private final TileFactory tileFactory; private int lowestX = Integer.MAX_VALUE, highestX = Integer.MIN_VALUE, lowestY = Integer.MAX_VALUE, highestY = Integer.MIN_VALUE; private Terrain subsurfaceMaterial = Terrain.STONE_MIX; @@ -2356,6 +2415,20 @@ public class Dimension extends InstanceKeeper implements TileProvider, Serializa return 31 * (31 * (31 * dim + role.hashCode()) + (invert ? 1 : 0)) + layer; } + /** + * Parse a string previously produced by {@link #toString()} into a new {@code Anchor} instance. + */ + public static Anchor fromString(String str) { + final String[] parts = str.split(" "); + final int dim = Integer.parseInt(parts[0]); + final Role role = Role.valueOf(parts[1]); + final boolean invert = (parts.length > 2) && parts[2].equals("CEILING"); + final int layer = invert + ? ((parts.length > 3) ? Integer.parseInt(parts[3]) : 0) + : ((parts.length > 2) ? Integer.parseInt(parts[2]) : 0); + return new Anchor(dim, role, invert, layer); + } + /** * The game dimension to which this anchor refers. See {@link Constants#DIM_NORMAL}, * {@link Constants#DIM_NETHER} and {@link Constants#DIM_END} for predefined values. Note that they don't diff --git a/WorldPainter/WPCore/src/main/java/org/pepsoft/worldpainter/Tile.java b/WorldPainter/WPCore/src/main/java/org/pepsoft/worldpainter/Tile.java index db20aa03..9691b81e 100644 --- a/WorldPainter/WPCore/src/main/java/org/pepsoft/worldpainter/Tile.java +++ b/WorldPainter/WPCore/src/main/java/org/pepsoft/worldpainter/Tile.java @@ -1346,6 +1346,50 @@ public class Tile extends InstanceKeeper implements Serializable, UndoListener, } } + synchronized void prepareForSaving() { + // Make sure all buffers are current, otherwise we may save out of date + // data to disk + ensureAllReadable(); + + // Take the opportunity to save memory and disk space by throwing away "empty" layer buffers. Since this is + // functionally a null operation there is no need to notify listeners, make the buffer writable or otherwise + // notify the undo manager + for (Iterator> i = bitLayerData.entrySet().iterator(); i.hasNext(); ) { + final Map.Entry entry = i.next(); + if (entry.getValue().isEmpty()) { + i.remove(); + cachedLayers = null; + } + } + layerLoop: + for (Iterator> i = layerData.entrySet().iterator(); i.hasNext(); ) { + Map.Entry entry = i.next(); + final Layer layer = entry.getKey(); + final byte[] buffer = entry.getValue(); + if (layer.getDataSize() == NIBBLE) { + final byte defaultByte = (byte) (layer.getDefaultValue() << 4 | layer.getDefaultValue()); + for (byte bufferByte: buffer) { + if (bufferByte != defaultByte) { + continue layerLoop; + } + } + // If we reach here all bytes were default bytes + i.remove(); + cachedLayers = null; + } else if (layer.getDataSize() == BYTE) { + final byte defaultByte = (byte) layer.getDefaultValue(); + for (byte bufferByte: buffer) { + if (bufferByte != defaultByte) { + continue layerLoop; + } + } + // If we reach here all bytes were default bytes + i.remove(); + cachedLayers = null; + } + } + } + private boolean getBitPerBlockLayerValue(BitSet bitSet, int x, int y) { return bitSet.get(x | (y << TILE_SIZE_BITS)); } @@ -1565,48 +1609,7 @@ public class Tile extends InstanceKeeper implements Serializable, UndoListener, } private synchronized void writeObject(ObjectOutputStream out) throws IOException { - // Make sure all buffers are current, otherwise we may save out of date - // data to disk - ensureAllReadable(); - - // Take the opportunity to save memory and disk space by throwing away "empty" layer buffers. Since this is - // functionally a null operation there is no need to notify listeners, make the buffer writable or otherwise - // notify the undo manager - for (Iterator> i = bitLayerData.entrySet().iterator(); i.hasNext(); ) { - final Map.Entry entry = i.next(); - if (entry.getValue().isEmpty()) { - i.remove(); - cachedLayers = null; - } - } - layerLoop: - for (Iterator> i = layerData.entrySet().iterator(); i.hasNext(); ) { - Map.Entry entry = i.next(); - final Layer layer = entry.getKey(); - final byte[] buffer = entry.getValue(); - if (layer.getDataSize() == NIBBLE) { - final byte defaultByte = (byte) (layer.getDefaultValue() << 4 | layer.getDefaultValue()); - for (byte bufferByte: buffer) { - if (bufferByte != defaultByte) { - continue layerLoop; - } - } - // If we reach here all bytes were default bytes - i.remove(); - cachedLayers = null; - } else if (layer.getDataSize() == BYTE) { - final byte defaultByte = (byte) layer.getDefaultValue(); - for (byte bufferByte: buffer) { - if (bufferByte != defaultByte) { - continue layerLoop; - } - } - // If we reach here all bytes were default bytes - i.remove(); - cachedLayers = null; - } - } - + prepareForSaving(); out.defaultWriteObject(); } diff --git a/WorldPainter/WPCore/src/main/java/org/pepsoft/worldpainter/World2.java b/WorldPainter/WPCore/src/main/java/org/pepsoft/worldpainter/World2.java index 96d954c1..775b0015 100644 --- a/WorldPainter/WPCore/src/main/java/org/pepsoft/worldpainter/World2.java +++ b/WorldPainter/WPCore/src/main/java/org/pepsoft/worldpainter/World2.java @@ -24,6 +24,8 @@ import java.beans.PropertyChangeSupport; import java.io.*; import java.util.List; import java.util.*; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; import static java.util.stream.Collectors.toSet; import static org.pepsoft.minecraft.Material.WOOL_MAGENTA; @@ -541,6 +543,29 @@ public class World2 extends InstanceKeeper implements Serializable, Cloneable { return MemoryUtils.getSize(this, new HashSet<>(Arrays.asList(UndoManager.class, Dimension.Listener.class, PropertyChangeSupport.class, Layer.class, Terrain.class))); } + public synchronized void save(ZipOutputStream out) throws IOException { + // First serialise everything but the dimensions to a separate file + out.putNextEntry(new ZipEntry("world-data.bin")); + try { + final Map savedDimensions = dimensionsByAnchor; + try { + dimensionsByAnchor = null; + final ObjectOutputStream dataout = new ObjectOutputStream(out); + dataout.writeObject(this); + dataout.flush(); + } finally { + dimensionsByAnchor = savedDimensions; + } + } finally { + out.closeEntry(); + } + + // Then serialise the dimensions individually + for (Dimension dimension: dimensionsByAnchor.values()) { + dimension.save(out); + } + } + /** * Get the set of warnings generated during loading, if any. * @@ -563,6 +588,10 @@ public class World2 extends InstanceKeeper implements Serializable, Cloneable { in.defaultReadObject(); propertyChangeSupport = new PropertyChangeSupport(this); + if (dimensionsByAnchor == null) { + dimensionsByAnchor = new HashMap<>(); + } + // Legacy maps if (wpVersion < 1) { if (maxheight == 0) { @@ -759,7 +788,6 @@ public class World2 extends InstanceKeeper implements Serializable, Cloneable { generatorOptions = null; } if (wpVersion < 10) { - dimensionsByAnchor = new HashMap<>(); dimensions.values().forEach(dimension -> dimensionsByAnchor.put(dimension.getAnchor(), dimension)); dimensions = null; } @@ -859,6 +887,11 @@ public class World2 extends InstanceKeeper implements Serializable, Cloneable { */ public static final String METADATA_KEY_PLUGINS = "org.pepsoft.worldpainter.plugins"; + /** + * A string containing the name of the world. + */ + public static final String METADATA_KEY_NAME = "name"; + private static final int CURRENT_WP_VERSION = 10; private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(World2.class); diff --git a/WorldPainter/WPCore/src/main/java/org/pepsoft/worldpainter/WorldIO.java b/WorldPainter/WPCore/src/main/java/org/pepsoft/worldpainter/WorldIO.java index acd5747b..c7a9a99a 100644 --- a/WorldPainter/WPCore/src/main/java/org/pepsoft/worldpainter/WorldIO.java +++ b/WorldPainter/WPCore/src/main/java/org/pepsoft/worldpainter/WorldIO.java @@ -1,5 +1,7 @@ package org.pepsoft.worldpainter; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.jetbrains.annotations.NotNull; import org.pepsoft.minecraft.Direction; import org.pepsoft.minecraft.SeededGenerator; import org.pepsoft.minecraft.SuperflatGenerator; @@ -17,10 +19,10 @@ import org.pepsoft.worldpainter.vo.EventVO; import java.io.*; import java.util.*; -import java.util.zip.GZIPInputStream; -import java.util.zip.GZIPOutputStream; -import java.util.zip.ZipException; +import java.util.zip.*; +import static com.fasterxml.jackson.core.JsonGenerator.Feature.AUTO_CLOSE_TARGET; +import static com.fasterxml.jackson.databind.SerializationFeature.WRITE_DATES_AS_TIMESTAMPS; import static org.pepsoft.minecraft.Material.*; import static org.pepsoft.worldpainter.DefaultPlugin.JAVA_MCREGION; import static org.pepsoft.worldpainter.Dimension.Anchor.NORMAL_DETAIL; @@ -62,24 +64,38 @@ public class WorldIO { */ public void save(OutputStream out) throws IOException { try (ObjectOutputStream wrappedOut = new ObjectOutputStream(new GZIPOutputStream(out))) { - Map metadata = new HashMap<>(); - metadata.put(World2.METADATA_KEY_WP_VERSION, Version.VERSION); - metadata.put(World2.METADATA_KEY_WP_BUILD, Version.BUILD); - metadata.put(World2.METADATA_KEY_TIMESTAMP, new Date()); - if (WPPluginManager.getInstance() != null) { - List pluginArray = new ArrayList<>(); - WPPluginManager.getInstance().getAllPlugins().stream() - .filter(plugin -> ! plugin.getClass().getName().startsWith("org.pepsoft.worldpainter")) - .forEach(plugin -> pluginArray.add(new String[]{plugin.getName(), plugin.getVersion()})); - if (! pluginArray.isEmpty()) { - metadata.put(World2.METADATA_KEY_PLUGINS, pluginArray.toArray(new String[pluginArray.size()][])); - } - } - wrappedOut.writeObject(metadata); + wrappedOut.writeObject(getMetadata()); wrappedOut.writeObject(world); } } + /** + * Save the world to a binary stream, such that it can later be loaded using {@link #load(InputStream)}. The stream + * is closed before returning. + * + *

This version uses a format that compresses every region separately, for faster access to individual regions + * without having to laod the entire world. + * + * @param out The stream to which to save the world. + * @throws IOException If an I/O error occurred saving the world. + */ + // TODO this saves multiple copies of layers, etc.! Either solve that on loading, or else use this only for + // exporting + public void saveCompartmentalised(OutputStream out) throws IOException { + try (ZipOutputStream wrappedOut = new ZipOutputStream(out)) { + final ObjectMapper objectMapper = new ObjectMapper() + .disable(AUTO_CLOSE_TARGET) + .disable(WRITE_DATES_AS_TIMESTAMPS); + wrappedOut.putNextEntry(new ZipEntry("metadata.json")); + try { + objectMapper.writeValue(wrappedOut, getMetadata()); + } finally { + wrappedOut.closeEntry(); + } + world.save(wrappedOut); + } + } + /** * Load a world from a binary stream to which it was previously saved by * {@link #save(OutputStream)}, or by a previous version of WorldPainter. @@ -124,6 +140,25 @@ public class WorldIO { } } + @NotNull + private Map getMetadata() { + final Map metadata = new HashMap<>(); + metadata.put(World2.METADATA_KEY_NAME, world.getName()); + metadata.put(World2.METADATA_KEY_WP_VERSION, Version.VERSION); + metadata.put(World2.METADATA_KEY_WP_BUILD, Version.BUILD); + metadata.put(World2.METADATA_KEY_TIMESTAMP, new Date()); + if (WPPluginManager.getInstance() != null) { + final List pluginArray = new ArrayList<>(); + WPPluginManager.getInstance().getAllPlugins().stream() + .filter(plugin -> ! plugin.getClass().getName().startsWith("org.pepsoft.worldpainter")) + .forEach(plugin -> pluginArray.add(new String[]{plugin.getName(), plugin.getVersion()})); + if (! pluginArray.isEmpty()) { + metadata.put(World2.METADATA_KEY_PLUGINS, pluginArray.toArray(new String[pluginArray.size()][])); + } + } + return metadata; + } + private World2 migrate(Object object) { if (object instanceof World) { World oldWorld = (World) object; diff --git a/WorldPainter/WPCore/src/main/java/org/pepsoft/worldpainter/exporting/JavaWorldExporter.java b/WorldPainter/WPCore/src/main/java/org/pepsoft/worldpainter/exporting/JavaWorldExporter.java index 42004f20..38962768 100644 --- a/WorldPainter/WPCore/src/main/java/org/pepsoft/worldpainter/exporting/JavaWorldExporter.java +++ b/WorldPainter/WPCore/src/main/java/org/pepsoft/worldpainter/exporting/JavaWorldExporter.java @@ -57,33 +57,7 @@ public class JavaWorldExporter extends AbstractWorldExporter { // TODO can this } @SuppressWarnings("ConstantConditions") // Clarity - @Override - public Map export(File baseDir, String name, File backupDir, ProgressReceiver progressReceiver) throws IOException, ProgressReceiver.OperationCancelled { - // Sanity checks - final Set selectedTiles = world.getTilesToExport(); - final Set selectedDimensions = world.getDimensionsToExport(); - if ((selectedTiles != null) && ((selectedDimensions == null) || (selectedDimensions.size() != 1))) { - throw new IllegalArgumentException("If a tile selection is active then exactly one dimension must be selected"); - } - - // Backup existing level - File worldDir = new File(baseDir, FileUtils.sanitiseName(name)); - logger.info("Exporting world " + world.getName() + " to map at " + worldDir + " in " + platform.displayName + " format"); - if (worldDir.isDirectory()) { - if (backupDir != null) { - logger.info("Directory already exists; backing up to " + backupDir); - if (!worldDir.renameTo(backupDir)) { - throw new FileInUseException("Could not move " + worldDir + " to " + backupDir); - } - } else { - throw new IllegalStateException("Directory already exists and no backup directory specified"); - } - } - - // Record start of export - long start = System.currentTimeMillis(); - - // Export dimensions + protected JavaLevel createWorld(File worldDir, String name) throws IOException { Dimension dim0 = world.getDimension(NORMAL_DETAIL); JavaLevel level = JavaLevel.create(platform, world.getMaxHeight()); level.setSeed(dim0.getMinecraftSeed()); @@ -208,6 +182,37 @@ public class JavaWorldExporter extends AbstractWorldExporter { // TODO can this // Save the level.dat file. This will also create a session.lock file, hopefully kicking out any Minecraft // instances which may have the map open: level.save(worldDir); + return level; + } + + @Override + public Map export(File baseDir, String name, File backupDir, ProgressReceiver progressReceiver) throws IOException, ProgressReceiver.OperationCancelled { + // Sanity checks + final Set selectedTiles = world.getTilesToExport(); + final Set selectedDimensions = world.getDimensionsToExport(); + if ((selectedTiles != null) && ((selectedDimensions == null) || (selectedDimensions.size() != 1))) { + throw new IllegalArgumentException("If a tile selection is active then exactly one dimension must be selected"); + } + + // Backup existing level + File worldDir = new File(baseDir, FileUtils.sanitiseName(name)); + logger.info("Exporting world " + world.getName() + " to map at " + worldDir + " in " + platform.displayName + " format"); + if (worldDir.isDirectory()) { + if (backupDir != null) { + logger.info("Directory already exists; backing up to " + backupDir); + if (!worldDir.renameTo(backupDir)) { + throw new FileInUseException("Could not move " + worldDir + " to " + backupDir); + } + } else { + throw new IllegalStateException("Directory already exists and no backup directory specified"); + } + } + + // Record start of export + long start = System.currentTimeMillis(); + + // Create the level.dat file + final JavaLevel level = createWorld(worldDir, name); // Lock the level.dat file, to keep Minecraft out until we are done File levelDatFile = new File(worldDir, "level.dat"); @@ -264,7 +269,7 @@ public class JavaWorldExporter extends AbstractWorldExporter { // TODO can this event.setAttribute(ATTRIBUTE_KEY_MAP_FEATURES, world.isMapFeatures()); event.setAttribute(ATTRIBUTE_KEY_GAME_TYPE_NAME, world.getGameType().name()); event.setAttribute(ATTRIBUTE_KEY_ALLOW_CHEATS, world.isAllowCheats()); - event.setAttribute(ATTRIBUTE_KEY_GENERATOR, dim0.getGenerator().getType().name()); + event.setAttribute(ATTRIBUTE_KEY_GENERATOR, world.getDimension(NORMAL_DETAIL).getGenerator().getType().name()); Dimension dimension = world.getDimension(NORMAL_DETAIL); event.setAttribute(ATTRIBUTE_KEY_TILES, dimension.getTileCount()); logLayers(dimension, event, "");