From ed7df341b4b4681daa627c2f967801fc0db96186 Mon Sep 17 00:00:00 2001 From: Jesse Boyd Date: Sat, 2 Nov 2019 08:07:40 +0100 Subject: [PATCH] Make mca file reusable --- .../com/boydti/fawe/jnbt/anvil/MCAChunk.java | 4 + .../com/boydti/fawe/jnbt/anvil/MCAFile.java | 247 ++++++++++-------- .../clipboard/CPUOptimizedClipboard.java | 12 +- .../clipboard/MemoryOptimizedClipboard.java | 12 +- .../extent/clipboard/io/SchematicReader.java | 13 +- 5 files changed, 166 insertions(+), 122 deletions(-) diff --git a/worldedit-core/src/main/java/com/boydti/fawe/jnbt/anvil/MCAChunk.java b/worldedit-core/src/main/java/com/boydti/fawe/jnbt/anvil/MCAChunk.java index f4747c03b..f37801ba4 100644 --- a/worldedit-core/src/main/java/com/boydti/fawe/jnbt/anvil/MCAChunk.java +++ b/worldedit-core/src/main/java/com/boydti/fawe/jnbt/anvil/MCAChunk.java @@ -88,6 +88,10 @@ public class MCAChunk implements IChunkSet { public MCAChunk(NBTInputStream nis, int chunkX, int chunkZ, boolean readPos) throws IOException { this.chunkX = chunkX; this.chunkZ = chunkZ; + read(nis, readPos); + } + + public void read(NBTInputStream nis, boolean readPos) throws IOException { StreamDelegate root = createDelegate(nis, readPos); nis.readNamedTagLazy(root); } diff --git a/worldedit-core/src/main/java/com/boydti/fawe/jnbt/anvil/MCAFile.java b/worldedit-core/src/main/java/com/boydti/fawe/jnbt/anvil/MCAFile.java index 40f61411b..21a36c430 100644 --- a/worldedit-core/src/main/java/com/boydti/fawe/jnbt/anvil/MCAFile.java +++ b/worldedit-core/src/main/java/com/boydti/fawe/jnbt/anvil/MCAFile.java @@ -1,6 +1,7 @@ package com.boydti.fawe.jnbt.anvil; import com.boydti.fawe.Fawe; +import com.boydti.fawe.beta.Trimable; import com.boydti.fawe.jnbt.streamer.StreamDelegate; import com.boydti.fawe.object.RunnableVal; import com.boydti.fawe.object.RunnableVal4; @@ -25,6 +26,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ForkJoinPool; import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; import java.util.zip.Inflater; import java.util.zip.InflaterInputStream; @@ -32,7 +34,7 @@ import java.util.zip.InflaterInputStream; * Chunk format: http://minecraft.gamepedia.com/Chunk_format#Entity_format * e.g.: `.Level.Entities.#` (Starts with a . as the root tag is unnamed) */ -public class MCAFile { +public class MCAFile implements Trimable { private static Field fieldBuf2; private static Field fieldBuf3; @@ -48,13 +50,17 @@ public class MCAFile { } } - private final World world; - private final File file; + private final ForkJoinPool pool; + private final byte[] locations; + private boolean readLocations; + + private File file; private RandomAccessFile raf; - private byte[] locations; + private boolean deleted; - private final int X, Z; - private final Int2ObjectOpenHashMap chunks = new Int2ObjectOpenHashMap<>(); + private int X, Z; + private MCAChunk[] chunks; + private boolean[] chunkInitialized; final ThreadLocal byteStore1 = new ThreadLocal() { @Override @@ -75,26 +81,93 @@ public class MCAFile { } }; - public MCAFile(World world, File file) throws FileNotFoundException { - this.world = world; + public MCAFile(ForkJoinPool pool) { + this.pool = pool; + this.locations = new byte[4096]; + this.chunks = new MCAChunk[32 * 32]; + this.chunkInitialized = new boolean[this.chunks.length]; + } + + @Override + public boolean trim(boolean aggressive) { + boolean hasChunk = false; + for (int i = 0; i < chunkInitialized.length; i++) { + if (!chunkInitialized[i]) { + chunks[i] = null; + } else { + hasChunk = true; + } + } + CleanableThreadLocal.clean(byteStore1); + CleanableThreadLocal.clean(byteStore2); + CleanableThreadLocal.clean(byteStore3); + return !hasChunk; + } + + public MCAFile init(File file) throws FileNotFoundException { + String[] split = file.getName().split("\\."); + X = Integer.parseInt(split[1]); + Z = Integer.parseInt(split[2]); + return init(file, X, Z); + } + + public MCAFile init(File file, int mcrX, int mcrZ) throws FileNotFoundException { + if (raf != null) { + synchronized (raf) { + if (raf != null) { + flush(pool); + for (int i = 0; i < 4096; i++) { + locations[i] = 0; + } + try { + raf.close(); + } catch (IOException e) { + e.printStackTrace(); + } + raf = null; + deleted = false; + Arrays.fill(chunkInitialized, false); + readLocations = false; + } + } + this.X = mcrX; + this.Z = mcrZ; + } + this.file = file; if (!file.exists()) { throw new FileNotFoundException(file.getName()); } - String[] split = file.getName().split("\\."); - X = Integer.parseInt(split[1]); - Z = Integer.parseInt(split[2]); + return this; } - public MCAFile(World world, int mcrX, int mcrZ) { - this(world, mcrX, mcrZ, new File(world.getStoragePath().toFile(), "r." + mcrX + "." + mcrZ + ".mca")); + public MCAFile init(World world, int mcrX, int mcrZ) throws FileNotFoundException { + return init(new File(world.getStoragePath().toFile(), File.separator + "regions" + File.separator + "r." + mcrX + "." + mcrZ + ".mca")); } - public MCAFile(World world, int mcrX, int mcrZ, File file) { - this.world = world; - this.file = file; - X = mcrX; - Z = mcrZ; + public int getIndex(int chunkX, int chunkZ) { + return ((chunkX & 31) << 2) + ((chunkZ & 31) << 7); + } + + + private RandomAccessFile getRaf() throws FileNotFoundException { + if (this.raf == null) { + this.raf = new RandomAccessFile(file, "rw"); + } + return this.raf; + } + + private void readHeader() throws IOException { + if (!readLocations) { + readLocations = true; + getRaf(); + if (raf.length() < 8192) { + raf.setLength(8192); + } else { + raf.seek(0); + raf.readFully(locations); + } + } } public void clear() { @@ -106,12 +179,8 @@ public class MCAFile { } } synchronized (chunks) { - chunks.clear(); + Arrays.fill(chunkInitialized, false); } - locations = null; - CleanableThreadLocal.clean(byteStore1); - CleanableThreadLocal.clean(byteStore2); - CleanableThreadLocal.clean(byteStore3); } @Override @@ -130,32 +199,6 @@ public class MCAFile { return deleted; } - public World getWorld() { - return world; - } - - /** - * Loads the location header from disk - */ - public void init() { - try { - if (raf == null) { - this.locations = new byte[4096]; - if (file != null) { - this.raf = new RandomAccessFile(file, "rw"); - if (raf.length() < 8192) { - raf.setLength(8192); - } else { - raf.seek(0); - raf.readFully(locations); - } - } - } - } catch (Throwable e) { - e.printStackTrace(); - } - } - public int getX() { return X; } @@ -173,44 +216,46 @@ public class MCAFile { } public MCAChunk getCachedChunk(int cx, int cz) { - int pair = MathMan.pair((short) (cx & 31), (short) (cz & 31)); - synchronized (chunks) { - return chunks.get(pair); + int pair = getIndex(cx, cz); + MCAChunk chunk = chunks[pair]; + if (chunk != null && chunkInitialized[pair]) { + return chunk; } + return null; } public void setChunk(MCAChunk chunk) { int cx = chunk.getX(); int cz = chunk.getZ(); - int pair = MathMan.pair((short) (cx & 31), (short) (cz & 31)); - synchronized (chunks) { - chunks.put(pair, chunk); - } + int pair = getIndex(cx, cz); + chunks[pair] = chunk; } public MCAChunk getChunk(int cx, int cz) throws IOException { - MCAChunk cached = getCachedChunk(cx, cz); - if (cached != null) { - return cached; - } else { - return readChunk(cx, cz); + int pair = getIndex(cx, cz); + MCAChunk chunk = chunks[pair]; + if (chunk == null) { + chunk = new MCAChunk(); + chunk.setPosition(cx, cz); + chunks[pair] = chunk; + } else if (chunkInitialized[pair]) { + return chunk; } + readChunk(chunk, pair); + chunkInitialized[pair] = true; + return chunk; } - public MCAChunk readChunk(int cx, int cz) throws IOException { - int i = ((cx & 31) << 2) + ((cz & 31) << 7); + private MCAChunk readChunk(MCAChunk chunk, int i) throws IOException { int offset = (((locations[i] & 0xFF) << 16) + ((locations[i + 1] & 0xFF) << 8) + ((locations[i + 2] & 0xFF))) << 12; - int size = (locations[i + 3] & 0xFF) << 12; if (offset == 0) { return null; } - NBTInputStream nis = getChunkIS(offset); - MCAChunk chunk = new MCAChunk(nis, cx, cz, false); - nis.close(); - int pair = MathMan.pair((short) (cx & 31), (short) (cz & 31)); - synchronized (chunks) { - chunks.put(pair, chunk); + int size = (locations[i + 3] & 0xFF) << 12; + try (NBTInputStream nis = getChunkIS(offset)) { + chunk.read(nis, false); } + System.out.println("TODO multithreaded"); // TODO return chunk; } @@ -266,7 +311,7 @@ public class MCAFile { } } - public void forEachChunk(RunnableVal onEach) { + public void forEachChunk(Consumer onEach) { int i = 0; for (int z = 0; z < 32; z++) { for (int x = 0; x < 32; x++, i += 4) { @@ -274,7 +319,7 @@ public class MCAFile { int size = locations[i + 3] & 0xFF; if (size != 0) { try { - onEach.run(getChunk(x, z)); + onEach.accept(getChunk(x, z)); } catch (Throwable ignore) { } } @@ -283,28 +328,16 @@ public class MCAFile { } public int getOffset(int cx, int cz) { - int i = ((cx & 31) << 2) + ((cz & 31) << 7); + int i = getIndex(cx, cz); int offset = (((locations[i] & 0xFF) << 16) + ((locations[i + 1] & 0xFF) << 8) + ((locations[i + 2] & 0xFF))); return offset << 12; } public int getSize(int cx, int cz) { - int i = ((cx & 31) << 2) + ((cz & 31) << 7); + int i = getIndex(cx, cz); return (locations[i + 3] & 0xFF) << 12; } - public List getChunks() { - final List values; - synchronized (chunks) { - values = new ArrayList<>(chunks.size()); - } - for (int i = 0; i < locations.length; i += 4) { - int offset = (((locations[i] & 0xFF) << 16) + ((locations[i + 1] & 0xFF) << 8) + ((locations[i + 2] & 0xFF))); - values.add(offset); - } - return values; - } - public byte[] getChunkCompressedBytes(int offset) throws IOException { if (offset == 0) { return null; @@ -363,25 +396,28 @@ public class MCAFile { /** * @param onEach chunk */ - public void forEachCachedChunk(RunnableVal onEach) { - synchronized (chunks) { - for (Map.Entry entry : chunks.entrySet()) { - onEach.run(entry.getValue()); + public void forEachCachedChunk(Consumer onEach) { + for (int i = 0; i < chunks.length; i++) { + MCAChunk chunk = chunks[i]; + if (chunk != null && this.chunkInitialized[i]) { + onEach.accept(chunk); } } } public List getCachedChunks() { - synchronized (chunks) { - return new ArrayList<>(chunks.values()); + int size = 0; + for (int i = 0; i < chunks.length; i++) { + if (chunks[i] != null && this.chunkInitialized[i]) size++; } - } - - public void uncache(int cx, int cz) { - int pair = MathMan.pair((short) (cx & 31), (short) (cz & 31)); - synchronized (chunks) { - chunks.remove(pair); + ArrayList list = new ArrayList<>(size); + for (int i = 0; i < chunks.length; i++) { + MCAChunk chunk = chunks[i]; + if (chunk != null && this.chunkInitialized[i]) { + list.add(chunk); + } } + return list; } private byte[] toBytes(MCAChunk chunk) throws Exception { @@ -420,7 +456,7 @@ public class MCAFile { } private void writeHeader(RandomAccessFile raf, int cx, int cz, int offsetMedium, int sizeByte, boolean writeTime) throws IOException { - int i = ((cx & 31) << 2) + ((cz & 31) << 7); + int i = getIndex(cx, cz); locations[i] = (byte) (offsetMedium >> 16); locations[i + 1] = (byte) (offsetMedium >> 8); locations[i + 2] = (byte) (offsetMedium); @@ -449,7 +485,6 @@ public class MCAFile { e.printStackTrace(); } raf = null; - locations = null; } } } @@ -459,10 +494,12 @@ public class MCAFile { return true; } synchronized (chunks) { - for (Int2ObjectMap.Entry entry : chunks.int2ObjectEntrySet()) { - MCAChunk chunk = entry.getValue(); - if (chunk.isModified() || chunk.isDeleted()) { - return true; + for (int i = 0; i < chunks.length; i++) { + MCAChunk chunk = chunks[i]; + if (chunk != null && this.chunkInitialized[i]) { + if (chunk.isModified() || chunk.isDeleted()) { + return true; + } } } } @@ -571,7 +608,7 @@ public class MCAFile { nextOffset += size; end = Math.min(start + size, end); - int pair = MathMan.pair((short) (cx & 31), (short) (cz & 31)); + int pair = getIndex(cx, cz); byte[] newBytes = relocate.get(pair); // newBytes is null if the chunk isn't modified or marked for moving diff --git a/worldedit-core/src/main/java/com/boydti/fawe/object/clipboard/CPUOptimizedClipboard.java b/worldedit-core/src/main/java/com/boydti/fawe/object/clipboard/CPUOptimizedClipboard.java index 9d0b6d8d1..506f39ed5 100644 --- a/worldedit-core/src/main/java/com/boydti/fawe/object/clipboard/CPUOptimizedClipboard.java +++ b/worldedit-core/src/main/java/com/boydti/fawe/object/clipboard/CPUOptimizedClipboard.java @@ -17,6 +17,7 @@ import com.sk89q.worldedit.world.block.BlockState; import com.sk89q.worldedit.world.block.BlockStateHolder; import javax.annotation.Nullable; +import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -67,10 +68,15 @@ public class CPUOptimizedClipboard extends LinearClipboard { public void streamBiomes(IntValueReader task) { if (!hasBiomes()) return; int index = 0; - for (int z = 0; z < getLength(); z++) { - for (int x = 0; x < getWidth(); x++, index++) { - task.applyInt(index, biomes[index].getInternalId()); + try { + for (int z = 0; z < getLength(); z++) { + for (int x = 0; x < getWidth(); x++, index++) { + task.applyInt(index, biomes[index].getInternalId()); + } } + } catch (IOException e) { + e.printStackTrace(); + throw new RuntimeException(e); } } diff --git a/worldedit-core/src/main/java/com/boydti/fawe/object/clipboard/MemoryOptimizedClipboard.java b/worldedit-core/src/main/java/com/boydti/fawe/object/clipboard/MemoryOptimizedClipboard.java index 659374cc6..5cedda7ad 100644 --- a/worldedit-core/src/main/java/com/boydti/fawe/object/clipboard/MemoryOptimizedClipboard.java +++ b/worldedit-core/src/main/java/com/boydti/fawe/object/clipboard/MemoryOptimizedClipboard.java @@ -22,6 +22,7 @@ import com.sk89q.worldedit.world.block.BlockType; import com.sk89q.worldedit.world.block.BlockTypes; import javax.annotation.Nullable; +import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -102,10 +103,15 @@ public class MemoryOptimizedClipboard extends LinearClipboard { public void streamBiomes(IntValueReader task) { if (!hasBiomes()) return; int index = 0; - for (int z = 0; z < getLength(); z++) { - for (int x = 0; x < getWidth(); x++, index++) { - task.applyInt(index, biomes[index] & 0xFF); + try { + for (int z = 0; z < getLength(); z++) { + for (int x = 0; x < getWidth(); x++, index++) { + task.applyInt(index, biomes[index] & 0xFF); + } } + } catch (IOException e) { + e.printStackTrace(); + throw new RuntimeException(e); } } diff --git a/worldedit-core/src/main/java/com/sk89q/worldedit/extent/clipboard/io/SchematicReader.java b/worldedit-core/src/main/java/com/sk89q/worldedit/extent/clipboard/io/SchematicReader.java index 6cdd56acb..421a48874 100644 --- a/worldedit-core/src/main/java/com/sk89q/worldedit/extent/clipboard/io/SchematicReader.java +++ b/worldedit-core/src/main/java/com/sk89q/worldedit/extent/clipboard/io/SchematicReader.java @@ -21,11 +21,6 @@ package com.sk89q.worldedit.extent.clipboard.io; import com.boydti.fawe.Fawe; import com.boydti.fawe.FaweCache; -import com.boydti.fawe.jnbt.CorruptSchematicStreamer; -import com.boydti.fawe.jnbt.SchematicStreamer; - -import static com.google.common.base.Preconditions.checkNotNull; - import com.boydti.fawe.jnbt.streamer.InfoReader; import com.boydti.fawe.jnbt.streamer.IntValueReader; import com.boydti.fawe.jnbt.streamer.StreamDelegate; @@ -35,14 +30,9 @@ import com.boydti.fawe.object.FaweOutputStream; import com.boydti.fawe.object.clipboard.LinearClipboard; import com.boydti.fawe.object.io.FastByteArrayOutputStream; import com.boydti.fawe.object.io.FastByteArraysInputStream; -import com.google.common.collect.ImmutableList; import com.sk89q.jnbt.CompoundTag; -import com.sk89q.jnbt.IntTag; import com.sk89q.jnbt.NBTInputStream; -import com.sk89q.jnbt.StringTag; -import com.sk89q.jnbt.Tag; import com.sk89q.worldedit.entity.BaseEntity; -import com.sk89q.worldedit.extension.input.InputParseException; import com.sk89q.worldedit.extent.clipboard.Clipboard; import com.sk89q.worldedit.extent.clipboard.io.legacycompat.EntityNBTCompatibilityHandler; import com.sk89q.worldedit.extent.clipboard.io.legacycompat.FlowerPotCompatibilityHandler; @@ -55,7 +45,6 @@ import com.sk89q.worldedit.math.BlockVector3; import com.sk89q.worldedit.registry.state.PropertyKey; import com.sk89q.worldedit.util.Direction; import com.sk89q.worldedit.util.Location; -import com.sk89q.worldedit.world.DataFixer; import com.sk89q.worldedit.world.biome.BiomeType; import com.sk89q.worldedit.world.biome.BiomeTypes; import com.sk89q.worldedit.world.block.BaseBlock; @@ -81,6 +70,8 @@ import java.util.Map; import java.util.UUID; import java.util.function.Function; +import static com.google.common.base.Preconditions.checkNotNull; + /** * Reads schematic files based that are compatible with MCEdit and other editors. */