From 3e00ce36d2f5789386d8b33a1e1af7670bfd62b2 Mon Sep 17 00:00:00 2001 From: Jesse Boyd Date: Fri, 1 Nov 2019 19:11:05 +0100 Subject: [PATCH] Add barebones Anvil API not anvil commands, just the core of the api could do with some optimization --- .../main/java/com/boydti/fawe/FaweCache.java | 2 + .../com/boydti/fawe/jnbt/anvil/MCAChunk.java | 522 ++++++++++++++ .../com/boydti/fawe/jnbt/anvil/MCAFile.java | 675 ++++++++++++++++++ .../fawe/jnbt/streamer/StreamDelegate.java | 4 +- .../java/com/sk89q/jnbt/NBTInputStream.java | 33 +- 5 files changed, 1218 insertions(+), 18 deletions(-) create mode 100644 worldedit-core/src/main/java/com/boydti/fawe/jnbt/anvil/MCAChunk.java create mode 100644 worldedit-core/src/main/java/com/boydti/fawe/jnbt/anvil/MCAFile.java diff --git a/worldedit-core/src/main/java/com/boydti/fawe/FaweCache.java b/worldedit-core/src/main/java/com/boydti/fawe/FaweCache.java index 07a6fe0b9..d8d113b5d 100644 --- a/worldedit-core/src/main/java/com/boydti/fawe/FaweCache.java +++ b/worldedit-core/src/main/java/com/boydti/fawe/FaweCache.java @@ -209,6 +209,8 @@ public enum FaweCache implements Trimable { */ public final CleanableThreadLocal CHUNK_FLAG = new CleanableThreadLocal<>(AtomicBoolean::new); // resets to false + public final CleanableThreadLocal LONG_BUFFER_1024 = new CleanableThreadLocal<>(() -> new long[1024]); + public final CleanableThreadLocal BYTE_BUFFER_8192 = new CleanableThreadLocal<>(() -> new byte[8192]); public final CleanableThreadLocal BLOCK_TO_PALETTE = new CleanableThreadLocal<>(() -> { diff --git a/worldedit-core/src/main/java/com/boydti/fawe/jnbt/anvil/MCAChunk.java b/worldedit-core/src/main/java/com/boydti/fawe/jnbt/anvil/MCAChunk.java new file mode 100644 index 000000000..e5335f720 --- /dev/null +++ b/worldedit-core/src/main/java/com/boydti/fawe/jnbt/anvil/MCAChunk.java @@ -0,0 +1,522 @@ +package com.boydti.fawe.jnbt.anvil; + +import com.boydti.fawe.FaweCache; +import com.boydti.fawe.beta.IChunkSet; +import com.boydti.fawe.jnbt.streamer.StreamDelegate; +import com.boydti.fawe.jnbt.streamer.ValueReader; +import com.boydti.fawe.object.collection.BitArray4096; +import com.boydti.fawe.object.collection.BlockVector3ChunkMap; +import com.boydti.fawe.object.io.FastByteArrayOutputStream; +import com.boydti.fawe.util.MathMan; +import com.sk89q.jnbt.CompoundTag; +import com.sk89q.jnbt.ListTag; +import com.sk89q.jnbt.NBTConstants; +import com.sk89q.jnbt.NBTInputStream; +import com.sk89q.jnbt.NBTOutputStream; +import com.sk89q.worldedit.math.BlockVector2; +import com.sk89q.worldedit.math.BlockVector3; +import com.sk89q.worldedit.registry.state.Property; +import com.sk89q.worldedit.world.biome.BiomeType; +import com.sk89q.worldedit.world.biome.BiomeTypes; +import com.sk89q.worldedit.world.block.BlockID; +import com.sk89q.worldedit.world.block.BlockState; +import com.sk89q.worldedit.world.block.BlockStateHolder; +import com.sk89q.worldedit.world.block.BlockType; +import com.sk89q.worldedit.world.block.BlockTypes; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +public class MCAChunk implements IChunkSet { + public final boolean[] hasSections = new boolean[16]; + + public boolean hasBiomes = false; + public final byte[] biomes = new byte[256]; + + public final char[] blocks = new char[65536]; + + public final BlockVector3ChunkMap tiles = new BlockVector3ChunkMap(); + public final Map entities = new HashMap<>(); + public long inhabitedTime = System.currentTimeMillis(); + public long lastUpdate; + + public int modified; + public boolean deleted; + + public int chunkX; + public int chunkZ; + + public MCAChunk() {} + + private boolean readLayer(Section section) { + if (section.palette == null || section.layer == -1 || section.blocksLength == -1 || section.palette[section.palette.length - 1] == null || section.blocks == null) { + // not initialized + return false; + } + + int bitsPerEntry = MathMan.log2nlz(section.palette.length - 1); + BitArray4096 bitArray = new BitArray4096(section.blocks, bitsPerEntry); + char[] buffer = FaweCache.IMP.SECTION_BITS_TO_CHAR.get(); + bitArray.toRaw(buffer); + int offset = section.layer << 12; + for (int i = 0; i < buffer.length; i++) { + BlockState block = section.palette[buffer[i]]; + blocks[offset + i] = block.getOrdinalChar(); + } + + section.layer = -1; + section.blocksLength = -1; + section.blocks = null; + section.palette = null; + return true; + } + + private static class Section { + public int layer = -1; + public long[] blocks; + public int blocksLength = -1; + public BlockState[] palette; + } + + public MCAChunk(NBTInputStream nis, int chunkX, int chunkZ, boolean readPos) throws IOException { + this.chunkX = chunkX; + this.chunkZ = chunkZ; + StreamDelegate root = createDelegate(nis, readPos); + nis.readNamedTagLazy(root); + } + + public StreamDelegate createDelegate(NBTInputStream nis, boolean readPos) { + StreamDelegate root = new StreamDelegate(); + StreamDelegate level = root.add("").add("Level"); + + level.add("InhabitedTime").withLong((i, v) -> inhabitedTime = v); + level.add("LastUpdate").withLong((i, v) -> lastUpdate = v); + + if (readPos) { + level.add("xPos").withInt((i, v) -> MCAChunk.this.chunkX = v); + level.add("zPos").withInt((i, v) -> MCAChunk.this.chunkZ = v); + } + + Section section = new Section(); + + StreamDelegate layers = level.add("Sections"); + StreamDelegate layer = layers.add(); + layer.withInfo((length, type) -> { + section.layer = -1; + section.blocksLength = -1; + }); + layer.add("Y").withInt((i, y) -> section.layer = y); + layer.add("Palette").withValue((ValueReader>) (index, map) -> { + String name = (String) map.get("Name"); + BlockType type = BlockTypes.get(name); + BlockState state = type.getDefaultState(); + Map properties = (Map) map.get("Properties"); + if (properties != null) { + for (Map.Entry entry : properties.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + Property property = type.getProperty(key); + state = state.with(property, property.getValueFor(value)); + } + } + section.palette[index] = state; + readLayer(section); + }); + StreamDelegate blockStates = layer.add("BlockStates"); + blockStates.withInfo((length, type) -> { + if (section.blocks == null) { + section.blocks = FaweCache.IMP.LONG_BUFFER_1024.get(); + } + section.blocksLength = length; + }); + blockStates.withLong((index, value) -> section.blocks[index] = value); + level.add("TileEntities").withValue((ValueReader>) (index, value) -> { + CompoundTag tile = FaweCache.IMP.asTag(value); + int x = tile.getInt("x") & 15; + int y = tile.getInt("y"); + int z = tile.getInt("z") & 15; + tiles.put(x, y, z, tile); + }); + level.add("Entities").withValue((ValueReader>) (index, value) -> { + CompoundTag entity = FaweCache.IMP.asTag(value); + entities.put(entity.getUUID(), entity); + }); + level.add("Biomes").withInt((index, value) -> biomes[index] = (byte) value); + + return root; + } + + public int getX() { + return chunkX; + } + + public int getZ() { + return chunkZ; + } + + @Override + public boolean hasSection(int layer) { + return hasSections[layer]; + } + + public void setPosition(int X, int Z) { + this.chunkX = X; + this.chunkZ = Z; + } + + @Override + public IChunkSet reset() { + return this.reset(true); + } + + public IChunkSet reset(boolean full) { + if (!tiles.isEmpty()) { + tiles.clear(); + } + if (!entities.isEmpty()) { + entities.clear(); + } + modified = 0; + deleted = false; + hasBiomes = false; + if (full) { + for (int i = 0; i < 65536; i++) { + blocks[i] = BlockID.AIR; + } + } + Arrays.fill(hasSections, false); + return this; + } + + public void write(NBTOutputStream nbtOut) throws IOException { + int[] blockToPalette = FaweCache.IMP.BLOCK_TO_PALETTE.get(); + int[] paletteToBlock = FaweCache.IMP.PALETTE_TO_BLOCK.get(); + long[] blockstates = FaweCache.IMP.BLOCK_STATES.get(); + int[] blocksCopy = FaweCache.IMP.SECTION_BLOCKS.get(); + + nbtOut.writeNamedTagName("", NBTConstants.TYPE_COMPOUND); + nbtOut.writeNamedTag("DataVersion", 1631); + nbtOut.writeLazyCompoundTag("Level", out -> { + out.writeNamedTag("Status", "decorated"); + out.writeNamedTag("xPos", getX()); + out.writeNamedTag("zPos", getZ()); + if (entities.isEmpty()) { + out.writeNamedEmptyList("Entities"); + } else { + out.writeNamedTag("Entities", new ListTag(CompoundTag.class, new ArrayList<>(entities.values()))); + } + if (tiles.isEmpty()) { + out.writeNamedEmptyList("TileEntities"); + } else { + out.writeNamedTag("TileEntities", new ListTag(CompoundTag.class, + new ArrayList<>(tiles.values()))); + } + out.writeNamedTag("InhabitedTime", inhabitedTime); + out.writeNamedTag("LastUpdate", lastUpdate); + if (hasBiomes) { + out.writeNamedTag("Biomes", biomes); + } + int len = 0; + for (boolean hasSection : hasSections) { + if (hasSection) { + len++; + } + } + out.writeNamedTagName("Sections", NBTConstants.TYPE_LIST); + nbtOut.writeByte(NBTConstants.TYPE_COMPOUND); + nbtOut.writeInt(len); + + for (int layer = 0; layer < hasSections.length; layer++) { + if (!hasSections[layer]) { + continue; + } + out.writeNamedTag("Y", (byte) layer); + + int blockIndexStart = layer << 12; + int blockIndexEnd = blockIndexStart + 4096; + int num_palette = 0; + try { + for (int i = blockIndexStart, j = 0; i < blockIndexEnd; i++, j++) { + int ordinal = blocks[i]; + int palette = blockToPalette[ordinal]; + if (palette == Integer.MAX_VALUE) { +// BlockState state = BlockTypes.states[ordinal]; + blockToPalette[ordinal] = palette = num_palette; + paletteToBlock[num_palette] = ordinal; + num_palette++; + } + blocksCopy[j] = palette; + } + + for (int i = 0; i < num_palette; i++) { + blockToPalette[paletteToBlock[i]] = Integer.MAX_VALUE; + } + + out.writeNamedTagName("Palette", NBTConstants.TYPE_LIST); + out.writeByte(NBTConstants.TYPE_COMPOUND); + out.writeInt(num_palette); + + for (int i = 0; i < num_palette; i++) { + int ordinal = paletteToBlock[i]; + BlockState state = BlockTypes.states[ordinal]; + BlockType type = state.getBlockType(); + out.writeNamedTag("Name", type.getId()); + + // Has no properties + if (type.getDefaultState() != state) { + // Write properties + out.writeNamedTagName("Properties", NBTConstants.TYPE_COMPOUND); + for (Property property : type.getProperties()) { + String key = property.getName(); + Object value = state.getState(property); + String valueStr = value.toString(); + if (Character.isUpperCase(valueStr.charAt(0))) { + System.out.println("Invalid uppercase value " + value); + valueStr = valueStr.toLowerCase(); + } + out.writeNamedTag(key, valueStr); + } + out.writeEndTag(); + } + out.writeEndTag(); + } + + + // BlockStates + int bitsPerEntry = MathMan.log2nlz(num_palette - 1); + int blockBitArrayEnd = (bitsPerEntry * 4096) >> 6; + if (num_palette == 1) { + // Set a value, because minecraft needs it for some reason + blockstates[0] = 0; + blockBitArrayEnd = 1; + } else { + BitArray4096 bitArray = new BitArray4096(blockstates, bitsPerEntry); + bitArray.fromRaw(blocksCopy); + } + + out.writeNamedTagName("BlockStates", NBTConstants.TYPE_LONG_ARRAY); + out.writeInt(blockBitArrayEnd); + for (int i = 0; i < blockBitArrayEnd; i++) { + out.writeLong(blockstates[i]); + } + + +// out.writeNamedTagName("BlockLight", NBTConstants.TYPE_BYTE_ARRAY); +// out.writeInt(2048); +// out.write(blockLight, layer << 11, 1 << 11); +// +// out.writeNamedTagName("SkyLight", NBTConstants.TYPE_BYTE_ARRAY); +// out.writeInt(2048); +// out.write(skyLight, layer << 11, 1 << 11); + + + out.writeEndTag(); + + // cleanup + } catch (Throwable e) { + Arrays.fill(blockToPalette, Integer.MAX_VALUE); + e.printStackTrace(); + throw e; + } + } + }); + nbtOut.writeEndTag(); + } + + public byte[] toBytes(byte[] buffer) throws IOException { + if (buffer == null) { + buffer = new byte[8192]; + } + FastByteArrayOutputStream buffered = new FastByteArrayOutputStream(buffer); + try (NBTOutputStream nbtOut = new NBTOutputStream(buffered)) { + write(nbtOut); + } + return buffered.toByteArray(); + } + + public long getInhabitedTime() { + return inhabitedTime; + } + + public long getLastUpdate() { + return lastUpdate; + } + + public void setInhabitedTime(long inhabitedTime) { + this.inhabitedTime = inhabitedTime; + } + + public void setLastUpdate(long lastUpdate) { + this.lastUpdate = lastUpdate; + } + + public void setDeleted(boolean deleted) { + setModified(); + this.deleted = deleted; + } + + public boolean isDeleted() { + return deleted; + } + + @Override + public boolean isEmpty() { + if (deleted) return true; + for (boolean hasSection : hasSections) { + if (hasSection) return false; + } + return true; + } + + public boolean isModified() { + return modified != 0; + } + + public int getModified() { + return modified; + } + + public final void setModified() { + this.modified++; + } + + public int getBitMask() { + int bitMask = 0; + for (int section = 0; section < hasSections.length; section++) { + if (hasSections[section]) { + bitMask += 1 << section; + } + } + return bitMask; + } + + @Override + public boolean setTile(int x, int y, int z, CompoundTag tile) { + setModified(); + if (tile != null) { + tiles.put(x, y, z, tile); + } else { + if (tiles.remove(x, y, z) == null) { + return false; + } + } + return true; + } + + public void setEntity(CompoundTag entityTag) { + setModified(); + long least = entityTag.getLong("UUIDLeast"); + long most = entityTag.getLong("UUIDMost"); + entities.put(new UUID(most, least), entityTag); + } + + @Override + public BiomeType getBiomeType(int x, int z) { + return BiomeTypes.get(this.biomes[(z << 4) | x] & 0xFF); + } + + @Override + public BiomeType[] getBiomes() { + BiomeType[] tmp = new BiomeType[256]; + for (int i = 0; i < 256; i++) { + tmp[i] = BiomeTypes.get(this.biomes[i] & 0xFF); + } + return tmp; + } + + @Override + public boolean setBiome(BlockVector2 pos, BiomeType biome) { + return this.setBiome(pos.getX(), 0, pos.getZ(), biome); + } + + @Override + public boolean setBiome(int x, int y, int z, BiomeType biome) { + setModified(); + biomes[x + (z << 4)] = (byte) biome.getInternalId(); + return true; + } + + public Set getEntities() { + return new HashSet<>(entities.values()); + } + + @Override + public Map getTiles() { + return tiles == null ? Collections.emptyMap() : tiles; + } + + public CompoundTag getTile(int x, int y, int z) { + if (tiles == null || tiles.isEmpty()) { + return null; + } + short pair = MathMan.tripleBlockCoord(x, y, z); + return tiles.get(pair); + } + + private final int getIndex(int x, int y, int z) { + return x | (z << 4) | (y << 8); + } + + public int getBlockOrdinal(int x, int y, int z) { + return blocks[x | (z << 4) | (y << 8)]; + } + + @Override + public BlockState getBlock(int x, int y, int z) { + int ordinal = getBlockOrdinal(x, y, z); + return BlockState.getFromOrdinal(ordinal); + } + + public Set getEntityRemoves() { + return new HashSet<>(); + } + + @Override + public boolean setBlock(int x, int y, int z, BlockStateHolder holder) { + setBlock(x, y, z, holder.getOrdinalChar()); + holder.applyTileEntity(this, x, y, z); + return true; + } + + @Override + public void setBlocks(int layer, char[] data) { + int offset = layer << 12; + for (int i = 0; i < 4096; i++) { + blocks[offset + i] = data[i]; + } + } + + @Override + public char[] getArray(int layer) { + char[] tmp = FaweCache.IMP.SECTION_BITS_TO_CHAR.get(); + int offset = layer << 12; + for (int i = 0; i < 4096; i++) { + tmp[i] = blocks[offset + i]; + } + return tmp; + } + + public void setBlock(int x, int y, int z, char ordinal) { + blocks[getIndex(x, y, z)] = ordinal; + } + + public void setBiome(BiomeType biome) { + Arrays.fill(biomes, (byte) biome.getInternalId()); + } + + public void removeEntity(UUID uuid) { + entities.remove(uuid); + } + + @Override + public boolean trim(boolean aggressive) { + return isEmpty(); + } +} diff --git a/worldedit-core/src/main/java/com/boydti/fawe/jnbt/anvil/MCAFile.java b/worldedit-core/src/main/java/com/boydti/fawe/jnbt/anvil/MCAFile.java new file mode 100644 index 000000000..40f61411b --- /dev/null +++ b/worldedit-core/src/main/java/com/boydti/fawe/jnbt/anvil/MCAFile.java @@ -0,0 +1,675 @@ +package com.boydti.fawe.jnbt.anvil; + +import com.boydti.fawe.Fawe; +import com.boydti.fawe.jnbt.streamer.StreamDelegate; +import com.boydti.fawe.object.RunnableVal; +import com.boydti.fawe.object.RunnableVal4; +import com.boydti.fawe.object.collection.CleanableThreadLocal; +import com.boydti.fawe.object.io.BufferedRandomAccessFile; +import com.boydti.fawe.object.io.FastByteArrayInputStream; +import com.boydti.fawe.util.MainUtil; +import com.boydti.fawe.util.MathMan; +import com.sk89q.jnbt.NBTInputStream; +import com.sk89q.worldedit.world.World; +import it.unimi.dsi.fastutil.ints.Int2ObjectMap; +import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.TimeUnit; +import java.util.zip.Inflater; +import java.util.zip.InflaterInputStream; + +/** + * Chunk format: http://minecraft.gamepedia.com/Chunk_format#Entity_format + * e.g.: `.Level.Entities.#` (Starts with a . as the root tag is unnamed) + */ +public class MCAFile { + + private static Field fieldBuf2; + private static Field fieldBuf3; + + static { + try { + fieldBuf2 = InflaterInputStream.class.getDeclaredField("buf"); + fieldBuf2.setAccessible(true); + fieldBuf3 = NBTInputStream.class.getDeclaredField("buf"); + fieldBuf3.setAccessible(true); + } catch (Throwable e) { + e.printStackTrace(); + } + } + + private final World world; + private final File file; + private RandomAccessFile raf; + private byte[] locations; + private boolean deleted; + private final int X, Z; + private final Int2ObjectOpenHashMap chunks = new Int2ObjectOpenHashMap<>(); + + final ThreadLocal byteStore1 = new ThreadLocal() { + @Override + protected byte[] initialValue() { + return new byte[4096]; + } + }; + final ThreadLocal byteStore2 = new ThreadLocal() { + @Override + protected byte[] initialValue() { + return new byte[4096]; + } + }; + final ThreadLocal byteStore3 = new ThreadLocal() { + @Override + protected byte[] initialValue() { + return new byte[1024]; + } + }; + + public MCAFile(World world, File file) throws FileNotFoundException { + this.world = world; + this.file = file; + if (!file.exists()) { + throw new FileNotFoundException(file.getName()); + } + String[] split = file.getName().split("\\."); + X = Integer.parseInt(split[1]); + Z = Integer.parseInt(split[2]); + } + + public MCAFile(World world, int mcrX, int mcrZ) { + this(world, mcrX, mcrZ, new File(world.getStoragePath().toFile(), "r." + mcrX + "." + mcrZ + ".mca")); + } + + public MCAFile(World world, int mcrX, int mcrZ, File file) { + this.world = world; + this.file = file; + X = mcrX; + Z = mcrZ; + } + + public void clear() { + if (raf != null) { + try { + raf.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + synchronized (chunks) { + chunks.clear(); + } + locations = null; + CleanableThreadLocal.clean(byteStore1); + CleanableThreadLocal.clean(byteStore2); + CleanableThreadLocal.clean(byteStore3); + } + + @Override + protected void finalize() throws Throwable { + CleanableThreadLocal.clean(byteStore1); + CleanableThreadLocal.clean(byteStore2); + CleanableThreadLocal.clean(byteStore3); + super.finalize(); + } + + public void setDeleted(boolean deleted) { + this.deleted = deleted; + } + + public boolean isDeleted() { + return deleted; + } + + public World getWorld() { + return world; + } + + /** + * Loads the location header from disk + */ + public void init() { + try { + if (raf == null) { + this.locations = new byte[4096]; + if (file != null) { + this.raf = new RandomAccessFile(file, "rw"); + if (raf.length() < 8192) { + raf.setLength(8192); + } else { + raf.seek(0); + raf.readFully(locations); + } + } + } + } catch (Throwable e) { + e.printStackTrace(); + } + } + + public int getX() { + return X; + } + + public int getZ() { + return Z; + } + + public RandomAccessFile getRandomAccessFile() { + return raf; + } + + public File getFile() { + return file; + } + + public MCAChunk getCachedChunk(int cx, int cz) { + int pair = MathMan.pair((short) (cx & 31), (short) (cz & 31)); + synchronized (chunks) { + return chunks.get(pair); + } + } + + public void setChunk(MCAChunk chunk) { + int cx = chunk.getX(); + int cz = chunk.getZ(); + int pair = MathMan.pair((short) (cx & 31), (short) (cz & 31)); + synchronized (chunks) { + chunks.put(pair, chunk); + } + } + + public MCAChunk getChunk(int cx, int cz) throws IOException { + MCAChunk cached = getCachedChunk(cx, cz); + if (cached != null) { + return cached; + } else { + return readChunk(cx, cz); + } + } + + public MCAChunk readChunk(int cx, int cz) throws IOException { + int i = ((cx & 31) << 2) + ((cz & 31) << 7); + int offset = (((locations[i] & 0xFF) << 16) + ((locations[i + 1] & 0xFF) << 8) + ((locations[i + 2] & 0xFF))) << 12; + int size = (locations[i + 3] & 0xFF) << 12; + if (offset == 0) { + return null; + } + NBTInputStream nis = getChunkIS(offset); + MCAChunk chunk = new MCAChunk(nis, cx, cz, false); + nis.close(); + int pair = MathMan.pair((short) (cx & 31), (short) (cz & 31)); + synchronized (chunks) { + chunks.put(pair, chunk); + } + return chunk; + } + + /** + * CX, CZ, OFFSET, SIZE + * + * @param onEach + * @throws IOException + */ + public void forEachSortedChunk(RunnableVal4 onEach) throws IOException { + char[] offsets = new char[(int) (raf.length() / 4096) - 2]; + Arrays.fill(offsets, Character.MAX_VALUE); + char i = 0; + for (int z = 0; z < 32; z++) { + for (int x = 0; x < 32; x++, i += 4) { + int offset = (((locations[i] & 0xFF) << 16) + ((locations[i + 1] & 0xFF) << 8) + ((locations[i + 2] & 0xFF))) - 2; + int size = locations[i + 3] & 0xFF; + if (size != 0) { + if (offset < offsets.length) { + offsets[offset] = i; + } else { + Fawe.debug("Ignoring invalid offset " + offset); + } + } + } + } + for (i = 0; i < offsets.length; i++) { + int index = offsets[i]; + if (index != Character.MAX_VALUE) { + int offset = i + 2; + int size = locations[index + 3] & 0xFF; + int index2 = index >> 2; + int x = (index2) & 31; + int z = (index2) >> 5; + onEach.run(x, z, offset << 12, size << 12); + } + } + } + + /** + * @param onEach cx, cz, offset, size + */ + public void forEachChunk(RunnableVal4 onEach) { + int i = 0; + for (int z = 0; z < 32; z++) { + for (int x = 0; x < 32; x++, i += 4) { + int offset = (((locations[i] & 0xFF) << 16) + ((locations[i + 1] & 0xFF) << 8) + ((locations[i + 2] & 0xFF))); + int size = locations[i + 3] & 0xFF; + if (size != 0) { + onEach.run(x, z, offset << 12, size << 12); + } + } + } + } + + public void forEachChunk(RunnableVal onEach) { + int i = 0; + for (int z = 0; z < 32; z++) { + for (int x = 0; x < 32; x++, i += 4) { + int offset = (((locations[i] & 0xFF) << 16) + ((locations[i + 1] & 0xFF) << 8) + ((locations[i + 2] & 0xFF))); + int size = locations[i + 3] & 0xFF; + if (size != 0) { + try { + onEach.run(getChunk(x, z)); + } catch (Throwable ignore) { + } + } + } + } + } + + public int getOffset(int cx, int cz) { + int i = ((cx & 31) << 2) + ((cz & 31) << 7); + int offset = (((locations[i] & 0xFF) << 16) + ((locations[i + 1] & 0xFF) << 8) + ((locations[i + 2] & 0xFF))); + return offset << 12; + } + + public int getSize(int cx, int cz) { + int i = ((cx & 31) << 2) + ((cz & 31) << 7); + return (locations[i + 3] & 0xFF) << 12; + } + + public List getChunks() { + final List values; + synchronized (chunks) { + values = new ArrayList<>(chunks.size()); + } + for (int i = 0; i < locations.length; i += 4) { + int offset = (((locations[i] & 0xFF) << 16) + ((locations[i + 1] & 0xFF) << 8) + ((locations[i + 2] & 0xFF))); + values.add(offset); + } + return values; + } + + public byte[] getChunkCompressedBytes(int offset) throws IOException { + if (offset == 0) { + return null; + } + synchronized (raf) { + raf.seek(offset); + int size = raf.readInt(); + int compression = raf.read(); + byte[] data = new byte[size]; + raf.readFully(data); + return data; + } + } + + private NBTInputStream getChunkIS(int offset) throws IOException { + try { + byte[] data = getChunkCompressedBytes(offset); + FastByteArrayInputStream bais = new FastByteArrayInputStream(data); + InflaterInputStream iis = new InflaterInputStream(bais, new Inflater(), 1); + fieldBuf2.set(iis, byteStore2.get()); + BufferedInputStream bis = new BufferedInputStream(iis); + NBTInputStream nis = new NBTInputStream(bis); + fieldBuf3.set(nis, byteStore3.get()); + return nis; + } catch (IllegalAccessException unlikely) { + unlikely.printStackTrace(); + return null; + } + } + + public void streamChunk(int cx, int cz, StreamDelegate delegate) throws IOException { + streamChunk(getOffset(cx, cz), delegate); + } + + public void streamChunk(int offset, StreamDelegate delegate) throws IOException { + byte[] data = getChunkCompressedBytes(offset); + streamChunk(data, delegate); + } + + public void streamChunk(byte[] data, StreamDelegate delegate) throws IOException { + if (data != null) { + try { + FastByteArrayInputStream bais = new FastByteArrayInputStream(data); + InflaterInputStream iis = new InflaterInputStream(bais, new Inflater(), 1); + fieldBuf2.set(iis, byteStore2.get()); + BufferedInputStream bis = new BufferedInputStream(iis); + NBTInputStream nis = new NBTInputStream(bis); + fieldBuf3.set(nis, byteStore3.get()); + nis.readNamedTagLazy(delegate); + } catch (IllegalAccessException unlikely) { + unlikely.printStackTrace(); + } + } + } + + /** + * @param onEach chunk + */ + public void forEachCachedChunk(RunnableVal onEach) { + synchronized (chunks) { + for (Map.Entry entry : chunks.entrySet()) { + onEach.run(entry.getValue()); + } + } + } + + public List getCachedChunks() { + synchronized (chunks) { + return new ArrayList<>(chunks.values()); + } + } + + public void uncache(int cx, int cz) { + int pair = MathMan.pair((short) (cx & 31), (short) (cz & 31)); + synchronized (chunks) { + chunks.remove(pair); + } + } + + private byte[] toBytes(MCAChunk chunk) throws Exception { + if (chunk.isDeleted()) { + return null; + } + byte[] uncompressed = chunk.toBytes(byteStore3.get()); + byte[] compressed = MainUtil.compress(uncompressed, byteStore2.get(), null); + return compressed; + } + + private byte[] getChunkBytes(int cx, int cz) throws Exception { + MCAChunk mca = getCachedChunk(cx, cz); + if (mca == null) { + int offset = getOffset(cx, cz); + if (offset == 0) { + return null; + } + return getChunkCompressedBytes(offset); + } + return toBytes(mca); + } + + + private void writeSafe(RandomAccessFile raf, int offset, byte[] data) throws IOException { + int len = data.length + 5; + raf.seek(offset); + if (raf.length() - offset < len) { + raf.setLength(((offset + len + 4095) / 4096) * 4096); + } + // Length of remaining data + raf.writeInt(data.length + 1); + // Compression type + raf.write(2); + raf.write(data); + } + + private void writeHeader(RandomAccessFile raf, int cx, int cz, int offsetMedium, int sizeByte, boolean writeTime) throws IOException { + int i = ((cx & 31) << 2) + ((cz & 31) << 7); + locations[i] = (byte) (offsetMedium >> 16); + locations[i + 1] = (byte) (offsetMedium >> 8); + locations[i + 2] = (byte) (offsetMedium); + locations[i + 3] = (byte) sizeByte; + raf.seek(i); + raf.write((offsetMedium >> 16)); + raf.write((offsetMedium >> 8)); + raf.write((offsetMedium >> 0)); + raf.write(sizeByte); + raf.seek(i + 4096); + if (offsetMedium == 0 && sizeByte == 0) { + raf.writeInt(0); + } else { + raf.writeInt((int) (System.currentTimeMillis() / 1000L)); + } + } + + public void close(ForkJoinPool pool) { + if (raf == null) return; + synchronized (raf) { + if (raf != null) { + flush(pool); + try { + raf.close(); + } catch (IOException e) { + e.printStackTrace(); + } + raf = null; + locations = null; + } + } + } + + public boolean isModified() { + if (isDeleted()) { + return true; + } + synchronized (chunks) { + for (Int2ObjectMap.Entry entry : chunks.int2ObjectEntrySet()) { + MCAChunk chunk = entry.getValue(); + if (chunk.isModified() || chunk.isDeleted()) { + return true; + } + } + } + return false; + } + + /** + * Write the chunk to the file + * @param pool + */ + public void flush(ForkJoinPool pool) { + synchronized (raf) { + // If the file is marked as deleted, nothing is written + if (isDeleted()) { + clear(); + file.delete(); + return; + } + + boolean wait; // If the flush method needs to wait for the pool + if (pool == null) { + wait = true; + pool = new ForkJoinPool(); + } else wait = false; + + // Chunks that need to be relocated + Int2ObjectOpenHashMap relocate = new Int2ObjectOpenHashMap<>(); + // The position of each chunk + final Int2ObjectOpenHashMap offsetMap = new Int2ObjectOpenHashMap<>(); // Offset -> + // The data of each modified chunk + final Int2ObjectOpenHashMap compressedMap = new Int2ObjectOpenHashMap<>(); + // The data of each chunk that needs to be moved + final Int2ObjectOpenHashMap append = new Int2ObjectOpenHashMap<>(); + boolean modified = false; + // Get the current time for the chunk timestamp + long now = System.currentTimeMillis(); + + // Load the chunks into the append or compressed map + for (MCAChunk chunk : getCachedChunks()) { + if (chunk.isModified() || chunk.isDeleted()) { + modified = true; + chunk.setLastUpdate(now); + if (!chunk.isDeleted()) { + pool.submit(new Runnable() { + @Override + public void run() { + try { + byte[] compressed = toBytes(chunk); + int pair = MathMan.pair((short) (chunk.getX() & 31), (short) (chunk.getZ() & 31)); + Int2ObjectOpenHashMap map; + if (getOffset(chunk.getX(), chunk.getZ()) == 0) { + map = append; + } else { + map = compressedMap; + } + synchronized (map) { + map.put(pair, compressed); + } + } catch (Throwable e) { + e.printStackTrace(); + } + } + }); + } + } + } + + // If any changes were detected + if (modified) { + file.setLastModified(now); + + // Load the offset data into the offset map + forEachChunk(new RunnableVal4() { + @Override + public void run(Integer cx, Integer cz, Integer offset, Integer size) { + short pair1 = MathMan.pairByte((byte) (cx & 31), (byte) (cz & 31)); + short pair2 = (short) (size >> 12); + offsetMap.put((int) offset, (Integer) MathMan.pair(pair1, pair2)); + } + }); + // Wait for previous tasks + pool.awaitQuiescence(Long.MAX_VALUE, TimeUnit.MILLISECONDS); + + + int start = 8192; + int written = start; + int end = 8192; + int nextOffset = 8192; + try { + for (int count = 0; count < offsetMap.size(); count++) { + // Get the previous position of the next chunk + Integer loc = offsetMap.get(nextOffset); + while (loc == null) { + nextOffset += 4096; + loc = offsetMap.get(nextOffset); + } + int offset = nextOffset; + + // Get the x/z from the paired location + short cxz = MathMan.unpairX(loc); + int cx = MathMan.unpairShortX(cxz); + int cz = MathMan.unpairShortY(cxz); + + // Get the size from the pair + int size = MathMan.unpairY(loc) << 12; + + nextOffset += size; + end = Math.min(start + size, end); + int pair = MathMan.pair((short) (cx & 31), (short) (cz & 31)); + byte[] newBytes = relocate.get(pair); + + // newBytes is null if the chunk isn't modified or marked for moving + if (newBytes == null) { + MCAChunk cached = getCachedChunk(cx, cz); + // If the previous offset marks the current write position (start) then we only write the header + if (offset == start) { + if (cached == null || !cached.isModified()) { + writeHeader(raf, cx, cz, start >> 12, size >> 12, true); + start += size; + written = start + size; + continue; + } else { + newBytes = compressedMap.get(pair); + } + } else { + // The chunk needs to be moved, fetch the data if necessary + newBytes = compressedMap.get(pair); + if (newBytes == null) { + if (cached == null || !cached.isDeleted()) { + newBytes = getChunkCompressedBytes(getOffset(cx, cz)); + } + } + } + } + + if (newBytes == null) { + writeHeader(raf, cx, cz, 0, 0, false); + continue; + } + + // The length to be written (compressed data + 5 byte chunk header) + int len = newBytes.length + 5; + int oldSize = (size + 4095) >> 12; + int newSize = (len + 4095) >> 12; + int nextOffset2 = end; + + // If the current write position (start) + length of data to write (len) are longer than the position of the next chunk, we need to move the next chunks + while (start + len > end) { + Integer nextLoc = offsetMap.get(nextOffset2); + if (nextLoc != null) { + short nextCXZ = MathMan.unpairX(nextLoc); + int nextCX = MathMan.unpairShortX(nextCXZ); + int nextCZ = MathMan.unpairShortY(nextCXZ); + MCAChunk cached = getCachedChunk(nextCX, nextCZ); + if (cached == null || !cached.isModified()) { + byte[] nextBytes = getChunkCompressedBytes(nextOffset2); + relocate.put(MathMan.pair((short) (nextCX & 31), (short) (nextCZ & 31)), nextBytes); + } + int nextSize = MathMan.unpairY(nextLoc) << 12; + end += nextSize; + nextOffset2 += nextSize; + } else { + end += 4096; + nextOffset2 += 4096; + } + } + // Write the chunk + chunk header + writeSafe(raf, start, newBytes); + // Write the location data (beginning of file) + writeHeader(raf, cx, cz, start >> 12, newSize, true); + + written = start + newBytes.length + 5; + start += newSize << 12; + } + + // Write all the chunks which need to be appended + if (!append.isEmpty()) { + for (Int2ObjectMap.Entry entry : append.int2ObjectEntrySet()) { + int pair = entry.getIntKey(); + short cx = MathMan.unpairX(pair); + short cz = MathMan.unpairY(pair); + byte[] bytes = entry.getValue(); + int len = bytes.length + 5; + int newSize = (len + 4095) >> 12; + writeSafe(raf, start, bytes); + writeHeader(raf, cx, cz, start >> 12, newSize, true); + written = start + bytes.length + 5; + start += newSize << 12; + } + } + // Round the file length, since the vanilla server doesn't like it for some reason + raf.setLength(4096 * ((written + 4095) / 4096)); + if (raf instanceof BufferedRandomAccessFile) { + ((BufferedRandomAccessFile) raf).flush(); + } + raf.close(); + } catch (Throwable e) { + e.printStackTrace(); + } + if (wait) { + pool.shutdown(); + pool.awaitQuiescence(Long.MAX_VALUE, TimeUnit.MILLISECONDS); + } + } + } + CleanableThreadLocal.clean(byteStore1); + CleanableThreadLocal.clean(byteStore2); + CleanableThreadLocal.clean(byteStore3); + } +} \ No newline at end of file diff --git a/worldedit-core/src/main/java/com/boydti/fawe/jnbt/streamer/StreamDelegate.java b/worldedit-core/src/main/java/com/boydti/fawe/jnbt/streamer/StreamDelegate.java index 7503db628..58f8cef4f 100644 --- a/worldedit-core/src/main/java/com/boydti/fawe/jnbt/streamer/StreamDelegate.java +++ b/worldedit-core/src/main/java/com/boydti/fawe/jnbt/streamer/StreamDelegate.java @@ -190,10 +190,10 @@ public class StreamDelegate { if (lazyReader != null) { lazyReader.apply(0, is); } else if (elemReader != null) { - Object raw = is.readTagPaylodRaw(type, depth); + Object raw = is.readTagPayloadRaw(type, depth); elemReader.apply(0, raw); } else if (valueReader != null) { - Object raw = is.readTagPaylodRaw(type, depth); + Object raw = is.readTagPayloadRaw(type, depth); valueReader.apply(0, raw); } else { is.readTagPaylodLazy(type, depth + 1, this); diff --git a/worldedit-core/src/main/java/com/sk89q/jnbt/NBTInputStream.java b/worldedit-core/src/main/java/com/sk89q/jnbt/NBTInputStream.java index b8e76867d..414b00635 100644 --- a/worldedit-core/src/main/java/com/sk89q/jnbt/NBTInputStream.java +++ b/worldedit-core/src/main/java/com/sk89q/jnbt/NBTInputStream.java @@ -283,7 +283,7 @@ public final class NBTInputStream implements Closeable { ValueReader valueReader = scope.getValueReader(); if (valueReader != null) { for (int i = 0; i < length; ++i) { - valueReader.apply(i, readTagPaylodRaw(childType, depth + 1)); + valueReader.apply(i, readTagPayloadRaw(childType, depth + 1)); } return; } @@ -316,7 +316,7 @@ public final class NBTInputStream implements Closeable { return; } is.skipBytes(is.readShort() & 0xFFFF); - Object child = readTagPaylodRaw(childType, depth); + Object child = readTagPayloadRaw(childType, depth); elem.apply(i, child); } } @@ -390,7 +390,7 @@ public final class NBTInputStream implements Closeable { } } - public Object readTagPaylodRaw(int type, int depth) throws IOException { + public Object readTagPayloadRaw(int type, int depth) throws IOException { switch (type) { case NBTConstants.TYPE_END: if (depth == 0) { @@ -421,30 +421,31 @@ public final class NBTInputStream implements Closeable { bytes = new byte[length]; is.readFully(bytes); return (new String(bytes, NBTConstants.CHARSET)); - case NBTConstants.TYPE_LIST: + case NBTConstants.TYPE_LIST: { int childType = is.readByte(); length = is.readInt(); - List tagList = new ArrayList<>(); + List tagList = new ArrayList<>(length); for (int i = 0; i < length; ++i) { - Tag tag = readTagPayload(childType, depth + 1); - if (tag instanceof EndTag) { + Object tag = readTagPayloadRaw(childType, depth + 1); + if (tag == null) { throw new IOException("TAG_End not permitted in a list."); } tagList.add(tag); } return (tagList); - case NBTConstants.TYPE_COMPOUND: - Map tagMap = new HashMap<>(); + } + case NBTConstants.TYPE_COMPOUND: { + Map tagMap = new HashMap<>(); while (true) { - NamedTag namedTag = readNamedTag(depth + 1); - Tag tag = namedTag.getTag(); - if (tag instanceof EndTag) { - break; - } else { - tagMap.put(namedTag.getName(), tag); + int childType = is.readByte(); + if (childType == NBTConstants.TYPE_END) { + return tagMap; } + String name = readNamedTagName(childType); + Object value = readTagPayloadRaw(childType, depth + 1); + tagMap.put(name, value); } - return (tagMap); + } case NBTConstants.TYPE_INT_ARRAY: { length = is.readInt(); int[] data = new int[length];