async chunk loading

worldedit-bukkit/src/main/java/com/boydti/fawe/bukkit/beta/BukkitChunkHolder.java

@@ -6,7 +6,9 @@ import com.boydti.fawe.beta.IQueueExtent;
 import com.boydti.fawe.beta.implementation.blocks.CharSetBlocks;
 import com.boydti.fawe.beta.implementation.holder.ChunkHolder;
 
-public class BukkitChunkHolder extends ChunkHolder<Boolean> {
+import java.util.concurrent.Future;
+
+public class BukkitChunkHolder<T extends Future<T>> extends ChunkHolder {
     @Override
     public void init(final IQueueExtent extent, final int X, final int Z) {
         super.init(extent, X, Z);
@@ -19,30 +21,37 @@ public class BukkitChunkHolder extends ChunkHolder<Boolean> {
     }
 
     @Override
-    public boolean applyAsync() {
+    public T call() {
-        BukkitGetBlocks get = (BukkitGetBlocks) cachedGet();
+        BukkitQueue extent = (BukkitQueue) getExtent();
-        CharSetBlocks set = (CharSetBlocks) cachedSet();
+        BukkitGetBlocks get = (BukkitGetBlocks) getOrCreateGet();
-//        - getBlocks
+        CharSetBlocks set = (CharSetBlocks) getOrCreateSet();
-//            - set lock
+
-//            - synchronize on chunk object
+
-//            - verify section is same object as chunk's section
+
-//            - merge with setblocks
+        /*
-//        - set section
+
-//            - verify chunk is same
+ - getBlocks
-//            - verify section is same
+ - set ChunkSection lock with a tracking lock
-//            - Otherwise repeat steps on main thread
+ - synchronize on chunk object (so no other FAWE thread updates it at the same time)
-//            - set status to needs relighting
+ - verify cached section is same object as NMS chunk section
-//        - mark as dirty
+	otherwise, fetch the new section, set the tracking lock and reconstruct the getBlocks array
-//        - skip verification if main thread
+ - Merge raw getBlocks and setBlocks array
+ - Construct the ChunkSection
+ - In parallel on the main thread
+ - if the tracking lock has had no updates and the cached ChunkSection == the NMS chunk section
+ - Otherwise, reconstruct the ChunkSection (TODO: Benchmark if this is a performance concern)
+ - swap in the new ChunkSection
+ - Update tile entities/entities (if necessary)
+ - Merge the biome array (if necessary)
+ - set chunk status to needs relighting
+ - mark as dirty
+
+         */
+
         throw new UnsupportedOperationException("Not implemented");
 //        return true;
     }
 
-    @Override
-    public boolean applySync() {
-        return true;
-    }
-
     @Override
     public void set(final Filter filter) {
         // for each block

worldedit-bukkit/src/main/java/com/boydti/fawe/bukkit/beta/BukkitFullChunk.java

@@ -1,42 +0,0 @@
-package com.boydti.fawe.bukkit.beta;
-
-import com.boydti.fawe.beta.Filter;
-import com.boydti.fawe.beta.IQueueExtent;
-import com.boydti.fawe.beta.ISetBlocks;
-import com.boydti.fawe.beta.implementation.holder.ChunkHolder;
-
-public class BukkitFullChunk extends ChunkHolder {
-    public BukkitFullChunk() {
-        throw new UnsupportedOperationException("Not implemented");
-    }
-
-    @Override
-    public void init(IQueueExtent extent, int X, int Z) {
-
-    }
-
-    @Override
-    public boolean applyAsync() {
-        return false;
-    }
-
-    @Override
-    public boolean applySync() {
-        return false;
-    }
-
-    @Override
-    public void set(Filter filter) {
-
-    }
-
-    @Override
-    public Object get() {
-        return null;
-    }
-
-    @Override
-    public ISetBlocks set() {
-        return null;
-    }
-}

worldedit-bukkit/src/main/java/com/boydti/fawe/bukkit/beta/BukkitGetBlocks.java

@@ -24,10 +24,10 @@ import net.minecraft.server.v1_13_R2.World;
 import static com.boydti.fawe.bukkit.v0.BukkitQueue_0.getAdapter;
 
 public class BukkitGetBlocks extends CharGetBlocks {
-    private ChunkSection[] sections;
+    public ChunkSection[] sections;
-    private Chunk nmsChunk;
+    public Chunk nmsChunk;
-    private World nmsWorld;
+    public World nmsWorld;
-    private int X, Z;
+    public int X, Z;
 
     public BukkitGetBlocks(World nmsWorld, int X, int Z) {/*d*/
         this.nmsWorld = nmsWorld;
@@ -48,12 +48,12 @@ public class BukkitGetBlocks extends CharGetBlocks {
     }
 
     @Override
-    protected char[] load(int layer) {
+    public char[] load(int layer) {
         return load(layer, null);
     }
 
     @Override
-    protected char[] load(int layer, char[] data) {
+    public char[] load(int layer, char[] data) {
         ChunkSection section = getSections()[layer];
         // Section is null, return empty array
         if (section == null) {
@@ -136,7 +136,7 @@ public class BukkitGetBlocks extends CharGetBlocks {
         return data;
     }
 
-    private ChunkSection[] getSections() {
+    public ChunkSection[] getSections() {
         ChunkSection[] tmp = sections;
         if (tmp == null) {
             Chunk chunk = getChunk();
@@ -145,16 +145,10 @@ public class BukkitGetBlocks extends CharGetBlocks {
         return tmp;
     }
 
-    private Chunk getChunk() {
+    public Chunk getChunk() {
         Chunk tmp = nmsChunk;
         if (tmp == null) {
-            ChunkProviderServer provider = (ChunkProviderServer) nmsWorld.getChunkProvider();
+            nmsChunk = tmp = BukkitQueue.ensureLoaded(nmsWorld, X, Z);
-            nmsChunk = tmp = provider.chunks.get(ChunkCoordIntPair.a(X, Z));
-            if (tmp == null) {
-                System.out.println("SYNC");
-                // TOOD load with paper
-                nmsChunk = tmp = TaskManager.IMP.sync(() -> nmsWorld.getChunkAt(X, Z));
-            }
         }
         return tmp;
     }

worldedit-bukkit/src/main/java/com/boydti/fawe/bukkit/beta/BukkitQueue.java

@@ -1,16 +1,43 @@
 package com.boydti.fawe.bukkit.beta;
 
+import com.boydti.fawe.Fawe;
+import com.boydti.fawe.FaweCache;
 import com.boydti.fawe.beta.IChunk;
 import com.boydti.fawe.beta.implementation.SimpleCharQueueExtent;
 import com.boydti.fawe.beta.implementation.SingleThreadQueueExtent;
 import com.boydti.fawe.beta.implementation.WorldChunkCache;
+import com.boydti.fawe.bukkit.adapter.v1_13_1.BlockMaterial_1_13;
+import com.boydti.fawe.config.Settings;
+import com.boydti.fawe.jnbt.anvil.BitArray4096;
 import com.boydti.fawe.object.collection.IterableThreadLocal;
+import com.boydti.fawe.util.MathMan;
+import com.boydti.fawe.util.TaskManager;
 import com.sk89q.worldedit.bukkit.BukkitWorld;
 import com.sk89q.worldedit.world.World;
+import com.sk89q.worldedit.world.block.BlockID;
+import com.sk89q.worldedit.world.block.BlockState;
+import com.sk89q.worldedit.world.block.BlockTypes;
+import net.minecraft.server.v1_13_R2.Block;
+import net.minecraft.server.v1_13_R2.Chunk;
+import net.minecraft.server.v1_13_R2.ChunkCoordIntPair;
+import net.minecraft.server.v1_13_R2.ChunkProviderServer;
+import net.minecraft.server.v1_13_R2.ChunkSection;
+import net.minecraft.server.v1_13_R2.DataBits;
+import net.minecraft.server.v1_13_R2.DataPalette;
+import net.minecraft.server.v1_13_R2.DataPaletteBlock;
+import net.minecraft.server.v1_13_R2.DataPaletteLinear;
+import net.minecraft.server.v1_13_R2.GameProfileSerializer;
+import net.minecraft.server.v1_13_R2.IBlockData;
 import net.minecraft.server.v1_13_R2.WorldServer;
 import org.bukkit.Bukkit;
+import org.bukkit.craftbukkit.v1_13_R2.CraftChunk;
 import org.bukkit.craftbukkit.v1_13_R2.CraftWorld;
 
+import java.lang.reflect.Field;
+import java.util.Arrays;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+
 import static com.google.common.base.Preconditions.checkNotNull;
 
 public class BukkitQueue extends SimpleCharQueueExtent {
@@ -45,19 +72,177 @@ public class BukkitQueue extends SimpleCharQueueExtent {
         super.reset();
     }
 
-    private static final IterableThreadLocal<BukkitFullChunk> FULL_CHUNKS = new IterableThreadLocal<BukkitFullChunk>() {
+//    private static final IterableThreadLocal<BukkitFullChunk> FULL_CHUNKS = new IterableThreadLocal<BukkitFullChunk>() {
-        @Override
+//        @Override
-        public BukkitFullChunk init() {
+//        public BukkitFullChunk init() {
-            return new BukkitFullChunk();
+//            return new BukkitFullChunk();
-        }
+//        }
-    };
+//    };
 
     @Override
     public IChunk create(boolean full) {
-        if (full) {
+//        if (full) {
-            // TODO implement
+//            //TODO implement
 //            return FULL_CHUNKS.get();
-        }
+//        }
         return new BukkitChunkHolder();
     }
+
+    /*
+    NMS fields
+    */
+    public final static Field fieldBits;
+    public final static Field fieldPalette;
+    public final static Field fieldSize;
+
+    public final static Field fieldFluidCount;
+    public final static Field fieldTickingBlockCount;
+    public final static Field fieldNonEmptyBlockCount;
+
+    static {
+        try {
+            fieldSize = DataPaletteBlock.class.getDeclaredField("i");
+            fieldSize.setAccessible(true);
+            fieldBits = DataPaletteBlock.class.getDeclaredField("a");
+            fieldBits.setAccessible(true);
+            fieldPalette = DataPaletteBlock.class.getDeclaredField("h");
+            fieldPalette.setAccessible(true);
+
+            fieldFluidCount = ChunkSection.class.getDeclaredField("e");
+            fieldFluidCount.setAccessible(true);
+            fieldTickingBlockCount = ChunkSection.class.getDeclaredField("tickingBlockCount");
+            fieldTickingBlockCount.setAccessible(true);
+            fieldNonEmptyBlockCount = ChunkSection.class.getDeclaredField("nonEmptyBlockCount");
+            fieldNonEmptyBlockCount.setAccessible(true);
+        } catch (RuntimeException e) {
+            throw e;
+        } catch (Throwable rethrow) {
+            rethrow.printStackTrace();
+            throw new RuntimeException(rethrow);
+        }
+    }
+
+    private static boolean PAPER = true;
+
+    public Chunk ensureLoaded(int X, int Z) {
+        return ensureLoaded(nmsWorld, X, Z);
+    }
+
+    public static Chunk ensureLoaded(net.minecraft.server.v1_13_R2.World nmsWorld, int X, int Z) {
+        ChunkProviderServer provider = (ChunkProviderServer) nmsWorld.getChunkProvider();
+        Chunk nmsChunk = provider.chunks.get(ChunkCoordIntPair.a(X, Z));
+        if (nmsChunk != null) {
+            return nmsChunk;
+        }
+        if (Fawe.isMainThread()) {
+            return nmsWorld.getChunkAt(X, Z);
+        }
+        if (PAPER) {
+            CraftWorld craftWorld = nmsWorld.getWorld();
+            CompletableFuture<org.bukkit.Chunk> future = craftWorld.getChunkAtAsync(X, Z, true);
+            try {
+                CraftChunk chunk = (CraftChunk) future.get();
+                return chunk.getHandle();
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            } catch (ExecutionException e) {
+                e.printStackTrace();
+            } catch (Throwable e) {
+                System.out.println("Error, cannot load chunk async (paper not installed?)");
+                PAPER = false;
+            }
+        }
+        // TODO optimize
+        return TaskManager.IMP.sync(() -> nmsWorld.getChunkAt(X, Z));
+    }
+
+    /*
+    NMS conversion
+     */
+
+    public static ChunkSection newChunkSection(final int y2, final boolean flag, final char[] blocks) {
+        ChunkSection section = new ChunkSection(y2 << 4, flag);
+        if (blocks == null) {
+            return section;
+        }
+        final int[] blockToPalette = FaweCache.BLOCK_TO_PALETTE.get();
+        final int[] paletteToBlock = FaweCache.PALETTE_TO_BLOCK.get();
+        final long[] blockstates = FaweCache.BLOCK_STATES.get();
+        final int[] blocksCopy = FaweCache.SECTION_BLOCKS.get();
+        try {
+            int num_palette = 0;
+            int air = 0;
+            for (int i = 0; i < 4096; i++) {
+                char ordinal = blocks[i];
+                switch (ordinal) {
+                    case 0:
+                    case BlockID.AIR:
+                    case BlockID.CAVE_AIR:
+                    case BlockID.VOID_AIR:
+                        air++;
+                }
+                int palette = blockToPalette[ordinal];
+                if (palette == Integer.MAX_VALUE) {
+                    blockToPalette[ordinal] = palette = num_palette;
+                    paletteToBlock[num_palette] = ordinal;
+                    num_palette++;
+                }
+                blocksCopy[i] = palette;
+            }
+
+            // BlockStates
+            int bitsPerEntry = MathMan.log2nlz(num_palette - 1);
+            if (Settings.IMP.PROTOCOL_SUPPORT_FIX || num_palette != 1) {
+                bitsPerEntry = Math.max(bitsPerEntry, 4); // Protocol support breaks <4 bits per entry
+            } else {
+                bitsPerEntry = Math.max(bitsPerEntry, 1); // For some reason minecraft needs 4096 bits to store 0 entries
+            }
+
+            final int blockBitArrayEnd = (bitsPerEntry * 4096) >> 6;
+            if (num_palette == 1) {
+                for (int i = 0; i < blockBitArrayEnd; i++) blockstates[i] = 0;
+            } else {
+                final BitArray4096 bitArray = new BitArray4096(blockstates, bitsPerEntry);
+                bitArray.fromRaw(blocksCopy);
+            }
+
+            // set palette & data bits
+            final DataPaletteBlock<IBlockData> dataPaletteBlocks = section.getBlocks();
+            // private DataPalette<T> h;
+            // protected DataBits a;
+            final long[] bits = Arrays.copyOfRange(blockstates, 0, blockBitArrayEnd);
+            final DataBits nmsBits = new DataBits(bitsPerEntry, 4096, bits);
+            final DataPalette<IBlockData> palette;
+//                palette = new DataPaletteHash<>(Block.REGISTRY_ID, bitsPerEntry, dataPaletteBlocks, GameProfileSerializer::d, GameProfileSerializer::a);
+            palette = new DataPaletteLinear<>(Block.REGISTRY_ID, bitsPerEntry, dataPaletteBlocks, GameProfileSerializer::d);
+
+            // set palette
+            for (int i = 0; i < num_palette; i++) {
+                final int ordinal = paletteToBlock[i];
+                blockToPalette[ordinal] = Integer.MAX_VALUE;
+                final BlockState state = BlockTypes.states[ordinal];
+                final IBlockData ibd = ((BlockMaterial_1_13) state.getMaterial()).getState();
+                palette.a(ibd);
+            }
+            try {
+                fieldBits.set(dataPaletteBlocks, nmsBits);
+                fieldPalette.set(dataPaletteBlocks, palette);
+                fieldSize.set(dataPaletteBlocks, bitsPerEntry);
+                setCount(0, 4096 - air, section);
+            } catch (final IllegalAccessException | NoSuchFieldException e) {
+                throw new RuntimeException(e);
+            }
+
+            return section;
+        } catch (final Throwable e){
+            Arrays.fill(blockToPalette, Integer.MAX_VALUE);
+            throw e;
+        }
+    }
+
+    public static void setCount(final int tickingBlockCount, final int nonEmptyBlockCount, final ChunkSection section) throws NoSuchFieldException, IllegalAccessException {
+        fieldFluidCount.set(section, 0); // TODO FIXME
+        fieldTickingBlockCount.set(section, tickingBlockCount);
+        fieldNonEmptyBlockCount.set(section, nonEmptyBlockCount);
+    }
 }

worldedit-bukkit/src/main/java/com/boydti/fawe/bukkit/beta/DelegateLock.java

@@ -0,0 +1,78 @@
+package com.boydti.fawe.bukkit.beta;
+
+import java.util.Collection;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.ReentrantLock;
+
+public class DelegateLock extends ReentrantLock {
+    private final ReentrantLock parent;
+    private volatile boolean modified;
+
+    public DelegateLock(ReentrantLock parent) {
+        this.parent = parent;
+    }
+
+    @Override
+    public void lock() {
+        modified = true;
+        parent.lock();
+    }
+
+    @Override
+    public synchronized void lockInterruptibly() throws InterruptedException {
+        parent.lockInterruptibly();
+    }
+
+    @Override
+    public synchronized boolean tryLock() {
+        return parent.tryLock();
+    }
+
+    @Override
+    public synchronized boolean tryLock(long timeout, TimeUnit unit) throws InterruptedException {
+        return parent.tryLock(timeout, unit);
+    }
+
+    @Override
+    public synchronized void unlock() {
+        modified = true;
+        parent.unlock();
+        this.notifyAll();
+    }
+
+    @Override
+    public synchronized Condition newCondition() {
+        return parent.newCondition();
+    }
+
+    @Override
+    public synchronized int getHoldCount() {
+        return parent.getHoldCount();
+    }
+
+    @Override
+    public synchronized boolean isHeldByCurrentThread() {
+        return parent.isHeldByCurrentThread();
+    }
+
+    @Override
+    public synchronized boolean isLocked() {
+        return parent.isLocked();
+    }
+
+    @Override
+    public synchronized boolean hasWaiters(Condition condition) {
+        return parent.hasWaiters(condition);
+    }
+
+    @Override
+    public synchronized int getWaitQueueLength(Condition condition) {
+        return parent.getWaitQueueLength(condition);
+    }
+
+    @Override
+    public synchronized String toString() {
+        return parent.toString();
+    }
+}

worldedit-bukkit/src/main/java/com/boydti/fawe/bukkit/v1_13/BukkitQueue_1_13.java

@@ -70,7 +70,6 @@ public class BukkitQueue_1_13 extends BukkitQueue_0<net.minecraft.server.v1_13_R
 
     public final static Field fieldBits;
     public final static Field fieldPalette;
-
     protected final static Field fieldSize;
 
     protected final static Field fieldHashBlocks;
@@ -88,6 +87,7 @@ public class BukkitQueue_1_13 extends BukkitQueue_0<net.minecraft.server.v1_13_R
     protected final static Field fieldFluidCount;
     protected final static Field fieldTickingBlockCount;
     protected final static Field fieldNonEmptyBlockCount;
+
     protected final static Field fieldSection;
     protected final static Field fieldLiquidCount;
     protected final static Field fieldEmittedLight;
@@ -847,85 +847,84 @@ public class BukkitQueue_1_13 extends BukkitQueue_0<net.minecraft.server.v1_13_R
     }
 
     public static ChunkSection newChunkSection(final int y2, final boolean flag, final int[] blocks) {
+        ChunkSection section = new ChunkSection(y2 << 4, flag);
         if (blocks == null) {
-            return new ChunkSection(y2 << 4, flag);
+            return section;
-        } else {
+        }
-            final ChunkSection section = new ChunkSection(y2 << 4, flag);
+        final int[] blockToPalette = FaweCache.BLOCK_TO_PALETTE.get();
-            final int[] blockToPalette = FaweCache.BLOCK_TO_PALETTE.get();
+        final int[] paletteToBlock = FaweCache.PALETTE_TO_BLOCK.get();
-            final int[] paletteToBlock = FaweCache.PALETTE_TO_BLOCK.get();
+        final long[] blockstates = FaweCache.BLOCK_STATES.get();
-            final long[] blockstates = FaweCache.BLOCK_STATES.get();
+        final int[] blocksCopy = FaweCache.SECTION_BLOCKS.get();
-            final int[] blocksCopy = FaweCache.SECTION_BLOCKS.get();
+        try {
-            try {
+            int num_palette = 0;
-                int num_palette = 0;
+            int air = 0;
-                int air = 0;
+            for (int i = 0; i < 4096; i++) {
-                for (int i = 0; i < 4096; i++) {
+                int stateId = blocks[i];
-                    int stateId = blocks[i];
+                switch (stateId) {
-                    switch (stateId) {
+                    case 0:
-                        case 0:
+                    case BlockID.AIR:
-                        case BlockID.AIR:
+                    case BlockID.CAVE_AIR:
-                        case BlockID.CAVE_AIR:
+                    case BlockID.VOID_AIR:
-                        case BlockID.VOID_AIR:
+                        stateId = BlockID.AIR;
-                            stateId = BlockID.AIR;
+                        air++;
-                            air++;
-                    }
-                    final int ordinal = BlockState.getFromInternalId(stateId).getOrdinal(); // TODO fixme Remove all use of BlockTypes.BIT_OFFSET so that this conversion isn't necessary
-                    int palette = blockToPalette[ordinal];
-                    if (palette == Integer.MAX_VALUE) {
-                        blockToPalette[ordinal] = palette = num_palette;
-                        paletteToBlock[num_palette] = ordinal;
-                        num_palette++;
-                    }
-                    blocksCopy[i] = palette;
                 }
-
+                final int ordinal = BlockState.getFromInternalId(stateId).getOrdinal(); // TODO fixme Remove all use of BlockTypes.BIT_OFFSET so that this conversion isn't necessary
-                // BlockStates
+                int palette = blockToPalette[ordinal];
-                int bitsPerEntry = MathMan.log2nlz(num_palette - 1);
+                if (palette == Integer.MAX_VALUE) {
-                if (Settings.IMP.PROTOCOL_SUPPORT_FIX || num_palette != 1) {
+                    blockToPalette[ordinal] = palette = num_palette;
-                    bitsPerEntry = Math.max(bitsPerEntry, 4); // Protocol support breaks <4 bits per entry
+                    paletteToBlock[num_palette] = ordinal;
-                } else {
+                    num_palette++;
-                    bitsPerEntry = Math.max(bitsPerEntry, 1); // For some reason minecraft needs 4096 bits to store 0 entries
                 }
-
+                blocksCopy[i] = palette;
-                final int blockBitArrayEnd = (bitsPerEntry * 4096) >> 6;
-                if (num_palette == 1) {
-                    for (int i = 0; i < blockBitArrayEnd; i++) blockstates[i] = 0;
-                } else {
-                    final BitArray4096 bitArray = new BitArray4096(blockstates, bitsPerEntry);
-                    bitArray.fromRaw(blocksCopy);
-                }
-
-                // set palette & data bits
-                final DataPaletteBlock<IBlockData> dataPaletteBlocks = section.getBlocks();
-                // private DataPalette<T> h;
-                // protected DataBits a;
-                final long[] bits = Arrays.copyOfRange(blockstates, 0, blockBitArrayEnd);
-                final DataBits nmsBits = new DataBits(bitsPerEntry, 4096, bits);
-                final DataPalette<IBlockData> palette;
-//                palette = new DataPaletteHash<>(Block.REGISTRY_ID, bitsPerEntry, dataPaletteBlocks, GameProfileSerializer::d, GameProfileSerializer::a);
-                palette = new DataPaletteLinear<>(Block.REGISTRY_ID, bitsPerEntry, dataPaletteBlocks, GameProfileSerializer::d);
-
-                // set palette
-                for (int i = 0; i < num_palette; i++) {
-                    final int ordinal = paletteToBlock[i];
-                    blockToPalette[ordinal] = Integer.MAX_VALUE;
-                    final BlockState state = BlockTypes.states[ordinal];
-                    final IBlockData ibd = ((BlockMaterial_1_13) state.getMaterial()).getState();
-                    palette.a(ibd);
-                }
-                try {
-                    fieldBits.set(dataPaletteBlocks, nmsBits);
-                    fieldPalette.set(dataPaletteBlocks, palette);
-                    fieldSize.set(dataPaletteBlocks, bitsPerEntry);
-                    setCount(0, 4096 - air, section);
-                } catch (final IllegalAccessException | NoSuchFieldException e) {
-                    throw new RuntimeException(e);
-                }
-
-                return section;
-            } catch (final Throwable e){
-                Arrays.fill(blockToPalette, Integer.MAX_VALUE);
-                throw e;
             }
+
+            // BlockStates
+            int bitsPerEntry = MathMan.log2nlz(num_palette - 1);
+            if (Settings.IMP.PROTOCOL_SUPPORT_FIX || num_palette != 1) {
+                bitsPerEntry = Math.max(bitsPerEntry, 4); // Protocol support breaks <4 bits per entry
+            } else {
+                bitsPerEntry = Math.max(bitsPerEntry, 1); // For some reason minecraft needs 4096 bits to store 0 entries
+            }
+
+            final int blockBitArrayEnd = (bitsPerEntry * 4096) >> 6;
+            if (num_palette == 1) {
+                for (int i = 0; i < blockBitArrayEnd; i++) blockstates[i] = 0;
+            } else {
+                final BitArray4096 bitArray = new BitArray4096(blockstates, bitsPerEntry);
+                bitArray.fromRaw(blocksCopy);
+            }
+
+            // set palette & data bits
+            final DataPaletteBlock<IBlockData> dataPaletteBlocks = section.getBlocks();
+            // private DataPalette<T> h;
+            // protected DataBits a;
+            final long[] bits = Arrays.copyOfRange(blockstates, 0, blockBitArrayEnd);
+            final DataBits nmsBits = new DataBits(bitsPerEntry, 4096, bits);
+            final DataPalette<IBlockData> palette;
+//                palette = new DataPaletteHash<>(Block.REGISTRY_ID, bitsPerEntry, dataPaletteBlocks, GameProfileSerializer::d, GameProfileSerializer::a);
+            palette = new DataPaletteLinear<>(Block.REGISTRY_ID, bitsPerEntry, dataPaletteBlocks, GameProfileSerializer::d);
+
+            // set palette
+            for (int i = 0; i < num_palette; i++) {
+                final int ordinal = paletteToBlock[i];
+                blockToPalette[ordinal] = Integer.MAX_VALUE;
+                final BlockState state = BlockTypes.states[ordinal];
+                final IBlockData ibd = ((BlockMaterial_1_13) state.getMaterial()).getState();
+                palette.a(ibd);
+            }
+            try {
+                fieldBits.set(dataPaletteBlocks, nmsBits);
+                fieldPalette.set(dataPaletteBlocks, palette);
+                fieldSize.set(dataPaletteBlocks, bitsPerEntry);
+                setCount(0, 4096 - air, section);
+            } catch (final IllegalAccessException | NoSuchFieldException e) {
+                throw new RuntimeException(e);
+            }
+
+            return section;
+        } catch (final Throwable e){
+            Arrays.fill(blockToPalette, Integer.MAX_VALUE);
+            throw e;
         }
     }
 

worldedit-core/src/main/java/com/boydti/fawe/FaweCache.java

@@ -1,6 +1,7 @@
 package com.boydti.fawe;
 
 import com.boydti.fawe.beta.Trimable;
+import com.boydti.fawe.config.Settings;
 import com.boydti.fawe.jnbt.anvil.BitArray4096;
 import com.boydti.fawe.object.collection.IterableThreadLocal;
 import com.boydti.fawe.util.MathMan;
@@ -11,6 +12,12 @@ import com.sk89q.worldedit.world.block.BlockTypes;
 
 import java.lang.reflect.Field;
 import java.util.*;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
 
 public class FaweCache implements Trimable {
     public static final char[] EMPTY_CHAR_4096 = new char[4096];
@@ -326,4 +333,16 @@ public class FaweCache implements Trimable {
         if (clazz == null) clazz = EndTag.class;
         return new ListTag(clazz, list);
     }
+
+    /*
+    Thread stuff
+     */
+    public static ThreadPoolExecutor newBlockingExecutor() {
+        int nThreads = Settings.IMP.QUEUE.PARALLEL_THREADS;
+        ArrayBlockingQueue<Runnable> queue = new ArrayBlockingQueue<>(nThreads);
+        return new ThreadPoolExecutor(nThreads, nThreads,
+                0L, TimeUnit.MILLISECONDS, queue
+                , Executors.defaultThreadFactory(),
+                new ThreadPoolExecutor.CallerRunsPolicy());
+    }
 }

worldedit-core/src/main/java/com/boydti/fawe/beta/CharFilterBlock.java

@@ -13,12 +13,12 @@ public class CharFilterBlock implements FilterBlock {
     private char[] section;
 
     @Override
-    public void init(IQueueExtent queue) {
+    public final void init(IQueueExtent queue) {
         this.queue = queue;
     }
 
     @Override
-    public void init(int X, int Z, IGetBlocks chunk) {
+    public final void init(int X, int Z, IGetBlocks chunk) {
         this.chunk = (CharGetBlocks) chunk;
         this.X = X;
         this.Z = Z;
@@ -26,68 +26,85 @@ public class CharFilterBlock implements FilterBlock {
         this.zz = Z << 4;
     }
 
-    public void init(char[] section, int layer) {
+    // local
-        this.section = section;
+    private int layer, index, x, y, z, xx, yy, zz, X, Z;
-        this.layer = layer;
+
-        this.yy = layer << 4;
+    public final void filter(CharGetBlocks blocks, Filter filter) {
+        for (int layer = 0; layer < 16; layer++) {
+            if (!blocks.hasSection(layer)) continue;
+            char[] arr = blocks.sections[layer].get(blocks, layer);
+
+            this.section = arr;
+            this.layer = layer;
+            this.yy = layer << 4;
+
+            for (y = 0, index = 0; y < 16; y++) {
+                for (z = 0; z < 16; z++) {
+                    for (x = 0; x < 16; x++, index++) {
+                        filter.applyBlock(this);
+                    }
+                }
+            }
+        }
     }
 
-    // local
-    public int layer, index, x, y, z, xx, yy, zz, X, Z;
-
     @Override
-    public int getX() {
+    public final int getX() {
         return xx + x;
     }
 
     @Override
-    public int getY() {
+    public final int getY() {
         return yy + y;
     }
 
     @Override
-    public int getZ() {
+    public final int getZ() {
         return zz + z;
     }
 
     @Override
-    public  int getLocalX() {
+    public final int getLocalX() {
         return x;
     }
 
     @Override
-    public  int getLocalY() {
+    public final int getLocalY() {
         return y;
     }
 
     @Override
-    public  int getLocalZ() {
+    public final int getLocalZ() {
         return z;
     }
 
     @Override
-    public  int getChunkX() {
+    public final int getChunkX() {
         return X;
     }
 
     @Override
-    public  int getChunkZ() {
+    public final int getChunkZ() {
         return Z;
     }
 
-    @Override
+    public final char getOrdinalChar() {
-    public int getOrdinal() {
         return section[index];
     }
 
     @Override
-    public BlockState getState() {
+    public final int getOrdinal() {
+        return section[index];
+    }
+
+    @Override
+    public final BlockState getState() {
         int ordinal = section[index];
         return BlockTypes.states[ordinal];
     }
 
     @Override
-    public BaseBlock getBaseBlock() {
+    public final BaseBlock getBaseBlock() {
         BlockState state = getState();
         BlockMaterial material = state.getMaterial();
         if (material.hasContainer()) {
@@ -98,11 +115,11 @@ public class CharFilterBlock implements FilterBlock {
     }
 
     @Override
-    public CompoundTag getTag() {
+    public final CompoundTag getTag() {
         return null;
     }
 
-    public BlockState getOrdinalBelow() {
+    public final BlockState getOrdinalBelow() {
         if (y > 0) {
             return states[section[index - 256]];
         }
@@ -114,7 +131,7 @@ public class CharFilterBlock implements FilterBlock {
         return BlockTypes.__RESERVED__.getDefaultState();
     }
 
-    public BlockState getStateAbove() {
+    public final BlockState getStateAbove() {
         if (y < 16) {
             return states[section[index + 256]];
         }
@@ -126,7 +143,7 @@ public class CharFilterBlock implements FilterBlock {
         return BlockTypes.__RESERVED__.getDefaultState();
     }
 
-    public BlockState getStateRelativeY(int y) {
+    public final BlockState getStateRelativeY(int y) {
         int newY = this.y + y;
         int layerAdd = newY >> 4;
         switch (layerAdd) {
@@ -180,7 +197,7 @@ public class CharFilterBlock implements FilterBlock {
         return BlockTypes.__RESERVED__.getDefaultState();
     }
 
-    public BlockState getStateRelative(final int x, final int y, final int z) {
+    public final BlockState getStateRelative(final int x, final int y, final int z) {
         int newX = this.x + x;
         if (newX >> 4 == 0) {
             int newZ = this.z + z;
@@ -189,7 +206,7 @@ public class CharFilterBlock implements FilterBlock {
                 int layerAdd = newY >> 4;
                 switch (layerAdd) {
                     case 0:
-                        return states[section[this.index + ((y << 8) | (z << 4) | x)]];
+                        return states[section[this.index + ((y << 8) + (z << 4) + x)]];
                     case 1:
                     case 2:
                     case 3:
@@ -207,7 +224,7 @@ public class CharFilterBlock implements FilterBlock {
                     case 15: {
                         int newLayer = layer + layerAdd;
                         if (newLayer < 16) {
-                            int index = this.index + (((y & 15) << 8) | (z << 4) | x);
+                            int index = ((newY & 15) << 8) + (newZ << 4) + newX;
                             return states[chunk.sections[newLayer].get(chunk, newLayer, index)];
                         }
                         break;
@@ -229,7 +246,7 @@ public class CharFilterBlock implements FilterBlock {
                     case -15: {
                         int newLayer = layer + layerAdd;
                         if (newLayer >= 0) {
-                            int index = this.index + (((y & 15) << 8) | (z << 4) | x);
+                            int index = ((newY & 15) << 8) + (newZ << 4) + newX;
                             return states[chunk.sections[newLayer].get(chunk, newLayer, index)];
                         }
                         break;

worldedit-core/src/main/java/com/boydti/fawe/beta/IChunk.java

@@ -5,11 +5,16 @@ import com.sk89q.worldedit.world.block.BaseBlock;
 import com.sk89q.worldedit.world.block.BlockState;
 import com.sk89q.worldedit.world.block.BlockStateHolder;
 
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.function.Supplier;
+
 /**
  * Represents a chunk in the queue {@link IQueueExtent}
  * Used for getting and setting blocks / biomes / entities
  */
-public interface IChunk extends Trimable {
+public interface IChunk<T extends Future<T>> extends Trimable, Callable<T> {
     /**
      * Initialize at the location
      * @param extent
@@ -22,6 +27,8 @@ public interface IChunk extends Trimable {
 
     int getZ();
 
+
+
     /**
      * If the chunk is a delegate, returns it's paren'ts root
      * @return root IChunk
@@ -36,16 +43,33 @@ public interface IChunk extends Trimable {
     boolean isEmpty();
 
     /**
-     * Apply the queued async changes to the world
+     * Spend time optimizing for apply<br>
-     * @return false if applySync needs to run
+     * default behavior: do nothing
      */
-    boolean applyAsync();
+    default void optimize() {
+
+    }
 
     /**
-     * Apply the queued sync changes to the world
+     * Apply the queued changes to the world<br>
-     * @return true
+     * The future returned may return another future<br>
+     * To ensure completion keep calling {@link Future#get()} on each result
+     * @return Futures
      */
-    boolean applySync();
+    T call();
+
+    /**
+     * Call and join
+     * @throws ExecutionException
+     * @throws InterruptedException
+     */
+    default void join() throws ExecutionException, InterruptedException {
+        T future = call();
+        while (future != null) {
+            future = future.get();
+        }
+        return;
+    }
 
     /* set - queues a change */
     boolean setBiome(int x, int y, int z, BiomeType biome);

worldedit-core/src/main/java/com/boydti/fawe/beta/IDelegateChunk.java

@@ -5,6 +5,9 @@ import com.sk89q.worldedit.world.block.BaseBlock;
 import com.sk89q.worldedit.world.block.BlockState;
 import com.sk89q.worldedit.world.block.BlockStateHolder;
 
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+
 /**
  * Delegate for IChunk
  * @param <U> parent class
@@ -67,13 +70,13 @@ public interface IDelegateChunk<U extends IChunk> extends IChunk {
     }
 
     @Override
-    default boolean applySync() {
+    default Future call() {
-        return getParent().applySync();
+        return getParent().call();
     }
 
     @Override
-    default boolean applyAsync() {
+    default void join() throws ExecutionException, InterruptedException {
-        return getParent().applyAsync();
+        getParent().join();
     }
 
     @Override

worldedit-core/src/main/java/com/boydti/fawe/beta/IGetBlocks.java

@@ -21,4 +21,8 @@ public interface IGetBlocks extends IBlocks, Trimable {
     boolean trim(boolean aggressive);
 
     void filter(Filter filter, FilterBlock block);
+
+    default void optimize() {
+
+    }
 }

worldedit-core/src/main/java/com/boydti/fawe/beta/IQueueExtent.java

@@ -34,7 +34,7 @@ public interface IQueueExtent extends Flushable, Trimable {
      * @param chunk
      * @return result
      */
-    Future<?> submit(IChunk chunk);
+    <T extends Future<T>> T submit(IChunk<T> chunk);
 
     default boolean setBlock(final int x, final int y, final int z, final BlockStateHolder state) {
         final IChunk chunk = getCachedChunk(x >> 4, z >> 4);

worldedit-core/src/main/java/com/boydti/fawe/beta/ISetBlocks.java

@@ -12,4 +12,8 @@ public interface ISetBlocks extends IBlocks {
     boolean setBlock(int x, int y, int z, BlockStateHolder holder);
 
     boolean isEmpty();
+
+    default void optimize() {
+
+    }
 }

worldedit-core/src/main/java/com/boydti/fawe/beta/implementation/QueueHandler.java

@@ -1,6 +1,6 @@
 package com.boydti.fawe.beta.implementation;
 
-import com.boydti.fawe.Fawe;
+import com.boydti.fawe.FaweCache;
 import com.boydti.fawe.beta.Filter;
 import com.boydti.fawe.beta.FilterBlock;
 import com.boydti.fawe.beta.IChunk;
@@ -8,9 +8,9 @@ import com.boydti.fawe.beta.IQueueExtent;
 import com.boydti.fawe.beta.Trimable;
 import com.boydti.fawe.config.Settings;
 import com.boydti.fawe.object.collection.IterableThreadLocal;
-import com.boydti.fawe.util.MathMan;
+import com.boydti.fawe.util.MemUtil;
-import com.boydti.fawe.util.TaskManager;
 import com.boydti.fawe.wrappers.WorldWrapper;
+import com.google.common.util.concurrent.Futures;
 import com.sk89q.worldedit.math.BlockVector2;
 import com.sk89q.worldedit.regions.Region;
 import com.sk89q.worldedit.world.World;
@@ -20,32 +20,36 @@ import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutorService;
 import java.util.concurrent.ForkJoinPool;
 import java.util.concurrent.ForkJoinTask;
 import java.util.concurrent.Future;
+import java.util.concurrent.ThreadPoolExecutor;
 
 /**
  * Class which handles all the queues {@link IQueueExtent}
  */
 public abstract class QueueHandler implements Trimable {
-    private Map<World, WeakReference<WorldChunkCache>> chunkCache = new HashMap<>();
+    private ForkJoinPool forkJoinPoolPrimary = new ForkJoinPool();
+    private ForkJoinPool forkJoinPoolSecondary = new ForkJoinPool();
+    private ThreadPoolExecutor blockingExecutor = FaweCache.newBlockingExecutor();
+    private ConcurrentLinkedQueue<Runnable> syncTasks = new ConcurrentLinkedQueue();
 
-    private IterableThreadLocal<IQueueExtent> pool = new IterableThreadLocal<IQueueExtent>() {
+    private Map<World, WeakReference<WorldChunkCache>> chunkCache = new HashMap<>();
+    private IterableThreadLocal<IQueueExtent> queuePool = new IterableThreadLocal<IQueueExtent>() {
         @Override
         public IQueueExtent init() {
             return create();
         }
     };
 
-    public Future<?> submit(IChunk chunk) {
+    public <T extends Future<T>> T submit(IChunk<T> chunk) {
-        if (Fawe.isMainThread()) {
+        if (MemUtil.isMemoryFree()) {
-            if (!chunk.applyAsync()) {
+//            return (T) forkJoinPoolSecondary.submit(chunk);
-                chunk.applySync();
-            }
-            return null;
         }
-        // TODO return future
+        return (T) blockingExecutor.submit(chunk);
-        return null;
+
     }
 
     /**
@@ -73,7 +77,7 @@ public abstract class QueueHandler implements Trimable {
     public abstract IQueueExtent create();
 
     public IQueueExtent getQueue(World world) {
-        IQueueExtent queue = pool.get();
+        IQueueExtent queue = queuePool.get();
         queue.init(getOrCreate(world));
         return queue;
     }
@@ -103,62 +107,63 @@ public abstract class QueueHandler implements Trimable {
         final Iterator<BlockVector2> chunksIter = chunks.iterator();
 
         // Get a pool, to operate on the chunks in parallel
-        final ForkJoinPool pool = TaskManager.IMP.getPublicForkJoinPool();
         final int size = Math.min(chunks.size(), Settings.IMP.QUEUE.PARALLEL_THREADS);
-        final ForkJoinTask[] tasks = new ForkJoinTask[size];
+        ForkJoinTask[] tasks = new ForkJoinTask[size];
-
         for (int i = 0; i < size; i++) {
-            tasks[i] = pool.submit(new Runnable() {
+            tasks[i] = forkJoinPoolPrimary.submit(new Runnable() {
                 @Override
                 public void run() {
                     Filter newFilter = filter.fork();
                     // Create a chunk that we will reuse/reset for each operation
                     IQueueExtent queue = getQueue(world);
-                    FilterBlock block = null;
+                    synchronized (queue) {
+                        FilterBlock block = null;
 
-                    while (true) {
+                        while (true) {
-                        // Get the next chunk pos
+                            // Get the next chunk pos
-                        final BlockVector2 pos;
+                            final BlockVector2 pos;
-                        synchronized (chunksIter) {
+                            synchronized (chunksIter) {
-                            if (!chunksIter.hasNext()) return;
+                                if (!chunksIter.hasNext()) break;
-                            pos = chunksIter.next();
+                                pos = chunksIter.next();
-                        }
-                        final int X = pos.getX();
-                        final int Z = pos.getZ();
-                        // TODO create full
-                        IChunk chunk = queue.getCachedChunk(X, Z);
-                        // Initialize
-                        chunk.init(queue, X, Z);
-                        try {
-                            if (!newFilter.appliesChunk(X, Z)) {
-                                continue;
                             }
-                            chunk = newFilter.applyChunk(chunk);
+                            final int X = pos.getX();
+                            final int Z = pos.getZ();
+                            IChunk chunk = queue.getCachedChunk(X, Z);
+                            // Initialize
+                            chunk.init(queue, X, Z);
+                            try {
+                                if (!newFilter.appliesChunk(X, Z)) {
+                                    continue;
+                                }
+                                chunk = newFilter.applyChunk(chunk);
 
-                            if (chunk == null) continue;
+                                if (chunk == null) continue;
 
-                            if (block == null) block = queue.initFilterBlock();
+                                if (block == null) block = queue.initFilterBlock();
-                            chunk.filter(newFilter, block);
+                                chunk.filter(newFilter, block);
 
-                            newFilter.finishChunk(chunk);
+                                newFilter.finishChunk(chunk);
 
-                            queue.submit(chunk);
+                                queue.submit(chunk);
-                        } finally
+                            } finally {
-                        {
+                                if (filter != newFilter) {
-                            if (filter != newFilter) {
+                                    synchronized (filter) {
-                                synchronized (filter) {
+                                        newFilter.join(filter);
-                                    newFilter.join(filter);
+                                    }
                                 }
                             }
                         }
+                        queue.flush();
                     }
                 }
             });
         }
-
+        // Join filters
-        // Join the tasks
+        for (int i = 0; i < tasks.length; i++) {
-        for (final ForkJoinTask task : tasks) {
+            ForkJoinTask task = tasks[i];
-            task.join();
+            if (task != null) {
+                task.quietlyJoin();
+            }
         }
     }
 }

worldedit-core/src/main/java/com/boydti/fawe/beta/implementation/SingleThreadQueueExtent.java

@@ -5,17 +5,12 @@ import com.boydti.fawe.beta.IChunk;
 import com.boydti.fawe.beta.IQueueExtent;
 import com.boydti.fawe.beta.implementation.holder.ReferenceChunk;
 import com.boydti.fawe.config.Settings;
-import com.boydti.fawe.util.MathMan;
 import com.boydti.fawe.util.MemUtil;
-import com.boydti.fawe.util.SetQueue;
+import com.google.common.util.concurrent.Futures;
-import com.boydti.fawe.util.TaskManager;
 import it.unimi.dsi.fastutil.longs.Long2ObjectLinkedOpenHashMap;
 
-import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ForkJoinPool;
-import java.util.concurrent.ForkJoinTask;
 import java.util.concurrent.Future;
 
 import static com.google.common.base.Preconditions.checkNotNull;
@@ -29,6 +24,7 @@ import static com.google.common.base.Preconditions.checkNotNull;
 public abstract class SingleThreadQueueExtent implements IQueueExtent {
     private WorldChunkCache cache;
     private Thread currentThread;
+    private ConcurrentLinkedQueue<Future> submissions = new ConcurrentLinkedQueue<>();
 
     /**
      * Safety check to ensure that the thread being used matches the one being initialized on
@@ -66,7 +62,7 @@ public abstract class SingleThreadQueueExtent implements IQueueExtent {
      */
     @Override
     public synchronized void init(final WorldChunkCache cache) {
-        if (cache != null) {
+        if (this.cache != null) {
             reset();
         }
         currentThread = Thread.currentThread();
@@ -83,19 +79,17 @@ public abstract class SingleThreadQueueExtent implements IQueueExtent {
     private static final ConcurrentLinkedQueue<IChunk> CHUNK_POOL = new ConcurrentLinkedQueue<>();
 
     @Override
-    public Future<?> submit(final IChunk chunk) {
+    public <T extends Future<T>> T submit(final IChunk<T> chunk) {
         if (chunk.isEmpty()) {
             CHUNK_POOL.add(chunk);
-            return null;
+            return (T) (Future) Futures.immediateFuture(null);
         }
+
         if (Fawe.isMainThread()) {
-            if (!chunk.applyAsync()) {
+            return chunk.call();
-                chunk.applySync();
-            }
-            return null;
         }
-        QueueHandler handler = Fawe.get().getQueueHandler();
+
-        return handler.submit(chunk);
+        return Fawe.get().getQueueHandler().submit(chunk);
     }
 
     @Override
@@ -107,6 +101,13 @@ public abstract class SingleThreadQueueExtent implements IQueueExtent {
             lastPair = Long.MAX_VALUE;
             return chunks.isEmpty();
         }
+        if (!submissions.isEmpty()) {
+            if (aggressive) {
+                pollSubmissions(0, aggressive);
+            } else {
+                pollSubmissions(Settings.IMP.QUEUE.PARALLEL_THREADS, aggressive);
+            }
+        }
         synchronized (this) {
             return currentThread == null;
         }
@@ -121,7 +122,10 @@ public abstract class SingleThreadQueueExtent implements IQueueExtent {
      */
     private IChunk poolOrCreate(final int X, final int Z) {
         IChunk next = CHUNK_POOL.poll();
-        if (next == null) next = create(false);
+        if (next == null) {
+            System.out.println("Create");
+            next = create(false);
+        }
         next.init(this, X, Z);
         return next;
     }
@@ -145,10 +149,19 @@ public abstract class SingleThreadQueueExtent implements IQueueExtent {
 
         checkThread();
         final int size = chunks.size();
-        if (size > Settings.IMP.QUEUE.TARGET_SIZE || MemUtil.isMemoryLimited()) {
+        boolean lowMem = MemUtil.isMemoryLimited();
-            if (size > Settings.IMP.QUEUE.PARALLEL_THREADS * 2 + 16) {
+        if (lowMem || size > Settings.IMP.QUEUE.TARGET_SIZE) {
-                chunk = chunks.removeFirst();
+            chunk = chunks.removeFirst();
-                submit(chunk);
+            Future future = submit(chunk);
+            if (future != null && !future.isDone()) {
+                int targetSize;
+                if (lowMem) {
+                    targetSize = Settings.IMP.QUEUE.PARALLEL_THREADS;
+                } else {
+                    targetSize = Settings.IMP.QUEUE.TARGET_SIZE;
+                }
+                pollSubmissions(targetSize, true);
+                submissions.add(future);
             }
         }
         chunk = poolOrCreate(X, Z);
@@ -161,27 +174,59 @@ public abstract class SingleThreadQueueExtent implements IQueueExtent {
         return chunk;
     }
 
+    private void pollSubmissions(int targetSize, boolean aggressive) {
+        int overflow = submissions.size() - targetSize;
+        if (aggressive) {
+            for (int i = 0; i < overflow; i++) {
+                Future first = submissions.poll();
+                try {
+                    while ((first = (Future) first.get()) != null) ;
+                } catch (InterruptedException | ExecutionException e) {
+                    e.printStackTrace();
+                }
+            }
+        } else {
+            for (int i = 0; i < overflow; i++) {
+                Future next = submissions.peek();
+                while (next != null) {
+                    if (next.isDone()) {
+                        try {
+                            next = (Future) next.get();
+                        } catch (InterruptedException | ExecutionException e) {
+                            e.printStackTrace();
+                        }
+                    } else {
+                        return;
+                    }
+                }
+                submissions.poll();
+            }
+        }
+    }
+
     @Override
     public synchronized void flush() {
         checkThread();
         if (!chunks.isEmpty()) {
-            final Future[] tasks = new ForkJoinTask[chunks.size()];
+            if (MemUtil.isMemoryLimited()) {
-            int i = 0;
+                for (IChunk chunk : chunks.values()) {
-            for (final IChunk chunk : chunks.values()) {
+                    Future future = submit(chunk);
-                tasks[i++] = submit(chunk);
+                    if (future != null && !future.isDone()) {
-            }
+                        pollSubmissions(Settings.IMP.QUEUE.PARALLEL_THREADS, true);
-            chunks.clear();
+                        submissions.add(future);
-            for (final Future task : tasks) {
+                    }
-                if (task != null) {
+                }
-                    try {
+            } else {
-                        task.get();
+                for (final IChunk chunk : chunks.values()) {
-                    } catch (InterruptedException | ExecutionException e) {
+                    Future future = submit(chunk);
-                        e.printStackTrace();
+                    if (future != null && !future.isDone()) {
-                        throw new RuntimeException(e);
+                        submissions.add(future);
                     }
                 }
             }
+            chunks.clear();
         }
+        pollSubmissions(0, true);
         reset();
     }
 }

worldedit-core/src/main/java/com/boydti/fawe/beta/implementation/blocks/CharGetBlocks.java

@@ -22,18 +22,7 @@ public abstract class CharGetBlocks extends CharBlocks implements IGetBlocks {
     @Override
     public void filter(Filter filter, FilterBlock block) {
         CharFilterBlock b = (CharFilterBlock) block;
-        for (int layer = 0; layer < 16; layer++) {
+        b.filter(this, filter);
-            if (!hasSection(layer)) continue;
-            char[] arr = sections[layer].get(this, layer);
-            b.init(arr, layer);
-            for (b.y = 0, b.index = 0; b.y < 16; b.y++) {
-                for (b.z = 0; b.z < 16; b.z++) {
-                    for (b.x = 0; b.x < 16; b.x++, b.index++) {
-                        filter.applyBlock(b);
-                    }
-                }
-            }
-        }
     }
 
     @Override

worldedit-core/src/main/java/com/boydti/fawe/beta/implementation/holder/ChunkHolder.java

@@ -1,10 +1,8 @@
 package com.boydti.fawe.beta.implementation.holder;
 
-import com.boydti.fawe.beta.CharFilterBlock;
 import com.boydti.fawe.beta.Filter;
 import com.boydti.fawe.beta.FilterBlock;
 import com.boydti.fawe.beta.IQueueExtent;
-import com.boydti.fawe.beta.implementation.blocks.CharGetBlocks;
 import com.boydti.fawe.beta.implementation.blocks.CharSetBlocks;
 import com.boydti.fawe.beta.IChunk;
 import com.boydti.fawe.beta.IGetBlocks;
@@ -22,7 +20,7 @@ import java.util.function.Supplier;
 /**
  * Abstract IChunk class that implements basic get/set blocks
  */
-public abstract class ChunkHolder<T> implements IChunk, Supplier<IGetBlocks> {
+public abstract class ChunkHolder implements IChunk, Supplier<IGetBlocks> {
     private IGetBlocks get;
     private ISetBlocks set;
     private IBlockDelegate delegate;
@@ -40,7 +38,7 @@ public abstract class ChunkHolder<T> implements IChunk, Supplier<IGetBlocks> {
     @Override
     public void filter(Filter filter, FilterBlock block) {
         block.init(X, Z, get);
-        IGetBlocks get = cachedGet();
+        IGetBlocks get = getOrCreateGet();
         get.filter(filter, block);
     }
 
@@ -73,12 +71,12 @@ public abstract class ChunkHolder<T> implements IChunk, Supplier<IGetBlocks> {
         return set == null || set.isEmpty();
     }
 
-    public final IGetBlocks cachedGet() {
+    public final IGetBlocks getOrCreateGet() {
         if (get == null) get = newGet();
         return get;
     }
 
-    public final ISetBlocks cachedSet() {
+    public final ISetBlocks getOrCreateSet() {
         if (set == null) set = set();
         return set;
     }
@@ -95,6 +93,13 @@ public abstract class ChunkHolder<T> implements IChunk, Supplier<IGetBlocks> {
         return get();
     }
 
+    @Override
+    public void optimize() {
+        if (set != null) {
+            set.optimize();
+        }
+    }
+
     @Override
     public void init(IQueueExtent extent, final int X, final int Z) {
         this.extent = extent;
@@ -163,35 +168,35 @@ public abstract class ChunkHolder<T> implements IChunk, Supplier<IGetBlocks> {
     public static final IBlockDelegate NULL = new IBlockDelegate() {
         @Override
         public boolean setBiome(final ChunkHolder chunk, final int x, final int y, final int z, final BiomeType biome) {
-            chunk.cachedSet();
+            chunk.getOrCreateSet();
             chunk.delegate = SET;
             return chunk.setBiome(x, y, z, biome);
         }
 
         @Override
         public boolean setBlock(final ChunkHolder chunk, final int x, final int y, final int z, final BlockStateHolder block) {
-            chunk.cachedSet();
+            chunk.getOrCreateSet();
             chunk.delegate = SET;
             return chunk.setBlock(x, y, z, block);
         }
 
         @Override
         public BiomeType getBiome(final ChunkHolder chunk, final int x, final int z) {
-            chunk.cachedGet();
+            chunk.getOrCreateGet();
             chunk.delegate = GET;
             return chunk.getBiome(x, z);
         }
 
         @Override
         public BlockState getBlock(final ChunkHolder chunk, final int x, final int y, final int z) {
-            chunk.cachedGet();
+            chunk.getOrCreateGet();
             chunk.delegate = GET;
             return chunk.getBlock(x, y, z);
         }
 
         @Override
         public BaseBlock getFullBlock(final ChunkHolder chunk, final int x, final int y, final int z) {
-            chunk.cachedGet();
+            chunk.getOrCreateGet();
             chunk.delegate = GET;
             return chunk.getFullBlock(x, y, z);
         }
@@ -200,14 +205,14 @@ public abstract class ChunkHolder<T> implements IChunk, Supplier<IGetBlocks> {
     public static final IBlockDelegate GET = new IBlockDelegate() {
         @Override
         public boolean setBiome(final ChunkHolder chunk, final int x, final int y, final int z, final BiomeType biome) {
-            chunk.cachedSet();
+            chunk.getOrCreateSet();
             chunk.delegate = BOTH;
             return chunk.setBiome(x, y, z, biome);
         }
 
         @Override
         public boolean setBlock(final ChunkHolder chunk, final int x, final int y, final int z, final BlockStateHolder block) {
-            chunk.cachedSet();
+            chunk.getOrCreateSet();
             chunk.delegate = BOTH;
             return chunk.setBlock(x, y, z, block);
         }
@@ -241,21 +246,21 @@ public abstract class ChunkHolder<T> implements IChunk, Supplier<IGetBlocks> {
 
         @Override
         public BiomeType getBiome(final ChunkHolder chunk, final int x, final int z) {
-            chunk.cachedGet();
+            chunk.getOrCreateGet();
             chunk.delegate = BOTH;
             return chunk.getBiome(x, z);
         }
 
         @Override
         public BlockState getBlock(final ChunkHolder chunk, final int x, final int y, final int z) {
-            chunk.cachedGet();
+            chunk.getOrCreateGet();
             chunk.delegate = BOTH;
             return chunk.getBlock(x, y, z);
         }
 
         @Override
         public BaseBlock getFullBlock(final ChunkHolder chunk, final int x, final int y, final int z) {
-            chunk.cachedGet();
+            chunk.getOrCreateGet();
             chunk.delegate = BOTH;
             return chunk.getFullBlock(x, y, z);
         }

worldedit-core/src/main/java/com/sk89q/worldedit/command/RegionCommands.java

@@ -281,7 +281,10 @@ public class RegionCommands extends MethodCommands {
         QueueHandler queueHandler = Fawe.get().getQueueHandler();
         World world = player.getWorld();
         CountFilter filter = new CountFilter();
+        long start = System.currentTimeMillis();
         queueHandler.apply(world, region, filter);
+        long diff = System.currentTimeMillis() - start;
+        System.out.println(diff);
     }
 
     @Command(

worldedit-core/src/main/java/com/sk89q/worldedit/world/block/BlockTypes.java

@@ -62,6 +62,10 @@ import java.util.stream.Stream;
 public final class BlockTypes {
     // Doesn't really matter what the hardcoded values are, as FAWE will update it on load
     @Nullable public static final BlockType __RESERVED__ = null;
+    @Nullable public static final BlockType AIR = null;
+    @Nullable public static final BlockType CAVE_AIR = null;
+    @Nullable public static final BlockType VOID_AIR = null;
+
     @Nullable public static final BlockType ACACIA_BUTTON = null;
     @Nullable public static final BlockType ACACIA_DOOR = null;
     @Nullable public static final BlockType ACACIA_FENCE = null;
@@ -76,7 +80,6 @@ public final class BlockTypes {
     @Nullable public static final BlockType ACACIA_TRAPDOOR = null;
     @Nullable public static final BlockType ACACIA_WOOD = null;
     @Nullable public static final BlockType ACTIVATOR_RAIL = null;
-    @Nullable public static final BlockType AIR = null;
     @Nullable public static final BlockType ALLIUM = null;
     @Nullable public static final BlockType ANDESITE = null;
     @Nullable public static final BlockType ANVIL = null;
@@ -160,7 +163,6 @@ public final class BlockTypes {
     @Nullable public static final BlockType CARROTS = null;
     @Nullable public static final BlockType CARVED_PUMPKIN = null;
     @Nullable public static final BlockType CAULDRON = null;
-    @Nullable public static final BlockType CAVE_AIR = null;
     @Nullable public static final BlockType CHAIN_COMMAND_BLOCK = null;
     @Nullable public static final BlockType CHEST = null;
     @Nullable public static final BlockType CHIPPED_ANVIL = null;
@@ -625,7 +627,6 @@ public final class BlockTypes {
     @Nullable public static final BlockType TUBE_CORAL_WALL_FAN = null;
     @Nullable public static final BlockType TURTLE_EGG = null;
     @Nullable public static final BlockType VINE = null;
-    @Nullable public static final BlockType VOID_AIR = null;
     @Nullable public static final BlockType WALL_SIGN = null;
     @Nullable public static final BlockType WALL_TORCH = null;
     @Nullable public static final BlockType WATER = null;