From d763ab374caa102b15c07681bdb3f43277f825e8 Mon Sep 17 00:00:00 2001 From: wizjany Date: Sat, 22 Jun 2019 14:20:14 -0400 Subject: [PATCH] Re-add delchunks command (#481) The new command now writes a json file to WorldEdit's working directory with instructions on which chunks to delete, which is read by the plugin/mod at startup and calls the ChunkDeleter. The chunk deleter parses the json and iterates the instructions, backing up .mca files as it goes and overwriting the offset headers with 0 wherever a chunk needs to be deleted. This allows Minecraft to reclaim the space used for that chunk, as well as forcing it to be generated from scratch next time the area is loaded. --- .../sk89q/worldedit/bukkit/BukkitWorld.java | 6 + .../worldedit/bukkit/WorldEditPlugin.java | 10 + .../bukkit/adapter/BukkitImplLoader.java | 3 - .../worldedit/command/ChunkCommands.java | 170 ++++----- .../internal/anvil/ChunkDeleter.java | 350 ++++++++++++++++++ .../internal/anvil/ChunkDeletionInfo.java | 48 +++ .../internal/anvil/RegionAccess.java | 101 +++++ .../sk89q/worldedit/world/AbstractWorld.java | 6 + .../java/com/sk89q/worldedit/world/World.java | 12 + .../world/storage/McRegionReader.java | 9 +- .../com/sk89q/worldedit/forge/ForgeWorld.java | 10 + .../sk89q/worldedit/forge/ForgeWorldEdit.java | 14 +- .../sk89q/worldedit/sponge/SpongeWorld.java | 6 + .../worldedit/sponge/SpongeWorldEdit.java | 9 + 14 files changed, 650 insertions(+), 104 deletions(-) create mode 100644 worldedit-core/src/main/java/com/sk89q/worldedit/internal/anvil/ChunkDeleter.java create mode 100644 worldedit-core/src/main/java/com/sk89q/worldedit/internal/anvil/ChunkDeletionInfo.java create mode 100644 worldedit-core/src/main/java/com/sk89q/worldedit/internal/anvil/RegionAccess.java diff --git a/worldedit-bukkit/src/main/java/com/sk89q/worldedit/bukkit/BukkitWorld.java b/worldedit-bukkit/src/main/java/com/sk89q/worldedit/bukkit/BukkitWorld.java index 1613f46eb..977985df7 100644 --- a/worldedit-bukkit/src/main/java/com/sk89q/worldedit/bukkit/BukkitWorld.java +++ b/worldedit-bukkit/src/main/java/com/sk89q/worldedit/bukkit/BukkitWorld.java @@ -53,6 +53,7 @@ import org.slf4j.Logger; import javax.annotation.Nullable; import java.lang.ref.WeakReference; +import java.nio.file.Path; import java.util.ArrayList; import java.util.EnumMap; import java.util.HashMap; @@ -158,6 +159,11 @@ public class BukkitWorld extends AbstractWorld { return getWorld().getName(); } + @Override + public Path getStoragePath() { + return getWorld().getWorldFolder().toPath(); + } + @Override public int getBlockLightLevel(BlockVector3 pt) { return getWorld().getBlockAt(pt.getBlockX(), pt.getBlockY(), pt.getBlockZ()).getLightLevel(); diff --git a/worldedit-bukkit/src/main/java/com/sk89q/worldedit/bukkit/WorldEditPlugin.java b/worldedit-bukkit/src/main/java/com/sk89q/worldedit/bukkit/WorldEditPlugin.java index 4ac9a3ee1..a81aab7c5 100644 --- a/worldedit-bukkit/src/main/java/com/sk89q/worldedit/bukkit/WorldEditPlugin.java +++ b/worldedit-bukkit/src/main/java/com/sk89q/worldedit/bukkit/WorldEditPlugin.java @@ -38,6 +38,7 @@ import com.sk89q.worldedit.extension.platform.Capability; import com.sk89q.worldedit.extension.platform.Platform; import com.sk89q.worldedit.extent.inventory.BlockBag; import com.sk89q.worldedit.internal.command.CommandUtil; +import com.sk89q.worldedit.internal.anvil.ChunkDeleter; import com.sk89q.worldedit.registry.state.Property; import com.sk89q.worldedit.world.biome.BiomeType; import com.sk89q.worldedit.world.block.BlockCategory; @@ -73,6 +74,9 @@ import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.List; import java.util.Locale; import java.util.Map; @@ -82,6 +86,7 @@ import java.util.logging.Level; import java.util.zip.ZipEntry; import static com.google.common.base.Preconditions.checkNotNull; +import static com.sk89q.worldedit.internal.anvil.ChunkDeleter.DELCHUNKS_FILE_NAME; /** * Plugin for Bukkit. @@ -109,6 +114,11 @@ public class WorldEditPlugin extends JavaPlugin implements TabCompleter { // Setup platform server = new BukkitServerInterface(this, getServer()); worldEdit.getPlatformManager().register(server); + + Path delChunks = Paths.get(getDataFolder().getPath(), DELCHUNKS_FILE_NAME); + if (Files.exists(delChunks)) { + ChunkDeleter.runFromFile(delChunks, true); + } } /** diff --git a/worldedit-bukkit/src/main/java/com/sk89q/worldedit/bukkit/adapter/BukkitImplLoader.java b/worldedit-bukkit/src/main/java/com/sk89q/worldedit/bukkit/adapter/BukkitImplLoader.java index 950e4f862..5d464181f 100644 --- a/worldedit-bukkit/src/main/java/com/sk89q/worldedit/bukkit/adapter/BukkitImplLoader.java +++ b/worldedit-bukkit/src/main/java/com/sk89q/worldedit/bukkit/adapter/BukkitImplLoader.java @@ -156,9 +156,6 @@ public class BukkitImplLoader { if (cls.isSynthetic()) continue; if (BukkitImplAdapter.class.isAssignableFrom(cls)) { return (BukkitImplAdapter) cls.newInstance(); - } else { - log.debug("Failed to load the Bukkit adapter class '" + className + - "' because it does not implement " + BukkitImplAdapter.class.getCanonicalName()); } } catch (ClassNotFoundException e) { log.warn("Failed to load the Bukkit adapter class '" + className + diff --git a/worldedit-core/src/main/java/com/sk89q/worldedit/command/ChunkCommands.java b/worldedit-core/src/main/java/com/sk89q/worldedit/command/ChunkCommands.java index af1e4e95d..450547740 100644 --- a/worldedit-core/src/main/java/com/sk89q/worldedit/command/ChunkCommands.java +++ b/worldedit-core/src/main/java/com/sk89q/worldedit/command/ChunkCommands.java @@ -19,10 +19,7 @@ package com.sk89q.worldedit.command; -import static com.google.common.base.Preconditions.checkNotNull; -import static com.sk89q.worldedit.command.util.Logging.LogMode.REGION; - -import com.sk89q.worldedit.LocalConfiguration; +import com.google.gson.JsonIOException; import com.sk89q.worldedit.LocalSession; import com.sk89q.worldedit.WorldEdit; import com.sk89q.worldedit.WorldEditException; @@ -30,22 +27,36 @@ import com.sk89q.worldedit.command.util.CommandPermissions; import com.sk89q.worldedit.command.util.CommandPermissionsConditionGenerator; import com.sk89q.worldedit.command.util.Logging; import com.sk89q.worldedit.entity.Player; +import com.sk89q.worldedit.internal.anvil.ChunkDeleter; +import com.sk89q.worldedit.internal.anvil.ChunkDeletionInfo; import com.sk89q.worldedit.math.BlockVector2; -import com.sk89q.worldedit.math.MathUtils; +import com.sk89q.worldedit.regions.CuboidRegion; +import com.sk89q.worldedit.regions.Region; import com.sk89q.worldedit.util.Location; import com.sk89q.worldedit.util.formatting.component.PaginationBox; +import com.sk89q.worldedit.util.formatting.text.TextComponent; +import com.sk89q.worldedit.util.formatting.text.event.ClickEvent; +import com.sk89q.worldedit.util.formatting.text.format.TextColor; import com.sk89q.worldedit.world.storage.LegacyChunkStore; import com.sk89q.worldedit.world.storage.McRegionChunkStore; import org.enginehub.piston.annotation.Command; import org.enginehub.piston.annotation.CommandContainer; import org.enginehub.piston.annotation.param.ArgFlag; +import org.enginehub.piston.exception.StopExecutionException; -import java.io.FileOutputStream; +import java.io.File; import java.io.IOException; -import java.io.OutputStreamWriter; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.ZonedDateTime; +import java.util.ArrayList; import java.util.Set; import java.util.stream.Collectors; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.sk89q.worldedit.command.util.Logging.LogMode.REGION; +import static com.sk89q.worldedit.internal.anvil.ChunkDeleter.DELCHUNKS_FILE_NAME; + /** * Commands for working with chunks. */ @@ -69,15 +80,10 @@ public class ChunkCommands { int chunkX = (int) Math.floor(pos.getBlockX() / 16.0); int chunkZ = (int) Math.floor(pos.getBlockZ() / 16.0); - String folder1 = Integer.toString(MathUtils.divisorMod(chunkX, 64), 36); - String folder2 = Integer.toString(MathUtils.divisorMod(chunkZ, 64), 36); - String filename = "c." + Integer.toString(chunkX, 36) - + "." + Integer.toString(chunkZ, 36) + ".dat"; - + final BlockVector2 chunkPos = BlockVector2.at(chunkX, chunkZ); player.print("Chunk: " + chunkX + ", " + chunkZ); - player.print("Old format: " + folder1 + "/" + folder2 + "/" + filename); - player.print("McRegion: region/" + McRegionChunkStore.getFilename( - BlockVector2.at(chunkX, chunkZ))); + player.print("Old format: " + LegacyChunkStore.getFilename(chunkPos)); + player.print("McRegion: region/" + McRegionChunkStore.getFilename(chunkPos)); } @Command( @@ -86,7 +92,7 @@ public class ChunkCommands { ) @CommandPermissions("worldedit.listchunks") public void listChunks(Player player, LocalSession session, - @ArgFlag(name = 'p', desc = "Page number.", def = "1") int page) throws WorldEditException { + @ArgFlag(name = 'p', desc = "Page number.", def = "1") int page) throws WorldEditException { Set chunks = session.getSelection(player.getWorld()).getChunks(); PaginationBox paginationBox = PaginationBox.fromStrings("Selected Chunks", "/listchunks -p %page%", @@ -100,82 +106,64 @@ public class ChunkCommands { ) @CommandPermissions("worldedit.delchunks") @Logging(REGION) - public void deleteChunks(Player player, LocalSession session) throws WorldEditException { - player.print("Note that this command does not yet support the mcregion format."); - LocalConfiguration config = worldEdit.getConfiguration(); - - Set chunks = session.getSelection(player.getWorld()).getChunks(); - FileOutputStream out = null; - - if (config.shellSaveType == null) { - player.printError("Shell script type must be configured: 'bat' or 'bash' expected."); - } else if (config.shellSaveType.equalsIgnoreCase("bat")) { - try { - out = new FileOutputStream("worldedit-delchunks.bat"); - OutputStreamWriter writer = new OutputStreamWriter(out, "UTF-8"); - writer.write("@ECHO off\r\n"); - writer.write("ECHO This batch file was generated by WorldEdit.\r\n"); - writer.write("ECHO It contains a list of chunks that were in the selected region\r\n"); - writer.write("ECHO at the time that the /delchunks command was used. Run this file\r\n"); - writer.write("ECHO in order to delete the chunk files listed in this file.\r\n"); - writer.write("ECHO.\r\n"); - writer.write("PAUSE\r\n"); - - for (BlockVector2 chunk : chunks) { - String filename = LegacyChunkStore.getFilename(chunk); - writer.write("ECHO " + filename + "\r\n"); - writer.write("DEL \"world/" + filename + "\"\r\n"); - } - - writer.write("ECHO Complete.\r\n"); - writer.write("PAUSE\r\n"); - writer.close(); - player.print("worldedit-delchunks.bat written. Run it when no one is near the region."); - } catch (IOException e) { - player.printError("Error occurred: " + e.getMessage()); - } finally { - if (out != null) { - try { - out.close(); - } catch (IOException ignored) { } - } - } - } else if (config.shellSaveType.equalsIgnoreCase("bash")) { - try { - out = new FileOutputStream("worldedit-delchunks.sh"); - OutputStreamWriter writer = new OutputStreamWriter(out, "UTF-8"); - writer.write("#!/bin/bash\n"); - writer.write("echo This shell file was generated by WorldEdit.\n"); - writer.write("echo It contains a list of chunks that were in the selected region\n"); - writer.write("echo at the time that the /delchunks command was used. Run this file\n"); - writer.write("echo in order to delete the chunk files listed in this file.\n"); - writer.write("echo\n"); - writer.write("read -p \"Press any key to continue...\"\n"); - - for (BlockVector2 chunk : chunks) { - String filename = LegacyChunkStore.getFilename(chunk); - writer.write("echo " + filename + "\n"); - writer.write("rm \"world/" + filename + "\"\n"); - } - - writer.write("echo Complete.\n"); - writer.write("read -p \"Press any key to continue...\"\n"); - writer.close(); - player.print("worldedit-delchunks.sh written. Run it when no one is near the region."); - player.print("You will have to chmod it to be executable."); - } catch (IOException e) { - player.printError("Error occurred: " + e.getMessage()); - } finally { - if (out != null) { - try { - out.close(); - } catch (IOException ignored) { - } - } - } - } else { - player.printError("Shell script type must be configured: 'bat' or 'bash' expected."); + public void deleteChunks(Player player, LocalSession session, + @ArgFlag(name = 'o', desc = "Only delete chunks older than the specified time.", def = "") + ZonedDateTime beforeTime) throws WorldEditException { + Path worldDir = player.getWorld().getStoragePath(); + if (worldDir == null) { + throw new StopExecutionException(TextComponent.of("Couldn't find world folder for this world.")); } + + File chunkFile = worldEdit.getWorkingDirectoryFile(DELCHUNKS_FILE_NAME); + Path chunkPath = chunkFile.toPath(); + ChunkDeletionInfo currentInfo = null; + if (Files.exists(chunkPath)) { + try { + currentInfo = ChunkDeleter.readInfo(chunkFile.toPath()); + } catch (IOException e) { + throw new StopExecutionException(TextComponent.of("Error reading existing chunk file.")); + } + } + if (currentInfo == null) { + currentInfo = new ChunkDeletionInfo(); + currentInfo.batches = new ArrayList<>(); + } + + ChunkDeletionInfo.ChunkBatch newBatch = new ChunkDeletionInfo.ChunkBatch(); + newBatch.worldPath = worldDir.toAbsolutePath().normalize().toString(); + newBatch.backup = true; + final Region selection = session.getSelection(player.getWorld()); + int chunkCount; + if (selection instanceof CuboidRegion) { + newBatch.minChunk = BlockVector2.at(selection.getMinimumPoint().getBlockX() >> 4, selection.getMinimumPoint().getBlockZ() >> 4); + newBatch.maxChunk = BlockVector2.at(selection.getMaximumPoint().getBlockX() >> 4, selection.getMaximumPoint().getBlockZ() >> 4); + final BlockVector2 dist = newBatch.maxChunk.subtract(newBatch.minChunk).add(1, 1); + chunkCount = dist.getBlockX() * dist.getBlockZ(); + } else { + // this has a possibility to OOM for very large selections still + Set chunks = selection.getChunks(); + newBatch.chunks = new ArrayList<>(chunks); + chunkCount = chunks.size(); + } + if (beforeTime != null) { + newBatch.deletionPredicates = new ArrayList<>(); + ChunkDeletionInfo.DeletionPredicate timePred = new ChunkDeletionInfo.DeletionPredicate(); + timePred.property = "modification"; + timePred.comparison = "<"; + timePred.value = String.valueOf((int) beforeTime.toOffsetDateTime().toEpochSecond()); + newBatch.deletionPredicates.add(timePred); + } + currentInfo.batches.add(newBatch); + + try { + ChunkDeleter.writeInfo(currentInfo, chunkPath); + } catch (IOException | JsonIOException e) { + throw new StopExecutionException(TextComponent.of("Failed to write chunk list: " + e.getMessage())); + } + + player.print(String.format("%d chunk(s) have been marked for deletion and will be deleted the next time the server starts.", chunkCount)); + player.print(TextComponent.of("You can mark more chunks for deletion, or to stop the server now, run: ", TextColor.LIGHT_PURPLE) + .append(TextComponent.of("/stop", TextColor.AQUA).clickEvent(ClickEvent.of(ClickEvent.Action.SUGGEST_COMMAND, "/stop")))); } } diff --git a/worldedit-core/src/main/java/com/sk89q/worldedit/internal/anvil/ChunkDeleter.java b/worldedit-core/src/main/java/com/sk89q/worldedit/internal/anvil/ChunkDeleter.java new file mode 100644 index 000000000..71106c3b3 --- /dev/null +++ b/worldedit-core/src/main/java/com/sk89q/worldedit/internal/anvil/ChunkDeleter.java @@ -0,0 +1,350 @@ +/* + * WorldEdit, a Minecraft world manipulation toolkit + * Copyright (C) sk89q + * Copyright (C) WorldEdit team and contributors + * + * This program is free software: you can redistribute it and/or modify it + * under the terms of the GNU Lesser General Public License as published by the + * Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +package com.sk89q.worldedit.internal.anvil; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonIOException; +import com.google.gson.JsonSyntaxException; +import com.google.gson.TypeAdapter; +import com.google.gson.reflect.TypeToken; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonWriter; +import com.sk89q.worldedit.math.BlockVector2; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedWriter; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.BiPredicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public final class ChunkDeleter { + + public static final String DELCHUNKS_FILE_NAME = "delete_chunks.json"; + private static final Logger logger = LoggerFactory.getLogger(ChunkDeleter.class); + + private static final Comparator chunkSorter = Comparator.comparing( + pos -> (pos.getBlockX() & 31) + (pos.getBlockZ() & 31) * 32); + + private static Gson chunkDeleterGson = new GsonBuilder() + .registerTypeAdapter(BlockVector2.class, new BlockVector2Adapter()) + .setPrettyPrinting() + .create(); + + public static ChunkDeletionInfo readInfo(Path chunkFile) throws IOException, JsonSyntaxException { + String json = new String(Files.readAllBytes(chunkFile), StandardCharsets.UTF_8); + return chunkDeleterGson.fromJson(json, ChunkDeletionInfo.class); + } + + public static void writeInfo(ChunkDeletionInfo info, Path chunkFile) throws IOException, JsonIOException { + String json = chunkDeleterGson.toJson(info, new TypeToken() {}.getType()); + try (BufferedWriter writer = Files.newBufferedWriter(chunkFile, StandardOpenOption.CREATE)) { + writer.write(json); + } + } + + public static void runFromFile(Path chunkFile, boolean deleteOnSuccess) { + ChunkDeleter chunkDeleter; + try { + chunkDeleter = createFromFile(chunkFile); + } catch (JsonSyntaxException | IOException e) { + logger.error("Could not parse chunk deletion file. Invalid file?", e); + return; + } + logger.info("Found chunk deletions. Proceeding with deletion..."); + long start = System.currentTimeMillis(); + if (chunkDeleter.runDeleter()) { + logger.info("Successfully deleted {} matching chunks (out of {}, taking {} ms).", + chunkDeleter.getDeletedChunkCount(), chunkDeleter.getDeletionsRequested(), + System.currentTimeMillis() - start); + if (deleteOnSuccess) { + boolean deletedFile = false; + try { + deletedFile = Files.deleteIfExists(chunkFile); + } catch (IOException ignored) { + } + if (!deletedFile) { + logger.warn("Chunk deletion file could not be cleaned up. This may have unintended consequences" + + " on next startup, or if /delchunks is used again."); + } + } + } else { + logger.error("Error occurred while deleting chunks. " + + "If world errors occur, stop the server and restore the *.bak backup files."); + } + } + + private ChunkDeleter(ChunkDeletionInfo chunkDeletionInfo) { + this.chunkDeletionInfo = chunkDeletionInfo; + } + + private static ChunkDeleter createFromFile(Path chunkFile) throws IOException { + ChunkDeletionInfo info = readInfo(chunkFile); + if (info == null) { + throw new IOException("Read null json. Empty file?"); + } + return new ChunkDeleter(info); + } + + private final ChunkDeletionInfo chunkDeletionInfo; + private Set backedUpRegions = new HashSet<>(); + private boolean shouldPreload; + private int debugRate = 100; + private int totalChunksDeleted = 0; + private int deletionsRequested = 0; + + private boolean runDeleter() { + return chunkDeletionInfo.batches.stream().allMatch(this::runBatch); + } + + private boolean runBatch(ChunkDeletionInfo.ChunkBatch chunkBatch) { + logger.debug("Processing deletion batch."); + final Map> regionToChunkList = groupChunks(chunkBatch); + BiPredicate predicate = createPredicates(chunkBatch.deletionPredicates); + shouldPreload = chunkBatch.chunks == null; + return regionToChunkList.entrySet().stream().allMatch(entry -> { + Path regionPath = entry.getKey(); + if (!Files.exists(regionPath)) return true; + if (chunkBatch.backup && !backedUpRegions.contains(regionPath)) { + try { + backupRegion(regionPath); + } catch (IOException e) { + logger.warn("Error backing up region file: " + regionPath + ". Aborting the process.", e); + return false; + } + } + return deleteChunks(regionPath, entry.getValue(), predicate); + }); + } + + private Map> groupChunks(ChunkDeletionInfo.ChunkBatch chunkBatch) { + Path worldPath = Paths.get(chunkBatch.worldPath); + if (chunkBatch.chunks != null) { + deletionsRequested += chunkBatch.chunks.size(); + debugRate = chunkBatch.chunks.size() / 10; + return chunkBatch.chunks.stream() + .collect(Collectors.groupingBy(RegionFilePos::new)) + .entrySet().stream().collect(Collectors.toMap( + e -> worldPath.resolve("region").resolve(e.getKey().getFileName()), + e -> e.getValue().stream().sorted(chunkSorter))); + } else { + final BlockVector2 minChunk = chunkBatch.minChunk; + final BlockVector2 maxChunk = chunkBatch.maxChunk; + final RegionFilePos minRegion = new RegionFilePos(minChunk); + final RegionFilePos maxRegion = new RegionFilePos(maxChunk); + Map> groupedChunks = new HashMap<>(); + for (int regX = minRegion.getX(); regX <= maxRegion.getX(); regX++) { + for (int regZ = minRegion.getZ(); regZ <= maxRegion.getZ(); regZ++) { + final Path regionPath = worldPath.resolve("region").resolve(new RegionFilePos(regX, regZ).getFileName()); + if (!Files.exists(regionPath)) continue; + int startX = regX << 5; + int endX = (regX << 5) + 31; + int startZ = regZ << 5; + int endZ = (regZ << 5) + 31; + + int minX = Math.max(Math.min(startX, endX), minChunk.getBlockX()); + int minZ = Math.max(Math.min(startZ, endZ), minChunk.getBlockZ()); + int maxX = Math.min(Math.max(startX, endX), maxChunk.getBlockX()); + int maxZ = Math.min(Math.max(startZ, endZ), maxChunk.getBlockZ()); + Stream stream = Stream.iterate(BlockVector2.at(minX, minZ), + bv2 -> { + int nextX = bv2.getBlockX(); + int nextZ = bv2.getBlockZ(); + if (++nextX > maxX) { + nextX = minX; + if (++nextZ > maxZ) { + return null; + } + } + return BlockVector2.at(nextX, nextZ); + }); + groupedChunks.put(regionPath, stream); + } + } + final BlockVector2 dist = maxChunk.subtract(minChunk).add(1, 1); + final int batchSize = dist.getBlockX() * dist.getBlockZ(); + debugRate = batchSize / 10; + this.deletionsRequested += batchSize; + return groupedChunks; + } + } + + private BiPredicate createPredicates(List deletionPredicates) { + if (deletionPredicates == null) return (r, p) -> true; + return deletionPredicates.stream() + .map(this::createPredicate) + .reduce(BiPredicate::and) + .orElse((r, p) -> true); + } + + private BiPredicate createPredicate(ChunkDeletionInfo.DeletionPredicate deletionPredicate) { + if ("modification".equals(deletionPredicate.property)) { + int time; + try { + time = Integer.parseInt(deletionPredicate.value); + } catch (NumberFormatException e) { + throw new IllegalStateException("Modification time predicate specified invalid time: " + deletionPredicate.value); + } + switch (deletionPredicate.comparison) { + case "<": + return (r, p) -> { + try { + return r.getModificationTime(p) < time; + } catch (IOException e) { + return false; + } + }; + case ">": + return (r, p) -> { + try { + return r.getModificationTime(p) > time; + } catch (IOException e) { + return false; + } + }; + default: + throw new IllegalStateException("Unexpected comparison value: " + deletionPredicate.comparison); + } + } + throw new IllegalStateException("Unexpected property value: " + deletionPredicate.property); + } + + private void backupRegion(Path regionFile) throws IOException { + Path backupFile = regionFile.resolveSibling(regionFile.getFileName() + ".bak"); + Files.copy(regionFile, backupFile, StandardCopyOption.REPLACE_EXISTING); + backedUpRegions.add(backupFile); + } + + private boolean deleteChunks(Path regionFile, Stream chunks, + BiPredicate deletionPredicate) { + try (RegionAccess region = new RegionAccess(regionFile, shouldPreload)) { + for (Iterator iterator = chunks.iterator(); iterator.hasNext();) { + BlockVector2 chunk = iterator.next(); + if (chunk == null) break; + if (deletionPredicate.test(region, chunk)) { + region.deleteChunk(chunk); + totalChunksDeleted++; + if (totalChunksDeleted % debugRate == 0) { + logger.debug("Deleted {} chunks so far.", totalChunksDeleted); + } + } else { + logger.debug("Chunk did not match predicates: " + chunk); + } + } + return true; + } catch (IOException e) { + logger.warn("Error deleting chunks from region: " + regionFile + ". Aborting the process.", e); + return false; + } + } + + public int getDeletedChunkCount() { + return totalChunksDeleted; + } + + public int getDeletionsRequested() { + return deletionsRequested; + } + + private static class BlockVector2Adapter extends TypeAdapter { + @Override + public void write(JsonWriter out, BlockVector2 value) throws IOException { + out.beginArray(); + out.value(value.getBlockX()); + out.value(value.getBlockZ()); + out.endArray(); + } + + @Override + public BlockVector2 read(JsonReader in) throws IOException { + in.beginArray(); + int x = in.nextInt(); + int z = in.nextInt(); + in.endArray(); + return BlockVector2.at(x, z); + } + } + + private static class RegionFilePos { + private final int x; + private final int z; + + RegionFilePos(BlockVector2 chunk) { + this.x = chunk.getBlockX() >> 5; + this.z = chunk.getBlockZ() >> 5; + } + + RegionFilePos(int regX, int regZ) { + this.x = regX; + this.z = regZ; + } + + public int getX() { + return x; + } + + public int getZ() { + return z; + } + + public String getFileName() { + return "r." + x + "." + z + ".mca"; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + RegionFilePos that = (RegionFilePos) o; + + if (x != that.x) return false; + return z == that.z; + + } + + @Override + public int hashCode() { + int result = x; + result = 31 * result + z; + return result; + } + + @Override + public String toString() { + return "(" + x + ", " + z + ")"; + } + } +} diff --git a/worldedit-core/src/main/java/com/sk89q/worldedit/internal/anvil/ChunkDeletionInfo.java b/worldedit-core/src/main/java/com/sk89q/worldedit/internal/anvil/ChunkDeletionInfo.java new file mode 100644 index 000000000..7e03ba502 --- /dev/null +++ b/worldedit-core/src/main/java/com/sk89q/worldedit/internal/anvil/ChunkDeletionInfo.java @@ -0,0 +1,48 @@ +/* + * WorldEdit, a Minecraft world manipulation toolkit + * Copyright (C) sk89q + * Copyright (C) WorldEdit team and contributors + * + * This program is free software: you can redistribute it and/or modify it + * under the terms of the GNU Lesser General Public License as published by the + * Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +package com.sk89q.worldedit.internal.anvil; + +import com.sk89q.worldedit.math.BlockVector2; + +import java.util.List; + +/** + * Internal class. Subject to changes. + */ +public class ChunkDeletionInfo { + + public List batches; + + public static class ChunkBatch { + public String worldPath; + public boolean backup; + public List deletionPredicates; + // specify either list of chunks, or min-max + public List chunks; + public BlockVector2 minChunk; + public BlockVector2 maxChunk; + } + + public static class DeletionPredicate { + public String property; + public String comparison; + public String value; + } +} diff --git a/worldedit-core/src/main/java/com/sk89q/worldedit/internal/anvil/RegionAccess.java b/worldedit-core/src/main/java/com/sk89q/worldedit/internal/anvil/RegionAccess.java new file mode 100644 index 000000000..9446773a6 --- /dev/null +++ b/worldedit-core/src/main/java/com/sk89q/worldedit/internal/anvil/RegionAccess.java @@ -0,0 +1,101 @@ +/* + * WorldEdit, a Minecraft world manipulation toolkit + * Copyright (C) sk89q + * Copyright (C) WorldEdit team and contributors + * + * This program is free software: you can redistribute it and/or modify it + * under the terms of the GNU Lesser General Public License as published by the + * Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +package com.sk89q.worldedit.internal.anvil; + +import com.sk89q.worldedit.math.BlockVector2; + +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.file.Path; + +/** + * Internal class. Subject to changes. + */ +class RegionAccess implements AutoCloseable { + + private RandomAccessFile raf; + private int[] offsets; + private int[] timestamps; + + RegionAccess(Path file) throws IOException { + this(file, false); + } + + RegionAccess(Path file, boolean preload) throws IOException { + raf = new RandomAccessFile(file.toFile(), "rw"); + if (preload) { + readHeaders(); + } + } + + private void readHeaders() throws IOException { + offsets = new int[1024]; + timestamps = new int[1024]; + for (int idx = 0; idx < 1024; ++idx) { + offsets[idx] = raf.readInt(); + } + for (int idx = 0; idx < 1024; ++idx) { + timestamps[idx] = raf.readInt(); + } + } + + private static int indexChunk(BlockVector2 pos) { + int x = pos.getBlockX() & 31; + int z = pos.getBlockZ() & 31; + return x + z * 32; + } + + int getModificationTime(BlockVector2 pos) throws IOException { + int idx = indexChunk(pos); + if (timestamps != null) { + return timestamps[idx]; + } + raf.seek(idx * 4L + 4096); + return raf.readInt(); + } + + int getChunkSize(BlockVector2 pos) throws IOException { + int idx = indexChunk(pos); + if (offsets != null) { + return offsets[idx] & 0xFF; + } + raf.seek(idx * 4L); + // 3 bytes for offset + raf.read(); + raf.read(); + raf.read(); + // one byte for size - note, yes, could do raf.readInt() & 0xFF but that does extra checks + return raf.read(); + } + + void deleteChunk(BlockVector2 pos) throws IOException { + int idx = indexChunk(pos); + raf.seek(idx * 4L); + raf.writeInt(0); + if (offsets != null) { + offsets[idx] = 0; + } + } + + @Override + public void close() throws IOException { + raf.close(); + } +} diff --git a/worldedit-core/src/main/java/com/sk89q/worldedit/world/AbstractWorld.java b/worldedit-core/src/main/java/com/sk89q/worldedit/world/AbstractWorld.java index 5fb92af03..201cac554 100644 --- a/worldedit-core/src/main/java/com/sk89q/worldedit/world/AbstractWorld.java +++ b/worldedit-core/src/main/java/com/sk89q/worldedit/world/AbstractWorld.java @@ -34,6 +34,7 @@ import com.sk89q.worldedit.world.block.BlockStateHolder; import com.sk89q.worldedit.world.block.BlockType; import com.sk89q.worldedit.world.block.BlockTypes; +import java.nio.file.Path; import java.util.PriorityQueue; import javax.annotation.Nullable; @@ -56,6 +57,11 @@ public abstract class AbstractWorld implements World { return setBlock(pt, block, true); } + @Override + public Path getStoragePath() { + return null; + } + @Override public int getMaxY() { return getMaximumPoint().getBlockY(); diff --git a/worldedit-core/src/main/java/com/sk89q/worldedit/world/World.java b/worldedit-core/src/main/java/com/sk89q/worldedit/world/World.java index cad0be0f5..471279281 100644 --- a/worldedit-core/src/main/java/com/sk89q/worldedit/world/World.java +++ b/worldedit-core/src/main/java/com/sk89q/worldedit/world/World.java @@ -38,6 +38,9 @@ import com.sk89q.worldedit.world.block.BlockStateHolder; import com.sk89q.worldedit.world.block.BlockType; import com.sk89q.worldedit.world.weather.WeatherType; +import javax.annotation.Nullable; +import java.nio.file.Path; + /** * Represents a world (dimension). */ @@ -50,6 +53,15 @@ public interface World extends Extent { */ String getName(); + /** + * Get the folder in which this world is stored. May return null if unknown + * or if this world is not serialized to disk. + * + * @return world storage path + */ + @Nullable + Path getStoragePath(); + /** * Get the maximum Y. * diff --git a/worldedit-core/src/main/java/com/sk89q/worldedit/world/storage/McRegionReader.java b/worldedit-core/src/main/java/com/sk89q/worldedit/world/storage/McRegionReader.java index e991216ae..29f37df31 100644 --- a/worldedit-core/src/main/java/com/sk89q/worldedit/world/storage/McRegionReader.java +++ b/worldedit-core/src/main/java/com/sk89q/worldedit/world/storage/McRegionReader.java @@ -100,10 +100,9 @@ public class McRegionReader { /** * Read the header. * - * @throws DataException * @throws IOException */ - private void readHeader() throws DataException, IOException { + private void readHeader() throws IOException { offsets = new int[SECTOR_INTS]; for (int i = 0; i < SECTOR_INTS; ++i) { @@ -124,10 +123,6 @@ public class McRegionReader { int x = position.getBlockX() & 31; int z = position.getBlockZ() & 31; - if (x < 0 || x >= 32 || z < 0 || z >= 32) { - throw new DataException("MCRegion file does not contain " + x + "," + z); - } - int offset = getOffset(x, z); // The chunk hasn't been generated @@ -138,7 +133,7 @@ public class McRegionReader { int sectorNumber = offset >> 8; int numSectors = offset & 0xFF; - stream.seek(sectorNumber * SECTOR_BYTES); + stream.seek((long) sectorNumber * SECTOR_BYTES); int length = dataStream.readInt(); if (length > SECTOR_BYTES * numSectors) { diff --git a/worldedit-forge/src/main/java/com/sk89q/worldedit/forge/ForgeWorld.java b/worldedit-forge/src/main/java/com/sk89q/worldedit/forge/ForgeWorld.java index 3c2d374b4..0e2c50ad7 100644 --- a/worldedit-forge/src/main/java/com/sk89q/worldedit/forge/ForgeWorld.java +++ b/worldedit-forge/src/main/java/com/sk89q/worldedit/forge/ForgeWorld.java @@ -97,6 +97,7 @@ import net.minecraft.world.storage.WorldInfo; import javax.annotation.Nullable; import java.io.File; import java.lang.ref.WeakReference; +import java.nio.file.Path; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -167,6 +168,15 @@ public class ForgeWorld extends AbstractWorld { return getWorld().getWorldInfo().getWorldName(); } + @Override + public Path getStoragePath() { + final World world = getWorld(); + if (world instanceof ServerWorld) { + return ((ServerWorld) world).getSaveHandler().getWorldDirectory().toPath(); + } + return null; + } + @Override public > boolean setBlock(BlockVector3 position, B block, boolean notifyAndLight) throws WorldEditException { checkNotNull(position); diff --git a/worldedit-forge/src/main/java/com/sk89q/worldedit/forge/ForgeWorldEdit.java b/worldedit-forge/src/main/java/com/sk89q/worldedit/forge/ForgeWorldEdit.java index e7cc7d4f0..a3ad1c9a0 100644 --- a/worldedit-forge/src/main/java/com/sk89q/worldedit/forge/ForgeWorldEdit.java +++ b/worldedit-forge/src/main/java/com/sk89q/worldedit/forge/ForgeWorldEdit.java @@ -31,6 +31,7 @@ import com.sk89q.worldedit.forge.net.packet.LeftClickAirEventMessage; import com.sk89q.worldedit.forge.proxy.ClientProxy; import com.sk89q.worldedit.forge.proxy.CommonProxy; import com.sk89q.worldedit.forge.proxy.ServerProxy; +import com.sk89q.worldedit.internal.anvil.ChunkDeleter; import com.sk89q.worldedit.util.Location; import com.sk89q.worldedit.world.biome.BiomeType; import com.sk89q.worldedit.world.block.BlockCategory; @@ -45,7 +46,6 @@ import net.minecraft.tags.ItemTags; import net.minecraft.util.Hand; import net.minecraft.util.ResourceLocation; import net.minecraft.world.World; -import net.minecraftforge.api.distmarker.Dist; import net.minecraftforge.common.MinecraftForge; import net.minecraftforge.event.CommandEvent; import net.minecraftforge.event.entity.player.PlayerInteractEvent; @@ -58,11 +58,10 @@ import net.minecraftforge.fml.ModContainer; import net.minecraftforge.fml.ModLoadingContext; import net.minecraftforge.fml.common.Mod; import net.minecraftforge.fml.event.lifecycle.FMLCommonSetupEvent; -import net.minecraftforge.fml.event.lifecycle.FMLLoadCompleteEvent; +import net.minecraftforge.fml.event.server.FMLServerAboutToStartEvent; import net.minecraftforge.fml.event.server.FMLServerStartedEvent; import net.minecraftforge.fml.event.server.FMLServerStoppingEvent; import net.minecraftforge.fml.javafmlmod.FMLJavaModLoadingContext; -import net.minecraftforge.fml.loading.FMLLoader; import net.minecraftforge.fml.loading.FMLPaths; import net.minecraftforge.registries.ForgeRegistries; import org.apache.logging.log4j.LogManager; @@ -76,6 +75,7 @@ import java.nio.file.Path; import static com.google.common.base.Preconditions.checkNotNull; import static com.sk89q.worldedit.forge.ForgeAdapter.adaptPlayer; +import static com.sk89q.worldedit.internal.anvil.ChunkDeleter.DELCHUNKS_FILE_NAME; /** * The Forge implementation of WorldEdit. @@ -180,6 +180,14 @@ public class ForgeWorldEdit { } } + @SubscribeEvent + public void serverAboutToStart(FMLServerAboutToStartEvent event) { + final Path delChunks = workingDir.resolve(DELCHUNKS_FILE_NAME); + if (Files.exists(delChunks)) { + ChunkDeleter.runFromFile(delChunks, true); + } + } + @SubscribeEvent public void serverStopping(FMLServerStoppingEvent event) { WorldEdit worldEdit = WorldEdit.getInstance(); diff --git a/worldedit-sponge/src/main/java/com/sk89q/worldedit/sponge/SpongeWorld.java b/worldedit-sponge/src/main/java/com/sk89q/worldedit/sponge/SpongeWorld.java index 743bf7198..04b1961ff 100644 --- a/worldedit-sponge/src/main/java/com/sk89q/worldedit/sponge/SpongeWorld.java +++ b/worldedit-sponge/src/main/java/com/sk89q/worldedit/sponge/SpongeWorld.java @@ -57,6 +57,7 @@ import org.spongepowered.api.world.World; import org.spongepowered.api.world.weather.Weather; import java.lang.ref.WeakReference; +import java.nio.file.Path; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -116,6 +117,11 @@ public abstract class SpongeWorld extends AbstractWorld { return getWorld().getName(); } + @Override + public Path getStoragePath() { + return getWorld().getDirectory(); + } + @SuppressWarnings("WeakerAccess") protected BlockState getBlockState(BlockStateHolder block) { if (block instanceof com.sk89q.worldedit.world.block.BlockState) { diff --git a/worldedit-sponge/src/main/java/com/sk89q/worldedit/sponge/SpongeWorldEdit.java b/worldedit-sponge/src/main/java/com/sk89q/worldedit/sponge/SpongeWorldEdit.java index d875d4938..711648a3c 100644 --- a/worldedit-sponge/src/main/java/com/sk89q/worldedit/sponge/SpongeWorldEdit.java +++ b/worldedit-sponge/src/main/java/com/sk89q/worldedit/sponge/SpongeWorldEdit.java @@ -20,6 +20,7 @@ package com.sk89q.worldedit.sponge; import static com.google.common.base.Preconditions.checkNotNull; +import static com.sk89q.worldedit.internal.anvil.ChunkDeleter.DELCHUNKS_FILE_NAME; import com.google.inject.Inject; import com.sk89q.worldedit.LocalSession; @@ -29,6 +30,7 @@ import com.sk89q.worldedit.event.platform.PlatformReadyEvent; import com.sk89q.worldedit.extension.platform.Actor; import com.sk89q.worldedit.extension.platform.Capability; import com.sk89q.worldedit.extension.platform.Platform; +import com.sk89q.worldedit.internal.anvil.ChunkDeleter; import com.sk89q.worldedit.sponge.adapter.AdapterLoadException; import com.sk89q.worldedit.sponge.adapter.SpongeImplAdapter; import com.sk89q.worldedit.sponge.adapter.SpongeImplLoader; @@ -62,6 +64,8 @@ import org.spongepowered.api.world.World; import java.io.File; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Optional; import java.util.concurrent.TimeUnit; @@ -134,6 +138,11 @@ public class SpongeWorldEdit { WorldEdit.getInstance().getPlatformManager().unregister(platform); } + final Path delChunks = workingDir.toPath().resolve(DELCHUNKS_FILE_NAME); + if (Files.exists(delChunks)) { + ChunkDeleter.runFromFile(delChunks, true); + } + this.platform = new SpongePlatform(this); this.provider = new SpongePermissionsProvider();