Properly close all files when dealing with archives (#1274)

* Properly close all files when dealing with archives

* Move file utils to SafeFiles class

* Licenses

(cherry picked from commit a600266d41151eec4f2239cf90e202bb99fa3a8b)
This commit is contained in:
Octavia Togami
2020-04-05 12:17:26 -04:00
committed by MattBDev
parent 8d1efcfb21
commit 374ad992a2
13 changed files with 897 additions and 130 deletions

View File

@ -0,0 +1,104 @@
/*
* WorldEdit, a Minecraft world manipulation toolkit
* Copyright (C) sk89q <http://www.sk89q.com>
* Copyright (C) WorldEdit team and contributors
*
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU Lesser General Public License as published by the
* Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License
* for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package com.sk89q.worldedit.world.snapshot.experimental.fs;
import com.sk89q.worldedit.util.io.Closer;
import com.sk89q.worldedit.util.io.file.ArchiveDir;
import com.sk89q.worldedit.util.io.file.ArchiveNioSupport;
import com.sk89q.worldedit.world.snapshot.experimental.Snapshot;
import javax.annotation.Nullable;
import java.io.IOException;
import java.net.URI;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import java.util.stream.Stream;
import static com.google.common.base.Preconditions.checkArgument;
import static java.util.stream.Collectors.toList;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* Context class for using a {@link FileSystemSnapshotDatabase}.
*/
class FSSDContext {
final ArchiveNioSupport archiveNioSupport;
final FileSystemSnapshotDatabase db;
FSSDContext(ArchiveNioSupport archiveNioSupport, Path root) {
this.archiveNioSupport = archiveNioSupport;
this.db = new FileSystemSnapshotDatabase(root, archiveNioSupport);
}
Path path(String first, String... more) {
Path p = db.getRoot().resolve(Paths.get(first, more));
checkArgument(p.startsWith(db.getRoot()), "Escaping root!");
return p;
}
URI nameUri(String name) {
return FileSystemSnapshotDatabase.createUri(name);
}
Snapshot requireSnapshot(String name) throws IOException {
return requireSnapshot(name, db.getSnapshot(nameUri(name)).orElse(null));
}
Snapshot requireListsSnapshot(String name) throws IOException {
// World name is the last element of the path
String worldName = Paths.get(name).getFileName().toString();
// Without an extension
worldName = worldName.split("\\.")[0];
List<Snapshot> snapshots;
try (Stream<Snapshot> snapshotStream = db.getSnapshots(worldName)) {
snapshots = snapshotStream.collect(toList());
}
try {
assertTrue(snapshots.size() <= 1,
"Too many snapshots matched for " + worldName);
return requireSnapshot(name, snapshots.stream().findAny().orElse(null));
} catch (Throwable t) {
Closer closer = Closer.create();
snapshots.forEach(closer::register);
throw closer.rethrowAndClose(t);
}
}
Snapshot requireSnapshot(String name, @Nullable Snapshot snapshot) throws IOException {
assertNotNull(snapshot, "No snapshot for " + name);
try {
assertEquals(name, snapshot.getInfo().getDisplayName());
} catch (Throwable t) {
Closer closer = Closer.create();
closer.register(snapshot);
throw closer.rethrowAndClose(t);
}
return snapshot;
}
ArchiveDir getRootOfArchive(Path archive) throws IOException {
return archiveNioSupport.tryOpenAsDir(archive)
.orElseThrow(() -> new AssertionError("No archive opener for " + archive));
}
}

View File

@ -0,0 +1,291 @@
/*
* WorldEdit, a Minecraft world manipulation toolkit
* Copyright (C) sk89q <http://www.sk89q.com>
* Copyright (C) WorldEdit team and contributors
*
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU Lesser General Public License as published by the
* Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License
* for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package com.sk89q.worldedit.world.snapshot.experimental.fs;
import com.google.common.collect.ImmutableList;
import com.sk89q.worldedit.math.BlockVector3;
import com.sk89q.worldedit.util.io.file.ArchiveDir;
import com.sk89q.worldedit.world.DataException;
import com.sk89q.worldedit.world.snapshot.experimental.Snapshot;
import org.junit.jupiter.api.DynamicNode;
import org.junit.jupiter.api.DynamicTest;
import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.FileTime;
import java.time.ZonedDateTime;
import java.util.List;
import java.util.concurrent.ThreadLocalRandom;
import java.util.stream.Stream;
import static com.sk89q.worldedit.world.snapshot.experimental.fs.FileSystemSnapshotDatabaseTest.CHUNK_POS;
import static com.sk89q.worldedit.world.snapshot.experimental.fs.FileSystemSnapshotDatabaseTest.CHUNK_TAG;
import static com.sk89q.worldedit.world.snapshot.experimental.fs.FileSystemSnapshotDatabaseTest.TIME_ONE;
import static com.sk89q.worldedit.world.snapshot.experimental.fs.FileSystemSnapshotDatabaseTest.TIME_TWO;
import static com.sk89q.worldedit.world.snapshot.experimental.fs.FileSystemSnapshotDatabaseTest.WORLD_ALPHA;
import static com.sk89q.worldedit.world.snapshot.experimental.fs.FileSystemSnapshotDatabaseTest.WORLD_BETA;
import static java.util.stream.Collectors.toList;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.DynamicContainer.dynamicContainer;
import static org.junit.jupiter.api.DynamicTest.dynamicTest;
enum FSSDTestType {
EMPTY {
@Override
List<DynamicTest> getTests(FSSDContext context) {
return ImmutableList.of(
dynamicTest("return an empty stream from getSnapshots(worldName)",
() -> context.db.getSnapshots(WORLD_ALPHA)),
dynamicTest("return an empty optional from getSnapshot(name)",
() -> context.db.getSnapshot(context.nameUri(WORLD_ALPHA)))
);
}
},
WORLD_ONLY_DIR {
@Override
List<DynamicTest> getTests(FSSDContext context) throws IOException {
Path worldFolder = EntryMaker.WORLD_DIR.createEntry(context.db.getRoot(), WORLD_ALPHA);
Files.setLastModifiedTime(worldFolder, FileTime.from(TIME_ONE.toInstant()));
return singleSnapTest(context, WORLD_ALPHA, TIME_ONE);
}
},
WORLD_ONLY_DIM_DIR {
@Override
List<DynamicTest> getTests(FSSDContext context) throws IOException {
int dim = ThreadLocalRandom.current().nextInt();
Path worldFolder = EntryMaker.WORLD_DIM_DIR
.createEntry(context.db.getRoot(), new EntryMaker.DimInfo(WORLD_ALPHA, dim));
Files.setLastModifiedTime(worldFolder, FileTime.from(TIME_ONE.toInstant()));
return singleSnapTest(context, WORLD_ALPHA, TIME_ONE);
}
},
WORLD_ONLY_NO_REGION_DIR {
@Override
List<DynamicTest> getTests(FSSDContext context) throws IOException {
Path worldFolder = EntryMaker.WORLD_NO_REGION_DIR
.createEntry(context.db.getRoot(), WORLD_ALPHA);
Files.setLastModifiedTime(worldFolder, FileTime.from(TIME_ONE.toInstant()));
return singleSnapTest(context, WORLD_ALPHA, TIME_ONE);
}
},
WORLD_LEGACY_DIR {
@Override
List<DynamicTest> getTests(FSSDContext context) throws IOException {
Path worldFolder = EntryMaker.WORLD_LEGACY_DIR
.createEntry(context.db.getRoot(), WORLD_ALPHA);
Files.setLastModifiedTime(worldFolder, FileTime.from(TIME_ONE.toInstant()));
return singleSnapTest(context, WORLD_ALPHA, TIME_ONE);
}
},
WORLD_ONLY_ARCHIVE {
@Override
List<DynamicTest> getTests(FSSDContext context) throws IOException {
Path worldArchive = EntryMaker.WORLD_ARCHIVE
.createEntry(context.db.getRoot(), WORLD_ALPHA);
try (ArchiveDir rootOfArchive = context.getRootOfArchive(worldArchive)) {
Files.setLastModifiedTime(
rootOfArchive.getPath(),
FileTime.from(TIME_ONE.toInstant())
);
}
return singleSnapTest(context, WORLD_ALPHA + ".zip", TIME_ONE);
}
},
TIMESTAMPED_DIR {
@Override
List<? extends DynamicNode> getTests(FSSDContext context) throws IOException {
Path root = context.db.getRoot();
Path timestampedDir = EntryMaker.TIMESTAMPED_DIR
.createEntry(root, TIME_ONE);
EntryMaker.WORLD_DIR.createEntry(timestampedDir, WORLD_ALPHA);
EntryMaker.WORLD_ARCHIVE.createEntry(timestampedDir, WORLD_BETA);
return ImmutableList.of(
dynamicContainer("world dir",
singleSnapTest(context,
root.relativize(timestampedDir) + File.separator + WORLD_ALPHA,
TIME_ONE)
),
dynamicContainer("world archive",
singleSnapTest(context,
root.relativize(timestampedDir) + File.separator + WORLD_BETA + ".zip",
TIME_ONE)
)
);
}
},
TIMESTAMPED_ARCHIVE {
@Override
List<? extends DynamicNode> getTests(FSSDContext context) throws IOException {
Path root = context.db.getRoot();
Path timestampedArchive = EntryMaker.TIMESTAMPED_ARCHIVE
.createEntry(root, TIME_ONE);
try (ArchiveDir timestampedDir = context.getRootOfArchive(timestampedArchive)) {
EntryMaker.WORLD_DIR.createEntry(timestampedDir.getPath(), WORLD_ALPHA);
EntryMaker.WORLD_ARCHIVE.createEntry(timestampedDir.getPath(), WORLD_BETA);
}
return ImmutableList.of(
dynamicContainer("world dir",
singleSnapTest(context,
root.relativize(timestampedArchive) + File.separator + WORLD_ALPHA,
TIME_ONE)
)
);
}
},
WORLD_DIR_TIMESTAMPED_DIR {
@Override
List<? extends DynamicNode> getTests(FSSDContext context) throws IOException {
Path root = context.db.getRoot();
Path timestampedDirA = EntryMaker.TIMESTAMPED_DIR
.createEntry(root.resolve(WORLD_ALPHA), TIME_ONE);
Path timestampedDirB = EntryMaker.TIMESTAMPED_DIR
.createEntry(root.resolve(WORLD_BETA), TIME_ONE);
EntryMaker.WORLD_DIR.createEntry(timestampedDirA, WORLD_ALPHA);
EntryMaker.WORLD_ARCHIVE.createEntry(timestampedDirB, WORLD_BETA);
return ImmutableList.of(
dynamicContainer("world dir",
singleSnapTest(context,
root.relativize(timestampedDirA) + File.separator + WORLD_ALPHA,
TIME_ONE)
),
dynamicContainer("world archive",
singleSnapTest(context,
root.relativize(timestampedDirB) + File.separator + WORLD_BETA + ".zip",
TIME_ONE)
)
);
}
},
TWO_TIMESTAMPED_WORLD_DIR {
@Override
List<DynamicTest> getTests(FSSDContext context) throws IOException {
Path root = context.db.getRoot();
Path timestampedDirA = EntryMaker.TIMESTAMPED_DIR
.createEntry(root, TIME_ONE);
EntryMaker.WORLD_DIR.createEntry(timestampedDirA, WORLD_ALPHA);
Path timestampedDirB = EntryMaker.TIMESTAMPED_DIR
.createEntry(root, TIME_TWO);
EntryMaker.WORLD_DIR.createEntry(timestampedDirB, WORLD_ALPHA);
return ImmutableList.of(
dynamicTest("list both snapshots in order (newest first)", () -> {
List<Snapshot> snapshots = context.db
.getSnapshotsNewestFirst(WORLD_ALPHA).collect(toList());
assertEquals(2, snapshots.size());
assertValidSnapshot(TIME_ONE, snapshots.get(0));
assertValidSnapshot(TIME_TWO, snapshots.get(1));
}),
dynamicTest("list both snapshots in order (oldest first)", () -> {
List<Snapshot> snapshots = context.db
.getSnapshotsOldestFirst(WORLD_ALPHA).collect(toList());
assertEquals(2, snapshots.size());
assertValidSnapshot(TIME_TWO, snapshots.get(0));
assertValidSnapshot(TIME_ONE, snapshots.get(1));
}),
dynamicTest("list only 1 if getting AFTER 2", () -> {
List<Snapshot> snapshots = context.db
.getSnapshotsAfter(WORLD_ALPHA, TIME_TWO).collect(toList());
assertEquals(1, snapshots.size());
assertValidSnapshot(TIME_ONE, snapshots.get(0));
}),
dynamicTest("list only 2 if getting BEFORE 1", () -> {
List<Snapshot> snapshots = context.db
.getSnapshotsBefore(WORLD_ALPHA, TIME_ONE).collect(toList());
assertEquals(1, snapshots.size());
assertValidSnapshot(TIME_TWO, snapshots.get(0));
}),
dynamicTest("list both if AFTER time before 2", () -> {
List<Snapshot> snapshots = context.db
.getSnapshotsAfter(WORLD_ALPHA, TIME_TWO.minusSeconds(1))
.collect(toList());
assertEquals(2, snapshots.size());
// sorted newest first
assertValidSnapshot(TIME_ONE, snapshots.get(0));
assertValidSnapshot(TIME_TWO, snapshots.get(1));
}),
dynamicTest("list both if BEFORE time after 1", () -> {
List<Snapshot> snapshots = context.db
.getSnapshotsBefore(WORLD_ALPHA, TIME_ONE.plusSeconds(1))
.collect(toList());
assertEquals(2, snapshots.size());
// sorted oldest first
assertValidSnapshot(TIME_TWO, snapshots.get(0));
assertValidSnapshot(TIME_ONE, snapshots.get(1));
}
)
);
}
},
TWO_WORLD_ONLY_DIR {
@Override
List<? extends DynamicNode> getTests(FSSDContext context) throws IOException {
Path worldFolderA = EntryMaker.WORLD_DIR
.createEntry(context.db.getRoot(), WORLD_ALPHA);
Files.setLastModifiedTime(worldFolderA, FileTime.from(TIME_ONE.toInstant()));
Path worldFolderB = EntryMaker.WORLD_DIR
.createEntry(context.db.getRoot(), WORLD_BETA);
Files.setLastModifiedTime(worldFolderB, FileTime.from(TIME_TWO.toInstant()));
return Stream.of(
singleSnapTest(context, WORLD_ALPHA, TIME_ONE),
singleSnapTest(context, WORLD_BETA, TIME_TWO)
).flatMap(List::stream).collect(toList());
}
};
List<DynamicTest> singleSnapTest(FSSDContext context, String name,
ZonedDateTime time) {
return ImmutableList.of(
dynamicTest("return a valid snapshot for " + name, () -> {
try (Snapshot snapshot = context.requireSnapshot(name)) {
assertValidSnapshot(time, snapshot);
}
}),
dynamicTest("list a valid snapshot for " + name, () -> {
try (Snapshot snapshot = context.requireListsSnapshot(name)) {
assertValidSnapshot(time, snapshot);
}
})
);
}
private static void assertValidSnapshot(ZonedDateTime time,
Snapshot snapshot) throws IOException, DataException {
assertEquals(time, snapshot.getInfo().getDateTime());
// MCA file
assertEquals(CHUNK_TAG.toString(), snapshot.getChunkTag(CHUNK_POS).toString());
// MCR file
BlockVector3 offsetChunkPos = CHUNK_POS.add(32, 0, 32);
assertEquals(CHUNK_TAG.toString(), snapshot.getChunkTag(offsetChunkPos).toString());
}
abstract List<? extends DynamicNode> getTests(FSSDContext context) throws IOException;
Stream<DynamicNode> getNamedTests(FSSDContext context) throws IOException {
return Stream.of(dynamicContainer(
name(),
URI.create("method:" + getClass().getName() +
"#getTests(" + FSSDContext.class.getName() + ")"),
getTests(context).stream()
));
}
}

View File

@ -0,0 +1,195 @@
/*
* WorldEdit, a Minecraft world manipulation toolkit
* Copyright (C) sk89q <http://www.sk89q.com>
* Copyright (C) WorldEdit team and contributors
*
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU Lesser General Public License as published by the
* Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License
* for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package com.sk89q.worldedit.world.snapshot.experimental.fs;
import com.google.common.io.ByteStreams;
import com.google.common.io.Resources;
import com.sk89q.jnbt.CompoundTag;
import com.sk89q.worldedit.math.BlockVector2;
import com.sk89q.worldedit.math.BlockVector3;
import com.sk89q.worldedit.util.io.file.ArchiveNioSupport;
import com.sk89q.worldedit.util.io.file.ArchiveNioSupports;
import com.sk89q.worldedit.util.io.file.TrueVfsArchiveNioSupport;
import com.sk89q.worldedit.util.io.file.ZipArchiveNioSupport;
import com.sk89q.worldedit.world.DataException;
import com.sk89q.worldedit.world.storage.ChunkStoreHelper;
import com.sk89q.worldedit.world.storage.McRegionReader;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.DynamicNode;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestFactory;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.UncheckedIOException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.time.Instant;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.function.Function;
import java.util.stream.IntStream;
import java.util.stream.Stream;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.DynamicContainer.dynamicContainer;
@DisplayName("A FS Snapshot Database")
class FileSystemSnapshotDatabaseTest {
static byte[] REGION_DATA;
static byte[] CHUNK_DATA;
static CompoundTag CHUNK_TAG;
static BlockVector3 CHUNK_POS;
static final String WORLD_ALPHA = "World Alpha";
static final String WORLD_BETA = "World Beta";
static final DateTimeFormatter FORMATTER =
DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH_mm_ss");
static final ZonedDateTime TIME_ONE = Instant.parse("2018-01-01T12:00:00.00Z")
.atZone(ZoneId.systemDefault());
static final ZonedDateTime TIME_TWO = TIME_ONE.minusDays(1);
private static Path TEMP_DIR;
@BeforeAll
static void setUpStatic() throws IOException, DataException {
try (InputStream in = Resources.getResource("world_region.mca.gzip").openStream();
GZIPInputStream gzIn = new GZIPInputStream(in)) {
REGION_DATA = ByteStreams.toByteArray(gzIn);
}
McRegionReader reader = new McRegionReader(new ByteArrayInputStream(REGION_DATA));
try {
// Find the single chunk
BlockVector2 chunkPos = IntStream.range(0, 32).mapToObj(
x -> IntStream.range(0, 32).filter(z -> reader.hasChunk(x, z))
.mapToObj(z -> BlockVector2.at(x, z))
).flatMap(Function.identity())
.findAny()
.orElseThrow(() -> new AssertionError("No chunk in region file."));
ByteArrayOutputStream cap = new ByteArrayOutputStream();
try (InputStream in = reader.getChunkInputStream(chunkPos);
GZIPOutputStream gzOut = new GZIPOutputStream(cap)) {
ByteStreams.copy(in, gzOut);
}
CHUNK_DATA = cap.toByteArray();
CHUNK_TAG = ChunkStoreHelper.readCompoundTag(() -> new GZIPInputStream(
new ByteArrayInputStream(CHUNK_DATA)
));
CHUNK_POS = chunkPos.toBlockVector3();
} finally {
reader.close();
}
TEMP_DIR = Files.createTempDirectory("worldedit-fs-snap-dbs");
}
@AfterAll
static void afterAll() throws IOException {
deleteTree(TEMP_DIR);
}
private static Path newTempDb() throws IOException {
return Files.createTempDirectory(TEMP_DIR, "db");
}
private static void deleteTree(Path root) throws IOException {
Files.walkFileTree(root, new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
Files.deleteIfExists(file);
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
Files.deleteIfExists(dir);
return FileVisitResult.CONTINUE;
}
});
}
@DisplayName("makes the root directory absolute if needed")
@Test
void rootIsAbsoluteDir() throws IOException {
Path root = newTempDb();
try {
Path relative = root.getFileSystem().getPath("relative");
Files.createDirectories(relative);
FileSystemSnapshotDatabase db2 = new FileSystemSnapshotDatabase(relative,
ArchiveNioSupports.combined());
assertEquals(root.getFileSystem().getPath(".").toRealPath()
.resolve(relative), db2.getRoot());
Path absolute = root.resolve("absolute");
Files.createDirectories(absolute);
FileSystemSnapshotDatabase db3 = new FileSystemSnapshotDatabase(absolute,
ArchiveNioSupports.combined());
assertEquals(absolute, db3.getRoot());
} finally {
deleteTree(root);
}
}
@DisplayName("with a specific NIO support:")
@TestFactory
Stream<DynamicNode> withSpecificNioSupport() {
return Stream.of(
ZipArchiveNioSupport.getInstance(), TrueVfsArchiveNioSupport.getInstance()
)
.map(nioSupport -> {
Stream<? extends DynamicNode> nodes = Stream.of(FSSDTestType.values())
.flatMap(type -> {
try {
return getTests(nioSupport, type);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
});
return dynamicContainer(
nioSupport.getClass().getSimpleName() + ", can, for format:",
nodes
);
});
}
private static Stream<? extends DynamicNode> getTests(ArchiveNioSupport nioSupport,
FSSDTestType type) throws IOException {
Path root = newTempDb();
try {
Path dbRoot = root.resolve("snapshots");
Files.createDirectories(dbRoot);
return type.getNamedTests(new FSSDContext(nioSupport, dbRoot));
} catch (Throwable t) {
deleteTree(root);
throw t;
}
}
}