Properly close all files when dealing with archives (#1274)

* Properly close all files when dealing with archives

* Move file utils to SafeFiles class

* Licenses

(cherry picked from commit a600266d41151eec4f2239cf90e202bb99fa3a8b)
This commit is contained in:
Octavia Togami 2020-04-05 12:17:26 -04:00 committed by MattBDev
parent 8d1efcfb21
commit 374ad992a2
13 changed files with 897 additions and 130 deletions

View File

@ -0,0 +1,44 @@
/*
* WorldEdit, a Minecraft world manipulation toolkit
* Copyright (C) sk89q <http://www.sk89q.com>
* Copyright (C) WorldEdit team and contributors
*
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU Lesser General Public License as published by the
* Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License
* for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package com.sk89q.worldedit.util.function;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.function.Function;
/**
* I/O function type.
*/
@FunctionalInterface
public interface IOFunction<T, R> {
static <T, R> Function<T, R> unchecked(IOFunction<T, R> function) {
return param -> {
try {
return function.apply(param);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
};
}
R apply(T param) throws IOException;
}

View File

@ -20,6 +20,7 @@
package com.sk89q.worldedit.util.function;
import java.io.IOException;
import java.io.UncheckedIOException;
/**
* I/O runnable type.
@ -27,6 +28,16 @@ import java.io.IOException;
@FunctionalInterface
public interface IORunnable {
static Runnable unchecked(IORunnable runnable) {
return () -> {
try {
runnable.run();
} catch (IOException e) {
throw new UncheckedIOException(e);
}
};
}
void run() throws IOException;
}

View File

@ -0,0 +1,33 @@
/*
* WorldEdit, a Minecraft world manipulation toolkit
* Copyright (C) sk89q <http://www.sk89q.com>
* Copyright (C) WorldEdit team and contributors
*
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU Lesser General Public License as published by the
* Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License
* for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package com.sk89q.worldedit.util.io.file;
import java.io.Closeable;
import java.nio.file.Path;
/**
* Represents an archive opened as a directory. This must be closed after work on the Path is
* done.
*/
public interface ArchiveDir extends Closeable {
Path getPath();
}

View File

@ -35,6 +35,6 @@ public interface ArchiveNioSupport {
* @param archive the archive to open
* @return the path for the root of the archive, if available
*/
Optional<Path> tryOpenAsDir(Path archive) throws IOException;
Optional<ArchiveDir> tryOpenAsDir(Path archive) throws IOException;
}

View File

@ -45,9 +45,9 @@ public class ArchiveNioSupports {
.build();
}
public static Optional<Path> tryOpenAsDir(Path archive) throws IOException {
public static Optional<ArchiveDir> tryOpenAsDir(Path archive) throws IOException {
for (ArchiveNioSupport support : SUPPORTS) {
Optional<Path> fs = support.tryOpenAsDir(archive);
Optional<ArchiveDir> fs = support.tryOpenAsDir(archive);
if (fs.isPresent()) {
return fs;
}

View File

@ -0,0 +1,69 @@
/*
* WorldEdit, a Minecraft world manipulation toolkit
* Copyright (C) sk89q <http://www.sk89q.com>
* Copyright (C) WorldEdit team and contributors
*
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU Lesser General Public License as published by the
* Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License
* for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package com.sk89q.worldedit.util.io.file;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
public class SafeFiles {
/**
* A version of {@link Files#list(Path)} that won't leak resources.
*
* <p>
* Instead, it immediately consumes the entire listing into a {@link List} and
* calls {@link List#stream()}.
* </p>
*
* @param dir the directory to list
* @return an I/O-resource-free stream of the files in the directory
* @throws IOException if an I/O error occurs
*/
public static Stream<Path> noLeakFileList(Path dir) throws IOException {
try (Stream<Path> stream = Files.list(dir)) {
return stream.collect(Collectors.toList()).stream();
}
}
/**
* {@link Path#getFileName()} includes a slash sometimes for some reason.
* This will get rid of it.
*
* @param path the path to get the file name for
* @return the file name of the given path
*/
public static String canonicalFileName(Path path) {
return dropSlash(path.getFileName().toString());
}
private static String dropSlash(String name) {
if (name.isEmpty() || name.codePointBefore(name.length()) != '/') {
return name;
}
return name.substring(0, name.length() - 1);
}
private SafeFiles() {
}
}

View File

@ -22,6 +22,7 @@ package com.sk89q.worldedit.util.io.file;
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableSet;
import net.java.truevfs.access.TArchiveDetector;
import net.java.truevfs.access.TFileSystem;
import net.java.truevfs.access.TPath;
import java.io.IOException;
@ -45,15 +46,28 @@ public final class TrueVfsArchiveNioSupport implements ArchiveNioSupport {
}
@Override
public Optional<Path> tryOpenAsDir(Path archive) throws IOException {
public Optional<ArchiveDir> tryOpenAsDir(Path archive) throws IOException {
String fileName = archive.getFileName().toString();
int dot = fileName.indexOf('.');
if (dot < 0 || dot >= fileName.length() || !ALLOWED_EXTENSIONS.contains(fileName.substring(dot + 1))) {
if (dot < 0 || dot >= fileName.length() || !ALLOWED_EXTENSIONS
.contains(fileName.substring(dot + 1))) {
return Optional.empty();
}
TPath root = new TPath(archive).getFileSystem().getPath("/");
return Optional.of(ArchiveNioSupports.skipRootSameName(
TFileSystem fileSystem = new TPath(archive).getFileSystem();
TPath root = fileSystem.getPath("/");
Path realRoot = ArchiveNioSupports.skipRootSameName(
root, fileName.substring(0, dot)
));
);
return Optional.of(new ArchiveDir() {
@Override
public Path getPath() {
return realRoot;
}
@Override
public void close() throws IOException {
fileSystem.close();
}
});
}
}

View File

@ -37,17 +37,28 @@ public final class ZipArchiveNioSupport implements ArchiveNioSupport {
}
@Override
public Optional<Path> tryOpenAsDir(Path archive) throws IOException {
public Optional<ArchiveDir> tryOpenAsDir(Path archive) throws IOException {
if (!archive.getFileName().toString().endsWith(".zip")) {
return Optional.empty();
}
FileSystem zipFs = FileSystems.newFileSystem(
archive, getClass().getClassLoader()
);
return Optional.of(ArchiveNioSupports.skipRootSameName(
Path root = ArchiveNioSupports.skipRootSameName(
zipFs.getPath("/"), archive.getFileName().toString()
.replaceFirst("\\.zip$", "")
));
);
return Optional.of(new ArchiveDir() {
@Override
public Path getPath() {
return root;
}
@Override
public void close() throws IOException {
zipFs.close();
}
});
}
}

View File

@ -21,34 +21,31 @@ package com.sk89q.worldedit.world.snapshot.experimental.fs;
import com.google.common.collect.ImmutableList;
import com.google.common.net.UrlEscapers;
import com.sk89q.worldedit.util.function.IOFunction;
import com.sk89q.worldedit.util.function.IORunnable;
import com.sk89q.worldedit.util.io.Closer;
import com.sk89q.worldedit.util.io.file.ArchiveDir;
import com.sk89q.worldedit.util.io.file.ArchiveNioSupport;
import com.sk89q.worldedit.util.io.file.MorePaths;
import com.sk89q.worldedit.util.io.file.SafeFiles;
import com.sk89q.worldedit.util.time.FileNameDateTimeParser;
import com.sk89q.worldedit.util.time.ModificationDateTimeParser;
import com.sk89q.worldedit.util.time.SnapshotDateTimeParser;
import com.sk89q.worldedit.world.snapshot.experimental.Snapshot;
import com.sk89q.worldedit.world.snapshot.experimental.SnapshotDatabase;
import com.sk89q.worldedit.world.snapshot.experimental.SnapshotInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.Nullable;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.net.URI;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.ZonedDateTime;
import java.util.Iterator;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.ServiceLoader;
import java.util.function.Function;
import java.util.stream.Stream;
import static com.google.common.base.Preconditions.checkArgument;
@ -58,8 +55,6 @@ import static com.google.common.base.Preconditions.checkArgument;
*/
public class FileSystemSnapshotDatabase implements SnapshotDatabase {
private static final Logger logger = LoggerFactory.getLogger(FileSystemSnapshotDatabase.class);
private static final String SCHEME = "snapfs";
private static final List<SnapshotDateTimeParser> DATE_TIME_PARSERS =
@ -102,15 +97,24 @@ public class FileSystemSnapshotDatabase implements SnapshotDatabase {
this.archiveNioSupport = archiveNioSupport;
}
private SnapshotInfo createSnapshotInfo(Path fullPath, Path realPath) {
// Try full for parsing out of file name, real for parsing mod time.
ZonedDateTime date = tryParseDateInternal(fullPath).orElseGet(() -> tryParseDate(realPath));
return SnapshotInfo.create(createUri(fullPath.toString()), date);
/*
* When this code says "idPath" it is the path that uniquely identifies that snapshot.
* A snapshot can be looked up by its idPath.
*
* When the code says "ioPath" it is the path that holds the world data, and can actually
* be read from proper. The "idPath" may not even exist, it is purely for the path components
* and not for IO.
*/
private SnapshotInfo createSnapshotInfo(Path idPath, Path ioPath) {
// Try ID for parsing out of file name, IO for parsing mod time.
ZonedDateTime date = tryParseDateInternal(idPath).orElseGet(() -> tryParseDate(ioPath));
return SnapshotInfo.create(createUri(idPath.toString()), date);
}
private Snapshot createSnapshot(Path fullPath, Path realPath, @Nullable IORunnable closeCallback) {
private Snapshot createSnapshot(Path idPath, Path ioPath, @Nullable Closer closeCallback) {
return new FolderSnapshot(
createSnapshotInfo(fullPath, realPath), realPath, closeCallback
createSnapshotInfo(idPath, ioPath), ioPath, closeCallback
);
}
@ -128,27 +132,31 @@ public class FileSystemSnapshotDatabase implements SnapshotDatabase {
if (!name.getScheme().equals(SCHEME)) {
return Optional.empty();
}
// drop the / in the path to make it absolute
Path rawResolved = root.resolve(name.getSchemeSpecificPart());
return getSnapshot(name.getSchemeSpecificPart());
}
private Optional<Snapshot> getSnapshot(String id) throws IOException {
Path rawResolved = root.resolve(id);
// Catch trickery with paths:
Path realPath = rawResolved.normalize();
if (!realPath.startsWith(root)) {
Path ioPath = rawResolved.normalize();
if (!ioPath.startsWith(root)) {
return Optional.empty();
}
Optional<Snapshot> result = tryRegularFileSnapshot(root.relativize(realPath), realPath);
Path idPath = root.relativize(ioPath);
Optional<Snapshot> result = tryRegularFileSnapshot(idPath);
if (result.isPresent()) {
return result;
}
if (!Files.isDirectory(realPath)) {
if (!Files.isDirectory(ioPath)) {
return Optional.empty();
}
return Optional.of(createSnapshot(root.relativize(realPath), realPath, null));
return Optional.of(createSnapshot(idPath, ioPath, null));
}
private Optional<Snapshot> tryRegularFileSnapshot(Path fullPath, Path realPath) throws IOException {
private Optional<Snapshot> tryRegularFileSnapshot(Path idPath) throws IOException {
Closer closer = Closer.create();
Path root = this.root;
Path relative = root.relativize(realPath);
Path relative = idPath;
Iterator<Path> iterator = null;
try {
while (true) {
@ -156,6 +164,7 @@ public class FileSystemSnapshotDatabase implements SnapshotDatabase {
iterator = MorePaths.iterPaths(relative).iterator();
}
if (!iterator.hasNext()) {
closer.close();
return Optional.empty();
}
Path relativeNext = iterator.next();
@ -164,18 +173,17 @@ public class FileSystemSnapshotDatabase implements SnapshotDatabase {
// This will never be it.
continue;
}
Optional<Path> newRootOpt = archiveNioSupport.tryOpenAsDir(next);
Optional<ArchiveDir> newRootOpt = archiveNioSupport.tryOpenAsDir(next);
if (newRootOpt.isPresent()) {
root = newRootOpt.get();
if (root.getFileSystem() != FileSystems.getDefault()) {
closer.register(root.getFileSystem());
}
ArchiveDir archiveDir = newRootOpt.get();
root = archiveDir.getPath();
closer.register(archiveDir);
// Switch path to path inside the archive
relative = root.resolve(relativeNext.relativize(relative).toString());
iterator = null;
// Check if it exists, if so open snapshot
if (Files.exists(relative)) {
return Optional.of(createSnapshot(fullPath, relative, closer::close));
return Optional.of(createSnapshot(idPath, relative, closer));
}
// Otherwise, we may have more archives to open.
// Keep searching!
@ -191,110 +199,97 @@ public class FileSystemSnapshotDatabase implements SnapshotDatabase {
/*
There are a few possible snapshot formats we accept:
- a world directory, identified by <worldName>/level.dat
- a directory with the world name, but no level.dat
- inside must be a timestamped directory/archive, which then has one of the two world
formats inside of it!
- a world archive, identified by <worldName>.ext
* does not need to have level.dat inside
- a timestamped directory, identified by <stamp>, that can have
- the two world formats described above, inside the directory
- a timestamped archive, identified by <stamp>.ext, that can have
- the same as timestamped directory, but inside the archive.
- a directory with the world name, but no level.dat
- inside must be timestamped directory/archive, with the world inside that
All archives may have a root directory with the same name as the archive,
minus the extensions. Due to extension detection methods, this won't work properly
with some files, e.g. world.qux.zip/world.qux is invalid, but world.qux.zip/world isn't.
*/
return Stream.of(
listWorldEntries(Paths.get(""), root, worldName),
listTimestampedEntries(Paths.get(""), root, worldName)
).flatMap(Function.identity());
return SafeFiles.noLeakFileList(root)
.flatMap(IOFunction.unchecked(entry -> {
String worldEntry = getWorldEntry(worldName, entry);
if (worldEntry != null) {
return Stream.of(worldEntry);
}
String fileName = SafeFiles.canonicalFileName(entry);
if (fileName.equals(worldName)
&& Files.isDirectory(entry)
&& !Files.exists(entry.resolve("level.dat"))) {
// world dir with timestamp entries
return listTimestampedEntries(worldName, entry)
.map(id -> worldName + "/" + id);
}
return getTimestampedEntries(worldName, entry);
}))
.map(IOFunction.unchecked(id ->
getSnapshot(id)
.orElseThrow(() ->
new AssertionError("Could not find discovered snapshot: " + id)
)
));
}
private Stream<Snapshot> listWorldEntries(Path fullPath, Path root, String worldName) throws IOException {
logger.debug("World check in: {}", root);
return Files.list(root)
.flatMap(candidate -> {
logger.debug("World trying: {}", candidate);
// Try world directory
String fileName = candidate.getFileName().toString();
if (isSameDirectoryName(fileName, worldName)) {
// Direct
if (Files.exists(candidate.resolve("level.dat"))) {
logger.debug("Direct!");
return Stream.of(createSnapshot(
fullPath.resolve(fileName), candidate, null
));
}
// Container for time-stamped entries
try {
return listTimestampedEntries(
fullPath.resolve(fileName), candidate, worldName
);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
// Try world archive
if (Files.isRegularFile(candidate)
&& fileName.startsWith(worldName + ".")) {
logger.debug("Archive!");
try {
return tryRegularFileSnapshot(
fullPath.resolve(fileName), candidate
).map(Stream::of).orElse(null);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
logger.debug("Nothing!");
return null;
});
private Stream<String> listTimestampedEntries(String worldName, Path directory) throws IOException {
return SafeFiles.noLeakFileList(directory)
.flatMap(IOFunction.unchecked(entry -> getTimestampedEntries(worldName, entry)));
}
private boolean isSameDirectoryName(String fileName, String worldName) {
if (fileName.lastIndexOf('/') == fileName.length() - 1) {
fileName = fileName.substring(0, fileName.length() - 1);
private Stream<String> getTimestampedEntries(String worldName, Path entry) throws IOException {
ZonedDateTime dateTime = FileNameDateTimeParser.getInstance().detectDateTime(entry);
if (dateTime == null) {
// nothing available at this path
return Stream.of();
}
return fileName.equalsIgnoreCase(worldName);
String fileName = SafeFiles.canonicalFileName(entry);
if (Files.isDirectory(entry)) {
// timestamped directory, find worlds inside
return listWorldEntries(worldName, entry)
.map(id -> fileName + "/" + id);
}
if (!Files.isRegularFile(entry)) {
// not an archive either?
return Stream.of();
}
Optional<ArchiveDir> asArchive = archiveNioSupport.tryOpenAsDir(entry);
if (asArchive.isPresent()) {
// timestamped archive
ArchiveDir dir = asArchive.get();
return listWorldEntries(worldName, dir.getPath())
.map(id -> fileName + "/" + id)
.onClose(IORunnable.unchecked(dir::close));
}
return Stream.of();
}
private Stream<Snapshot> listTimestampedEntries(Path fullPath, Path root, String worldName) throws IOException {
logger.debug("Timestamp check in: {}", root);
return Files.list(root)
.filter(candidate -> {
ZonedDateTime date = FileNameDateTimeParser.getInstance().detectDateTime(candidate);
return date != null;
})
.flatMap(candidate -> {
logger.debug("Timestamp trying: {}", candidate);
// Try timestamped directory
if (Files.isDirectory(candidate)) {
logger.debug("Timestamped directory");
try {
return listWorldEntries(
fullPath.resolve(candidate.getFileName().toString()), candidate, worldName
);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
// Otherwise archive, get it as a directory & unpack it
try {
Optional<Path> newRoot = archiveNioSupport.tryOpenAsDir(candidate);
if (!newRoot.isPresent()) {
logger.debug("Nothing!");
return null;
}
logger.debug("Timestamped archive!");
return listWorldEntries(
fullPath.resolve(candidate.getFileName().toString()),
newRoot.get(),
worldName
);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
});
private Stream<String> listWorldEntries(String worldName, Path directory) throws IOException {
return SafeFiles.noLeakFileList(directory)
.map(IOFunction.unchecked(entry -> getWorldEntry(worldName, entry)))
.filter(Objects::nonNull);
}
private String getWorldEntry(String worldName, Path entry) throws IOException {
String fileName = SafeFiles.canonicalFileName(entry);
if (fileName.equals(worldName) && Files.exists(entry.resolve("level.dat"))) {
// world directory
return worldName;
}
if (fileName.startsWith(worldName + ".") && Files.isRegularFile(entry)) {
Optional<ArchiveDir> asArchive = archiveNioSupport.tryOpenAsDir(entry);
if (asArchive.isPresent()) {
// world archive
asArchive.get().close();
return fileName;
}
}
return null;
}
}

View File

@ -22,7 +22,7 @@ package com.sk89q.worldedit.world.snapshot.experimental.fs;
import com.sk89q.jnbt.CompoundTag;
import com.sk89q.worldedit.math.BlockVector2;
import com.sk89q.worldedit.math.BlockVector3;
import com.sk89q.worldedit.util.function.IORunnable;
import com.sk89q.worldedit.util.io.Closer;
import com.sk89q.worldedit.world.DataException;
import com.sk89q.worldedit.world.snapshot.experimental.Snapshot;
import com.sk89q.worldedit.world.snapshot.experimental.SnapshotInfo;
@ -95,9 +95,9 @@ public class FolderSnapshot implements Snapshot {
private final SnapshotInfo info;
private final Path folder;
private final AtomicReference<Object> regionFolder = new AtomicReference<>();
private final @Nullable IORunnable closeCallback;
private final @Nullable Closer closeCallback;
public FolderSnapshot(SnapshotInfo info, Path folder, @Nullable IORunnable closeCallback) {
public FolderSnapshot(SnapshotInfo info, Path folder, @Nullable Closer closeCallback) {
this.info = info;
// This is required to force TrueVfs to properly resolve parents.
// Kinda odd, but whatever works.
@ -160,7 +160,7 @@ public class FolderSnapshot implements Snapshot {
@Override
public void close() throws IOException {
if (closeCallback != null) {
closeCallback.run();
closeCallback.close();
}
}
}

View File

@ -0,0 +1,104 @@
/*
* WorldEdit, a Minecraft world manipulation toolkit
* Copyright (C) sk89q <http://www.sk89q.com>
* Copyright (C) WorldEdit team and contributors
*
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU Lesser General Public License as published by the
* Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License
* for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package com.sk89q.worldedit.world.snapshot.experimental.fs;
import com.sk89q.worldedit.util.io.Closer;
import com.sk89q.worldedit.util.io.file.ArchiveDir;
import com.sk89q.worldedit.util.io.file.ArchiveNioSupport;
import com.sk89q.worldedit.world.snapshot.experimental.Snapshot;
import javax.annotation.Nullable;
import java.io.IOException;
import java.net.URI;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import java.util.stream.Stream;
import static com.google.common.base.Preconditions.checkArgument;
import static java.util.stream.Collectors.toList;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* Context class for using a {@link FileSystemSnapshotDatabase}.
*/
class FSSDContext {
final ArchiveNioSupport archiveNioSupport;
final FileSystemSnapshotDatabase db;
FSSDContext(ArchiveNioSupport archiveNioSupport, Path root) {
this.archiveNioSupport = archiveNioSupport;
this.db = new FileSystemSnapshotDatabase(root, archiveNioSupport);
}
Path path(String first, String... more) {
Path p = db.getRoot().resolve(Paths.get(first, more));
checkArgument(p.startsWith(db.getRoot()), "Escaping root!");
return p;
}
URI nameUri(String name) {
return FileSystemSnapshotDatabase.createUri(name);
}
Snapshot requireSnapshot(String name) throws IOException {
return requireSnapshot(name, db.getSnapshot(nameUri(name)).orElse(null));
}
Snapshot requireListsSnapshot(String name) throws IOException {
// World name is the last element of the path
String worldName = Paths.get(name).getFileName().toString();
// Without an extension
worldName = worldName.split("\\.")[0];
List<Snapshot> snapshots;
try (Stream<Snapshot> snapshotStream = db.getSnapshots(worldName)) {
snapshots = snapshotStream.collect(toList());
}
try {
assertTrue(snapshots.size() <= 1,
"Too many snapshots matched for " + worldName);
return requireSnapshot(name, snapshots.stream().findAny().orElse(null));
} catch (Throwable t) {
Closer closer = Closer.create();
snapshots.forEach(closer::register);
throw closer.rethrowAndClose(t);
}
}
Snapshot requireSnapshot(String name, @Nullable Snapshot snapshot) throws IOException {
assertNotNull(snapshot, "No snapshot for " + name);
try {
assertEquals(name, snapshot.getInfo().getDisplayName());
} catch (Throwable t) {
Closer closer = Closer.create();
closer.register(snapshot);
throw closer.rethrowAndClose(t);
}
return snapshot;
}
ArchiveDir getRootOfArchive(Path archive) throws IOException {
return archiveNioSupport.tryOpenAsDir(archive)
.orElseThrow(() -> new AssertionError("No archive opener for " + archive));
}
}

View File

@ -0,0 +1,291 @@
/*
* WorldEdit, a Minecraft world manipulation toolkit
* Copyright (C) sk89q <http://www.sk89q.com>
* Copyright (C) WorldEdit team and contributors
*
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU Lesser General Public License as published by the
* Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License
* for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package com.sk89q.worldedit.world.snapshot.experimental.fs;
import com.google.common.collect.ImmutableList;
import com.sk89q.worldedit.math.BlockVector3;
import com.sk89q.worldedit.util.io.file.ArchiveDir;
import com.sk89q.worldedit.world.DataException;
import com.sk89q.worldedit.world.snapshot.experimental.Snapshot;
import org.junit.jupiter.api.DynamicNode;
import org.junit.jupiter.api.DynamicTest;
import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.FileTime;
import java.time.ZonedDateTime;
import java.util.List;
import java.util.concurrent.ThreadLocalRandom;
import java.util.stream.Stream;
import static com.sk89q.worldedit.world.snapshot.experimental.fs.FileSystemSnapshotDatabaseTest.CHUNK_POS;
import static com.sk89q.worldedit.world.snapshot.experimental.fs.FileSystemSnapshotDatabaseTest.CHUNK_TAG;
import static com.sk89q.worldedit.world.snapshot.experimental.fs.FileSystemSnapshotDatabaseTest.TIME_ONE;
import static com.sk89q.worldedit.world.snapshot.experimental.fs.FileSystemSnapshotDatabaseTest.TIME_TWO;
import static com.sk89q.worldedit.world.snapshot.experimental.fs.FileSystemSnapshotDatabaseTest.WORLD_ALPHA;
import static com.sk89q.worldedit.world.snapshot.experimental.fs.FileSystemSnapshotDatabaseTest.WORLD_BETA;
import static java.util.stream.Collectors.toList;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.DynamicContainer.dynamicContainer;
import static org.junit.jupiter.api.DynamicTest.dynamicTest;
enum FSSDTestType {
EMPTY {
@Override
List<DynamicTest> getTests(FSSDContext context) {
return ImmutableList.of(
dynamicTest("return an empty stream from getSnapshots(worldName)",
() -> context.db.getSnapshots(WORLD_ALPHA)),
dynamicTest("return an empty optional from getSnapshot(name)",
() -> context.db.getSnapshot(context.nameUri(WORLD_ALPHA)))
);
}
},
WORLD_ONLY_DIR {
@Override
List<DynamicTest> getTests(FSSDContext context) throws IOException {
Path worldFolder = EntryMaker.WORLD_DIR.createEntry(context.db.getRoot(), WORLD_ALPHA);
Files.setLastModifiedTime(worldFolder, FileTime.from(TIME_ONE.toInstant()));
return singleSnapTest(context, WORLD_ALPHA, TIME_ONE);
}
},
WORLD_ONLY_DIM_DIR {
@Override
List<DynamicTest> getTests(FSSDContext context) throws IOException {
int dim = ThreadLocalRandom.current().nextInt();
Path worldFolder = EntryMaker.WORLD_DIM_DIR
.createEntry(context.db.getRoot(), new EntryMaker.DimInfo(WORLD_ALPHA, dim));
Files.setLastModifiedTime(worldFolder, FileTime.from(TIME_ONE.toInstant()));
return singleSnapTest(context, WORLD_ALPHA, TIME_ONE);
}
},
WORLD_ONLY_NO_REGION_DIR {
@Override
List<DynamicTest> getTests(FSSDContext context) throws IOException {
Path worldFolder = EntryMaker.WORLD_NO_REGION_DIR
.createEntry(context.db.getRoot(), WORLD_ALPHA);
Files.setLastModifiedTime(worldFolder, FileTime.from(TIME_ONE.toInstant()));
return singleSnapTest(context, WORLD_ALPHA, TIME_ONE);
}
},
WORLD_LEGACY_DIR {
@Override
List<DynamicTest> getTests(FSSDContext context) throws IOException {
Path worldFolder = EntryMaker.WORLD_LEGACY_DIR
.createEntry(context.db.getRoot(), WORLD_ALPHA);
Files.setLastModifiedTime(worldFolder, FileTime.from(TIME_ONE.toInstant()));
return singleSnapTest(context, WORLD_ALPHA, TIME_ONE);
}
},
WORLD_ONLY_ARCHIVE {
@Override
List<DynamicTest> getTests(FSSDContext context) throws IOException {
Path worldArchive = EntryMaker.WORLD_ARCHIVE
.createEntry(context.db.getRoot(), WORLD_ALPHA);
try (ArchiveDir rootOfArchive = context.getRootOfArchive(worldArchive)) {
Files.setLastModifiedTime(
rootOfArchive.getPath(),
FileTime.from(TIME_ONE.toInstant())
);
}
return singleSnapTest(context, WORLD_ALPHA + ".zip", TIME_ONE);
}
},
TIMESTAMPED_DIR {
@Override
List<? extends DynamicNode> getTests(FSSDContext context) throws IOException {
Path root = context.db.getRoot();
Path timestampedDir = EntryMaker.TIMESTAMPED_DIR
.createEntry(root, TIME_ONE);
EntryMaker.WORLD_DIR.createEntry(timestampedDir, WORLD_ALPHA);
EntryMaker.WORLD_ARCHIVE.createEntry(timestampedDir, WORLD_BETA);
return ImmutableList.of(
dynamicContainer("world dir",
singleSnapTest(context,
root.relativize(timestampedDir) + File.separator + WORLD_ALPHA,
TIME_ONE)
),
dynamicContainer("world archive",
singleSnapTest(context,
root.relativize(timestampedDir) + File.separator + WORLD_BETA + ".zip",
TIME_ONE)
)
);
}
},
TIMESTAMPED_ARCHIVE {
@Override
List<? extends DynamicNode> getTests(FSSDContext context) throws IOException {
Path root = context.db.getRoot();
Path timestampedArchive = EntryMaker.TIMESTAMPED_ARCHIVE
.createEntry(root, TIME_ONE);
try (ArchiveDir timestampedDir = context.getRootOfArchive(timestampedArchive)) {
EntryMaker.WORLD_DIR.createEntry(timestampedDir.getPath(), WORLD_ALPHA);
EntryMaker.WORLD_ARCHIVE.createEntry(timestampedDir.getPath(), WORLD_BETA);
}
return ImmutableList.of(
dynamicContainer("world dir",
singleSnapTest(context,
root.relativize(timestampedArchive) + File.separator + WORLD_ALPHA,
TIME_ONE)
)
);
}
},
WORLD_DIR_TIMESTAMPED_DIR {
@Override
List<? extends DynamicNode> getTests(FSSDContext context) throws IOException {
Path root = context.db.getRoot();
Path timestampedDirA = EntryMaker.TIMESTAMPED_DIR
.createEntry(root.resolve(WORLD_ALPHA), TIME_ONE);
Path timestampedDirB = EntryMaker.TIMESTAMPED_DIR
.createEntry(root.resolve(WORLD_BETA), TIME_ONE);
EntryMaker.WORLD_DIR.createEntry(timestampedDirA, WORLD_ALPHA);
EntryMaker.WORLD_ARCHIVE.createEntry(timestampedDirB, WORLD_BETA);
return ImmutableList.of(
dynamicContainer("world dir",
singleSnapTest(context,
root.relativize(timestampedDirA) + File.separator + WORLD_ALPHA,
TIME_ONE)
),
dynamicContainer("world archive",
singleSnapTest(context,
root.relativize(timestampedDirB) + File.separator + WORLD_BETA + ".zip",
TIME_ONE)
)
);
}
},
TWO_TIMESTAMPED_WORLD_DIR {
@Override
List<DynamicTest> getTests(FSSDContext context) throws IOException {
Path root = context.db.getRoot();
Path timestampedDirA = EntryMaker.TIMESTAMPED_DIR
.createEntry(root, TIME_ONE);
EntryMaker.WORLD_DIR.createEntry(timestampedDirA, WORLD_ALPHA);
Path timestampedDirB = EntryMaker.TIMESTAMPED_DIR
.createEntry(root, TIME_TWO);
EntryMaker.WORLD_DIR.createEntry(timestampedDirB, WORLD_ALPHA);
return ImmutableList.of(
dynamicTest("list both snapshots in order (newest first)", () -> {
List<Snapshot> snapshots = context.db
.getSnapshotsNewestFirst(WORLD_ALPHA).collect(toList());
assertEquals(2, snapshots.size());
assertValidSnapshot(TIME_ONE, snapshots.get(0));
assertValidSnapshot(TIME_TWO, snapshots.get(1));
}),
dynamicTest("list both snapshots in order (oldest first)", () -> {
List<Snapshot> snapshots = context.db
.getSnapshotsOldestFirst(WORLD_ALPHA).collect(toList());
assertEquals(2, snapshots.size());
assertValidSnapshot(TIME_TWO, snapshots.get(0));
assertValidSnapshot(TIME_ONE, snapshots.get(1));
}),
dynamicTest("list only 1 if getting AFTER 2", () -> {
List<Snapshot> snapshots = context.db
.getSnapshotsAfter(WORLD_ALPHA, TIME_TWO).collect(toList());
assertEquals(1, snapshots.size());
assertValidSnapshot(TIME_ONE, snapshots.get(0));
}),
dynamicTest("list only 2 if getting BEFORE 1", () -> {
List<Snapshot> snapshots = context.db
.getSnapshotsBefore(WORLD_ALPHA, TIME_ONE).collect(toList());
assertEquals(1, snapshots.size());
assertValidSnapshot(TIME_TWO, snapshots.get(0));
}),
dynamicTest("list both if AFTER time before 2", () -> {
List<Snapshot> snapshots = context.db
.getSnapshotsAfter(WORLD_ALPHA, TIME_TWO.minusSeconds(1))
.collect(toList());
assertEquals(2, snapshots.size());
// sorted newest first
assertValidSnapshot(TIME_ONE, snapshots.get(0));
assertValidSnapshot(TIME_TWO, snapshots.get(1));
}),
dynamicTest("list both if BEFORE time after 1", () -> {
List<Snapshot> snapshots = context.db
.getSnapshotsBefore(WORLD_ALPHA, TIME_ONE.plusSeconds(1))
.collect(toList());
assertEquals(2, snapshots.size());
// sorted oldest first
assertValidSnapshot(TIME_TWO, snapshots.get(0));
assertValidSnapshot(TIME_ONE, snapshots.get(1));
}
)
);
}
},
TWO_WORLD_ONLY_DIR {
@Override
List<? extends DynamicNode> getTests(FSSDContext context) throws IOException {
Path worldFolderA = EntryMaker.WORLD_DIR
.createEntry(context.db.getRoot(), WORLD_ALPHA);
Files.setLastModifiedTime(worldFolderA, FileTime.from(TIME_ONE.toInstant()));
Path worldFolderB = EntryMaker.WORLD_DIR
.createEntry(context.db.getRoot(), WORLD_BETA);
Files.setLastModifiedTime(worldFolderB, FileTime.from(TIME_TWO.toInstant()));
return Stream.of(
singleSnapTest(context, WORLD_ALPHA, TIME_ONE),
singleSnapTest(context, WORLD_BETA, TIME_TWO)
).flatMap(List::stream).collect(toList());
}
};
List<DynamicTest> singleSnapTest(FSSDContext context, String name,
ZonedDateTime time) {
return ImmutableList.of(
dynamicTest("return a valid snapshot for " + name, () -> {
try (Snapshot snapshot = context.requireSnapshot(name)) {
assertValidSnapshot(time, snapshot);
}
}),
dynamicTest("list a valid snapshot for " + name, () -> {
try (Snapshot snapshot = context.requireListsSnapshot(name)) {
assertValidSnapshot(time, snapshot);
}
})
);
}
private static void assertValidSnapshot(ZonedDateTime time,
Snapshot snapshot) throws IOException, DataException {
assertEquals(time, snapshot.getInfo().getDateTime());
// MCA file
assertEquals(CHUNK_TAG.toString(), snapshot.getChunkTag(CHUNK_POS).toString());
// MCR file
BlockVector3 offsetChunkPos = CHUNK_POS.add(32, 0, 32);
assertEquals(CHUNK_TAG.toString(), snapshot.getChunkTag(offsetChunkPos).toString());
}
abstract List<? extends DynamicNode> getTests(FSSDContext context) throws IOException;
Stream<DynamicNode> getNamedTests(FSSDContext context) throws IOException {
return Stream.of(dynamicContainer(
name(),
URI.create("method:" + getClass().getName() +
"#getTests(" + FSSDContext.class.getName() + ")"),
getTests(context).stream()
));
}
}

View File

@ -0,0 +1,195 @@
/*
* WorldEdit, a Minecraft world manipulation toolkit
* Copyright (C) sk89q <http://www.sk89q.com>
* Copyright (C) WorldEdit team and contributors
*
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU Lesser General Public License as published by the
* Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License
* for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package com.sk89q.worldedit.world.snapshot.experimental.fs;
import com.google.common.io.ByteStreams;
import com.google.common.io.Resources;
import com.sk89q.jnbt.CompoundTag;
import com.sk89q.worldedit.math.BlockVector2;
import com.sk89q.worldedit.math.BlockVector3;
import com.sk89q.worldedit.util.io.file.ArchiveNioSupport;
import com.sk89q.worldedit.util.io.file.ArchiveNioSupports;
import com.sk89q.worldedit.util.io.file.TrueVfsArchiveNioSupport;
import com.sk89q.worldedit.util.io.file.ZipArchiveNioSupport;
import com.sk89q.worldedit.world.DataException;
import com.sk89q.worldedit.world.storage.ChunkStoreHelper;
import com.sk89q.worldedit.world.storage.McRegionReader;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.DynamicNode;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestFactory;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.UncheckedIOException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.time.Instant;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.function.Function;
import java.util.stream.IntStream;
import java.util.stream.Stream;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.DynamicContainer.dynamicContainer;
@DisplayName("A FS Snapshot Database")
class FileSystemSnapshotDatabaseTest {
static byte[] REGION_DATA;
static byte[] CHUNK_DATA;
static CompoundTag CHUNK_TAG;
static BlockVector3 CHUNK_POS;
static final String WORLD_ALPHA = "World Alpha";
static final String WORLD_BETA = "World Beta";
static final DateTimeFormatter FORMATTER =
DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH_mm_ss");
static final ZonedDateTime TIME_ONE = Instant.parse("2018-01-01T12:00:00.00Z")
.atZone(ZoneId.systemDefault());
static final ZonedDateTime TIME_TWO = TIME_ONE.minusDays(1);
private static Path TEMP_DIR;
@BeforeAll
static void setUpStatic() throws IOException, DataException {
try (InputStream in = Resources.getResource("world_region.mca.gzip").openStream();
GZIPInputStream gzIn = new GZIPInputStream(in)) {
REGION_DATA = ByteStreams.toByteArray(gzIn);
}
McRegionReader reader = new McRegionReader(new ByteArrayInputStream(REGION_DATA));
try {
// Find the single chunk
BlockVector2 chunkPos = IntStream.range(0, 32).mapToObj(
x -> IntStream.range(0, 32).filter(z -> reader.hasChunk(x, z))
.mapToObj(z -> BlockVector2.at(x, z))
).flatMap(Function.identity())
.findAny()
.orElseThrow(() -> new AssertionError("No chunk in region file."));
ByteArrayOutputStream cap = new ByteArrayOutputStream();
try (InputStream in = reader.getChunkInputStream(chunkPos);
GZIPOutputStream gzOut = new GZIPOutputStream(cap)) {
ByteStreams.copy(in, gzOut);
}
CHUNK_DATA = cap.toByteArray();
CHUNK_TAG = ChunkStoreHelper.readCompoundTag(() -> new GZIPInputStream(
new ByteArrayInputStream(CHUNK_DATA)
));
CHUNK_POS = chunkPos.toBlockVector3();
} finally {
reader.close();
}
TEMP_DIR = Files.createTempDirectory("worldedit-fs-snap-dbs");
}
@AfterAll
static void afterAll() throws IOException {
deleteTree(TEMP_DIR);
}
private static Path newTempDb() throws IOException {
return Files.createTempDirectory(TEMP_DIR, "db");
}
private static void deleteTree(Path root) throws IOException {
Files.walkFileTree(root, new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
Files.deleteIfExists(file);
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
Files.deleteIfExists(dir);
return FileVisitResult.CONTINUE;
}
});
}
@DisplayName("makes the root directory absolute if needed")
@Test
void rootIsAbsoluteDir() throws IOException {
Path root = newTempDb();
try {
Path relative = root.getFileSystem().getPath("relative");
Files.createDirectories(relative);
FileSystemSnapshotDatabase db2 = new FileSystemSnapshotDatabase(relative,
ArchiveNioSupports.combined());
assertEquals(root.getFileSystem().getPath(".").toRealPath()
.resolve(relative), db2.getRoot());
Path absolute = root.resolve("absolute");
Files.createDirectories(absolute);
FileSystemSnapshotDatabase db3 = new FileSystemSnapshotDatabase(absolute,
ArchiveNioSupports.combined());
assertEquals(absolute, db3.getRoot());
} finally {
deleteTree(root);
}
}
@DisplayName("with a specific NIO support:")
@TestFactory
Stream<DynamicNode> withSpecificNioSupport() {
return Stream.of(
ZipArchiveNioSupport.getInstance(), TrueVfsArchiveNioSupport.getInstance()
)
.map(nioSupport -> {
Stream<? extends DynamicNode> nodes = Stream.of(FSSDTestType.values())
.flatMap(type -> {
try {
return getTests(nioSupport, type);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
});
return dynamicContainer(
nioSupport.getClass().getSimpleName() + ", can, for format:",
nodes
);
});
}
private static Stream<? extends DynamicNode> getTests(ArchiveNioSupport nioSupport,
FSSDTestType type) throws IOException {
Path root = newTempDb();
try {
Path dbRoot = root.resolve("snapshots");
Files.createDirectories(dbRoot);
return type.getNamedTests(new FSSDContext(nioSupport, dbRoot));
} catch (Throwable t) {
deleteTree(root);
throw t;
}
}
}