made world export/import use the EPK v2 format (faster)

sp-server/src/main/java/net/lax1dude/eaglercraft/sp/EPK2Compiler.java

@@ -0,0 +1,151 @@
+package net.lax1dude.eaglercraft.sp;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+import com.jcraft.jzlib.CRC32;
+
+public class EPK2Compiler {
+	
+	private final ByteArrayOutputStream os;
+	private final CRC32 checkSum = new CRC32();
+	private int lengthIntegerOffset = 0;
+	private int totalFileCount = 0;
+
+	public EPK2Compiler(String name, String owner, String type) {
+		os = new ByteArrayOutputStream(0x200000);
+		try {
+			
+			os.write(new byte[]{(byte)69,(byte)65,(byte)71,(byte)80,(byte)75,(byte)71,(byte)36,(byte)36}); // EAGPKG$$
+			os.write(new byte[]{(byte)6,(byte)118,(byte)101,(byte)114,(byte)50,(byte)46,(byte)48}); // 6 + ver2.0
+			Date d = new Date();
+			
+			byte[] filename = (name + ".epk").getBytes(StandardCharsets.UTF_8);
+			os.write(filename.length);
+			os.write(filename);
+			
+			byte[] comment = ("\n\n #  Eagler EPK v2.0 (c) " + (new SimpleDateFormat("yyyy")).format(d) + " " +
+					owner + "\n #  export: on " + (new SimpleDateFormat("MM/dd/yyyy")).format(d) + " at " +
+					(new SimpleDateFormat("hh:mm:ss aa")).format(d) + "\n\n #  world name: " + name + "\n\n")
+					.getBytes(StandardCharsets.UTF_8);
+
+			os.write((comment.length >> 8) & 255);
+			os.write(comment.length & 255);
+			os.write(comment);
+			
+			writeLong(d.getTime(), os);
+			
+			lengthIntegerOffset = os.size();
+			os.write(new byte[]{(byte)255,(byte)255,(byte)255,(byte)255}); // this will be replaced with the file count
+			
+			os.write('0'); // compression type: none
+			
+			os.write(new byte[]{(byte)72,(byte)69,(byte)65,(byte)68}); // HEAD
+			os.write(new byte[]{(byte)9,(byte)102,(byte)105,(byte)108,(byte)101,(byte)45,(byte)116,(byte)121,
+					(byte)112,(byte)101}); // 9 + file-type
+			
+			byte[] typeBytes = type.getBytes(StandardCharsets.UTF_8);
+			writeInt(typeBytes.length, os);
+			os.write(typeBytes); // write type
+			os.write('>');
+			
+			++totalFileCount;
+			
+			os.write(new byte[]{(byte)72,(byte)69,(byte)65,(byte)68}); // HEAD
+			os.write(new byte[]{(byte)10,(byte)119,(byte)111,(byte)114,(byte)108,(byte)100,(byte)45,(byte)110,
+					(byte)97,(byte)109,(byte)101}); // 10 + world-name
+			
+			byte[] nameBytes = name.getBytes(StandardCharsets.UTF_8);
+			writeInt(nameBytes.length, os);
+			os.write(nameBytes); // write name
+			os.write('>');
+			
+			++totalFileCount;
+			
+			os.write(new byte[]{(byte)72,(byte)69,(byte)65,(byte)68}); // HEAD
+			os.write(new byte[]{(byte)11,(byte)119,(byte)111,(byte)114,(byte)108,(byte)100,(byte)45,(byte)111,
+					(byte)119,(byte)110,(byte)101,(byte)114}); // 11 + world-owner
+			
+			byte[] ownerBytes = owner.getBytes(StandardCharsets.UTF_8);
+			writeInt(ownerBytes.length, os);
+			os.write(ownerBytes); // write owner
+			os.write('>');
+			
+			++totalFileCount;
+			
+		}catch(IOException ex) {
+			throw new RuntimeException("This happened somehow", ex);
+		}
+	}
+	
+	public void append(String name, byte[] dat) {
+		try {
+			
+			checkSum.reset();
+			checkSum.update(dat, 0, dat.length);
+			long sum = checkSum.getValue();
+			
+			os.write(new byte[]{(byte)70,(byte)73,(byte)76,(byte)69}); // FILE
+			
+			byte[] nameBytes = name.getBytes(StandardCharsets.UTF_8);
+			os.write(nameBytes.length);
+			os.write(nameBytes);
+			
+			writeInt(dat.length + 5, os);
+			writeInt((int)sum, os);
+			
+			os.write(dat);
+			
+			os.write(':');
+			os.write('>');
+			
+			++totalFileCount;
+			
+		}catch(IOException ex) {
+			throw new RuntimeException("This happened somehow", ex);
+		}
+	}
+	
+	public byte[] complete() {
+		try {
+			
+			os.write(new byte[]{(byte)69,(byte)78,(byte)68,(byte)36}); // END$
+			os.write(new byte[]{(byte)58,(byte)58,(byte)58,(byte)89,(byte)69,(byte)69,(byte)58,(byte)62}); // :::YEE:>
+			
+			byte[] ret = os.toByteArray();
+
+			ret[lengthIntegerOffset] = (byte)((totalFileCount >> 24) & 0xFF);
+			ret[lengthIntegerOffset + 1] = (byte)((totalFileCount >> 16) & 0xFF);
+			ret[lengthIntegerOffset + 2] = (byte)((totalFileCount >> 8) & 0xFF);
+			ret[lengthIntegerOffset + 3] = (byte)(totalFileCount & 0xFF);
+			
+			return ret;
+			
+		}catch(IOException ex) {
+			throw new RuntimeException("This happened somehow", ex);
+		}
+	}
+	
+	public static void writeInt(int i, OutputStream os) throws IOException {
+		os.write((i >> 24) & 0xFF);
+		os.write((i >> 16) & 0xFF);
+		os.write((i >> 8) & 0xFF);
+		os.write(i & 0xFF);
+	}
+	
+	public static void writeLong(long i, OutputStream os) throws IOException {
+		os.write((int)((i >> 56) & 0xFF));
+		os.write((int)((i >> 48) & 0xFF));
+		os.write((int)((i >> 40) & 0xFF));
+		os.write((int)((i >> 32) & 0xFF));
+		os.write((int)((i >> 24) & 0xFF));
+		os.write((int)((i >> 16) & 0xFF));
+		os.write((int)((i >> 8) & 0xFF));
+		os.write((int)(i & 0xFF));
+	}
+	
+}

sp-server/src/main/java/net/lax1dude/eaglercraft/sp/EPKCompiler.java

@@ -1,57 +0,0 @@
-package net.lax1dude.eaglercraft.sp;
-
-import java.io.ByteArrayOutputStream;
-import java.io.DataOutputStream;
-import java.nio.charset.Charset;
-
-import com.jcraft.jzlib.Deflater;
-import com.jcraft.jzlib.DeflaterOutputStream;
-
-public class EPKCompiler {
-	
-	private final ByteArrayOutputStream osb = new ByteArrayOutputStream();
-	private DataOutputStream os;
-	private Deflater d;
-	private final SHA1Digest dig = new SHA1Digest();
-	
-	public EPKCompiler(String name) {
-		try {
-			os = new DataOutputStream(osb);
-			os.write("EAGPKG!!".getBytes(Charset.forName("UTF-8")));
-			os.writeUTF("\n\n #  eaglercraft package file - " + name + "\n #  eagler eagler eagler eagler eagler eagler eagler\n\n");
-			d = new Deflater(9);
-			os = new DataOutputStream(new DeflaterOutputStream(osb, d));
-		}catch(Throwable t) {
-			throw new RuntimeException("this happened somehow", t);
-		}
-	}
-	
-	public void append(String name, byte[] dat) {
-		try {
-			os.writeUTF("<file>");
-			os.writeUTF(name);
-			byte[] v = dat;
-			dig.update(v, 0, v.length);
-			byte[] final_ = new byte[20];
-			dig.doFinal(final_, 0);
-			os.write(final_);
-			os.writeInt(v.length);
-			os.write(v);
-			os.writeUTF("</file>");
-		}catch(Throwable t) {
-			throw new RuntimeException("this happened somehow", t);
-		}
-	}
-	
-	public byte[] complete() {
-		try {
-			os.writeUTF(" end");
-			os.flush();
-			os.close();
-			return osb.toByteArray();
-		}catch(Throwable t) {
-			throw new RuntimeException("this happened somehow", t);
-		}
-	}
-	
-}

sp-server/src/main/java/net/lax1dude/eaglercraft/sp/EPKDecompiler.java

@@ -3,38 +3,194 @@ package net.lax1dude.eaglercraft.sp;
 import java.io.ByteArrayInputStream;
 import java.io.DataInputStream;
 import java.io.IOException;
-import java.nio.charset.Charset;
+import java.io.InputStream;
 import java.util.Arrays;
 
+import com.jcraft.jzlib.CRC32;
+import com.jcraft.jzlib.GZIPInputStream;
 import com.jcraft.jzlib.InflaterInputStream;
 
 public class EPKDecompiler {
 	
 	public static class FileEntry {
+		public final String type;
 		public final String name;
 		public final byte[] data;
-		protected FileEntry(String name, byte[] data) {
+		protected FileEntry(String type, String name, byte[] data) {
+			this.type = type;
 			this.name = name;
 			this.data = data;
 		}
 	}
 	
-	private final ByteArrayInputStream in2;
+	private ByteArrayInputStream in2;
 	private DataInputStream in;
-	private SHA1Digest dg = new SHA1Digest();
+	private InputStream zis;
+	private SHA1Digest dg;
+	private CRC32 crc32;
+	private int numFiles;
 	private boolean isFinished = false;
+	private boolean isOldFormat = false;
 	
 	public EPKDecompiler(byte[] data) throws IOException {
 		in2 = new ByteArrayInputStream(data);
-		in = new DataInputStream(in2);
+		
 		byte[] header = new byte[8];
-		in.read(header);
+		in2.read(header);
-		if(!"EAGPKG!!".equals(new String(header, Charset.forName("UTF-8")))) throw new IOException("invalid epk file");
+		
+		if(Arrays.equals(header, new byte[]{(byte)69,(byte)65,(byte)71,(byte)80,(byte)75,(byte)71,(byte)36,(byte)36})) {
+			byte[] endCode = new byte[] { (byte)':', (byte)':', (byte)':', (byte)'Y',
+					(byte)'E', (byte)'E', (byte)':', (byte)'>' };
+			for(int i = 0; i < 8; ++i) {
+				if(data[data.length - 8 + i] != endCode[i]) {
+					throw new IOException("EPK file is missing EOF code (:::YEE:>)");
+				}
+			}
+			in2 = new ByteArrayInputStream(data, 8, data.length - 16);
+			initNew();
+		}else if(Arrays.equals(header, new byte[]{(byte)69,(byte)65,(byte)71,(byte)80,(byte)75,(byte)71,(byte)33,(byte)33})) {
+			initOld();
+		}
+		
+	}
+	
+	public boolean isOld() {
+		return isOldFormat;
+	}
+	
+	public FileEntry readFile() throws IOException {
+		if(!isOldFormat) {
+			return readFileNew();
+		}else {
+			return readFileOld();
+		}
+	}
+	
+	private void initNew() throws IOException {
+		InputStream is = in2;
+		
+		String vers = readASCII(is);
+		if(!vers.startsWith("ver2.")) {
+			throw new IOException("Unknown or invalid EPK version: " + vers);
+		}
+		
+		is.skip(is.read()); // skip filename
+		is.skip(loadShort(is)); // skip comment
+		is.skip(8); // skip millis date
+		
+		numFiles = loadInt(is);
+		
+		char compressionType = (char)is.read();
+		
+		switch(compressionType) {
+		case 'G':
+			zis = new GZIPInputStream(is);
+			break;
+		case 'Z':
+			zis = new InflaterInputStream(is);
+			break;
+		case '0':
+			zis = is;
+			break;
+		default:
+			throw new IOException("Invalid or unsupported EPK compression: " + compressionType);
+		}
+		
+		crc32 = new CRC32();
+		
+	}
+	
+	private FileEntry readFileNew() throws IOException {
+		if(isFinished) {
+			return null;
+		}
+		
+		byte[] typeBytes = new byte[4];
+		zis.read(typeBytes);
+		String type = readASCII(typeBytes);
+		
+		if(numFiles == 0) {
+			if(!"END$".equals(type)) {
+				throw new IOException("EPK file is missing END code (END$)");
+			}
+			isFinished = true;
+			return null;
+		}else {
+			if("END$".equals(type)) {
+				throw new IOException("Unexpected END when there are still " + numFiles + " files remaining");
+			}else {
+				String name = readASCII(zis);
+				int len = loadInt(zis);
+				byte[] data;
+				
+				if("FILE".equals(type)) {
+					if(len < 5) {
+						throw new IOException("File '" + name + "' is incomplete (no crc)");
+					}
+					
+					int loadedCrc = loadInt(zis);
+					
+					data = new byte[len - 5];
+					zis.read(data);
+					
+					crc32.reset();
+					crc32.update(data, 0, data.length);
+					if((int)crc32.getValue() != loadedCrc) {
+						throw new IOException("File '" + name + "' has an invalid checksum");
+					}
+					
+					if(zis.read() != ':') {
+						throw new IOException("File '" + name + "' is incomplete");
+					}
+				}else {
+					data = new byte[len];
+					zis.read(data);
+				}
+				
+				if(zis.read() != '>') {
+					throw new IOException("Object '" + name + "' is incomplete");
+				}
+				
+				--numFiles;
+				return new FileEntry(type, name, data);
+			}
+		}
+	}
+	
+	private static final int loadShort(InputStream is) throws IOException {
+		return (is.read() << 8) | is.read();
+	}
+	
+	private static final int loadInt(InputStream is) throws IOException {
+		return (is.read() << 24) | (is.read() << 16) | (is.read() << 8) | is.read();
+	}
+	
+	public static final String readASCII(byte[] bytesIn) throws IOException {
+		char[] charIn = new char[bytesIn.length];
+		for(int i = 0; i < bytesIn.length; ++i) {
+			charIn[i] = (char)((int)bytesIn[i] & 0xFF);
+		}
+		return new String(charIn);
+	}
+	
+	private static final String readASCII(InputStream bytesIn) throws IOException {
+		int len = bytesIn.read();
+		char[] charIn = new char[len];
+		for(int i = 0; i < len; ++i) {
+			charIn[i] = (char)(bytesIn.read() & 0xFF);
+		}
+		return new String(charIn);
+	}
+	
+	private void initOld() throws IOException {
+		isOldFormat = true;
+		dg = new SHA1Digest();
+		in = new DataInputStream(in2);
 		in.readUTF();
 		in = new DataInputStream(new InflaterInputStream(in2));
 	}
 	
-	public FileEntry readFile() throws IOException {
+	private FileEntry readFileOld() throws IOException {
 		if(isFinished) {
 			return null;
 		}
@@ -55,7 +211,7 @@ public class EPKDecompiler {
 		dg.update(file, 0, len); dg.doFinal(digest2, 0);
 		if(!Arrays.equals(digest, digest2)) throw new IOException("invalid file hash for "+path);
 		if(!"</file>".equals(in.readUTF())) throw new IOException("invalid epk file");
-		return new FileEntry(path, file);
+		return new FileEntry("FILE", path, file);
 	}
 	
 }

sp-server/src/main/java/net/lax1dude/eaglercraft/sp/IntegratedServer.java

@@ -266,12 +266,20 @@ public class IntegratedServer {
 						break;
 					case IPCPacket05RequestData.ID: {
 							IPCPacket05RequestData pkt = (IPCPacket05RequestData)packet;
-							if(pkt.request == IPCPacket05RequestData.REQUEST_LEVEL_EAG) {
+							if(pkt.request == IPCPacket05RequestData.REQUEST_LEVEL_EAG) {String realWorldName = pkt.worldName;
+								String worldOwner = "UNKNOWN";
+								String splitter = new String(new char[] { (char)253, (char)233, (char)233 });
+								if(realWorldName.contains(splitter)) {
+									int i = realWorldName.lastIndexOf(splitter);
+									worldOwner = realWorldName.substring(i + 3);
+									realWorldName = realWorldName.substring(0, i);
+								}
 								try {
+									
 									final int[] bytesWritten = new int[1];
 									final int[] lastUpdate = new int[1];
-									String pfx = "worlds/" + pkt.worldName + "/";
+									String pfx = "worlds/" + realWorldName + "/";
-									EPKCompiler c = new EPKCompiler("contains backup of world '" + pkt.worldName + "'");
+									EPK2Compiler c = new EPK2Compiler(realWorldName, worldOwner, "epk/world152");
 									SYS.VFS.iterateFiles(pfx, false, (i) -> {
 										byte[] b = i.getAllBytes();
 										c.append(i.path.substring(pfx.length()), b);
@@ -283,7 +291,7 @@ public class IntegratedServer {
 									});
 									sendIPCPacket(new IPCPacket09RequestResponse(c.complete()));
 								} catch (Throwable t) {
-									throwExceptionToClient("Failed to export world '" + pkt.worldName + "' as EPK", t);
+									throwExceptionToClient("Failed to export world '" + realWorldName + "' as EPK", t);
 									sendTaskFailed();
 								}
 							}else if(pkt.request == IPCPacket05RequestData.REQUEST_LEVEL_MCA) {
@@ -401,27 +409,38 @@ public class IntegratedServer {
 							IPCPacket07ImportWorld pkt = (IPCPacket07ImportWorld)packet;
 							if(isServerStopped()) {
 								if(pkt.worldFormat == IPCPacket07ImportWorld.WORLD_FORMAT_EAG) {
+									String folder = VFSSaveHandler.worldNameToFolderName(pkt.worldName);
 									try {
-										String folder = VFSSaveHandler.worldNameToFolderName(pkt.worldName);
 										VFile dir = new VFile("worlds", folder);
 										EPKDecompiler dc = new EPKDecompiler(pkt.worldData);
 										EPKDecompiler.FileEntry f = null;
 										int lastProgUpdate = 0;
 										int prog = 0;
+										boolean hasReadType = dc.isOld();
 										while((f = dc.readFile()) != null) {
 											byte[] b = f.data;
-											if(f.name.equals("level.dat")) {
+											if(!hasReadType) {
-												NBTTagCompound worldDatNBT = CompressedStreamTools.decompress(b);
+												if(f.type.equals("HEAD") && f.name.equals("file-type") && EPKDecompiler.readASCII(f.data).equals("epk/world152")) {
-												worldDatNBT.getCompoundTag("Data").setString("LevelName", pkt.worldName);
+													hasReadType = true;
-												worldDatNBT.getCompoundTag("Data").setLong("LastPlayed", System.currentTimeMillis());
+													continue;
-												b = CompressedStreamTools.compress(worldDatNBT);
+												}else {
+													throw new IOException("file does not contain a singleplayer 1.5.2 world!");
+												}
 											}
-											VFile ff = new VFile(dir, f.name);
+											if(f.type.equals("FILE")) {
-											ff.setAllBytes(b);
+												if(f.name.equals("level.dat")) {
-											prog += b.length;
+													NBTTagCompound worldDatNBT = CompressedStreamTools.decompress(b);
-											if(prog - lastProgUpdate > 10000) {
+													worldDatNBT.getCompoundTag("Data").setString("LevelName", pkt.worldName);
-												lastProgUpdate = prog;
+													worldDatNBT.getCompoundTag("Data").setLong("LastPlayed", System.currentTimeMillis());
-												updateStatusString("selectWorld.progress.importing." + pkt.worldFormat, prog);
+													b = CompressedStreamTools.compress(worldDatNBT);
+												}
+												VFile ff = new VFile(dir, f.name);
+												ff.setAllBytes(b);
+												prog += b.length;
+												if(prog - lastProgUpdate > 10000) {
+													lastProgUpdate = prog;
+													updateStatusString("selectWorld.progress.importing." + pkt.worldFormat, prog);
+												}
 											}
 										}
 										String[] worldsTxt = SYS.VFS.getFile("worlds.txt").getAllLines();
@@ -436,12 +455,13 @@ public class IntegratedServer {
 										SYS.VFS.getFile("worlds.txt").setAllChars(String.join("\n", worldsTxt));
 										sendIPCPacket(new IPCPacketFFProcessKeepAlive(IPCPacket07ImportWorld.ID));
 									}catch(Throwable t) {
+										SYS.VFS.deleteFiles("worlds/" + folder + "/");
 										throwExceptionToClient("Failed to import world '" + pkt.worldName + "' as EPK", t);
 										sendTaskFailed();
 									}
 								}else if(pkt.worldFormat == IPCPacket07ImportWorld.WORLD_FORMAT_MCA) {
+									String folder = VFSSaveHandler.worldNameToFolderName(pkt.worldName);
 									try {
-										String folder = VFSSaveHandler.worldNameToFolderName(pkt.worldName);
 										VFile dir = new VFile("worlds", folder);
 										ZipInputStream folderNames = new ZipInputStream(new ByteArrayInputStream(pkt.worldData));
 										ZipEntry folderNameFile = null;
@@ -504,6 +524,7 @@ public class IntegratedServer {
 										SYS.VFS.getFile("worlds.txt").setAllChars(String.join("\n", worldsTxt));
 										sendIPCPacket(new IPCPacketFFProcessKeepAlive(IPCPacket07ImportWorld.ID));
 									}catch(Throwable t) {
+										SYS.VFS.deleteFiles("worlds/" + folder + "/");
 										throwExceptionToClient("Failed to import world '" + pkt.worldName + "' as MCA", t);
 										sendTaskFailed();
 									}

src/main/java/net/lax1dude/eaglercraft/AssetRepository.java

@@ -108,7 +108,7 @@ public class AssetRepository {
 			blockType = loadInt(zis);
 			
 			if(blockType == blockEnd) {
-				throw new IOException("Unexpected EOF when there are still " + (numFiles - i) + " files remaining");
+				throw new IOException("Unexpected END when there are still " + (numFiles - i) + " files remaining");
 			}
 			
 			String name = readASCII(zis);

src/main/java/net/lax1dude/eaglercraft/IntegratedServer.java

@@ -231,6 +231,9 @@ public class IntegratedServer {
 	public static void exportWorld(String name, int format) {
 		ensureReady();
 		statusState = IntegratedState.WORLD_EXPORTING;
+		if(format == IPCPacket05RequestData.REQUEST_LEVEL_EAG) {
+			name = name + (new String(new char[] { (char)253, (char)233, (char)233 })) + EaglerProfile.username;
+		}
 		sendIPCPacket(new IPCPacket05RequestData(name, (byte)format));
 	}
 	

src/main/java/net/minecraft/client/Minecraft.java

@@ -1478,7 +1478,7 @@ public class Minecraft implements Runnable {
 				}
 				
 				if(messageOnLoginCounter == 150 && isSingleplayerOrLAN()) {
-					displayEaglercraftText(EnumChatFormatting.AQUA + "Especially in new worlds, if no chunks show give the game up to 120 seconds before \"giving up\" on a new world");
+					displayEaglercraftText(EnumChatFormatting.AQUA + "Especially in new worlds, if no chunks show give the game up to 5 straight minutes before \"giving up\" on a new world");
 				}
 			}