redesigned the EPK file format

epkcompiler/src/CompilePackage.java

@@ -1,61 +1,161 @@
 import java.io.ByteArrayOutputStream;
-import java.io.DataOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.OutputStream;
 import java.nio.charset.Charset;
-import java.security.MessageDigest;
+import java.nio.charset.StandardCharsets;
-import java.security.NoSuchAlgorithmException;
+import java.text.SimpleDateFormat;
 import java.util.ArrayList;
+import java.util.Date;
 
+import com.jcraft.jzlib.CRC32;
 import com.jcraft.jzlib.Deflater;
 import com.jcraft.jzlib.DeflaterOutputStream;
+import com.jcraft.jzlib.GZIPOutputStream;
 
 public class CompilePackage {
 	
 	private static ArrayList<File> files = new ArrayList();
 	
-	public static void main(String[] args) throws IOException, NoSuchAlgorithmException {
+	public static void main(String[] args) throws IOException {
-		if(args.length != 2) {
+		if(args.length < 2 || args.length > 4) {
-			System.out.print("Usage: java -jar CompilePackage.jar <input directory> <output file>");
+			System.out.println("Usage: java -jar CompilePackage.jar <input directory> <output file> [gzip|zlib|none] [file-type]");
 			return;
 		}
+		
 		File root = new File(args[0]);
+		File output = new File(args[1]);
+		char compressionType;
+		
+		if(args.length > 2) {
+			if(args[2].equalsIgnoreCase("gzip")) {
+				compressionType = 'G';
+			}else if(args[2].equalsIgnoreCase("zlib")) {
+				compressionType = 'Z';
+			}else if(args[2].equalsIgnoreCase("none")) {
+				compressionType = '0';
+			}else {
+				throw new IllegalArgumentException("Unknown compression method: " + args[2]);
+			}
+		}else {
+			compressionType = 'G';
+		}
+		
 		listDirectory(root);
 		ByteArrayOutputStream osb = new ByteArrayOutputStream();
-		DataOutputStream os = new DataOutputStream(osb);
 		String start = root.getAbsolutePath();
-		os.write("EAGPKG!!".getBytes(Charset.forName("UTF-8")));
-		os.writeUTF("\n\n #  eaglercraft package file - assets copyright mojang ab\n #  eagler eagler eagler eagler eagler eagler eagler\n\n");
-		Deflater d = new Deflater(9);
-		os = new DataOutputStream(new DeflaterOutputStream(osb, d));
-		MessageDigest md = MessageDigest.getInstance("SHA-1"); 
-		for(File f : files) {
-			os.writeUTF("<file>");
-			String p = f.getAbsolutePath().replace(start, "").replace('\\', '/');
-			if(p.startsWith("/")) p = p.substring(1);
-			os.writeUTF(p);
 		
+		osb.write("EAGPKG$$".getBytes(Charset.forName("UTF-8")));
+		
+		String chars = "ver2.0";
+		osb.write(chars.length());
+		osb.write(chars.getBytes(StandardCharsets.US_ASCII));
+		
+		Date d = new Date();
+		
+		String comment = "\n\n #  Eagler EPK v2.0 (c) " + (new SimpleDateFormat("yyyy")).format(d) + " Calder Young\n" +
+				" #  update: on " + (new SimpleDateFormat("MM/dd/yyyy")).format(d) + " at " +
+				(new SimpleDateFormat("hh:mm:ss aa")).format(d) + "\n\n";
+
+		String nm = output.getName();
+		osb.write(nm.length());
+		osb.write(nm.getBytes(StandardCharsets.US_ASCII));
+		
+		writeShort(comment.length(), osb);
+		osb.write(comment.getBytes(StandardCharsets.US_ASCII));
+		
+		writeLong(d.getTime(), osb);
+		writeInt(files.size() + 1, osb);
+		
+		osb.write(compressionType);
+		
+		OutputStream os;
+		
+		if(compressionType == 'G') {
+			os = new GZIPOutputStream(osb, new Deflater(9, 15+16), 16384, true);
+		}else if(compressionType == 'Z') {
+			os = new DeflaterOutputStream(osb, new Deflater(9), 16384, true);
+		}else {
+			os = osb;
+		}
+		
+		os.write("HEAD".getBytes(StandardCharsets.US_ASCII));
+		String key = "file-type";
+		os.write(key.length());
+		os.write(key.getBytes(StandardCharsets.US_ASCII));
+		String value;
+		if(args.length > 3) {
+			value = args[3];
+		}else {
+			value = "epk/resources";
+		}
+		writeInt(value.length(), os);
+		os.write(value.getBytes(StandardCharsets.US_ASCII));
+		os.write('>');
+		
+		CRC32 checkSum = new CRC32();
+		for(File f : files) {
 			InputStream stream = new FileInputStream(f);
-			byte[] targetArray = new byte[stream.available()];
+			byte[] targetArray = new byte[(int)f.length()];
 			stream.read(targetArray);
 			stream.close();
 			
-			os.write(md.digest(targetArray));
+			checkSum.reset();
-			os.writeInt(targetArray.length);
+			checkSum.update(targetArray, 0, targetArray.length);
+			int ch = (int)checkSum.getValue();
+			
+			os.write("FILE".getBytes(StandardCharsets.US_ASCII));
+			
+			String p = f.getAbsolutePath().replace(start, "").replace('\\', '/');
+			if(p.startsWith("/")) {
+				p = p.substring(1);
+			}
+			os.write(p.length());
+			os.write(p.getBytes(StandardCharsets.US_ASCII));
+			writeInt(targetArray.length + 5, os);
+			writeInt(ch, os);
+			
 			os.write(targetArray);
-			os.writeUTF("</file>");
+			os.write(':');
+			os.write('>');
 		}
-		os.writeUTF(" end");
+		
-		os.flush();
+		os.write("END$".getBytes(StandardCharsets.US_ASCII));
 		os.close();
-		FileOutputStream out = new FileOutputStream(new File(args[1]));
+		
+		osb.write(":::YEE:>".getBytes(StandardCharsets.US_ASCII));
+		
+		FileOutputStream out = new FileOutputStream(output);
 		out.write(osb.toByteArray());
 		out.close();
 	}
 	
+	public static void writeShort(int i, OutputStream os) throws IOException {
+		os.write((i >> 8) & 0xFF);
+		os.write(i & 0xFF);
+	}
+	
+	public static void writeInt(int i, OutputStream os) throws IOException {
+		os.write((i >> 24) & 0xFF);
+		os.write((i >> 16) & 0xFF);
+		os.write((i >> 8) & 0xFF);
+		os.write(i & 0xFF);
+	}
+	
+	public static void writeLong(long i, OutputStream os) throws IOException {
+		os.write((int)((i >> 56) & 0xFF));
+		os.write((int)((i >> 48) & 0xFF));
+		os.write((int)((i >> 40) & 0xFF));
+		os.write((int)((i >> 32) & 0xFF));
+		os.write((int)((i >> 24) & 0xFF));
+		os.write((int)((i >> 16) & 0xFF));
+		os.write((int)((i >> 8) & 0xFF));
+		os.write((int)(i & 0xFF));
+	}
+	
 	public static void listDirectory(File dir) {
 		for(File f : dir.listFiles()) {
 			if(f.isDirectory()) {

src/main/java/net/lax1dude/eaglercraft/AssetRepository.java

@@ -3,10 +3,12 @@ package net.lax1dude.eaglercraft;
 import java.io.ByteArrayInputStream;
 import java.io.DataInputStream;
 import java.io.IOException;
-import java.nio.charset.Charset;
+import java.io.InputStream;
 import java.util.Arrays;
 import java.util.HashMap;
 
+import com.jcraft.jzlib.CRC32;
+import com.jcraft.jzlib.GZIPInputStream;
 import com.jcraft.jzlib.InflaterInputStream;
 
 public class AssetRepository {
@@ -14,13 +16,163 @@ public class AssetRepository {
 	private static final HashMap<String,byte[]> filePool = new HashMap();
 	
 	public static final void install(byte[] pkg) throws IOException {
-		ByteArrayInputStream in2 = new ByteArrayInputStream(pkg);
+		ByteArrayInputStream in = new ByteArrayInputStream(pkg);
-		DataInputStream in = new DataInputStream(in2);
+		
 		byte[] header = new byte[8];
 		in.read(header);
-		if(!"EAGPKG!!".equals(new String(header, Charset.forName("UTF-8")))) throw new IOException("invalid epk file");
+		String type = readASCII(header);
+		
+		if("EAGPKG$$".equals(type)) {
+			int l = pkg.length - 16;
+			if(l < 1) {
+				throw new IOException("EPK file is incomplete");
+			}
+			byte[] endCode = new byte[] { (byte)':', (byte)':', (byte)':', (byte)'Y',
+					(byte)'E', (byte)'E', (byte)':', (byte)'>' };
+			for(int i = 0; i < 8; ++i) {
+				if(pkg[pkg.length - 8 + i] != endCode[i]) {
+					throw new IOException("EPK file is missing EOF code (:::YEE:>)");
+				}
+			}
+			loadNew(new ByteArrayInputStream(pkg, 8, pkg.length - 16));
+		}else if("EAGPKG!!".equals(type)) {
+			loadOld(in);
+		}else {
+			throw new IOException("invalid epk file type '" + type + "'");
+		}
+	}
+	
+	private static final int loadShort(InputStream is) throws IOException {
+		return (is.read() << 8) | is.read();
+	}
+	
+	private static final int loadInt(InputStream is) throws IOException {
+		return (is.read() << 24) | (is.read() << 16) | (is.read() << 8) | is.read();
+	}
+	
+	private static final String readASCII(byte[] bytesIn) throws IOException {
+		char[] charIn = new char[bytesIn.length];
+		for(int i = 0; i < bytesIn.length; ++i) {
+			charIn[i] = (char)((int)bytesIn[i] & 0xFF);
+		}
+		return new String(charIn);
+	}
+	
+	private static final String readASCII(InputStream bytesIn) throws IOException {
+		int len = bytesIn.read();
+		char[] charIn = new char[len];
+		for(int i = 0; i < len; ++i) {
+			charIn[i] = (char)(bytesIn.read() & 0xFF);
+		}
+		return new String(charIn);
+	}
+	
+	public static final void loadNew(InputStream is) throws IOException {
+		
+		String vers = readASCII(is);
+		if(!vers.startsWith("ver2.")) {
+			throw new IOException("Unknown or invalid EPK version: " + vers);
+		}
+		
+		is.skip(is.read()); // skip filename
+		is.skip(loadShort(is)); // skip comment
+		is.skip(8); // skip millis date
+		
+		int numFiles = loadInt(is);
+		
+		char compressionType = (char)is.read();
+		
+		InputStream zis;
+		switch(compressionType) {
+		case 'G':
+			zis = new GZIPInputStream(is);
+			break;
+		case 'Z':
+			zis = new InflaterInputStream(is);
+			break;
+		case '0':
+			zis = is;
+			break;
+		default:
+			throw new IOException("Invalid or unsupported EPK compression: " + compressionType);
+		}
+
+		int blockFile = ('F' << 24) | ('I' << 16) | ('L' << 8) | 'E';
+		int blockEnd = ('E' << 24) | ('N' << 16) | ('D' << 8) | '$';
+		int blockHead = ('H' << 24) | ('E' << 16) | ('A' << 8) | 'D';
+		
+		CRC32 crc32 = new CRC32();
+		int blockType;
+		for(int i = 0; i < numFiles; ++i) {
+			
+			blockType = loadInt(zis);
+			
+			if(blockType == blockEnd) {
+				throw new IOException("Unexpected EOF when there are still " + (numFiles - i) + " files remaining");
+			}
+			
+			String name = readASCII(zis);
+			int len = loadInt(zis);
+			
+			if(i == 0) {
+				if(blockType == blockHead) {
+					byte[] readType = new byte[len];
+					zis.read(readType);
+					if(!"file-type".equals(name) || !"epk/resources".equals(readASCII(readType))) {
+						throw new IOException("EPK is not of file-type 'epk/resources'!");
+					}
+					if(zis.read() != '>') {
+						throw new IOException("Object '" + name + "' is incomplete");
+					}
+					continue;
+				}else {
+					throw new IOException("File '" + name + "' did not have a file-type block as the first entry in the file");
+				}
+			}
+			
+			if(blockType == blockFile) {
+				if(len < 5) {
+					throw new IOException("File '" + name + "' is incomplete");
+				}
+				
+				int expectedCRC = loadInt(zis);
+				
+				byte[] load = new byte[len - 5];
+				zis.read(load);
+
+				if(len > 5) {
+					crc32.reset();
+					crc32.update(load, 0, load.length);
+					if(expectedCRC != (int)crc32.getValue()) {
+						throw new IOException("File '" + name + "' has an invalid checksum");
+					}
+				}
+				
+				if(zis.read() != ':') {
+					throw new IOException("File '" + name + "' is incomplete");
+				}
+				
+				filePool.put(name, load);
+			}else {
+				zis.skip(len);
+			}
+
+			if(zis.read() != '>') {
+				throw new IOException("Object '" + name + "' is incomplete");
+			}
+		}
+		
+		if(loadInt(zis) != blockEnd) {
+			throw new IOException("EPK missing END$ object");
+		}
+		
+		zis.close();
+	}
+	
+	public static final void loadOld(InputStream is) throws IOException {
+		DataInputStream in = new DataInputStream(is);
 		in.readUTF();
-		in = new DataInputStream(new InflaterInputStream(in2));
+		in = new DataInputStream(new InflaterInputStream(is));
 		String s = null;
 		SHA1Digest dg = new SHA1Digest();
 		while("<file>".equals(s = in.readUTF())) {