diff --git a/pom.xml b/pom.xml index 85ee2ae471..ccb775cbae 100644 --- a/pom.xml +++ b/pom.xml @@ -24,6 +24,11 @@ commons-compress 1.10 + + edu.ucla.sspace + sspace + 2.0.4 + org.slf4j diff --git a/src/main/java/net/runelite/deob/Deob.java b/src/main/java/net/runelite/deob/Deob.java index 2672b7d1d2..8f1745b9c6 100644 --- a/src/main/java/net/runelite/deob/Deob.java +++ b/src/main/java/net/runelite/deob/Deob.java @@ -16,7 +16,7 @@ import java.util.jar.Manifest; import net.runelite.deob.deobfuscators.ConstantParameter; import net.runelite.deob.deobfuscators.IllegalStateExceptions; import net.runelite.deob.deobfuscators.MethodInliner; -import net.runelite.deob.deobfuscators.Rename; +import net.runelite.deob.deobfuscators.rename.Rename; import net.runelite.deob.deobfuscators.RenameUnique; import net.runelite.deob.deobfuscators.RuntimeExceptions; import net.runelite.deob.deobfuscators.UnreachedCode; @@ -31,12 +31,13 @@ import net.runelite.deob.deobfuscators.arithmetic.MultiplyZeroDeobfuscator; import net.runelite.deob.execution.Execution; // XXX something to detect final fields and evaluate them +// XXX ORDER IN WHICH FIELDS ARE ACCESSED public class Deob { public static void main(String[] args) throws IOException { - //merge(); if(true) return; + merge(); if(true) return; long start = System.currentTimeMillis(); diff --git a/src/main/java/net/runelite/deob/Interfaces.java b/src/main/java/net/runelite/deob/Interfaces.java index f8a877721e..42503916dd 100644 --- a/src/main/java/net/runelite/deob/Interfaces.java +++ b/src/main/java/net/runelite/deob/Interfaces.java @@ -7,6 +7,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; public class Interfaces { @@ -51,6 +52,11 @@ public class Interfaces return l; } + public List getNonMyInterfaces() + { + return interfaces.stream().filter(clazz -> classFile.getGroup().findClass(clazz.getName()) == null).collect(Collectors.toList()); + } + public void write(DataOutputStream out) throws IOException { out.writeShort(interfaces.size()); diff --git a/src/main/java/net/runelite/deob/deobfuscators/Rename.java b/src/main/java/net/runelite/deob/deobfuscators/Rename.java deleted file mode 100644 index dd35ca5486..0000000000 --- a/src/main/java/net/runelite/deob/deobfuscators/Rename.java +++ /dev/null @@ -1,74 +0,0 @@ -package net.runelite.deob.deobfuscators; - -import java.util.List; -import java.util.stream.Collectors; -import net.runelite.deob.ClassGroup; -import net.runelite.deob.attributes.code.Instruction; -import net.runelite.deob.attributes.code.instruction.types.ComparisonInstruction; -import net.runelite.deob.attributes.code.instruction.types.InvokeInstruction; -import net.runelite.deob.attributes.code.instruction.types.LVTInstruction; -import net.runelite.deob.attributes.code.instruction.types.SetFieldInstruction; -import net.runelite.deob.execution.Execution; -import net.runelite.deob.execution.Frame; - -public class Rename -{ - private static boolean isExpressionInstruction(Instruction in) - { - return - in instanceof SetFieldInstruction || - (in instanceof LVTInstruction && ((LVTInstruction) in).store()) || - in instanceof InvokeInstruction || - in instanceof ComparisonInstruction; - } - - private static List getExprIns(Frame frame) - { - return frame.getInstructions().stream().map(i -> i.getInstruction()).filter(i -> isExpressionInstruction(i)).collect(Collectors.toList()); - } - - private static boolean equalsFrames(Frame one, Frame two) - { - List oneIns = getExprIns(one), - twoIns = getExprIns(two); - - if (oneIns.size() != twoIns.size() || oneIns.isEmpty()) - return false; - - for (int i = 0; i < oneIns.size(); ++i) - { - Instruction i1 = oneIns.get(i), - i2 = twoIns.get(i); - - if (i1.getType() != i2.getType()) - return false; - } - - if (one.getMethod().getName().startsWith("method") && two.getMethod().getName().startsWith("method")) - { - int i =5; - } - return true; - } - - public void run(ClassGroup one, ClassGroup two) - { - Execution ex1 = new Execution(one); - ex1.populateInitialMethods(); - ex1.run(); - - Execution ex2 = new Execution(two); - ex2.populateInitialMethods(); - ex2.run(); - - for (Frame f : ex1.processedFrames) - for (Frame f2 : ex2.processedFrames) - { - if (f.getMethod().getName().equals("vmethod2976") && f2.getMethod().getName().equals("vmethod2973")) - { - if (equalsFrames(f, f2)) - System.out.println(f.getMethod().getName() + " " + f2.getMethod().getName()); - } - } - } -} diff --git a/src/main/java/net/runelite/deob/deobfuscators/rename/Rename.java b/src/main/java/net/runelite/deob/deobfuscators/rename/Rename.java new file mode 100644 index 0000000000..4fd1534c54 --- /dev/null +++ b/src/main/java/net/runelite/deob/deobfuscators/rename/Rename.java @@ -0,0 +1,253 @@ +package net.runelite.deob.deobfuscators.rename; + +import edu.ucla.sspace.graph.Graph; +import edu.ucla.sspace.graph.isomorphism.IsomorphismTester; +import edu.ucla.sspace.graph.isomorphism.TypedVF2IsomorphismTester; +import java.nio.ByteBuffer; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.stream.Collectors; +import net.runelite.deob.ClassFile; +import net.runelite.deob.ClassGroup; +import net.runelite.deob.Field; +import net.runelite.deob.Method; +import net.runelite.deob.attributes.code.Instruction; +import net.runelite.deob.attributes.code.instruction.types.InvokeInstruction; +import net.runelite.deob.execution.Execution; +import net.runelite.deob.execution.Frame; +import net.runelite.deob.signature.Signature; +import net.runelite.deob.signature.Type; + +public class Rename +{ + public static int hash(Object... objects) + { + MessageDigest md; + try + { + md = MessageDigest.getInstance("MD5"); + } + catch (NoSuchAlgorithmException ex) + { + throw new RuntimeException(ex); + } + for (Object o : objects) + md.update(o.toString().getBytes()); + byte[] b = md.digest(); + + ByteBuffer buf = ByteBuffer.wrap(b); + return buf.getInt() ^ buf.getInt() ^ buf.getInt() ^ buf.getInt(); + } + + public static int hashType(Type type) + { + int dimms = type.getArrayDims(); + String t = type.getType(); + if (t.startsWith("L")) + t = "class"; + return hash(dimms, t); + } + + public static int fingerprintField(Field f) + { + return hashType(f.getType()); + } + + private int fingerprintMethod(Method m) + { + int type = hashType(m.getNameAndType().getDescriptorType()); + + Signature sig = m.getDescriptor(); + for (int i = 0; i < sig.size(); ++i) + { + Type t = sig.getTypeOfArg(i); + type ^= hashType(t); + } + + return type; + } + + private int fingerprintClass(ClassFile cf) + { + int hash = 0; + + ClassFile f = cf.getParent(); + if (f != null) + hash ^= fingerprintClass(f); + + for (ClassFile i : cf.getInterfaces().getMyInterfaces()) + hash ^= fingerprintClass(i); + + int count = 0; + for (Field fi : cf.getFields().getFields()) + { + if (fi.isStatic()) + continue; + + hash ^= fingerprintField(fi) << (count++ % 16); + } + + return hash; + } +// private static boolean isExpressionInstruction(Instruction in) +// { +// return +// in instanceof SetFieldInstruction || +// (in instanceof LVTInstruction && ((LVTInstruction) in).store()) || +// in instanceof InvokeInstruction || +// in instanceof ComparisonInstruction; +// } +// +// private static List getExprIns(Frame frame) +// { +// return frame.getInstructions().stream().map(i -> i.getInstruction()).filter(i -> isExpressionInstruction(i)).collect(Collectors.toList()); +// } +// +// private static boolean equalsFrames(Frame one, Frame two) +// { +// List oneIns = getExprIns(one), +// twoIns = getExprIns(two); +// +// if (oneIns.size() != twoIns.size() || oneIns.isEmpty()) +// return false; +// +// for (int i = 0; i < oneIns.size(); ++i) +// { +// Instruction i1 = oneIns.get(i), +// i2 = twoIns.get(i); +// +// if (i1.getType() != i2.getType()) +// return false; +// } +// +// if (one.getMethod().getName().startsWith("method") && two.getMethod().getName().startsWith("method")) +// { +// int i =5; +// } +// return true; +// } + + private void printInfo(ClassGroup group) + { + int fields = 0, methods = 0; + for (ClassFile cf : group.getClasses()) + { + methods += cf.getMethods().getMethods().stream().filter(m -> !m.isStatic()).count(); + fields += cf.getFields().getFields().stream().filter(f -> !f.isStatic()).count(); + } + System.out.println("Classes: " + group.getClasses().size() + " fields " + fields + " methods " + methods); + } + + public void run(ClassGroup one, ClassGroup two) + { + Execution eone = new Execution(one); + eone.populateInitialMethods(); + eone.run(); + + Execution etwo = new Execution(two); + etwo.populateInitialMethods(); + etwo.run(); + + List f1 = eone.processedFrames.stream().filter(f -> f.getMethod() == one.findClass("client").findMethod("init")).collect(Collectors.toList()); + List f2 = etwo.processedFrames.stream().filter(f -> f.getMethod() == two.findClass("client").findMethod("init")).collect(Collectors.toList()); + + Frame ff1 = f1.get(0), ff2 = f2.get(0); + + Graph g1 = ff1.getGraph(), g2 = ff2.getGraph(); + + IsomorphismTester isoTest = new TypedVF2IsomorphismTester(); + System.out.println(isoTest.areIsomorphic(g1, g2)); + + Map mapping = isoTest.findIsomorphism(g1, g2); + Map map1 = ff1.getIdMap(), map2 = ff2.getIdMap(); + + for (Entry e : mapping.entrySet()) + { + if (e.getKey() == null || e.getValue() == null) + { + assert e.getKey() == e.getValue(); + continue; + } + + Instruction i1 = map1.get(e.getKey()); + Instruction i2 = map2.get(e.getValue()); + + assert i1.getClass() == i2.getClass(); + + InvokeInstruction ii1 = (InvokeInstruction) i1, ii2 = (InvokeInstruction) i2; + System.out.println("MATCH " + ii1.getMethods().get(0).getName() + " -> " + ii2.getMethods().get(0).getName()); + } + + System.out.print(eone); +// for (ClassFile cf : one.getClasses()) +// { +// if (!cf.getFields().getFields().stream().anyMatch(f -> !f.isStatic())) +// continue; +// +// ClassInfo info = new ClassInfo(cf); +// info.build(); +// +// for (ClassFile cf2 : two.getClasses()) +// { +// if (!cf2.getFields().getFields().stream().anyMatch(f -> !f.isStatic())) +// { +// continue; +// } +// +// ClassInfo info2 = new ClassInfo(cf2); +// info2.build(); +// +// if (info.equals(info2)) +// { +// System.out.println("cl match " + cf.getName() + " -> " + cf2.getName()); +// } +// } +// } +// GraphInfo graphOne = new GraphInfo(one); +// graphOne.populate(); +// +// GraphInfo graphTwo = new GraphInfo(two); +// graphTwo.populate(); +// +// //Map mapping = new HashMap<>(); +// +//// for (ClassFile cf : one.getClasses()) +//// { +//// ClassFile cf2 = two.findClass(cf.getName()); +//// assert cf2 != null; +//// mapping.put(graphOne.getIds().toId(cf), graphTwo.getIds().toId(cf2)); +//// } +//// +// //graphOne.getGraph(). +// System.out.println(graphOne.getGraph().size()); +// System.out.println(graphTwo.getGraph().size()); +// +// printInfo(one); +// printInfo(two); +// +// //graphOne.getGraph(). +// +// IsomorphismTester isoTest = new TypedVF2IsomorphismTester(); +// System.out.println(isoTest.areIsomorphic(graphOne.getGraph(), graphTwo.getGraph())); +// Execution ex1 = new Execution(one); +// ex1.populateInitialMethods(); +// ex1.run(); +// +// Execution ex2 = new Execution(two); +// ex2.populateInitialMethods(); +// ex2.run(); +// +// for (Frame f : ex1.processedFrames) +// for (Frame f2 : ex2.processedFrames) +// { +// if (f.getMethod().getName().equals("vmethod2976") && f2.getMethod().getName().equals("method3115")) +// { +// if (equalsFrames(f, f2)) +// System.out.println(f.getMethod().getName() + " " + f2.getMethod().getName()); +// } +// } + } +} diff --git a/src/main/java/net/runelite/deob/execution/Frame.java b/src/main/java/net/runelite/deob/execution/Frame.java index 22089a9e4d..b15194a01a 100644 --- a/src/main/java/net/runelite/deob/execution/Frame.java +++ b/src/main/java/net/runelite/deob/execution/Frame.java @@ -1,6 +1,10 @@ package net.runelite.deob.execution; import com.google.common.collect.Lists; +import edu.ucla.sspace.graph.DirectedEdge; +import edu.ucla.sspace.graph.Graph; +import edu.ucla.sspace.graph.SimpleDirectedEdge; +import edu.ucla.sspace.graph.SparseDirectedGraph; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -11,13 +15,9 @@ import net.runelite.deob.attributes.Code; import net.runelite.deob.attributes.code.Exception; import net.runelite.deob.attributes.code.Instruction; import net.runelite.deob.attributes.code.Instructions; -import net.runelite.deob.attributes.code.instructions.LookupSwitch; -import net.runelite.deob.attributes.code.instructions.TableSwitch; import net.runelite.deob.pool.NameAndType; -import java.util.HashSet; -import java.util.Set; import net.runelite.deob.attributes.code.instruction.types.InvokeInstruction; -import org.apache.commons.collections4.MultiMap; +import net.runelite.deob.util.IdGen; import org.apache.commons.collections4.map.MultiValueMap; public class Frame @@ -31,6 +31,11 @@ public class Frame private List instructions = new ArrayList<>(); // instructions executed in this frame private MultiValueMap visited = new MultiValueMap<>(); // shared + private IdGen ids = new IdGen(); + private Map idMap = new HashMap<>(); + private Graph graph = new SparseDirectedGraph(); // shared. + private int prevVertex = -1; + public static long num; public Frame(Execution execution, Method method) @@ -100,6 +105,11 @@ public class Frame this.stack = new Stack(other.stack); this.variables = new Variables(other.variables); this.visited = other.visited; + + this.ids = other.ids; + this.idMap = other.idMap; + this.graph = other.graph; + this.prevVertex = other.prevVertex; } public Frame dup() @@ -190,6 +200,8 @@ public class Frame if (!executing) break; + buildGraph(oldCur); + if (oldCur == cur) { int idx = instructions.indexOf(cur); @@ -255,4 +267,55 @@ public class Frame cur = to; } + + public Graph getGraph() + { + return graph; + } + + public Map getIdMap() + { + return idMap; + } + + private void buildGraph(Instruction i) + { + if (i instanceof InvokeInstruction) + { + InvokeInstruction ii = (InvokeInstruction) i; + + List methods = ii.getMethods(); + if (methods.isEmpty()) + return; + } +// else if (i instanceof FieldInstruction) +// { +// FieldInstruction fi = (FieldInstruction) i; +// +// if (fi.getMyField() == null) +// return; +// } + else + { + return; + } + + if (prevVertex == -1) + { + int id = ids.get(); + graph.add(id); + prevVertex = id; + this.idMap.put(id, i); + return; + } + + int id = ids.get(); + graph.add(id); + idMap.put(id, i); + + DirectedEdge edge = new SimpleDirectedEdge(prevVertex, id); + graph.add(edge); + + prevVertex = id; + } } diff --git a/src/main/java/net/runelite/deob/signature/Type.java b/src/main/java/net/runelite/deob/signature/Type.java index 216bf16964..26e0974ffc 100644 --- a/src/main/java/net/runelite/deob/signature/Type.java +++ b/src/main/java/net/runelite/deob/signature/Type.java @@ -1,5 +1,7 @@ package net.runelite.deob.signature; +import java.util.Objects; + public class Type { private String type; @@ -57,6 +59,17 @@ public class Type return 1; } + public boolean isPrimitive() + { + assert type.startsWith("L") == type.endsWith(";"); + return !type.startsWith("L"); + } + + public boolean isObfuscatedType() + { + return type.startsWith("Lclass"); + } + @Override public boolean equals(Object other) { @@ -66,6 +79,15 @@ public class Type Type a = (Type) other; return type.equals(a.type) && arrayDimms == a.arrayDimms; } + + @Override + public int hashCode() + { + int hash = 7; + hash = 23 * hash + Objects.hashCode(this.type); + hash = 23 * hash + this.arrayDimms; + return hash; + } @Override public String toString()