From f540d76b479947695827bf49a6038bd9afc648ac Mon Sep 17 00:00:00 2001 From: Adam Date: Tue, 24 Nov 2015 22:20:12 -0600 Subject: [PATCH] Idr this. Some fields have different accessors (at least public) between versions and is messing with this. --- .../java/net/runelite/deob/ClassFile.java | 7 + src/main/java/net/runelite/deob/Deob.java | 44 ++-- src/main/java/net/runelite/deob/Field.java | 9 +- src/main/java/net/runelite/deob/Method.java | 6 + .../deob/attributes/code/Instruction.java | 8 + .../deobfuscators/rename/FieldWrapper.java | 60 +++++ .../deobfuscators/rename/InstructionList.java | 139 +++++++++++ .../deob/deobfuscators/rename/Rename2.java | 224 +++++++++++++----- .../deob/deobfuscators/rename/graph/Edge.java | 10 +- .../deobfuscators/rename/graph/Graph.java | 51 +++- .../deobfuscators/rename/graph/Vertex.java | 144 ++++++++--- .../rename/graph/VertexType.java | 3 +- .../runelite/deob/execution/Execution.java | 30 ++- .../net/runelite/deob/signature/Type.java | 5 +- .../runelite/deob/execution/FrameTest.java | 133 ----------- .../runelite/deob/execution/GraphTest.java | 138 +++++++++++ .../deob/execution/one/TestClass.java | 30 +++ .../deob/execution/two/TestClass.java | 30 +++ 18 files changed, 804 insertions(+), 267 deletions(-) create mode 100644 src/main/java/net/runelite/deob/deobfuscators/rename/FieldWrapper.java create mode 100644 src/main/java/net/runelite/deob/deobfuscators/rename/InstructionList.java delete mode 100644 src/test/java/net/runelite/deob/execution/FrameTest.java create mode 100644 src/test/java/net/runelite/deob/execution/GraphTest.java create mode 100644 src/test/java/net/runelite/deob/execution/one/TestClass.java create mode 100644 src/test/java/net/runelite/deob/execution/two/TestClass.java diff --git a/src/main/java/net/runelite/deob/ClassFile.java b/src/main/java/net/runelite/deob/ClassFile.java index efb09ee0b9..5f6c8be6b2 100644 --- a/src/main/java/net/runelite/deob/ClassFile.java +++ b/src/main/java/net/runelite/deob/ClassFile.java @@ -137,6 +137,13 @@ public class ClassFile this.name = new Class(name); } + public String getClassName() + { + String n = getName(); + int i = n.lastIndexOf('/'); + return n.substring(i + 1); + } + public String getSuperName() { return super_class.getName(); diff --git a/src/main/java/net/runelite/deob/Deob.java b/src/main/java/net/runelite/deob/Deob.java index 7c5cd3ac88..5a967aec2c 100644 --- a/src/main/java/net/runelite/deob/Deob.java +++ b/src/main/java/net/runelite/deob/Deob.java @@ -25,7 +25,7 @@ public class Deob { public static void main(String[] args) throws IOException { - //merge(); if(true) return; + merge(); if(true) return; long start = System.currentTimeMillis(); @@ -73,25 +73,25 @@ public class Deob // // run(group, new UnusedClass()); - ModArith mod = new ModArith(); - mod.run(group); - - int last = -1, cur; - while ((cur = mod.runOnce()) > 0) - { - new MultiplicationDeobfuscator().run(group); - - new MultiplyOneDeobfuscator().run(group); - - new MultiplyZeroDeobfuscator().run(group); - - if (last == cur) - break; - - last = cur; - } - - mod.annotateEncryption(); +// ModArith mod = new ModArith(); +// mod.run(group); +// +// int last = -1, cur; +// while ((cur = mod.runOnce()) > 0) +// { +// new MultiplicationDeobfuscator().run(group); +// +// new MultiplyOneDeobfuscator().run(group); +// +// new MultiplyZeroDeobfuscator().run(group); +// +// if (last == cur) +// break; +// +// last = cur; +// } +// +// mod.annotateEncryption(); // eval constant fields (only set once to a constant in ctor) maybe just inline them @@ -105,8 +105,8 @@ public class Deob private static void merge() throws IOException { - ClassGroup group1 = JarUtil.loadJar(new File("d:/rs/07/adamin1.jar")), - group2 = JarUtil.loadJar(new File("d:/rs/07/adamin2.jar")); + ClassGroup group1 = JarUtil.loadJar(new File("/Users/adam/w/rs/07/adamin1.jar")), + group2 = JarUtil.loadJar(new File("/Users/adam/w/rs/07/adamin2.jar")); Rename2 rename = new Rename2(); rename.run(group1, group2); diff --git a/src/main/java/net/runelite/deob/Field.java b/src/main/java/net/runelite/deob/Field.java index 05bb4aeff5..80b535119a 100644 --- a/src/main/java/net/runelite/deob/Field.java +++ b/src/main/java/net/runelite/deob/Field.java @@ -1,14 +1,11 @@ package net.runelite.deob; import net.runelite.deob.attributes.Attributes; -import net.runelite.deob.attributes.code.Instruction; import net.runelite.deob.signature.Type; import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; -import java.util.ArrayList; -import java.util.Objects; import net.runelite.deob.pool.NameAndType; public class Field @@ -124,4 +121,10 @@ public class Field { return name.hashCode(); } + + @Override + public String toString() + { + return this.type + " " + this.getFields().getClassFile().getName() + "." + this.getName(); + } } \ No newline at end of file diff --git a/src/main/java/net/runelite/deob/Method.java b/src/main/java/net/runelite/deob/Method.java index adfe22ba28..1243bf3f90 100644 --- a/src/main/java/net/runelite/deob/Method.java +++ b/src/main/java/net/runelite/deob/Method.java @@ -53,6 +53,12 @@ public class Method attributes = new Attributes(this); } + @Override + public String toString() + { + return this.getMethods().getClassFile().getName() + "." + this.name + this.arguments; + } + public void write(DataOutputStream out) throws IOException { assert methods.getMethods().contains(this); diff --git a/src/main/java/net/runelite/deob/attributes/code/Instruction.java b/src/main/java/net/runelite/deob/attributes/code/Instruction.java index 9eb98fc056..b027e9b21a 100644 --- a/src/main/java/net/runelite/deob/attributes/code/Instruction.java +++ b/src/main/java/net/runelite/deob/attributes/code/Instruction.java @@ -8,6 +8,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import net.runelite.deob.Method; public abstract class Instruction implements Cloneable { @@ -27,6 +28,13 @@ public abstract class Instruction implements Cloneable this.pc = pc; } + @Override + public String toString() + { + Method m = this.getInstructions().getCode().getAttributes().getMethod(); + return super.toString() + " in " + m; + } + @Override public Instruction clone() { diff --git a/src/main/java/net/runelite/deob/deobfuscators/rename/FieldWrapper.java b/src/main/java/net/runelite/deob/deobfuscators/rename/FieldWrapper.java new file mode 100644 index 0000000000..546d66e643 --- /dev/null +++ b/src/main/java/net/runelite/deob/deobfuscators/rename/FieldWrapper.java @@ -0,0 +1,60 @@ +package net.runelite.deob.deobfuscators.rename; + +import java.util.Objects; +import net.runelite.deob.Field; +import net.runelite.deob.attributes.code.instruction.types.FieldInstruction; +import net.runelite.deob.signature.Type; + +public class FieldWrapper +{ + private FieldInstruction fi; + public Field field; + private Type type; + private short accessFlags; + + public FieldWrapper(FieldInstruction fi ,Field field) + { + this.fi = fi; + this.field = field; + this.type = field.getType(); + this.accessFlags = field.getAccessFlags(); + } + + @Override + public String toString() + { + return field.toString() + " access from instruction " + fi; + } + + @Override + public int hashCode() + { + int hash = 3; + hash = 29 * hash + Objects.hashCode(this.type); + hash = 29 * hash + this.accessFlags; + return hash; + } + + @Override + public boolean equals(Object obj) + { + if (obj == null) + { + return false; + } + if (getClass() != obj.getClass()) + { + return false; + } + final FieldWrapper other = (FieldWrapper) obj; + if (!Objects.equals(this.type, other.type)) + { + return false; + } + if (this.accessFlags != other.accessFlags) + { + return false; + } + return true; + } +} diff --git a/src/main/java/net/runelite/deob/deobfuscators/rename/InstructionList.java b/src/main/java/net/runelite/deob/deobfuscators/rename/InstructionList.java new file mode 100644 index 0000000000..e8ae859f7e --- /dev/null +++ b/src/main/java/net/runelite/deob/deobfuscators/rename/InstructionList.java @@ -0,0 +1,139 @@ +package net.runelite.deob.deobfuscators.rename; + +import com.google.common.collect.HashMultiset; +import com.google.common.collect.Multiset; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import net.runelite.deob.Field; +import net.runelite.deob.Method; +import net.runelite.deob.attributes.code.Instruction; +import net.runelite.deob.attributes.code.instruction.types.GetFieldInstruction; +import net.runelite.deob.attributes.code.instruction.types.PushConstantInstruction; +import net.runelite.deob.attributes.code.instruction.types.SetFieldInstruction; +import net.runelite.deob.attributes.code.instructions.InvokeVirtual; +import net.runelite.deob.deobfuscators.arithmetic.DMath; +import net.runelite.deob.pool.PoolEntry; +import net.runelite.deob.signature.Signature; +import net.runelite.deob.signature.Type; + +public class InstructionList +{ + private final List instructions; + + public InstructionList(List instructions) + { + this.instructions = instructions; + } + + public boolean couldBeEqual(InstructionList other) + { + Multiset sig1 = HashMultiset.create(), + sig2 = HashMultiset.create(); + + // check signatures and field types + instructions.stream().filter(i -> i instanceof InvokeVirtual).forEach(i -> { + InvokeVirtual iv = (InvokeVirtual) i; + for (Method m : iv.getMethods()) + sig1.add(m.getDescriptor()); + }); + + other.instructions.stream().filter(i -> i instanceof InvokeVirtual).forEach(i -> { + InvokeVirtual iv = (InvokeVirtual) i; + for (Method m : iv.getMethods()) + sig2.add(m.getDescriptor()); + }); + + if (!sig1.equals(sig2)) + return false; + + Set type1 = new HashSet<>(), + type2 = new HashSet<>(); + + instructions.stream().filter(i -> i instanceof GetFieldInstruction).forEach(i -> { + GetFieldInstruction gfi = (GetFieldInstruction) i; + Field f = gfi.getMyField(); + if (f != null) + type1.add(new FieldWrapper(gfi, f)); + }); + + other.instructions.stream().filter(i -> i instanceof GetFieldInstruction).forEach(i -> { + GetFieldInstruction gfi = (GetFieldInstruction) i; + Field f = gfi.getMyField(); + if (f != null) + type2.add(new FieldWrapper(gfi, f)); + }); + + if (!type1.equals(type2)) + { + for (FieldWrapper fw : type1) + { + if (!type2.contains(fw)) + { + // 2726 -> 2738 + System.out.println(fw + " not in type2"); + for (FieldWrapper fw2 : type2) + { + if (fw2.field.getName().equals("field2738")) + { + int i= 5; + } + } + } + } + return false; + } + + Multiset ms1 = HashMultiset.create(), + ms2 = HashMultiset.create(); + + instructions.stream().filter(i -> i instanceof SetFieldInstruction).forEach(i -> { + SetFieldInstruction sfi = (SetFieldInstruction) i; + Field f = sfi.getMyField(); + if (f != null) + ms1.add(new FieldWrapper(sfi, f)); + }); + + other.instructions.stream().filter(i -> i instanceof SetFieldInstruction).forEach(i -> { + SetFieldInstruction sfi = (SetFieldInstruction) i; + Field f = sfi.getMyField(); + if (f != null) + ms2.add(new FieldWrapper(sfi, f)); + }); + + if (!ms1.equals(ms2)) + return false; + + Set constants1 = new HashSet<>(), + constants2 = new HashSet<>(); + + instructions.stream().filter(i -> i instanceof PushConstantInstruction).forEach(i -> { + PushConstantInstruction pci = (PushConstantInstruction) i; + PoolEntry e = pci.getConstant(); + Object o = e.getObject(); + + if (o instanceof Integer || o instanceof Long) + if (DMath.isBig((Number) o)) + return; + + constants1.add(o); + }); + + other.instructions.stream().filter(i -> i instanceof PushConstantInstruction).forEach(i -> { + PushConstantInstruction pci = (PushConstantInstruction) i; + PoolEntry e = pci.getConstant(); + Object o = e.getObject(); + + if (o instanceof Integer || o instanceof Long) + if (DMath.isBig((Number) o)) + return; + + constants2.add(o); + }); + + if (!constants1.equals(constants2)) + return false; + + return true; + } +} diff --git a/src/main/java/net/runelite/deob/deobfuscators/rename/Rename2.java b/src/main/java/net/runelite/deob/deobfuscators/rename/Rename2.java index c35bce6d4a..96049ba1ec 100644 --- a/src/main/java/net/runelite/deob/deobfuscators/rename/Rename2.java +++ b/src/main/java/net/runelite/deob/deobfuscators/rename/Rename2.java @@ -8,16 +8,19 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Optional; import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; import net.runelite.deob.ClassFile; import net.runelite.deob.ClassGroup; +import net.runelite.deob.Deob; import net.runelite.deob.Field; import net.runelite.deob.Method; import net.runelite.deob.attributes.code.instruction.types.SetFieldInstruction; import net.runelite.deob.deobfuscators.Renamer; +import net.runelite.deob.deobfuscators.rename.graph.Edge; import net.runelite.deob.deobfuscators.rename.graph.Graph; import net.runelite.deob.deobfuscators.rename.graph.Vertex; import net.runelite.deob.deobfuscators.rename.graph.VertexType; @@ -43,10 +46,11 @@ public class Rename2 if (o1 instanceof Method) { + Method m0 = (Method) o0; Method m1 = (Method) o1; Method m2 = (Method) o2; - System.out.println("COLLISION " + mname(m1) + " -> " + mname(m2)); + System.out.println("COLLISION on " + mname(m0) + ": " + mname(m1) + " -> " + mname(m2)); } else if (o1 instanceof Field) { @@ -82,7 +86,7 @@ public class Rename2 return set; } - void mapClassMethods(Map one, Map two) + private void mapClassMethods(Map one, Map two) { if (!one.keySet().equals(two.keySet())) return; @@ -102,6 +106,27 @@ public class Rename2 } } + private void mapDeobfuscatedMethods(ClassFile cf1, ClassFile cf2) + { + List m1 = cf1.getMethods().getMethods().stream().filter(m -> !Deob.isObfuscated(m.getName())).collect(Collectors.toList()), + m2 = cf2.getMethods().getMethods().stream().filter(m -> !Deob.isObfuscated(m.getName())).collect(Collectors.toList()); + + for (Method m : m1) + { + Optional opt = m2.stream().filter(m2m -> m.getName().equals(m2m.getName()) && m.getDescriptor().equals(m2m.getDescriptor())).findAny(); + if (!opt.isPresent()) + continue; + + Vertex v1 = g1.getVertexFor(m); + Vertex v2 = g2.getVertexFor(opt.get()); + + v1.is(v2); + v2.is(v1); + + System.out.println(mname(m) + " is " + mname(opt.get())); + } + } + private List getClientFields(ClassGroup group, Execution e) { Method clinit = group.findClass("client").findMethod(""); @@ -131,22 +156,58 @@ public class Rename2 { Vertex other = s.getOther(); - s.getEdges().stream() - .filter(e -> e.getTo().getOther() == null) // only get vertexes that aren't solved yet - .forEach(e -> - e.getTo().merge( - other.getEdges().stream() - .filter(e2 -> e2.getTo().getOther() == null) - .filter(e2 -> e.getTo().couldBeEqual(e2.getTo())) - .filter(e2 -> e.couldBeEqual(e2)) - .map(e2 -> e2.getTo()) - .collect(Collectors.toList()) - ) - ); + assert s.getGraph() != other.getGraph(); + + for (Edge e : s.getEdges()) + { + assert e.getFrom() == s; + + if (e.getTo().getOther() != null) + continue; // skip solved edges + + Vertex v = e.getTo(); // end of edge in g1 + + Method m = (Method) v.getObject(); + if (m.getName().equals("vmethod3054")) + { + int i = 5; + } + + List l = new ArrayList<>(); + for (Edge e2 : other.getEdges()) + { + if (e2.getTo().getOther() != null) + continue; // skip solved edges + + if (e.getTo().toString().equals("Vertex{object=client.vmethod3054()V}") + && e2.getTo().toString().equals("Vertex{object=client.vmethod2973()V}")) + { + int i= 5; + } + + if (!e.getTo().couldBeEqual(e2.getTo())) + { + System.out.println(e.getTo() + " != " + e2.getTo()); + continue; + } + + if (!e.couldBeEqual(e2)) + { + System.out.println(e + " != " + e2); + continue; + } + + Vertex v2 = e2.getTo(); + + l.add(v2); + } + + v.merge(l); + } } } - public void run(ClassGroup one, ClassGroup two) + public NameMappings run(ClassGroup one, ClassGroup two) { Execution eone = new Execution(one); eone.setBuildGraph(true); @@ -166,34 +227,51 @@ public class Rename2 for (int i = 0; i < Math.min(one.getClasses().size(), two.getClasses().size()); ++i) { + ClassFile c1 = one.getClasses().get(i); + ClassFile c2 = two.getClasses().get(i); + Map m1 = this.find(one.getClasses().get(i)); Map m2 = this.find(two.getClasses().get(i)); - mapClassMethods(m1, m2); + // mapClassMethods(m1, m2); + + mapDeobfuscatedMethods(c1, c2); } - List fl1 = getClientFields(one, eone); - List fl2 = getClientFields(two, etwo); + //List fl1 = getClientFields(one, eone); + //List fl2 = getClientFields(two, etwo); - for (int i = 0; i < Math.min(fl1.size(), fl2.size()); ++i) - { - Field f1 = fl1.get(i), f2 = fl2.get(i); - - Vertex v1 = g1.getVertexFor(f1); - Vertex v2 = g2.getVertexFor(f2); - - v1.is(v2); - v2.is(v1); - - System.out.println(fname(f1) + " is " + fname(f2)); - } +// for (int i = 0; i < Math.min(fl1.size(), fl2.size()); ++i) +// { +// Field f1 = fl1.get(i), f2 = fl2.get(i); +// +// Vertex v1 = g1.getVertexFor(f1); +// Vertex v2 = g2.getVertexFor(f2); +// +// v1.is(v2); +// v2.is(v1); +// +// System.out.println(fname(f1) + " is " + fname(f2)); +// } + + System.out.println("g1 verticies " + g1.getVerticies().size() + " reachable " + g1.reachableVerticiesFromSolvedVerticies().size()); + Set reachable = g1.reachableVerticiesFromSolvedVerticies(); + for (Vertex v : g1.getVerticies()) + if (!reachable.contains(v)) + { + System.out.println("unreachable " + v); + } for (;;) { int before = g1.solved(null); System.out.println("Before " + before); + solve(); + g1.getVerticies().forEach(v -> v.finish()); + //g2 + int after = g1.solved(null); System.out.println("After " + after); @@ -214,16 +292,20 @@ public class Rename2 show(mappings); - rename(mappings, two); + System.out.println("Solved methods "+ g1.solved(VertexType.METHOD) + ", total " + g1.getVerticies().size()); + + //rename(mappings, two); try { - JarUtil.saveJar(two, new File("d:/rs/07/adamout.jar")); + JarUtil.saveJar(two, new File("/Users/adam/w/rs/07/adamout.jar")); } catch (IOException ex) { Logger.getLogger(Rename2.class.getName()).log(Level.SEVERE, null, ex); } + + return mappings; } private void show(NameMappings mappings) @@ -244,6 +326,8 @@ public class Rename2 System.out.println("FINAL " + n + " -> " + f.getNameAndType().getName()); } } + + System.out.println("Mappins size " + mappings.getMap().size()); } private NameMappings buildCollisionMap(ClassGroup one, ClassGroup two) @@ -255,6 +339,9 @@ public class Rename2 { for (Method m : cf.getMethods().getMethods()) { + if (m.isStatic() && !m.getName().equals("")) + continue; + Vertex v = g2.getVertexFor(m); Vertex other = v.getOther(); @@ -266,25 +353,30 @@ public class Rename2 Method m2 = (Method) other.getObject(); + if (m.getName().equals(m2.getName())) + continue; // already correct + Method existingMethod = cf.findMethod(m2.getName()); if (existingMethod != null) + { mappings.map(existingMethod.getPoolMethod(), "collidedMethod" + count++); + } } - for (Field f : cf.getFields().getFields()) - { - Vertex v = g2.getVertexFor(f); - Vertex other = v.getOther(); - - if (other == null) - continue; - - Field f2 = (Field) other.getObject(); - - Field existingField = cf.findField(f2.getName()); - if (existingField != null) - mappings.map(existingField.getPoolField(), "collidedField" + count++); - } +// for (Field f : cf.getFields().getFields()) +// { +// Vertex v = g2.getVertexFor(f); +// Vertex other = v.getOther(); +// +// if (other == null) +// continue; +// +// Field f2 = (Field) other.getObject(); +// +// Field existingField = cf.findField(f2.getName()); +// if (existingField != null) +// mappings.map(existingField.getPoolField(), "collidedField" + count++); +// } } return mappings; @@ -298,6 +390,9 @@ public class Rename2 { for (Method m : cf.getMethods().getMethods()) { + if (m.isStatic() && !m.getName().equals("")) + continue; + Vertex v = g2.getVertexFor(m); Vertex other = v.getOther(); @@ -309,27 +404,30 @@ public class Rename2 Method m2 = (Method) other.getObject(); - Method existingMethod = cf.findMethod(m2.getName()); - assert existingMethod == null; + if (!m.getName().equals(m2.getName())) + { + Method existingMethod = cf.findMethod(m2.getName()); + assert existingMethod == null; + } mappings.map(m.getPoolMethod(), m2.getName()); } - for (Field f : cf.getFields().getFields()) - { - Vertex v = g2.getVertexFor(f); - Vertex other = v.getOther(); - - if (other == null) - continue; - - Field f2 = (Field) other.getObject(); - - Field existingField = cf.findField(f2.getName()); - assert existingField == null; - - mappings.map(f.getPoolField(), f2.getName()); - } +// for (Field f : cf.getFields().getFields()) +// { +// Vertex v = g2.getVertexFor(f); +// Vertex other = v.getOther(); +// +// if (other == null) +// continue; +// +// Field f2 = (Field) other.getObject(); +// +// Field existingField = cf.findField(f2.getName()); +// assert existingField == null; +// +// mappings.map(f.getPoolField(), f2.getName()); +// } } return mappings; diff --git a/src/main/java/net/runelite/deob/deobfuscators/rename/graph/Edge.java b/src/main/java/net/runelite/deob/deobfuscators/rename/graph/Edge.java index 0bd768ca3f..1e9e895b0c 100644 --- a/src/main/java/net/runelite/deob/deobfuscators/rename/graph/Edge.java +++ b/src/main/java/net/runelite/deob/deobfuscators/rename/graph/Edge.java @@ -42,6 +42,12 @@ public class Edge return weight; } + @Override + public String toString() + { + return "Edge{" + "from=" + from + ", to=" + to + '}'; + } + @Override public int hashCode() { @@ -88,8 +94,8 @@ public class Edge if (this.type != other.type) return false; -// if (this.weight != other.weight) -// return false; + if (this.weight != other.weight) + return false; return true; } diff --git a/src/main/java/net/runelite/deob/deobfuscators/rename/graph/Graph.java b/src/main/java/net/runelite/deob/deobfuscators/rename/graph/Graph.java index 72cde7b536..38ae410a8c 100644 --- a/src/main/java/net/runelite/deob/deobfuscators/rename/graph/Graph.java +++ b/src/main/java/net/runelite/deob/deobfuscators/rename/graph/Graph.java @@ -2,8 +2,10 @@ package net.runelite.deob.deobfuscators.rename.graph; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; public class Graph { @@ -11,15 +13,36 @@ public class Graph private Map o2v = new HashMap<>(); +// public Vertex getVertexFor(Object o) +// { +// Vertex v = o2v.get(o); +// if (v != null) +// return v; +// +// v = new Vertex(this, o); +// o2v.put(o, v); +// verticies.add(v); +// return v; +// } + public Vertex getVertexFor(Object o) { Vertex v = o2v.get(o); - if (v != null) - return v; + assert v != null; + return v; + } + + public Vertex addVertex(Object o, VertexType type) + { + assert o2v.get(o) == null; + + Vertex v = new Vertex(this, o); + //v.setType(type); + assert type == v.getType(); - v = new Vertex(this, o); - o2v.put(o, v); verticies.add(v); + o2v.put(o, v); + return v; } @@ -69,4 +92,24 @@ public class Graph ++solved; return solved; } + + private void recurse(Vertex v, Set verticies) + { + if (verticies.contains(v)) + return; + + verticies.add(v); + + for (Edge e : v.getEdges()) + recurse(e.getTo(), verticies); + } + + public Set reachableVerticiesFromSolvedVerticies() + { + Set verticies = new HashSet<>(); + for (Vertex v : this.verticies) + if (v.getOther() != null) + recurse(v, verticies); + return verticies; + } } diff --git a/src/main/java/net/runelite/deob/deobfuscators/rename/graph/Vertex.java b/src/main/java/net/runelite/deob/deobfuscators/rename/graph/Vertex.java index 9680f35e76..7a15008c0b 100644 --- a/src/main/java/net/runelite/deob/deobfuscators/rename/graph/Vertex.java +++ b/src/main/java/net/runelite/deob/deobfuscators/rename/graph/Vertex.java @@ -1,9 +1,11 @@ package net.runelite.deob.deobfuscators.rename.graph; import com.google.common.base.Objects; +import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import net.runelite.deob.ClassFile; @@ -13,15 +15,16 @@ import net.runelite.deob.attributes.AttributeType; import net.runelite.deob.attributes.Code; import net.runelite.deob.attributes.ConstantValue; import net.runelite.deob.attributes.code.Instruction; -import net.runelite.deob.attributes.code.instruction.types.PushConstantInstruction; +import net.runelite.deob.attributes.code.instruction.types.InvokeInstruction; +import net.runelite.deob.attributes.code.instructions.InvokeStatic; +import net.runelite.deob.deobfuscators.rename.InstructionList; import net.runelite.deob.deobfuscators.rename.Rename2; -import net.runelite.deob.pool.PoolEntry; import org.apache.commons.collections4.CollectionUtils; public class Vertex { private Graph graph; - private Object object; + private final Object object; private VertexType type; private final Map edges = new HashMap<>(); @@ -37,10 +40,45 @@ public class Vertex type = VertexType.METHOD; else if (object instanceof Field) type = VertexType.FIELD; + else if (object instanceof ClassFile) + type = VertexType.CLASS; else assert false; } + @Override + public String toString() + { + return "Vertex{" + "object=" + object + '}'; + } + + @Override + public int hashCode() + { + int hash = 7; + hash = 79 * hash + java.util.Objects.hashCode(this.object); + return hash; + } + + @Override + public boolean equals(Object obj) + { + if (obj == null) + { + return false; + } + if (getClass() != obj.getClass()) + { + return false; + } + final Vertex other = (Vertex) obj; + if (!java.util.Objects.equals(this.object, other.object)) + { + return false; + } + return true; + } + public Graph getGraph() { return graph; @@ -90,11 +128,32 @@ public class Vertex public void finish() { + if (mightBe == null) + return; + + if (mightBe != null && mightBe.size() == 2) + { + System.out.println("Can't decide for " + this); + + for(Vertex v : mightBe) + System.out.println(v); + int i = 5; + } + if (mightBe.isEmpty()) + { + System.out.println("empty " + this); + int i = 5; + } if (mightBe != null && mightBe.size() == 1) { - is(mightBe.stream().findAny().get()); - is.is(this); - mightBe = null; + Vertex v = mightBe.stream().findAny().get(); + //if (v.getOther() == null || v.getOther() == this) + { + is(v); + is.is(this); + mightBe = null; + System.out.println(this + " is " + is); + } } } @@ -119,7 +178,7 @@ public class Vertex private boolean couldBeEqual(ClassFile cf1, ClassFile cf2) { - if (!cf1.getName().equals(cf2.getName())) + if (!cf1.getClassName().equals(cf2.getClassName())) return false; if (!cf1.getInterfaces().getInterfaces().equals(cf2.getInterfaces().getInterfaces())) @@ -131,35 +190,45 @@ public class Vertex return true; } + private List getInstructionsInMethodInclStatic(Method method, Set visited) + { + List ilist = new ArrayList<>(); + + if (visited.contains(method)) + return ilist; + visited.add(method); + + Code code = method.getCode(); + if (code == null) + return ilist; + + for (Instruction i : code.getInstructions().getInstructions()) + { + if (i instanceof InvokeStatic) + { + InvokeInstruction ii = (InvokeInstruction) i; + List methods = ii.getMethods(); + + if (methods.isEmpty()) + continue; + + Method m = methods.get(0); + ilist.addAll(this.getInstructionsInMethodInclStatic(m, visited)); + } + else + { + ilist.add(i); + } + } + + return ilist; + } + private boolean couldBeEqual(Method m1, Method m2) { - Set h1 = new HashSet<>(), - h2 = new HashSet<>(); - - if (m1.getCode() == null) - return true; - - for (Instruction i : m1.getCode().getInstructions().getInstructions()) - { - if (i instanceof PushConstantInstruction) - { - PushConstantInstruction pci = (PushConstantInstruction) i; - h1.add(pci.getConstant()); - } - } - - for (Instruction i : m2.getCode().getInstructions().getInstructions()) - { - if (i instanceof PushConstantInstruction) - { - PushConstantInstruction pci = (PushConstantInstruction) i; - h2.add(pci.getConstant()); - } - } - - boolean b = h1.equals(h2); - return b; - //return true; + InstructionList il1 = new InstructionList(this.getInstructionsInMethodInclStatic(m1, new HashSet())), + il2 = new InstructionList(this.getInstructionsInMethodInclStatic(m2, new HashSet())); + return il1.couldBeEqual(il2); } public boolean couldBeEqual(Vertex other) @@ -197,8 +266,8 @@ public class Vertex if (!couldBeEqual(cf1, cf2)) return false; -// if (!couldBeEqual(m1, m2)) -// return false; + if (!couldBeEqual(m1, m2)) + return false; } else if (type == VertexType.FIELD) { @@ -207,7 +276,8 @@ public class Vertex if (!f1.getType().equals(f2.getType())) return false; - + + access flags can change sometimes from non public to public like 2726 -> 2738 if (f1.isStatic() != f2.isStatic() || f1.getAccessFlags() != f2.getAccessFlags()) return false; diff --git a/src/main/java/net/runelite/deob/deobfuscators/rename/graph/VertexType.java b/src/main/java/net/runelite/deob/deobfuscators/rename/graph/VertexType.java index 8a13981063..2d09ce81bc 100644 --- a/src/main/java/net/runelite/deob/deobfuscators/rename/graph/VertexType.java +++ b/src/main/java/net/runelite/deob/deobfuscators/rename/graph/VertexType.java @@ -4,5 +4,6 @@ package net.runelite.deob.deobfuscators.rename.graph; public enum VertexType { METHOD, - FIELD; + FIELD, + CLASS; } diff --git a/src/main/java/net/runelite/deob/execution/Execution.java b/src/main/java/net/runelite/deob/execution/Execution.java index 13781e5e49..2e16d878cd 100644 --- a/src/main/java/net/runelite/deob/execution/Execution.java +++ b/src/main/java/net/runelite/deob/execution/Execution.java @@ -14,12 +14,14 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; +import net.runelite.deob.Field; import net.runelite.deob.attributes.code.instruction.types.FieldInstruction; import net.runelite.deob.attributes.code.instruction.types.GetFieldInstruction; import net.runelite.deob.attributes.code.instruction.types.InvokeInstruction; import net.runelite.deob.attributes.code.instructions.InvokeStatic; import net.runelite.deob.deobfuscators.rename.graph.EdgeType; import net.runelite.deob.deobfuscators.rename.graph.Graph; +import net.runelite.deob.deobfuscators.rename.graph.VertexType; import org.apache.commons.collections4.map.MultiValueMap; public class Execution @@ -117,6 +119,8 @@ public class Execution public void run() { + initializeGraph(); + int fcount = 0; while (!frames.isEmpty()) { @@ -161,6 +165,30 @@ public class Execution this.buildGraph = buildGraph; } + private void initializeGraph() + { + if (!isBuildGraph()) + return; + + for (ClassFile cf : this.group.getClasses()) + { + //graph.addVertex(cf, VertexType.CLASS); + + for (Method m : cf.getMethods().getMethods()) + { + if (m.isStatic() && !m.getName().equals("")) + continue; + + graph.addVertex(m, VertexType.METHOD); + } + + for (Field f : cf.getFields().getFields()) + { + // graph.addVertex(f, VertexType.FIELD); + } + } + } + protected void buildGraph(Frame frame, Instruction i) { if (!isBuildGraph()) @@ -190,7 +218,7 @@ public class Execution return; EdgeType type = fi instanceof GetFieldInstruction ? EdgeType.GETFIELD : EdgeType.SETFIELD; - graph.addEdge(frame.nonStatic, fi.getMyField(), type); + //graph.addEdge(frame.nonStatic, fi.getMyField(), type); } } diff --git a/src/main/java/net/runelite/deob/signature/Type.java b/src/main/java/net/runelite/deob/signature/Type.java index 26e0974ffc..e692fb21b9 100644 --- a/src/main/java/net/runelite/deob/signature/Type.java +++ b/src/main/java/net/runelite/deob/signature/Type.java @@ -92,6 +92,9 @@ public class Type @Override public String toString() { - return getFullType(); + String type = this.type; + for (int i = 0; i < this.getArrayDims(); ++i) + type += "[]"; + return type; } } diff --git a/src/test/java/net/runelite/deob/execution/FrameTest.java b/src/test/java/net/runelite/deob/execution/FrameTest.java deleted file mode 100644 index 8a8bf7e9ac..0000000000 --- a/src/test/java/net/runelite/deob/execution/FrameTest.java +++ /dev/null @@ -1,133 +0,0 @@ -//package net.runelite.deob.execution; -// -//import net.runelite.deob.ClassGroup; -//import net.runelite.deob.ClassGroupFactory; -//import net.runelite.deob.Method; -//import net.runelite.deob.attributes.Code; -//import net.runelite.deob.attributes.code.Instruction; -//import net.runelite.deob.attributes.code.Instructions; -//import net.runelite.deob.attributes.code.instructions.Goto; -//import net.runelite.deob.attributes.code.instructions.IConst_0; -//import net.runelite.deob.attributes.code.instructions.If0; -//import net.runelite.deob.attributes.code.instructions.InvokeStatic; -//import net.runelite.deob.attributes.code.instructions.NOP; -//import net.runelite.deob.attributes.code.instructions.VReturn; -//import net.runelite.deob.deobfuscators.rename.graph.Graph; -//import org.junit.Assert; -//import org.junit.Test; -// -//public class FrameTest -//{ -// // invoke instruction, -// // conditional jump out, -// // both jump in, -// // jump, -// // invoke -// // check that num edges = 4 -// @Test -// public void testGraph() -// { -// ClassGroup group = ClassGroupFactory.generateGroup(); -// Code code = group.findClass("test").findMethod("func").getCode(); -// Instructions ins = code.getInstructions(); -// -// code.setMaxStack(1); -// -// Method void1 = group.findClass("test").findMethod("void1"), -// void2 = group.findClass("test").findMethod("void2"), -// void3 = group.findClass("test").findMethod("void3"), -// void4 = group.findClass("test").findMethod("void4"); -// -// NOP label1 = new NOP(ins), -// label2 = new NOP(ins), -// label3 = new NOP(ins); -// -// Instruction body[] = { -// new InvokeStatic(ins, void1.getPoolMethod()), -// -// new IConst_0(ins), -// new If0(ins, label1), -// -// new InvokeStatic(ins, void2.getPoolMethod()), -// new Goto(ins, label2), -// -// label1, -// new InvokeStatic(ins, void3.getPoolMethod()), -// -// label2, -// // do something dumb -// -// new Goto(ins, label3), -// label3, -// -// new InvokeStatic(ins, void4.getPoolMethod()), -// -// new VReturn(ins) -// }; -// -// for (Instruction i : body) -// ins.addInstruction(i); -// -// Execution e = new Execution(group); -// e.setBuildGraph(true); -// e.populateInitialMethods(); -// e.run(); -// -// Graph graph = e.processedFrames.get(0).getMethodCtx().getGraph(); -// Assert.assertEquals(4, graph.size()); -// } -// -// // invoke instruction, -// // conditional jump out, -// // both jump in, -// // invoke -// // check that num edges = 4 -// @Test -// public void testGraph2() -// { -// ClassGroup group = ClassGroupFactory.generateGroup(); -// Code code = group.findClass("test").findMethod("func").getCode(); -// Instructions ins = code.getInstructions(); -// -// code.setMaxStack(1); -// -// Method void1 = group.findClass("test").findMethod("void1"), -// void2 = group.findClass("test").findMethod("void2"), -// void3 = group.findClass("test").findMethod("void3"), -// void4 = group.findClass("test").findMethod("void4"); -// -// NOP label1 = new NOP(ins), -// label2 = new NOP(ins); -// -// Instruction body[] = { -// new InvokeStatic(ins, void1.getPoolMethod()), -// -// new IConst_0(ins), -// new If0(ins, label1), -// -// new InvokeStatic(ins, void2.getPoolMethod()), -// new Goto(ins, label2), -// -// label1, -// new InvokeStatic(ins, void3.getPoolMethod()), -// -// label2, -// // do something dumb -// -// new InvokeStatic(ins, void4.getPoolMethod()), -// -// new VReturn(ins) -// }; -// -// for (Instruction i : body) -// ins.addInstruction(i); -// -// Execution e = new Execution(group); -// e.setBuildGraph(true); -// e.populateInitialMethods(); -// e.run(); -// -// Graph graph = e.processedFrames.get(0).getMethodCtx().getGraph(); -// Assert.assertEquals(4, graph.size()); -// } -//} \ No newline at end of file diff --git a/src/test/java/net/runelite/deob/execution/GraphTest.java b/src/test/java/net/runelite/deob/execution/GraphTest.java new file mode 100644 index 0000000000..aca997d12d --- /dev/null +++ b/src/test/java/net/runelite/deob/execution/GraphTest.java @@ -0,0 +1,138 @@ +package net.runelite.deob.execution; + +import java.io.DataInputStream; +import java.io.IOException; +import java.io.InputStream; +import net.runelite.deob.ClassFile; +import net.runelite.deob.ClassGroup; +import net.runelite.deob.Method; +import net.runelite.deob.deobfuscators.rename.InstructionList; +import net.runelite.deob.deobfuscators.rename.Rename2; +import net.runelite.deob.deobfuscators.rename.graph.Graph; +import net.runelite.deob.deobfuscators.rename.graph.Vertex; +import net.runelite.deob.util.NameMappings; +import org.junit.Assert; +import org.junit.Test; + +class TestClass2 +{ + int array[]; + int count; + + void method1() + { + array[++count - 1] = 1; + array[++count - 1] = 2; + } + + void method2() + { + array[++count - 1] = 1; + array[++count - 1] = 2; + array[++count - 1] = 3; + array[++count - 1] = 4; + } +} + +public class GraphTest +{ + private ClassGroup loadClassGroup(String name) throws IOException + { + ClassGroup group = new ClassGroup(); + + ClassFile cf = this.loadClass(name); + group.addClass(cf); + + return group; + } + + private ClassFile loadClass(String name) throws IOException + { + InputStream in = this.getClass().getClassLoader().getResourceAsStream("net/runelite/deob/execution/" + name + ".class"); + Assert.assertNotNull(in); + + ClassGroup group = new ClassGroup(); + ClassFile cf = new ClassFile(group, new DataInputStream(in)); + group.addClass(cf); + return cf; + } + + //@Test + public void test() throws IOException + { + ClassGroup group1 = this.loadClassGroup("TestClass"), group2 = this.loadClassGroup("TestClass"); + + Execution e = new Execution(group1); + e.setBuildGraph(true); + e.populateInitialMethods(); + e.run(); + + Execution e2 = new Execution(group2); + e2.setBuildGraph(true); + e2.populateInitialMethods(); + e2.run(); + + Graph graph = e.getGraph(); + Graph graph2 = e2.getGraph(); + + Assert.assertEquals(4, graph.getVerticies().size()); + + Method m = group1.getClasses().get(0).findMethod("init"); + Vertex v = graph.getVertexFor(m); + + Assert.assertEquals(1, v.getEdges().size()); + + m = group1.getClasses().get(0).findMethod("method2"); + v = graph.getVertexFor(m); + + Assert.assertEquals(1, v.getEdges().size()); + } + + @Test + public void test2() throws IOException + { + ClassGroup group1 = this.loadClassGroup("one/TestClass"), group2 = this.loadClassGroup("two/TestClass"); + Rename2 rename2 = new Rename2(); + NameMappings mappings = rename2.run(group1, group2); // 2->1 + + ClassFile cf1 = group1.getClasses().get(0), + cf2 = group2.getClasses().get(0); + + Method m2 = cf2.findMethod("init"); + Assert.assertTrue(mappings.get(m2.getPoolMethod()).equals("init")); + + m2 = cf2.findMethod("method6"); + String to = mappings.get(m2.getPoolMethod()); + Assert.assertNotNull(to); + Assert.assertTrue(to.equals("method2")); + } + + //@Test + public void testVertexEquals() throws IOException + { + ClassGroup group1 = this.loadClassGroup("one/TestClass"), group2 = this.loadClassGroup("two/TestClass"); + + ClassFile cf1 = group1.getClasses().get(0), + cf2 = group2.getClasses().get(0); + + Graph g1 = new Graph(), g2 = new Graph(); + + Vertex v1 = new Vertex(g1, cf2.findMethod("method6")), + v2 = new Vertex(g2, cf1.findMethod("method2")); + + Assert.assertTrue(v1.couldBeEqual(v2)); + } + + //@Test + public void test3() throws IOException + { + ClassFile cf1 = this.loadClass("TestClass2"), cf2 = this.loadClass("TestClass2"); + + Method m1 = cf1.findMethod("method1"), m2 = cf2.findMethod("method2"); + + InstructionList il1 = new InstructionList(m1.getCode().getInstructions().getInstructions()), + il2 = new InstructionList(m2.getCode().getInstructions().getInstructions()); + + Assert.assertFalse(il1.couldBeEqual(il2)); + } +} diff --git a/src/test/java/net/runelite/deob/execution/one/TestClass.java b/src/test/java/net/runelite/deob/execution/one/TestClass.java new file mode 100644 index 0000000000..bf4fa9ea7c --- /dev/null +++ b/src/test/java/net/runelite/deob/execution/one/TestClass.java @@ -0,0 +1,30 @@ +package net.runelite.deob.execution.one; + +public class TestClass +{ + int i; + + public void init() + { + method1(this); + } + + static void method1(TestClass tc) + { + tc.method2(); + } + + void method2() + { + if (i > 42) + { + int i = 5 + 6; + method3(); + } + } + + void method3() + { + + } +} \ No newline at end of file diff --git a/src/test/java/net/runelite/deob/execution/two/TestClass.java b/src/test/java/net/runelite/deob/execution/two/TestClass.java new file mode 100644 index 0000000000..3941d65c94 --- /dev/null +++ b/src/test/java/net/runelite/deob/execution/two/TestClass.java @@ -0,0 +1,30 @@ +package net.runelite.deob.execution.two; + +public class TestClass +{ + int i; + + public void init() + { + method5(this); + } + + static void method5(TestClass tc) + { + tc.method6(); + } + + void method6() + { + if (i > 42) + { + int i = 5 + 6; + method7(); + } + } + + void method7() + { + + } +} \ No newline at end of file