From 6bb02614ca5c49dec1e8cf1e1d2f40bd1f1cc337 Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 31 Dec 2015 18:49:48 -0500 Subject: [PATCH] Skip ctors and class initializers, and exception frames for now in parallel executor. Found a case of ifeq with one side null vs ifnull, so, need to write some if logic. --- .../types/MappableInstruction.java | 3 +- .../deob/attributes/code/instructions/If.java | 5 +- .../attributes/code/instructions/If0.java | 3 +- .../code/instructions/InvokeInterface.java | 8 +- .../code/instructions/InvokeSpecial.java | 10 +- .../code/instructions/InvokeStatic.java | 8 +- .../code/instructions/InvokeVirtual.java | 8 +- .../code/instructions/PutField.java | 22 +++- .../code/instructions/PutStatic.java | 5 +- .../rename/MappingExecutorUtil.java | 112 ++++++++++++++++++ .../rename/ParallelExecutorMapping.java | 24 ++++ .../deob/deobfuscators/rename/Rename2.java | 51 ++++++++ .../net/runelite/deob/execution/Frame.java | 3 + .../deob/deobfuscators/rename/MapTest.java | 84 +------------ 14 files changed, 252 insertions(+), 94 deletions(-) create mode 100644 src/main/java/net/runelite/deob/deobfuscators/rename/MappingExecutorUtil.java create mode 100644 src/main/java/net/runelite/deob/deobfuscators/rename/ParallelExecutorMapping.java diff --git a/src/main/java/net/runelite/deob/attributes/code/instruction/types/MappableInstruction.java b/src/main/java/net/runelite/deob/attributes/code/instruction/types/MappableInstruction.java index b07377d78c..af1bed491a 100644 --- a/src/main/java/net/runelite/deob/attributes/code/instruction/types/MappableInstruction.java +++ b/src/main/java/net/runelite/deob/attributes/code/instruction/types/MappableInstruction.java @@ -1,8 +1,9 @@ package net.runelite.deob.attributes.code.instruction.types; +import net.runelite.deob.deobfuscators.rename.ParallelExecutorMapping; import net.runelite.deob.execution.InstructionContext; public interface MappableInstruction { - void map(InstructionContext ctx, InstructionContext other); + void map(ParallelExecutorMapping mappings, InstructionContext ctx, InstructionContext other); } diff --git a/src/main/java/net/runelite/deob/attributes/code/instructions/If.java b/src/main/java/net/runelite/deob/attributes/code/instructions/If.java index 3453bccb09..333e969e85 100644 --- a/src/main/java/net/runelite/deob/attributes/code/instructions/If.java +++ b/src/main/java/net/runelite/deob/attributes/code/instructions/If.java @@ -16,6 +16,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; import net.runelite.deob.attributes.code.instruction.types.MappableInstruction; +import net.runelite.deob.deobfuscators.rename.ParallelExecutorMapping; public abstract class If extends Instruction implements JumpingInstruction, ComparisonInstruction, MappableInstruction { @@ -96,7 +97,7 @@ public abstract class If extends Instruction implements JumpingInstruction, Comp } @Override - public final/*XXX tmp*/ void map(InstructionContext ctx, InstructionContext other) + public final/*XXX tmp*/ void map(ParallelExecutorMapping mapping, InstructionContext ctx, InstructionContext other) { InstructionContext oneLhs = ctx.getPops().get(0).getPushed().resolve(ctx.getPops().get(0)), oneRhs = ctx.getPops().get(1).getPushed().resolve(ctx.getPops().get(1)), @@ -116,7 +117,5 @@ public abstract class If extends Instruction implements JumpingInstruction, Comp branch2.other = branch1; // we can map these if they are getfield instructions? - - // } } diff --git a/src/main/java/net/runelite/deob/attributes/code/instructions/If0.java b/src/main/java/net/runelite/deob/attributes/code/instructions/If0.java index d63bee10d0..a67fe11b4c 100644 --- a/src/main/java/net/runelite/deob/attributes/code/instructions/If0.java +++ b/src/main/java/net/runelite/deob/attributes/code/instructions/If0.java @@ -16,6 +16,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; import net.runelite.deob.attributes.code.instruction.types.MappableInstruction; +import net.runelite.deob.deobfuscators.rename.ParallelExecutorMapping; public abstract class If0 extends Instruction implements JumpingInstruction, ComparisonInstruction, MappableInstruction { @@ -97,7 +98,7 @@ public abstract class If0 extends Instruction implements JumpingInstruction, Com } @Override - public final/*XXX tmp*/ void map(InstructionContext ctx, InstructionContext other) + public final/*XXX tmp*/ void map(ParallelExecutorMapping mapping, InstructionContext ctx, InstructionContext other) { InstructionContext one = ctx.getPops().get(0).getPushed().resolve(ctx.getPops().get(0)), two = other.getPops().get(0).getPushed().resolve(other.getPops().get(0)); diff --git a/src/main/java/net/runelite/deob/attributes/code/instructions/InvokeInterface.java b/src/main/java/net/runelite/deob/attributes/code/instructions/InvokeInterface.java index a5e3192942..dcc4c5d5ca 100644 --- a/src/main/java/net/runelite/deob/attributes/code/instructions/InvokeInterface.java +++ b/src/main/java/net/runelite/deob/attributes/code/instructions/InvokeInterface.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import net.runelite.deob.deobfuscators.rename.ParallelExecutorMapping; import net.runelite.deob.execution.Execution; import net.runelite.deob.execution.Value; @@ -159,9 +160,14 @@ public class InvokeInterface extends Instruction implements InvokeInstruction } @Override - public void map(InstructionContext ctx, InstructionContext other) + public void map(ParallelExecutorMapping mapping, InstructionContext ctx, InstructionContext other) { List myMethods = this.getMethods(), otherMethods = ((InvokeInterface) other.getInstruction()).getMethods(); + + assert myMethods.size() == otherMethods.size(); + + for (int i = 0; i < myMethods.size(); ++i) + mapping.map(myMethods.get(i), otherMethods.get(i)); } } diff --git a/src/main/java/net/runelite/deob/attributes/code/instructions/InvokeSpecial.java b/src/main/java/net/runelite/deob/attributes/code/instructions/InvokeSpecial.java index a51b15742a..41303a95d5 100644 --- a/src/main/java/net/runelite/deob/attributes/code/instructions/InvokeSpecial.java +++ b/src/main/java/net/runelite/deob/attributes/code/instructions/InvokeSpecial.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import net.runelite.deob.deobfuscators.rename.ParallelExecutorMapping; import net.runelite.deob.execution.Execution; import net.runelite.deob.execution.Value; @@ -152,9 +153,16 @@ public class InvokeSpecial extends Instruction implements InvokeInstruction } @Override - public void map(InstructionContext ctx, InstructionContext other) + public void map(ParallelExecutorMapping mapping, InstructionContext ctx, InstructionContext other) { List myMethods = this.getMethods(), otherMethods = ((InvokeSpecial) other.getInstruction()).getMethods(); + + List m1 = this.myMethods; + List m2 = ((InvokeSpecial) other.getInstruction()).myMethods; + assert myMethods.size() == otherMethods.size(); + + for (int i = 0; i < myMethods.size(); ++i) + mapping.map(myMethods.get(i), otherMethods.get(i)); } } diff --git a/src/main/java/net/runelite/deob/attributes/code/instructions/InvokeStatic.java b/src/main/java/net/runelite/deob/attributes/code/instructions/InvokeStatic.java index e51741d05a..e72518c8fb 100644 --- a/src/main/java/net/runelite/deob/attributes/code/instructions/InvokeStatic.java +++ b/src/main/java/net/runelite/deob/attributes/code/instructions/InvokeStatic.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import net.runelite.deob.deobfuscators.rename.ParallelExecutorMapping; import net.runelite.deob.execution.Execution; import net.runelite.deob.execution.Value; @@ -162,9 +163,14 @@ public class InvokeStatic extends Instruction implements InvokeInstruction } @Override - public void map(InstructionContext ctx, InstructionContext other) + public void map(ParallelExecutorMapping mapping, InstructionContext ctx, InstructionContext other) { List myMethods = this.getMethods(), otherMethods = ((InvokeStatic) other.getInstruction()).getMethods(); + + assert myMethods.size() == otherMethods.size(); + + for (int i = 0; i < myMethods.size(); ++i) + mapping.map(myMethods.get(i), otherMethods.get(i)); } } diff --git a/src/main/java/net/runelite/deob/attributes/code/instructions/InvokeVirtual.java b/src/main/java/net/runelite/deob/attributes/code/instructions/InvokeVirtual.java index c64444ee75..099e8ce342 100644 --- a/src/main/java/net/runelite/deob/attributes/code/instructions/InvokeVirtual.java +++ b/src/main/java/net/runelite/deob/attributes/code/instructions/InvokeVirtual.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import net.runelite.deob.deobfuscators.rename.ParallelExecutorMapping; import net.runelite.deob.execution.Execution; import net.runelite.deob.execution.Value; @@ -163,9 +164,14 @@ public class InvokeVirtual extends Instruction implements InvokeInstruction } @Override - public void map(InstructionContext ctx, InstructionContext other) + public void map(ParallelExecutorMapping mapping, InstructionContext ctx, InstructionContext other) { List myMethods = this.getMethods(), otherMethods = ((InvokeVirtual) other.getInstruction()).getMethods(); + + assert myMethods.size() == otherMethods.size(); + + for (int i = 0; i < myMethods.size(); ++i) + mapping.map(myMethods.get(i), otherMethods.get(i)); } } diff --git a/src/main/java/net/runelite/deob/attributes/code/instructions/PutField.java b/src/main/java/net/runelite/deob/attributes/code/instructions/PutField.java index e8b5de8966..024285e9e2 100644 --- a/src/main/java/net/runelite/deob/attributes/code/instructions/PutField.java +++ b/src/main/java/net/runelite/deob/attributes/code/instructions/PutField.java @@ -17,6 +17,8 @@ import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; import net.runelite.deob.Method; +import net.runelite.deob.attributes.code.instruction.types.PushConstantInstruction; +import net.runelite.deob.deobfuscators.rename.ParallelExecutorMapping; public class PutField extends Instruction implements SetFieldInstruction { @@ -94,13 +96,29 @@ public class PutField extends Instruction implements SetFieldInstruction if (myField != null) field = myField.getPoolField(); } + + private boolean isConstantAssignment(InstructionContext ctx) + { + return ctx.getPops().get(0).getPushed().getInstruction() instanceof PushConstantInstruction; + } @Override - public void map(InstructionContext ctx, InstructionContext other) + public void map(ParallelExecutorMapping mapping, InstructionContext ctx, InstructionContext other) { net.runelite.deob.Field myField = this.getMyField(), otherField = ((PutField) other.getInstruction()).getMyField(); - System.out.println("MAPPING " + myField + " -> " + otherField); + // it appears ConstantValue field attributes are inlined into the constructor + // and their orders scrambled, so don't accept constant value assignments? +// if (ctx.getFrame().getMethod().getName().equals("")) +// { +// //assert isConstantAssignment(ctx) == isConstantAssignment(other); +// //if (isConstantAssignment(ctx)) +// return; +// } + + // XXX field types must be the same + + mapping.map(myField, otherField); } } diff --git a/src/main/java/net/runelite/deob/attributes/code/instructions/PutStatic.java b/src/main/java/net/runelite/deob/attributes/code/instructions/PutStatic.java index e9c2aa3ade..515624eaaf 100644 --- a/src/main/java/net/runelite/deob/attributes/code/instructions/PutStatic.java +++ b/src/main/java/net/runelite/deob/attributes/code/instructions/PutStatic.java @@ -17,6 +17,7 @@ import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; import net.runelite.deob.Method; +import net.runelite.deob.deobfuscators.rename.ParallelExecutorMapping; public class PutStatic extends Instruction implements SetFieldInstruction { @@ -95,11 +96,11 @@ public class PutStatic extends Instruction implements SetFieldInstruction } @Override - public void map(InstructionContext ctx, InstructionContext other) + public void map(ParallelExecutorMapping mapping, InstructionContext ctx, InstructionContext other) { net.runelite.deob.Field myField = this.getMyField(), otherField = ((PutStatic) other.getInstruction()).getMyField(); - System.out.println("MAPPING " + myField + " -> " + otherField); + mapping.map(myField, otherField); } } diff --git a/src/main/java/net/runelite/deob/deobfuscators/rename/MappingExecutorUtil.java b/src/main/java/net/runelite/deob/deobfuscators/rename/MappingExecutorUtil.java new file mode 100644 index 0000000000..330bc2e887 --- /dev/null +++ b/src/main/java/net/runelite/deob/deobfuscators/rename/MappingExecutorUtil.java @@ -0,0 +1,112 @@ +package net.runelite.deob.deobfuscators.rename; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import net.runelite.deob.ClassGroup; +import net.runelite.deob.Method; +import net.runelite.deob.attributes.code.Instruction; +import net.runelite.deob.attributes.code.instruction.types.MappableInstruction; +import net.runelite.deob.execution.Execution; +import net.runelite.deob.execution.Frame; +import net.runelite.deob.execution.InstructionContext; +import net.runelite.deob.execution.ParallellMappingExecutor; +import net.runelite.deob.util.JarUtil; + +public class MappingExecutorUtil +{ + public static boolean isMappable(Method m1, Method m2) + { + assert (m1.getCode() == null) == (m2.getCode() == null); + + if (m1.getCode() == null || m2.getCode() == null) + return false; + + List l1 = m1.getCode().getInstructions().getInstructions().stream().filter(i -> i instanceof MappableInstruction).collect(Collectors.toList()); + List l2 = m2.getCode().getInstructions().getInstructions().stream().filter(i -> i instanceof MappableInstruction).collect(Collectors.toList()); + + if (l1.size() != l2.size()) + return false; + + Map map1 = new HashMap<>(), + map2 = new HashMap<>(); + + for (int i = 0; i < l1.size(); ++i) + { + Instruction i1 = l1.get(i); + + Integer c = map1.get(i1.getClass()); + if (c == null) + map1.put(i1.getClass(), 1); + else + map1.put(i1.getClass(), c + 1); + } + + for (int i = 0; i < l2.size(); ++i) + { + Instruction i2 = l2.get(i); + + Integer c = map2.get(i2.getClass()); + if (c == null) + map2.put(i2.getClass(), 1); + else + map2.put(i2.getClass(), c + 1); + } + + return map1.equals(map2); + } + + public static ParallelExecutorMapping map(Method m1, Method m2) + { + ClassGroup group1 = m1.getMethods().getClassFile().getGroup(); + ClassGroup group2 = m2.getMethods().getClassFile().getGroup(); + + Execution e = new Execution(group1); + e.step = true; + Frame frame = new Frame(e, m1); + frame.initialize(); + e.frames.add(frame); + + Execution e2 = new Execution(group2); + e2.step = true; + Frame frame2 = new Frame(e2, m2); + frame2.initialize(); + e2.frames.add(frame2); + + frame.other = frame2; + frame2.other = frame; + + ParallellMappingExecutor parallel = new ParallellMappingExecutor(e, e2); + ParallelExecutorMapping mappings = new ParallelExecutorMapping(); + + while (parallel.step()) + { + // get what each frame is paused/exited on + InstructionContext p1 = parallel.getP1(), p2 = parallel.getP2(); + + assert e.paused; + assert e2.paused; + + System.out.println(p1.getInstruction() + " <-> " + p2.getInstruction()); + + assert p1.getInstruction().getInstructions().getCode().getAttributes().getMethod() == m1; + assert p2.getInstruction().getInstructions().getCode().getAttributes().getMethod() == m2; + + assert p1.getInstruction() instanceof MappableInstruction; + assert p2.getInstruction() instanceof MappableInstruction; + + assert p1.getInstruction().getClass().equals(p2.getInstruction().getClass()); + + MappableInstruction mi1 = (MappableInstruction) p1.getInstruction(); + + mi1.map(mappings, p1, p2); + + e.paused = e2.paused = false; + } + + return mappings; + } +} diff --git a/src/main/java/net/runelite/deob/deobfuscators/rename/ParallelExecutorMapping.java b/src/main/java/net/runelite/deob/deobfuscators/rename/ParallelExecutorMapping.java new file mode 100644 index 0000000000..3d35119933 --- /dev/null +++ b/src/main/java/net/runelite/deob/deobfuscators/rename/ParallelExecutorMapping.java @@ -0,0 +1,24 @@ +package net.runelite.deob.deobfuscators.rename; + +import java.util.HashMap; +import java.util.Map; + +public class ParallelExecutorMapping +{ + private Map map = new HashMap<>(); + + public void map(Object one, Object two) + { + if (one.toString().equals("Z class6.field120")) + { + int i =5; + } + assert !map.containsKey(one) || map.get(one) == two; + map.put(one, two); + } + + public Map getMap() + { + return map; + } +} diff --git a/src/main/java/net/runelite/deob/deobfuscators/rename/Rename2.java b/src/main/java/net/runelite/deob/deobfuscators/rename/Rename2.java index eb30c49237..297a494c99 100644 --- a/src/main/java/net/runelite/deob/deobfuscators/rename/Rename2.java +++ b/src/main/java/net/runelite/deob/deobfuscators/rename/Rename2.java @@ -172,6 +172,8 @@ public class Rename2 v2.is(v1); System.out.println(mname(m) + " is " + mname(opt.get())); + + executeMethod(m, opt.get()); } } @@ -307,6 +309,53 @@ public class Rename2 } } + private void executeNewMethods() + { + for (Vertex v : g1.getVerticies()) + { + Vertex other = v.getOther(); + + if (other == null || !(v.getObject() instanceof Method)) + continue; + + executeMethod((Method) v.getObject(), (Method) other.getObject()); + } + } + + static int count = 0; + private void executeMethod(Method m1, Method m2) + { +// assert (m1.getCode() == null) == (m2.getCode() == null); +// +// if (m1.getCode() == null) +// return; +// + if (!MappingExecutorUtil.isMappable(m1, m2)) + return; + + if (m1.getName().equals("") || m1.getName().equals("")) + return; + + ParallelExecutorMapping mapping = MappingExecutorUtil.map(m1, m2); + System.out.println("EXEC " + count++ + " " + mname(m1) + " " + mname(m2) + " " + mapping); + + for (Entry e : mapping.getMap().entrySet()) + { + if (e.getKey() instanceof Method) + { + Method m = (Method) e.getKey(); + if (m.isStatic()) + continue; + } + Vertex v1 = g1.getVertexFor(e.getKey()), + v2 = g2.getVertexFor(e.getValue()); + + v1.is(v2); + v2.is(v1); + } + // XXX next is use mappings. and then use executeMethod() on more than just the initial method mappings + } + public NameMappings run(ClassGroup one, ClassGroup two) { Execution eone = new Execution(one); @@ -389,6 +438,8 @@ public class Rename2 int after = g1.solved(null); System.out.println("After " + after); + executeNewMethods(); + if (before == after) break; } diff --git a/src/main/java/net/runelite/deob/execution/Frame.java b/src/main/java/net/runelite/deob/execution/Frame.java index 841607b463..760750eb97 100644 --- a/src/main/java/net/runelite/deob/execution/Frame.java +++ b/src/main/java/net/runelite/deob/execution/Frame.java @@ -238,6 +238,9 @@ public class Frame private void processExceptions(Instruction i) { + if (this.execution.step) + return; // no frame.other + Code code = method.getCode(); InstructionContext ictx = instructions.get(instructions.size() - 1); diff --git a/src/test/java/net/runelite/deob/deobfuscators/rename/MapTest.java b/src/test/java/net/runelite/deob/deobfuscators/rename/MapTest.java index e6ec117d2d..e989bc4256 100644 --- a/src/test/java/net/runelite/deob/deobfuscators/rename/MapTest.java +++ b/src/test/java/net/runelite/deob/deobfuscators/rename/MapTest.java @@ -21,52 +21,13 @@ import org.junit.Test; public class MapTest { - public boolean isMappable(Method m1, Method m2) - { - // must have same number of each type - // invokes - - List l1 = m1.getCode().getInstructions().getInstructions().stream().filter(i -> i instanceof MappableInstruction).collect(Collectors.toList()); - List l2 = m2.getCode().getInstructions().getInstructions().stream().filter(i -> i instanceof MappableInstruction).collect(Collectors.toList()); - - if (l1.size() != l2.size()) - return false; - - Map map1 = new HashMap<>(), - map2 = new HashMap<>(); - - for (int i = 0; i < l1.size(); ++i) - { - Instruction i1 = l1.get(i); - - Integer c = map1.get(i1.getClass()); - if (c == null) - map1.put(i1.getClass(), 1); - else - map1.put(i1.getClass(), c + 1); - } - - for (int i = 0; i < l2.size(); ++i) - { - Instruction i2 = l2.get(i); - - Integer c = map2.get(i2.getClass()); - if (c == null) - map2.put(i2.getClass(), 1); - else - map2.put(i2.getClass(), c + 1); - } - - return true; - } - - //@Test + @Test public void testMappable() throws IOException { ClassGroup group1 = JarUtil.loadJar(new File("d:/rs/07/adamin1.jar")); ClassGroup group2 = JarUtil.loadJar(new File("d:/rs/07/adamin2.jar")); - Assert.assertTrue(isMappable( + Assert.assertTrue(MappingExecutorUtil.isMappable( group1.findClass("class99").findMethod("method2220"), group2.findClass("class99").findMethod("method2149") )); @@ -79,47 +40,8 @@ public class MapTest ClassGroup group2 = JarUtil.loadJar(new File("d:/rs/07/adamin2.jar")); Method m1 = group1.findClass("class99").findMethod("method2220"); - Execution e = new Execution(group1); - e.step = true; - Frame frame = new Frame(e, m1); - frame.initialize(); - e.frames.add(frame); - Method m2 = group2.findClass("class99").findMethod("method2149"); - Execution e2 = new Execution(group2); - e2.step = true; - Frame frame2 = new Frame(e2, m2); - frame2.initialize(); - e2.frames.add(frame2); - frame.other = frame2; - frame2.other = frame; - - ParallellMappingExecutor parallel = new ParallellMappingExecutor(e, e2); - - while (parallel.step()) - { - // get what each frame is paused/exited on - InstructionContext p1 = parallel.getP1(), p2 = parallel.getP2(); - - assert e.paused; - assert e2.paused; - - System.out.println(p1.getInstruction() + " <-> " + p2.getInstruction()); - - assert p1.getInstruction().getInstructions().getCode().getAttributes().getMethod() == m1; - assert p2.getInstruction().getInstructions().getCode().getAttributes().getMethod() == m2; - - assert p1.getInstruction() instanceof MappableInstruction; - assert p2.getInstruction() instanceof MappableInstruction; - - assert p1.getInstruction().getClass().equals(p2.getInstruction().getClass()); - - MappableInstruction mi1 = (MappableInstruction) p1.getInstruction(); - - mi1.map(p1, p2); - - e.paused = e2.paused = false; - } + ParallelExecutorMapping mappings = MappingExecutorUtil.map(m1, m2); } }