From 19f2807c8629e320eb455c15d7a2889105f49ac4 Mon Sep 17 00:00:00 2001 From: Adam Date: Sat, 27 Feb 2016 11:38:44 -0500 Subject: [PATCH] Mapping packet handlers works some. Tried to add inlining of results of static methods. Doesnt work well because it inf loops so its disabled usually. I think instead I can track this separately on the stack context and fix resolve() to deal with it. --- .../deob/attributes/code/instructions/If.java | 34 ++++- .../rename/MappingExecutorUtil.java | 136 ++++++++---------- .../deobfuscators/rename/PacketHandler.java | 57 ++++++++ .../rename/ParallelExecutorMapping.java | 20 ++- .../net/runelite/deob/execution/Frame.java | 29 ++-- .../execution/ParallellMappingExecutor.java | 50 ++++++- .../runelite/deob/execution/StackContext.java | 1 + .../deobfuscators/rename/MapStaticTest.java | 104 ++++++++++++-- 8 files changed, 324 insertions(+), 107 deletions(-) create mode 100644 src/main/java/net/runelite/deob/deobfuscators/rename/PacketHandler.java diff --git a/src/main/java/net/runelite/deob/attributes/code/instructions/If.java b/src/main/java/net/runelite/deob/attributes/code/instructions/If.java index 42b952cc1f..62dc6c7ce1 100644 --- a/src/main/java/net/runelite/deob/attributes/code/instructions/If.java +++ b/src/main/java/net/runelite/deob/attributes/code/instructions/If.java @@ -18,7 +18,9 @@ import java.util.List; import net.runelite.deob.Field; import net.runelite.deob.attributes.code.instruction.types.GetFieldInstruction; import net.runelite.deob.attributes.code.instruction.types.MappableInstruction; +import net.runelite.deob.attributes.code.instruction.types.PushConstantInstruction; import net.runelite.deob.deobfuscators.rename.MappingExecutorUtil; +import net.runelite.deob.deobfuscators.rename.PacketHandler; import net.runelite.deob.deobfuscators.rename.ParallelExecutorMapping; import net.runelite.deob.execution.Execution; @@ -196,8 +198,16 @@ public abstract class If extends Instruction implements JumpingInstruction, Comp if (f1.packetHandler && f2.packetHandler) { - mapping.packetHandler1.add(this); - mapping.packetHandler2.add((If) other.getInstruction()); + int pc1 = this.getConstantInstruction(ctx), + pc2 = this.getConstantInstruction(other); + + assert (pc1 != -1) == (pc2 != -1); + + if (pc1 == -1 && pc2 == -1) + return; + + mapping.packetHandler1.add(new PacketHandler(this, pc1)); + mapping.packetHandler2.add(new PacketHandler((If) other.getInstruction(), pc2)); } } @@ -223,6 +233,26 @@ public abstract class If extends Instruction implements JumpingInstruction, Comp return gfi.getMyField(); } + + private Integer getConstantInstruction(InstructionContext ctx) + { + PushConstantInstruction gfi = null; + + for (StackContext sctx : ctx.getPops()) + { + InstructionContext base = MappingExecutorUtil.resolve(sctx.getPushed(), sctx); + + if (base.getInstruction() instanceof PushConstantInstruction) + { + if (gfi != null) + return null; + + gfi = (PushConstantInstruction) base.getInstruction(); + } + } + + return (Integer) gfi.getConstant().getObject(); + } protected boolean isSameField(InstructionContext thisIc, InstructionContext otherIc) { diff --git a/src/main/java/net/runelite/deob/deobfuscators/rename/MappingExecutorUtil.java b/src/main/java/net/runelite/deob/deobfuscators/rename/MappingExecutorUtil.java index 88d26709bd..6171018984 100644 --- a/src/main/java/net/runelite/deob/deobfuscators/rename/MappingExecutorUtil.java +++ b/src/main/java/net/runelite/deob/deobfuscators/rename/MappingExecutorUtil.java @@ -90,9 +90,6 @@ public class MappingExecutorUtil frame.other = frame2; frame2.other = frame; - MethodContext ctx1 = frame.getMethodCtx(), - ctx2 = frame2.getMethodCtx(); - ParallellMappingExecutor parallel = new ParallellMappingExecutor(e, e2); ParallelExecutorMapping mappings = new ParallelExecutorMapping(m1.getMethods().getClassFile().getGroup(), m2.getMethods().getClassFile().getGroup()); @@ -128,96 +125,79 @@ public class MappingExecutorUtil p2.getFrame().stop(); e.paused = e2.paused = false; continue; -// if (!hit) -// { -// hit = true; -// -// throw new MappingException(); -// } -// -// System.out.println("ERROR mapping " + p1 + " to " + p2); -// -// // methods don't map. find common static method and back out. -// -// Frame c1 = p1.getFrame(), c2 = p2.getFrame(); -// -// while (c1 != null && c1.otherStatic == null) -// c1 = c1.returnTo; -// -// while (c2 != null && c2.otherStatic == null) -// c2 = c2.returnTo; -// -// // otherStatic would point to the original frame of the method, which the other might not be. we don't -// // care just compare the method. -// if (c1 == null || c2 == null || c1.otherStatic.getMethod() != c2.getMethod() || c2.otherStatic.getMethod() != c1.getMethod()) -// { -// throw new MappingException(); -// } -// -// // c1/c2 are top frames of static methods that we can't map. -// // return out of frames -// c1 = c1.returnTo; -// c2 = c2.returnTo; -// -// if (c1 == null || c2 == null) -// throw new MappingException(); -// -// // Back execution out to c1 and c2. -// // When something is stepped into, the calling frame is removed. -// // Remove all frames from the respective method, add frame from good method to continue -// parallel.removeFramesFromMethod(p1.getFrame().getMethod()); -// parallel.removeFramesFromMethod(p2.getFrame().getMethod()); -// -// assert c1.other == null; -// assert c2.other == null; -// -// c1.other = c2; -// c2.other = c1; -// -// parallel.addFrame(c1, c2); -// continue; } -// try -// { mi1.map(mappings, p1, p2); -// } -// catch (Throwable ex) -// { -// p1.getFrame().stop(); -// p2.getFrame().stop(); -// ex.printStackTrace(); -// } - e.paused = e2.paused = false; } -// if (mappings.getMap().isEmpty() == false) -// { -// checkReturns(m1, ctx1); -// } - return mappings; } - - private static boolean checkReturns(Method method, MethodContext ctx) + + public static ParallelExecutorMapping mapFrame(ClassGroup group1, ClassGroup group2, Instruction i1, Instruction i2) { - List ins = method.getCode().getInstructions().getInstructions().stream().filter(i -> i instanceof ReturnInstruction).collect(Collectors.toList()); - List exc = ctx.instructions.stream().map(i -> i.getInstruction()).collect(Collectors.toList()); - - for (Instruction i : ins) + Execution e = new Execution(group1); + e.step = true; + Frame frame = new Frame(e, i1.getInstructions().getCode().getAttributes().getMethod(), i1); + //frame.initialize(); + e.frames.add(frame); + + Execution e2 = new Execution(group2); + e2.step = true; + //Frame frame2 = new Frame(e2, m2); + Frame frame2 = new Frame(e2, i2.getInstructions().getCode().getAttributes().getMethod(), i2); + //frame2.initialize(); + e2.frames.add(frame2); + + frame.other = frame2; + frame2.other = frame; + + ParallellMappingExecutor parallel = new ParallellMappingExecutor(e, e2); + ParallelExecutorMapping mappings = new ParallelExecutorMapping(group1, group2); + + //mappings.m1 = m1; + //mappings.m2 = m2; + + parallel.mappings = mappings; + + int compare = 0; + while (parallel.step()) { - if (!exc.contains(i)) + // get what each frame is paused/exited on + InstructionContext p1 = parallel.getP1(), p2 = parallel.getP2(); + + assert e.paused; + assert e2.paused; + ++compare; + + //System.out.println(p1.getInstruction() + " <-> " + p2.getInstruction()); + + //assert p1.getInstruction().getInstructions().getCode().getAttributes().getMethod() == m1; + //assert p2.getInstruction().getInstructions().getCode().getAttributes().getMethod() == m2; + + assert p1.getInstruction() instanceof MappableInstruction; + assert p2.getInstruction() instanceof MappableInstruction; + + MappableInstruction mi1 = (MappableInstruction) p1.getInstruction(), + mi2 = (MappableInstruction) p2.getInstruction(); + + if (!mi1.isSame(p1, p2)) { - return false; + mappings.crashed = true; + mi1.isSame(p1, p2); + p1.getFrame().stop(); + p2.getFrame().stop(); + e.paused = e2.paused = false; + continue; } + + mi1.map(mappings, p1, p2); + e.paused = e2.paused = false; } - - return true; + + return mappings; } - //private static boolean containsMappableInstruction - public static boolean isMappable(InvokeInstruction ii) { String className; diff --git a/src/main/java/net/runelite/deob/deobfuscators/rename/PacketHandler.java b/src/main/java/net/runelite/deob/deobfuscators/rename/PacketHandler.java new file mode 100644 index 0000000000..186d561844 --- /dev/null +++ b/src/main/java/net/runelite/deob/deobfuscators/rename/PacketHandler.java @@ -0,0 +1,57 @@ +package net.runelite.deob.deobfuscators.rename; + +import java.util.List; +import net.runelite.deob.attributes.code.Instruction; +import net.runelite.deob.attributes.code.instructions.If; +import net.runelite.deob.attributes.code.instructions.IfICmpEq; +import net.runelite.deob.attributes.code.instructions.IfICmpNe; + +public class PacketHandler +{ + private final If branchInstruction; + private final int packetId; + + public PacketHandler(If branchInstruction, int packetId) + { + this.branchInstruction = branchInstruction; + this.packetId = packetId; + } + + public If getBranchInstruction() + { + return branchInstruction; + } + + public int getPacketId() + { + return packetId; + } + + @Override + public String toString() + { + return "PacketHandler{" + "packetId=" + packetId + '}'; + } + + public Instruction getFirstInsOfHandler() + { + if (branchInstruction instanceof IfICmpNe) + { + List ins = branchInstruction.getInstructions().getInstructions(); + int idx = ins.indexOf(branchInstruction); + assert idx != -1; + return ins.get(idx + 1); + } + else if (branchInstruction instanceof IfICmpEq) + { + List jumps = branchInstruction.getJumps(); + assert jumps.size() == 1; + return jumps.get(0); + } + else + { + assert false; + return null; + } + } +} diff --git a/src/main/java/net/runelite/deob/deobfuscators/rename/ParallelExecutorMapping.java b/src/main/java/net/runelite/deob/deobfuscators/rename/ParallelExecutorMapping.java index 9175702563..faf34308be 100644 --- a/src/main/java/net/runelite/deob/deobfuscators/rename/ParallelExecutorMapping.java +++ b/src/main/java/net/runelite/deob/deobfuscators/rename/ParallelExecutorMapping.java @@ -16,8 +16,8 @@ public class ParallelExecutorMapping //private List order = new ArrayList<>(); public Method m1, m2; public boolean crashed; - public List packetHandler1 = new ArrayList<>(); - public List packetHandler2 = new ArrayList<>(); + public List packetHandler1 = new ArrayList<>(); + public List packetHandler2 = new ArrayList<>(); public ParallelExecutorMapping(ClassGroup group, ClassGroup group2) { @@ -73,4 +73,20 @@ public class ParallelExecutorMapping else assert false; } + + public PacketHandler findPacketHandler1(int id) + { + for (PacketHandler p : this.packetHandler1) + if (p.getPacketId() == id) + return p; + return null; + } + + public PacketHandler findPacketHandler2(int id) + { + for (PacketHandler p : this.packetHandler2) + if (p.getPacketId() == id) + return p; + return null; + } } diff --git a/src/main/java/net/runelite/deob/execution/Frame.java b/src/main/java/net/runelite/deob/execution/Frame.java index 09ff882a32..6cc119cd14 100644 --- a/src/main/java/net/runelite/deob/execution/Frame.java +++ b/src/main/java/net/runelite/deob/execution/Frame.java @@ -53,6 +53,22 @@ public class Frame nonStatic = method; } + public Frame(Execution execution, Method method, Instruction i) + { + this.execution = execution; + this.method = method; + + Code code = method.getCode(); + + stack = new Stack(code.getMaxStack()); + variables = new Variables(code.getMaxLocals()); + + ctx = new MethodContext(execution); + nonStatic = method; + + cur = i; + } + @Override public String toString() { @@ -113,8 +129,7 @@ public class Frame Code code = method.getCode(); cur = code.getInstructions().getInstructions().get(0); } - - static List ffs = new ArrayList(); + protected Frame(Frame other) { iscopy=true; @@ -127,21 +142,11 @@ public class Frame this.ctx = other.ctx; this.nonStatic = other.nonStatic; this.caller = other.caller; - ffs.add(this); - if (ffs.size() == 10) - { - for (Frame f : ffs) - { - System.out.println(f.method); - } - int i = 5; - } if (other.returnTo != null) { this.returnTo = new Frame(other.returnTo); this.returnTo.instructions.addAll(other.returnTo.instructions); } - ffs.remove(this); this.created = other.created; this.forking = other.forking; this.otherStatic = other.otherStatic; diff --git a/src/main/java/net/runelite/deob/execution/ParallellMappingExecutor.java b/src/main/java/net/runelite/deob/execution/ParallellMappingExecutor.java index c08ef61465..3742ade3c0 100644 --- a/src/main/java/net/runelite/deob/execution/ParallellMappingExecutor.java +++ b/src/main/java/net/runelite/deob/execution/ParallellMappingExecutor.java @@ -1,5 +1,6 @@ package net.runelite.deob.execution; +import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -7,6 +8,7 @@ import java.util.stream.Collectors; import net.runelite.deob.Method; import net.runelite.deob.attributes.code.instruction.types.ReturnInstruction; import net.runelite.deob.attributes.code.instructions.InvokeStatic; +import net.runelite.deob.attributes.code.instructions.Return; import net.runelite.deob.deobfuscators.rename.MappingExecutorUtil; import net.runelite.deob.deobfuscators.rename.ParallelExecutorMapping; @@ -294,7 +296,9 @@ public class ParallellMappingExecutor return f2; } - + + public static boolean enable = false; + public static List returnStacks = new ArrayList<>(); private Frame popStack(Frame f) { Execution e = f.getExecution(); @@ -308,10 +312,54 @@ public class ParallellMappingExecutor InstructionContext i = f.getInstructions().get(f.getInstructions().size() - 1); if (!(i.getInstruction() instanceof ReturnInstruction)) return f; + + StackContext returnValue = null; + if (enable&& i.getInstruction() instanceof Return) + { + assert i.getPops().size() == 1; + + returnValue = i.getPops().get(0); + } Frame r = popStackForce(f); f.returnTo = null; + + // last ins must be an invokestatic + InstructionContext i2 = r.getInstructions().get(r.getInstructions().size() - 1); + assert i2.getInstruction() instanceof InvokeStatic; + if (returnValue != null) + { + // if the function retunred something, we must have pushed + assert i2.getPushes().size() == 1; + + StackContext invokePushed = i2.getPushes().get(0); + + if (invokePushed.getPushed().getInstruction() != i2.getInstruction()) + //if (!(invokePushed.getPushed().getInstruction() instanceof InvokeStatic)) + { + return r; + } + + //returnStacks.add(invokePushed); + returnStacks.add(returnValue); + boolean b = returnStacks.contains(invokePushed); + assert invokePushed.getPopped().isEmpty(); + + // replace invokePushed with returnValue? + i2.getPushes().remove(invokePushed); + i2.getPushes().add(returnValue); + + //invokePushed.setpushed = null + + Stack stack = r.getStack(); + StackContext s = stack.pop(); + assert s == invokePushed; + stack.push(returnValue); + + //assert invokePushed.getPushed().getPushes().contains(invokePushed); + //invokePushed.getpu + } // step return frame //r.execute(); diff --git a/src/main/java/net/runelite/deob/execution/StackContext.java b/src/main/java/net/runelite/deob/execution/StackContext.java index def515a237..7e5bb25c2d 100644 --- a/src/main/java/net/runelite/deob/execution/StackContext.java +++ b/src/main/java/net/runelite/deob/execution/StackContext.java @@ -47,6 +47,7 @@ public class StackContext public void addPopped(InstructionContext popped) { +// assert ParallellMappingExecutor.returnStacks.contains(this) == false; if (!this.poppeds.contains(popped)) this.poppeds.add(popped); } diff --git a/src/test/java/net/runelite/deob/deobfuscators/rename/MapStaticTest.java b/src/test/java/net/runelite/deob/deobfuscators/rename/MapStaticTest.java index 7fe47d7cc3..d08aacaa24 100644 --- a/src/test/java/net/runelite/deob/deobfuscators/rename/MapStaticTest.java +++ b/src/test/java/net/runelite/deob/deobfuscators/rename/MapStaticTest.java @@ -20,6 +20,12 @@ import net.runelite.deob.Method; import net.runelite.deob.attributes.Annotations; import net.runelite.deob.attributes.AttributeType; import net.runelite.deob.attributes.annotation.Annotation; +import net.runelite.deob.attributes.code.Instruction; +import net.runelite.deob.attributes.code.instructions.If; +import net.runelite.deob.attributes.code.instructions.IfICmpEq; +import net.runelite.deob.attributes.code.instructions.IfICmpNe; +import net.runelite.deob.execution.Execution; +import net.runelite.deob.execution.Frame; import net.runelite.deob.execution.ParallellMappingExecutor; import net.runelite.deob.signature.Type; import net.runelite.deob.util.JarUtil; @@ -93,8 +99,8 @@ public class MapStaticTest ClassGroup group1 = JarUtil.loadJar(new File(JAR1)); ClassGroup group2 = JarUtil.loadJar(new File(JAR2)); - Method m1 = group1.findClass("class222").findMethod("method4107"); - Method m2 = group2.findClass("class222").findMethod("method3980"); + Method m1 = group1.findClass("class6").findMethod("method112"); + Method m2 = group2.findClass("class20").findMethod("method551"); ParallelExecutorMapping mappings = MappingExecutorUtil.map(m1, m2); @@ -104,7 +110,7 @@ public class MapStaticTest Object value = mappings.get(o); System.out.println(o + " <-> " + value); } - System.out.println("END OF MAPPINGS " + mappings.getMap().size()); + System.out.println("END OF MAPPINGS " + mappings.getMap().size() + " " + mappings.crashed); // I think because this is an array store //Object other = mappings.get(group1.findClass("class136").findField("field2098")); @@ -211,7 +217,7 @@ public class MapStaticTest // assert m1.getPoolMethod().equals(m2.getPoolMethod()); // // HashMap all = new HashMap(); -// map(all, pmes, m1, m2); +// map(all, pmes, m1, m2);/fil // } ParallelExecutorMapping finalm = new ParallelExecutorMapping(group1, group2); @@ -221,6 +227,7 @@ public class MapStaticTest finalm.merge(testStaticMapperMap(group1, group2)); finalm.merge(testMapperMap(group1, group2)); + finalm.merge(this.testPackets(group1, group2)); for (int i = -1; i < 250; ++i) { @@ -465,6 +472,8 @@ public class MapStaticTest rename.run(); ParallelExecutorMapping mapping = rename.getMapping(); + + mapping.merge(this.testPackets(group1, group2)); summary(rename.getMapping(), group1); @@ -514,11 +523,11 @@ public class MapStaticTest return list; } - @Test - public void testPackets() throws IOException + //@Test + public ParallelExecutorMapping testPackets(ClassGroup group1, ClassGroup group2) throws IOException { - ClassGroup group1 = JarUtil.loadJar(new File(JAR1)); - ClassGroup group2 = JarUtil.loadJar(new File(JAR2)); + //ClassGroup group1 = JarUtil.loadJar(new File(JAR1)); + //ClassGroup group2 = JarUtil.loadJar(new File(JAR2)); group1.findClass("client").findField("field446").packetHandler = true; group2.findClass("client").findField("field324").packetHandler = true; @@ -537,9 +546,80 @@ public class MapStaticTest System.out.println("END OF MAPPINGS " + mappings.getMap().size()); System.out.println(mappings.packetHandler1.size() + " vs " + mappings.packetHandler2.size() + " handlers"); - - // I think because this is an array store - //Object other = mappings.get(group1.findClass("class136").findField("field2098")); - //Assert.assertNotNull(other); + + assert mappings.packetHandler1.size() == mappings.packetHandler2.size(); + + ParallellMappingExecutor.enable = true; + ParallelExecutorMapping all = new ParallelExecutorMapping(group1, group2); + + for (int i = 0; i < mappings.packetHandler1.size(); ++i) + { + PacketHandler if1 = mappings.packetHandler1.get(i); + + PacketHandler highestHandler = null; + ParallelExecutorMapping highest = null; + + for (int j = 0; j < mappings.packetHandler2.size(); ++j) + { + PacketHandler if2 = mappings.packetHandler2.get(j); + + Instruction i1 = if1.getFirstInsOfHandler(), + i2 = if2.getFirstInsOfHandler(); + + ParallelExecutorMapping mapping = MappingExecutorUtil.mapFrame(group1, group2, i1, i2); + + if (mapping.getMap().isEmpty()) + continue; + + if (highest == null || mapping.getMap().size() > highest.getMap().size()) + { + highest = mapping; + highestHandler = if2; + } + + +// Execution e1 = new Execution(group1); +// Execution e2 = new Execution(group2); +// +// Frame f1 = new Frame(e1, i1.getInstructions().getCode().getAttributes().getMethod(), i1); +// Frame f2 = new Frame(e2, i2.getInstructions().getCode().getAttributes().getMethod(), i2); + + //e1.frames.add(f1); + //e2.frames.add(f2); + } + + System.out.println(if1 + " <-> " + highestHandler + " <-> " + highest.getMap().size() + " " + highest.crashed); + all.merge(highest); + } + + ParallellMappingExecutor.enable = false; + return all; + } + + private static int handlers[][] = { + { 74, 187 } + }; + + @Test + public void testPacket() throws IOException + { + ClassGroup group1 = JarUtil.loadJar(new File(JAR1)); + ClassGroup group2 = JarUtil.loadJar(new File(JAR2)); + + group1.findClass("client").findField("field446").packetHandler = true; + group2.findClass("client").findField("field324").packetHandler = true; + + Method m1 = group1.findClass("client").findMethod("vmethod3096"); + Method m2 = group2.findClass("client").findMethod("vmethod2975"); + + ParallelExecutorMapping mappings = MappingExecutorUtil.map(m1, m2); + + // var55 = class17.method214(); vs var107 = class25.field625[++class25.field624 - 1]; + PacketHandler h1 = mappings.findPacketHandler1(127); + PacketHandler h2 = mappings.findPacketHandler2(160); + + ParallelExecutorMapping mapping = MappingExecutorUtil.mapFrame(group1, group2, h1.getFirstInsOfHandler(), h2.getFirstInsOfHandler()); + + System.out.println(h1 + " <-> " + h2 + " <-> " + mapping.getMap().size() + " " + mapping.crashed); } }