ra/runelite
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

playing with less memory

Browse Source
This commit is contained in:
Adam
2016-04-03 15:53:13 -04:00
parent c9bfbb1352
commit d6c7d7dc8a
6 changed files with 415 additions and 181 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
src/main/java/net/runelite/asm/execution/Execution.java
View File

@@ -17,14 +17,14 @@ import org.apache.commons.collections4.map.MultiValueMap;
public class Execution
{
private ClassGroup group;
public List<Frame> frames = new LinkedList<>(),
processedFrames = new LinkedList<>();
public List<Frame> frames = new LinkedList<>();
public Set<Method> methods = new HashSet<>(); // all methods
public Set<Instruction> executed = new HashSet<>(); // executed instructions
private MultiValueMap<InstructionContext, Method> invokes = new MultiValueMap<>();
public MultiValueMap<Instruction, InstructionContext> contexts = new MultiValueMap<>();
private MultiValueMap<WeakInstructionContext, Method> invokes = new MultiValueMap<>();
public MultiValueMap<Instruction, InstructionContext> contexts = new MultiValueMap<>(); // XXX this should move to method ctx probably
public boolean paused;
public boolean step = false;
private List<ExecutionVisitor> visitors = new ArrayList<>();
public Execution(ClassGroup group)
{
@@ -76,11 +76,11 @@ public class Execution
public boolean hasInvoked(InstructionContext from, Method to)
{
Collection<Method> methods = invokes.getCollection(from);
Collection<Method> methods = invokes.getCollection(from.toWeak());
if (methods != null && methods.contains(to))
return true;
invokes.put(from, to);
invokes.put(from.toWeak(), to);
return false;
}
@@ -128,7 +128,6 @@ public class Execution
assert !frame.isExecuting();
frames.remove(frame);
processedFrames.add(frame);
}
System.out.println("Processed " + fcount + " frames");
@@ -138,4 +137,14 @@ public class Execution
{
return contexts.getCollection(i);
}
public void addExecutionVisitor(ExecutionVisitor ev)
{
this.visitors.add(ev);
}
public void accept(InstructionContext ic)
{
visitors.forEach(v -> v.visit(ic));
}
}

6
src/main/java/net/runelite/asm/execution/ExecutionVisitor.java Normal file
View File

@@ -0,0 +1,6 @@
package net.runelite.asm.execution;
public interface ExecutionVisitor
{
void visit(InstructionContext context);
}

2
src/main/java/net/runelite/asm/execution/Frame.java
View File

@@ -239,6 +239,8 @@ public class Frame
execution.contexts.put(oldCur, ictx);
execution.executed.add(oldCur);
execution.accept(ictx);
processExceptions(ictx);

16
src/main/java/net/runelite/asm/execution/InstructionContext.java
View File

@@ -209,4 +209,20 @@ public class InstructionContext
return ctx;
}
public WeakInstructionContext toWeak()
{
WeakInstructionContext wic = new WeakInstructionContext(this.getInstruction());
for (StackContext sctx : stack.getStack())
{
if (sctx == null)
break;
InstructionContext i = sctx.getPushed();
wic.addStack(i.getInstruction());
}
return wic;
}
}

58
src/main/java/net/runelite/asm/execution/WeakInstructionContext.java Normal file
View File

@@ -0,0 +1,58 @@
package net.runelite.asm.execution;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import net.runelite.asm.attributes.code.Instruction;
public class WeakInstructionContext
{
private Instruction ins;
private List<Instruction> stack = new ArrayList<>();
public WeakInstructionContext(Instruction ins)
{
this.ins = ins;
}
public void addStack(Instruction i)
{
stack.add(i);
}
@Override
public int hashCode()
{
int hash = 3;
hash = 37 * hash + Objects.hashCode(this.ins);
hash = 37 * hash + Objects.hashCode(this.stack);
return hash;
}
@Override
public boolean equals(Object obj)
{
if (this == obj)
{
return true;
}
if (obj == null)
{
return false;
}
if (getClass() != obj.getClass())
{
return false;
}
final WeakInstructionContext other = (WeakInstructionContext) obj;
if (!Objects.equals(this.ins, other.ins))
{
return false;
}
if (!Objects.equals(this.stack, other.stack))
{
return false;
}
return true;
}
}

491
src/main/java/net/runelite/deob/deobfuscators/ConstantParameter.java
View File

@@ -39,6 +39,8 @@ class ConstantMethodParameter
public int paramIndex;
public int lvtIndex;
public Object value;
List<InstructionContext> operations = new ArrayList<>();
Boolean result;
@Override
public int hashCode()
@@ -88,6 +90,8 @@ class MethodGroup
public class ConstantParameter implements Deobfuscator
{
private List<ConstantMethodParameter> parameters = new ArrayList<>();
private MultiValueMap<Method, ConstantMethodParameter> mparams = new MultiValueMap<>();
private MultiValueMap<Method, Integer> nonconst = new MultiValueMap<>(); // methods with non const parameters
private List<MethodGroup> methodGroups = new ArrayList<>();
@@ -105,8 +109,10 @@ public class ConstantParameter implements Deobfuscator
}
}
private void findConstantParameter(Execution execution, List<Method> methods, InstructionContext invokeCtx)
private List<ConstantMethodParameter> findConstantParameter(List<Method> methods, InstructionContext invokeCtx)
{
List<ConstantMethodParameter> list = new ArrayList<>();
checkMethodsAreConsistent(methods);
Method method = methods.get(0); // all methods must have the same signature etc
@@ -149,6 +155,10 @@ public class ConstantParameter implements Deobfuscator
continue outer;
parameters.add(cmp);
for (Method m : methods)
mparams.put(m, cmp);
list.add(cmp);
}
else
{
@@ -162,28 +172,152 @@ public class ConstantParameter implements Deobfuscator
if (c.methods.equals(methods) && c.lvtIndex == lvtOffset)
{
it.remove();
list.remove(c);
}
}
}
}
return list;
}
private void findParameters(Execution execution)
private void findParameters(InstructionContext ins)
{
for (Frame frame : execution.processedFrames)
for (InstructionContext ins : frame.getInstructions())
{
if (!(ins.getInstruction() instanceof InvokeInstruction))
continue;
List<Method> methods = ((InvokeInstruction) ins.getInstruction()).getMethods();
if (methods.isEmpty())
continue;
findConstantParameter(execution, methods, ins);
}
if (!(ins.getInstruction() instanceof InvokeInstruction))
return;
List<Method> methods = ((InvokeInstruction) ins.getInstruction()).getMethods();
if (methods.isEmpty())
return;
findConstantParameter(methods, ins);
//findDeadParameters(ins, c);
}
private List<ConstantMethodParameter> findParametersForMethod(Method m)
{
Collection c = mparams.getCollection(m);
if (c == null) return new ArrayList();
return new ArrayList(c);
// List<ConstantMethodParameter> list = new ArrayList<>();
// for (ConstantMethodParameter c : parameters)
// if (c.methods.contains(m))
// list.add(c);
// return list;
}
private void findDeadParameters(InstructionContext ins)
{
List<ConstantMethodParameter> parameters = this.findParametersForMethod(ins.getFrame().getMethod());
for (ConstantMethodParameter parameter : parameters)
{
int lvtIndex = parameter.lvtIndex;
Object value = parameter.value;
if (ins.getInstruction() instanceof LVTInstruction)
{
LVTInstruction lvt = (LVTInstruction) ins.getInstruction();
if (lvt.getVariableIndex() == lvtIndex && lvt.store())
{
this.parameters.remove(parameter);
continue; // parameter is used
}
}
if (!(ins.getInstruction() instanceof ComparisonInstruction))
{
continue;
}
// assume that this will always be variable index #paramIndex comp with a constant.
ComparisonInstruction comp = (ComparisonInstruction) ins.getInstruction();
StackContext one, two = null;
if (comp instanceof If0)
{
one = ins.getPops().get(0);
}
else if (comp instanceof If)
{
one = ins.getPops().get(0);
two = ins.getPops().get(1);
}
else
{
throw new RuntimeException("Unknown comp ins");
}
// find if one is a lvt ins
LVTInstruction lvt = null;
StackContext other = null;
if (one.getPushed().getInstruction() instanceof LVTInstruction)
{
lvt = (LVTInstruction) one.getPushed().getInstruction();
other = two;
}
else if (two != null && two.getPushed().getInstruction() instanceof LVTInstruction)
{
lvt = (LVTInstruction) two.getPushed().getInstruction();
other = one;
}
assert lvt == null || !lvt.store();
if (lvt == null || lvt.getVariableIndex() != lvtIndex)
{
continue;
}
Object otherValue = null;
if (two != null) // two is null for if0
{
if (!(other.getPushed().getInstruction() instanceof PushConstantInstruction))
{
continue;
}
PushConstantInstruction pc = (PushConstantInstruction) other.getPushed().getInstruction();
otherValue = pc.getConstant().getObject();
}
// the result of the comparison doesn't matter, only that it always goes the same direction for every invocation
boolean result = doLogicalComparison(value, comp, otherValue);
if (parameter.result != null && parameter.result != result)
{
this.parameters.remove(parameter);
}
else
{
parameter.operations.add(ins);
parameter.result = result;
}
//
// Boolean b = deadOps.get(ins.getInstruction());
// if (b != null)
// {
// if (b != result)
// {
// //deadOps.remove(ins.getInstruction());
// this.parameters.remove(parameter);
// //invalidDeadOps.add(ins.getInstruction());
// }
// }
// else
// {
// deadOps.put(ins.getInstruction(), result);
// }
}
}
//private Map<Instruction, Boolean> deadOps = new HashMap<>();
// private Set<Instruction> invalidDeadOps = new HashSet<>();
private boolean doLogicalComparison(Object value, ComparisonInstruction comparison, Object otherValue)
{
Instruction ins = (Instruction) comparison;
@@ -252,93 +386,93 @@ public class ConstantParameter implements Deobfuscator
}
// find all comparisons of lvtIndex in method and record branch taken
private List<LogicallyDeadOp> isLogicallyDead(Execution execution, Method method, int lvtIndex, Object value)
{
List<LogicallyDeadOp> ops = new ArrayList<>();
for (Frame frame : execution.processedFrames)
{
if (frame.getMethod() != method)
continue;
for (InstructionContext ins : frame.getInstructions())
{
if (ins.getInstruction() instanceof LVTInstruction)
{
LVTInstruction lvt = (LVTInstruction) ins.getInstruction();
if (lvt.getVariableIndex() == lvtIndex && lvt.store())
{
return null;
}
}
if (!(ins.getInstruction() instanceof ComparisonInstruction))
continue;
// assume that this will always be variable index #paramIndex comp with a constant.
ComparisonInstruction comp = (ComparisonInstruction) ins.getInstruction();
StackContext one, two = null;
if (comp instanceof If0)
{
one = ins.getPops().get(0);
}
else if (comp instanceof If)
{
one = ins.getPops().get(0);
two = ins.getPops().get(1);
}
else
{
throw new RuntimeException("Unknown comp ins");
}
// find if one is a lvt ins
LVTInstruction lvt = null;
StackContext other = null;
if (one.getPushed().getInstruction() instanceof LVTInstruction)
{
lvt = (LVTInstruction) one.getPushed().getInstruction();
other = two;
}
else if (two != null && two.getPushed().getInstruction() instanceof LVTInstruction)
{
lvt = (LVTInstruction) two.getPushed().getInstruction();
other = one;
}
assert lvt == null || !lvt.store();
if (lvt == null || lvt.getVariableIndex() != lvtIndex)
continue;
Object otherValue = null;
if (two != null) // two is null for if0
{
if (!(other.getPushed().getInstruction() instanceof PushConstantInstruction))
continue;
PushConstantInstruction pc = (PushConstantInstruction) other.getPushed().getInstruction();
otherValue = pc.getConstant().getObject();
}
// the result of the comparison doesn't matter, only that it always goes the same direction for every invocation
boolean result = doLogicalComparison(value, comp, otherValue);
LogicallyDeadOp deadOp = new LogicallyDeadOp();
deadOp.compCtx = ins;
deadOp.branch = result;
ops.add(deadOp);
}
}
return ops;
}
// private List<LogicallyDeadOp> isLogicallyDead(Execution execution, Method method, int lvtIndex, Object value)
// {
// List<LogicallyDeadOp> ops = new ArrayList<>();
//
// for (Frame frame : execution.processedFrames)
// {
// if (frame.getMethod() != method)
// continue;
//
// for (InstructionContext ins : frame.getInstructions())
// {
// if (ins.getInstruction() instanceof LVTInstruction)
// {
// LVTInstruction lvt = (LVTInstruction) ins.getInstruction();
//
// if (lvt.getVariableIndex() == lvtIndex && lvt.store())
// {
// return null;
// }
// }
//
// if (!(ins.getInstruction() instanceof ComparisonInstruction))
// continue;
//
// // assume that this will always be variable index #paramIndex comp with a constant.
//
// ComparisonInstruction comp = (ComparisonInstruction) ins.getInstruction();
//
// StackContext one, two = null;
//
// if (comp instanceof If0)
// {
// one = ins.getPops().get(0);
// }
// else if (comp instanceof If)
// {
// one = ins.getPops().get(0);
// two = ins.getPops().get(1);
// }
// else
// {
// throw new RuntimeException("Unknown comp ins");
// }
//
// // find if one is a lvt ins
// LVTInstruction lvt = null;
// StackContext other = null;
//
// if (one.getPushed().getInstruction() instanceof LVTInstruction)
// {
// lvt = (LVTInstruction) one.getPushed().getInstruction();
// other = two;
// }
// else if (two != null && two.getPushed().getInstruction() instanceof LVTInstruction)
// {
// lvt = (LVTInstruction) two.getPushed().getInstruction();
// other = one;
// }
//
// assert lvt == null || !lvt.store();
//
// if (lvt == null || lvt.getVariableIndex() != lvtIndex)
// continue;
//
// Object otherValue = null;
//
// if (two != null) // two is null for if0
// {
// if (!(other.getPushed().getInstruction() instanceof PushConstantInstruction))
// continue;
//
// PushConstantInstruction pc = (PushConstantInstruction) other.getPushed().getInstruction();
// otherValue = pc.getConstant().getObject();
// }
//
// // the result of the comparison doesn't matter, only that it always goes the same direction for every invocation
// boolean result = doLogicalComparison(value, comp, otherValue);
//
// LogicallyDeadOp deadOp = new LogicallyDeadOp();
// deadOp.compCtx = ins;
// deadOp.branch = result;
// ops.add(deadOp);
// }
// }
//
// return ops;
// }
private static class MethodLvtPair
{
@@ -368,20 +502,25 @@ public class ConstantParameter implements Deobfuscator
@Override
public boolean equals(Object obj)
{
if (obj == null) {
if (obj == null)
{
return false;
}
if (getClass() != obj.getClass()) {
if (getClass() != obj.getClass())
{
return false;
}
final MethodLvtPair other = (MethodLvtPair) obj;
if (!Objects.equals(this.method, other.method)) {
if (!Objects.equals(this.method, other.method))
{
return false;
}
if (this.lvtIndex != other.lvtIndex) {
if (this.lvtIndex != other.lvtIndex)
{
return false;
}
if (this.paramIndex != other.paramIndex) {
if (this.paramIndex != other.paramIndex)
{
return false;
}
return true;
@@ -389,71 +528,67 @@ public class ConstantParameter implements Deobfuscator
}
private Map<MethodLvtPair, List<LogicallyDeadOp> > deadops = new HashMap<>();
private Set<MethodLvtPair> invalidDeadops = new HashSet<>();
//
// private Map<MethodLvtPair, List<LogicallyDeadOp> > deadops = new HashMap<>();
// private Set<MethodLvtPair> invalidDeadops = new HashSet<>();
// check every method parameter that we've identified as being passed constants to see if it's logically dead
private void findLogicallyDeadOperations(Execution execution)
{
outer:
for (ConstantMethodParameter cmp : parameters)
{
for (Method method : cmp.methods)
{
MethodLvtPair pair = new MethodLvtPair(method, cmp.lvtIndex, cmp.paramIndex, cmp.value);
if (invalidDeadops.contains(pair))
continue;
// the dead comparisons must be the same and branch the same way for every call to this method.
List<LogicallyDeadOp> deadOps = isLogicallyDead(execution, method, cmp.lvtIndex, cmp.value);
if (deadOps == null)
{
deadops.remove(pair);
invalidDeadops.add(pair);
continue; // lvt store
}
if (deadOps.isEmpty())
continue; // no ops to compare
// this must be per method,lvtindex
List<LogicallyDeadOp> existing = deadops.get(pair);
if (existing != null)
if (!existing.equals(deadOps))
{
// one of the branches taken differs because of the value, skip it
deadops.remove(pair);
invalidDeadops.add(pair);
continue;
}
else
{
continue;
}
deadops.put(pair, deadOps);
}
}
}
// private void findLogicallyDeadOperations(Execution execution)
// {
// for (ConstantMethodParameter cmp : parameters)
// {
// for (Method method : cmp.methods)
// {
// MethodLvtPair pair = new MethodLvtPair(method, cmp.lvtIndex, cmp.paramIndex, cmp.value);
//
// if (invalidDeadops.contains(pair))
// continue;
//
// // the dead comparisons must be the same and branch the same way for every call to this method.
// List<LogicallyDeadOp> deadOps = isLogicallyDead(execution, method, cmp.lvtIndex, cmp.value);
//
// if (deadOps == null)
// {
// deadops.remove(pair);
// invalidDeadops.add(pair);
// continue; // lvt store
// }
//
// if (deadOps.isEmpty())
// continue; // no ops to compare
//
// // this must be per method,lvtindex
// List<LogicallyDeadOp> existing = deadops.get(pair);
// if (existing != null)
// if (!existing.equals(deadOps))
// {
// // one of the branches taken differs because of the value, skip it
// deadops.remove(pair);
// invalidDeadops.add(pair);
// continue;
// }
// else
// {
// continue;
// }
//
// deadops.put(pair, deadOps);
// }
// }
// }
// remove logically dead comparisons
private int removeDeadOperations()
{
int count = 0;
for (MethodLvtPair mvp : deadops.keySet())
for (ConstantMethodParameter cmp : parameters)
{
List<LogicallyDeadOp> ops = deadops.get(mvp);
annotateObfuscatedSignature(cmp);
annotateObfuscatedSignature(mvp);
for (LogicallyDeadOp op : ops)
for (InstructionContext ctx : cmp.operations) // comparisons
{
InstructionContext ctx = op.compCtx; // comparison
Instruction ins = ctx.getInstruction();
boolean branch = op.branch; // branch that is always taken
boolean branch = cmp.result; // branch that is always taken
if (ins.getInstructions() == null)
continue; // ins already removed?
@@ -512,23 +647,25 @@ public class ConstantParameter implements Deobfuscator
private static final Type OBFUSCATED_SIGNATURE = new Type("Lnet/runelite/mapping/ObfuscatedSignature;");
private void annotateObfuscatedSignature(MethodLvtPair mvp)
private void annotateObfuscatedSignature(ConstantMethodParameter parameter)
{
Method m = mvp.method;
Object value = mvp.value;
for (Method m : parameter.methods)
{
Object value = parameter.value;
Attributes attributes = m.getAttributes();
Annotations annotations = attributes.getAnnotations();
Attributes attributes = m.getAttributes();
Annotations annotations = attributes.getAnnotations();
if (annotations != null && annotations.find(OBFUSCATED_SIGNATURE) != null)
return;
if (annotations != null && annotations.find(OBFUSCATED_SIGNATURE) != null)
return;
Annotation annotation = attributes.addAnnotation(OBFUSCATED_SIGNATURE, "signature", new net.runelite.asm.pool.UTF8(m.getDescriptor().toString()));
Annotation annotation = attributes.addAnnotation(OBFUSCATED_SIGNATURE, "signature", new net.runelite.asm.pool.UTF8(m.getDescriptor().toString()));
Element element = new Element(annotation);
element.setType(new Type("garbageValue"));
element.setValue(new net.runelite.asm.pool.UTF8(value.toString()));
annotation.addElement(element);
Element element = new Element(annotation);
element.setType(new Type("garbageValue"));
element.setValue(new net.runelite.asm.pool.UTF8(value.toString()));
annotation.addElement(element);
}
}
@Override
@@ -537,11 +674,17 @@ public class ConstantParameter implements Deobfuscator
group.buildClassGraph(); // required for getMethods in the invoke stuff by execution...
Execution execution = new Execution(group);
execution.addExecutionVisitor((i) -> findParameters(i));
execution.addExecutionVisitor((i) -> findDeadParameters(i));
execution.populateInitialMethods();
execution.run();
// execution = new Execution(group);
// execution.addExecutionVisitor((i) -> findDeadParameters(i));
// execution.populateInitialMethods();
// execution.run();
findParameters(execution);
findLogicallyDeadOperations(execution);
//findLogicallyDeadOperations(execution);
int count = removeDeadOperations();
System.out.println("Removed " + count + " logically dead conditional jumps");