Deob: Make mapping dumper dump (theoretically) all you need for updating (#1191)

* Deob: Make mapping dumper dump (theoretically) all you need for updating

* Update ClassFile.java
This commit is contained in:
Lucwousin
2019-07-30 17:01:15 +02:00
committed by GitHub
parent 252b3bb06b
commit c6fbe5940b
22 changed files with 644 additions and 328 deletions

View File

@@ -30,6 +30,7 @@ import net.runelite.asm.attributes.Annotations;
import net.runelite.asm.attributes.annotation.Annotation;
import net.runelite.asm.pool.Class;
import net.runelite.asm.signature.Signature;
import static net.runelite.deob.DeobAnnotations.*;
import org.objectweb.asm.AnnotationVisitor;
import org.objectweb.asm.ClassVisitor;
import org.objectweb.asm.FieldVisitor;
@@ -308,6 +309,20 @@ public class ClassFile
return null;
}
public Method findObfStaticMethod(String name, Signature type)
{
for (Method m : methods)
{
if (m.isStatic() &&
name.equals(getObfuscatedName(m.getAnnotations())) &&
type.equals(getObfuscatedSignature(m)))
{
return m;
}
}
return findMethodDeepStatic(name, type);
}
public Method findMethod(String name)
{
for (Method m : methods)

View File

@@ -31,6 +31,7 @@ import java.util.List;
import java.util.Map;
import net.runelite.asm.attributes.Code;
import net.runelite.asm.signature.Signature;
import static net.runelite.deob.DeobAnnotations.*;
public class ClassGroup
{
@@ -142,4 +143,17 @@ public class ClassGroup
return m;
}
public ClassFile findObfuscatedName(String name)
{
for (ClassFile cf : classes)
{
if (name.equals(getObfuscatedName(cf.getAnnotations())))
{
return cf;
}
}
return findClass(name);
}
}

View File

@@ -26,6 +26,7 @@ package net.runelite.asm;
import net.runelite.asm.attributes.Annotations;
import net.runelite.asm.attributes.annotation.Annotation;
import net.runelite.deob.DeobAnnotations;
import org.objectweb.asm.AnnotationVisitor;
import org.objectweb.asm.FieldVisitor;
import org.objectweb.asm.Opcodes;
@@ -131,6 +132,17 @@ public class Field
this.type = type;
}
public Type getObfuscatedType()
{
Type type = DeobAnnotations.getObfuscatedType(this);
if (type == null)
{
type = getType();
}
return type;
}
public Object getValue()
{
return value;

View File

@@ -35,6 +35,7 @@ import net.runelite.asm.attributes.code.LocalVariable;
import net.runelite.asm.attributes.code.Parameter;
import net.runelite.asm.attributes.code.instruction.types.LVTInstruction;
import net.runelite.asm.signature.Signature;
import net.runelite.deob.DeobAnnotations;
import org.objectweb.asm.AnnotationVisitor;
import org.objectweb.asm.Label;
import org.objectweb.asm.MethodVisitor;
@@ -193,6 +194,17 @@ public class Method
this.arguments = signature;
}
public Signature getObfuscatedSignature()
{
Signature sig = DeobAnnotations.getObfuscatedSignature(this);
if (sig == null)
{
sig = arguments;
}
return sig;
}
public boolean isNative()
{
return (accessFlags & ACC_NATIVE) != 0;

View File

@@ -25,10 +25,13 @@
package net.runelite.asm.attributes;
import java.util.ArrayList;
import java.util.List;
import net.runelite.asm.Method;
import net.runelite.asm.attributes.code.Exceptions;
import net.runelite.asm.attributes.code.Instruction;
import net.runelite.asm.attributes.code.Instructions;
import net.runelite.asm.attributes.code.Label;
import net.runelite.asm.attributes.code.instruction.types.LVTInstruction;
import net.runelite.asm.signature.Signature;
@@ -110,4 +113,28 @@ public class Code
{
return instructions;
}
public List<Integer> getLineNumbers()
{
final List<Integer> lineNumbers = new ArrayList<>();
for (Instruction i : instructions.getInstructions())
{
if (!(i instanceof Label))
{
continue;
}
Integer lineNumber = ((Label) i).getLineNumber();
if (lineNumber == null)
{
continue;
}
lineNumbers.add(lineNumber);
}
lineNumbers.sort(Integer::compareTo);
return lineNumbers;
}
}

View File

@@ -109,6 +109,11 @@ public class Label extends NOP
this.lineNumber = lineNumber;
}
public Integer getLineNumber()
{
return this.lineNumber;
}
public Instruction next()
{
Instructions ins = this.getInstructions();

View File

@@ -45,7 +45,7 @@ public class Field
@Override
public String toString()
{
return "Field{" + "clazz=" + clazz + ", name=" + name + ", type=" + type + '}';
return clazz.getName() + '.' + name + " " + type;
}
@Override

View File

@@ -43,7 +43,7 @@ public class Method
@Override
public String toString()
{
return clazz + "." + name + type;
return clazz.getName() + "." + name + type;
}
@Override

View File

@@ -75,7 +75,7 @@ public class Deob
System.exit(-1);
}
//logger.info("Deobfuscator revision {}", DeobProperties.getRevision());
logger.info("Deobfuscator revision {}", DeobProperties.getRevision());
Stopwatch stopwatch = Stopwatch.createStarted();

View File

@@ -24,18 +24,39 @@
*/
package net.runelite.deob;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;
public class DeobProperties
{
public static String getRevision() throws IOException
private static final Properties properties;
static
{
Properties properties = new Properties();
InputStream resourceAsStream = DeobProperties.class.getResourceAsStream("/deob.properties");
properties.load(resourceAsStream);
properties = new Properties();
try (InputStream resourceAsStream = DeobProperties.class.getResourceAsStream("/deob.properties"))
{
properties.load(resourceAsStream);
}
catch (IOException e)
{
//yes
}
}
return "420";
public static String getRevision()
{
return properties.getProperty("rs.version");
}
public static File getVanilla()
{
return new File(properties.getProperty("vanilla.jar"));
}
public static File getRsClient()
{
return new File(properties.getProperty("rs.client"));
}
}

View File

@@ -0,0 +1,130 @@
package net.runelite.deob.updater;
import com.google.common.base.Stopwatch;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.stream.JsonWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Map;
import net.runelite.asm.ClassFile;
import net.runelite.asm.ClassGroup;
import net.runelite.asm.pool.Field;
import net.runelite.asm.pool.Method;
import net.runelite.deob.DeobProperties;
import net.runelite.deob.updater.mappingdumper.MappedClass;
import net.runelite.deob.updater.mappingdumper.MappedField;
import net.runelite.deob.updater.mappingdumper.MappedMethod;
import net.runelite.deob.updater.mappingdumper.MappingDump;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class MappingDumper
{
private static ClassGroup group;
private final Logger log = LoggerFactory.getLogger(MappingDumper.class);
private static final Map<ClassFile, MappedClass> classMap = new HashMap<>();
private static final Map<Field, MappedField> fieldMap = new HashMap<>();
private static final Map<Method, MappedMethod> methodMap = new HashMap<>();
public MappingDumper(ClassGroup group)
{
MappingDumper.group = group;
}
public void dump(final File outputFile)
{
Stopwatch st = Stopwatch.createStarted();
group.buildClassGraph();
// MappingDump.of(ClassGroup) dumps everything completely
final MappingDump dump = new MappingDump().visitGroup(group);
dump.revision = Integer.parseInt(DeobProperties.getRevision());
log.info("RS version {}. Dump took {}", dump.revision, st.toString());
log.info("Total classes: {}. Total mapped classes: {}. ({}%)", dump.totalClasses, dump.totalNamedClasses, dump.totalNamedClasses * 100 / dump.totalClasses);
log.info("Total non static methods: {}. Total mapped non static methods: {}. ({}%)", dump.totalNonStaticMethods, dump.totalNamedNonStaticMethods, dump.totalNamedNonStaticMethods * 100 / dump.totalNamedMethods);
log.info("Total methods: {}. Total mapped methods: {}. ({}%)", dump.totalMethods, dump.totalNamedMethods, dump.totalNamedMethods * 100 / dump.totalMethods);
log.info("Total fields: {}. Total mapped fields: {}. ({}%)", dump.totalFields, dump.totalNamedFields, dump.totalNamedFields * 100 / dump.totalFields);
log.info("Total non static fields: {}. Total mapped non static fields: {}. ({}%)", dump.totalNonStaticFields, dump.totalNamedNonStaticFields, dump.totalNamedNonStaticFields * 100 / dump.totalNamedFields);
writeJson(dump, outputFile);
}
// Without this stack'll overflow :P
private void writeJson(MappingDump dump, File outputFile)
{
final Gson gson = new GsonBuilder().setPrettyPrinting().create();
try (JsonWriter writer = new JsonWriter(new OutputStreamWriter(new FileOutputStream(outputFile), StandardCharsets.UTF_8)))
{
writer.setIndent(" ");
writer.beginObject();
writer.name("revision").value(dump.revision);
writer.name("totalClasses").value(dump.totalClasses);
writer.name("totalNamedClasses").value(dump.totalNamedClasses);
writer.name("totalFields").value(dump.totalFields);
writer.name("totalNamedFields").value(dump.totalNamedFields);
writer.name("totalNonStaticFields").value(dump.totalNonStaticFields);
writer.name("totalNamedNonStaticFields").value(dump.totalNamedNonStaticFields);
writer.name("totalStaticFields").value(dump.totalStaticFields);
writer.name("totalNamedStaticFields").value(dump.totalNamedStaticFields);
writer.name("totalMethods").value(dump.totalMethods);
writer.name("totalNamedMethods").value(dump.totalNamedMethods);
writer.name("totalNonStaticMethods").value(dump.totalNonStaticMethods);
writer.name("totalNamedNonStaticMethods").value(dump.totalNamedNonStaticMethods);
writer.name("totalStaticMethods").value(dump.totalStaticMethods);
writer.name("totalNamedStaticMethods").value(dump.totalNamedStaticMethods);
writer.name("mappedClasses");
writer.beginArray();
for (MappedClass mc : dump.classes)
{
gson.toJson(mc, MappedClass.class, writer);
}
writer.endArray();
writer.endObject();
}
catch (IOException e)
{
log.error("Error saving json: ", e);
}
}
public static ClassGroup getGroup()
{
return group;
}
public static void putMap(ClassFile clazz, MappedClass mc)
{
classMap.put(clazz, mc);
}
public static MappedClass getMap(ClassFile clazz)
{
return classMap.get(clazz);
}
public static void putMap(Field field, MappedField mf)
{
fieldMap.put(field, mf);
}
public static MappedField getMap(Field field)
{
return fieldMap.get(field);
}
public static void putMap(Method method, MappedMethod mm)
{
methodMap.put(method, mm);
}
public static MappedMethod getMap(Method method)
{
return methodMap.get(method);
}
}

View File

@@ -0,0 +1,68 @@
package net.runelite.deob.updater.mappingdumper;
import com.google.gson.annotations.SerializedName;
import java.util.List;
import java.util.stream.Collectors;
import net.runelite.asm.ClassFile;
import net.runelite.deob.DeobAnnotations;
import net.runelite.deob.updater.MappingDumper;
public class MappedClass
{
@SerializedName("class")
public String implementingName;
@SerializedName("name")
public String obfuscatedName;
@SerializedName("super")
public String superClass;
public int access;
public List<String> interfaces;
public List<MappedField> fields;
public List<MappedMethod> methods;
public List<MappedMethod> constructors;
public MappedClass visitClass(final ClassFile c, final MappingDump dump)
{
MappingDumper.putMap(c, this);
implementingName = DeobAnnotations.getImplements(c);
obfuscatedName = DeobAnnotations.getObfuscatedName(c.getAnnotations());
if (obfuscatedName == null)
{
obfuscatedName = c.getName();
}
ClassFile parent = c.getParent();
if (parent != null)
{
superClass = DeobAnnotations.getObfuscatedName(parent.getAnnotations());
}
access = c.getAccess();
interfaces = c.getInterfaces()
.getMyInterfaces()
.stream()
.map(ClassFile::getAnnotations)
.map(DeobAnnotations::getObfuscatedName)
.collect(Collectors.toList());
fields = c.getFields()
.stream()
.map(f -> new MappedField().visitField(f, dump))
.collect(Collectors.toList());
methods = c.getMethods()
.stream()
.map(m -> new MappedMethod().visitMethod(m, dump))
.collect(Collectors.toList());
constructors = methods
.stream()
.filter(m -> m.obfuscatedName.endsWith("init>"))
.collect(Collectors.toList());
return this;
}
}

View File

@@ -0,0 +1,47 @@
package net.runelite.deob.updater.mappingdumper;
import com.google.gson.annotations.SerializedName;
import java.util.HashMap;
import java.util.Map;
import net.runelite.asm.Field;
import net.runelite.asm.pool.Method;
import net.runelite.deob.DeobAnnotations;
import net.runelite.deob.updater.MappingDumper;
public class MappedField
{
@SerializedName("field")
public String exportedName;
public String owner;
@SerializedName("name")
public String obfuscatedName;
public int access;
public String descriptor;
public Number decoder;
// method name, amt of times
public Map<Method, Integer> puts = new HashMap<>();
public Map<Method, Integer> gets = new HashMap<>();
public MappedField visitField(final Field f, final MappingDump dump)
{
MappingDumper.putMap(f.getPoolField(), this);
exportedName = DeobAnnotations.getExportedName(f.getAnnotations());
owner = MappingDumper.getMap(f.getClassFile()).obfuscatedName;
obfuscatedName = DeobAnnotations.getObfuscatedName(f.getAnnotations());
if (obfuscatedName == null)
{
obfuscatedName = f.getName();
}
access = f.getAccessFlags();
descriptor = f.getObfuscatedType().toString();
decoder = DeobAnnotations.getObfuscatedGetter(f);
return this;
}
}

View File

@@ -0,0 +1,112 @@
package net.runelite.deob.updater.mappingdumper;
import com.google.gson.annotations.SerializedName;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import net.runelite.asm.Method;
import net.runelite.asm.attributes.Code;
import net.runelite.asm.attributes.code.Instruction;
import net.runelite.asm.attributes.code.Instructions;
import net.runelite.asm.attributes.code.Parameter;
import net.runelite.asm.attributes.code.instruction.types.GetFieldInstruction;
import net.runelite.asm.attributes.code.instruction.types.InvokeInstruction;
import net.runelite.asm.attributes.code.instruction.types.SetFieldInstruction;
import net.runelite.asm.pool.Class;
import net.runelite.asm.pool.Field;
import net.runelite.deob.DeobAnnotations;
import net.runelite.deob.updater.MappingDumper;
public class MappedMethod
{
@SerializedName("method")
public String exportedName;
public String owner;
@SerializedName("name")
public String obfuscatedName;
public int access;
public List<String> parameters;
public String descriptor;
public String garbageValue;
public List<Integer> lineNumbers;
public Map<Field, Integer> fieldGets = new HashMap<>();
public Map<Field, Integer> fieldPuts = new HashMap<>();
public Map<net.runelite.asm.pool.Method, Integer> dependencies = new HashMap<>();
public MappedMethod visitMethod(final Method m, final MappingDump dump)
{
MappingDumper.putMap(m.getPoolMethod(), this);
exportedName = DeobAnnotations.getExportedName(m.getAnnotations());
owner = MappingDumper.getMap(m.getClassFile()).obfuscatedName;
obfuscatedName = DeobAnnotations.getObfuscatedName(m.getAnnotations());
if (obfuscatedName == null)
{
obfuscatedName = m.getName();
}
access = m.getAccessFlags();
parameters = m.getParameters()
.stream()
.map(Parameter::getName)
.collect(Collectors.toList());
descriptor = m.getObfuscatedSignature().toString();
garbageValue = DeobAnnotations.getDecoder(m);
Code c = m.getCode();
if (c != null)
{
visitCode(c);
}
return this;
}
private void visitCode(Code c)
{
lineNumbers = c.getLineNumbers();
Instructions ins = c.getInstructions();
for (Instruction i : ins.getInstructions())
{
if (i instanceof GetFieldInstruction)
{
Field k = ((GetFieldInstruction) i).getField();
int v = fieldGets.getOrDefault(k, 0) + 1;
fieldGets.put(k, v);
}
else if (i instanceof SetFieldInstruction)
{
Field k = ((SetFieldInstruction) i).getField();
int v = fieldPuts.getOrDefault(k, 0) + 1;
fieldPuts.put(k, v);
}
else if (i instanceof InvokeInstruction)
{
List<Method> met = ((InvokeInstruction) i).getMethods();
net.runelite.asm.pool.Method k;
if (met.size() > 0)
{
Method mme = met.get(0);
k = new net.runelite.asm.pool.Method(
new Class(DeobAnnotations.getObfuscatedName(mme.getClassFile().getAnnotations())),
DeobAnnotations.getObfuscatedName(mme.getAnnotations()),
mme.getObfuscatedSignature() != null ? mme.getObfuscatedSignature() : mme.getDescriptor()
);
}
else
{
k = ((InvokeInstruction) i).getMethod();
}
int v = dependencies.getOrDefault(k, 0) + 1;
dependencies.put(k, v);
}
}
}
}

View File

@@ -0,0 +1,148 @@
package net.runelite.deob.updater.mappingdumper;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import net.runelite.asm.ClassFile;
import net.runelite.asm.ClassGroup;
import net.runelite.asm.pool.Class;
import net.runelite.asm.pool.Field;
import net.runelite.asm.pool.Method;
import net.runelite.asm.signature.Signature;
import net.runelite.deob.updater.MappingDumper;
import static org.objectweb.asm.Opcodes.ACC_STATIC;
public class MappingDump
{
public int revision;
public int totalClasses = 0;
public int totalNamedClasses = 0;
public int totalFields = 0;
public int totalNamedFields = 0;
public int totalNonStaticFields = 0;
public int totalNamedNonStaticFields = 0;
public int totalStaticFields = 0;
public int totalNamedStaticFields = 0;
public int totalMethods = 0;
public int totalNamedMethods = 0;
public int totalNonStaticMethods = 0;
public int totalNamedNonStaticMethods = 0;
public int totalStaticMethods = 0;
public int totalNamedStaticMethods = 0;
public List<MappedClass> classes;
private ClassGroup group;
public MappingDump visitGroup(ClassGroup group)
{
this.group = group;
classes = new ArrayList<>();
for (ClassFile c : group.getClasses())
{
if (c.getName().contains("runelite"))
{
continue;
}
final MappedClass mc = new MappedClass();
classes.add(mc.visitClass(c, this));
}
totalClasses = classes.size();
for (MappedClass c : classes)
{
if (c.implementingName != null)
{
totalNamedClasses++;
}
for (MappedMethod mm : c.methods)
{
for (Map.Entry<Field, Integer> entry : mm.fieldGets.entrySet())
{
MappedField mf = MappingDumper.getMap(entry.getKey());
if (mf == null)
{
continue;
}
mf.gets.put(
new Method(
new Class(mm.owner),
mm.obfuscatedName,
new Signature(mm.descriptor)),
entry.getValue());
}
for (Map.Entry<Field, Integer> entry : mm.fieldPuts.entrySet())
{
MappedField mf = MappingDumper.getMap(entry.getKey());
if (mf == null)
{
continue;
}
mf.puts.put(
new Method(
new Class(mm.owner),
mm.obfuscatedName,
new Signature(mm.descriptor)),
entry.getValue());
}
}
grabAmountInfo(c);
}
totalNonStaticFields = totalFields - totalStaticFields;
totalNamedNonStaticFields = totalNamedFields - totalNamedStaticFields;
totalNonStaticMethods = totalMethods - totalStaticMethods;
totalNamedNonStaticMethods = totalNamedMethods - totalNamedStaticMethods;
return this;
}
private void grabAmountInfo(MappedClass c)
{
totalFields += c.fields.size();
totalNamedFields += c.fields
.stream()
.filter(f -> f.exportedName != null)
.count();
totalStaticFields += c.fields
.stream()
.filter(f -> (f.access & ACC_STATIC) != 0)
.count();
totalNamedStaticFields += c.fields
.stream()
.filter(f -> f.exportedName != null
&& (f.access & ACC_STATIC) != 0)
.count();
totalMethods += c.methods.size();
totalNamedMethods += c.methods
.stream()
.filter(f -> f.exportedName != null)
.count();
totalStaticMethods += c.methods
.stream()
.filter(f -> (f.access & ACC_STATIC) != 0)
.count();
totalNamedStaticMethods += c.methods
.stream()
.filter(f -> f.exportedName != null
&& (f.access & ACC_STATIC) != 0)
.count();
}
ClassGroup getGroup()
{
return group;
}
}