Nixyri/ghidra
1
0
Fork 0
mirror of https://github.com/NationalSecurityAgency/ghidra.git synced 2024-11-12 23:23:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

GT-2848: Refactor DependencyGraph with Deterministic version

Browse Source
This commit is contained in:
ghizard 2019-05-07 10:24:41 -04:00
parent dace9682fb
commit fa558af9c2
11 changed files with 744 additions and 343 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Ghidra/Features/Decompiler/src/main/java/ghidra/app/cmd/function/DecompilerParameterIdCmd.java
View File

@ -37,7 +37,7 @@ import ghidra.program.model.util.AcyclicCallGraphBuilder;
import ghidra.util.Msg;
import ghidra.util.exception.CancelledException;
import ghidra.util.exception.InvalidInputException;
import ghidra.util.graph.DependencyGraph;
import ghidra.util.graph.AbstractDependencyGraph;
import ghidra.util.task.TaskMonitor;
public class DecompilerParameterIdCmd extends BackgroundCommand {
@ -74,7 +74,7 @@ public class DecompilerParameterIdCmd extends BackgroundCommand {
monitor.setMessage("Analyzing Call Hierarchy...");
AcyclicCallGraphBuilder builder =
new AcyclicCallGraphBuilder(program, entryPoints, true);
DependencyGraph<Address> graph = builder.getDependencyGraph(monitor);
AbstractDependencyGraph<Address> graph = builder.getDependencyGraph(monitor);
if (graph.isEmpty()) {
return true;
}

4
Ghidra/Features/Decompiler/src/main/java/ghidra/app/plugin/core/analysis/DecompilerCallConventionAnalyzer.java
View File

@ -33,7 +33,7 @@ import ghidra.program.model.symbol.SourceType;
import ghidra.program.model.util.AcyclicCallGraphBuilder;
import ghidra.util.Msg;
import ghidra.util.exception.CancelledException;
import ghidra.util.graph.DependencyGraph;
import ghidra.util.graph.AbstractDependencyGraph;
import ghidra.util.task.TaskMonitor;
public class DecompilerCallConventionAnalyzer extends AbstractAnalyzer {
@ -138,7 +138,7 @@ public class DecompilerCallConventionAnalyzer extends AbstractAnalyzer {
monitor.setMessage("Analyzing Call Hierarchy...");
AcyclicCallGraphBuilder builder =
new AcyclicCallGraphBuilder(program, functionEntries, true);
DependencyGraph<Address> graph = builder.getDependencyGraph(monitor);
AbstractDependencyGraph<Address> graph = builder.getDependencyGraph(monitor);
if (graph.isEmpty()) {
return;
}

6
Ghidra/Framework/Generic/src/main/java/generic/concurrent/ConcurrentGraphQ.java
View File

@ -15,16 +15,16 @@
*/
package generic.concurrent;
import ghidra.util.graph.DependencyGraph;
import ghidra.util.graph.AbstractDependencyGraph;
import ghidra.util.task.TaskMonitor;
import java.util.Set;
public class ConcurrentGraphQ<I> {
private ConcurrentQ<I, Object> queue;
private DependencyGraph<I> graph;
private AbstractDependencyGraph<I> graph;
public ConcurrentGraphQ(QRunnable<I> runnable, DependencyGraph<I> graph, GThreadPool pool,
public ConcurrentGraphQ(QRunnable<I> runnable, AbstractDependencyGraph<I> graph, GThreadPool pool,
TaskMonitor monitor) {
this.graph = graph;
// @formatter:off

354
Ghidra/Framework/Generic/src/main/java/ghidra/util/graph/AbstractDependencyGraph.java Normal file
View File

@ -0,0 +1,354 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.util.graph;
import java.util.*;
/**
* Class for managing the visiting (processing) of a set of values where some values depend
* on other values being process before them. In other words, an acyclic directed graph will
* be formed where the vertexes are the values and the edges represent dependencies. Values can
* only be removed if they have no dependencies. Since the graph is acyclic, as values are removed
* that have no dependencies, other nodes that depend on those nodes will become eligible for
* processing and removal. If cycles are introduced, they will eventually cause an IllegalState
* exception to occur when removing and processing values. There is also a hasCycles() method
* that can be called before processing to find cycle problems up front without wasting time
* processing values.
*
* @param <T> the type of value. Some concrete classes might have restrictions on T.
*
* @see DependencyGraph
* @see DeterministicDependencyGraph
*/
public abstract class AbstractDependencyGraph<T> {
protected Map<T, DependencyNode> nodeMap;
protected Set<T> unvisitedIndependentSet;
private int visitedButNotDeletedCount = 0;
public AbstractDependencyGraph() {
nodeMap = createNodeMap();
unvisitedIndependentSet = createNodeSet();
}
public Map<T, DependencyNode> getNodeMap() {
return nodeMap;
}
/**
* Creates the Map of Nodes to {@link DependencyNode}s appropriate for the implementer.
* @return a new Map of Nodes to {@link DependencyNode}s.
*/
protected abstract Map<T, DependencyNode> createNodeMap();
/**
* Creates the Set of Nodes appropriate for the implementer.
* @return a new Set of Nodes.
*/
protected abstract Set<T> createNodeSet();
/**
* Creates the Set of {@link DependencyNode}s appropriate for the implementer.
* @return a new Set of {@link DependencyNode}s.
*/
protected abstract Set<DependencyNode> createDependencyNodeSet();
/**
* Returns a copy of this graph.
* @return a copy of this graph.
*/
public abstract AbstractDependencyGraph<T> copy();
/**
* Copy constructor
* Cannot be abstracted here. Each individual implementer must implement their own.
* @param other the other DependencyGraph to copy
*/
/**
* Adds the value to this graph.
* @param value the value to add
*/
public synchronized void addValue(T value) {
getOrCreateDependencyNode(value);
}
/**
* Returns the number of values in this graph.
* @return the number of values in this graph.
*/
public synchronized int size() {
return nodeMap.size();
}
/**
* Returns true if the graph has no values;
* @return true if the graph has no values;
*/
public synchronized boolean isEmpty() {
return nodeMap.isEmpty();
}
/**
* Returns true if this graph has the given key.
* @param value the value to check if its in this graph
* @return true if this graph has the given key.
*/
public synchronized boolean contains(T value) {
return nodeMap.containsKey(value);
}
/**
* Returns the set of values in this graph.
* @return the set of values in this graph.
*/
public synchronized Set<T> getValues() {
return new HashSet<>(nodeMap.keySet());
}
/**
* Returns the set of values in this graph.
* @return the set of values in this graph.
*/
public abstract Set<T> getNodeMapValues();
private DependencyNode getOrCreateDependencyNode(T value) {
DependencyNode dependencyNode = nodeMap.get(value);
if (dependencyNode == null) {
dependencyNode = new DependencyNode(value);
nodeMap.put(value, dependencyNode);
unvisitedIndependentSet.add(value);
}
return dependencyNode;
}
/**
* Add a dependency such that value1 depends on value2. Both value1 and value2 will be
* added to the graph if they are not already in the graph.
* @param value1 the value that depends on value2
* @param value2 the value that value1 is depending on
*/
public synchronized void addDependency(T value1, T value2) {
DependencyNode valueNode1 = getOrCreateDependencyNode(value1);
DependencyNode valueNode2 = getOrCreateDependencyNode(value2);
valueNode2.addNodeThatDependsOnMe(valueNode1);
}
/**
* Returns true if there are unvisited values ready (no dependencies) for processing.
*
* @return true if there are unvisited values ready for processing.
*
* @exception IllegalStateException is thrown if the graph is not empty and there are no nodes
* without dependency which indicates there is a cycle in the graph.
*/
public synchronized boolean hasUnVisitedIndependentValues() {
if (!unvisitedIndependentSet.isEmpty()) {
return true;
}
checkCycleState();
return false;
}
/**
* Removes and returns a value that has no dependencies from the graph. If the graph is empty
* or all the nodes without dependencies are currently visited, then null will be returned.
* NOTE: If the getUnvisitedIndependentValues() method has been called(), this method may
* return null until all those "visited" nodes are removed from the graph.
* @return return an arbitrary value that has no dependencies and hasn't been visited or null.
*/
public synchronized T pop() {
checkCycleState();
if (unvisitedIndependentSet.isEmpty()) {
return null;
}
T value = unvisitedIndependentSet.iterator().next();
unvisitedIndependentSet.remove(value);
remove(value);
return value;
}
private void checkCycleState() {
if (!isEmpty() && unvisitedIndependentSet.isEmpty() && visitedButNotDeletedCount == 0) {
throw new IllegalStateException("Cycle detected!");
}
}
/**
* Checks if this graph has cycles. Normal processing of this graph will eventually reveal
* a cycle and throw an exception at the time it is detected. This method allows for a
* "fail fast" way to detect cycles.
* @return true if cycles exist in the graph.
*/
public synchronized boolean hasCycles() {
try {
Set<T> visited = createNodeSet();
while (!unvisitedIndependentSet.isEmpty()) {
Collection<T> values = getUnvisitedIndependentValues();
visited.addAll(values);
for (T k : values) {
DependencyNode node = nodeMap.get(k);
node.releaseDependencies();
}
}
if (visited.size() != nodeMap.size()) {
return true;
}
}
finally {
reset();
}
return false;
}
private void reset() {
visitedButNotDeletedCount = 0;
for (DependencyNode node : nodeMap.values()) {
node.numberOfNodesThatIDependOn = 0;
}
for (DependencyNode node : nodeMap.values()) {
if (node.setOfNodesThatDependOnMe != null) {
for (DependencyNode child : node.setOfNodesThatDependOnMe) {
unvisitedIndependentSet.remove(child.value);
child.numberOfNodesThatIDependOn++;
}
}
}
unvisitedIndependentSet = getAllIndependentValues();
}
/**
* Returns a set of all values that have no dependencies. As values are removed from the
* graph, dependencies will be removed and additional values will be eligible to be returned
* by this method. Once a value has been retrieved using this method, it will be considered
* "visited" and future calls to this method will not include those values. To continue
* processing the values in the graph, all values return from this method should eventually
* be deleted from the graph to "free up" other values. NOTE: values retrieved by this method
* will no longer be eligible for return by the pop() method.
*
* @return the set of values without dependencies that have never been returned by this method
* before.
*/
public synchronized Set<T> getUnvisitedIndependentValues() {
checkCycleState();
visitedButNotDeletedCount += unvisitedIndependentSet.size();
Set<T> returnCollection = unvisitedIndependentSet;
unvisitedIndependentSet = createNodeSet();
return returnCollection;
}
/**
* Returns the set of all values that have no dependencies regardless of whether or not
* they have been "visited" (by the getUnvisitedIndependentValues() method.
* @return return the set of all values that have no dependencies.
*/
public synchronized Set<T> getAllIndependentValues() {
Set<T> set = createNodeSet();
for (DependencyNode node : nodeMap.values()) {
if (node.numberOfNodesThatIDependOn == 0) {
set.add(node.value);
}
}
return set;
}
/**
* Removes the value from the graph. Any dependency from this node to another will be removed,
* possible allowing nodes that depend on this node to be eligible for processing.
* @param value the value to remove from the graph.
*/
public synchronized void remove(T value) {
DependencyNode node = nodeMap.remove(value);
if (node != null) {
node.releaseDependencies();
if (unvisitedIndependentSet.remove(value)) {
visitedButNotDeletedCount--;
}
}
}
/**
* Returns a set of values that depend on the given value.
* @param value the value that other values may depend on.
* @return a set of values that depend on the given value.
*/
public synchronized Set<T> getDependentValues(T value) {
Set<T> set = createNodeSet();
DependencyNode node = nodeMap.get(value);
if (node != null && node.setOfNodesThatDependOnMe != null) {
for (DependencyNode child : node.setOfNodesThatDependOnMe) {
set.add(child.value);
}
}
return set;
}
protected class DependencyNode {
private final T value;
private Set<DependencyNode> setOfNodesThatDependOnMe;
private int numberOfNodesThatIDependOn = 0;
DependencyNode(T value) {
this.value = value;
}
public T getValue() {
return value;
}
public Set<DependencyNode> getSetOfNodesThatDependOnMe() {
return setOfNodesThatDependOnMe;
}
public int getNumberOfNodesThatIDependOn() {
return numberOfNodesThatIDependOn;
}
public void releaseDependencies() {
if (setOfNodesThatDependOnMe == null) {
return;
}
for (DependencyNode node : setOfNodesThatDependOnMe) {
if (--node.numberOfNodesThatIDependOn == 0) {
unvisitedIndependentSet.add(node.value);
}
}
}
public void addNodeThatDependsOnMe(DependencyNode node) {
if (setOfNodesThatDependOnMe == null) {
setOfNodesThatDependOnMe = createDependencyNodeSet();
}
if (setOfNodesThatDependOnMe.add(node)) {
// if not already added, increment the dependent node's count so that it knows
// how many nodes it depends on.
node.numberOfNodesThatIDependOn++;
unvisitedIndependentSet.remove(node.value); // it has at least one dependency now
}
}
@Override
public String toString() {
return value == null ? "" : value.toString();
}
}
}

305
Ghidra/Framework/Generic/src/main/java/ghidra/util/graph/DependencyGraph.java
View File

@ -15,33 +15,23 @@
*/
package ghidra.util.graph;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.*;
/**
* Class for managing the visiting (processing) of a set of values where some values depend
* on other values being process before them. In other words, an acyclic directed graph will
* be formed where the vertexes are the values and the edges represent dependencies. Values can
* only be removed if they have no dependencies. Since the graph is acyclic, as values are removed
* that have no dependencies, other nodes that depend on those nodes will become eligible for
* processing and removal. If cycles are introduced, they will eventually cause an IllegalState
* exception to occur when removing and processing values. There is also a hasCycles() method
* that can be called before processing to find cycle problems up front without wasting time
* processing values.
* Original Dependency Graph implementation that uses {@link HashMap}s and {@link HashSet}s.
* Side affect of these is that data pulled from the graph ({@link #pop()}) is not performed
* in a deterministic order. However, load time for the graph is O(1).
*
* @param <T> the type of value. This class uses the values as keys in HashSets, so the value
* type must be meet the equals() and hashCode() requirements for hashing.
*
* @see AbstractDependencyGraph
* @see DeterministicDependencyGraph
*/
public class DependencyGraph<T> {
private Map<T, DependencyNode> nodeMap = new HashMap<T, DependencyNode>();
private Set<T> unvisitedIndependentSet = new HashSet<T>();
private int visitedButNotDeletedCount = 0;
public class DependencyGraph<T> extends AbstractDependencyGraph<T> {
public DependencyGraph() {
super();
}
/**
@ -51,278 +41,39 @@ public class DependencyGraph<T> {
public DependencyGraph(DependencyGraph<T> other) {
synchronized (other) {
for (DependencyNode node : other.nodeMap.values()) {
addValue(node.value);
if (node.setOfNodesThatDependOnMe != null) {
for (DependencyNode child : node.setOfNodesThatDependOnMe) {
addDependency(child.value, node.value);
addValue(node.getValue());
if (node.getSetOfNodesThatDependOnMe() != null) {
for (DependencyNode child : node.getSetOfNodesThatDependOnMe()) {
addDependency(child.getValue(), node.getValue());
}
}
}
}
}
/**
* Adds the value to this graph.
* @param value the value to add
*/
public synchronized void addValue(T value) {
getOrCreateDependencyNode(value);
@Override
public AbstractDependencyGraph<T> copy() {
return new DependencyGraph<>(this);
}
/**
* Returns the number of values in this graph.
* @return the number of values in this graph.
*/
public synchronized int size() {
return nodeMap.size();
@Override
protected Map<T, DependencyNode> createNodeMap() {
return new HashMap<>();
}
/**
* Returns true if the graph has no values;
* @return true if the graph has no values;
*/
public synchronized boolean isEmpty() {
return nodeMap.isEmpty();
@Override
protected Set<T> createNodeSet() {
return new HashSet<>();
}
/**
* Returns true if this graph has the given key.
* @param value the value to check if its in this graph
* @return true if this graph has the given key.
*/
public synchronized boolean contains(T value) {
return nodeMap.containsKey(value);
@Override
protected Set<DependencyNode> createDependencyNodeSet() {
return new HashSet<>();
}
/**
* Returns the set of values in this graph.
* @return the set of values in this graph.
*/
public synchronized Set<T> getValues() {
return new HashSet<T>(nodeMap.keySet());
}
/**
* Returns a copy of this graph.
* @return a copy of this graph.
*/
public synchronized DependencyGraph<T> copy() {
return new DependencyGraph<T>(this);
}
private DependencyNode getOrCreateDependencyNode(T value) {
DependencyNode dependencyNode = nodeMap.get(value);
if (dependencyNode == null) {
dependencyNode = new DependencyNode(value);
nodeMap.put(value, dependencyNode);
unvisitedIndependentSet.add(value);
}
return dependencyNode;
}
/**
* Add a dependency such that value1 depends on value2. Both value1 and value2 will be
* added to the graph if they are not already in the graph.
* @param value1 the value that depends on value2
* @param value2 the value that value1 is depending on
*/
public synchronized void addDependency(T value1, T value2) {
DependencyNode valueNode1 = getOrCreateDependencyNode(value1);
DependencyNode valueNode2 = getOrCreateDependencyNode(value2);
valueNode2.addNodeThatDependsOnMe(valueNode1);
}
/**
* Returns true if there are unvisited values ready (no dependencies) for processing.
*
* @return true if there are unvisited values ready for processing.
*
* @exception IllegalStateException is thrown if the graph is not empty and there are no nodes
* without dependency which indicates there is a cycle in the graph.
*/
public synchronized boolean hasUnVisitedIndependentValues() {
if (!unvisitedIndependentSet.isEmpty()) {
return true;
}
checkCycleState();
return false;
}
/**
* Removes and returns a value that has no dependencies from the graph. If the graph is empty
* or all the nodes without dependencies are currently visited, then null will be returned.
* NOTE: If the getUnvisitedIndependentValues() method has been called(), this method may
* return null until all those "visited" nodes are removed from the graph.
* @return return an arbitrary value that has no dependencies and hasn't been visited or null.
*/
public synchronized T pop() {
checkCycleState();
if (unvisitedIndependentSet.isEmpty()) {
return null;
}
T value = unvisitedIndependentSet.iterator().next();
unvisitedIndependentSet.remove(value);
remove(value);
return value;
}
private void checkCycleState() {
if (!isEmpty() && unvisitedIndependentSet.isEmpty() && visitedButNotDeletedCount == 0) {
throw new IllegalStateException("Cycle detected!");
}
}
/**
* Checks if this graph has cycles. Normal processing of this graph will eventually reveal
* a cycle and throw an exception at the time it is detected. This method allows for a
* "fail fast" way to detect cycles.
* @return true if cycles exist in the graph.
*/
public synchronized boolean hasCycles() {
try {
Set<T> visited = new HashSet<T>();
while (!unvisitedIndependentSet.isEmpty()) {
Collection<T> values = getUnvisitedIndependentValues();
visited.addAll(values);
for (T k : values) {
DependencyNode node = nodeMap.get(k);
node.releaseDependencies();
}
}
if (visited.size() != nodeMap.size()) {
return true;
}
}
finally {
reset();
}
return false;
}
private void reset() {
visitedButNotDeletedCount = 0;
for (DependencyNode node : nodeMap.values()) {
node.numberOfNodesThatIDependOn = 0;
}
for (DependencyNode node : nodeMap.values()) {
if (node.setOfNodesThatDependOnMe != null) {
for (DependencyNode child : node.setOfNodesThatDependOnMe) {
unvisitedIndependentSet.remove(child.value);
child.numberOfNodesThatIDependOn++;
}
}
}
unvisitedIndependentSet = getAllIndependentValues();
}
/**
* Returns a set of all values that have no dependencies. As values are removed from the
* graph, dependencies will be removed and additional values will be eligible to be returned
* by this method. Once a value has been retrieved using this method, it will be considered
* "visited" and future calls to this method will not include those values. To continue
* processing the values in the graph, all values return from this method should eventually
* be deleted from the graph to "free up" other values. NOTE: values retrieved by this method
* will no longer be eligible for return by the pop() method.
*
* @return the set of values without dependencies that have never been returned by this method
* before.
*/
public synchronized Set<T> getUnvisitedIndependentValues() {
checkCycleState();
visitedButNotDeletedCount += unvisitedIndependentSet.size();
Set<T> returnCollection = unvisitedIndependentSet;
unvisitedIndependentSet = new HashSet<T>();
return returnCollection;
}
/**
* Returns the set of all values that have no dependencies regardless of whether or not
* they have been "visited" (by the getUnvisitedIndependentValues() method.
* @return return the set of all values that have no dependencies.
*/
public synchronized Set<T> getAllIndependentValues() {
Set<T> set = new HashSet<T>();
for (DependencyNode node : nodeMap.values()) {
if (node.numberOfNodesThatIDependOn == 0) {
set.add(node.value);
}
}
return set;
}
/**
* Removes the value from the graph. Any dependency from this node to another will be removed,
* possible allowing nodes that depend on this node to be eligible for processing.
* @param value the value to remove from the graph.
*/
public synchronized void remove(T value) {
DependencyNode node = nodeMap.remove(value);
if (node != null) {
node.releaseDependencies();
if (unvisitedIndependentSet.remove(value)) {
visitedButNotDeletedCount--;
}
}
}
/**
* Returns a set of values that depend on the given value.
* @param value the value that other values may depend on.
* @return a set of values that depend on the given value.
*/
public synchronized Set<T> getDependentValues(T value) {
Set<T> set = new HashSet<T>();
DependencyNode node = nodeMap.get(value);
if (node != null && node.setOfNodesThatDependOnMe != null) {
for (DependencyNode child : node.setOfNodesThatDependOnMe) {
set.add(child.value);
}
}
return set;
}
private class DependencyNode {
private final T value;
private Set<DependencyNode> setOfNodesThatDependOnMe;
private int numberOfNodesThatIDependOn = 0;
DependencyNode(T value) {
this.value = value;
}
public void releaseDependencies() {
if (setOfNodesThatDependOnMe == null) {
return;
}
for (DependencyNode node : setOfNodesThatDependOnMe) {
if (--node.numberOfNodesThatIDependOn == 0) {
unvisitedIndependentSet.add(node.value);
}
}
}
public void addNodeThatDependsOnMe(DependencyNode node) {
if (setOfNodesThatDependOnMe == null) {
setOfNodesThatDependOnMe = new HashSet<DependencyGraph<T>.DependencyNode>();
}
if (setOfNodesThatDependOnMe.add(node)) {
// if not already added, increment the dependent node's count so that it knows
// how many nodes it depends on.
node.numberOfNodesThatIDependOn++;
unvisitedIndependentSet.remove(node.value); // it has at least one dependency now
}
}
@Override
public String toString() {
return value == null ? "" : value.toString();
}
@Override
public synchronized Set<T> getNodeMapValues() {
return new HashSet<>(nodeMap.keySet());
}
}

84
Ghidra/Framework/Generic/src/main/java/ghidra/util/graph/DeterministicDependencyGraph.java Normal file
View File

@ -0,0 +1,84 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.util.graph;
import java.util.*;
import org.apache.commons.collections4.set.ListOrderedSet;
/**
* Dependency Graph that uses {@link TreeMap}s and {@link ListOrderedSet}s to provide
* determinism in pulling ({@link #pop()}) from the graph. This class seems to consume more
* memory than {@link DependencyGraph}, and if memory is not an issue, it also seems to be
* slightly faster as well.
* <P>
* This class was implemented to provide determinism while doing
* developmental debugging.
*
* @param <T> the type of value.
*
* @see AbstractDependencyGraph
* @see DependencyGraph
*/
public class DeterministicDependencyGraph<T> extends AbstractDependencyGraph<T> {
public DeterministicDependencyGraph() {
super();
}
/**
* Copy constructor
* @param other the other DependencyGraph to copy
*/
public DeterministicDependencyGraph(DeterministicDependencyGraph<T> other) {
synchronized (other) {
for (DependencyNode node : other.nodeMap.values()) {
addValue(node.getValue());
if (node.getSetOfNodesThatDependOnMe() != null) {
for (DependencyNode child : node.getSetOfNodesThatDependOnMe()) {
addDependency(child.getValue(), node.getValue());
}
}
}
}
}
@Override
public AbstractDependencyGraph<T> copy() {
return new DeterministicDependencyGraph<>(this);
}
@Override
protected Map<T, DependencyNode> createNodeMap() {
return new TreeMap<>();
}
@Override
protected Set<T> createNodeSet() {
return new ListOrderedSet<>();
}
@Override
protected Set<DependencyNode> createDependencyNodeSet() {
return new ListOrderedSet<>();
}
@Override
public synchronized Set<T> getNodeMapValues() {
return ListOrderedSet.listOrderedSet(nodeMap.keySet());
}
}

21
Ghidra/Framework/Generic/src/test/java/generic/concurrent/ConcurrentGraphQTest.java
View File

@ -23,6 +23,7 @@ import org.junit.Assert;
import org.junit.Test;
import generic.test.AbstractGenericTest;
import ghidra.util.graph.AbstractDependencyGraph;
import ghidra.util.graph.DependencyGraph;
import ghidra.util.task.TaskMonitor;
@ -32,11 +33,11 @@ public class ConcurrentGraphQTest extends AbstractGenericTest {
super();
}
@Test
public void test() throws InterruptedException, Exception {
final ArrayList<String> completionOrder = new ArrayList<String>();
@Test
public void test() throws InterruptedException, Exception {
final ArrayList<String> completionOrder = new ArrayList<>();
DependencyGraph<String> graph = new DependencyGraph<String>();
AbstractDependencyGraph<String> graph = new DependencyGraph<>();
graph.addDependency("@0", "A8");
graph.addDependency("@1", "A1");
graph.addDependency("@2", "A7");
@ -71,10 +72,10 @@ public class ConcurrentGraphQTest extends AbstractGenericTest {
assertTrue(!graph.hasCycles());
DependencyGraph<String> savedGraph = graph.copy();
AbstractDependencyGraph<String> savedGraph = graph.copy();
GThreadPool pool = GThreadPool.getPrivateThreadPool("ConcurrentGraphQ Test");
QRunnable<String> runnable = new QRunnable<String>() {
QRunnable<String> runnable = new QRunnable<>() {
@Override
public void run(String item, TaskMonitor monitor) throws Exception {
@ -87,7 +88,7 @@ public class ConcurrentGraphQTest extends AbstractGenericTest {
}
};
ConcurrentGraphQ<String> queue = new ConcurrentGraphQ<String>(runnable, graph, pool, null);
ConcurrentGraphQ<String> queue = new ConcurrentGraphQ<>(runnable, graph, pool, null);
queue.execute();
checkOrderSatisfiesDependencies(savedGraph, completionOrder);
}
@ -101,7 +102,7 @@ public class ConcurrentGraphQTest extends AbstractGenericTest {
* @param visitedOrder the actual execution order to be tested
* @return
*/
public void checkOrderSatisfiesDependencies(DependencyGraph<String> dependencyGraph,
public void checkOrderSatisfiesDependencies(AbstractDependencyGraph<String> dependencyGraph,
List<String> visitedOrder) {
if (visitedOrder.size() > dependencyGraph.size()) {
@ -111,12 +112,12 @@ public class ConcurrentGraphQTest extends AbstractGenericTest {
Assert.fail("Not all items in the graph were visited");
}
HashSet<String> items = new HashSet<String>(visitedOrder);
HashSet<String> items = new HashSet<>(visitedOrder);
if (items.size() != visitedOrder.size()) {
Assert.fail("duplicate item(s) in linearOrder\n");
}
HashMap<String, Integer> visitedOrderMap = new HashMap<String, Integer>();
HashMap<String, Integer> visitedOrderMap = new HashMap<>();
for (int i = 0; i < visitedOrder.size(); i++) {
visitedOrderMap.put(visitedOrder.get(i), i);
}

150
Ghidra/Framework/Generic/src/test/java/ghidra/util/datastruct/DependencyGraphPerformanceTest.java Normal file
View File

@ -0,0 +1,150 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.util.datastruct;
import java.util.*;
import ghidra.util.graph.DependencyGraph;
import ghidra.util.graph.DeterministicDependencyGraph;
/**
* Performs computation of dependency graph
* Does not test the accuracy/functionality of Dependency Graphs.
*/
public class DependencyGraphPerformanceTest {
// Specifying the graph edge density ratio: number of edges over the
// number of possible edges (|E|/P(|V|,2).
// For sparse graphs, where |E|=O(n), can do |E|/|V|.
// TODO: What is good for testing.
private static final long GRAPH_SEED = 12345L;
private static final double GRAPH_EDGE_DENSITY = 0.20;
private static final int NUM_DEPENDENCIES = 30000;
private static final int GRAPH_SIZE = (int) (NUM_DEPENDENCIES / GRAPH_EDGE_DENSITY);
private static final boolean GRAPH_ALLOW_CYCLES = false;
private final List<DependencyRelation> testRelationships;
public DependencyGraphPerformanceTest() {
testRelationships = constructRandomRelationships(GRAPH_SEED, NUM_DEPENDENCIES, GRAPH_SIZE,
GRAPH_ALLOW_CYCLES);
}
// Not intended for nightly or continuous testing. Comment in when needed during development.
// @Test
public void testLargeDependencyGraph() {
Timer timer = new Timer("DependencyGraph");
timer.mark();
DependencyGraph<String> graph = new DependencyGraph<>();
for (DependencyRelation relation : testRelationships) {
graph.addDependency(relation.dependent, relation.dependee);
}
timer.mark();
while (!graph.isEmpty()) {
graph.pop();
}
timer.mark();
System.out.println(timer);
}
// Not intended for nightly or continuous testing. Comment in when needed during development.
// @Test
public void testLargeDeterministicDependencyGraph() {
Timer timer = new Timer("DeterministicDependencyGraph");
timer.mark();
DeterministicDependencyGraph<String> graph = new DeterministicDependencyGraph<>();
for (DependencyRelation relation : testRelationships) {
graph.addDependency(relation.dependent, relation.dependee);
}
timer.mark();
while (!graph.isEmpty()) {
graph.pop();
}
timer.mark();
System.out.println(timer);
}
private class Timer {
private String testName;
private List<Long> times = new ArrayList<>();
public Timer(String testName) {
this.testName = testName;
}
public void mark() {
times.add(System.currentTimeMillis());
}
@Override
public String toString() {
String report = testName + " (milliseconds)\n";
if (!times.isEmpty()) {
long prev = times.get(0);
long total = times.get(times.size() - 1) - prev;
for (int i = 1; i < times.size(); i++) {
long current = times.get(i);
long diff = current - prev;
report += String.format(" %03d: %d\n", i, diff);
prev = current;
}
report += String.format("total: %d\n", total);
}
return report;
}
}
private class DependencyRelation {
public String dependent;
public String dependee;
public DependencyRelation(int dependentId, int dependeeId) {
dependent = "V" + dependentId;
dependee = "V" + dependeeId;
}
}
private List<DependencyRelation> constructRandomRelationships(long seed, int numDependencies,
int graphSize, boolean allowCycles) {
List<DependencyRelation> relationships = new ArrayList<>();
Random generator = new Random(seed);
// Not taking a dependent beyond 90% of graph size; similar dependee only in latter
// 90%.
double factor = 0.90;
int limit = (int) (factor * graphSize);
int dependeeOffset = graphSize - limit;
assert limit != graphSize;
// TODO: currently ignoring allowCycles parameter.
// TODO: Do no cycles for now... ask if would need both types for performance testing.
// To disallow cycles, we simply are preventing the dependee number from being less than
// the dependent number. This weights the graph in an interesting way: dependents with
// smaller numbers will tend to have more dependees.
for (int i = 0; i < numDependencies; i++) {
int dependentId = generator.nextInt(limit);
int dependeeId;
do {
dependeeId = generator.nextInt(limit) + dependeeOffset;
}
while (dependeeId <= dependentId);
DependencyRelation relation = new DependencyRelation(dependentId, dependeeId);
relationships.add(relation);
}
return relationships;
}
}

132
Ghidra/Framework/Generic/src/test/java/ghidra/util/datastruct/DependencyGraphTest.java
View File

@ -22,14 +22,95 @@ import java.util.*;
import org.junit.Assert;
import org.junit.Test;
import ghidra.util.graph.DependencyGraph;
import ghidra.util.graph.*;
public class DependencyGraphTest {
@Test
public void testSimpleCase() {
DependencyGraph<Integer> graph = new DependencyGraph<Integer>();
public void testSimpleCaseDependencyGraph() {
AbstractDependencyGraph<Integer> graph = new DependencyGraph<>();
runSimpleCase(graph);
}
@Test
public void testSimpleCaseDeterministicDependencyGraph() {
AbstractDependencyGraph<Integer> graph = new DeterministicDependencyGraph<>();
runSimpleCase(graph);
}
@Test
public void testMultipleDependencyCaseDependencyGraph() {
AbstractDependencyGraph<Integer> graph = new DependencyGraph<>();
runMultipleDependencyCase(graph);
}
@Test
public void testMultipleDependencyCaseDeterministicDependencyGraph() {
AbstractDependencyGraph<Integer> graph = new DeterministicDependencyGraph<>();
runMultipleDependencyCase(graph);
}
@Test
public void testPopDependencyGraph() {
AbstractDependencyGraph<Integer> graph = new DependencyGraph<>();
runPop(graph);
}
@Test
public void testPopDeterministicDependencyGraph() {
AbstractDependencyGraph<Integer> graph = new DeterministicDependencyGraph<>();
runPop(graph);
}
@Test
public void testPopWithCycleDependencyGraph() {
AbstractDependencyGraph<Integer> graph = new DependencyGraph<>();
runPopWithCycle(graph);
}
@Test
public void testPopWithCycleDeterministicDependencyGraph() {
AbstractDependencyGraph<Integer> graph = new DeterministicDependencyGraph<>();
runPopWithCycle(graph);
}
@Test
public void testCycleDetectionDependencyGraph() {
AbstractDependencyGraph<Integer> graph = new DependencyGraph<>();
runCycleDetection(graph);
}
@Test
public void testCycleDetectionDeterministicDependencyGraph() {
AbstractDependencyGraph<Integer> graph = new DeterministicDependencyGraph<>();
runCycleDetection(graph);
}
@Test
public void testCycleDetectionDoesNotCorruptGraphDependencyGraph() {
AbstractDependencyGraph<Integer> graph = new DependencyGraph<>();
runCycleDetectionDoesNotCorruptGraph(graph);
}
@Test
public void testCycleDetectionDoesNotCorruptGraphDeterministicDependencyGraph() {
AbstractDependencyGraph<Integer> graph = new DeterministicDependencyGraph<>();
runCycleDetectionDoesNotCorruptGraph(graph);
}
@Test
public void testRandomProcessingOfDependenciesSimulationDependencyGraph() {
AbstractDependencyGraph<String> graph = new DependencyGraph<>();
runRandomProcessingOfDependenciesSimulation(graph);
}
@Test
public void testRandomProcessingOfDependenciesSimulationDeterministicDependencyGraph() {
AbstractDependencyGraph<String> graph = new DeterministicDependencyGraph<>();
runRandomProcessingOfDependenciesSimulation(graph);
}
private void runSimpleCase(AbstractDependencyGraph<Integer> graph) {
graph.addDependency(1, 2);
graph.addDependency(2, 3);
graph.addDependency(3, 4);
@ -56,13 +137,9 @@ public class DependencyGraphTest {
graph.remove(1);
set = graph.getUnvisitedIndependentValues();
assertTrue(set.isEmpty());
}
@Test
public void testMultipleDependencyCase() {
DependencyGraph<Integer> graph = new DependencyGraph<Integer>();
private void runMultipleDependencyCase(AbstractDependencyGraph<Integer> graph) {
graph.addDependency(1, 2);
graph.addDependency(2, 3);
graph.addDependency(1, 3);
@ -84,13 +161,9 @@ public class DependencyGraphTest {
graph.remove(1);
set = graph.getUnvisitedIndependentValues();
assertTrue(set.isEmpty());
}
@Test
public void testPop() {
DependencyGraph<Integer> graph = new DependencyGraph<Integer>();
private void runPop(AbstractDependencyGraph<Integer> graph) {
graph.addDependency(1, 2);
graph.addDependency(2, 3);
graph.addDependency(3, 4);
@ -103,10 +176,7 @@ public class DependencyGraphTest {
assertNull(graph.pop());
}
@Test
public void testPopWithCycle() {
DependencyGraph<Integer> graph = new DependencyGraph<Integer>();
private void runPopWithCycle(AbstractDependencyGraph<Integer> graph) {
graph.addDependency(1, 2);
graph.addDependency(2, 3);
graph.addDependency(3, 4);
@ -121,13 +191,9 @@ public class DependencyGraphTest {
catch (IllegalStateException e) {
// expected
}
}
@Test
public void testCycleDetection() {
DependencyGraph<Integer> graph = new DependencyGraph<Integer>();
private void runCycleDetection(AbstractDependencyGraph<Integer> graph) {
graph.addDependency(1, 2);
graph.addDependency(2, 3);
graph.addDependency(3, 4);
@ -137,13 +203,9 @@ public class DependencyGraphTest {
graph.addDependency(4, 1);
assertTrue(graph.hasCycles());
}
@Test
public void testCycleDetectionDoesNotCorruptGraph() {
DependencyGraph<Integer> graph = new DependencyGraph<Integer>();
private void runCycleDetectionDoesNotCorruptGraph(AbstractDependencyGraph<Integer> graph) {
graph.addDependency(1, 2);
graph.addDependency(2, 3);
graph.addDependency(3, 4);
@ -172,14 +234,12 @@ public class DependencyGraphTest {
graph.remove(1);
set = graph.getUnvisitedIndependentValues();
assertTrue(set.isEmpty());
}
@Test
public void testRandomProcessingOfDependenciesSimulation() {
final ArrayList<String> completionOrder = new ArrayList<String>();
private void runRandomProcessingOfDependenciesSimulation(
AbstractDependencyGraph<String> graph) {
final ArrayList<String> completionOrder = new ArrayList<>();
DependencyGraph<String> graph = new DependencyGraph<String>();
graph.addDependency("@0", "A8");
graph.addDependency("@1", "A1");
graph.addDependency("@2", "A7");
@ -214,7 +274,7 @@ public class DependencyGraphTest {
assertTrue(!graph.hasCycles());
DependencyGraph<String> savedGraph = graph.copy();
AbstractDependencyGraph<String> savedGraph = graph.copy();
while (!graph.isEmpty()) {
completionOrder.add(graph.pop());
@ -231,7 +291,7 @@ public class DependencyGraphTest {
* @param visitedOrder the actual execution order to be tested
* @return
*/
public void checkOrderSatisfiesDependencies(DependencyGraph<String> dependencyGraph,
private void checkOrderSatisfiesDependencies(AbstractDependencyGraph<String> dependencyGraph,
List<String> visitedOrder) {
if (visitedOrder.size() > dependencyGraph.size()) {
@ -241,12 +301,12 @@ public class DependencyGraphTest {
Assert.fail("Not all items in the graph were visited");
}
HashSet<String> items = new HashSet<String>(visitedOrder);
HashSet<String> items = new HashSet<>(visitedOrder);
if (items.size() != visitedOrder.size()) {
Assert.fail("duplicate item(s) in linearOrder\n");
}
HashMap<String, Integer> visitedOrderMap = new HashMap<String, Integer>();
HashMap<String, Integer> visitedOrderMap = new HashMap<>();
for (int i = 0; i < visitedOrder.size(); i++) {
visitedOrderMap.put(visitedOrder.get(i), i);
}

7
Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/util/AcyclicCallGraphBuilder.java
View File

@ -22,6 +22,7 @@ import ghidra.program.model.listing.*;
import ghidra.program.model.symbol.Reference;
import ghidra.program.model.symbol.ReferenceManager;
import ghidra.util.exception.CancelledException;
import ghidra.util.graph.AbstractDependencyGraph;
import ghidra.util.graph.DependencyGraph;
import ghidra.util.task.TaskMonitor;
@ -84,10 +85,10 @@ public class AcyclicCallGraphBuilder {
* @return the DependencyGraph for the acyclic call graph represented by this object.
* @throws CancelledException if the monitor was cancelled.
*/
public DependencyGraph<Address> getDependencyGraph(TaskMonitor monitor)
public AbstractDependencyGraph<Address> getDependencyGraph(TaskMonitor monitor)
throws CancelledException {
DependencyGraph<Address> graph = new DependencyGraph<>();
AbstractDependencyGraph<Address> graph = new DependencyGraph<>();
Deque<Address> startPoints = findStartPoints();
Set<Address> unprocessed = new TreeSet<>(functionSet); // reliable processing order
@ -158,7 +159,7 @@ public class AcyclicCallGraphBuilder {
children.toArray(node.children);
}
private void processForward(DependencyGraph<Address> graph, Set<Address> unprocessed,
private void processForward(AbstractDependencyGraph<Address> graph, Set<Address> unprocessed,
Address startFunction, TaskMonitor monitor) throws CancelledException {
VisitStack stack = new VisitStack(startFunction);
StackNode curnode = stack.peek();

20
Ghidra/Framework/SoftwareModeling/src/test/java/ghidra/program/model/util/AcyclicCallGraphBuilderTest.java
View File

@ -27,7 +27,7 @@ import ghidra.program.model.*;
import ghidra.program.model.address.*;
import ghidra.program.model.listing.*;
import ghidra.program.model.symbol.ReferenceManager;
import ghidra.util.graph.DependencyGraph;
import ghidra.util.graph.AbstractDependencyGraph;
public class AcyclicCallGraphBuilderTest extends AbstractGenericTest {
@ -102,7 +102,7 @@ public class AcyclicCallGraphBuilderTest extends AbstractGenericTest {
node(3, 4);
AcyclicCallGraphBuilder builder = new AcyclicCallGraphBuilder(program, functions, false);
DependencyGraph<Address> graph = builder.getDependencyGraph(DUMMY_MONITOR);
AbstractDependencyGraph<Address> graph = builder.getDependencyGraph(DUMMY_MONITOR);
Assert.assertEquals(4, graph.size());
@ -119,7 +119,7 @@ public class AcyclicCallGraphBuilderTest extends AbstractGenericTest {
node(2,3);
AcyclicCallGraphBuilder builder = new AcyclicCallGraphBuilder(program,functions,false);
DependencyGraph<Address> graph = builder.getDependencyGraph(DUMMY_MONITOR);
AbstractDependencyGraph<Address> graph = builder.getDependencyGraph(DUMMY_MONITOR);
Assert.assertEquals(3, graph.size());
@ -135,7 +135,7 @@ public class AcyclicCallGraphBuilderTest extends AbstractGenericTest {
node(3, 1);
AcyclicCallGraphBuilder builder = new AcyclicCallGraphBuilder(program, functions, false);
DependencyGraph<Address> graph = builder.getDependencyGraph(DUMMY_MONITOR);
AbstractDependencyGraph<Address> graph = builder.getDependencyGraph(DUMMY_MONITOR);
Assert.assertEquals(3, graph.size());
@ -152,7 +152,7 @@ public class AcyclicCallGraphBuilderTest extends AbstractGenericTest {
node(2, 2);
AcyclicCallGraphBuilder builder = new AcyclicCallGraphBuilder(program, functions, false);
DependencyGraph<Address> graph = builder.getDependencyGraph(DUMMY_MONITOR);
AbstractDependencyGraph<Address> graph = builder.getDependencyGraph(DUMMY_MONITOR);
Assert.assertEquals(3, graph.size());
@ -168,7 +168,7 @@ public class AcyclicCallGraphBuilderTest extends AbstractGenericTest {
node(3, 1);
AcyclicCallGraphBuilder builder = new AcyclicCallGraphBuilder(program, functions, false);
DependencyGraph<Address> graph = builder.getDependencyGraph(DUMMY_MONITOR);
AbstractDependencyGraph<Address> graph = builder.getDependencyGraph(DUMMY_MONITOR);
Assert.assertEquals(3, graph.size());
@ -197,7 +197,7 @@ public class AcyclicCallGraphBuilderTest extends AbstractGenericTest {
thunkNode(17, 18, false);
AcyclicCallGraphBuilder builder = new AcyclicCallGraphBuilder(program, functions, false);
DependencyGraph<Address> graph = builder.getDependencyGraph(DUMMY_MONITOR);
AbstractDependencyGraph<Address> graph = builder.getDependencyGraph(DUMMY_MONITOR);
Assert.assertEquals(18, graph.size());
@ -241,7 +241,7 @@ public class AcyclicCallGraphBuilderTest extends AbstractGenericTest {
thunkNode(17, 18, false);
AcyclicCallGraphBuilder builder = new AcyclicCallGraphBuilder(program, functions, true);
DependencyGraph<Address> graph = builder.getDependencyGraph(DUMMY_MONITOR);
AbstractDependencyGraph<Address> graph = builder.getDependencyGraph(DUMMY_MONITOR);
Assert.assertEquals(8, graph.size());
@ -265,7 +265,7 @@ public class AcyclicCallGraphBuilderTest extends AbstractGenericTest {
thunkNode(5, 3, true); // Thunk node hits recursion from different point
AcyclicCallGraphBuilder builder = new AcyclicCallGraphBuilder(program, functions, true);
DependencyGraph<Address> graph = builder.getDependencyGraph(DUMMY_MONITOR);
AbstractDependencyGraph<Address> graph = builder.getDependencyGraph(DUMMY_MONITOR);
Assert.assertEquals(4, graph.size());
assertDependents(graph, 2, 1);
@ -274,7 +274,7 @@ public class AcyclicCallGraphBuilderTest extends AbstractGenericTest {
Assert.assertFalse(graph.hasCycles());
}
private void assertDependents(DependencyGraph<Address> graph, int fromID, int... toIDs) {
private void assertDependents(AbstractDependencyGraph<Address> graph, int fromID, int... toIDs) {
Set<Address> expectedSet = new HashSet<Address>();
for (int toAddr : toIDs) {
expectedSet.add(functionAddress(toAddr));