OSchip
/
llvm-project
mirror of https://github.com/THU-DSP-LAB/llvm-project.git
13
0
Fork 6
Code Issues Packages Projects Releases Wiki Activity
llvm-project/llvm/lib/Transforms/Utils/SampleProfileInference.cpp

846 lines
29 KiB
C++
Raw Blame History

//===- SampleProfileInference.cpp - Adjust sample profiles in the IR ------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file implements a profile inference algorithm. Given an incomplete and
// possibly imprecise block counts, the algorithm reconstructs realistic block
// and edge counts that satisfy flow conservation rules, while minimally modify
// input block counts.
//
//===----------------------------------------------------------------------===//
#include "llvm/Transforms/Utils/SampleProfileInference.h"
#include "llvm/Support/Debug.h"
#include <queue>
#include <set>
using namespace llvm;
#define DEBUG_TYPE "sample-profile-inference"
namespace {
/// A value indicating an infinite flow/capacity/weight of a block/edge.
/// Not using numeric_limits<int64_t>::max(), as the values can be summed up
/// during the execution.
static constexpr int64_t INF = ((int64_t)1) << 50;
/// The minimum-cost maximum flow algorithm.
///
/// The algorithm finds the maximum flow of minimum cost on a given (directed)
/// network using a modified version of the classical Moore-Bellman-Ford
/// approach. The algorithm applies a number of augmentation iterations in which
/// flow is sent along paths of positive capacity from the source to the sink.
/// The worst-case time complexity of the implementation is O(v(f)*m*n), where
/// where m is the number of edges, n is the number of vertices, and v(f) is the
/// value of the maximum flow. However, the observed running time on typical
/// instances is sub-quadratic, that is, o(n^2).
///
/// The input is a set of edges with specified costs and capacities, and a pair
/// of nodes (source and sink). The output is the flow along each edge of the
/// minimum total cost respecting the given edge capacities.
class MinCostMaxFlow {
public:
// Initialize algorithm's data structures for a network of a given size.
void initialize(uint64_t NodeCount, uint64_t SourceNode, uint64_t SinkNode) {
Source = SourceNode;
Target = SinkNode;
Nodes = std::vector<Node>(NodeCount);
Edges = std::vector<std::vector<Edge>>(NodeCount, std::vector<Edge>());
}
// Run the algorithm.
int64_t run() {
// Find an augmenting path and update the flow along the path
size_t AugmentationIters = 0;
while (findAugmentingPath()) {
augmentFlowAlongPath();
AugmentationIters++;
}
// Compute the total flow and its cost
int64_t TotalCost = 0;
int64_t TotalFlow = 0;
for (uint64_t Src = 0; Src < Nodes.size(); Src++) {
for (auto &Edge : Edges[Src]) {
if (Edge.Flow > 0) {
TotalCost += Edge.Cost * Edge.Flow;
if (Src == Source)
TotalFlow += Edge.Flow;
}
}
}
LLVM_DEBUG(dbgs() << "Completed profi after " << AugmentationIters
<< " iterations with " << TotalFlow << " total flow"
<< " of " << TotalCost << " cost\n");
(void)TotalFlow;
return TotalCost;
}
/// Adding an edge to the network with a specified capacity and a cost.
/// Multiple edges between a pair of nodes are allowed but self-edges
/// are not supported.
void addEdge(uint64_t Src, uint64_t Dst, int64_t Capacity, int64_t Cost) {
assert(Capacity > 0 && "adding an edge of zero capacity");
assert(Src != Dst && "loop edge are not supported");
Edge SrcEdge;
SrcEdge.Dst = Dst;
SrcEdge.Cost = Cost;
SrcEdge.Capacity = Capacity;
SrcEdge.Flow = 0;
SrcEdge.RevEdgeIndex = Edges[Dst].size();
Edge DstEdge;
DstEdge.Dst = Src;
DstEdge.Cost = -Cost;
DstEdge.Capacity = 0;
DstEdge.Flow = 0;
DstEdge.RevEdgeIndex = Edges[Src].size();
Edges[Src].push_back(SrcEdge);
Edges[Dst].push_back(DstEdge);
}
/// Adding an edge to the network of infinite capacity and a given cost.
void addEdge(uint64_t Src, uint64_t Dst, int64_t Cost) {
addEdge(Src, Dst, INF, Cost);
}
/// Get the total flow from a given source node.
/// Returns a list of pairs (target node, amount of flow to the target).
const std::vector<std::pair<uint64_t, int64_t>> getFlow(uint64_t Src) const {
std::vector<std::pair<uint64_t, int64_t>> Flow;
for (auto &Edge : Edges[Src]) {
if (Edge.Flow > 0)
Flow.push_back(std::make_pair(Edge.Dst, Edge.Flow));
}
return Flow;
}
/// Get the total flow between a pair of nodes.
int64_t getFlow(uint64_t Src, uint64_t Dst) const {
int64_t Flow = 0;
for (auto &Edge : Edges[Src]) {
if (Edge.Dst == Dst) {
Flow += Edge.Flow;
}
}
return Flow;
}
/// A cost of increasing a block's count by one.
static constexpr int64_t AuxCostInc = 10;
/// A cost of decreasing a block's count by one.
static constexpr int64_t AuxCostDec = 20;
/// A cost of increasing a count of zero-weight block by one.
static constexpr int64_t AuxCostIncZero = 11;
/// A cost of increasing the entry block's count by one.
static constexpr int64_t AuxCostIncEntry = 40;
/// A cost of decreasing the entry block's count by one.
static constexpr int64_t AuxCostDecEntry = 10;
/// A cost of taking an unlikely jump.
static constexpr int64_t AuxCostUnlikely = ((int64_t)1) << 20;
private:
/// Check for existence of an augmenting path with a positive capacity.
bool findAugmentingPath() {
// Initialize data structures
for (auto &Node : Nodes) {
Node.Distance = INF;
Node.ParentNode = uint64_t(-1);
Node.ParentEdgeIndex = uint64_t(-1);
Node.Taken = false;
}
std::queue<uint64_t> Queue;
Queue.push(Source);
Nodes[Source].Distance = 0;
Nodes[Source].Taken = true;
while (!Queue.empty()) {
uint64_t Src = Queue.front();
Queue.pop();
Nodes[Src].Taken = false;
// Although the residual network contains edges with negative costs
// (in particular, backward edges), it can be shown that there are no
// negative-weight cycles and the following two invariants are maintained:
// (i) Dist[Source, V] >= 0 and (ii) Dist[V, Target] >= 0 for all nodes V,
// where Dist is the length of the shortest path between two nodes. This
// allows to prune the search-space of the path-finding algorithm using
// the following early-stop criteria:
// -- If we find a path with zero-distance from Source to Target, stop the
// search, as the path is the shortest since Dist[Source, Target] >= 0;
// -- If we have Dist[Source, V] > Dist[Source, Target], then do not
// process node V, as it is guaranteed _not_ to be on a shortest path
// from Source to Target; it follows from inequalities
// Dist[Source, Target] >= Dist[Source, V] + Dist[V, Target]
// >= Dist[Source, V]
if (Nodes[Target].Distance == 0)
break;
if (Nodes[Src].Distance > Nodes[Target].Distance)
continue;
// Process adjacent edges
for (uint64_t EdgeIdx = 0; EdgeIdx < Edges[Src].size(); EdgeIdx++) {
auto &Edge = Edges[Src][EdgeIdx];
if (Edge.Flow < Edge.Capacity) {
uint64_t Dst = Edge.Dst;
int64_t NewDistance = Nodes[Src].Distance + Edge.Cost;
if (Nodes[Dst].Distance > NewDistance) {
// Update the distance and the parent node/edge
Nodes[Dst].Distance = NewDistance;
Nodes[Dst].ParentNode = Src;
Nodes[Dst].ParentEdgeIndex = EdgeIdx;
// Add the node to the queue, if it is not there yet
if (!Nodes[Dst].Taken) {
Queue.push(Dst);
Nodes[Dst].Taken = true;
}
}
}
}
}
return Nodes[Target].Distance != INF;
}
/// Update the current flow along the augmenting path.
void augmentFlowAlongPath() {
// Find path capacity
int64_t PathCapacity = INF;
uint64_t Now = Target;
while (Now != Source) {
uint64_t Pred = Nodes[Now].ParentNode;
auto &Edge = Edges[Pred][Nodes[Now].ParentEdgeIndex];
PathCapacity = std::min(PathCapacity, Edge.Capacity - Edge.Flow);
Now = Pred;
}
assert(PathCapacity > 0 && "found an incorrect augmenting path");
// Update the flow along the path
Now = Target;
while (Now != Source) {
uint64_t Pred = Nodes[Now].ParentNode;
auto &Edge = Edges[Pred][Nodes[Now].ParentEdgeIndex];
auto &RevEdge = Edges[Now][Edge.RevEdgeIndex];
Edge.Flow += PathCapacity;
RevEdge.Flow -= PathCapacity;
Now = Pred;
}
}
/// An node in a flow network.
struct Node {
/// The cost of the cheapest path from the source to the current node.
int64_t Distance;
/// The node preceding the current one in the path.
uint64_t ParentNode;
/// The index of the edge between ParentNode and the current node.
uint64_t ParentEdgeIndex;
/// An indicator of whether the current node is in a queue.
bool Taken;
};
/// An edge in a flow network.
struct Edge {
/// The cost of the edge.
int64_t Cost;
/// The capacity of the edge.
int64_t Capacity;
/// The current flow on the edge.
int64_t Flow;
/// The destination node of the edge.
uint64_t Dst;
/// The index of the reverse edge between Dst and the current node.
uint64_t RevEdgeIndex;
};
/// The set of network nodes.
std::vector<Node> Nodes;
/// The set of network edges.
std::vector<std::vector<Edge>> Edges;
/// Source node of the flow.
uint64_t Source;
/// Target (sink) node of the flow.
uint64_t Target;
};
/// A post-processing adjustment of control flow. It applies two steps by
/// rerouting some flow and making it more realistic:
///
/// - First, it removes all isolated components ("islands") with a positive flow
/// that are unreachable from the entry block. For every such component, we
/// find the shortest from the entry to an exit passing through the component,
/// and increase the flow by one unit along the path.
///
/// - Second, it identifies all "unknown subgraphs" consisting of basic blocks
/// with no sampled counts. Then it rebalnces the flow that goes through such
/// a subgraph so that each branch is taken with probability 50%.
/// An unknown subgraph is such that for every two nodes u and v:
/// - u dominates v and u is not unknown;
/// - v post-dominates u; and
/// - all inner-nodes of all (u,v)-paths are unknown.
///
class FlowAdjuster {
public:
FlowAdjuster(FlowFunction &Func) : Func(Func) {
assert(Func.Blocks[Func.Entry].isEntry() &&
"incorrect index of the entry block");
}
// Run the post-processing
void run() {
/// Adjust the flow to get rid of isolated components.
joinIsolatedComponents();
/// Rebalance the flow inside unknown subgraphs.
rebalanceUnknownSubgraphs();
}
/// The probability for the first successor of a unknown subgraph
static constexpr double UnknownFirstSuccProbability = 0.5;
private:
void joinIsolatedComponents() {
// Find blocks that are reachable from the source
auto Visited = std::vector<bool>(NumBlocks(), false);
findReachable(Func.Entry, Visited);
// Iterate over all non-reachable blocks and adjust their weights
for (uint64_t I = 0; I < NumBlocks(); I++) {
auto &Block = Func.Blocks[I];
if (Block.Flow > 0 && !Visited[I]) {
// Find a path from the entry to an exit passing through the block I
auto Path = findShortestPath(I);
// Increase the flow along the path
assert(Path.size() > 0 && Path[0]->Source == Func.Entry &&
"incorrectly computed path adjusting control flow");
Func.Blocks[Func.Entry].Flow += 1;
for (auto &Jump : Path) {
Jump->Flow += 1;
Func.Blocks[Jump->Target].Flow += 1;
// Update reachability
findReachable(Jump->Target, Visited);
}
}
}
}
/// Run BFS from a given block along the jumps with a positive flow and mark
/// all reachable blocks.
void findReachable(uint64_t Src, std::vector<bool> &Visited) {
if (Visited[Src])
return;
std::queue<uint64_t> Queue;
Queue.push(Src);
Visited[Src] = true;
while (!Queue.empty()) {
Src = Queue.front();
Queue.pop();
for (auto Jump : Func.Blocks[Src].SuccJumps) {
uint64_t Dst = Jump->Target;
if (Jump->Flow > 0 && !Visited[Dst]) {
Queue.push(Dst);
Visited[Dst] = true;
}
}
}
}
/// Find the shortest path from the entry block to an exit block passing
/// through a given block.
std::vector<FlowJump *> findShortestPath(uint64_t BlockIdx) {
// A path from the entry block to BlockIdx
auto ForwardPath = findShortestPath(Func.Entry, BlockIdx);
// A path from BlockIdx to an exit block
auto BackwardPath = findShortestPath(BlockIdx, AnyExitBlock);
// Concatenate the two paths
std::vector<FlowJump *> Result;
Result.insert(Result.end(), ForwardPath.begin(), ForwardPath.end());
Result.insert(Result.end(), BackwardPath.begin(), BackwardPath.end());
return Result;
}
/// Apply the Dijkstra algorithm to find the shortest path from a given
/// Source to a given Target block.
/// If Target == -1, then the path ends at an exit block.
std::vector<FlowJump *> findShortestPath(uint64_t Source, uint64_t Target) {
// Quit early, if possible
if (Source == Target)
return std::vector<FlowJump *>();
if (Func.Blocks[Source].isExit() && Target == AnyExitBlock)
return std::vector<FlowJump *>();
// Initialize data structures
auto Distance = std::vector<int64_t>(NumBlocks(), INF);
auto Parent = std::vector<FlowJump *>(NumBlocks(), nullptr);
Distance[Source] = 0;
std::set<std::pair<uint64_t, uint64_t>> Queue;
Queue.insert(std::make_pair(Distance[Source], Source));
// Run the Dijkstra algorithm
while (!Queue.empty()) {
uint64_t Src = Queue.begin()->second;
Queue.erase(Queue.begin());
// If we found a solution, quit early
if (Src == Target ||
(Func.Blocks[Src].isExit() && Target == AnyExitBlock))
break;
for (auto Jump : Func.Blocks[Src].SuccJumps) {
uint64_t Dst = Jump->Target;
int64_t JumpDist = jumpDistance(Jump);
if (Distance[Dst] > Distance[Src] + JumpDist) {
Queue.erase(std::make_pair(Distance[Dst], Dst));
Distance[Dst] = Distance[Src] + JumpDist;
Parent[Dst] = Jump;
Queue.insert(std::make_pair(Distance[Dst], Dst));
}
}
}
// If Target is not provided, find the closest exit block
if (Target == AnyExitBlock) {
for (uint64_t I = 0; I < NumBlocks(); I++) {
if (Func.Blocks[I].isExit() && Parent[I] != nullptr) {
if (Target == AnyExitBlock || Distance[Target] > Distance[I]) {
Target = I;
}
}
}
}
assert(Parent[Target] != nullptr && "a path does not exist");
// Extract the constructed path
std::vector<FlowJump *> Result;
uint64_t Now = Target;
while (Now != Source) {
assert(Now == Parent[Now]->Target && "incorrect parent jump");
Result.push_back(Parent[Now]);
Now = Parent[Now]->Source;
}
// Reverse the path, since it is extracted from Target to Source
std::reverse(Result.begin(), Result.end());
return Result;
}
/// A distance of a path for a given jump.
/// In order to incite the path to use blocks/jumps with large positive flow,
/// and avoid changing branch probability of outgoing edges drastically,
/// set the distance as follows:
/// if Jump.Flow > 0, then distance = max(100 - Jump->Flow, 0)
/// if Block.Weight > 0, then distance = 1
/// otherwise distance >> 1
int64_t jumpDistance(FlowJump *Jump) const {
int64_t BaseDistance = 100;
if (Jump->IsUnlikely)
return MinCostMaxFlow::AuxCostUnlikely;
if (Jump->Flow > 0)
return std::max(BaseDistance - (int64_t)Jump->Flow, (int64_t)0);
if (Func.Blocks[Jump->Target].Weight > 0)
return BaseDistance;
return BaseDistance * (NumBlocks() + 1);
};
uint64_t NumBlocks() const { return Func.Blocks.size(); }
/// Rebalance unknown subgraphs so as each branch splits with probabilities
/// UnknownFirstSuccProbability and 1 - UnknownFirstSuccProbability
void rebalanceUnknownSubgraphs() {
assert(UnknownFirstSuccProbability >= 0.0 &&
UnknownFirstSuccProbability <= 1.0 &&
"the share of the unknown successor should be between 0 and 1");
// Try to find unknown subgraphs from each non-unknown block
for (uint64_t I = 0; I < Func.Blocks.size(); I++) {
auto SrcBlock = &Func.Blocks[I];
// Do not attempt to find unknown successors from a unknown or a
// zero-flow block
if (SrcBlock->UnknownWeight || SrcBlock->Flow == 0)
continue;
std::vector<FlowBlock *> UnknownSuccs;
FlowBlock *DstBlock = nullptr;
// Find a unknown subgraphs starting at block SrcBlock
if (!findUnknownSubgraph(SrcBlock, DstBlock, UnknownSuccs))
continue;
// At the moment, we do not rebalance subgraphs containing cycles among
// unknown blocks
if (!isAcyclicSubgraph(SrcBlock, DstBlock, UnknownSuccs))
continue;
// Rebalance the flow
rebalanceUnknownSubgraph(SrcBlock, DstBlock, UnknownSuccs);
}
}
/// Find a unknown subgraph starting at block SrcBlock.
/// If the search is successful, the method sets DstBlock and UnknownSuccs.
bool findUnknownSubgraph(FlowBlock *SrcBlock, FlowBlock *&DstBlock,
std::vector<FlowBlock *> &UnknownSuccs) {
// Run BFS from SrcBlock and make sure all paths are going through unknown
// blocks and end at a non-unknown DstBlock
auto Visited = std::vector<bool>(NumBlocks(), false);
std::queue<uint64_t> Queue;
DstBlock = nullptr;
Queue.push(SrcBlock->Index);
Visited[SrcBlock->Index] = true;
while (!Queue.empty()) {
auto &Block = Func.Blocks[Queue.front()];
Queue.pop();
// Process blocks reachable from Block
for (auto Jump : Block.SuccJumps) {
uint64_t Dst = Jump->Target;
if (Visited[Dst])
continue;
Visited[Dst] = true;
if (!Func.Blocks[Dst].UnknownWeight) {
// If we see non-unique non-unknown block reachable from SrcBlock,
// stop processing and skip rebalancing
FlowBlock *CandidateDstBlock = &Func.Blocks[Dst];
if (DstBlock != nullptr && DstBlock != CandidateDstBlock)
return false;
DstBlock = CandidateDstBlock;
} else {
Queue.push(Dst);
UnknownSuccs.push_back(&Func.Blocks[Dst]);
}
}
}
// If the list of unknown blocks is empty, we don't need rebalancing
if (UnknownSuccs.empty())
return false;
// If all reachable nodes from SrcBlock are unknown, skip rebalancing
if (DstBlock == nullptr)
return false;
// If any of the unknown blocks is an exit block, skip rebalancing
for (auto Block : UnknownSuccs) {
if (Block->isExit())
return false;
}
return true;
}
/// Verify if the given unknown subgraph is acyclic, and if yes, reorder
/// UnknownSuccs in the topological order (so that all jumps are "forward").
bool isAcyclicSubgraph(FlowBlock *SrcBlock, FlowBlock *DstBlock,
std::vector<FlowBlock *> &UnknownSuccs) {
// Extract local in-degrees in the considered subgraph
auto LocalInDegree = std::vector<uint64_t>(NumBlocks(), 0);
for (auto Jump : SrcBlock->SuccJumps) {
LocalInDegree[Jump->Target]++;
}
for (uint64_t I = 0; I < UnknownSuccs.size(); I++) {
for (auto Jump : UnknownSuccs[I]->SuccJumps) {
LocalInDegree[Jump->Target]++;
}
}
// A loop containing SrcBlock
if (LocalInDegree[SrcBlock->Index] > 0)
return false;
std::vector<FlowBlock *> AcyclicOrder;
std::queue<uint64_t> Queue;
Queue.push(SrcBlock->Index);
while (!Queue.empty()) {
auto &Block = Func.Blocks[Queue.front()];
Queue.pop();
// Stop propagation once we reach DstBlock
if (Block.Index == DstBlock->Index)
break;
AcyclicOrder.push_back(&Block);
// Add to the queue all successors with zero local in-degree
for (auto Jump : Block.SuccJumps) {
uint64_t Dst = Jump->Target;
LocalInDegree[Dst]--;
if (LocalInDegree[Dst] == 0) {
Queue.push(Dst);
}
}
}
// If there is a cycle in the subgraph, AcyclicOrder contains only a subset
// of all blocks
if (UnknownSuccs.size() + 1 != AcyclicOrder.size())
return false;
UnknownSuccs = AcyclicOrder;
return true;
}
/// Rebalance a given subgraph.
void rebalanceUnknownSubgraph(FlowBlock *SrcBlock, FlowBlock *DstBlock,
std::vector<FlowBlock *> &UnknownSuccs) {
assert(SrcBlock->Flow > 0 && "zero-flow block in unknown subgraph");
assert(UnknownSuccs.front() == SrcBlock && "incorrect order of unknowns");
for (auto Block : UnknownSuccs) {
// Block's flow is the sum of incoming flows
uint64_t TotalFlow = 0;
if (Block == SrcBlock) {
TotalFlow = Block->Flow;
} else {
for (auto Jump : Block->PredJumps) {
TotalFlow += Jump->Flow;
}
Block->Flow = TotalFlow;
}
// Process all successor jumps and update corresponding flow values
for (uint64_t I = 0; I < Block->SuccJumps.size(); I++) {
auto Jump = Block->SuccJumps[I];
if (I + 1 == Block->SuccJumps.size()) {
Jump->Flow = TotalFlow;
continue;
}
uint64_t Flow = uint64_t(TotalFlow * UnknownFirstSuccProbability);
Jump->Flow = Flow;
TotalFlow -= Flow;
}
}
}
/// A constant indicating an arbitrary exit block of a function.
static constexpr uint64_t AnyExitBlock = uint64_t(-1);
/// The function.
FlowFunction &Func;
};
/// Initializing flow network for a given function.
///
/// Every block is split into three nodes that are responsible for (i) an
/// incoming flow, (ii) an outgoing flow, and (iii) penalizing an increase or
/// reduction of the block weight.
void initializeNetwork(MinCostMaxFlow &Network, FlowFunction &Func) {
uint64_t NumBlocks = Func.Blocks.size();
assert(NumBlocks > 1 && "Too few blocks in a function");
LLVM_DEBUG(dbgs() << "Initializing profi for " << NumBlocks << " blocks\n");
// Pre-process data: make sure the entry weight is at least 1
if (Func.Blocks[Func.Entry].Weight == 0) {
Func.Blocks[Func.Entry].Weight = 1;
}
// Introducing dummy source/sink pairs to allow flow circulation.
// The nodes corresponding to blocks of Func have indicies in the range
// [0..3 * NumBlocks); the dummy nodes are indexed by the next four values.
uint64_t S = 3 * NumBlocks;
uint64_t T = S + 1;
uint64_t S1 = S + 2;
uint64_t T1 = S + 3;
Network.initialize(3 * NumBlocks + 4, S1, T1);
// Create three nodes for every block of the function
for (uint64_t B = 0; B < NumBlocks; B++) {
auto &Block = Func.Blocks[B];
assert((!Block.UnknownWeight || Block.Weight == 0 || Block.isEntry()) &&
"non-zero weight of a block w/o weight except for an entry");
// Split every block into two nodes
uint64_t Bin = 3 * B;
uint64_t Bout = 3 * B + 1;
uint64_t Baux = 3 * B + 2;
if (Block.Weight > 0) {
Network.addEdge(S1, Bout, Block.Weight, 0);
Network.addEdge(Bin, T1, Block.Weight, 0);
}
// Edges from S and to T
assert((!Block.isEntry() || !Block.isExit()) &&
"a block cannot be an entry and an exit");
if (Block.isEntry()) {
Network.addEdge(S, Bin, 0);
} else if (Block.isExit()) {
Network.addEdge(Bout, T, 0);
}
// An auxiliary node to allow increase/reduction of block counts:
// We assume that decreasing block counts is more expensive than increasing,
// and thus, setting separate costs here. In the future we may want to tune
// the relative costs so as to maximize the quality of generated profiles.
int64_t AuxCostInc = MinCostMaxFlow::AuxCostInc;
int64_t AuxCostDec = MinCostMaxFlow::AuxCostDec;
if (Block.UnknownWeight) {
// Do not penalize changing weights of blocks w/o known profile count
AuxCostInc = 0;
AuxCostDec = 0;
} else {
// Increasing the count for "cold" blocks with zero initial count is more
// expensive than for "hot" ones
if (Block.Weight == 0) {
AuxCostInc = MinCostMaxFlow::AuxCostIncZero;
}
// Modifying the count of the entry block is expensive
if (Block.isEntry()) {
AuxCostInc = MinCostMaxFlow::AuxCostIncEntry;
AuxCostDec = MinCostMaxFlow::AuxCostDecEntry;
}
}
// For blocks with self-edges, do not penalize a reduction of the count,
// as all of the increase can be attributed to the self-edge
if (Block.HasSelfEdge) {
AuxCostDec = 0;
}
Network.addEdge(Bin, Baux, AuxCostInc);
Network.addEdge(Baux, Bout, AuxCostInc);
if (Block.Weight > 0) {
Network.addEdge(Bout, Baux, AuxCostDec);
Network.addEdge(Baux, Bin, AuxCostDec);
}
}
// Creating edges for every jump
for (auto &Jump : Func.Jumps) {
uint64_t Src = Jump.Source;
uint64_t Dst = Jump.Target;
if (Src != Dst) {
uint64_t SrcOut = 3 * Src + 1;
uint64_t DstIn = 3 * Dst;
uint64_t Cost = Jump.IsUnlikely ? MinCostMaxFlow::AuxCostUnlikely : 0;
Network.addEdge(SrcOut, DstIn, Cost);
}
}
// Make sure we have a valid flow circulation
Network.addEdge(T, S, 0);
}
/// Extract resulting block and edge counts from the flow network.
void extractWeights(MinCostMaxFlow &Network, FlowFunction &Func) {
uint64_t NumBlocks = Func.Blocks.size();
// Extract resulting block counts
for (uint64_t Src = 0; Src < NumBlocks; Src++) {
auto &Block = Func.Blocks[Src];
uint64_t SrcOut = 3 * Src + 1;
int64_t Flow = 0;
for (auto &Adj : Network.getFlow(SrcOut)) {
uint64_t DstIn = Adj.first;
int64_t DstFlow = Adj.second;
bool IsAuxNode = (DstIn < 3 * NumBlocks && DstIn % 3 == 2);
if (!IsAuxNode || Block.HasSelfEdge) {
Flow += DstFlow;
}
}
Block.Flow = Flow;
assert(Flow >= 0 && "negative block flow");
}
// Extract resulting jump counts
for (auto &Jump : Func.Jumps) {
uint64_t Src = Jump.Source;
uint64_t Dst = Jump.Target;
int64_t Flow = 0;
if (Src != Dst) {
uint64_t SrcOut = 3 * Src + 1;
uint64_t DstIn = 3 * Dst;
Flow = Network.getFlow(SrcOut, DstIn);
} else {
uint64_t SrcOut = 3 * Src + 1;
uint64_t SrcAux = 3 * Src + 2;
int64_t AuxFlow = Network.getFlow(SrcOut, SrcAux);
if (AuxFlow > 0)
Flow = AuxFlow;
}
Jump.Flow = Flow;
assert(Flow >= 0 && "negative jump flow");
}
}
#ifndef NDEBUG
/// Verify that the computed flow values satisfy flow conservation rules
void verifyWeights(const FlowFunction &Func) {
const uint64_t NumBlocks = Func.Blocks.size();
auto InFlow = std::vector<uint64_t>(NumBlocks, 0);
auto OutFlow = std::vector<uint64_t>(NumBlocks, 0);
for (auto &Jump : Func.Jumps) {
InFlow[Jump.Target] += Jump.Flow;
OutFlow[Jump.Source] += Jump.Flow;
}
uint64_t TotalInFlow = 0;
uint64_t TotalOutFlow = 0;
for (uint64_t I = 0; I < NumBlocks; I++) {
auto &Block = Func.Blocks[I];
if (Block.isEntry()) {
TotalInFlow += Block.Flow;
assert(Block.Flow == OutFlow[I] && "incorrectly computed control flow");
} else if (Block.isExit()) {
TotalOutFlow += Block.Flow;
assert(Block.Flow == InFlow[I] && "incorrectly computed control flow");
} else {
assert(Block.Flow == OutFlow[I] && "incorrectly computed control flow");
assert(Block.Flow == InFlow[I] && "incorrectly computed control flow");
}
}
assert(TotalInFlow == TotalOutFlow && "incorrectly computed control flow");
// Verify that there are no isolated flow components
// One could modify FlowFunction to hold edges indexed by the sources, which
// will avoid a creation of the object
auto PositiveFlowEdges = std::vector<std::vector<uint64_t>>(NumBlocks);
for (auto &Jump : Func.Jumps) {
if (Jump.Flow > 0) {
PositiveFlowEdges[Jump.Source].push_back(Jump.Target);
}
}
// Run BFS from the source along edges with positive flow
std::queue<uint64_t> Queue;
auto Visited = std::vector<bool>(NumBlocks, false);
Queue.push(Func.Entry);
Visited[Func.Entry] = true;
while (!Queue.empty()) {
uint64_t Src = Queue.front();
Queue.pop();
for (uint64_t Dst : PositiveFlowEdges[Src]) {
if (!Visited[Dst]) {
Queue.push(Dst);
Visited[Dst] = true;
}
}
}
// Verify that every block that has a positive flow is reached from the source
// along edges with a positive flow
for (uint64_t I = 0; I < NumBlocks; I++) {
auto &Block = Func.Blocks[I];
assert((Visited[I] || Block.Flow == 0) && "an isolated flow component");
}
}
#endif
} // end of anonymous namespace
/// Apply the profile inference algorithm for a given flow function
void llvm::applyFlowInference(FlowFunction &Func) {
// Create and apply an inference network model
auto InferenceNetwork = MinCostMaxFlow();
initializeNetwork(InferenceNetwork, Func);
InferenceNetwork.run();
// Extract flow values for every block and every edge
extractWeights(InferenceNetwork, Func);
// Post-processing adjustments to the flow
auto Adjuster = FlowAdjuster(Func);
Adjuster.run();
#ifndef NDEBUG
// Verify the result
verifyWeights(Func);
#endif
}
Reference in New Issue View Git Blame Copy Permalink