//===- MetadataLoader.cpp - Internal BitcodeReader implementation ---------===//

//

// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

// See https://llvm.org/LICENSE.txt for license information.

// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

//

//===----------------------------------------------------------------------===//

#include "MetadataLoader.h"

#include "ValueList.h"

#include "llvm/ADT/APFloat.h"

#include "llvm/ADT/APInt.h"

#include "llvm/ADT/ArrayRef.h"

#include "llvm/ADT/DenseMap.h"

#include "llvm/ADT/DenseSet.h"

#include "llvm/ADT/None.h"

#include "llvm/ADT/STLExtras.h"

#include "llvm/ADT/SmallString.h"

#include "llvm/ADT/SmallVector.h"

#include "llvm/ADT/Statistic.h"

#include "llvm/ADT/StringRef.h"

#include "llvm/ADT/Twine.h"

#include "llvm/Bitcode/BitcodeReader.h"

#include "llvm/Bitstream/BitstreamReader.h"

#include "llvm/Bitcode/LLVMBitCodes.h"

#include "llvm/IR/Argument.h"

#include "llvm/IR/Attributes.h"

#include "llvm/IR/AutoUpgrade.h"

#include "llvm/IR/BasicBlock.h"

#include "llvm/IR/CallingConv.h"

#include "llvm/IR/Comdat.h"

#include "llvm/IR/Constant.h"

#include "llvm/IR/Constants.h"

#include "llvm/IR/DebugInfo.h"

#include "llvm/IR/DebugInfoMetadata.h"

#include "llvm/IR/DebugLoc.h"

#include "llvm/IR/DerivedTypes.h"

#include "llvm/IR/DiagnosticPrinter.h"

#include "llvm/IR/Function.h"

#include "llvm/IR/GVMaterializer.h"

#include "llvm/IR/GlobalAlias.h"

#include "llvm/IR/GlobalIFunc.h"

#include "llvm/IR/GlobalIndirectSymbol.h"

#include "llvm/IR/GlobalObject.h"

#include "llvm/IR/GlobalValue.h"

#include "llvm/IR/GlobalVariable.h"

#include "llvm/IR/InlineAsm.h"

#include "llvm/IR/InstrTypes.h"

#include "llvm/IR/Instruction.h"

#include "llvm/IR/Instructions.h"

#include "llvm/IR/IntrinsicInst.h"

#include "llvm/IR/Intrinsics.h"

#include "llvm/IR/LLVMContext.h"

#include "llvm/IR/Module.h"

#include "llvm/IR/ModuleSummaryIndex.h"

#include "llvm/IR/OperandTraits.h"

#include "llvm/IR/TrackingMDRef.h"

#include "llvm/IR/Type.h"

#include "llvm/IR/ValueHandle.h"

#include "llvm/Support/AtomicOrdering.h"

#include "llvm/Support/Casting.h"

#include "llvm/Support/CommandLine.h"

#include "llvm/Support/Compiler.h"

#include "llvm/Support/Debug.h"

#include "llvm/Support/Error.h"

#include "llvm/Support/ErrorHandling.h"

#include "llvm/Support/ManagedStatic.h"

#include "llvm/Support/MemoryBuffer.h"

#include "llvm/Support/raw_ostream.h"

#include <algorithm>

#include <cassert>

#include <cstddef>

#include <cstdint>

#include <deque>

#include <limits>

#include <map>

#include <memory>

#include <string>

#include <system_error>

#include <tuple>

#include <utility>

#include <vector>

using namespace llvm;

#define DEBUG_TYPE "bitcode-reader"

STATISTIC(NumMDStringLoaded, "Number of MDStrings loaded");

STATISTIC(NumMDNodeTemporary, "Number of MDNode::Temporary created");

STATISTIC(NumMDRecordLoaded, "Number of Metadata records loaded");

/// Flag whether we need to import full type definitions for ThinLTO.

/// Currently needed for Darwin and LLDB.

static cl::opt<bool> ImportFullTypeDefinitions(

"import-full-type-definitions", cl::init(false), cl::Hidden,

cl::desc("Import full type definitions for ThinLTO."));

static cl::opt<bool> DisableLazyLoading(

"disable-ondemand-mds-loading", cl::init(false), cl::Hidden,

cl::desc("Force disable the lazy-loading on-demand of metadata when "

"loading bitcode for importing."));

namespace {

static int64_t unrotateSign(uint64_t U) { return (U & 1) ? ~(U >> 1) : U >> 1; }

class BitcodeReaderMetadataList {

/// Array of metadata references.

///

/// Don't use std::vector here. Some versions of libc++ copy (instead of

/// move) on resize, and TrackingMDRef is very expensive to copy.

SmallVector<TrackingMDRef, 1> MetadataPtrs;

/// The set of indices in MetadataPtrs above of forward references that were

/// generated.

SmallDenseSet<unsigned, 1> ForwardReference;

/// The set of indices in MetadataPtrs above of Metadata that need to be

/// resolved.

SmallDenseSet<unsigned, 1> UnresolvedNodes;

/// Structures for resolving old type refs.

struct {

SmallDenseMap<MDString *, TempMDTuple, 1> Unknown;

SmallDenseMap<MDString *, DICompositeType *, 1> Final;

SmallDenseMap<MDString *, DICompositeType *, 1> FwdDecls;

SmallVector<std::pair<TrackingMDRef, TempMDTuple>, 1> Arrays;

} OldTypeRefs;

LLVMContext &Context;

/// Maximum number of valid references. Forward references exceeding the

/// maximum must be invalid.

unsigned RefsUpperBound;

public:

BitcodeReaderMetadataList(LLVMContext &C, size_t RefsUpperBound)

: Context(C),

RefsUpperBound(std::min((size_t)std::numeric_limits<unsigned>::max(),

RefsUpperBound)) {}

// vector compatibility methods

unsigned size() const { return MetadataPtrs.size(); }

void resize(unsigned N) { MetadataPtrs.resize(N); }

void push_back(Metadata *MD) { MetadataPtrs.emplace_back(MD); }

void clear() { MetadataPtrs.clear(); }

Metadata *back() const { return MetadataPtrs.back(); }

void pop_back() { MetadataPtrs.pop_back(); }

bool empty() const { return MetadataPtrs.empty(); }

Metadata *operator[](unsigned i) const {

assert(i < MetadataPtrs.size());

return MetadataPtrs[i];

}

Metadata *lookup(unsigned I) const {

if (I < MetadataPtrs.size())

return MetadataPtrs[I];

return nullptr;

}

void shrinkTo(unsigned N) {

assert(N <= size() && "Invalid shrinkTo request!");

assert(ForwardReference.empty() && "Unexpected forward refs");

assert(UnresolvedNodes.empty() && "Unexpected unresolved node");

MetadataPtrs.resize(N);

}

/// Return the given metadata, creating a replaceable forward reference if

/// necessary.

Metadata *getMetadataFwdRef(unsigned Idx);

/// Return the given metadata only if it is fully resolved.

///

/// Gives the same result as \a lookup(), unless \a MDNode::isResolved()

/// would give \c false.

Metadata *getMetadataIfResolved(unsigned Idx);

MDNode *getMDNodeFwdRefOrNull(unsigned Idx);

void assignValue(Metadata *MD, unsigned Idx);

void tryToResolveCycles();

bool hasFwdRefs() const { return !ForwardReference.empty(); }

int getNextFwdRef() {

assert(hasFwdRefs());

return *ForwardReference.begin();

}

/// Upgrade a type that had an MDString reference.

void addTypeRef(MDString &UUID, DICompositeType &CT);

/// Upgrade a type that had an MDString reference.

Metadata *upgradeTypeRef(Metadata *MaybeUUID);

/// Upgrade a type ref array that may have MDString references.

Metadata *upgradeTypeRefArray(Metadata *MaybeTuple);

private:

Metadata *resolveTypeRefArray(Metadata *MaybeTuple);

};

void BitcodeReaderMetadataList::assignValue(Metadata *MD, unsigned Idx) {

if (auto *MDN = dyn_cast<MDNode>(MD))

if (!MDN->isResolved())

UnresolvedNodes.insert(Idx);

if (Idx == size()) {

push_back(MD);

return;

}

if (Idx >= size())

resize(Idx + 1);

TrackingMDRef &OldMD = MetadataPtrs[Idx];

if (!OldMD) {

OldMD.reset(MD);

return;

}

// If there was a forward reference to this value, replace it.

TempMDTuple PrevMD(cast<MDTuple>(OldMD.get()));

PrevMD->replaceAllUsesWith(MD);

ForwardReference.erase(Idx);

}

Metadata *BitcodeReaderMetadataList::getMetadataFwdRef(unsigned Idx) {

// Bail out for a clearly invalid value.

if (Idx >= RefsUpperBound)

return nullptr;

if (Idx >= size())

resize(Idx + 1);

if (Metadata *MD = MetadataPtrs[Idx])

return MD;

// Track forward refs to be resolved later.

ForwardReference.insert(Idx);

// Create and return a placeholder, which will later be RAUW'd.

++NumMDNodeTemporary;

Metadata *MD = MDNode::getTemporary(Context, None).release();

MetadataPtrs[Idx].reset(MD);

return MD;

}

Metadata *BitcodeReaderMetadataList::getMetadataIfResolved(unsigned Idx) {

Metadata *MD = lookup(Idx);

if (auto *N = dyn_cast_or_null<MDNode>(MD))

if (!N->isResolved())

return nullptr;

return MD;

}

MDNode *BitcodeReaderMetadataList::getMDNodeFwdRefOrNull(unsigned Idx) {

return dyn_cast_or_null<MDNode>(getMetadataFwdRef(Idx));

}

void BitcodeReaderMetadataList::tryToResolveCycles() {

if (!ForwardReference.empty())

// Still forward references... can't resolve cycles.

return;

// Give up on finding a full definition for any forward decls that remain.

for (const auto &Ref : OldTypeRefs.FwdDecls)

OldTypeRefs.Final.insert(Ref);

OldTypeRefs.FwdDecls.clear();

// Upgrade from old type ref arrays. In strange cases, this could add to

// OldTypeRefs.Unknown.

for (const auto &Array : OldTypeRefs.Arrays)

Array.second->replaceAllUsesWith(resolveTypeRefArray(Array.first.get()));

OldTypeRefs.Arrays.clear();

// Replace old string-based type refs with the resolved node, if possible.

// If we haven't seen the node, leave it to the verifier to complain about

// the invalid string reference.

for (const auto &Ref : OldTypeRefs.Unknown) {

if (DICompositeType *CT = OldTypeRefs.Final.lookup(Ref.first))

Ref.second->replaceAllUsesWith(CT);

else

Ref.second->replaceAllUsesWith(Ref.first);

}

OldTypeRefs.Unknown.clear();

if (UnresolvedNodes.empty())

// Nothing to do.

return;

// Resolve any cycles.

for (unsigned I : UnresolvedNodes) {

auto &MD = MetadataPtrs[I];

auto *N = dyn_cast_or_null<MDNode>(MD);

if (!N)

continue;

assert(!N->isTemporary() && "Unexpected forward reference");

N->resolveCycles();

}

// Make sure we return early again until there's another unresolved ref.

UnresolvedNodes.clear();

}

void BitcodeReaderMetadataList::addTypeRef(MDString &UUID,

DICompositeType &CT) {

assert(CT.getRawIdentifier() == &UUID && "Mismatched UUID");

if (CT.isForwardDecl())

OldTypeRefs.FwdDecls.insert(std::make_pair(&UUID, &CT));

else

OldTypeRefs.Final.insert(std::make_pair(&UUID, &CT));

}

Metadata *BitcodeReaderMetadataList::upgradeTypeRef(Metadata *MaybeUUID) {

auto *UUID = dyn_cast_or_null<MDString>(MaybeUUID);

if (LLVM_LIKELY(!UUID))

return MaybeUUID;

if (auto *CT = OldTypeRefs.Final.lookup(UUID))

return CT;

auto &Ref = OldTypeRefs.Unknown[UUID];

if (!Ref)

Ref = MDNode::getTemporary(Context, None);

return Ref.get();

}

Metadata *BitcodeReaderMetadataList::upgradeTypeRefArray(Metadata *MaybeTuple) {

auto *Tuple = dyn_cast_or_null<MDTuple>(MaybeTuple);

if (!Tuple || Tuple->isDistinct())

return MaybeTuple;

// Look through the array immediately if possible.

if (!Tuple->isTemporary())

return resolveTypeRefArray(Tuple);

// Create and return a placeholder to use for now. Eventually

// resolveTypeRefArrays() will be resolve this forward reference.

OldTypeRefs.Arrays.emplace_back(

std::piecewise_construct, std::forward_as_tuple(Tuple),

std::forward_as_tuple(MDTuple::getTemporary(Context, None)));

return OldTypeRefs.Arrays.back().second.get();

}

Metadata *BitcodeReaderMetadataList::resolveTypeRefArray(Metadata *MaybeTuple) {

auto *Tuple = dyn_cast_or_null<MDTuple>(MaybeTuple);

if (!Tuple || Tuple->isDistinct())

return MaybeTuple;

// Look through the DITypeRefArray, upgrading each DIType *.

SmallVector<Metadata *, 32> Ops;

Ops.reserve(Tuple->getNumOperands());

for (Metadata *MD : Tuple->operands())

Ops.push_back(upgradeTypeRef(MD));

return MDTuple::get(Context, Ops);

}

namespace {

class PlaceholderQueue {

// Placeholders would thrash around when moved, so store in a std::deque

// instead of some sort of vector.

std::deque<DistinctMDOperandPlaceholder> PHs;

public:

~PlaceholderQueue() {

assert(empty() && "PlaceholderQueue hasn't been flushed before being destroyed");

}

bool empty() { return PHs.empty(); }

DistinctMDOperandPlaceholder &getPlaceholderOp(unsigned ID);

void flush(BitcodeReaderMetadataList &MetadataList);

/// Return the list of temporaries nodes in the queue, these need to be

/// loaded before we can flush the queue.

void getTemporaries(BitcodeReaderMetadataList &MetadataList,

DenseSet<unsigned> &Temporaries) {

for (auto &PH : PHs) {

auto ID = PH.getID();

auto *MD = MetadataList.lookup(ID);

if (!MD) {

Temporaries.insert(ID);

continue;

}

auto *N = dyn_cast_or_null<MDNode>(MD);

if (N && N->isTemporary())

Temporaries.insert(ID);

}

}

};

} // end anonymous namespace

DistinctMDOperandPlaceholder &PlaceholderQueue::getPlaceholderOp(unsigned ID) {

PHs.emplace_back(ID);

return PHs.back();

}

void PlaceholderQueue::flush(BitcodeReaderMetadataList &MetadataList) {

while (!PHs.empty()) {

auto *MD = MetadataList.lookup(PHs.front().getID());

assert(MD && "Flushing placeholder on unassigned MD");

#ifndef NDEBUG

if (auto *MDN = dyn_cast<MDNode>(MD))

assert(MDN->isResolved() &&

"Flushing Placeholder while cycles aren't resolved");

#endif

PHs.front().replaceUseWith(MD);

PHs.pop_front();

}

}

} // anonynous namespace

static Error error(const Twine &Message) {

return make_error<StringError>(

Message, make_error_code(BitcodeError::CorruptedBitcode));

}

class MetadataLoader::MetadataLoaderImpl {

BitcodeReaderMetadataList MetadataList;

BitcodeReaderValueList &ValueList;

BitstreamCursor &Stream;

LLVMContext &Context;

Module &TheModule;

std::function<Type *(unsigned)> getTypeByID;

/// Cursor associated with the lazy-loading of Metadata. This is the easy way

/// to keep around the right "context" (Abbrev list) to be able to jump in

/// the middle of the metadata block and load any record.

BitstreamCursor IndexCursor;

/// Index that keeps track of MDString values.

std::vector<StringRef> MDStringRef;

/// On-demand loading of a single MDString. Requires the index above to be

/// populated.

MDString *lazyLoadOneMDString(unsigned Idx);

/// Index that keeps track of where to find a metadata record in the stream.

std::vector<uint64_t> GlobalMetadataBitPosIndex;

/// Populate the index above to enable lazily loading of metadata, and load

/// the named metadata as well as the transitively referenced global

/// Metadata.

Expected<bool> lazyLoadModuleMetadataBlock();

/// On-demand loading of a single metadata. Requires the index above to be

/// populated.

void lazyLoadOneMetadata(unsigned Idx, PlaceholderQueue &Placeholders);

// Keep mapping of seens pair of old-style CU <-> SP, and update pointers to

// point from SP to CU after a block is completly parsed.

std::vector<std::pair<DICompileUnit *, Metadata *>> CUSubprograms;

/// Functions that need to be matched with subprograms when upgrading old

/// metadata.

SmallDenseMap<Function *, DISubprogram *, 16> FunctionsWithSPs;

// Map the bitcode's custom MDKind ID to the Module's MDKind ID.

DenseMap<unsigned, unsigned> MDKindMap;

bool StripTBAA = false;

bool HasSeenOldLoopTags = false;

bool NeedUpgradeToDIGlobalVariableExpression = false;

bool NeedDeclareExpressionUpgrade = false;

/// True if metadata is being parsed for a module being ThinLTO imported.

bool IsImporting = false;

Error parseOneMetadata(SmallVectorImpl<uint64_t> &Record, unsigned Code,

PlaceholderQueue &Placeholders, StringRef Blob,

unsigned &NextMetadataNo);

Error parseMetadataStrings(ArrayRef<uint64_t> Record, StringRef Blob,

function_ref<void(StringRef)> CallBack);

Error parseGlobalObjectAttachment(GlobalObject &GO,

ArrayRef<uint64_t> Record);

Error parseMetadataKindRecord(SmallVectorImpl<uint64_t> &Record);

void resolveForwardRefsAndPlaceholders(PlaceholderQueue &Placeholders);

/// Upgrade old-style CU <-> SP pointers to point from SP to CU.

void upgradeCUSubprograms() {

for (auto CU_SP : CUSubprograms)

if (auto *SPs = dyn_cast_or_null<MDTuple>(CU_SP.second))

for (auto &Op : SPs->operands())

if (auto *SP = dyn_cast_or_null<DISubprogram>(Op))

SP->replaceUnit(CU_SP.first);

CUSubprograms.clear();

}

/// Upgrade old-style bare DIGlobalVariables to DIGlobalVariableExpressions.

void upgradeCUVariables() {

if (!NeedUpgradeToDIGlobalVariableExpression)

return;

// Upgrade list of variables attached to the CUs.

if (NamedMDNode *CUNodes = TheModule.getNamedMetadata("llvm.dbg.cu"))

for (unsigned I = 0, E = CUNodes->getNumOperands(); I != E; ++I) {

auto *CU = cast<DICompileUnit>(CUNodes->getOperand(I));

if (auto *GVs = dyn_cast_or_null<MDTuple>(CU->getRawGlobalVariables()))

for (unsigned I = 0; I < GVs->getNumOperands(); I++)

if (auto *GV =

dyn_cast_or_null<DIGlobalVariable>(GVs->getOperand(I))) {

auto *DGVE = DIGlobalVariableExpression::getDistinct(

Context, GV, DIExpression::get(Context, {}));

GVs->replaceOperandWith(I, DGVE);

}

}

// Upgrade variables attached to globals.

for (auto &GV : TheModule.globals()) {

SmallVector<MDNode *, 1> MDs;

GV.getMetadata(LLVMContext::MD_dbg, MDs);

GV.eraseMetadata(LLVMContext::MD_dbg);

for (auto *MD : MDs)

if (auto *DGV = dyn_cast<DIGlobalVariable>(MD)) {

auto *DGVE = DIGlobalVariableExpression::getDistinct(

Context, DGV, DIExpression::get(Context, {}));

GV.addMetadata(LLVMContext::MD_dbg, *DGVE);

} else

GV.addMetadata(LLVMContext::MD_dbg, *MD);

}

}

/// Remove a leading DW_OP_deref from DIExpressions in a dbg.declare that

/// describes a function argument.

void upgradeDeclareExpressions(Function &F) {

if (!NeedDeclareExpressionUpgrade)

return;

for (auto &BB : F)

for (auto &I : BB)

if (auto *DDI = dyn_cast<DbgDeclareInst>(&I))

if (auto *DIExpr = DDI->getExpression())

if (DIExpr->startsWithDeref() &&

dyn_cast_or_null<Argument>(DDI->getAddress())) {

SmallVector<uint64_t, 8> Ops;

Ops.append(std::next(DIExpr->elements_begin()),

DIExpr->elements_end());

auto *E = DIExpression::get(Context, Ops);

DDI->setOperand(2, MetadataAsValue::get(Context, E));

}

}

/// Upgrade the expression from previous versions.

Error upgradeDIExpression(uint64_t FromVersion,

MutableArrayRef<uint64_t> &Expr,

SmallVectorImpl<uint64_t> &Buffer) {

auto N = Expr.size();

switch (FromVersion) {

default:

return error("Invalid record");

case 0:

if (N >= 3 && Expr[N - 3] == dwarf::DW_OP_bit_piece)

Expr[N - 3] = dwarf::DW_OP_LLVM_fragment;

LLVM_FALLTHROUGH;

case 1:

// Move DW_OP_deref to the end.

if (N && Expr[0] == dwarf::DW_OP_deref) {

auto End = Expr.end();

if (Expr.size() >= 3 &&

*std::prev(End, 3) == dwarf::DW_OP_LLVM_fragment)

End = std::prev(End, 3);

std::move(std::next(Expr.begin()), End, Expr.begin());

*std::prev(End) = dwarf::DW_OP_deref;

}

NeedDeclareExpressionUpgrade = true;

LLVM_FALLTHROUGH;

case 2: {

// Change DW_OP_plus to DW_OP_plus_uconst.

// Change DW_OP_minus to DW_OP_uconst, DW_OP_minus

auto SubExpr = ArrayRef<uint64_t>(Expr);

while (!SubExpr.empty()) {

// Skip past other operators with their operands

// for this version of the IR, obtained from

// from historic DIExpression::ExprOperand::getSize().

size_t HistoricSize;

switch (SubExpr.front()) {

default:

HistoricSize = 1;

break;

case dwarf::DW_OP_constu:

case dwarf::DW_OP_minus:

case dwarf::DW_OP_plus:

HistoricSize = 2;

break;

case dwarf::DW_OP_LLVM_fragment:

HistoricSize = 3;

break;

}

// If the expression is malformed, make sure we don't

// copy more elements than we should.

HistoricSize = std::min(SubExpr.size(), HistoricSize);

ArrayRef<uint64_t> Args = SubExpr.slice(1, HistoricSize-1);

switch (SubExpr.front()) {

case dwarf::DW_OP_plus:

Buffer.push_back(dwarf::DW_OP_plus_uconst);

Buffer.append(Args.begin(), Args.end());

break;

case dwarf::DW_OP_minus:

Buffer.push_back(dwarf::DW_OP_constu);

Buffer.append(Args.begin(), Args.end());

Buffer.push_back(dwarf::DW_OP_minus);

break;

default:

Buffer.push_back(*SubExpr.begin());

Buffer.append(Args.begin(), Args.end());

break;

}

// Continue with remaining elements.

SubExpr = SubExpr.slice(HistoricSize);

}

Expr = MutableArrayRef<uint64_t>(Buffer);

LLVM_FALLTHROUGH;

}

case 3:

// Up-to-date!

break;

}

return Error::success();

}

void upgradeDebugInfo() {

upgradeCUSubprograms();

upgradeCUVariables();

}

public:

MetadataLoaderImpl(BitstreamCursor &Stream, Module &TheModule,

BitcodeReaderValueList &ValueList,

std::function<Type *(unsigned)> getTypeByID,

bool IsImporting)

: MetadataList(TheModule.getContext(), Stream.SizeInBytes()),

ValueList(ValueList), Stream(Stream), Context(TheModule.getContext()),

TheModule(TheModule), getTypeByID(std::move(getTypeByID)),

IsImporting(IsImporting) {}

Error parseMetadata(bool ModuleLevel);

bool hasFwdRefs() const { return MetadataList.hasFwdRefs(); }

Metadata *getMetadataFwdRefOrLoad(unsigned ID) {

if (ID < MDStringRef.size())

return lazyLoadOneMDString(ID);

if (auto *MD = MetadataList.lookup(ID))

return MD;

// If lazy-loading is enabled, we try recursively to load the operand

// instead of creating a temporary.

if (ID < (MDStringRef.size() + GlobalMetadataBitPosIndex.size())) {

PlaceholderQueue Placeholders;

lazyLoadOneMetadata(ID, Placeholders);

resolveForwardRefsAndPlaceholders(Placeholders);

return MetadataList.lookup(ID);

}

return MetadataList.getMetadataFwdRef(ID);

}

DISubprogram *lookupSubprogramForFunction(Function *F) {

return FunctionsWithSPs.lookup(F);

}

bool hasSeenOldLoopTags() { return HasSeenOldLoopTags; }

Error parseMetadataAttachment(

Function &F, const SmallVectorImpl<Instruction *> &InstructionList);

Error parseMetadataKinds();

void setStripTBAA(bool Value) { StripTBAA = Value; }

bool isStrippingTBAA() { return StripTBAA; }

unsigned size() const { return MetadataList.size(); }

void shrinkTo(unsigned N) { MetadataList.shrinkTo(N); }

void upgradeDebugIntrinsics(Function &F) { upgradeDeclareExpressions(F); }

};

Expected<bool>

MetadataLoader::MetadataLoaderImpl::lazyLoadModuleMetadataBlock() {

IndexCursor = Stream;

SmallVector<uint64_t, 64> Record;

// Get the abbrevs, and preload record positions to make them lazy-loadable.

while (true) {

Expected<BitstreamEntry> MaybeEntry = IndexCursor.advanceSkippingSubblocks(

BitstreamCursor::AF_DontPopBlockAtEnd);

if (!MaybeEntry)

return MaybeEntry.takeError();

BitstreamEntry Entry = MaybeEntry.get();

switch (Entry.Kind) {

case BitstreamEntry::SubBlock: // Handled for us already.

case BitstreamEntry::Error:

return error("Malformed block");

case BitstreamEntry::EndBlock: {

return true;

}

case BitstreamEntry::Record: {

// The interesting case.

++NumMDRecordLoaded;

uint64_t CurrentPos = IndexCursor.GetCurrentBitNo();

Expected<unsigned> MaybeCode = IndexCursor.skipRecord(Entry.ID);

if (!MaybeCode)

return MaybeCode.takeError();

unsigned Code = MaybeCode.get();

switch (Code) {

case bitc::METADATA_STRINGS: {

// Rewind and parse the strings.

if (Error Err = IndexCursor.JumpToBit(CurrentPos))

return std::move(Err);

StringRef Blob;

Record.clear();

if (Expected<unsigned> MaybeRecord =

IndexCursor.readRecord(Entry.ID, Record, &Blob))

;

else

return MaybeRecord.takeError();

unsigned NumStrings = Record[0];

MDStringRef.reserve(NumStrings);

auto IndexNextMDString = [&](StringRef Str) {

MDStringRef.push_back(Str);

};

if (auto Err = parseMetadataStrings(Record, Blob, IndexNextMDString))

return std::move(Err);

break;

}

case bitc::METADATA_INDEX_OFFSET: {

// This is the offset to the index, when we see this we skip all the

// records and load only an index to these.

if (Error Err = IndexCursor.JumpToBit(CurrentPos))

return std::move(Err);

Record.clear();

if (Expected<unsigned> MaybeRecord =

IndexCursor.readRecord(Entry.ID, Record))

;

else

return MaybeRecord.takeError();

if (Record.size() != 2)

return error("Invalid record");

auto Offset = Record[0] + (Record[1] << 32);

auto BeginPos = IndexCursor.GetCurrentBitNo();

if (Error Err = IndexCursor.JumpToBit(BeginPos + Offset))

return std::move(Err);

Expected<BitstreamEntry> MaybeEntry =

IndexCursor.advanceSkippingSubblocks(

BitstreamCursor::AF_DontPopBlockAtEnd);

if (!MaybeEntry)

return MaybeEntry.takeError();

Entry = MaybeEntry.get();

assert(Entry.Kind == BitstreamEntry::Record &&

"Corrupted bitcode: Expected `Record` when trying to find the "

"Metadata index");

Record.clear();

if (Expected<unsigned> MaybeCode =

IndexCursor.readRecord(Entry.ID, Record))

assert(MaybeCode.get() == bitc::METADATA_INDEX &&

"Corrupted bitcode: Expected `METADATA_INDEX` when trying to "

"find the Metadata index");

else

return MaybeCode.takeError();

// Delta unpack

auto CurrentValue = BeginPos;

GlobalMetadataBitPosIndex.reserve(Record.size());

for (auto &Elt : Record) {

CurrentValue += Elt;

GlobalMetadataBitPosIndex.push_back(CurrentValue);

}

break;

}

case bitc::METADATA_INDEX:

// We don't expect to get there, the Index is loaded when we encounter

// the offset.

return error("Corrupted Metadata block");

case bitc::METADATA_NAME: {

// Named metadata need to be materialized now and aren't deferred.

if (Error Err = IndexCursor.JumpToBit(CurrentPos))

return std::move(Err);

Record.clear();

unsigned Code;

if (Expected<unsigned> MaybeCode =

IndexCursor.readRecord(Entry.ID, Record)) {

Code = MaybeCode.get();

assert(Code == bitc::METADATA_NAME);

} else

return MaybeCode.takeError();

// Read name of the named metadata.

SmallString<8> Name(Record.begin(), Record.end());

if (Expected<unsigned> MaybeCode = IndexCursor.ReadCode())

Code = MaybeCode.get();

else

return MaybeCode.takeError();

// Named Metadata comes in two parts, we expect the name to be followed

// by the node

Record.clear();

if (Expected<unsigned> MaybeNextBitCode =

IndexCursor.readRecord(Code, Record))

assert(MaybeNextBitCode.get() == bitc::METADATA_NAMED_NODE);

else

return MaybeNextBitCode.takeError();

// Read named metadata elements.

unsigned Size = Record.size();

NamedMDNode *NMD = TheModule.getOrInsertNamedMetadata(Name);

for (unsigned i = 0; i != Size; ++i) {

// FIXME: We could use a placeholder here, however NamedMDNode are

// taking MDNode as operand and not using the Metadata infrastructure.

// It is acknowledged by 'TODO: Inherit from Metadata' in the

// NamedMDNode class definition.

MDNode *MD = MetadataList.getMDNodeFwdRefOrNull(Record[i]);

assert(MD && "Invalid metadata: expect fwd ref to MDNode");

NMD->addOperand(MD);

}

break;

}

case bitc::METADATA_GLOBAL_DECL_ATTACHMENT: {

// FIXME: we need to do this early because we don't materialize global

// value explicitly.

if (Error Err = IndexCursor.JumpToBit(CurrentPos))

return std::move(Err);

Record.clear();

if (Expected<unsigned> MaybeRecord =

IndexCursor.readRecord(Entry.ID, Record))

;

else

return MaybeRecord.takeError();

if (Record.size() % 2 == 0)

return error("Invalid record");

unsigned ValueID = Record[0];

if (ValueID >= ValueList.size())

return error("Invalid record");

if (auto *GO = dyn_cast<GlobalObject>(ValueList[ValueID]))

if (Error Err = parseGlobalObjectAttachment(

*GO, ArrayRef<uint64_t>(Record).slice(1)))

return std::move(Err);

break;

}

case bitc::METADATA_KIND:

case bitc::METADATA_STRING_OLD:

case bitc::METADATA_OLD_FN_NODE:

case bitc::METADATA_OLD_NODE:

case bitc::METADATA_VALUE:

case bitc::METADATA_DISTINCT_NODE:

case bitc::METADATA_NODE:

case bitc::METADATA_LOCATION:

case bitc::METADATA_GENERIC_DEBUG:

case bitc::METADATA_SUBRANGE:

case bitc::METADATA_ENUMERATOR:

case bitc::METADATA_BASIC_TYPE:

case bitc::METADATA_DERIVED_TYPE:

case bitc::METADATA_COMPOSITE_TYPE:

case bitc::METADATA_SUBROUTINE_TYPE:

case bitc::METADATA_MODULE:

case bitc::METADATA_FILE:

case bitc::METADATA_COMPILE_UNIT:

case bitc::METADATA_SUBPROGRAM:

case bitc::METADATA_LEXICAL_BLOCK:

case bitc::METADATA_LEXICAL_BLOCK_FILE:

case bitc::METADATA_NAMESPACE:

case bitc::METADATA_COMMON_BLOCK:

case bitc::METADATA_MACRO:

case bitc::METADATA_MACRO_FILE:

case bitc::METADATA_TEMPLATE_TYPE:

case bitc::METADATA_TEMPLATE_VALUE:

case bitc::METADATA_GLOBAL_VAR:

case bitc::METADATA_LOCAL_VAR:

case bitc::METADATA_LABEL:

case bitc::METADATA_EXPRESSION:

case bitc::METADATA_OBJC_PROPERTY:

case bitc::METADATA_IMPORTED_ENTITY:

case bitc::METADATA_GLOBAL_VAR_EXPR:

// We don't expect to see any of these, if we see one, give up on

// lazy-loading and fallback.

MDStringRef.clear();

GlobalMetadataBitPosIndex.clear();

return false;

}

break;

}

}

}

}

/// Parse a METADATA_BLOCK. If ModuleLevel is true then we are parsing

/// module level metadata.

Error MetadataLoader::MetadataLoaderImpl::parseMetadata(bool ModuleLevel) {

if (!ModuleLevel && MetadataList.hasFwdRefs())

return error("Invalid metadata: fwd refs into function blocks");

// Record the entry position so that we can jump back here and efficiently

// skip the whole block in case we lazy-load.

auto EntryPos = Stream.GetCurrentBitNo();

if (Error Err = Stream.EnterSubBlock(bitc::METADATA_BLOCK_ID))

return Err;

SmallVector<uint64_t, 64> Record;

PlaceholderQueue Placeholders;

// We lazy-load module-level metadata: we build an index for each record, and

// then load individual record as needed, starting with the named metadata.

if (ModuleLevel && IsImporting && MetadataList.empty() &&

!DisableLazyLoading) {

auto SuccessOrErr = lazyLoadModuleMetadataBlock();

if (!SuccessOrErr)

return SuccessOrErr.takeError();

if (SuccessOrErr.get()) {

// An index was successfully created and we will be able to load metadata

// on-demand.

MetadataList.resize(MDStringRef.size() +

GlobalMetadataBitPosIndex.size());

// Reading the named metadata created forward references and/or

// placeholders, that we flush here.

resolveForwardRefsAndPlaceholders(Placeholders);

upgradeDebugInfo();

// Return at the beginning of the block, since it is easy to skip it

// entirely from there.

Stream.ReadBlockEnd(); // Pop the abbrev block context.

if (Error Err = IndexCursor.JumpToBit(EntryPos))

return Err;

if (Error Err = Stream.SkipBlock()) {

// FIXME this drops the error on the floor, which

// ThinLTO/X86/debuginfo-cu-import.ll relies on.

consumeError(std::move(Err));

return Error::success();

}

return Error::success();

}

// Couldn't load an index, fallback to loading all the block "old-style".

}

unsigned NextMetadataNo = MetadataList.size();

// Read all the records.

while (true) {

Expected<BitstreamEntry> MaybeEntry = Stream.advanceSkippingSubblocks();

if (!MaybeEntry)

return MaybeEntry.takeError();

BitstreamEntry Entry = MaybeEntry.get();

switch (Entry.Kind) {

case BitstreamEntry::SubBlock: // Handled for us already.

case BitstreamEntry::Error:

return error("Malformed block");

case BitstreamEntry::EndBlock:

resolveForwardRefsAndPlaceholders(Placeholders);

upgradeDebugInfo();

return Error::success();

case BitstreamEntry::Record:

// The interesting case.

break;

}

// Read a record.

Record.clear();

StringRef Blob;

++NumMDRecordLoaded;

if (Expected<unsigned> MaybeCode =

Stream.readRecord(Entry.ID, Record, &Blob)) {

if (Error Err = parseOneMetadata(Record, MaybeCode.get(), Placeholders,

Blob, NextMetadataNo))

return Err;

} else

return MaybeCode.takeError();

}

}

MDString *MetadataLoader::MetadataLoaderImpl::lazyLoadOneMDString(unsigned ID) {

++NumMDStringLoaded;

if (Metadata *MD = MetadataList.lookup(ID))

return cast<MDString>(MD);

auto MDS = MDString::get(Context, MDStringRef[ID]);

MetadataList.assignValue(MDS, ID);

return MDS;

}

void MetadataLoader::MetadataLoaderImpl::lazyLoadOneMetadata(

unsigned ID, PlaceholderQueue &Placeholders) {

assert(ID < (MDStringRef.size()) + GlobalMetadataBitPosIndex.size());

assert(ID >= MDStringRef.size() && "Unexpected lazy-loading of MDString");

// Lookup first if the metadata hasn't already been loaded.

if (auto *MD = MetadataList.lookup(ID)) {

auto *N = cast<MDNode>(MD);

if (!N->isTemporary())

return;

}

SmallVector<uint64_t, 64> Record;

StringRef Blob;

if (Error Err = IndexCursor.JumpToBit(

GlobalMetadataBitPosIndex[ID - MDStringRef.size()]))

report_fatal_error("lazyLoadOneMetadata failed jumping: " +

toString(std::move(Err)));

Expected<BitstreamEntry> MaybeEntry = IndexCursor.advanceSkippingSubblocks();