/**************************************************************************
*
* Copyright 2010 VMware, Inc.
* All Rights Reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sub license, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
* THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
* DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
* OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
* USE OR OTHER DEALINGS IN THE SOFTWARE.
*
* The above copyright notice and this permission notice (including the
* next paragraph) shall be included in all copies or substantial portions
* of the Software.
*
**************************************************************************/
/**
* The purpose of this module is to expose LLVM functionality not available
* through the C++ bindings.
*/
#ifndef __STDC_LIMIT_MACROS
#define __STDC_LIMIT_MACROS
#endif
#ifndef __STDC_CONSTANT_MACROS
#define __STDC_CONSTANT_MACROS
#endif
// Undef these vars just to silence warnings
#undef PACKAGE_BUGREPORT
#undef PACKAGE_NAME
#undef PACKAGE_STRING
#undef PACKAGE_TARNAME
#undef PACKAGE_VERSION
#include <stddef.h>
#include <llvm-c/Core.h>
#include <llvm-c/ExecutionEngine.h>
#include <llvm/Target/TargetOptions.h>
#include <llvm/ExecutionEngine/ExecutionEngine.h>
#include <llvm/ADT/Triple.h>
#if HAVE_LLVM < 0x0306
#include <llvm/ExecutionEngine/JITMemoryManager.h>
#else
#include <llvm/ExecutionEngine/SectionMemoryManager.h>
#endif
#include <llvm/Support/CommandLine.h>
#include <llvm/Support/Host.h>
#include <llvm/Support/PrettyStackTrace.h>
#include <llvm/Support/TargetSelect.h>
#include <llvm/IR/IRBuilder.h>
#include <llvm/IR/Module.h>
#include <llvm/Support/CBindingWrapping.h>
#include "pipe/p_config.h"
#include "util/u_debug.h"
#include "util/u_cpu_detect.h"
#include "lp_bld_misc.h"
namespace {
class LLVMEnsureMultithreaded {
public:
LLVMEnsureMultithreaded()
{
if (!LLVMIsMultithreaded()) {
LLVMStartMultithreaded();
}
}
};
static LLVMEnsureMultithreaded lLVMEnsureMultithreaded;
}
extern "C" void
lp_set_target_options(void)
{
#if HAVE_LLVM < 0x0304
/*
* By default LLVM adds a signal handler to output a pretty stack trace.
* This signal handler is never removed, causing problems when unloading the
* shared object where the gallium driver resides.
*/
llvm::DisablePrettyStackTrace = true;
#endif
// If we have a native target, initialize it to ensure it is linked in and
// usable by the JIT.
llvm::InitializeNativeTarget();
llvm::InitializeNativeTargetAsmPrinter();
llvm::InitializeNativeTargetDisassembler();
}
extern "C"
LLVMValueRef
lp_build_load_volatile(LLVMBuilderRef B, LLVMValueRef PointerVal,
const char *Name)
{
return llvm::wrap(llvm::unwrap(B)->CreateLoad(llvm::unwrap(PointerVal), true, Name));
}
extern "C"
void
lp_set_load_alignment(LLVMValueRef Inst,
unsigned Align)
{
llvm::unwrap<llvm::LoadInst>(Inst)->setAlignment(Align);
}
extern "C"
void
lp_set_store_alignment(LLVMValueRef Inst,
unsigned Align)
{
llvm::unwrap<llvm::StoreInst>(Inst)->setAlignment(Align);
}
#if HAVE_LLVM < 0x0306
typedef llvm::JITMemoryManager BaseMemoryManager;
#else
typedef llvm::RTDyldMemoryManager BaseMemoryManager;
#endif
/*
* Delegating is tedious but the default manager class is hidden in an
* anonymous namespace in LLVM, so we cannot just derive from it to change
* its behavior.
*/
class DelegatingJITMemoryManager : public BaseMemoryManager {
protected:
virtual BaseMemoryManager *mgr() const = 0;
public:
#if HAVE_LLVM < 0x0306
/*
* From JITMemoryManager
*/
virtual void setMemoryWritable() {
mgr()->setMemoryWritable();
}
virtual void setMemoryExecutable() {
mgr()->setMemoryExecutable();
}
virtual void setPoisonMemory(bool poison) {
mgr()->setPoisonMemory(poison);
}
virtual void AllocateGOT() {
mgr()->AllocateGOT();
/*
* isManagingGOT() is not virtual in base class so we can't delegate.
* Instead we mirror the value of HasGOT in our instance.
*/
HasGOT = mgr()->isManagingGOT();
}
virtual uint8_t *getGOTBase() const {
return mgr()->getGOTBase();
}
virtual uint8_t *startFunctionBody(const llvm::Function *F,
uintptr_t &ActualSize) {
return mgr()->startFunctionBody(F, ActualSize);
}
virtual uint8_t *allocateStub(const llvm::GlobalValue *F,
unsigned StubSize,
unsigned Alignment) {
return mgr()->allocateStub(F, StubSize, Alignment);
}
virtual void endFunctionBody(const llvm::Function *F,
uint8_t *FunctionStart,
uint8_t *FunctionEnd) {
mgr()->endFunctionBody(F, FunctionStart, FunctionEnd);
}
virtual uint8_t *allocateSpace(intptr_t Size, unsigned Alignment) {
return mgr()->allocateSpace(Size, Alignment);
}
virtual uint8_t *allocateGlobal(uintptr_t Size, unsigned Alignment) {
return mgr()->allocateGlobal(Size, Alignment);
}
virtual void deallocateFunctionBody(void *Body) {
mgr()->deallocateFunctionBody(Body);
}
#if HAVE_LLVM < 0x0304
virtual uint8_t *startExceptionTable(const llvm::Function *F,
uintptr_t &ActualSize) {
return mgr()->startExceptionTable(F, ActualSize);
}
virtual void endExceptionTable(const llvm::Function *F,
uint8_t *TableStart,
uint8_t *TableEnd,
uint8_t *FrameRegister) {
mgr()->endExceptionTable(F, TableStart, TableEnd,
FrameRegister);
}
virtual void deallocateExceptionTable(void *ET) {
mgr()->deallocateExceptionTable(ET);
}
#endif
virtual bool CheckInvariants(std::string &s) {
return mgr()->CheckInvariants(s);
}
virtual size_t GetDefaultCodeSlabSize() {
return mgr()->GetDefaultCodeSlabSize();
}
virtual size_t GetDefaultDataSlabSize() {
return mgr()->GetDefaultDataSlabSize();
}
virtual size_t GetDefaultStubSlabSize() {
return mgr()->GetDefaultStubSlabSize();
}
virtual unsigned GetNumCodeSlabs() {
return mgr()->GetNumCodeSlabs();
}
virtual unsigned GetNumDataSlabs() {
return mgr()->GetNumDataSlabs();
}
virtual unsigned GetNumStubSlabs() {
return mgr()->GetNumStubSlabs();
}
#endif
/*
* From RTDyldMemoryManager
*/
#if HAVE_LLVM >= 0x0304
virtual uint8_t *allocateCodeSection(uintptr_t Size,
unsigned Alignment,
unsigned SectionID,
llvm::StringRef SectionName) {
return mgr()->allocateCodeSection(Size, Alignment, SectionID,
SectionName);
}
#else
virtual uint8_t *allocateCodeSection(uintptr_t Size,
unsigned Alignment,
unsigned SectionID) {
return mgr()->allocateCodeSection(Size, Alignment, SectionID);
}
#endif
virtual uint8_t *allocateDataSection(uintptr_t Size,
unsigned Alignment,
unsigned SectionID,
#if HAVE_LLVM >= 0x0304
llvm::StringRef SectionName,
#endif
bool IsReadOnly) {
return mgr()->allocateDataSection(Size, Alignment, SectionID,
#if HAVE_LLVM >= 0x0304
SectionName,
#endif
IsReadOnly);
}
#if HAVE_LLVM >= 0x0304
virtual void registerEHFrames(uint8_t *Addr, uint64_t LoadAddr, size_t Size) {
mgr()->registerEHFrames(Addr, LoadAddr, Size);
}
virtual void deregisterEHFrames(uint8_t *Addr, uint64_t LoadAddr, size_t Size) {
mgr()->deregisterEHFrames(Addr, LoadAddr, Size);
}
#else
virtual void registerEHFrames(llvm::StringRef SectionData) {
mgr()->registerEHFrames(SectionData);
}
#endif
virtual void *getPointerToNamedFunction(const std::string &Name,
bool AbortOnFailure=true) {
return mgr()->getPointerToNamedFunction(Name, AbortOnFailure);
}
#if HAVE_LLVM <= 0x0303
virtual bool applyPermissions(std::string *ErrMsg = 0) {
return mgr()->applyPermissions(ErrMsg);
}
#else
virtual bool finalizeMemory(std::string *ErrMsg = 0) {
return mgr()->finalizeMemory(ErrMsg);
}
#endif
};
/*
* Delegate memory management to one shared manager for more efficient use
* of memory than creating a separate pool for each LLVM engine.
* Keep generated code until freeGeneratedCode() is called, instead of when
* memory manager is destroyed, which happens during engine destruction.
* This allows additional memory savings as we don't have to keep the engine
* around in order to use the code.
* All methods are delegated to the shared manager except destruction and
* deallocating code. For the latter we just remember what needs to be
* deallocated later. The shared manager is deleted once it is empty.
*/
class ShaderMemoryManager : public DelegatingJITMemoryManager {
BaseMemoryManager *TheMM;
struct GeneratedCode {
typedef std::vector<void *> Vec;
Vec FunctionBody, ExceptionTable;
BaseMemoryManager *TheMM;
GeneratedCode(BaseMemoryManager *MM) {
TheMM = MM;
}
~GeneratedCode() {
/*
* Deallocate things as previously requested and
* free shared manager when no longer used.
*/
#if HAVE_LLVM < 0x0306
Vec::iterator i;
assert(TheMM);
for ( i = FunctionBody.begin(); i != FunctionBody.end(); ++i )
TheMM->deallocateFunctionBody(*i);
#if HAVE_LLVM < 0x0304
for ( i = ExceptionTable.begin(); i != ExceptionTable.end(); ++i )
TheMM->deallocateExceptionTable(*i);
#endif /* HAVE_LLVM < 0x0304 */
#endif /* HAVE_LLVM < 0x0306 */
}
};
GeneratedCode *code;
BaseMemoryManager *mgr() const {
return TheMM;
}
public:
ShaderMemoryManager(BaseMemoryManager* MM) {
TheMM = MM;
code = new GeneratedCode(MM);
}
virtual ~ShaderMemoryManager() {
/*
* 'code' is purposely not deleted. It is the user's responsibility
* to call getGeneratedCode() and freeGeneratedCode().
*/
}
struct lp_generated_code *getGeneratedCode() {
return (struct lp_generated_code *) code;
}
static void freeGeneratedCode(struct lp_generated_code *code) {
delete (GeneratedCode *) code;
}
#if HAVE_LLVM < 0x0304
virtual void deallocateExceptionTable(void *ET) {
// remember for later deallocation
code->ExceptionTable.push_back(ET);
}
#endif
virtual void deallocateFunctionBody(void *Body) {
// remember for later deallocation
code->FunctionBody.push_back(Body);
}
};
/**
* Same as LLVMCreateJITCompilerForModule, but:
* - allows using MCJIT and enabling AVX feature where available.
* - set target options
*
* See also:
* - llvm/lib/ExecutionEngine/ExecutionEngineBindings.cpp
* - llvm/tools/lli/lli.cpp
* - http://markmail.org/message/ttkuhvgj4cxxy2on#query:+page:1+mid:aju2dggerju3ivd3+state:results
*/
extern "C"
LLVMBool
lp_build_create_jit_compiler_for_module(LLVMExecutionEngineRef *OutJIT,
lp_generated_code **OutCode,
LLVMModuleRef M,
LLVMMCJITMemoryManagerRef CMM,
unsigned OptLevel,
int useMCJIT,
char **OutError)
{
using namespace llvm;
std::string Error;
#if HAVE_LLVM >= 0x0306
EngineBuilder builder(std::unique_ptr<Module>(unwrap(M)));
#else
EngineBuilder builder(unwrap(M));
#endif
/**
* LLVM 3.1+ haven't more "extern unsigned llvm::StackAlignmentOverride" and
* friends for configuring code generation options, like stack alignment.
*/
TargetOptions options;
#if defined(PIPE_ARCH_X86)
options.StackAlignmentOverride = 4;
#if HAVE_LLVM < 0x0304
options.RealignStack = true;
#endif
#endif
#if defined(DEBUG) && HAVE_LLVM < 0x0307
options.JITEmitDebugInfo = true;
#endif
/* XXX: Workaround http://llvm.org/PR21435 */
#if defined(DEBUG) || defined(PROFILE) || \
(HAVE_LLVM >= 0x0303 && (defined(PIPE_ARCH_X86) || defined(PIPE_ARCH_X86_64)))
#if HAVE_LLVM < 0x0304
options.NoFramePointerElimNonLeaf = true;
#endif
options.NoFramePointerElim = true;
#endif
builder.setEngineKind(EngineKind::JIT)
.setErrorStr(&Error)
.setTargetOptions(options)
.setOptLevel((CodeGenOpt::Level)OptLevel);
if (useMCJIT) {
#if HAVE_LLVM < 0x0306
builder.setUseMCJIT(true);
#endif
#ifdef _WIN32
/*
* MCJIT works on Windows, but currently only through ELF object format.
*/
std::string targetTriple = llvm::sys::getProcessTriple();
targetTriple.append("-elf");
unwrap(M)->setTargetTriple(targetTriple);
#endif
}
llvm::SmallVector<std::string, 1> MAttrs;
if (util_cpu_caps.has_avx) {
/*
* AVX feature is not automatically detected from CPUID by the X86 target
* yet, because the old (yet default) JIT engine is not capable of
* emitting the opcodes. On newer llvm versions it is and at least some
* versions (tested with 3.3) will emit avx opcodes without this anyway.
*/
MAttrs.push_back("+avx");
if (util_cpu_caps.has_f16c) {
MAttrs.push_back("+f16c");
}
builder.setMAttrs(MAttrs);
}
#if HAVE_LLVM >= 0x0305
StringRef MCPU = llvm::sys::getHostCPUName();
/*
* The cpu bits are no longer set automatically, so need to set mcpu manually.
* Note that the MAttrs set above will be sort of ignored (since we should
* not set any which would not be set by specifying the cpu anyway).
* It ought to be safe though since getHostCPUName() should include bits
* not only from the cpu but environment as well (for instance if it's safe
* to use avx instructions which need OS support). According to
* http://llvm.org/bugs/show_bug.cgi?id=19429 however if I understand this
* right it may be necessary to specify older cpu (or disable mattrs) though
* when not using MCJIT so no instructions are generated which the old JIT
* can't handle. Not entirely sure if we really need to do anything yet.
*/
builder.setMCPU(MCPU);
#endif
ShaderMemoryManager *MM = NULL;
if (useMCJIT) {
#if HAVE_LLVM > 0x0303
BaseMemoryManager* JMM = reinterpret_cast<BaseMemoryManager*>(CMM);
MM = new ShaderMemoryManager(JMM);
*OutCode = MM->getGeneratedCode();
#if HAVE_LLVM >= 0x0306
builder.setMCJITMemoryManager(std::unique_ptr<RTDyldMemoryManager>(MM));
MM = NULL; // ownership taken by std::unique_ptr
#else
builder.setMCJITMemoryManager(MM);
#endif
#endif
} else {
#if HAVE_LLVM < 0x0306
BaseMemoryManager* JMM = reinterpret_cast<BaseMemoryManager*>(CMM);
MM = new ShaderMemoryManager(JMM);
*OutCode = MM->getGeneratedCode();
builder.setJITMemoryManager(MM);
#else
assert(0);
#endif
}
ExecutionEngine *JIT;
JIT = builder.create();
if (JIT) {
*OutJIT = wrap(JIT);
return 0;
}
lp_free_generated_code(*OutCode);
*OutCode = 0;
delete MM;
*OutError = strdup(Error.c_str());
return 1;
}
extern "C"
void
lp_free_generated_code(struct lp_generated_code *code)
{
ShaderMemoryManager::freeGeneratedCode(code);
}
extern "C"
LLVMMCJITMemoryManagerRef
lp_get_default_memory_manager()
{
BaseMemoryManager *mm;
#if HAVE_LLVM < 0x0306
mm = llvm::JITMemoryManager::CreateDefaultMemManager();
#else
mm = new llvm::SectionMemoryManager();
#endif
return reinterpret_cast<LLVMMCJITMemoryManagerRef>(mm);
}
extern "C"
void
lp_free_memory_manager(LLVMMCJITMemoryManagerRef memorymgr)
{
delete reinterpret_cast<BaseMemoryManager*>(memorymgr);
}