1/****************************************************************************
2 * Copyright (C) 2014-2018 Intel Corporation.   All Rights Reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * @file JitManager.cpp
24 *
25 * @brief Implementation if the Jit Manager.
26 *
27 * Notes:
28 *
29 ******************************************************************************/
30#include "jit_pch.hpp"
31
32#include "JitManager.h"
33#include "jit_api.h"
34#include "fetch_jit.h"
35
36#include "core/state.h"
37
38#include "gen_state_llvm.h"
39
40#include <sstream>
41#if defined(_WIN32)
42#include <psapi.h>
43#include <cstring>
44
45#define INTEL_OUTPUT_DIR "c:\\Intel"
46#define SWR_OUTPUT_DIR INTEL_OUTPUT_DIR "\\SWR"
47#define JITTER_OUTPUT_DIR SWR_OUTPUT_DIR "\\Jitter"
48#endif // _WIN32
49
50#if defined(__APPLE__) || defined(FORCE_LINUX) || defined(__linux__) || defined(__gnu_linux__)
51#include <pwd.h>
52#include <sys/stat.h>
53#endif
54
55
56using namespace llvm;
57using namespace SwrJit;
58
59//////////////////////////////////////////////////////////////////////////
60/// @brief Contructor for JitManager.
61/// @param simdWidth - SIMD width to be used in generated program.
62JitManager::JitManager(uint32_t simdWidth, const char* arch, const char* core) :
63    mContext(), mBuilder(mContext), mIsModuleFinalized(true), mJitNumber(0), mVWidth(simdWidth),
64    mArch(arch)
65{
66    mpCurrentModule = nullptr;
67    mpExec          = nullptr;
68
69    InitializeNativeTarget();
70    InitializeNativeTargetAsmPrinter();
71    InitializeNativeTargetDisassembler();
72
73
74    // force JIT to use the same CPU arch as the rest of swr
75    if (mArch.AVX512F())
76    {
77#if USE_SIMD16_SHADERS
78        if (mArch.AVX512ER())
79        {
80            mHostCpuName = StringRef("knl");
81        }
82        else
83        {
84            mHostCpuName = StringRef("skylake-avx512");
85        }
86        mUsingAVX512 = true;
87#else
88        mHostCpuName = StringRef("core-avx2");
89#endif
90        if (mVWidth == 0)
91        {
92            mVWidth = 8;
93        }
94    }
95    else if (mArch.AVX2())
96    {
97        mHostCpuName = StringRef("core-avx2");
98        if (mVWidth == 0)
99        {
100            mVWidth = 8;
101        }
102    }
103    else if (mArch.AVX())
104    {
105        if (mArch.F16C())
106        {
107            mHostCpuName = StringRef("core-avx-i");
108        }
109        else
110        {
111            mHostCpuName = StringRef("corei7-avx");
112        }
113        if (mVWidth == 0)
114        {
115            mVWidth = 8;
116        }
117    }
118    else
119    {
120        SWR_INVALID("Jitting requires at least AVX ISA support");
121    }
122
123
124    mOptLevel = CodeGenOpt::Aggressive;
125
126    if (KNOB_JIT_OPTIMIZATION_LEVEL >= CodeGenOpt::None &&
127        KNOB_JIT_OPTIMIZATION_LEVEL <= CodeGenOpt::Aggressive)
128    {
129        mOptLevel = CodeGenOpt::Level(KNOB_JIT_OPTIMIZATION_LEVEL);
130    }
131
132    if (KNOB_JIT_ENABLE_CACHE)
133    {
134        mCache.Init(this, mHostCpuName, mOptLevel);
135    }
136
137    SetupNewModule();
138    mIsModuleFinalized = true;
139
140    // fetch function signature
141#if USE_SIMD16_SHADERS
142    // typedef void(__cdecl *PFN_FETCH_FUNC)(SWR_FETCH_CONTEXT& fetchInfo, simd16vertex& out);
143#else
144    // typedef void(__cdecl *PFN_FETCH_FUNC)(SWR_FETCH_CONTEXT& fetchInfo, simdvertex& out);
145#endif
146    std::vector<Type*> fsArgs;
147
148    // llvm5 is picky and does not take a void * type
149    fsArgs.push_back(PointerType::get(Gen_SWR_FETCH_CONTEXT(this), 0));
150
151    fsArgs.push_back(Type::getInt8PtrTy(mContext));
152
153    fsArgs.push_back(PointerType::get(Gen_SWR_FETCH_CONTEXT(this), 0));
154#if USE_SIMD16_SHADERS
155    fsArgs.push_back(PointerType::get(Gen_simd16vertex(this), 0));
156#else
157    fsArgs.push_back(PointerType::get(Gen_simdvertex(this), 0));
158#endif
159
160    mFetchShaderTy = FunctionType::get(Type::getVoidTy(mContext), fsArgs, false);
161
162#if defined(_MSC_VER)
163    // explicitly instantiate used symbols from potentially staticly linked libs
164    sys::DynamicLibrary::AddSymbol("exp2f", &exp2f);
165    sys::DynamicLibrary::AddSymbol("log2f", &log2f);
166    sys::DynamicLibrary::AddSymbol("sinf", &sinf);
167    sys::DynamicLibrary::AddSymbol("cosf", &cosf);
168    sys::DynamicLibrary::AddSymbol("powf", &powf);
169#endif
170
171#if defined(_WIN32)
172    if (KNOB_DUMP_SHADER_IR)
173    {
174        CreateDirectoryPath(INTEL_OUTPUT_DIR);
175        CreateDirectoryPath(SWR_OUTPUT_DIR);
176        CreateDirectoryPath(JITTER_OUTPUT_DIR);
177    }
178#endif
179}
180
181void JitManager::CreateExecEngine(std::unique_ptr<Module> pModule)
182{
183    TargetOptions tOpts;
184    tOpts.AllowFPOpFusion = FPOpFusion::Fast;
185    tOpts.NoInfsFPMath    = false;
186    tOpts.NoNaNsFPMath    = false;
187    tOpts.UnsafeFPMath = false;
188
189    // tOpts.PrintMachineCode    = true;
190
191    mpExec = EngineBuilder(std::move(pModule))
192                 .setTargetOptions(tOpts)
193                 .setOptLevel(mOptLevel)
194                 .setMCPU(mHostCpuName)
195                 .create();
196
197    if (KNOB_JIT_ENABLE_CACHE)
198    {
199        mpExec->setObjectCache(&mCache);
200    }
201
202#if LLVM_USE_INTEL_JITEVENTS
203    JITEventListener* vTune = JITEventListener::createIntelJITEventListener();
204    mpExec->RegisterJITEventListener(vTune);
205#endif
206
207    mvExecEngines.push_back(mpExec);
208}
209
210//////////////////////////////////////////////////////////////////////////
211/// @brief Create new LLVM module.
212void JitManager::SetupNewModule()
213{
214    SWR_ASSERT(mIsModuleFinalized == true && "Current module is not finalized!");
215
216    std::unique_ptr<Module> newModule(new Module("", mContext));
217    mpCurrentModule = newModule.get();
218    mpCurrentModule->setTargetTriple(sys::getProcessTriple());
219    CreateExecEngine(std::move(newModule));
220    mIsModuleFinalized = false;
221}
222
223
224DIType*
225JitManager::CreateDebugStructType(StructType*                                          pType,
226                                  const std::string&                                   name,
227                                  DIFile*                                              pFile,
228                                  uint32_t                                             lineNum,
229                                  const std::vector<std::pair<std::string, uint32_t>>& members)
230{
231    DIBuilder                 builder(*mpCurrentModule);
232    SmallVector<Metadata*, 8> ElemTypes;
233    DataLayout                DL        = DataLayout(mpCurrentModule);
234    uint32_t                  size      = DL.getTypeAllocSizeInBits(pType);
235    uint32_t                  alignment = DL.getABITypeAlignment(pType);
236    DINode::DIFlags           flags     = DINode::DIFlags::FlagPublic;
237
238    DICompositeType* pDIStructTy = builder.createStructType(pFile,
239                                                            name,
240                                                            pFile,
241                                                            lineNum,
242                                                            size,
243                                                            alignment,
244                                                            flags,
245                                                            nullptr,
246                                                            builder.getOrCreateArray(ElemTypes));
247
248    // Register mapping now to break loops (in case struct contains itself or pointers to itself)
249    mDebugStructMap[pType] = pDIStructTy;
250
251    uint32_t idx = 0;
252    for (auto& elem : pType->elements())
253    {
254        std::string name       = members[idx].first;
255        uint32_t    lineNum    = members[idx].second;
256        size                   = DL.getTypeAllocSizeInBits(elem);
257        alignment              = DL.getABITypeAlignment(elem);
258        uint32_t      offset   = DL.getStructLayout(pType)->getElementOffsetInBits(idx);
259        llvm::DIType* pDebugTy = GetDebugType(elem);
260        ElemTypes.push_back(builder.createMemberType(
261            pDIStructTy, name, pFile, lineNum, size, alignment, offset, flags, pDebugTy));
262
263        idx++;
264    }
265
266    pDIStructTy->replaceElements(builder.getOrCreateArray(ElemTypes));
267    return pDIStructTy;
268}
269
270DIType* JitManager::GetDebugArrayType(Type* pTy)
271{
272    DIBuilder  builder(*mpCurrentModule);
273    DataLayout DL        = DataLayout(mpCurrentModule);
274    ArrayType* pArrayTy  = cast<ArrayType>(pTy);
275    uint32_t   size      = DL.getTypeAllocSizeInBits(pArrayTy);
276    uint32_t   alignment = DL.getABITypeAlignment(pArrayTy);
277
278    SmallVector<Metadata*, 8> Elems;
279    Elems.push_back(builder.getOrCreateSubrange(0, pArrayTy->getNumElements()));
280    return builder.createArrayType(
281        size, alignment, GetDebugType(pArrayTy->getElementType()), builder.getOrCreateArray(Elems));
282}
283
284// Create a DIType from llvm Type
285DIType* JitManager::GetDebugType(Type* pTy)
286{
287    DIBuilder    builder(*mpCurrentModule);
288    Type::TypeID id = pTy->getTypeID();
289
290    switch (id)
291    {
292    case Type::VoidTyID:
293        return builder.createUnspecifiedType("void");
294        break;
295    case Type::HalfTyID:
296        return builder.createBasicType("float16", 16, dwarf::DW_ATE_float);
297        break;
298    case Type::FloatTyID:
299        return builder.createBasicType("float", 32, dwarf::DW_ATE_float);
300        break;
301    case Type::DoubleTyID:
302        return builder.createBasicType("double", 64, dwarf::DW_ATE_float);
303        break;
304    case Type::IntegerTyID:
305        return GetDebugIntegerType(pTy);
306        break;
307    case Type::StructTyID:
308        return GetDebugStructType(pTy);
309        break;
310    case Type::ArrayTyID:
311        return GetDebugArrayType(pTy);
312        break;
313    case Type::PointerTyID:
314        return builder.createPointerType(GetDebugType(pTy->getPointerElementType()), 64, 64);
315        break;
316#if LLVM_VERSION_MAJOR >= 11
317    case Type::FixedVectorTyID:
318#else
319    case Type::VectorTyID:
320#endif
321        return GetDebugVectorType(pTy);
322        break;
323    case Type::FunctionTyID:
324        return GetDebugFunctionType(pTy);
325        break;
326    default:
327        SWR_ASSERT(false, "Unimplemented llvm type");
328    }
329    return nullptr;
330}
331
332// Create a DISubroutineType from an llvm FunctionType
333DIType* JitManager::GetDebugFunctionType(Type* pTy)
334{
335    SmallVector<Metadata*, 8> ElemTypes;
336    FunctionType*             pFuncTy = cast<FunctionType>(pTy);
337    DIBuilder                 builder(*mpCurrentModule);
338
339    // Add result type
340    ElemTypes.push_back(GetDebugType(pFuncTy->getReturnType()));
341
342    // Add arguments
343    for (auto& param : pFuncTy->params())
344    {
345        ElemTypes.push_back(GetDebugType(param));
346    }
347
348    return builder.createSubroutineType(builder.getOrCreateTypeArray(ElemTypes));
349}
350
351DIType* JitManager::GetDebugIntegerType(Type* pTy)
352{
353    DIBuilder    builder(*mpCurrentModule);
354    IntegerType* pIntTy = cast<IntegerType>(pTy);
355    switch (pIntTy->getBitWidth())
356    {
357    case 1:
358        return builder.createBasicType("int1", 1, dwarf::DW_ATE_unsigned);
359        break;
360    case 8:
361        return builder.createBasicType("int8", 8, dwarf::DW_ATE_signed);
362        break;
363    case 16:
364        return builder.createBasicType("int16", 16, dwarf::DW_ATE_signed);
365        break;
366    case 32:
367        return builder.createBasicType("int", 32, dwarf::DW_ATE_signed);
368        break;
369    case 64:
370        return builder.createBasicType("int64", 64, dwarf::DW_ATE_signed);
371        break;
372    case 128:
373        return builder.createBasicType("int128", 128, dwarf::DW_ATE_signed);
374        break;
375    default:
376        SWR_ASSERT(false, "Unimplemented integer bit width");
377    }
378    return nullptr;
379}
380
381DIType* JitManager::GetDebugVectorType(Type* pTy)
382{
383    DIBuilder                 builder(*mpCurrentModule);
384#if LLVM_VERSION_MAJOR >= 12
385    FixedVectorType*          pVecTy    = cast<FixedVectorType>(pTy);
386#elif LLVM_VERSION_MAJOR >= 11
387    VectorType*               pVecTy    = cast<VectorType>(pTy);
388#else
389    auto                      pVecTy    = pTy;
390#endif
391    DataLayout                DL        = DataLayout(mpCurrentModule);
392    uint32_t                  size      = DL.getTypeAllocSizeInBits(pVecTy);
393    uint32_t                  alignment = DL.getABITypeAlignment(pVecTy);
394    SmallVector<Metadata*, 1> Elems;
395
396#if LLVM_VERSION_MAJOR >= 11
397    Elems.push_back(builder.getOrCreateSubrange(0, pVecTy->getNumElements()));
398#else
399    Elems.push_back(builder.getOrCreateSubrange(0, pVecTy->getVectorNumElements()));
400#endif
401
402    return builder.createVectorType(size,
403                                    alignment,
404#if LLVM_VERSION_MAJOR >= 11
405                                    GetDebugType(pVecTy->getElementType()),
406#else
407                                    GetDebugType(pVecTy->getVectorElementType()),
408#endif
409                                    builder.getOrCreateArray(Elems));
410}
411
412//////////////////////////////////////////////////////////////////////////
413/// @brief Dump function x86 assembly to file.
414/// @note This should only be called after the module has been jitted to x86 and the
415///       module will not be further accessed.
416void JitManager::DumpAsm(Function* pFunction, const char* fileName)
417{
418    if (KNOB_DUMP_SHADER_IR)
419    {
420#if defined(_WIN32)
421        DWORD pid = GetCurrentProcessId();
422        char  procname[MAX_PATH];
423        GetModuleFileNameA(NULL, procname, MAX_PATH);
424        const char*       pBaseName = strrchr(procname, '\\');
425        std::stringstream outDir;
426        outDir << JITTER_OUTPUT_DIR << pBaseName << "_" << pid << std::ends;
427        CreateDirectoryPath(outDir.str().c_str());
428#endif
429
430        std::error_code EC;
431        Module*         pModule  = pFunction->getParent();
432        const char*     funcName = pFunction->getName().data();
433        char            fName[256];
434#if defined(_WIN32)
435        sprintf(fName, "%s\\%s.%s.asm", outDir.str().c_str(), funcName, fileName);
436#else
437        sprintf(fName, "%s.%s.asm", funcName, fileName);
438#endif
439
440        raw_fd_ostream filestream(fName, EC, llvm::sys::fs::F_None);
441
442        legacy::PassManager* pMPasses         = new legacy::PassManager();
443        auto*                pTarget          = mpExec->getTargetMachine();
444        pTarget->Options.MCOptions.AsmVerbose = true;
445#if LLVM_VERSION_MAJOR >= 10
446        pTarget->addPassesToEmitFile(
447            *pMPasses, filestream, nullptr, CGFT_AssemblyFile);
448#elif LLVM_VERSION_MAJOR >= 7
449        pTarget->addPassesToEmitFile(
450            *pMPasses, filestream, nullptr, TargetMachine::CGFT_AssemblyFile);
451#else
452        pTarget->addPassesToEmitFile(*pMPasses, filestream, TargetMachine::CGFT_AssemblyFile);
453#endif
454        pMPasses->run(*pModule);
455        delete pMPasses;
456        pTarget->Options.MCOptions.AsmVerbose = false;
457    }
458}
459
460std::string JitManager::GetOutputDir()
461{
462#if defined(_WIN32)
463    DWORD pid = GetCurrentProcessId();
464    char  procname[MAX_PATH];
465    GetModuleFileNameA(NULL, procname, MAX_PATH);
466    const char*       pBaseName = strrchr(procname, '\\');
467    std::stringstream outDir;
468    outDir << JITTER_OUTPUT_DIR << pBaseName << "_" << pid;
469    CreateDirectoryPath(outDir.str().c_str());
470    return outDir.str();
471#endif
472    return "";
473}
474
475//////////////////////////////////////////////////////////////////////////
476/// @brief Dump function to file.
477void JitManager::DumpToFile(Module*                         M,
478                            const char*                     fileName,
479                            llvm::AssemblyAnnotationWriter* annotater)
480{
481    if (KNOB_DUMP_SHADER_IR)
482    {
483        std::string outDir = GetOutputDir();
484
485        std::error_code EC;
486        const char*     funcName = M->getName().data();
487        char            fName[256];
488#if defined(_WIN32)
489        sprintf(fName, "%s\\%s.%s.ll", outDir.c_str(), funcName, fileName);
490#else
491        sprintf(fName, "%s.%s.ll", funcName, fileName);
492#endif
493        raw_fd_ostream fd(fName, EC, llvm::sys::fs::F_None);
494        M->print(fd, annotater);
495        fd.flush();
496    }
497}
498
499//////////////////////////////////////////////////////////////////////////
500/// @brief Dump function to file.
501void JitManager::DumpToFile(Function* f, const char* fileName)
502{
503    if (KNOB_DUMP_SHADER_IR)
504    {
505        std::string outDir = GetOutputDir();
506
507        std::error_code EC;
508        const char*     funcName = f->getName().data();
509        char            fName[256];
510#if defined(_WIN32)
511        sprintf(fName, "%s\\%s.%s.ll", outDir.c_str(), funcName, fileName);
512#else
513        sprintf(fName, "%s.%s.ll", funcName, fileName);
514#endif
515        raw_fd_ostream fd(fName, EC, llvm::sys::fs::F_None);
516        f->print(fd, nullptr);
517
518#if defined(_WIN32)
519        sprintf(fName, "%s\\cfg.%s.%s.dot", outDir.c_str(), funcName, fileName);
520#else
521        sprintf(fName, "cfg.%s.%s.dot", funcName, fileName);
522#endif
523        fd.flush();
524
525        raw_fd_ostream fd_cfg(fName, EC, llvm::sys::fs::F_Text);
526        WriteGraph(fd_cfg, (const Function*)f);
527
528        fd_cfg.flush();
529    }
530}
531
532extern "C" {
533bool g_DllActive = true;
534
535//////////////////////////////////////////////////////////////////////////
536/// @brief Create JIT context.
537/// @param simdWidth - SIMD width to be used in generated program.
538HANDLE JITCALL JitCreateContext(uint32_t targetSimdWidth, const char* arch, const char* core)
539{
540    return new JitManager(targetSimdWidth, arch, core);
541}
542
543//////////////////////////////////////////////////////////////////////////
544/// @brief Destroy JIT context.
545void JITCALL JitDestroyContext(HANDLE hJitContext)
546{
547    if (g_DllActive)
548    {
549        delete reinterpret_cast<JitManager*>(hJitContext);
550    }
551}
552}
553
554//////////////////////////////////////////////////////////////////////////
555/// JitCache
556//////////////////////////////////////////////////////////////////////////
557
558//////////////////////////////////////////////////////////////////////////
559/// JitCacheFileHeader
560//////////////////////////////////////////////////////////////////////////
561struct JitCacheFileHeader
562{
563    void Init(uint32_t           llCRC,
564              uint32_t           objCRC,
565              const std::string& moduleID,
566              const std::string& cpu,
567              uint32_t           optLevel,
568              uint64_t           objSize)
569    {
570        m_objSize = objSize;
571        m_llCRC   = llCRC;
572        m_objCRC  = objCRC;
573        strncpy(m_ModuleID, moduleID.c_str(), JC_STR_MAX_LEN - 1);
574        m_ModuleID[JC_STR_MAX_LEN - 1] = 0;
575        strncpy(m_Cpu, cpu.c_str(), JC_STR_MAX_LEN - 1);
576        m_Cpu[JC_STR_MAX_LEN - 1] = 0;
577        m_optLevel                = optLevel;
578    }
579
580
581    bool
582    IsValid(uint32_t llCRC, const std::string& moduleID, const std::string& cpu, uint32_t optLevel)
583    {
584        if ((m_MagicNumber != JC_MAGIC_NUMBER) || (m_llCRC != llCRC) ||
585            (m_platformKey != JC_PLATFORM_KEY) || (m_optLevel != optLevel))
586        {
587            return false;
588        }
589
590        m_ModuleID[JC_STR_MAX_LEN - 1] = 0;
591        if (strncmp(moduleID.c_str(), m_ModuleID, JC_STR_MAX_LEN - 1))
592        {
593            return false;
594        }
595
596        m_Cpu[JC_STR_MAX_LEN - 1] = 0;
597        if (strncmp(cpu.c_str(), m_Cpu, JC_STR_MAX_LEN - 1))
598        {
599            return false;
600        }
601
602        return true;
603    }
604
605    uint64_t GetObjectSize() const { return m_objSize; }
606    uint64_t GetObjectCRC() const { return m_objCRC; }
607
608private:
609    static const uint64_t JC_MAGIC_NUMBER = 0xfedcba9876543210ULL + 7;
610    static const size_t   JC_STR_MAX_LEN  = 32;
611    static const uint32_t JC_PLATFORM_KEY = (LLVM_VERSION_MAJOR << 24) |
612                                            (LLVM_VERSION_MINOR << 16) | (LLVM_VERSION_PATCH << 8) |
613                                            ((sizeof(void*) > sizeof(uint32_t)) ? 1 : 0);
614
615    uint64_t m_MagicNumber              = JC_MAGIC_NUMBER;
616    uint64_t m_objSize                  = 0;
617    uint32_t m_llCRC                    = 0;
618    uint32_t m_platformKey              = JC_PLATFORM_KEY;
619    uint32_t m_objCRC                   = 0;
620    uint32_t m_optLevel                 = 0;
621    char     m_ModuleID[JC_STR_MAX_LEN] = {};
622    char     m_Cpu[JC_STR_MAX_LEN]      = {};
623};
624
625static inline uint32_t ComputeModuleCRC(const llvm::Module* M)
626{
627    std::string        bitcodeBuffer;
628    raw_string_ostream bitcodeStream(bitcodeBuffer);
629
630#if LLVM_VERSION_MAJOR >= 7
631    llvm::WriteBitcodeToFile(*M, bitcodeStream);
632#else
633    llvm::WriteBitcodeToFile(M, bitcodeStream);
634#endif
635    // M->print(bitcodeStream, nullptr, false);
636
637    bitcodeStream.flush();
638
639    return ComputeCRC(0, bitcodeBuffer.data(), bitcodeBuffer.size());
640}
641
642/// constructor
643JitCache::JitCache()
644{
645#if defined(__APPLE__) || defined(FORCE_LINUX) || defined(__linux__) || defined(__gnu_linux__)
646    if (strncmp(KNOB_JIT_CACHE_DIR.c_str(), "~/", 2) == 0)
647    {
648        char* homedir;
649        if (!(homedir = getenv("HOME")))
650        {
651            homedir = getpwuid(getuid())->pw_dir;
652        }
653        mCacheDir = homedir;
654        mCacheDir += (KNOB_JIT_CACHE_DIR.c_str() + 1);
655    }
656    else
657#endif
658    {
659        mCacheDir = KNOB_JIT_CACHE_DIR;
660    }
661
662    // Create cache dir at startup to allow jitter to write debug.ll files
663    // to that directory.
664    if (!llvm::sys::fs::exists(mCacheDir.str()) &&
665        llvm::sys::fs::create_directories(mCacheDir.str()))
666    {
667        SWR_INVALID("Unable to create directory: %s", mCacheDir.c_str());
668    }
669
670}
671
672int ExecUnhookedProcess(const std::string& CmdLine, std::string* pStdOut, std::string* pStdErr)
673{
674
675    return ExecCmd(CmdLine, nullptr, pStdOut, pStdErr);
676}
677
678/// Calculate actual directory where module will be cached.
679/// This is always a subdirectory of mCacheDir.  Full absolute
680/// path name will be stored in mCurrentModuleCacheDir
681void JitCache::CalcModuleCacheDir()
682{
683    mModuleCacheDir.clear();
684
685    llvm::SmallString<MAX_PATH> moduleDir = mCacheDir;
686
687    // Create 4 levels of directory hierarchy based on CRC, 256 entries each
688    uint8_t* pCRC = (uint8_t*)&mCurrentModuleCRC;
689    for (uint32_t i = 0; i < 4; ++i)
690    {
691        llvm::sys::path::append(moduleDir, std::to_string((int)pCRC[i]));
692    }
693
694    mModuleCacheDir = moduleDir;
695}
696
697/// notifyObjectCompiled - Provides a pointer to compiled code for Module M.
698void JitCache::notifyObjectCompiled(const llvm::Module* M, llvm::MemoryBufferRef Obj)
699{
700    const std::string& moduleID = M->getModuleIdentifier();
701    if (!moduleID.length())
702    {
703        return;
704    }
705
706    if (!mModuleCacheDir.size())
707    {
708        SWR_INVALID("Unset module cache directory");
709        return;
710    }
711
712    if (!llvm::sys::fs::exists(mModuleCacheDir.str()) &&
713        llvm::sys::fs::create_directories(mModuleCacheDir.str()))
714    {
715        SWR_INVALID("Unable to create directory: %s", mModuleCacheDir.c_str());
716        return;
717    }
718
719    JitCacheFileHeader header;
720
721    llvm::SmallString<MAX_PATH> filePath = mModuleCacheDir;
722    llvm::sys::path::append(filePath, moduleID);
723
724    llvm::SmallString<MAX_PATH> objPath = filePath;
725    objPath += JIT_OBJ_EXT;
726
727    {
728        std::error_code      err;
729        llvm::raw_fd_ostream fileObj(objPath.c_str(), err, llvm::sys::fs::F_None);
730        fileObj << Obj.getBuffer();
731        fileObj.flush();
732    }
733
734
735    {
736        std::error_code      err;
737        llvm::raw_fd_ostream fileObj(filePath.c_str(), err, llvm::sys::fs::F_None);
738
739        uint32_t objcrc = ComputeCRC(0, Obj.getBufferStart(), Obj.getBufferSize());
740
741        header.Init(mCurrentModuleCRC, objcrc, moduleID, mCpu, mOptLevel, Obj.getBufferSize());
742
743        fileObj.write((const char*)&header, sizeof(header));
744        fileObj.flush();
745    }
746}
747
748/// Returns a pointer to a newly allocated MemoryBuffer that contains the
749/// object which corresponds with Module M, or 0 if an object is not
750/// available.
751std::unique_ptr<llvm::MemoryBuffer> JitCache::getObject(const llvm::Module* M)
752{
753    const std::string& moduleID = M->getModuleIdentifier();
754    mCurrentModuleCRC           = ComputeModuleCRC(M);
755
756    if (!moduleID.length())
757    {
758        return nullptr;
759    }
760
761    CalcModuleCacheDir();
762
763    if (!llvm::sys::fs::exists(mModuleCacheDir))
764    {
765        return nullptr;
766    }
767
768    llvm::SmallString<MAX_PATH> filePath = mModuleCacheDir;
769    llvm::sys::path::append(filePath, moduleID);
770
771    llvm::SmallString<MAX_PATH> objFilePath = filePath;
772    objFilePath += JIT_OBJ_EXT;
773
774    FILE* fpObjIn = nullptr;
775    FILE* fpIn    = fopen(filePath.c_str(), "rb");
776    if (!fpIn)
777    {
778        return nullptr;
779    }
780
781    std::unique_ptr<llvm::MemoryBuffer> pBuf = nullptr;
782    do
783    {
784        JitCacheFileHeader header;
785        if (!fread(&header, sizeof(header), 1, fpIn))
786        {
787            break;
788        }
789
790        if (!header.IsValid(mCurrentModuleCRC, moduleID, mCpu, mOptLevel))
791        {
792            break;
793        }
794
795        fpObjIn = fopen(objFilePath.c_str(), "rb");
796        if (!fpObjIn)
797        {
798            break;
799        }
800
801#if LLVM_VERSION_MAJOR < 6
802        pBuf = llvm::MemoryBuffer::getNewUninitMemBuffer(size_t(header.GetObjectSize()));
803#else
804        pBuf = llvm::WritableMemoryBuffer::getNewUninitMemBuffer(size_t(header.GetObjectSize()));
805#endif
806        if (!fread(const_cast<char*>(pBuf->getBufferStart()), header.GetObjectSize(), 1, fpObjIn))
807        {
808            pBuf = nullptr;
809            break;
810        }
811
812        if (header.GetObjectCRC() != ComputeCRC(0, pBuf->getBufferStart(), pBuf->getBufferSize()))
813        {
814            SWR_TRACE("Invalid object cache file, ignoring: %s", filePath.c_str());
815            pBuf = nullptr;
816            break;
817        }
818
819    } while (0);
820
821    fclose(fpIn);
822
823    if (fpObjIn)
824    {
825        fclose(fpObjIn);
826    }
827
828
829    return pBuf;
830}
831
832void InterleaveAssemblyAnnotater::emitInstructionAnnot(const llvm::Instruction*     pInst,
833                                                       llvm::formatted_raw_ostream& OS)
834{
835    auto dbgLoc = pInst->getDebugLoc();
836    if (dbgLoc)
837    {
838        unsigned int line = dbgLoc.getLine();
839        if (line != mCurrentLineNo)
840        {
841            if (line > 0 && line <= mAssembly.size())
842            {
843                // HACK: here we assume that OS is a formatted_raw_ostream(ods())
844                // and modify the color accordingly. We can't do the color
845                // modification on OS because formatted_raw_ostream strips
846                // the color information. The only way to fix this behavior
847                // is to patch LLVM.
848                OS << "\n; " << line << ": " << mAssembly[line - 1] << "\n";
849            }
850            mCurrentLineNo = line;
851        }
852    }
853}
854