diff --git a/0031-ACPO-ACPO-Infrastructure.patch b/0031-ACPO-ACPO-Infrastructure.patch new file mode 100644 index 0000000000000000000000000000000000000000..12d6b00c890af583c8a488d06f8370200d5dbe1b --- /dev/null +++ b/0031-ACPO-ACPO-Infrastructure.patch @@ -0,0 +1,6173 @@ +From 9773a0bfd29580c31867afccada947457617628e Mon Sep 17 00:00:00 2001 +From: tsczajkowski +Date: Thu, 22 Aug 2024 23:56:11 -0400 +Subject: [PATCH] [ACPO] ACPO Infrastructure + +This change introduces ACPO ML infrastructure to enable use of ML models +within LLVM compiler using a simple interface. +--- + ACPO_README.md | 36 + + llvm/CMakeLists.txt | 4 + + .../llvm/Analysis/ACPOCollectFeatures.h | 296 ++++ + llvm/include/llvm/Analysis/ACPOMLInterface.h | 482 ++++++ + llvm/include/llvm/Analysis/ACPOModel.h | 122 ++ + llvm/include/llvm/Analysis/ACPOModelRunner.h | 39 + + llvm/include/llvm/Analysis/AOTModelRunner.h | 203 +++ + llvm/include/llvm/Analysis/CallHeight.h | 72 + + llvm/include/llvm/Analysis/DumpCallsite.h | 27 + + llvm/include/llvm/Analysis/DumpFeature.h | 194 +++ + llvm/include/llvm/Analysis/LoopInfo.h | 11 + + .../llvm/Analysis/ModelDataCollector.h | 108 ++ + llvm/include/llvm/InitializePasses.h | 8 + + llvm/lib/Analysis/ACPOCollectFeatures.cpp | 1258 +++++++++++++++ + llvm/lib/Analysis/ACPOMLInterface.cpp | 1405 +++++++++++++++++ + llvm/lib/Analysis/ACPOModel.cpp | 63 + + llvm/lib/Analysis/CMakeLists.txt | 32 + + llvm/lib/Analysis/CallHeight.cpp | 89 ++ + llvm/lib/Analysis/DumpCallsite.cpp | 82 + + llvm/lib/Analysis/DumpFeature.cpp | 575 +++++++ + llvm/lib/Analysis/ModelDataCollector.cpp | 350 ++++ + llvm/lib/CodeGen/CMakeLists.txt | 2 +- + llvm/lib/IR/AsmWriter.cpp | 220 ++- + llvm/lib/Passes/PassBuilder.cpp | 6 + + llvm/lib/Passes/PassBuilderPipelines.cpp | 14 + + llvm/lib/Passes/PassRegistry.def | 10 + + 26 files changed, 5679 insertions(+), 29 deletions(-) + create mode 100644 ACPO_README.md + create mode 100644 llvm/include/llvm/Analysis/ACPOCollectFeatures.h + create mode 100644 llvm/include/llvm/Analysis/ACPOMLInterface.h + create mode 100644 llvm/include/llvm/Analysis/ACPOModel.h + create mode 100644 llvm/include/llvm/Analysis/ACPOModelRunner.h + create mode 100644 llvm/include/llvm/Analysis/AOTModelRunner.h + create mode 100644 llvm/include/llvm/Analysis/CallHeight.h + create mode 100644 llvm/include/llvm/Analysis/DumpCallsite.h + create mode 100644 llvm/include/llvm/Analysis/DumpFeature.h + create mode 100644 llvm/include/llvm/Analysis/ModelDataCollector.h + create mode 100644 llvm/lib/Analysis/ACPOCollectFeatures.cpp + create mode 100644 llvm/lib/Analysis/ACPOMLInterface.cpp + create mode 100644 llvm/lib/Analysis/ACPOModel.cpp + create mode 100644 llvm/lib/Analysis/CallHeight.cpp + create mode 100644 llvm/lib/Analysis/DumpCallsite.cpp + create mode 100644 llvm/lib/Analysis/DumpFeature.cpp + create mode 100644 llvm/lib/Analysis/ModelDataCollector.cpp + +diff --git a/llvm/CMakeLists.txt b/llvm/CMakeLists.txt +index 79de9eb2e3e7..b0afb47a7243 100644 +--- a/llvm/CMakeLists.txt ++++ b/llvm/CMakeLists.txt +@@ -1001,6 +1001,9 @@ endif() + # + set(TENSORFLOW_AOT_PATH "" CACHE PATH "Path to TensorFlow pip install dir") + ++set(LLVM_INLINER_MODEL_PATH "" CACHE PATH "Path to the inliner model") ++set(ACPO_AOT OFF CACHE BOOL "Whether or not ACPO AOT is enabled") ++ + if (NOT TENSORFLOW_AOT_PATH STREQUAL "") + set(LLVM_HAVE_TF_AOT "ON" CACHE BOOL "Tensorflow AOT available") + set(TENSORFLOW_AOT_COMPILER +@@ -1009,6 +1012,7 @@ if (NOT TENSORFLOW_AOT_PATH STREQUAL "") + include_directories(${TENSORFLOW_AOT_PATH}/include) + add_subdirectory(${TENSORFLOW_AOT_PATH}/xla_aot_runtime_src + ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}/tf_runtime) ++ target_compile_definitions(tf_xla_runtime_objects PUBLIC EIGEN_NEON_GEBP_NR=4) # Fix for issue https://github.com/tensorflow/tensorflow/issues/58481 + install(TARGETS tf_xla_runtime EXPORT LLVMExports + ARCHIVE DESTINATION lib${LLVM_LIBDIR_SUFFIX} COMPONENT tf_xla_runtime) + set_property(GLOBAL APPEND PROPERTY LLVM_EXPORTS tf_xla_runtime) +diff --git a/llvm/include/llvm/Analysis/ACPOCollectFeatures.h b/llvm/include/llvm/Analysis/ACPOCollectFeatures.h +new file mode 100644 +index 000000000000..ec62b559542d +--- /dev/null ++++ b/llvm/include/llvm/Analysis/ACPOCollectFeatures.h +@@ -0,0 +1,296 @@ ++//===- ACPOCollectFeatures.h - ACPO Class for Feature Collection ----------===// ++// ++// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. ++// See https://llvm.org/LICENSE.txt for license information. ++// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ++// ++//===----------------------------------------------------------------------===// ++// ++// This header file defines the type, scope, and number of features to be ++// collected on a given ACPOModel class from all available features. ++// ++//===----------------------------------------------------------------------===// ++#ifndef LLVM_ANALYSIS_ACPOCOLLECTFEATURES_H ++#define LLVM_ANALYSIS_ACPOCOLLECTFEATURES_H ++#include "llvm/Analysis/InlineAdvisor.h" ++#include "llvm/Analysis/LoopInfo.h" ++#include "llvm/IR/Function.h" ++#include "llvm/IR/Instructions.h" ++#include "llvm/IR/PassManager.h" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++namespace llvm { ++class ACPOCollectFeatures { ++public: ++ // A feature is related to one of the following scope ++ enum class Scope { ++ Module, ++ Function, ++ Loop, ++ Callgraph, ++ CallSite, ++ NumOfScope, ++ }; ++ ++ // In the future as more features are added, features can be calculated ++ // simultanously. ++ // Suppose feature A and B could be calculated in the same loop, ++ // then it would make sense to calculate both the features at the same time ++ // and save it in a cache system ++ // (which could be implemented similarly like Dumpfeatures.h/cpp). ++ enum class GroupID { ++ EdgeNodeCount, ++ FPIRelated, ++ HotColdCallSite, ++ InlineCostFeatureGroup, ++ ACPOFIExtendedFeatures, ++ NumOfGroupID ++ }; ++ ++ // List of features we support to be calculated. ++ // (1) For each feature there should be a corresponding scope on which it ++ // depends on for calculating. ++ // (2) A feature may belong in a group for which those features could be ++ // calculated together. ++ // (3) Once you decided to add a feature you should register it to all the ++ // static maps in the .cpp file. Except for some special indicator enum's ++ // like InlineCostFeatureGroupBegin/End ++ enum class FeatureIndex { ++ // Begin: InlineCostFeatureGroup ++ InlineCostFeatureGroupBegin, ++ SROASavings, ++ SROALosses, ++ LoadElimination, ++ CallPenalty, ++ CallArgumentSetup, ++ LoadRelativeIntrinsic, ++ LoweredCallArgSetup, ++ IndirectCallPenalty, ++ JumpTablePenalty, ++ CaseClusterPenalty, ++ SwitchPenalty, ++ UnsimplifiedCommonInstructions, ++ NumLoops, ++ DeadBlocks, ++ SimplifiedInstructions, ++ ConstantArgs, ++ ConstantOffsetPtrArgs, ++ CallSiteCost, ++ ColdCcPenalty, ++ LastCallToStaticBonus, ++ IsMultipleBlocks, ++ NestedInlines, ++ NestedInlineCostEstimate, ++ Threshold, ++ InlineCostFeatureGroupEnd, ++ // End: InlineCostFeatureGroup ++ ++ // Begin: FPIRelated ++ BasicBlockCount, ++ BlocksReachedFromConditionalInstruction, ++ Uses, ++ // End: FPIRelated ++ ++ // Begin: EdgeNodeCount ++ EdgeCount, ++ NodeCount, ++ // End: EdgeNodeCount ++ ++ // Begin: HotColdCallsite ++ ColdCallSite, ++ HotCallSite, ++ // End: HotColdCallsite ++ ++ // Begin: ACPOFIExtendedFeatures ++ ACPOFIExtendedFeaturesNamedFeatureBegin, ++ ACPOFIExtendedFeaturesInitialSize, ++ ACPOFIExtendedFeaturesBlocks, ++ ACPOFIExtendedFeaturesCalls, ++ ACPOFIExtendedFeaturesIsLocal, ++ ACPOFIExtendedFeaturesIsLinkOnceODR, ++ ACPOFIExtendedFeaturesIsLinkOnce, ++ ACPOFIExtendedFeaturesLoops, ++ ACPOFIExtendedFeaturesMaxLoopDepth, ++ ACPOFIExtendedFeaturesMaxDomTreeLevel, ++ ACPOFIExtendedFeaturesPtrArgs, ++ ACPOFIExtendedFeaturesPtrCallee, ++ ACPOFIExtendedFeaturesCallReturnPtr, ++ ACPOFIExtendedFeaturesConditionalBranch, ++ ACPOFIExtendedFeaturesCBwithArg, ++ ACPOFIExtendedFeaturesCallerHeight, ++ ACPOFIExtendedFeaturesCallUsage, ++ ACPOFIExtendedFeaturesIsRecursive, ++ ACPOFIExtendedFeaturesNumCallsiteInLoop, ++ ACPOFIExtendedFeaturesNumOfCallUsesInLoop, ++ ACPOFIExtendedFeaturesEntryBlockFreq, ++ ACPOFIExtendedFeaturesMaxCallsiteBlockFreq, ++ ACPOFIExtendedFeaturesNamedFeatureEnd, ++ ACPOFIExtendedFeaturesFloatFeatureBegin, ++ ACPOFIExtendedFeaturesInstructionPerBlock, ++ ACPOFIExtendedFeaturesSuccessorPerBlock, ++ ACPOFIExtendedFeaturesAvgVecInstr, ++ ACPOFIExtendedFeaturesAvgNestedLoopLevel, ++ ACPOFIExtendedFeaturesInstrPerLoop, ++ ACPOFIExtendedFeaturesBlockWithMultipleSuccecorsPerLoop, ++ ACPOFIExtendedFeaturesFloatFeatureEnd, ++ // End: ACPOFIExtendedFeatures ++ ++ CallerBlockFreq, ++ CallSiteHeight, ++ ConstantParam, ++ CostEstimate, ++ LoopLevel, ++ MandatoryKind, ++ MandatoryOnly, ++ OptCode, ++ IsIndirectCall, ++ IsInInnerLoop, ++ IsMustTailCall, ++ IsTailCall, ++ NumOfFeatures ++ }; ++ ++ struct AnalysisManagers { ++ FunctionAnalysisManager *FAM = nullptr; ++ ModuleAnalysisManager *MAM = nullptr; ++ }; ++ ++ // ScopeInfo is a struct that contains the correpsonding needed information to ++ // calculate the corresponding feature. ++ struct ScopeInfo { ++ Function *F = nullptr; ++ CallBase *CB = nullptr; ++ BasicBlock *BB = nullptr; ++ Module *M = nullptr; ++ Loop *L = nullptr; ++ // Can add Instructions or other types later. ++ }; ++ ++ struct OtherInfo { ++ bool MandatoryOnly = false; ++ InlineAdvisor *IA = nullptr; ++ }; ++ ++ // FeatureInfo should contain all the relevant information to calculate ++ // the corresponding FeatureIndex. ++ struct FeatureInfo { ++ // When Idx = NumOfFeatures. We assume this is a global FeatureInfo. ++ FeatureIndex Idx; ++ // Once we have the Idx we should know the following two attribute. ++ // Scope ScopeIdx // ++ // GroupID Group // ++ AnalysisManagers Managers; ++ ScopeInfo SI; ++ OtherInfo OI; ++ }; ++ ++ using FeatureValueMap = std::unordered_map; ++ using FeatureInfoMap = std::unordered_map; ++ using FeaturesInfo = std::vector; ++ using Scopes = std::vector; ++ using GroupIDs = std::vector; ++ typedef void (*CalculateFeatureFunction)(ACPOCollectFeatures &, ++ const FeatureInfo &); ++ ++ // Constructors/Destructors ++ ACPOCollectFeatures(); ++ ACPOCollectFeatures(FeatureInfo GlobalInfo); ++ ~ACPOCollectFeatures(); ++ ++ // Setters/getters ++ void setFeatureValue(FeatureIndex Idx, std::string Val); ++ ++ void setFeatureInfo(FeatureIndex Idx, FeatureInfo Info); ++ ++ void setFeatureValueAndInfo(FeatureIndex Idx, FeatureInfo Info, ++ std::string Val); ++ ++ void setGlobalFeatureInfo(FeatureInfo &Info); ++ ++ std::string getFeature(FeatureIndex Idx) const; ++ ++ // Check if the feature is alrady calculated. ++ bool containsFeature(FeatureIndex); ++ bool containsFeature(GroupID); ++ ++ static std::string getFeatureName(FeatureIndex Idx); ++ static GroupID getFeatureGroup(FeatureIndex Idx); ++ static Scope getFeatureScope(FeatureIndex Idx); ++ static std::set getGroupFeatures(GroupID Group); ++ static std::set getScopeFeatures(Scope S); ++ ++ void clearFeatureValueMap(); ++ bool registeredFeature(FeatureIndex Idx) const; ++ ++ // Calculate and Return the feature values specified by FeaturesInfo ++ FeatureValueMap getFeaturesPair(FeaturesInfo Features); ++ ++ // Calculate and Return the feature values specified from [Beg, End) ++ // TODO: Make a similar method for Scopes and GroupIDs ++ FeatureValueMap getFeaturesPair(FeatureIndex Beg, FeatureIndex End); ++ ++ // Calculate and Return the feature values specified by Scope. ++ FeatureValueMap getFeaturesPair(Scopes); ++ ++ // Calculate and Return the feature values specified by GroupID. ++ FeatureValueMap getFeaturesPair(GroupIDs); ++ ++ static InlineAdvisor::MandatoryInliningKind ++ getMandatoryKind(CallBase &CB, FunctionAnalysisManager &FAM, ++ OptimizationRemarkEmitter &ORE); ++ ++ static void clearFunctionLevel(); ++ static void insertFunctionLevel(const Function *, unsigned); ++ static std::optional getFunctionLevel(const Function *); ++ ++private: ++ // Global mappings. ++ // FeatureIndexToName and FeatureIndexToScope should be a one to one mapping. ++ static const std::unordered_map FeatureIndexToName; ++ static const std::unordered_map FeatureIndexToScope; ++ static const std::unordered_map FeatureIndexToGroup; ++ static const std::multimap GroupToFeatureIndices; ++ static const std::multimap ScopeToFeatureIndices; ++ // The CalculateFeatureMap maps each feature to a corresponding function that ++ // calculates the feature and also sets the feature value inside ++ // FeatureValues field. ++ static const std::unordered_map ++ CalculateFeatureMap; ++ ++ // TODO: ++ // Implement the cache systems here. See similar example in DumpFeature.cpp ++ // Notice we've only cached the FunctionLevels. ++ // But in the future this should be generalized for all features. ++ // One way to do this is to define a map from FeatureIndex -> Mapping. ++ // Inside this mapping, the key should be the Scope and a set of analysis it ++ // depends on. ++ ++ static std::map FunctionLevels; ++ ++ // Saved FeatureValues when we collect the features. ++ FeatureValueMap FeatureToValue; ++ FeatureInfoMap FeatureToInfo; ++ FeatureInfo GlobalFeatureInfo; ++}; ++ ++ACPOCollectFeatures::FeatureIndex operator+(ACPOCollectFeatures::FeatureIndex, ++ int); ++ACPOCollectFeatures::FeatureIndex operator-(ACPOCollectFeatures::FeatureIndex, ++ int); ++ACPOCollectFeatures::FeatureIndex & ++operator++(ACPOCollectFeatures::FeatureIndex &); ++ACPOCollectFeatures::FeatureIndex ++operator++(ACPOCollectFeatures::FeatureIndex &, int); ++ ++} // namespace llvm ++#endif // LLVM_ANALYSIS_ACPOCOLLECTFEATURES_H +diff --git a/llvm/include/llvm/Analysis/ACPOMLInterface.h b/llvm/include/llvm/Analysis/ACPOMLInterface.h +new file mode 100644 +index 000000000000..996f27ee32ba +--- /dev/null ++++ b/llvm/include/llvm/Analysis/ACPOMLInterface.h +@@ -0,0 +1,482 @@ ++//===- ACPOMLInterface.h - AI-Enabled Continuous Program Optimization -----===// ++// ++// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. ++// See https://llvm.org/LICENSE.txt for license information. ++// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ++// ++// Copyright (C) 2021-2022. Huawei Technologies Co., Ltd. All rights reserved. ++// ++//===----------------------------------------------------------------------===// ++ ++#ifndef LLVM_ANALYSIS_ACPOML_INTERFACE_H ++#define LLVM_ANALYSIS_ACPOML_INTERFACE_H ++ ++#include "llvm/Analysis/ACPOModelRunner.h" ++#include "llvm/IR/Constants.h" ++#include "llvm/IR/LLVMContext.h" ++#include "llvm/Support/Program.h" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++namespace llvm { ++ ++class ACPOModelRunner; ++ ++// This is class for storing information about a model. ++class Model { ++public: ++ // Constructors ++ Model() : NumFeatures{1}, NumOutputs{1} {} ++ Model(std::size_t NumFeatures) : NumFeatures{NumFeatures}, NumOutputs{1} {} ++ Model(std::size_t NumFeatures, int NumOutputs) ++ : NumFeatures{NumFeatures}, NumOutputs{NumOutputs} {} ++ ++ // Getters/Setters ++ std::size_t getNumFeatures() const { return NumFeatures; } ++ void setNumFeatures(int NumFeatures) { this->NumFeatures = NumFeatures; } ++ ++ int getNumOutputs() const { return NumOutputs; } ++ void setNumOutputs(int NumOutputs) { this->NumOutputs = NumOutputs; } ++ ++ std::string getSignature() const { return Signature; } ++ void setSignature(std::string Signature) { this->Signature = Signature; } ++ ++ // Register a feature into the NametoID and IDToIndex maps. ++ bool registerFeature(std::string FeatureName, uint64_t FeatureID, int Index); ++ ++ // Register an input into the map. ++ bool registerInput(std::string InputName, std::string InputType); ++ ++ // Register an output into the map. ++ bool registerOutput(std::string OutputName, std::string OutputType); ++ ++ // Return the index of a feature within the feature list used by inference(). ++ int getIndex(uint64_t FeatureID) const; ++ int getIndex(std::string FeatureName) const; ++ ++ // Return the name of a feature within the feature list used by inference(). ++ std::string getName(uint64_t FeatureID) const; ++ ++ // Return true if output name exists. ++ bool checkOutputExists(std::string OutputName) const; ++ ++ // Return the type of an input given its name. ++ std::string getInputType(std::string OutputName) const; ++ ++ // Return the type of an output given its name. ++ std::string getOutputType(std::string OutputName) const; ++ ++private: ++ std::size_t NumFeatures; ++ int NumOutputs; ++ std::string Signature; ++ std::unordered_map NameToID; ++ std::unordered_map IDToName; ++ std::unordered_map IDToIndex; ++ ++ // A map from input name to input type ++ std::unordered_map InputMap; ++ ++ // A map from output name to output type ++ std::unordered_map OutputMap; ++}; ++ ++// This is the base class to define an interface with an ML framework. ++class ACPOMLInterface { ++public: ++ // Constructor/Destructor. ++ ACPOMLInterface() {} ++ virtual ~ACPOMLInterface() {} ++ ++ // Getters/Setters ++ bool isInitialized() const { return Initialized; } ++ void setInitialized(bool Val) { Initialized = Val; } ++ ++ // Interface methods. ++ // Return the next available ID for a feature. ++ virtual uint64_t assignID() = 0; ++ ++ // Load a model by reading from the specified file. ++ // Return false if the operation failed. ++ virtual bool loadModel(std::string ModelSpecFile) = 0; ++ ++ // Insert a new model into the model map. ++ virtual bool registerModel(std::string ModelName, int NumFeatures) = 0; ++ virtual bool registerModel(std::string ModelName, int NumFeatures, ++ int NumOutputs) = 0; ++ ++ // Register a new feature for a given model. ++ virtual bool registerFeature(std::string ModelName, std::string FeatureName, ++ int Index) = 0; ++ ++ // Register a new output for a given model. ++ virtual bool registerOutput(std::string ModelName, std::string OutputName, ++ std::string OutputType) = 0; ++ ++ // Specify how many models are currently live in ML framework memory. ++ virtual int getNumLoadedModels() = 0; ++ ++ // Specify the input file to use as IR to be passed to the model (however ++ // it is processed afterwards). Return false if the operation failed. ++ virtual bool defineInputIR(std::string Filename) = 0; ++ ++ // Specify a custom feature for a model to use as input at the next model ++ // invocation. Return false if the operation failed. ++ virtual bool setCustomFeature(std::string ModelName, uint64_t FeatureID, ++ int FeatureValue) = 0; ++ virtual bool setCustomFeature(std::string ModelName, uint64_t FeatureID, ++ int64_t FeatureValue) = 0; ++ virtual bool setCustomFeature(std::string ModelName, uint64_t FeatureID, ++ double FeatureValue) = 0; ++ virtual bool setCustomFeature(std::string ModelName, uint64_t FeatureID, ++ float FeatureValue) = 0; ++ virtual bool setCustomFeature(std::string ModelName, uint64_t FeatureID, ++ bool FeatureValue) = 0; ++ virtual bool setCustomFeature(std::string ModelName, std::string FeatureName, ++ int FeatureValue) = 0; ++ virtual bool setCustomFeature(std::string ModelName, std::string FeatureName, ++ int64_t FeatureValue) = 0; ++ virtual bool setCustomFeature(std::string ModelName, std::string FeatureName, ++ double FeatureValue) = 0; ++ virtual bool setCustomFeature(std::string ModelName, std::string FeatureName, ++ float FeatureValue) = 0; ++ virtual bool setCustomFeature(std::string ModelName, std::string FeatureName, ++ bool FeatureValue) = 0; ++ ++ // Replace all features with the given feature values. ++ // Activate the specified model. ++ virtual bool initializeFeatures( ++ std::string ModelName, ++ const std::vector> &FeatureValues) = 0; ++ ++ virtual bool ++ initializeFeatures(std::string ModelName, ++ const std::vector> ++ &FeatureValues) = 0; ++ ++ // Set features with the specified feature values. ++ // Not changing the currently active model. ++ virtual bool setCustomFeatures( ++ std::string ModelName, ++ const std::vector> &FeatureValues) = 0; ++ ++ virtual bool ++ setCustomFeatures(std::string ModelName, ++ const std::vector> ++ &FeatureValues) = 0; ++ ++ // Run a model with specified name. Return false if the execution was not ++ // possible or an error was encountered. ++ virtual bool runModel(std::string ModelName) = 0; ++ ++ // Return the type of an output within the model specified by the name. ++ virtual std::string getOutputType(std::string ModelName, ++ std::string OutputName) = 0; ++ ++ // Return model results, based on the output name. ++ virtual int getModelResultI(std::string OutputName) = 0; ++ virtual int64_t getModelResultI64(std::string OutputName) = 0; ++ virtual float getModelResultF(std::string OutputName) = 0; ++ virtual double getModelResultD(std::string OutputName) = 0; ++ virtual bool getModelResultB(std::string OutputName) = 0; ++ ++ // Get status of the ML interface. Return zero if succeeded. ++ virtual int getStatus() = 0; ++ ++ // Free up memory taken by a model. ++ virtual bool releaseModel(std::string ModelName) = 0; ++ ++ // Close interface when done. Return false if the command was not successful. ++ // In some cases this just requires a constructor for this class to be called, ++ // but in others, additional steps that require feedback may be needed. ++ virtual bool closeMLInterface() = 0; ++ ++ // Set a flag to invoke closeMLInterface when the instance of the class is ++ // destroyed. ++ void setCloseOnDestruction() { CloseOnDestruction = true; } ++ ++protected: ++ bool CloseOnDestruction = false; ++ ++private: ++ bool Initialized = false; ++}; ++ ++class ACPOMLPythonInterface : public ACPOMLInterface { ++public: ++ ACPOMLPythonInterface(); ++ virtual ~ACPOMLPythonInterface(); ++ ++ // Interface methods. ++ // Return the next available ID for a feature. ++ virtual uint64_t assignID() override; ++ ++ // Load a model by reading from the specified file. ++ // Return false if the operation failed. ++ virtual bool loadModel(std::string ModelSpecFile) override; ++ ++ // Insert a new model into the model map. ++ virtual bool registerModel(std::string ModelName, int NumFeatures) override; ++ virtual bool registerModel(std::string ModelName, int NumFeatures, ++ int NumOutputs) override; ++ ++ // Register a new feature for a given model. ++ virtual bool registerFeature(std::string ModelName, std::string FeatureName, ++ int Index) override; ++ ++ // Register a new output for a given model. ++ virtual bool registerOutput(std::string ModelName, std::string OutputName, ++ std::string OutputType) override; ++ ++ // Specify how many models are currently live in ML framework memory. ++ virtual int getNumLoadedModels() override; ++ ++ // Specify the input file to use as IR to be passed to the model (however ++ // it is processed afterwards). Return false if the operation failed. ++ virtual bool defineInputIR(std::string Filename) override; ++ ++ // Specify a custom feature for a model to use as input at the next model ++ // invocation. Return false if the operation failed. ++ virtual bool setCustomFeature(std::string ModelName, uint64_t FeatureID, ++ int FeatureValue) override; ++ virtual bool setCustomFeature(std::string ModelName, uint64_t FeatureID, ++ int64_t FeatureValue) override; ++ virtual bool setCustomFeature(std::string ModelName, uint64_t FeatureID, ++ double FeatureValue) override; ++ virtual bool setCustomFeature(std::string ModelName, uint64_t FeatureID, ++ float FeatureValue) override; ++ virtual bool setCustomFeature(std::string ModelName, uint64_t FeatureID, ++ bool FeatureValue) override; ++ ++ virtual bool setCustomFeature(std::string ModelName, std::string FeatureName, ++ int FeatureValue) override; ++ virtual bool setCustomFeature(std::string ModelName, std::string FeatureName, ++ int64_t FeatureValue) override; ++ virtual bool setCustomFeature(std::string ModelName, std::string FeatureName, ++ double FeatureValue) override; ++ virtual bool setCustomFeature(std::string ModelName, std::string FeatureName, ++ float FeatureValue) override; ++ virtual bool setCustomFeature(std::string ModelName, std::string FeatureName, ++ bool FeatureValue) override; ++ ++ // Replace all features with the given feature values. ++ // Activate the specified model. ++ virtual bool ++ initializeFeatures(std::string ModelName, ++ const std::vector> ++ &FeatureValues) override; ++ ++ virtual bool ++ initializeFeatures(std::string ModelName, ++ const std::vector> ++ &FeatureValues) override; ++ ++ // Set features with the specified feature values. ++ // Not changing the currently active model. ++ virtual bool ++ setCustomFeatures(std::string ModelName, ++ const std::vector> ++ &FeatureValues) override; ++ ++ virtual bool ++ setCustomFeatures(std::string ModelName, ++ const std::vector> ++ &FeatureValues) override; ++ ++ // Run a model with the specified name. Return false if the execution was not ++ // possible or an error was encountered. ++ virtual bool runModel(std::string ModelName) override; ++ ++ // Return the type of an output within the model specified by the name. ++ virtual std::string getOutputType(std::string ModelName, ++ std::string OutputName) override; ++ ++ // Return model results, based on the output name. ++ virtual int getModelResultI(std::string OutputName) override; ++ virtual int64_t getModelResultI64(std::string OutputName) override; ++ virtual float getModelResultF(std::string OutputName) override; ++ virtual double getModelResultD(std::string OutputName) override; ++ virtual bool getModelResultB(std::string OutputName) override; ++ ++ // Get status of the ML interface. Return zero if succeeded. ++ virtual int getStatus() override; ++ ++ // Free up memory taken by a model. ++ virtual bool releaseModel(std::string ModelName) override; ++ ++ // Close interface when done. Return false if the command was not successful. ++ // In some cases this just requires a constructor for this class to be called, ++ // but in others, additional steps that require feedback may be needed. ++ virtual bool closeMLInterface() override; ++ ++protected: ++ void sendCommand(const std::string &Command); ++ void sendCommand(const std::vector &Features); ++ std::string getResponse(); ++ std::vector tokenize(const std::string &Line); ++ ++private: ++ llvm::sys::ProcessInfo SubProcess; ++ FILE *PipeOut = nullptr; ++ FILE *PipeIn = nullptr; ++ ++ uint64_t NextID; ++ ++ std::string CurrentlyActiveModel; ++ ++ // Mapping model names to their corresponding Model ++ std::unordered_map> ModelMap; ++}; ++ ++std::shared_ptr createPersistentPythonMLIF(); ++ ++class ACPOMLCPPInterface : public ACPOMLInterface { ++public: ++ ACPOMLCPPInterface(); ++ virtual ~ACPOMLCPPInterface(); ++ ++ // Interface methods. ++ // Return the next available ID for a feature. ++ virtual uint64_t assignID() override; ++ ++ // Load a model by reading from the specified file. ++ // Return false if the operation failed. ++ // For ACPOMLCompiledInterface, loadCompiledModel() should be used instead. ++ virtual bool loadModel(std::string ModelSpecFile) override; ++ ++ // Insert a new model into the model map. ++ virtual bool registerModel(std::string ModelName, int NumFeatures) override; ++ virtual bool registerModel(std::string ModelName, int NumFeatures, ++ int NumOutputs) override; ++ ++ // Register a new feature for a given model. ++ virtual bool registerFeature(std::string ModelName, std::string FeatureName, ++ int Index) override; ++ ++ // Register a new output for a given model. ++ virtual bool registerOutput(std::string ModelName, std::string OutputName, ++ std::string OutputType) override; ++ ++ // Specify how many models are currently live in ML framework memory. ++ virtual int getNumLoadedModels() override; ++ ++ // Specify the input file to use as IR to be passed to the model (however ++ // it is processed afterwards). Return false if the operation failed. ++ virtual bool defineInputIR(std::string Filename) override; ++ ++ // Specify a custom feature for a model to use as input at the next model ++ // invocation. Return false if the operation failed. ++ virtual bool setCustomFeature(std::string ModelName, uint64_t FeatureID, ++ int FeatureValue) override; ++ virtual bool setCustomFeature(std::string ModelName, uint64_t FeatureID, ++ int64_t FeatureValue) override; ++ virtual bool setCustomFeature(std::string ModelName, uint64_t FeatureID, ++ double FeatureValue) override; ++ virtual bool setCustomFeature(std::string ModelName, uint64_t FeatureID, ++ float FeatureValue) override; ++ virtual bool setCustomFeature(std::string ModelName, uint64_t FeatureID, ++ bool FeatureValue) override; ++ ++ virtual bool setCustomFeature(std::string ModelName, std::string FeatureName, ++ int FeatureValue) override; ++ virtual bool setCustomFeature(std::string ModelName, std::string FeatureName, ++ int64_t FeatureValue) override; ++ virtual bool setCustomFeature(std::string ModelName, std::string FeatureName, ++ double FeatureValue) override; ++ virtual bool setCustomFeature(std::string ModelName, std::string FeatureName, ++ float FeatureValue) override; ++ virtual bool setCustomFeature(std::string ModelName, std::string FeatureName, ++ bool FeatureValue) override; ++ ++ // Replace all features with the given feature values. ++ // Activate the specified model. ++ virtual bool ++ initializeFeatures(std::string ModelName, ++ const std::vector> ++ &FeatureValues) override; ++ ++ virtual bool ++ initializeFeatures(std::string ModelName, ++ const std::vector> ++ &FeatureValues) override; ++ ++ // Set features with the specified feature values. ++ // Not changing the currently active model. ++ virtual bool ++ setCustomFeatures(std::string ModelName, ++ const std::vector> ++ &FeatureValues) override; ++ ++ virtual bool ++ setCustomFeatures(std::string ModelName, ++ const std::vector> ++ &FeatureValues) override; ++ ++ // Run a model with the specified name. Return false if the execution was not ++ // possible or an error was encountered. ++ virtual bool runModel(std::string ModelName) override; ++ ++ // Return the type of an input within the model specified by the name. ++ virtual std::string getInputType(std::string ModelName, ++ std::string InputName); ++ ++ // Return the type of an output within the model specified by the name. ++ virtual std::string getOutputType(std::string ModelName, ++ std::string OutputName) override; ++ ++ // Return model results, based on the output name. ++ virtual int getModelResultI(std::string OutputName) override; ++ virtual int64_t getModelResultI64(std::string OutputName) override; ++ virtual float getModelResultF(std::string OutputName) override; ++ virtual double getModelResultD(std::string OutputName) override; ++ virtual bool getModelResultB(std::string OutputName) override; ++ ++ // Get status of the ML interface. Return zero if succeeded. ++ virtual int getStatus() override; ++ ++ // Free up memory taken by a model. ++ virtual bool releaseModel(std::string ModelName) override; ++ ++ // Close interface when done. Return false if the command was not successful. ++ // In some cases this just requires a constructor for this class to be called, ++ // but in others, additional steps that require feedback may be needed. ++ virtual bool closeMLInterface() override; ++ ++private: ++ uint64_t NextID; ++ ++ std::string CurrentlyActiveModel; ++ ++ // Mapping model names to their corresponding Model ++ std::unordered_map> ModelMap; ++ ++ // Mapping model names to their corresponding Runner ++ std::unordered_map> RunnerMap; ++ ++ std::string readModelParam(std::string FilePath, std::string Param); ++ ++ void readFeatures(std::string FilePath, ++ std::vector> &Features); ++ void readOutputs(std::string FilePath, ++ std::vector> &Outputs); ++ ++ typedef std::unique_ptr (*CreateModelRunnerFunction)( ++ std::vector>, ++ StringRef); // function pointer type ++ const static std::unordered_map ++ CreateModelRunnerMap; ++}; ++ ++std::shared_ptr createPersistentCompiledMLIF(); ++ ++} // namespace llvm ++ ++#endif // LLVM_ANALYSIS_ACPOML_INTERFACE_H +diff --git a/llvm/include/llvm/Analysis/ACPOModel.h b/llvm/include/llvm/Analysis/ACPOModel.h +new file mode 100644 +index 000000000000..34dbc0fdb8bf +--- /dev/null ++++ b/llvm/include/llvm/Analysis/ACPOModel.h +@@ -0,0 +1,122 @@ ++//===- ACPOModel.h - AI-Enabled Continuous Program Optimization -----------===// ++// ++// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. ++// See https://llvm.org/LICENSE.txt for license information. ++// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ++// ++//===----------------------------------------------------------------------===// ++ ++#ifndef LLVM_ANALYSIS_ACPOMODEL_H ++#define LLVM_ANALYSIS_ACPOMODEL_H ++ ++#include "llvm/Analysis/ACPOMLInterface.h" ++#include "llvm/IR/Constants.h" ++#include "llvm/IR/Type.h" ++#include ++#include ++#include ++#include ++ ++namespace llvm { ++ ++class OptimizationRemarkEmitter; ++class LLVMContext; ++ ++class ACPOAdvice { ++public: ++ struct FieldType { ++ Type::TypeID T; ++ Constant *Val; ++ }; ++ ++ ACPOAdvice() {} ++ ACPOAdvice(std::unique_ptr &ResultFormat); ++ virtual ~ACPOAdvice() {}; ++ ++ Constant *getField(std::string name) { ++ auto Search = FieldMap.find(name); ++ if (Search == FieldMap.end()) { ++ return nullptr; ++ } ++ return Search->second.Val; ++ } ++ ++ void reserveField(std::string name, Type::TypeID &ID) { ++ FieldMap[name].T = ID; ++ FieldMap[name].Val = nullptr; ++ } ++ ++ void addField(std::string name, Constant *Val) { ++ assert(Val != nullptr); ++ FieldMap[name].T = Val->getType()->getTypeID(); ++ FieldMap[name].Val = Val; ++ } ++ ++ std::unordered_map &getFieldMap() { ++ return FieldMap; ++ } ++ ++private: ++ std::unordered_map FieldMap; ++}; ++ ++class ACPOModel { ++public: ++ ACPOModel(OptimizationRemarkEmitter *OptReEm, bool UseML = true) : ++ ORE(OptReEm), ShouldUseML(UseML) { ++ ResultFormat = std::make_unique(); ++ assert(ResultFormat != nullptr); ++ } ++ ++ ~ACPOModel() {} ++ ++ bool isForcedToStop() const { return ForceStop; } ++ ++ // This is a interface method to return result of estimation either via an ML ++ // model or by employing a heuristic. The ML version should be implemented in ++ // the getAdviceML, which can be overwritten when necessary. The non-ML ++ // version should be implemented in getAdviceNoML and that should always be ++ // overwritten (and it will be marked as pure (=0) to force the programmer ++ // to do so). ++ std::unique_ptr getAdvice(); ++ void addRequiredResultField(std::string name, Type::TypeID &ID); ++ ++ void setContextPtr(LLVMContext *C) { Context = C; } ++ LLVMContext *getContextPtr() { return Context; } ++ ++ void setMLIF(std::shared_ptr ML) { MLIF = ML; } ++ std::shared_ptr getMLIF() { return MLIF; } ++ ++protected: ++ void addFeature(int64_t ID, Constant *Val); ++ virtual void sendCustomFeatures() {} ++ virtual void prepareModelInput(); ++ virtual bool runModel(std::unique_ptr &Result); ++ ++ virtual std::unique_ptr getAdviceML(); ++ virtual std::unique_ptr getAdviceNoML() = 0; ++ ++private: ++ // Pointer to means of feedback propagation ++ OptimizationRemarkEmitter *ORE; ++ ++ // We may need LLVMContext to set values of a Constant ++ LLVMContext *Context = nullptr; ++ ++ // Specify expected format of the ACPOAdvice result. ++ std::unique_ptr ResultFormat = nullptr; ++ ++ // Custom feature list. ++ std::unordered_map CustomFeatureMap; ++ ++ // Interface to ML framework. ++ std::shared_ptr MLIF = nullptr; ++ ++ // Specify if ML infra is in use ++ bool ShouldUseML = false; ++ bool ForceStop = false; ++}; ++ ++} // namespace llvm ++ ++#endif // LLVM_ANALYSIS_ACPOMODEL_H +diff --git a/llvm/include/llvm/Analysis/ACPOModelRunner.h b/llvm/include/llvm/Analysis/ACPOModelRunner.h +new file mode 100644 +index 000000000000..819e17f71103 +--- /dev/null ++++ b/llvm/include/llvm/Analysis/ACPOModelRunner.h +@@ -0,0 +1,39 @@ ++//===- ACPOModelRunner.h - AI-Enabled Continuous Program Optimization -----===// ++// ++// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. ++// See https://llvm.org/LICENSE.txt for license information. ++// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ++// ++//===----------------------------------------------------------------------===// ++ ++#ifndef LLVM_ANALYSIS_ACPOMODEL_H ++#define LLVM_ANALYSIS_ACPOMODEL_H ++ ++#include "llvm/Analysis/MLModelRunner.h" ++ ++namespace llvm { ++ ++class ACPOModelRunner : public MLModelRunner { ++public: ++ virtual bool setCustomFeature(int FeatureIndex, int FeatureValue) = 0; ++ virtual bool setCustomFeature(int FeatureIndex, int64_t FeatureValue) = 0; ++ virtual bool setCustomFeature(int FeatureIndex, double FeatureValue) = 0; ++ virtual bool setCustomFeature(int FeatureIndex, float FeatureValue) = 0; ++ virtual bool setCustomFeature(int FeatureIndex, bool FeatureValue) = 0; ++ ++ virtual bool runModel() = 0; ++ ++ virtual int getModelResultI(std::string OutputName) = 0; ++ virtual int64_t getModelResultI64(std::string OutputName) = 0; ++ virtual float getModelResultF(std::string OutputName) = 0; ++ virtual double getModelResultD(std::string OutputName) = 0; ++ virtual bool getModelResultB(std::string OutputName) = 0; ++ ++protected: ++ ACPOModelRunner(LLVMContext &Ctx, size_t NrInputs) ++ : MLModelRunner(Ctx, MLModelRunner::Kind::Release, NrInputs) {} ++}; ++ ++} // namespace llvm ++ ++#endif // LLVM_ANALYSIS_ACPOMODEL_H +diff --git a/llvm/include/llvm/Analysis/AOTModelRunner.h b/llvm/include/llvm/Analysis/AOTModelRunner.h +new file mode 100644 +index 000000000000..abc6258c4f09 +--- /dev/null ++++ b/llvm/include/llvm/Analysis/AOTModelRunner.h +@@ -0,0 +1,203 @@ ++//===- AOTModelRunner.h - AI-Enabled Continuous Program Optimization ------===// ++// ++// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. ++// See https://llvm.org/LICENSE.txt for license information. ++// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ++// ++//===----------------------------------------------------------------------===// ++ ++#ifndef LLVM_ANALYSIS_AOTMODEL_H ++#define LLVM_ANALYSIS_AOTMODEL_H ++ ++#include "llvm/Analysis/ACPOModelRunner.h" ++#include "llvm/Analysis/TensorSpec.h" ++ ++#define DEBUG_TYPE "acpo-aot" ++ ++namespace llvm { ++ ++template class AOTModelRunner : public ACPOModelRunner { ++public: ++ /// FeatureNames' type should be an indexed collection of std::string, like ++ /// std::array or std::vector, that has a size() method. ++ /// In the future, this method could be expanded to allow AOT models with ++ /// multiple outputs, by taking in a vector of string pairs similar to the ++ /// Features vector. ++ /// The current implementation does work for AOT models with a single output ++ /// which is a vector (or higher-dimensional tensor) of multiple values. ++ AOTModelRunner( ++ LLVMContext &Ctx, ++ const std::vector> &Features, ++ StringRef DecisionName, StringRef FeedPrefix = "feed_", ++ StringRef FetchPrefix = "fetch_") ++ : ACPOModelRunner(Ctx, Features.size()), ++ CompiledModel(std::make_unique()) { ++ assert(CompiledModel && "The CompiledModel should be valid"); ++ ++ for (size_t I = 0; I < Features.size(); ++I) { ++ const int Index = ++ CompiledModel->LookupArgIndex(FeedPrefix.str() + Features[I].first); ++ void *Buffer = nullptr; ++ if (Index >= 0) { ++ Buffer = CompiledModel->arg_data(Index); ++ } else { ++ LLVM_DEBUG(dbgs() << "Warning: AOTModelRunner was unable to find the " ++ "feature " ++ << (FeedPrefix.str() + Features[I].first) ++ << " in the compiled model\n"); ++ } ++ // The order of features passed to the model runner is important, it ++ // determines their index ++ TensorSpec Spec = makeSpec(Features[I].first, Features[I].second); ++ setUpBufferForTensor(I, Spec, Buffer); ++ } ++ ++ ResultIndex = CompiledModel->LookupResultIndex(FetchPrefix.str() + ++ DecisionName.str()); ++ assert(ResultIndex >= 0 && "Cannot find DecisionName in inlining model"); ++ } ++ ++ virtual ~AOTModelRunner() = default; ++ ++ static bool classof(const ACPOModelRunner *R) { ++ return R->getKind() == ACPOModelRunner::Kind::Release; ++ } ++ ++ bool setCustomFeature(int FeatureIndex, int FeatureValue) override { ++ LLVM_DEBUG(dbgs() << "AOTModelRunner: setting int feature " << FeatureIndex ++ << " to " << FeatureValue << "\n"); ++ *getTensor(FeatureIndex) = FeatureValue; ++ return true; ++ } ++ bool setCustomFeature(int FeatureIndex, int64_t FeatureValue) override { ++ LLVM_DEBUG(dbgs() << "AOTModelRunner: setting int64 feature " ++ << FeatureIndex << " to " << FeatureValue << "\n"); ++ *getTensor(FeatureIndex) = FeatureValue; ++ return true; ++ } ++ bool setCustomFeature(int FeatureIndex, double FeatureValue) override { ++ LLVM_DEBUG(dbgs() << "AOTModelRunner: setting double feature " ++ << FeatureIndex << " to " << FeatureValue << "\n"); ++ *getTensor(FeatureIndex) = FeatureValue; ++ return true; ++ } ++ bool setCustomFeature(int FeatureIndex, float FeatureValue) override { ++ LLVM_DEBUG(dbgs() << "AOTModelRunner: setting float feature " ++ << FeatureIndex << " to " << FeatureValue << "\n"); ++ *getTensor(FeatureIndex) = FeatureValue; ++ return true; ++ } ++ bool setCustomFeature(int FeatureIndex, bool FeatureValue) override { ++ // There are no bool tensors, so assume int for now ++ LLVM_DEBUG(dbgs() << "AOTModelRunner: setting bool feature " << FeatureIndex ++ << " to " << FeatureValue << "\n"); ++ *getTensor(FeatureIndex) = FeatureValue; ++ return true; ++ } ++ ++ bool runModel() override { ++ CompiledModel->Run(); ++ return true; ++ } ++ ++ int getModelResultI(std::string OutputName) override { ++ void *Data = CompiledModel->result_data(ResultIndex); ++ int Result = *reinterpret_cast(Data); ++ LLVM_DEBUG(dbgs() << "Returning int model result " << OutputName << " = " ++ << Result << "\n"); ++ return Result; ++ } ++ ++ int64_t getModelResultI64(std::string OutputName) override { ++ void *Data = CompiledModel->result_data(ResultIndex); ++ int64_t Result = *reinterpret_cast(Data); ++ LLVM_DEBUG(dbgs() << "Returning int64 model result " << OutputName << " = " ++ << Result << "\n"); ++ return Result; ++ } ++ ++ float getModelResultF(std::string OutputName) override { ++ void *Data = CompiledModel->result_data(ResultIndex); ++ float Result = *reinterpret_cast(Data); ++ LLVM_DEBUG(dbgs() << "Returning float model result " << OutputName << " = " ++ << Result << "\n"); ++ return Result; ++ } ++ ++ double getModelResultD(std::string OutputName) override { ++ void *Data = CompiledModel->result_data(ResultIndex); ++ double Result = *reinterpret_cast(Data); ++ LLVM_DEBUG(dbgs() << "Returning double model result " << OutputName << " = " ++ << Result << "\n"); ++ return Result; ++ } ++ ++ bool getModelResultB(std::string OutputName) override { ++ // Since there are no bool tensors, use int and return the corresponding ++ // result ++ void *Data = CompiledModel->result_data(ResultIndex); ++ bool Result = (*reinterpret_cast(Data)) > 0; ++ LLVM_DEBUG(dbgs() << "Returning bool model result " << OutputName << " = " ++ << Result << "\n"); ++ return Result; ++ } ++ ++protected: ++ std::unique_ptr CompiledModel; ++ ++private: ++ void *evaluateUntyped() override { ++ CompiledModel->Run(); ++ return CompiledModel->result_data(ResultIndex); ++ } ++ ++ llvm::TensorSpec makeSpec(std::string Name, std::string Type) { ++ std::vector Shape{}; ++ // If the string is of the form "float32[7][8]", read the value in brackets ++ // as the shape (read from left to right) ++ size_t RightBracket = 0; ++ size_t LeftBracket = 0; ++ do { ++ LeftBracket = Type.find("[", RightBracket + 1); ++ if (LeftBracket == std::string::npos) { ++ break; ++ } ++ RightBracket = Type.find("]", LeftBracket + 1); ++ size_t Value = std::stol( ++ Type.substr(LeftBracket + 1, RightBracket - LeftBracket - 1)); ++ Shape.push_back(Value); ++ } while (RightBracket != std::string::npos); ++ ++ // Remove array indices to just get type ++ if (Type.find("[") != std::string::npos) { ++ Type = Type.substr(0, Type.find("[")); ++ } ++ ++ if (Shape.size() == 0) ++ Shape.push_back(1); // Default shape is {1} ++ ++ if (Type == "int64") { ++ return TensorSpec::createSpec(Name, Shape); ++ } ++ if (Type == "int32") { ++ return TensorSpec::createSpec(Name, Shape); ++ } ++ if (Type == "int" || Type == "bool") { ++ // There are no bool tensors, so assume int for now ++ return TensorSpec::createSpec(Name, Shape); ++ } ++ if (Type == "float64") { ++ return TensorSpec::createSpec(Name, Shape); ++ } ++ if (Type == "float32") { ++ return TensorSpec::createSpec(Name, Shape); ++ } ++ assert(false && "ACPO AOT: received unknown feature type"); ++ } ++ ++ int32_t ResultIndex = -1; ++}; ++ ++} // namespace llvm ++ ++#endif // LLVM_ANALYSIS_AOTMODEL_H +diff --git a/llvm/include/llvm/Analysis/CallHeight.h b/llvm/include/llvm/Analysis/CallHeight.h +new file mode 100644 +index 000000000000..c1251081f525 +--- /dev/null ++++ b/llvm/include/llvm/Analysis/CallHeight.h +@@ -0,0 +1,72 @@ ++//===- CallHeight.h - Call height for function ------------------*- C++ -*-===// ++// ++// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. ++// See https://llvm.org/LICENSE.txt for license information. ++// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ++// ++//===----------------------------------------------------------------------===// ++// ++// This header file defines passes to get the call height of functions. ++// ++//===----------------------------------------------------------------------===// ++ ++#ifndef LLVM_ANALYSIS_CALLHEIGHT ++#define LLVM_ANALYSIS_CALLHEIGHT ++ ++#include "llvm/IR/Module.h" ++#include "llvm/IR/PassManager.h" ++#include "llvm/Pass.h" ++ ++#include ++#include ++ ++namespace llvm { ++ ++class CallHeight { ++private: ++ /// Map from function to its level (callheight) ++ std::unique_ptr> Levels; ++ ++public: ++ CallHeight(Module &M); ++ ++ // Change this to getHeight ++ unsigned getLevel(Function &F); ++ ++ bool invalidate(Module &, const PreservedAnalyses &PA, ++ ModuleAnalysisManager::Invalidator &) { ++ return false; ++ } ++}; ++ ++/// This analysis computes the mapping from function to level (callheight) ++/// for MLInliner ++class CallHeightAnalysis : public AnalysisInfoMixin { ++public: ++ static AnalysisKey Key; ++ using Result = CallHeight; ++ ++ Result run(Module &M, ModuleAnalysisManager &MAM); ++}; ++ ++/// Legacy wrapper pass to provide the CallHeightAnalysis object. ++class CallHeightAnalysisWrapper : public ModulePass { ++ std::unique_ptr Result; ++ ++public: ++ static char ID; ++ ++ CallHeightAnalysisWrapper() : ModulePass(ID) {} ++ ++ bool runOnModule(Module &M) override; ++ ++ llvm::CallHeight &getResult() { return *Result; } ++ const llvm::CallHeight &getResult() const { return *Result; } ++ void getAnalysisUsage(AnalysisUsage &AU) const override; ++}; ++ ++Pass *createCallHeightAnalysisWrapper(); ++ ++} // namespace llvm ++ ++#endif +diff --git a/llvm/include/llvm/Analysis/DumpCallsite.h b/llvm/include/llvm/Analysis/DumpCallsite.h +new file mode 100644 +index 000000000000..9f80fe1cb985 +--- /dev/null ++++ b/llvm/include/llvm/Analysis/DumpCallsite.h +@@ -0,0 +1,27 @@ ++//===- DumpCallSite.h - Dump information about a callsite -------*- C++ -*-===// ++// ++// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. ++// See https://llvm.org/LICENSE.txt for license information. ++// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ++// ++//===----------------------------------------------------------------------===// ++// ++// This header file defines the pass used to dump a callsite. ++// ++//===----------------------------------------------------------------------===// ++ ++#ifndef LLVM_ANALYSIS_DUMPCALLSITE ++#define LLVM_ANALYSIS_DUMPCALLSITE ++ ++#include "llvm/IR/PassManager.h" ++ ++namespace llvm { ++ ++class DumpCallsitePass : public PassInfoMixin { ++public: ++ PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM); ++}; ++ ++} // namespace llvm ++ ++#endif +diff --git a/llvm/include/llvm/Analysis/DumpFeature.h b/llvm/include/llvm/Analysis/DumpFeature.h +new file mode 100644 +index 000000000000..226e06cf5600 +--- /dev/null ++++ b/llvm/include/llvm/Analysis/DumpFeature.h +@@ -0,0 +1,194 @@ ++//===- DumpFeature.h - Dump features for a function -------------*- C++ -*-===// ++// ++// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. ++// See https://llvm.org/LICENSE.txt for license information. ++// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ++// ++//===----------------------------------------------------------------------===// ++// ++// This header file defines passes to dump features for functions in an scc. ++// ++//===----------------------------------------------------------------------===// ++ ++#ifndef LLVM_ANALYSIS_DUMPFEATURE ++#define LLVM_ANALYSIS_DUMPFEATURE ++ ++#include "llvm/Analysis/BlockFrequencyInfo.h" ++#include "llvm/Analysis/CGSCCPassManager.h" ++#include "llvm/Analysis/CallGraph.h" ++#include "llvm/Analysis/CallGraphSCCPass.h" ++#include "llvm/Analysis/LoopInfo.h" ++#include "llvm/Analysis/TargetTransformInfo.h" ++#include "llvm/IR/Dominators.h" ++#include "llvm/IR/PassManager.h" ++#include "llvm/Pass.h" ++ ++#include ++ ++// EnableFeatureDump - This boolean is set to true if '-enable-feature-dump' is ++// used as command line option. And we dump function features. ++extern bool EnableFeatureDump; ++ ++namespace llvm { ++ ++class DumpFeaturePass : public PassInfoMixin { ++public: ++ PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM, ++ LazyCallGraph &CG, CGSCCUpdateResult &UR); ++ ++private: ++ /// Get the caller height from cache or calculate from scratch ++ /// for a specific function F ++ int getCallHeight(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM, ++ LazyCallGraph &CG, Function *F); ++}; ++ ++class ACPOFIExtendedFeatures { ++public: ++ enum class NamedFeatureIndex : size_t { ++ InitialSize, ++ Blocks, ++ Calls, ++ IsLocal, ++ IsLinkOnceODR, ++ IsLinkOnce, ++ Loops, ++ MaxLoopDepth, ++ MaxDomTreeLevel, ++ PtrArgs, ++ PtrCallee, ++ CallReturnPtr, ++ ConditionalBranch, ++ CBwithArg, ++ CallerHeight, ++ CallUsage, ++ IsRecursive, ++ NumCallsiteInLoop, ++ NumOfCallUsesInLoop, ++ EntryBlockFreq, ++ MaxCallsiteBlockFreq, ++ NumNamedFeatures ++ }; ++ ++ enum class NamedFloatFeatureIndex : size_t { ++ InstructionPerBlock, ++ SuccessorPerBlock, ++ AvgVecInstr, ++ AvgNestedLoopLevel, ++ InstrPerLoop, ++ BlockWithMultipleSuccecorsPerLoop, ++ NumNamedFloatFeatures ++ }; ++ ++ struct FunctionFeatures { ++ static const size_t FeatureCount; ++ ++ std::array(NamedFeatureIndex::NumNamedFeatures)> ++ NamedFeatures = {{0}}; ++ std::array( ++ NamedFloatFeatureIndex::NumNamedFloatFeatures)> ++ NamedFloatFeatures = {{0}}; ++ std::vector InstructionHistogram; ++ std::vector InstructionPairHistogram; ++ ++ void fillTensor(int32_t *Ptr) const; ++ uint64_t &operator[](NamedFeatureIndex Pos) { ++ return NamedFeatures[static_cast(Pos)]; ++ } ++ float &operator[](NamedFloatFeatureIndex Pos) { ++ return NamedFloatFeatures[static_cast(Pos)]; ++ } ++ }; ++ ++ ACPOFIExtendedFeatures() = default; ++ ++ // Collect a number of features from the function F ++ static FunctionFeatures getFunctionFeatures( ++ Function &F, DominatorTree &DomTree, TargetTransformInfo &TTI, ++ LoopInfo &LI, FunctionAnalysisManager *FAM = nullptr, bool ValidSize = false, ++ bool ValidLoop = false, bool ValidTree = false); ++ ++private: ++ // Loop related features, will update FF ++ static void updateLoopRelatedFeatures(Function &F, LoopInfo &LI, ++ FunctionFeatures &FF); ++ // Instruction and BasicBlock related features, will update FF ++ static void updateInstBBRelatedFeatures(Function &F, FunctionFeatures &FF); ++ ++ // This function should mimic the behaviour of updating all features below at ++ // once: ++ // getMaxCallsiteBlockFreq ++ // updateCallsiteRelatedFeatures ++ // updateInstBBRelatedFeatures ++ static void ++ updateBBLoopCallsiteBFFeatures(Function &F, FunctionFeatures &FF, ++ LoopInfo &LI, ++ FunctionAnalysisManager *FAM = nullptr); ++}; ++ ++const std::map ++ NamedFeatureIndexToName = { ++ {ACPOFIExtendedFeatures::NamedFeatureIndex::InitialSize, "InitialSize"}, ++ {ACPOFIExtendedFeatures::NamedFeatureIndex::Blocks, "Blocks"}, ++ {ACPOFIExtendedFeatures::NamedFeatureIndex::Calls, "Calls"}, ++ {ACPOFIExtendedFeatures::NamedFeatureIndex::IsLocal, "IsLocal"}, ++ {ACPOFIExtendedFeatures::NamedFeatureIndex::IsLinkOnceODR, ++ "IsLinkOnceODR"}, ++ {ACPOFIExtendedFeatures::NamedFeatureIndex::IsLinkOnce, "IsLinkOnce"}, ++ {ACPOFIExtendedFeatures::NamedFeatureIndex::Loops, "Loops"}, ++ {ACPOFIExtendedFeatures::NamedFeatureIndex::MaxLoopDepth, ++ "MaxLoopDepth"}, ++ {ACPOFIExtendedFeatures::NamedFeatureIndex::MaxDomTreeLevel, ++ "MaxDomTreeLevel"}, ++ {ACPOFIExtendedFeatures::NamedFeatureIndex::PtrArgs, "PtrArgs"}, ++ {ACPOFIExtendedFeatures::NamedFeatureIndex::PtrCallee, "PtrCallee"}, ++ {ACPOFIExtendedFeatures::NamedFeatureIndex::CallReturnPtr, ++ "CallReturnPtr"}, ++ {ACPOFIExtendedFeatures::NamedFeatureIndex::ConditionalBranch, ++ "ConditionalBranch"}, ++ {ACPOFIExtendedFeatures::NamedFeatureIndex::CBwithArg, "CBwithArg"}, ++ {ACPOFIExtendedFeatures::NamedFeatureIndex::CallerHeight, ++ "CallerHeight"}, ++ {ACPOFIExtendedFeatures::NamedFeatureIndex::CallUsage, "CallUsage"}, ++ {ACPOFIExtendedFeatures::NamedFeatureIndex::IsRecursive, "IsRecursive"}, ++ {ACPOFIExtendedFeatures::NamedFeatureIndex::NumCallsiteInLoop, ++ "NumCallsiteInLoop"}, ++ {ACPOFIExtendedFeatures::NamedFeatureIndex::NumOfCallUsesInLoop, ++ "NumOfCallUsesInLoop"}, ++ {ACPOFIExtendedFeatures::NamedFeatureIndex::EntryBlockFreq, ++ "EntryBlockFreq"}, ++ {ACPOFIExtendedFeatures::NamedFeatureIndex::MaxCallsiteBlockFreq, ++ "MaxCallsiteBlockFreq"}}; ++ ++const std::map ++ FloatFeatureIndexToName = { ++ {ACPOFIExtendedFeatures::NamedFloatFeatureIndex::InstructionPerBlock, ++ "InstructionPerBlock"}, ++ {ACPOFIExtendedFeatures::NamedFloatFeatureIndex::SuccessorPerBlock, ++ "SuccessorPerBlock"}, ++ {ACPOFIExtendedFeatures::NamedFloatFeatureIndex::AvgVecInstr, ++ "AvgVecInstr"}, ++ {ACPOFIExtendedFeatures::NamedFloatFeatureIndex::AvgNestedLoopLevel, ++ "AvgNestedLoopLevel"}, ++ {ACPOFIExtendedFeatures::NamedFloatFeatureIndex::InstrPerLoop, ++ "InstrPerLoop"}, ++ {ACPOFIExtendedFeatures::NamedFloatFeatureIndex:: ++ BlockWithMultipleSuccecorsPerLoop, ++ "BlockWithMultipleSuccecorsPerLoop"}}; ++ ++ACPOFIExtendedFeatures::NamedFeatureIndex & ++operator++(ACPOFIExtendedFeatures::NamedFeatureIndex &n); ++ ++ACPOFIExtendedFeatures::NamedFeatureIndex ++operator++(ACPOFIExtendedFeatures::NamedFeatureIndex &n, int); ++ ++ACPOFIExtendedFeatures::NamedFloatFeatureIndex & ++operator++(ACPOFIExtendedFeatures::NamedFloatFeatureIndex &n); ++ ++ACPOFIExtendedFeatures::NamedFloatFeatureIndex ++operator++(ACPOFIExtendedFeatures::NamedFloatFeatureIndex &n, int); ++ ++} // namespace llvm ++ ++#endif +diff --git a/llvm/include/llvm/Analysis/LoopInfo.h b/llvm/include/llvm/Analysis/LoopInfo.h +index 9be3e056cf76..ea4cb7f7c684 100644 +--- a/llvm/include/llvm/Analysis/LoopInfo.h ++++ b/llvm/include/llvm/Analysis/LoopInfo.h +@@ -386,6 +386,17 @@ public: + void dump() const; + void dumpVerbose() const; + ++#if defined(ENABLE_ACPO) ++ /// Print loop IR wrapped in a dummy function ++ void printWithFunctionWrapper(raw_ostream &ROS, Function *F, ++ ArrayRef LoopBlocks, ++ BasicBlock *Header, ++ SmallVector ExitBlocks, ++ AssemblyAnnotationWriter *AAW, ++ bool ShouldPreserveUseListOrder, ++ bool IsForDebug) const; ++#endif ++ + /// Return the debug location of the start of this loop. + /// This looks for a BB terminating instruction with a known debug + /// location by looking at the preheader and header blocks. If it +diff --git a/llvm/include/llvm/Analysis/ModelDataCollector.h b/llvm/include/llvm/Analysis/ModelDataCollector.h +new file mode 100644 +index 000000000000..ad3fc476a9b2 +--- /dev/null ++++ b/llvm/include/llvm/Analysis/ModelDataCollector.h +@@ -0,0 +1,108 @@ ++//===- ModelDataCollector.h - Data collector for ML model -----------------===// ++// ++// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. ++// See https://llvm.org/LICENSE.txt for license information. ++// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ++// ++//===----------------------------------------------------------------------===// ++ ++#ifndef LLVM_ANALYSIS_MODELDATACOLLECTOR_H ++#define LLVM_ANALYSIS_MODELDATACOLLECTOR_H ++ ++#if defined(ENABLE_ACPO) ++#include "llvm/ADT/StringMap.h" ++#include "llvm/Analysis/ACPOCollectFeatures.h" ++#include "llvm/Analysis/LoopInfo.h" ++#include "llvm/Support/FormattedStream.h" ++#include "llvm/Support/raw_ostream.h" ++#include ++#include ++ ++namespace llvm { ++class ModelDataCollector { ++public: ++ enum DumpOption { function, loop, before, after }; ++ ++ ModelDataCollector(formatted_raw_ostream &OS, std::string OutputFileName = "") ++ : OutputFileName(OutputFileName), Out(OS) {} ++ ++ ~ModelDataCollector() {} ++ ++ std::string getDumpOptionAsString(DumpOption DO); ++ std::string getIRFileName(StringRef Key); ++ std::string getOutputFileName(); ++ bool isEmptyOutputFile(); ++ //std::string generateIRFileName(autotuning::CodeRegion CR); ++ std::string demangleName(const std::string &Name); ++ std::vector> getFeatures(); ++ std::unique_ptr ++ createFile(const Twine &FilePath, const Twine &FileName, std::error_code &EC); ++ StringMap getIRFileNameMap(); ++ void ++ setFeatures(std::vector> NewFeatures); ++ void setIRFileNameMap(StringMap IRFileNameMap); ++ void ++ addFeatures(std::vector> NewFeatures); ++ ++ // Print out the features ++ void printRow(bool printHeader = false); ++ ++ // Create the directory structure and store IR files in their corresponding ++ // directory ++ void writeIR(Loop *L, Function *F, std::string NewIRFileName, ++ std::string PassName, DumpOption DumpBeforeOrAfter, ++ bool PrintLoop, bool PrintFunction, ++ bool OverwriteIRFile = false); ++ ++ // Print the loop IR to a file ++ void createIRFileForLoop(Loop *L, const Twine &IRFilePath, ++ const Twine &NewIRFileName, bool OverwriteIRFile); ++ ++ // Print the function IR to a file ++ void createIRFileForFunction(Function *F, const Twine &IRFilePath, ++ const Twine &NewIRFileName, ++ bool OverwriteIRFile); ++ ++ virtual void collectFeatures(Loop *L, const std::string &ModuleName, ++ const std::string &FuncName, ++ const std::string &LoopName); ++ ++ virtual void collectFeatures(); ++ ++ // FeatureCollectInfo contains the information of registered feature. ++ struct FeatureCollectInfo { ++ std::unique_ptr FeaturesInfo; ++ std::unique_ptr RegisteredScopes; ++ std::unique_ptr RegisteredGroupIDs; ++ std::unique_ptr GlobalInfo; ++ std::unique_ptr FeatureCollector; ++ std::string Prefix; ++ std::string Postfix; ++ }; ++ ++ void registerFeature(ACPOCollectFeatures::FeaturesInfo, std::string = "", ++ std::string = ""); ++ void registerFeature(ACPOCollectFeatures::Scopes, ++ ACPOCollectFeatures::FeatureInfo, std::string = "", ++ std::string = ""); ++ void registerFeature(ACPOCollectFeatures::GroupIDs, ++ ACPOCollectFeatures::FeatureInfo, std::string = "", ++ std::string = ""); ++ void resetRegisteredFeatures(); ++ ++protected: ++ // Collected features ++ std::vector> Features; ++ // NOTE: OutputFileName being empty (null) is treated as stdout ++ std::string OutputFileName; ++ std::vector> FeatureCollectInfos; ++ ++private: ++ // Stream for dumping training data ++ formatted_raw_ostream &Out; ++ StringMap IRFileNames; ++}; ++} // namespace llvm ++ ++#endif // ENABLE_ACPO ++#endif // LLVM_ANALYSIS_MODELDATACOLLECTOR_H +diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h +index 80bec2d82e24..7fdb5db67c16 100644 +--- a/llvm/include/llvm/InitializePasses.h ++++ b/llvm/include/llvm/InitializePasses.h +@@ -100,6 +100,7 @@ void initializeDomPrinterWrapperPassPass(PassRegistry &); + void initializeDomViewerWrapperPassPass(PassRegistry &); + void initializeDominanceFrontierWrapperPassPass(PassRegistry&); + void initializeDominatorTreeWrapperPassPass(PassRegistry&); ++void initializeDumpCallsiteLegacyPass(PassRegistry &); + void initializeDwarfEHPrepareLegacyPassPass(PassRegistry &); + void initializeEarlyCSELegacyPassPass(PassRegistry&); + void initializeEarlyCSEMemSSALegacyPassPass(PassRegistry&); +@@ -124,6 +125,7 @@ void initializeFixIrreduciblePass(PassRegistry &); + void initializeFixupStatepointCallerSavedPass(PassRegistry&); + void initializeFlattenCFGLegacyPassPass(PassRegistry &); + void initializeFuncletLayoutPass(PassRegistry&); ++void initializeCallHeightAnalysisWrapperPass(PassRegistry &); + void initializeGCMachineCodeAnalysisPass(PassRegistry&); + void initializeGCModuleInfoPass(PassRegistry&); + void initializeGVNLegacyPassPass(PassRegistry&); +@@ -132,6 +134,7 @@ void initializeGlobalsAAWrapperPassPass(PassRegistry&); + void initializeGuardWideningLegacyPassPass(PassRegistry&); + void initializeHardwareLoopsLegacyPass(PassRegistry&); + void initializeMIRProfileLoaderPassPass(PassRegistry &); ++void initializeInlineAdvisorAnalysisWrapperPass(PassRegistry &); + void initializeIRSimilarityIdentifierWrapperPassPass(PassRegistry&); + void initializeIRTranslatorPass(PassRegistry&); + void initializeIVUsersWrapperPassPass(PassRegistry&); +@@ -149,6 +152,11 @@ void initializeInterleavedLoadCombinePass(PassRegistry &); + void initializeIntervalPartitionPass(PassRegistry&); + void initializeJMCInstrumenterPass(PassRegistry&); + void initializeKCFIPass(PassRegistry &); ++void initializeLegacyFAMPass(PassRegistry &); ++void initializeLegacyFunctionPropertiesAnalysisPass(PassRegistry &); ++void initializeLegacyInlinerPassPass(PassRegistry &); ++void initializeLegacyInlineSizeEstimatorAnalysisPass(PassRegistry &); ++void initializeLegacyModuleInlinerWrapperPassPass(PassRegistry &); + void initializeLCSSAVerificationPassPass(PassRegistry&); + void initializeLCSSAWrapperPassPass(PassRegistry&); + void initializeLazyBlockFrequencyInfoPassPass(PassRegistry&); +diff --git a/llvm/lib/Analysis/ACPOCollectFeatures.cpp b/llvm/lib/Analysis/ACPOCollectFeatures.cpp +new file mode 100644 +index 000000000000..f9de26483c76 +--- /dev/null ++++ b/llvm/lib/Analysis/ACPOCollectFeatures.cpp +@@ -0,0 +1,1258 @@ ++//===- ACPOCollectFeatures.cpp - ACPO Class for Feature Collection -------===// ++// ++// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. ++// See https://llvm.org/LICENSE.txt for license information. ++// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ++// ++//===----------------------------------------------------------------------===// ++// ++// This file implements ACPOCollectFeatures class ++// ++//===----------------------------------------------------------------------===// ++ ++#include "llvm/Analysis/ACPOCollectFeatures.h" ++#include "llvm/ADT/SCCIterator.h" ++// The ACPOFIModel.h currently contains only the cache system for ++// ACPOFIExtendedFeatures. ++#include "llvm/Analysis/ACPOFIModel.h" ++#include "llvm/Analysis/AssumptionCache.h" ++#include "llvm/Analysis/BlockFrequencyInfo.h" ++#include "llvm/Analysis/CallGraph.h" ++#include "llvm/Analysis/DumpFeature.h" ++#include "llvm/Analysis/FunctionPropertiesAnalysis.h" ++#include "llvm/Analysis/InlineAdvisor.h" ++#include "llvm/Analysis/InlineCost.h" ++#include "llvm/Analysis/OptimizationRemarkEmitter.h" ++#include "llvm/Analysis/TargetTransformInfo.h" ++#include "llvm/IR/Dominators.h" ++#include "llvm/IR/InstIterator.h" ++#include "llvm/IR/Instructions.h" ++#include "llvm/Support/Debug.h" ++ ++#define DEBUG_TYPE "ACPOCollectFeatures" ++ ++namespace llvm { ++ ++// Helper function that is used to calculate features and each function should ++// registered in the CalculateFeatureMap. ++static void calculateFPIRelated(ACPOCollectFeatures &ACF, ++ const ACPOCollectFeatures::FeatureInfo &info); ++static void ++calculateCallerBlockFreq(ACPOCollectFeatures &ACF, ++ const ACPOCollectFeatures::FeatureInfo &info); ++static void ++calculateCallSiteHeight(ACPOCollectFeatures &ACF, ++ const ACPOCollectFeatures::FeatureInfo &info); ++static void ++calculateConstantParam(ACPOCollectFeatures &ACF, ++ const ACPOCollectFeatures::FeatureInfo &info); ++static void calculateCostEstimate(ACPOCollectFeatures &ACF, ++ const ACPOCollectFeatures::FeatureInfo &info); ++static void ++calculateEdgeNodeCount(ACPOCollectFeatures &ACF, ++ const ACPOCollectFeatures::FeatureInfo &info); ++static void ++calculateHotColdCallSite(ACPOCollectFeatures &ACF, ++ const ACPOCollectFeatures::FeatureInfo &info); ++static void calculateLoopLevel(ACPOCollectFeatures &ACF, ++ const ACPOCollectFeatures::FeatureInfo &info); ++static void ++calculateMandatoryKind(ACPOCollectFeatures &ACF, ++ const ACPOCollectFeatures::FeatureInfo &info); ++static void ++calculateMandatoryOnly(ACPOCollectFeatures &ACF, ++ const ACPOCollectFeatures::FeatureInfo &info); ++static void ++calculateInlineCostFeatures(ACPOCollectFeatures &ACF, ++ const ACPOCollectFeatures::FeatureInfo &info); ++static void calculateACPOFIExtendedFeaturesFeatures( ++ ACPOCollectFeatures &ACF, const ACPOCollectFeatures::FeatureInfo &info); ++static void ++calculateIsIndirectCall(ACPOCollectFeatures &ACF, ++ const ACPOCollectFeatures::FeatureInfo &info); ++static void ++calculateIsInInnerLoop(ACPOCollectFeatures &ACF, ++ const ACPOCollectFeatures::FeatureInfo &info); ++static void ++calculateIsMustTailCall(ACPOCollectFeatures &ACF, ++ const ACPOCollectFeatures::FeatureInfo &info); ++static void calculateIsTailCall(ACPOCollectFeatures &ACF, ++ const ACPOCollectFeatures::FeatureInfo &info); ++static void calculateOptCode(ACPOCollectFeatures &ACF, ++ const ACPOCollectFeatures::FeatureInfo &info); ++ ++// Register FeatureIdx -> Feature name ++// FeatureIdx -> Scope, Scope -> FeatureIdx ++// FeatureIdx -> Group, Group -> FeatureIdx ++// FeatureIdx -> Calculating function ++#define REGISTER_NAME(INDEX_NAME, NAME) \ ++ { ACPOCollectFeatures::FeatureIndex::INDEX_NAME, NAME } ++const std::unordered_map ++ ACPOCollectFeatures::FeatureIndexToName{ ++ REGISTER_NAME(SROASavings, "sroa_savings"), ++ REGISTER_NAME(SROALosses, "sroa_losses"), ++ REGISTER_NAME(LoadElimination, "load_elimination"), ++ REGISTER_NAME(CallPenalty, "call_penalty"), ++ REGISTER_NAME(CallArgumentSetup, "call_argument_setup"), ++ REGISTER_NAME(LoadRelativeIntrinsic, "load_relative_intrinsic"), ++ REGISTER_NAME(LoweredCallArgSetup, "lowered_call_arg_setup"), ++ REGISTER_NAME(IndirectCallPenalty, "indirect_call_penalty"), ++ REGISTER_NAME(JumpTablePenalty, "jump_table_penalty"), ++ REGISTER_NAME(CaseClusterPenalty, "case_cluster_penalty"), ++ REGISTER_NAME(SwitchPenalty, "switch_penalty"), ++ REGISTER_NAME(UnsimplifiedCommonInstructions, ++ "unsimplified_common_instructions"), ++ REGISTER_NAME(NumLoops, "num_loops"), ++ REGISTER_NAME(DeadBlocks, "dead_blocks"), ++ REGISTER_NAME(SimplifiedInstructions, "simplified_instructions"), ++ REGISTER_NAME(ConstantArgs, "constant_args"), ++ REGISTER_NAME(ConstantOffsetPtrArgs, "constant_offset_ptr_args"), ++ REGISTER_NAME(CallSiteCost, "callsite_cost"), ++ REGISTER_NAME(ColdCcPenalty, "cold_cc_penalty"), ++ REGISTER_NAME(LastCallToStaticBonus, "last_call_to_static_bonus"), ++ REGISTER_NAME(IsMultipleBlocks, "is_multiple_blocks"), ++ REGISTER_NAME(NestedInlines, "nested_inlines"), ++ REGISTER_NAME(NestedInlineCostEstimate, "nested_inline_cost_estimate"), ++ REGISTER_NAME(Threshold, "threshold"), ++ REGISTER_NAME(BasicBlockCount, "basic_block_count"), ++ REGISTER_NAME(BlocksReachedFromConditionalInstruction, ++ "conditionally_executed_blocks"), ++ REGISTER_NAME(Uses, "users"), ++ REGISTER_NAME(EdgeCount, "edge_count"), ++ REGISTER_NAME(NodeCount, "node_count"), ++ REGISTER_NAME(ColdCallSite, "cold_callsite"), ++ REGISTER_NAME(HotCallSite, "hot_callsite"), ++ REGISTER_NAME(ACPOFIExtendedFeaturesInitialSize, "InitialSize"), ++ REGISTER_NAME(ACPOFIExtendedFeaturesBlocks, "Blocks"), ++ REGISTER_NAME(ACPOFIExtendedFeaturesCalls, "Calls"), ++ REGISTER_NAME(ACPOFIExtendedFeaturesIsLocal, "IsLocal"), ++ REGISTER_NAME(ACPOFIExtendedFeaturesIsLinkOnceODR, "IsLinkOnceODR"), ++ REGISTER_NAME(ACPOFIExtendedFeaturesIsLinkOnce, "IsLinkOnce"), ++ REGISTER_NAME(ACPOFIExtendedFeaturesLoops, "Loops"), ++ REGISTER_NAME(ACPOFIExtendedFeaturesMaxLoopDepth, "MaxLoopDepth"), ++ REGISTER_NAME(ACPOFIExtendedFeaturesMaxDomTreeLevel, "MaxDomTreeLevel"), ++ REGISTER_NAME(ACPOFIExtendedFeaturesPtrArgs, "PtrArgs"), ++ REGISTER_NAME(ACPOFIExtendedFeaturesPtrCallee, "PtrCallee"), ++ REGISTER_NAME(ACPOFIExtendedFeaturesCallReturnPtr, "CallReturnPtr"), ++ REGISTER_NAME(ACPOFIExtendedFeaturesConditionalBranch, ++ "ConditionalBranch"), ++ REGISTER_NAME(ACPOFIExtendedFeaturesCBwithArg, "CBwithArg"), ++ REGISTER_NAME(ACPOFIExtendedFeaturesCallerHeight, "CallerHeight"), ++ REGISTER_NAME(ACPOFIExtendedFeaturesCallUsage, "CallUsage"), ++ REGISTER_NAME(ACPOFIExtendedFeaturesIsRecursive, "IsRecursive"), ++ REGISTER_NAME(ACPOFIExtendedFeaturesNumCallsiteInLoop, ++ "NumCallsiteInLoop"), ++ REGISTER_NAME(ACPOFIExtendedFeaturesNumOfCallUsesInLoop, ++ "NumOfCallUsesInLoop"), ++ REGISTER_NAME(ACPOFIExtendedFeaturesEntryBlockFreq, "EntryBlockFreq"), ++ REGISTER_NAME(ACPOFIExtendedFeaturesMaxCallsiteBlockFreq, ++ "MaxCallsiteBlockFreq"), ++ REGISTER_NAME(ACPOFIExtendedFeaturesInstructionPerBlock, ++ "InstructionPerBlock"), ++ REGISTER_NAME(ACPOFIExtendedFeaturesSuccessorPerBlock, ++ "SuccessorPerBlock"), ++ REGISTER_NAME(ACPOFIExtendedFeaturesAvgVecInstr, "AvgVecInstr"), ++ REGISTER_NAME(ACPOFIExtendedFeaturesAvgNestedLoopLevel, ++ "AvgNestedLoopLevel"), ++ REGISTER_NAME(ACPOFIExtendedFeaturesInstrPerLoop, "InstrPerLoop"), ++ REGISTER_NAME(ACPOFIExtendedFeaturesBlockWithMultipleSuccecorsPerLoop, ++ "BlockWithMultipleSuccecorsPerLoop"), ++ REGISTER_NAME(CallerBlockFreq, "block_freq"), ++ REGISTER_NAME(CallSiteHeight, "callsite_height"), ++ REGISTER_NAME(ConstantParam, "nr_ctant_params"), ++ REGISTER_NAME(CostEstimate, "cost_estimate"), ++ REGISTER_NAME(LoopLevel, "loop_level"), ++ REGISTER_NAME(MandatoryKind, "mandatory_kind"), ++ REGISTER_NAME(MandatoryOnly, "mandatory_only"), ++ REGISTER_NAME(OptCode, "opt_code"), ++ REGISTER_NAME(IsIndirectCall, "is_indirect"), ++ REGISTER_NAME(IsInInnerLoop, "is_in_inner_loop"), ++ REGISTER_NAME(IsMustTailCall, "is_must_tail"), ++ REGISTER_NAME(IsTailCall, "is_tail"), ++ REGISTER_NAME(NumOfFeatures,"num_features"), ++ }; ++#undef REGISTER_NAME ++ ++#define REGISTER_SCOPE(INDEX_NAME, NAME) \ ++ { \ ++ ACPOCollectFeatures::FeatureIndex::INDEX_NAME, \ ++ ACPOCollectFeatures::Scope::NAME \ ++ } ++const std::unordered_map ++ ACPOCollectFeatures::FeatureIndexToScope{ ++ REGISTER_SCOPE(SROASavings, CallSite), ++ REGISTER_SCOPE(SROALosses, CallSite), ++ REGISTER_SCOPE(LoadElimination, CallSite), ++ REGISTER_SCOPE(CallPenalty, CallSite), ++ REGISTER_SCOPE(CallArgumentSetup, CallSite), ++ REGISTER_SCOPE(LoadRelativeIntrinsic, CallSite), ++ REGISTER_SCOPE(LoweredCallArgSetup, CallSite), ++ REGISTER_SCOPE(IndirectCallPenalty, CallSite), ++ REGISTER_SCOPE(JumpTablePenalty, CallSite), ++ REGISTER_SCOPE(CaseClusterPenalty, CallSite), ++ REGISTER_SCOPE(SwitchPenalty, CallSite), ++ REGISTER_SCOPE(UnsimplifiedCommonInstructions, CallSite), ++ REGISTER_SCOPE(NumLoops, CallSite), ++ REGISTER_SCOPE(DeadBlocks, CallSite), ++ REGISTER_SCOPE(SimplifiedInstructions, CallSite), ++ REGISTER_SCOPE(ConstantArgs, CallSite), ++ REGISTER_SCOPE(ConstantOffsetPtrArgs, CallSite), ++ REGISTER_SCOPE(CallSiteCost, CallSite), ++ REGISTER_SCOPE(ColdCcPenalty, CallSite), ++ REGISTER_SCOPE(LastCallToStaticBonus, CallSite), ++ REGISTER_SCOPE(IsMultipleBlocks, CallSite), ++ REGISTER_SCOPE(NestedInlines, CallSite), ++ REGISTER_SCOPE(NestedInlineCostEstimate, CallSite), ++ REGISTER_SCOPE(Threshold, CallSite), ++ REGISTER_SCOPE(BasicBlockCount, Function), ++ REGISTER_SCOPE(BlocksReachedFromConditionalInstruction, Function), ++ REGISTER_SCOPE(Uses, Function), ++ REGISTER_SCOPE(EdgeCount, Module), ++ REGISTER_SCOPE(NodeCount, Module), ++ REGISTER_SCOPE(ColdCallSite, CallSite), ++ REGISTER_SCOPE(HotCallSite, CallSite), ++ REGISTER_SCOPE(ACPOFIExtendedFeaturesInitialSize, Function), ++ REGISTER_SCOPE(ACPOFIExtendedFeaturesBlocks, Function), ++ REGISTER_SCOPE(ACPOFIExtendedFeaturesCalls, Function), ++ REGISTER_SCOPE(ACPOFIExtendedFeaturesIsLocal, Function), ++ REGISTER_SCOPE(ACPOFIExtendedFeaturesIsLinkOnceODR, Function), ++ REGISTER_SCOPE(ACPOFIExtendedFeaturesIsLinkOnce, Function), ++ REGISTER_SCOPE(ACPOFIExtendedFeaturesLoops, Function), ++ REGISTER_SCOPE(ACPOFIExtendedFeaturesMaxLoopDepth, Function), ++ REGISTER_SCOPE(ACPOFIExtendedFeaturesMaxDomTreeLevel, Function), ++ REGISTER_SCOPE(ACPOFIExtendedFeaturesPtrArgs, Function), ++ REGISTER_SCOPE(ACPOFIExtendedFeaturesPtrCallee, Function), ++ REGISTER_SCOPE(ACPOFIExtendedFeaturesCallReturnPtr, Function), ++ REGISTER_SCOPE(ACPOFIExtendedFeaturesConditionalBranch, Function), ++ REGISTER_SCOPE(ACPOFIExtendedFeaturesCBwithArg, Function), ++ REGISTER_SCOPE(ACPOFIExtendedFeaturesCallerHeight, Function), ++ REGISTER_SCOPE(ACPOFIExtendedFeaturesCallUsage, Function), ++ REGISTER_SCOPE(ACPOFIExtendedFeaturesIsRecursive, Function), ++ REGISTER_SCOPE(ACPOFIExtendedFeaturesNumCallsiteInLoop, Function), ++ REGISTER_SCOPE(ACPOFIExtendedFeaturesNumOfCallUsesInLoop, Function), ++ REGISTER_SCOPE(ACPOFIExtendedFeaturesEntryBlockFreq, Function), ++ REGISTER_SCOPE(ACPOFIExtendedFeaturesMaxCallsiteBlockFreq, Function), ++ REGISTER_SCOPE(ACPOFIExtendedFeaturesInstructionPerBlock, Function), ++ REGISTER_SCOPE(ACPOFIExtendedFeaturesSuccessorPerBlock, Function), ++ REGISTER_SCOPE(ACPOFIExtendedFeaturesAvgVecInstr, Function), ++ REGISTER_SCOPE(ACPOFIExtendedFeaturesAvgNestedLoopLevel, Function), ++ REGISTER_SCOPE(ACPOFIExtendedFeaturesInstrPerLoop, Function), ++ REGISTER_SCOPE(ACPOFIExtendedFeaturesBlockWithMultipleSuccecorsPerLoop, ++ Function), ++ REGISTER_SCOPE(CallerBlockFreq, CallSite), ++ REGISTER_SCOPE(CallSiteHeight, CallSite), ++ REGISTER_SCOPE(ConstantParam, CallSite), ++ REGISTER_SCOPE(CostEstimate, CallSite), ++ REGISTER_SCOPE(LoopLevel, CallSite), ++ REGISTER_SCOPE(MandatoryKind, CallSite), ++ REGISTER_SCOPE(MandatoryOnly, CallSite), ++ REGISTER_SCOPE(OptCode, CallSite), ++ REGISTER_SCOPE(IsIndirectCall, CallSite), ++ REGISTER_SCOPE(IsInInnerLoop, CallSite), ++ REGISTER_SCOPE(IsMustTailCall, CallSite), ++ REGISTER_SCOPE(IsTailCall, CallSite), ++ }; ++#undef REGISTER_SCOPE ++ ++#define REGISTER_GROUP(INDEX_NAME, NAME) \ ++ { \ ++ ACPOCollectFeatures::FeatureIndex::INDEX_NAME, \ ++ ACPOCollectFeatures::GroupID::NAME \ ++ } ++const std::unordered_map ++ ACPOCollectFeatures::FeatureIndexToGroup{ ++ REGISTER_GROUP(SROASavings, InlineCostFeatureGroup), ++ REGISTER_GROUP(SROALosses, InlineCostFeatureGroup), ++ REGISTER_GROUP(LoadElimination, InlineCostFeatureGroup), ++ REGISTER_GROUP(CallPenalty, InlineCostFeatureGroup), ++ REGISTER_GROUP(CallArgumentSetup, InlineCostFeatureGroup), ++ REGISTER_GROUP(LoadRelativeIntrinsic, InlineCostFeatureGroup), ++ REGISTER_GROUP(LoweredCallArgSetup, InlineCostFeatureGroup), ++ REGISTER_GROUP(IndirectCallPenalty, InlineCostFeatureGroup), ++ REGISTER_GROUP(JumpTablePenalty, InlineCostFeatureGroup), ++ REGISTER_GROUP(CaseClusterPenalty, InlineCostFeatureGroup), ++ REGISTER_GROUP(SwitchPenalty, InlineCostFeatureGroup), ++ REGISTER_GROUP(UnsimplifiedCommonInstructions, InlineCostFeatureGroup), ++ REGISTER_GROUP(NumLoops, InlineCostFeatureGroup), ++ REGISTER_GROUP(DeadBlocks, InlineCostFeatureGroup), ++ REGISTER_GROUP(SimplifiedInstructions, InlineCostFeatureGroup), ++ REGISTER_GROUP(ConstantArgs, InlineCostFeatureGroup), ++ REGISTER_GROUP(ConstantOffsetPtrArgs, InlineCostFeatureGroup), ++ REGISTER_GROUP(CallSiteCost, InlineCostFeatureGroup), ++ REGISTER_GROUP(ColdCcPenalty, InlineCostFeatureGroup), ++ REGISTER_GROUP(LastCallToStaticBonus, InlineCostFeatureGroup), ++ REGISTER_GROUP(IsMultipleBlocks, InlineCostFeatureGroup), ++ REGISTER_GROUP(NestedInlines, InlineCostFeatureGroup), ++ REGISTER_GROUP(NestedInlineCostEstimate, InlineCostFeatureGroup), ++ REGISTER_GROUP(Threshold, InlineCostFeatureGroup), ++ REGISTER_GROUP(BasicBlockCount, FPIRelated), ++ REGISTER_GROUP(BlocksReachedFromConditionalInstruction, FPIRelated), ++ REGISTER_GROUP(Uses, FPIRelated), ++ REGISTER_GROUP(EdgeCount, EdgeNodeCount), ++ REGISTER_GROUP(NodeCount, EdgeNodeCount), ++ REGISTER_GROUP(ColdCallSite, HotColdCallSite), ++ REGISTER_GROUP(HotCallSite, HotColdCallSite), ++ REGISTER_GROUP(ACPOFIExtendedFeaturesInitialSize, ++ ACPOFIExtendedFeatures), ++ REGISTER_GROUP(ACPOFIExtendedFeaturesBlocks, ACPOFIExtendedFeatures), ++ REGISTER_GROUP(ACPOFIExtendedFeaturesCalls, ACPOFIExtendedFeatures), ++ REGISTER_GROUP(ACPOFIExtendedFeaturesIsLocal, ACPOFIExtendedFeatures), ++ REGISTER_GROUP(ACPOFIExtendedFeaturesIsLinkOnceODR, ++ ACPOFIExtendedFeatures), ++ REGISTER_GROUP(ACPOFIExtendedFeaturesIsLinkOnce, ++ ACPOFIExtendedFeatures), ++ REGISTER_GROUP(ACPOFIExtendedFeaturesLoops, ACPOFIExtendedFeatures), ++ REGISTER_GROUP(ACPOFIExtendedFeaturesMaxLoopDepth, ++ ACPOFIExtendedFeatures), ++ REGISTER_GROUP(ACPOFIExtendedFeaturesMaxDomTreeLevel, ++ ACPOFIExtendedFeatures), ++ REGISTER_GROUP(ACPOFIExtendedFeaturesPtrArgs, ACPOFIExtendedFeatures), ++ REGISTER_GROUP(ACPOFIExtendedFeaturesPtrCallee, ACPOFIExtendedFeatures), ++ REGISTER_GROUP(ACPOFIExtendedFeaturesCallReturnPtr, ++ ACPOFIExtendedFeatures), ++ REGISTER_GROUP(ACPOFIExtendedFeaturesConditionalBranch, ++ ACPOFIExtendedFeatures), ++ REGISTER_GROUP(ACPOFIExtendedFeaturesCBwithArg, ACPOFIExtendedFeatures), ++ REGISTER_GROUP(ACPOFIExtendedFeaturesCallerHeight, ++ ACPOFIExtendedFeatures), ++ REGISTER_GROUP(ACPOFIExtendedFeaturesCallUsage, ACPOFIExtendedFeatures), ++ REGISTER_GROUP(ACPOFIExtendedFeaturesIsRecursive, ++ ACPOFIExtendedFeatures), ++ REGISTER_GROUP(ACPOFIExtendedFeaturesNumCallsiteInLoop, ++ ACPOFIExtendedFeatures), ++ REGISTER_GROUP(ACPOFIExtendedFeaturesNumOfCallUsesInLoop, ++ ACPOFIExtendedFeatures), ++ REGISTER_GROUP(ACPOFIExtendedFeaturesEntryBlockFreq, ++ ACPOFIExtendedFeatures), ++ REGISTER_GROUP(ACPOFIExtendedFeaturesMaxCallsiteBlockFreq, ++ ACPOFIExtendedFeatures), ++ REGISTER_GROUP(ACPOFIExtendedFeaturesInstructionPerBlock, ++ ACPOFIExtendedFeatures), ++ REGISTER_GROUP(ACPOFIExtendedFeaturesSuccessorPerBlock, ++ ACPOFIExtendedFeatures), ++ REGISTER_GROUP(ACPOFIExtendedFeaturesAvgVecInstr, ++ ACPOFIExtendedFeatures), ++ REGISTER_GROUP(ACPOFIExtendedFeaturesAvgNestedLoopLevel, ++ ACPOFIExtendedFeatures), ++ REGISTER_GROUP(ACPOFIExtendedFeaturesInstrPerLoop, ++ ACPOFIExtendedFeatures), ++ REGISTER_GROUP(ACPOFIExtendedFeaturesBlockWithMultipleSuccecorsPerLoop, ++ ACPOFIExtendedFeatures), ++ }; ++#undef REGISTER_GROUP ++ ++// Given a map that may not be one to one. Returns the inverse mapping. ++// EX: Input: A -> 1, B -> 1 ++// Output: 1 -> A, 1 -> B ++template ++static std::multimap inverseMap(std::unordered_map Map) { ++ std::multimap InverseMap; ++ for (const auto &It : Map) { ++ InverseMap.insert(std::pair(It.second, It.first)); ++ } ++ return InverseMap; ++} ++ ++const std::multimap ++ ACPOCollectFeatures::GroupToFeatureIndices{ ++ inverseMap(FeatureIndexToGroup)}; ++ ++const std::multimap ++ ACPOCollectFeatures::ScopeToFeatureIndices{ ++ inverseMap(FeatureIndexToScope)}; ++ ++#define REGISTER_FUNCTION(INDEX_NAME, NAME) \ ++ { ACPOCollectFeatures::FeatureIndex::INDEX_NAME, NAME } ++const std::unordered_map ++ ACPOCollectFeatures::CalculateFeatureMap{ ++ REGISTER_FUNCTION(SROASavings, calculateInlineCostFeatures), ++ REGISTER_FUNCTION(SROALosses, calculateInlineCostFeatures), ++ REGISTER_FUNCTION(LoadElimination, calculateInlineCostFeatures), ++ REGISTER_FUNCTION(CallPenalty, calculateInlineCostFeatures), ++ REGISTER_FUNCTION(CallArgumentSetup, calculateInlineCostFeatures), ++ REGISTER_FUNCTION(LoadRelativeIntrinsic, calculateInlineCostFeatures), ++ REGISTER_FUNCTION(LoweredCallArgSetup, calculateInlineCostFeatures), ++ REGISTER_FUNCTION(IndirectCallPenalty, calculateInlineCostFeatures), ++ REGISTER_FUNCTION(JumpTablePenalty, calculateInlineCostFeatures), ++ REGISTER_FUNCTION(CaseClusterPenalty, calculateInlineCostFeatures), ++ REGISTER_FUNCTION(SwitchPenalty, calculateInlineCostFeatures), ++ REGISTER_FUNCTION(UnsimplifiedCommonInstructions, ++ calculateInlineCostFeatures), ++ REGISTER_FUNCTION(NumLoops, calculateInlineCostFeatures), ++ REGISTER_FUNCTION(DeadBlocks, calculateInlineCostFeatures), ++ REGISTER_FUNCTION(SimplifiedInstructions, calculateInlineCostFeatures), ++ REGISTER_FUNCTION(ConstantArgs, calculateInlineCostFeatures), ++ REGISTER_FUNCTION(ConstantOffsetPtrArgs, calculateInlineCostFeatures), ++ REGISTER_FUNCTION(CallSiteCost, calculateInlineCostFeatures), ++ REGISTER_FUNCTION(ColdCcPenalty, calculateInlineCostFeatures), ++ REGISTER_FUNCTION(LastCallToStaticBonus, calculateInlineCostFeatures), ++ REGISTER_FUNCTION(IsMultipleBlocks, calculateInlineCostFeatures), ++ REGISTER_FUNCTION(NestedInlines, calculateInlineCostFeatures), ++ REGISTER_FUNCTION(NestedInlineCostEstimate, ++ calculateInlineCostFeatures), ++ REGISTER_FUNCTION(Threshold, calculateInlineCostFeatures), ++ REGISTER_FUNCTION(BasicBlockCount, calculateFPIRelated), ++ REGISTER_FUNCTION(BlocksReachedFromConditionalInstruction, ++ calculateFPIRelated), ++ REGISTER_FUNCTION(Uses, calculateFPIRelated), ++ REGISTER_FUNCTION(EdgeCount, calculateEdgeNodeCount), ++ REGISTER_FUNCTION(NodeCount, calculateEdgeNodeCount), ++ REGISTER_FUNCTION(ColdCallSite, calculateHotColdCallSite), ++ REGISTER_FUNCTION(HotCallSite, calculateHotColdCallSite), ++ REGISTER_FUNCTION(ACPOFIExtendedFeaturesInitialSize, ++ calculateACPOFIExtendedFeaturesFeatures), ++ REGISTER_FUNCTION(ACPOFIExtendedFeaturesBlocks, ++ calculateACPOFIExtendedFeaturesFeatures), ++ REGISTER_FUNCTION(ACPOFIExtendedFeaturesCalls, ++ calculateACPOFIExtendedFeaturesFeatures), ++ REGISTER_FUNCTION(ACPOFIExtendedFeaturesIsLocal, ++ calculateACPOFIExtendedFeaturesFeatures), ++ REGISTER_FUNCTION(ACPOFIExtendedFeaturesIsLinkOnceODR, ++ calculateACPOFIExtendedFeaturesFeatures), ++ REGISTER_FUNCTION(ACPOFIExtendedFeaturesIsLinkOnce, ++ calculateACPOFIExtendedFeaturesFeatures), ++ REGISTER_FUNCTION(ACPOFIExtendedFeaturesLoops, ++ calculateACPOFIExtendedFeaturesFeatures), ++ REGISTER_FUNCTION(ACPOFIExtendedFeaturesMaxLoopDepth, ++ calculateACPOFIExtendedFeaturesFeatures), ++ REGISTER_FUNCTION(ACPOFIExtendedFeaturesMaxDomTreeLevel, ++ calculateACPOFIExtendedFeaturesFeatures), ++ REGISTER_FUNCTION(ACPOFIExtendedFeaturesPtrArgs, ++ calculateACPOFIExtendedFeaturesFeatures), ++ REGISTER_FUNCTION(ACPOFIExtendedFeaturesPtrCallee, ++ calculateACPOFIExtendedFeaturesFeatures), ++ REGISTER_FUNCTION(ACPOFIExtendedFeaturesCallReturnPtr, ++ calculateACPOFIExtendedFeaturesFeatures), ++ REGISTER_FUNCTION(ACPOFIExtendedFeaturesConditionalBranch, ++ calculateACPOFIExtendedFeaturesFeatures), ++ REGISTER_FUNCTION(ACPOFIExtendedFeaturesCBwithArg, ++ calculateACPOFIExtendedFeaturesFeatures), ++ REGISTER_FUNCTION(ACPOFIExtendedFeaturesCallerHeight, ++ calculateACPOFIExtendedFeaturesFeatures), ++ REGISTER_FUNCTION(ACPOFIExtendedFeaturesCallUsage, ++ calculateACPOFIExtendedFeaturesFeatures), ++ REGISTER_FUNCTION(ACPOFIExtendedFeaturesIsRecursive, ++ calculateACPOFIExtendedFeaturesFeatures), ++ REGISTER_FUNCTION(ACPOFIExtendedFeaturesNumCallsiteInLoop, ++ calculateACPOFIExtendedFeaturesFeatures), ++ REGISTER_FUNCTION(ACPOFIExtendedFeaturesNumOfCallUsesInLoop, ++ calculateACPOFIExtendedFeaturesFeatures), ++ REGISTER_FUNCTION(ACPOFIExtendedFeaturesEntryBlockFreq, ++ calculateACPOFIExtendedFeaturesFeatures), ++ REGISTER_FUNCTION(ACPOFIExtendedFeaturesMaxCallsiteBlockFreq, ++ calculateACPOFIExtendedFeaturesFeatures), ++ REGISTER_FUNCTION(ACPOFIExtendedFeaturesInstructionPerBlock, ++ calculateACPOFIExtendedFeaturesFeatures), ++ REGISTER_FUNCTION(ACPOFIExtendedFeaturesSuccessorPerBlock, ++ calculateACPOFIExtendedFeaturesFeatures), ++ REGISTER_FUNCTION(ACPOFIExtendedFeaturesAvgVecInstr, ++ calculateACPOFIExtendedFeaturesFeatures), ++ REGISTER_FUNCTION(ACPOFIExtendedFeaturesAvgNestedLoopLevel, ++ calculateACPOFIExtendedFeaturesFeatures), ++ REGISTER_FUNCTION(ACPOFIExtendedFeaturesInstrPerLoop, ++ calculateACPOFIExtendedFeaturesFeatures), ++ REGISTER_FUNCTION( ++ ACPOFIExtendedFeaturesBlockWithMultipleSuccecorsPerLoop, ++ calculateACPOFIExtendedFeaturesFeatures), ++ REGISTER_FUNCTION(CallerBlockFreq, calculateCallerBlockFreq), ++ REGISTER_FUNCTION(CallSiteHeight, calculateCallSiteHeight), ++ REGISTER_FUNCTION(ConstantParam, calculateConstantParam), ++ REGISTER_FUNCTION(CostEstimate, calculateCostEstimate), ++ REGISTER_FUNCTION(LoopLevel, calculateLoopLevel), ++ REGISTER_FUNCTION(MandatoryKind, calculateMandatoryKind), ++ REGISTER_FUNCTION(MandatoryOnly, calculateMandatoryOnly), ++ REGISTER_FUNCTION(OptCode, calculateOptCode), ++ REGISTER_FUNCTION(IsIndirectCall, calculateIsIndirectCall), ++ REGISTER_FUNCTION(IsInInnerLoop, calculateIsInInnerLoop), ++ REGISTER_FUNCTION(IsMustTailCall, calculateIsMustTailCall), ++ REGISTER_FUNCTION(IsTailCall, calculateIsTailCall), ++ }; ++#undef REGISTER_FUNCTION ++ ++std::map ACPOCollectFeatures::FunctionLevels{}; ++ ++ACPOCollectFeatures::ACPOCollectFeatures() {} ++ ++ACPOCollectFeatures::ACPOCollectFeatures( ++ ACPOCollectFeatures::FeatureInfo GlobalInfo) ++ : GlobalFeatureInfo(GlobalInfo) { ++ assert(GlobalFeatureInfo.Idx == FeatureIndex::NumOfFeatures && ++ "When setting glboal FeatureInfo the Idx should always be " ++ "NumOfFeatures"); ++} ++ ++ACPOCollectFeatures::~ACPOCollectFeatures() {} ++ ++void ACPOCollectFeatures::setFeatureValue(ACPOCollectFeatures::FeatureIndex Idx, ++ std::string Val) { ++ FeatureToValue[Idx] = Val; ++} ++ ++void ACPOCollectFeatures::setFeatureInfo( ++ ACPOCollectFeatures::FeatureIndex Idx, ++ ACPOCollectFeatures::FeatureInfo Info) { ++ assert( ++ (Info.Idx == ACPOCollectFeatures::FeatureIndex::NumOfFeatures || ++ Info.Idx == Idx || getFeatureGroup(Info.Idx) == getFeatureGroup(Idx)) && ++ "When setting FeatureToInfo map the key and value pair should both refer " ++ "to the same Feature or the FeatureInfo.Idx should be NumOfFeatures."); ++ FeatureToInfo[Idx] = Info; ++} ++ ++void ACPOCollectFeatures::setFeatureValueAndInfo( ++ ACPOCollectFeatures::FeatureIndex Idx, ++ ACPOCollectFeatures::FeatureInfo Info, std::string Val) { ++ setFeatureValue(Idx, Val); ++ setFeatureInfo(Idx, Info); ++} ++ ++void ACPOCollectFeatures::setGlobalFeatureInfo( ++ ACPOCollectFeatures::FeatureInfo &Info) { ++ assert(Info.Idx == FeatureIndex::NumOfFeatures && ++ "When setting glboal FeatureInfo the Idx should always be " ++ "NumOfFeatures"); ++ GlobalFeatureInfo = Info; ++} ++ ++std::string ++ACPOCollectFeatures::getFeature(ACPOCollectFeatures::FeatureIndex Idx) const { ++ assert(registeredFeature(Idx) && "Feature not registered"); ++ return FeatureToValue.find(Idx)->second; ++} ++ ++std::string ++ACPOCollectFeatures::getFeatureName(ACPOCollectFeatures::FeatureIndex Idx) { ++ return FeatureIndexToName.find(Idx)->second; ++} ++ ++ACPOCollectFeatures::GroupID ++ACPOCollectFeatures::getFeatureGroup(ACPOCollectFeatures::FeatureIndex Idx) { ++ return FeatureIndexToGroup.find(Idx)->second; ++} ++ ++ACPOCollectFeatures::Scope ++ACPOCollectFeatures::getFeatureScope(ACPOCollectFeatures::FeatureIndex Idx) { ++ return FeatureIndexToScope.find(Idx)->second; ++} ++ ++std::set ++ACPOCollectFeatures::getGroupFeatures(ACPOCollectFeatures::GroupID Group) { ++ std::set FeatureIndices; ++ auto Range = GroupToFeatureIndices.equal_range(Group); ++ for (auto It = Range.first; It != Range.second; ++It) { ++ FeatureIndices.insert(It->second); ++ } ++ return FeatureIndices; ++} ++ ++std::set ++ACPOCollectFeatures::getScopeFeatures(ACPOCollectFeatures::Scope S) { ++ std::set FeatureIndices; ++ auto Range = ScopeToFeatureIndices.equal_range(S); ++ for (auto It = Range.first; It != Range.second; ++It) { ++ FeatureIndices.insert(It->second); ++ } ++ return FeatureIndices; ++} ++ ++bool ACPOCollectFeatures::containsFeature( ++ ACPOCollectFeatures::FeatureIndex Idx) { ++ return FeatureToValue.count(Idx) > 0; ++} ++ ++bool ACPOCollectFeatures::containsFeature( ++ ACPOCollectFeatures::GroupID GroupID) { ++ for (auto FeatureIdx : getGroupFeatures(GroupID)) { ++ if (!containsFeature(FeatureIdx)) ++ return false; ++ } ++ return true; ++} ++ ++void ACPOCollectFeatures::clearFeatureValueMap() { FeatureToValue.clear(); } ++ ++bool ACPOCollectFeatures::registeredFeature( ++ ACPOCollectFeatures::FeatureIndex Idx) const { ++ return FeatureToValue.find(Idx) != FeatureToValue.end(); ++} ++ ++void calculateFPIRelated(ACPOCollectFeatures &ACF, ++ const ACPOCollectFeatures::FeatureInfo &Info) { ++ assert(Info.Idx == ACPOCollectFeatures::FeatureIndex::NumOfFeatures || ++ Info.Idx == ACPOCollectFeatures::FeatureIndex::BasicBlockCount); ++ ++ auto *FAM = Info.Managers.FAM; ++ auto *F = Info.SI.F; ++ ++ assert(F && FAM && "Function or FAM is nullptr"); ++ ++ auto &FPI = FAM->getResult(*F); ++ ++ ACF.setFeatureValueAndInfo(ACPOCollectFeatures::FeatureIndex::BasicBlockCount, ++ Info, std::to_string(FPI.BasicBlockCount)); ++ ACF.setFeatureValueAndInfo( ++ ACPOCollectFeatures::FeatureIndex:: ++ BlocksReachedFromConditionalInstruction, ++ Info, std::to_string(FPI.BlocksReachedFromConditionalInstruction)); ++ ACF.setFeatureValueAndInfo(ACPOCollectFeatures::FeatureIndex::Uses, Info, ++ std::to_string(FPI.Uses)); ++} ++ ++void calculateCallerBlockFreq(ACPOCollectFeatures &ACF, ++ const ACPOCollectFeatures::FeatureInfo &Info) { ++ assert(Info.Idx == ACPOCollectFeatures::FeatureIndex::NumOfFeatures || ++ Info.Idx == ACPOCollectFeatures::FeatureIndex::CallerBlockFreq); ++ ++ auto *CB = Info.SI.CB; ++ auto *FAM = Info.Managers.FAM; ++ ++ assert(CB && FAM && "CallSite or FAM is nullptr"); ++ ++ Function *F = CB->getCaller(); ++ BasicBlock *BB = CB->getParent(); ++ BlockFrequencyInfo &BFI = FAM->getResult(*F); ++ ++ uint64_t CallerBlockFreq = BFI.getBlockFreq(BB).getFrequency(); ++ // The model uses signed 64-bit thus we need to take care of int overflow. ++ if (CallerBlockFreq >= std::numeric_limits::max()) { ++ CallerBlockFreq = std::numeric_limits::max() - 1; ++ } ++ ++ ACF.setFeatureValueAndInfo(ACPOCollectFeatures::FeatureIndex::CallerBlockFreq, ++ Info, std::to_string(CallerBlockFreq)); ++} ++ ++void calculateCallSiteHeight(ACPOCollectFeatures &ACF, ++ const ACPOCollectFeatures::FeatureInfo &Info) { ++ assert(Info.Idx == ACPOCollectFeatures::FeatureIndex::NumOfFeatures || ++ Info.Idx == ACPOCollectFeatures::FeatureIndex::CallSiteHeight); ++ ++ // Check if we already calculated the values. ++ if (ACF.containsFeature(ACPOCollectFeatures::FeatureIndex::CallSiteHeight)) ++ return; ++ ++ auto *CB = Info.SI.CB; ++ auto *IA = Info.OI.IA; ++ ++ assert(CB && IA && "CallSite or IA is nullptr"); ++ ++ if (IA) { ++ ACF.setFeatureValueAndInfo(ACPOCollectFeatures::FeatureIndex::CallSiteHeight, ++ Info, std::to_string(IA->getCallSiteHeight(CB))); ++ return; ++ } ++ LLVM_DEBUG(dbgs() << "IA was nullptr & callsite height is not set!" << "\n"); ++} ++ ++void calculateConstantParam(ACPOCollectFeatures &ACF, ++ const ACPOCollectFeatures::FeatureInfo &Info) { ++ assert(Info.Idx == ACPOCollectFeatures::FeatureIndex::NumOfFeatures || ++ Info.Idx == ACPOCollectFeatures::FeatureIndex::ConstantParam); ++ ++ // Check if we already calculated the values. ++ if (ACF.containsFeature(ACPOCollectFeatures::FeatureIndex::ConstantParam)) ++ return; ++ ++ auto *CB = Info.SI.CB; ++ assert(CB && "CallSite is nullptr"); ++ ++ size_t NrCtantParams = 0; ++ for (auto I = CB->arg_begin(), E = CB->arg_end(); I != E; ++I) { ++ NrCtantParams += (isa(*I)); ++ } ++ ++ ACF.setFeatureValueAndInfo(ACPOCollectFeatures::FeatureIndex::ConstantParam, ++ Info, std::to_string(NrCtantParams)); ++} ++ ++void calculateCostEstimate(ACPOCollectFeatures &ACF, ++ const ACPOCollectFeatures::FeatureInfo &Info) { ++ assert(Info.Idx == ACPOCollectFeatures::FeatureIndex::NumOfFeatures || ++ Info.Idx == ACPOCollectFeatures::FeatureIndex::CostEstimate); ++ ++ // Check if we already calculated the values. ++ if (ACF.containsFeature(ACPOCollectFeatures::FeatureIndex::CostEstimate)) ++ return; ++ ++ auto *CB = Info.SI.CB; ++ auto *FAM = Info.Managers.FAM; ++ ++ assert(CB && FAM && "CallBase or FAM is nullptr"); ++ ++ auto &Callee = *CB->getCalledFunction(); ++ auto &TIR = FAM->getResult(Callee); ++ ++ auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & { ++ return FAM->getResult(F); ++ }; ++ ++ int CostEstimate = 0; ++ auto IsCallSiteInlinable = ++ llvm::getInliningCostEstimate(*CB, TIR, GetAssumptionCache); ++ if (IsCallSiteInlinable) ++ CostEstimate = *IsCallSiteInlinable; ++ ++ ACF.setFeatureValueAndInfo(ACPOCollectFeatures::FeatureIndex::CostEstimate, ++ Info, std::to_string(CostEstimate)); ++} ++ ++int64_t getLocalCalls(Function &F, FunctionAnalysisManager &FAM) { ++ return FAM.getResult(F) ++ .DirectCallsToDefinedFunctions; ++} ++ ++void calculateEdgeNodeCount(ACPOCollectFeatures &ACF, ++ const ACPOCollectFeatures::FeatureInfo &Info) { ++ assert(Info.Idx == ACPOCollectFeatures::FeatureIndex::NumOfFeatures || ++ ACPOCollectFeatures::getFeatureGroup(Info.Idx) == ++ ACPOCollectFeatures::GroupID::EdgeNodeCount); ++ ++ // Check if we already calculated the values. ++ if (ACF.containsFeature(ACPOCollectFeatures::GroupID::EdgeNodeCount)) ++ return; ++ ++ auto *M = Info.SI.M; ++ auto *FAM = Info.Managers.FAM; ++ ++ assert(M && FAM && "Module or FAM is nullptr"); ++ ++ int NodeCount = 0; ++ int EdgeCount = 0; ++ for (auto &F : *M) ++ if (!F.isDeclaration()) { ++ ++NodeCount; ++ EdgeCount += getLocalCalls(F, *FAM); ++ } ++ ++ std::string EdgeCountStr = std::to_string(EdgeCount); ++ std::string NodeCountStr = std::to_string(NodeCount); ++ ACF.setFeatureValueAndInfo(ACPOCollectFeatures::FeatureIndex::EdgeCount, Info, ++ EdgeCountStr); ++ ACF.setFeatureValueAndInfo(ACPOCollectFeatures::FeatureIndex::NodeCount, Info, ++ NodeCountStr); ++} ++ ++void calculateHotColdCallSite(ACPOCollectFeatures &ACF, ++ const ACPOCollectFeatures::FeatureInfo &Info) { ++ assert(Info.Idx == ACPOCollectFeatures::FeatureIndex::NumOfFeatures || ++ ACPOCollectFeatures::getFeatureGroup(Info.Idx) == ++ ACPOCollectFeatures::GroupID::HotColdCallSite); ++ ++ // Check if we already calculated the values. ++ if (ACF.containsFeature(ACPOCollectFeatures::GroupID::HotColdCallSite)) ++ return; ++ ++ auto *CB = Info.SI.CB; ++ auto *FAM = Info.Managers.FAM; ++ ++ assert(CB && FAM && "Module or FAM is nullptr"); ++ ++ auto &Caller = *CB->getCaller(); ++ auto GetBFI = [&](Function &F) -> BlockFrequencyInfo & { ++ return FAM->getResult(F); ++ }; ++ ++ BlockFrequencyInfo &CallerBFI = GetBFI(Caller); ++ const BranchProbability ColdProb(2, 100); ++ auto *CallSiteBB = CB->getParent(); ++ auto CallSiteFreq = CallerBFI.getBlockFreq(CallSiteBB); ++ auto CallerEntryFreq = ++ CallerBFI.getBlockFreq(&(CB->getCaller()->getEntryBlock())); ++ bool ColdCallSite = CallSiteFreq < CallerEntryFreq * ColdProb; ++ auto CallerEntryFreqHot = CallerBFI.getEntryFreq(); ++ bool HotCallSite = (CallSiteFreq.getFrequency() >= CallerEntryFreqHot * 60); ++ ++ ACF.setFeatureValueAndInfo(ACPOCollectFeatures::FeatureIndex::ColdCallSite, ++ Info, std::to_string(ColdCallSite)); ++ ACF.setFeatureValueAndInfo(ACPOCollectFeatures::FeatureIndex::HotCallSite, ++ Info, std::to_string(HotCallSite)); ++} ++ ++void calculateLoopLevel(ACPOCollectFeatures &ACF, ++ const ACPOCollectFeatures::FeatureInfo &Info) { ++ assert(Info.Idx == ACPOCollectFeatures::FeatureIndex::NumOfFeatures || ++ Info.Idx == ACPOCollectFeatures::FeatureIndex::LoopLevel); ++ ++ // Check if we already calculated the values. ++ if (ACF.containsFeature(ACPOCollectFeatures::FeatureIndex::LoopLevel)) ++ return; ++ ++ auto *CB = Info.SI.CB; ++ auto *FAM = Info.Managers.FAM; ++ ++ assert(CB && FAM && "CallBase or FAM is nullptr"); ++ ++ Function *F = CB->getCaller(); ++ BasicBlock *BB = CB->getParent(); ++ LoopInfo &LI = FAM->getResult(*F); ++ ++ std::string OptCode = std::to_string(CB->getOpcode()); ++ ACF.setFeatureValueAndInfo(ACPOCollectFeatures::FeatureIndex::LoopLevel, Info, ++ std::to_string(LI.getLoopDepth(BB))); ++} ++ ++InlineAdvisor::MandatoryInliningKind ++ACPOCollectFeatures::getMandatoryKind(CallBase &CB, ++ FunctionAnalysisManager &FAM, ++ OptimizationRemarkEmitter &ORE) { ++ return InlineAdvisor::getMandatoryKind(CB, FAM, ORE); ++} ++ ++void calculateMandatoryKind(ACPOCollectFeatures &ACF, ++ const ACPOCollectFeatures::FeatureInfo &Info) { ++ assert(Info.Idx == ACPOCollectFeatures::FeatureIndex::NumOfFeatures || ++ Info.Idx == ACPOCollectFeatures::FeatureIndex::MandatoryKind); ++ ++ // Check if we already calculated the values. ++ if (ACF.containsFeature(ACPOCollectFeatures::FeatureIndex::MandatoryKind)) ++ return; ++ ++ auto *CB = Info.SI.CB; ++ auto *FAM = Info.Managers.FAM; ++ ++ assert(CB && FAM && "CallBase or FAM is nullptr"); ++ ++ auto &Caller = *CB->getCaller(); ++ auto &ORE = FAM->getResult(Caller); ++ auto MandatoryKind = ACPOCollectFeatures::getMandatoryKind(*CB, *FAM, ORE); ++ ++ ACF.setFeatureValueAndInfo(ACPOCollectFeatures::FeatureIndex::MandatoryKind, ++ Info, std::to_string((int)MandatoryKind)); ++} ++ ++void calculateMandatoryOnly(ACPOCollectFeatures &ACF, ++ const ACPOCollectFeatures::FeatureInfo &Info) { ++ assert(Info.Idx == ACPOCollectFeatures::FeatureIndex::NumOfFeatures || ++ Info.Idx == ACPOCollectFeatures::FeatureIndex::MandatoryOnly); ++ ++ // Check if we already calculated the values. ++ if (ACF.containsFeature(ACPOCollectFeatures::FeatureIndex::MandatoryOnly)) ++ return; ++ ++ ACF.setFeatureValueAndInfo(ACPOCollectFeatures::FeatureIndex::MandatoryOnly, ++ Info, std::to_string((int)Info.OI.MandatoryOnly)); ++} ++ ++void calculateOptCode(ACPOCollectFeatures &ACF, ++ const ACPOCollectFeatures::FeatureInfo &Info) { ++ assert(Info.Idx == ACPOCollectFeatures::FeatureIndex::NumOfFeatures || ++ Info.Idx == ACPOCollectFeatures::FeatureIndex::OptCode); ++ ++ // Check if we already calculated the values. ++ if (ACF.containsFeature(ACPOCollectFeatures::FeatureIndex::OptCode)) ++ return; ++ ++ auto *CB = Info.SI.CB; ++ ++ assert(CB && "CallBase is nullptr"); ++ ++ std::string OptCode = std::to_string(CB->getOpcode()); ++ ACF.setFeatureValueAndInfo(ACPOCollectFeatures::FeatureIndex::OptCode, Info, ++ OptCode); ++} ++ ++void calculateInlineCostFeatures(ACPOCollectFeatures &ACF, ++ const ACPOCollectFeatures::FeatureInfo &Info) { ++ assert(Info.Idx == ACPOCollectFeatures::FeatureIndex::NumOfFeatures || ++ (ACPOCollectFeatures::getFeatureGroup(Info.Idx) == ++ ACPOCollectFeatures::GroupID::InlineCostFeatureGroup)); ++ ++ // Check if we already calculated the values. ++ if (ACF.containsFeature(ACPOCollectFeatures::GroupID::InlineCostFeatureGroup)) ++ return; ++ ++ auto *CB = Info.SI.CB; ++ auto *FAM = Info.Managers.FAM; ++ ++ assert(CB && FAM && "CallBase or FAM is nullptr"); ++ ++ auto &Callee = *CB->getCalledFunction(); ++ auto &TIR = FAM->getResult(Callee); ++ ++ auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & { ++ return FAM->getResult(F); ++ }; ++ ++ const auto CostFeaturesOpt = ++ getInliningCostFeatures(*CB, TIR, GetAssumptionCache); ++ ++ for (auto Idx = ++ ACPOCollectFeatures::FeatureIndex::InlineCostFeatureGroupBegin + 1; ++ Idx != ACPOCollectFeatures::FeatureIndex::InlineCostFeatureGroupEnd; ++ ++Idx) { ++ size_t TmpIdx = ++ static_cast(Idx) - ++ static_cast( ++ ACPOCollectFeatures::FeatureIndex::InlineCostFeatureGroupBegin) - ++ 1; ++ ACF.setFeatureValueAndInfo( ++ Idx, Info, ++ std::to_string(CostFeaturesOpt ? CostFeaturesOpt.value()[TmpIdx] : 0)); ++ } ++} ++ ++static void ++checkValidFFCache(Function &F, ++ struct ACPOFIExtendedFeatures::FunctionFeatures &FF, ++ DominatorTree &Tree, TargetTransformInfo &TTI, LoopInfo &LI, ++ bool &ValidSize, bool &ValidLoop, bool &ValidTree) { ++ std::optional SizeCache = ACPOFIModel::getCachedSize( ++ &F, ACPOFIExtendedFeatures::NamedFeatureIndex::InitialSize); ++ auto TTIAnalysisCache = ACPOFIModel::getTTICachedAnalysis(&F); ++ if (SizeCache && TTIAnalysisCache == &TTI) { ++ ValidSize = true; ++ } ++ ++ std::optional MaxDomTreeLevelCache = ACPOFIModel::getCachedSize( ++ &F, ACPOFIExtendedFeatures::NamedFeatureIndex::MaxDomTreeLevel); ++ auto DomCache = ACPOFIModel::getDomCachedAnalysis(&F); ++ if (MaxDomTreeLevelCache && DomCache == &Tree) { ++ ValidTree = true; ++ } ++ ++ std::optional LoopNumCache = ACPOFIModel::getCachedSize( ++ &F, ACPOFIExtendedFeatures::NamedFeatureIndex::Loops); ++ auto LIAnalysisCache = ACPOFIModel::getLICachedAnalysis(&F); ++ if (LoopNumCache && LIAnalysisCache == &LI) { ++ ValidLoop = true; ++ } ++} ++ ++static void getCachedFF(Function &F, ++ struct ACPOFIExtendedFeatures::FunctionFeatures &FF, ++ DominatorTree &Tree, TargetTransformInfo &TTI, ++ LoopInfo &LI) { ++ std::optional SizeCache = ACPOFIModel::getCachedSize( ++ &F, ACPOFIExtendedFeatures::NamedFeatureIndex::InitialSize); ++ auto TTIAnalysisCache = ACPOFIModel::getTTICachedAnalysis(&F); ++ if (SizeCache && TTIAnalysisCache == &TTI) { ++ FF[ACPOFIExtendedFeatures::NamedFeatureIndex::InitialSize] = ++ SizeCache.value(); ++ } ++ ++ std::optional MaxDomTreeLevelCache = ACPOFIModel::getCachedSize( ++ &F, ACPOFIExtendedFeatures::NamedFeatureIndex::MaxDomTreeLevel); ++ auto DomCache = ACPOFIModel::getDomCachedAnalysis(&F); ++ if (MaxDomTreeLevelCache && DomCache == &Tree) { ++ FF[ACPOFIExtendedFeatures::NamedFeatureIndex::MaxDomTreeLevel] = ++ MaxDomTreeLevelCache.value(); ++ } ++ ++ std::optional LoopNumCache = ACPOFIModel::getCachedSize( ++ &F, ACPOFIExtendedFeatures::NamedFeatureIndex::Loops); ++ auto LIAnalysisCache = ACPOFIModel::getLICachedAnalysis(&F); ++ if (LoopNumCache && LIAnalysisCache == &LI) { ++ FF[ACPOFIExtendedFeatures::NamedFeatureIndex::Loops] = LoopNumCache.value(); ++ FF[ACPOFIExtendedFeatures::NamedFeatureIndex::MaxLoopDepth] = ++ ACPOFIModel::getCachedSize( ++ &F, ACPOFIExtendedFeatures::NamedFeatureIndex::MaxLoopDepth) ++ .value(); ++ if (LoopNumCache.value() != 0) { ++ FF[ACPOFIExtendedFeatures::NamedFloatFeatureIndex::InstrPerLoop] = ++ ACPOFIModel::getCachedFloat( ++ &F, ACPOFIExtendedFeatures::NamedFloatFeatureIndex::InstrPerLoop) ++ .value(); ++ FF[ACPOFIExtendedFeatures::NamedFloatFeatureIndex:: ++ BlockWithMultipleSuccecorsPerLoop] = ++ ACPOFIModel::getCachedFloat( ++ &F, ACPOFIExtendedFeatures::NamedFloatFeatureIndex:: ++ BlockWithMultipleSuccecorsPerLoop) ++ .value(); ++ FF[ACPOFIExtendedFeatures::NamedFloatFeatureIndex::AvgNestedLoopLevel] = ++ ACPOFIModel::getCachedFloat( ++ &F, ACPOFIExtendedFeatures::NamedFloatFeatureIndex:: ++ AvgNestedLoopLevel) ++ .value(); ++ } ++ } ++} ++ ++static void updateCachedFF(Function &F, ++ struct ACPOFIExtendedFeatures::FunctionFeatures &FF, ++ DominatorTree &Tree, TargetTransformInfo &TTI, ++ LoopInfo &LI) { ++ ACPOFIModel::insertSizeCache( ++ &F, ACPOFIExtendedFeatures::NamedFeatureIndex::InitialSize, ++ FF[ACPOFIExtendedFeatures::NamedFeatureIndex::InitialSize]); ++ ACPOFIModel::insertAnalysisCache(&F, &TTI); ++ ACPOFIModel::insertSizeCache( ++ &F, ACPOFIExtendedFeatures::NamedFeatureIndex::MaxDomTreeLevel, ++ FF[ACPOFIExtendedFeatures::NamedFeatureIndex::MaxDomTreeLevel]); ++ ACPOFIModel::insertAnalysisCache(&F, &Tree); ++ ACPOFIModel::insertSizeCache( ++ &F, ACPOFIExtendedFeatures::NamedFeatureIndex::Loops, ++ FF[ACPOFIExtendedFeatures::NamedFeatureIndex::Loops]); ++ ACPOFIModel::insertSizeCache( ++ &F, ACPOFIExtendedFeatures::NamedFeatureIndex::MaxLoopDepth, ++ FF[ACPOFIExtendedFeatures::NamedFeatureIndex::MaxLoopDepth]); ++ ACPOFIModel::insertFloatCache( ++ &F, ACPOFIExtendedFeatures::NamedFloatFeatureIndex::InstrPerLoop, ++ FF[ACPOFIExtendedFeatures::NamedFloatFeatureIndex::InstrPerLoop]); ++ ACPOFIModel::insertFloatCache( ++ &F, ++ ACPOFIExtendedFeatures::NamedFloatFeatureIndex:: ++ BlockWithMultipleSuccecorsPerLoop, ++ FF[ACPOFIExtendedFeatures::NamedFloatFeatureIndex:: ++ BlockWithMultipleSuccecorsPerLoop]); ++ ACPOFIModel::insertFloatCache( ++ &F, ACPOFIExtendedFeatures::NamedFloatFeatureIndex::AvgNestedLoopLevel, ++ FF[ACPOFIExtendedFeatures::NamedFloatFeatureIndex::AvgNestedLoopLevel]); ++ ACPOFIModel::insertAnalysisCache(&F, &LI); ++} ++ ++void calculateACPOFIExtendedFeaturesFeatures( ++ ACPOCollectFeatures &ACF, const ACPOCollectFeatures::FeatureInfo &Info) { ++ assert(Info.Idx == ACPOCollectFeatures::FeatureIndex::NumOfFeatures || ++ ACPOCollectFeatures::getFeatureGroup(Info.Idx) == ++ ACPOCollectFeatures::GroupID::ACPOFIExtendedFeatures); ++ ++ // Check if we already calculated the values. ++ if (ACF.containsFeature(ACPOCollectFeatures::GroupID::ACPOFIExtendedFeatures)) ++ return; ++ ++ auto F = Info.SI.F; ++ auto *FAM = Info.Managers.FAM; ++ ++ assert(F && FAM && "F or FAM is nullptr"); ++ ++ struct ACPOFIExtendedFeatures::FunctionFeatures FF; ++ auto &DomTree = FAM->getResult(*F); ++ auto &TTI = FAM->getResult(*F); ++ auto &LI = FAM->getResult(*F); ++ bool ValidSize = false; ++ bool ValidLoop = false; ++ bool ValidTree = false; ++ checkValidFFCache(*F, FF, DomTree, TTI, LI, ValidSize, ValidLoop, ValidTree); ++ FF = ACPOFIExtendedFeatures::getFunctionFeatures( ++ *F, DomTree, TTI, LI, FAM, ValidSize, ValidLoop, ValidTree); ++ getCachedFF(*F, FF, DomTree, TTI, LI); ++ updateCachedFF(*F, FF, DomTree, TTI, LI); ++ ++ for (auto Idx = ACPOCollectFeatures::FeatureIndex:: ++ ACPOFIExtendedFeaturesNamedFeatureBegin + ++ 1; ++ Idx != ++ ACPOCollectFeatures::FeatureIndex::ACPOFIExtendedFeaturesNamedFeatureEnd; ++ ++Idx) { ++ size_t TmpIdx = ++ static_cast(Idx) - ++ static_cast(ACPOCollectFeatures::FeatureIndex:: ++ ACPOFIExtendedFeaturesNamedFeatureBegin) - ++ 1; ++ ACF.setFeatureValueAndInfo(Idx, Info, ++ std::to_string(FF.NamedFeatures[TmpIdx])); ++ } ++ for (auto Idx = ACPOCollectFeatures::FeatureIndex:: ++ ACPOFIExtendedFeaturesFloatFeatureBegin + ++ 1; ++ Idx != ++ ACPOCollectFeatures::FeatureIndex::ACPOFIExtendedFeaturesFloatFeatureEnd; ++ ++Idx) { ++ size_t TmpIdx = ++ static_cast(Idx) - ++ static_cast(ACPOCollectFeatures::FeatureIndex:: ++ ACPOFIExtendedFeaturesFloatFeatureBegin) - ++ 1; ++ ACF.setFeatureValueAndInfo(Idx, Info, ++ std::to_string(FF.NamedFloatFeatures[TmpIdx])); ++ } ++} ++ ++void calculateIsIndirectCall(ACPOCollectFeatures &ACF, ++ const ACPOCollectFeatures::FeatureInfo &Info) { ++ assert(Info.Idx == ACPOCollectFeatures::FeatureIndex::NumOfFeatures || ++ Info.Idx == ACPOCollectFeatures::FeatureIndex::IsIndirectCall); ++ ++ // Check if we already calculated the values. ++ if (ACF.containsFeature(ACPOCollectFeatures::FeatureIndex::IsIndirectCall)) ++ return; ++ ++ auto *CB = Info.SI.CB; ++ ++ assert(CB && "CallBase is nullptr"); ++ ++ ACF.setFeatureValueAndInfo(ACPOCollectFeatures::FeatureIndex::IsIndirectCall, ++ Info, std::to_string(CB->isIndirectCall())); ++} ++ ++void calculateIsInInnerLoop(ACPOCollectFeatures &ACF, ++ const ACPOCollectFeatures::FeatureInfo &Info) { ++ assert(Info.Idx == ACPOCollectFeatures::FeatureIndex::NumOfFeatures || ++ Info.Idx == ACPOCollectFeatures::FeatureIndex::IsInInnerLoop); ++ ++ // Check if we already calculated the values. ++ if (ACF.containsFeature(ACPOCollectFeatures::FeatureIndex::IsInInnerLoop)) ++ return; ++ ++ auto *CB = Info.SI.CB; ++ auto *FAM = Info.Managers.FAM; ++ ++ assert(CB && FAM && "CallBase or FAM is nullptr"); ++ ++ auto &Caller = *CB->getCaller(); ++ auto &CallerLI = FAM->getResult(Caller); ++ ++ // Get loop for CB's BB. And check whether the loop is an inner most loop. ++ bool CallSiteInInnerLoop = false; ++ for (auto &L : CallerLI) { ++ if (L->isInnermost() && L->contains(CB)) ++ CallSiteInInnerLoop = true; ++ } ++ ++ ACF.setFeatureValueAndInfo(ACPOCollectFeatures::FeatureIndex::IsInInnerLoop, ++ Info, std::to_string(CallSiteInInnerLoop)); ++} ++ ++void calculateIsMustTailCall(ACPOCollectFeatures &ACF, ++ const ACPOCollectFeatures::FeatureInfo &Info) { ++ assert(Info.Idx == ACPOCollectFeatures::FeatureIndex::NumOfFeatures || ++ Info.Idx == ACPOCollectFeatures::FeatureIndex::IsMustTailCall); ++ ++ // Check if we already calculated the values. ++ if (ACF.containsFeature(ACPOCollectFeatures::FeatureIndex::IsMustTailCall)) ++ return; ++ ++ auto *CB = Info.SI.CB; ++ ++ assert(CB && "CallBase is nullptr"); ++ ++ ACF.setFeatureValueAndInfo(ACPOCollectFeatures::FeatureIndex::IsMustTailCall, ++ Info, std::to_string(CB->isMustTailCall())); ++} ++ ++void calculateIsTailCall(ACPOCollectFeatures &ACF, ++ const ACPOCollectFeatures::FeatureInfo &Info) { ++ assert(Info.Idx == ACPOCollectFeatures::FeatureIndex::NumOfFeatures || ++ Info.Idx == ACPOCollectFeatures::FeatureIndex::IsTailCall); ++ ++ // Check if we already calculated the values. ++ if (ACF.containsFeature(ACPOCollectFeatures::FeatureIndex::IsTailCall)) ++ return; ++ ++ auto *CB = Info.SI.CB; ++ ++ assert(CB && "CallBase is nullptr"); ++ ++ ACF.setFeatureValueAndInfo(ACPOCollectFeatures::FeatureIndex::IsTailCall, ++ Info, std::to_string(CB->isTailCall())); ++} ++ ++ACPOCollectFeatures::FeatureValueMap ACPOCollectFeatures::getFeaturesPair( ++ ACPOCollectFeatures::FeaturesInfo FeatureInfoVec) { ++ clearFeatureValueMap(); ++ for (auto &FeatureInfo : FeatureInfoVec) { ++ auto It = CalculateFeatureMap.find(FeatureInfo.Idx); ++ if (It == CalculateFeatureMap.end()) { ++ assert("Could not find the corresponding function to calculate feature"); ++ } ++ auto CalculateFunction = It->second; ++ CalculateFunction(*this, FeatureInfo); ++ LLVM_DEBUG(dbgs() << "ACPO Feature " << getFeatureName(FeatureInfo.Idx) ++ << ": " << FeatureToValue[FeatureInfo.Idx] << "\n"); ++ } ++ ++ return FeatureToValue; ++} ++ ++ACPOCollectFeatures::FeatureValueMap ++ACPOCollectFeatures::getFeaturesPair(ACPOCollectFeatures::Scopes ScopeVec) { ++ clearFeatureValueMap(); ++ for (auto Scope : ScopeVec) { ++ for (auto FeatureIdx : getScopeFeatures(Scope)) { ++ auto It = CalculateFeatureMap.find(FeatureIdx); ++ if (It == CalculateFeatureMap.end()) { ++ assert( ++ "Could not find the corresponding function to calculate feature"); ++ } ++ auto CalculateFunction = It->second; ++ CalculateFunction(*this, GlobalFeatureInfo); ++ LLVM_DEBUG(dbgs() << "ACPO Feature " << getFeatureName(FeatureIdx) ++ << ": " << FeatureToValue[FeatureIdx] << "\n"); ++ } ++ } ++ ++ return FeatureToValue; ++} ++ ++ACPOCollectFeatures::FeatureValueMap ++ACPOCollectFeatures::getFeaturesPair(ACPOCollectFeatures::GroupIDs GroupIDVec) { ++ clearFeatureValueMap(); ++ for (auto GroupID : GroupIDVec) { ++ for (auto FeatureIdx : getGroupFeatures(GroupID)) { ++ auto It = CalculateFeatureMap.find(FeatureIdx); ++ if (It == CalculateFeatureMap.end()) { ++ assert( ++ "Could not find the corresponding function to calculate feature"); ++ } ++ auto CalculateFunction = It->second; ++ CalculateFunction(*this, GlobalFeatureInfo); ++ LLVM_DEBUG(dbgs() << "ACPO Feature " << getFeatureName(FeatureIdx) ++ << ": " << FeatureToValue[FeatureIdx] << "\n"); ++ } ++ } ++ ++ return FeatureToValue; ++} ++ ++ACPOCollectFeatures::FeatureValueMap ++ACPOCollectFeatures::getFeaturesPair(ACPOCollectFeatures::FeatureIndex Beg, ++ ACPOCollectFeatures::FeatureIndex End) { ++ assert(Beg <= End); ++ for (auto Idx = Beg; Idx != End; ++Idx) { ++ auto It = CalculateFeatureMap.find(Idx); ++ if (It == CalculateFeatureMap.end()) { ++ assert("Could not find the corresponding function to calculate feature"); ++ } ++ auto CalculateFunction = It->second; ++ CalculateFunction(*this, GlobalFeatureInfo); ++ } ++ ++ return FeatureToValue; ++} ++ ++void ACPOCollectFeatures::clearFunctionLevel() { FunctionLevels.clear(); } ++ ++void ACPOCollectFeatures::insertFunctionLevel(const Function *F, unsigned FL) { ++ FunctionLevels[F] = FL; ++} ++ ++std::optional ++ACPOCollectFeatures::getFunctionLevel(const Function *F) { ++ auto It = FunctionLevels.find(F); ++ if (It == FunctionLevels.end()) { ++ return std::nullopt; ++ } else { ++ return It->second; ++ } ++} ++ ++ACPOCollectFeatures::FeatureIndex operator+(ACPOCollectFeatures::FeatureIndex N, ++ int Counter) { ++ return static_cast((int)N + Counter); ++} ++ ++ACPOCollectFeatures::FeatureIndex operator-(ACPOCollectFeatures::FeatureIndex N, ++ int Counter) { ++ return static_cast((int)N - Counter); ++} ++ ++ACPOCollectFeatures::FeatureIndex & ++operator++(ACPOCollectFeatures::FeatureIndex &N) { ++ return N = static_cast((int)N + 1); ++} ++ ++ACPOCollectFeatures::FeatureIndex ++operator++(ACPOCollectFeatures::FeatureIndex &N, int) { ++ ACPOCollectFeatures::FeatureIndex Res = N; ++ ++N; ++ return Res; ++} ++ ++} // namespace llvm +diff --git a/llvm/lib/Analysis/ACPOMLInterface.cpp b/llvm/lib/Analysis/ACPOMLInterface.cpp +new file mode 100644 +index 000000000000..271dcfe7d851 +--- /dev/null ++++ b/llvm/lib/Analysis/ACPOMLInterface.cpp +@@ -0,0 +1,1405 @@ ++//===- ACPOMLInterface.cpp - AI-Enabled Continuous Program Optimization ---===// ++// ++// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. ++// See https://llvm.org/LICENSE.txt for license information. ++// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ++// ++//===----------------------------------------------------------------------===// ++// ++// This file implements an interface to the ML framework. ++// ++//===----------------------------------------------------------------------===// ++ ++#include "llvm/Analysis/ACPOMLInterface.h" ++#include "llvm/Analysis/ACPOModelRunner.h" ++#include "llvm/Analysis/FIModelRunner.h" ++#include "llvm/Analysis/TensorSpec.h" ++#include "llvm/Support/Process.h" ++#include "llvm/Support/Program.h" ++#include "llvm/Support/raw_ostream.h" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#ifdef _WIN32 ++#include ++#else ++#include ++#endif ++ ++using namespace llvm; ++ ++#define DEBUG_TYPE "acpo" ++ ++#define ACPO_ENV_VAR_DIR "ACPO_DIR" ++#define ACPO_ML_PYTHON_INTERFACE_PY "MLInterface.py" ++#define ACPO_PYTHON_EXECUTABLE "python" ++#define ACPO_PIPE_PREFIX "ACPO_Pipe" ++ ++#define RESPONSE_MODEL_LOADED "Model loaded" ++#define RESPONSE_ALREADY_IN_DICT "already in dict" ++#define RESPONSE_FEATURE_SET "Feature set" ++#define RESPONSE_FEATURES_INITIALIZED "Features initialized" ++#define RESPONSE_FEATURES_SET "Features set" ++#define RESPONSE_COMPLETED "Completed" ++#define RESPONSE_ACTIVE "Active" ++#define RESPONSE_ERROR "ERROR" ++ ++// Static variables ++ ++static std::shared_ptr PersistentMLIF = nullptr; ++ ++// Class definitions ++ ++bool Model::registerFeature(std::string FeatureName, uint64_t FeatureID, ++ int Index) { ++ auto Find1 = NameToID.find(FeatureName); ++ if (Find1 != NameToID.end()) { ++ LLVM_DEBUG(dbgs() << "ERROR in registerFeature: Feature " << FeatureName ++ << " already exists\n"); ++ return false; ++ } ++ NameToID.insert(std::make_pair(FeatureName, FeatureID)); ++ IDToName.insert(std::make_pair(FeatureID, FeatureName)); ++ auto Find2 = IDToIndex.find(FeatureID); ++ if (Find2 != IDToIndex.end()) { ++ LLVM_DEBUG(dbgs() << "ERROR in registerFeature: Feature with ID " ++ << FeatureID << " already exists\n"); ++ return false; ++ } ++ IDToIndex.insert(std::make_pair(FeatureID, Index)); ++ return true; ++} ++ ++bool Model::registerInput(std::string InputName, std::string InputType) { ++ auto Find = InputMap.find(InputName); ++ if (Find != InputMap.end()) { ++ LLVM_DEBUG(dbgs() << "ERROR in registerInput: Input " << InputName ++ << " already exists\n"); ++ return false; ++ } ++ InputMap.insert(std::make_pair(InputName, InputType)); ++ return true; ++} ++ ++bool Model::registerOutput(std::string OutputName, std::string OutputType) { ++ auto Find = OutputMap.find(OutputName); ++ if (Find != OutputMap.end()) { ++ LLVM_DEBUG(dbgs() << "ERROR in registerOutput: Output " << OutputName ++ << " already exists\n"); ++ return false; ++ } ++ OutputMap.insert(std::make_pair(OutputName, OutputType)); ++ return true; ++} ++ ++int Model::getIndex(uint64_t FeatureID) const { ++ auto Find = IDToIndex.find(FeatureID); ++ assert(Find != IDToIndex.end()); ++ return Find->second; ++} ++ ++int Model::getIndex(std::string FeatureName) const { ++ auto Find = NameToID.find(FeatureName); ++ assert(Find != NameToID.end()); ++ uint64_t ID = Find->second; ++ return getIndex(ID); ++} ++ ++std::string Model::getName(uint64_t FeatureID) const { ++ auto Find = IDToName.find(FeatureID); ++ assert(Find != IDToName.end()); ++ return Find->second; ++} ++ ++bool Model::checkOutputExists(std::string OutputName) const { ++ return (OutputMap.find(OutputName) != OutputMap.end()); ++} ++ ++std::string Model::getInputType(std::string InputName) const { ++ auto Find = InputMap.find(InputName); ++ assert(Find != InputMap.end()); ++ return Find->second; ++} ++ ++std::string Model::getOutputType(std::string OutputName) const { ++ auto Find = OutputMap.find(OutputName); ++ assert(Find != OutputMap.end()); ++ return Find->second; ++} ++ ++ACPOMLPythonInterface::ACPOMLPythonInterface() : NextID{0} { ++ std::optional Env = llvm::sys::Process::GetEnv(ACPO_ENV_VAR_DIR); ++ if (!Env || *Env == "") { ++ std::optional LLVMDIROpt = ++ llvm::sys::Process::GetEnv("LLVM_DIR"); ++ if (LLVMDIROpt) { ++ Env = *LLVMDIROpt + "/acpo/"; ++ } else { ++ return; ++ } ++ } ++ ++ int32_t PID = (int32_t) llvm::sys::Process::getProcessId(); ++ std::string ExecPython = "/usr/bin/python3"; ++ std::string ++ PythonScript = *Env + "/" + std::string(ACPO_ML_PYTHON_INTERFACE_PY); ++ std::string PIDStr = std::to_string(PID); ++ std::string TimeStr = std::to_string(time(nullptr)); ++ std::string NameOut = ++ *Env + "/" + ACPO_PIPE_PREFIX + "_CMD_" + PIDStr + "_" + TimeStr; ++ std::string NameIn = ++ *Env + "/" + ACPO_PIPE_PREFIX + "_RESP_" + PIDStr + "_" + TimeStr; ++ StringRef Args[] = { ExecPython, PythonScript, NameOut, NameIn }; ++ ++ // Start a process and don't wait for it to finish. We want it running in ++ // tandem. ++ std::string ErrMsg; ++ SubProcess = ++ sys::ExecuteNoWait(ExecPython, Args, std::nullopt, {}, 0, &ErrMsg); ++ if (!SubProcess.Pid) { ++ // Print out error message if the process fails to start. ++ LLVM_DEBUG(dbgs() << ErrMsg << "\n"); ++ return; ++ } ++ // Yield to Python Process to set up pipes. ++ const int PythonProcessStartupLatency = 100; ++ usleep(PythonProcessStartupLatency); ++ ++ // Now link to named pipes created by the process we just started. Note that ++ // because the creation of this file as a pipe was done elsewhere, the ++ // interface here is simple. ++ ++ // First check that the response pipe has been created by attempting to open a ++ // file for reading. If this is not successful, then sleep for 100us to allow ++ // the ML interface the time to create named pipes and open the response pipe ++ // for writing. Once that is done, the fopen call will pass here. ++ ++ // FIXME: Support library provides robust and portable APIs for opening files ++ // and creating input/output streams. Use them instead of calling libc ++ // functions. ++ PipeIn = fopen(NameIn.c_str(), "r"); ++ if (PipeIn == nullptr) { ++ do { ++ usleep(100); ++ PipeIn = fopen(NameIn.c_str(), "r"); ++ } while (PipeIn == nullptr); ++ } ++ ++ // Once the response FIFO is created, then open the command FIFO for writing. ++ // This will complete the handshake with the MLInterface in Python. ++ PipeOut = fopen(NameOut.c_str(), "w"); ++ // Now open named pipes to the new process. ++ setInitialized(true); ++} ++ ++ACPOMLPythonInterface::~ACPOMLPythonInterface() { ++ if (SubProcess.Pid) ++ closeMLInterface(); ++ if (PipeIn) ++ fclose(PipeIn); ++ if (PipeOut) ++ fclose(PipeOut); ++ if (SubProcess.Pid) { ++ // Wait for the MLInterface 3 seconds and kill it. ++ sys::Wait(SubProcess, 3) ; ++ SubProcess = sys::ProcessInfo{}; ++ } ++ setInitialized(false); ++} ++ ++uint64_t ACPOMLPythonInterface::assignID() { ++ NextID++; ++ return NextID - 1; ++} ++ ++bool ACPOMLPythonInterface::loadModel(std::string ModelSpecFile) { ++ sendCommand("LoadModel " + ModelSpecFile); ++ std::string Response = getResponse(); ++ std::vector Tokens = tokenize(Response); ++ if (Tokens[0] != RESPONSE_MODEL_LOADED) { ++ return false; ++ } ++ if (Tokens[1] == RESPONSE_ALREADY_IN_DICT) { ++ LLVM_DEBUG(dbgs() << "loadModel: the model specified in " << ModelSpecFile ++ << " has already been loaded\n"); ++ return true; ++ } ++ std::string ModelName = Tokens[1]; ++ int NumFeatures = std::stoi(Tokens[2]); ++ LLVM_DEBUG(dbgs() << "Registering features: " << NumFeatures << "\n"); ++ registerModel(ModelName, NumFeatures); ++ auto ModelPtr = ModelMap.find(ModelName)->second; ++ std::string FeatureName = ""; ++ for (int I = 0; I < NumFeatures; I++) { ++ FeatureName = Tokens[I + 3]; ++ if (!registerFeature(ModelName, FeatureName, I)) { ++ return false; ++ } ++ } ++ int OutputStart = 3 + NumFeatures; ++ int NumOutputs = std::stoi(Tokens[OutputStart]); ++ ModelPtr->setNumOutputs(NumOutputs); ++ OutputStart++; ++ std::string OutputName; ++ std::string OutputType; ++ for (int I = 0; I < NumOutputs; I++) { ++ std::istringstream IS(Tokens[OutputStart + I]); ++ IS >> OutputName >> OutputType; ++ if (!registerOutput(ModelName, OutputName, OutputType)) { ++ return false; ++ } ++ } ++ std::string Signature = Tokens[OutputStart + NumOutputs]; ++ ModelPtr->setSignature(Signature); ++ return true; ++} ++ ++bool ACPOMLPythonInterface::registerModel(std::string ModelName, ++ int NumFeatures) { ++ auto Find = ModelMap.find(ModelName); ++ if (Find != ModelMap.end()) { ++ LLVM_DEBUG(dbgs() << "registerModel: Model " << ModelName ++ << " already exists\n"); ++ return false; ++ } ++ std::shared_ptr NewModel = std::make_shared(NumFeatures); ++ ModelMap.insert(std::make_pair(ModelName, NewModel)); ++ return true; ++} ++ ++bool ACPOMLPythonInterface::registerModel(std::string ModelName, ++ int NumFeatures, int NumOutputs) { ++ auto Find = ModelMap.find(ModelName); ++ if (Find != ModelMap.end()) { ++ LLVM_DEBUG(dbgs() << "registerModel: Model " << ModelName ++ << " already exists\n"); ++ return false; ++ } ++ std::shared_ptr NewModel = ++ std::make_shared(NumFeatures, NumOutputs); ++ ModelMap.insert(std::make_pair(ModelName, NewModel)); ++ return true; ++} ++ ++bool ACPOMLPythonInterface::registerFeature(std::string ModelName, ++ std::string FeatureName, ++ int Index) { ++ auto Find = ModelMap.find(ModelName); ++ assert(Find != ModelMap.end()); ++ if (Find == ModelMap.end()) { ++ LLVM_DEBUG(dbgs() << "ERROR in registerFeature: Model " << ModelName ++ << " has not been loaded\n"); ++ return false; ++ } ++ uint64_t ID = assignID(); ++ return Find->second->registerFeature(FeatureName, ID, Index); ++} ++ ++bool ACPOMLPythonInterface::registerOutput(std::string ModelName, ++ std::string OutputName, ++ std::string OutputType) { ++ auto Find = ModelMap.find(ModelName); ++ if (Find == ModelMap.end()) { ++ LLVM_DEBUG(dbgs() << "ERROR in registerOutput: Model " << ModelName ++ << " has not been loaded\n"); ++ return false; ++ } ++ return Find->second->registerOutput(OutputName, OutputType); ++} ++ ++int ACPOMLPythonInterface::getNumLoadedModels() { return ModelMap.size(); } ++ ++bool ACPOMLPythonInterface::defineInputIR(std::string Filename) { ++ return false; ++} ++ ++bool ACPOMLPythonInterface::setCustomFeature(std::string ModelName, ++ uint64_t FeatureID, ++ int FeatureValue) { ++ auto Find = ModelMap.find(ModelName); ++ if (Find == ModelMap.end()) { ++ LLVM_DEBUG(dbgs() << "ERROR in setCustomFeature: Model " << ModelName ++ << " has not been loaded\n"); ++ return false; ++ } ++ int Index = Find->second->getIndex(FeatureID); ++ sendCommand("SetCustomFeature " + std::to_string(Index) + " " + ++ std::to_string(FeatureValue)); ++ std::string Response = getResponse(); ++ return (Response.find(RESPONSE_FEATURE_SET) == 0); ++} ++ ++bool ACPOMLPythonInterface::setCustomFeature(std::string ModelName, ++ uint64_t FeatureID, ++ int64_t FeatureValue) { ++ auto Find = ModelMap.find(ModelName); ++ if (Find == ModelMap.end()) { ++ LLVM_DEBUG(dbgs() << "ERROR in setCustomFeature: Model " << ModelName ++ << " has not been loaded\n"); ++ return false; ++ } ++ int Index = Find->second->getIndex(FeatureID); ++ sendCommand("SetCustomFeature " + std::to_string(Index) + " " + ++ std::to_string(FeatureValue)); ++ std::string Response = getResponse(); ++ return (Response.find(RESPONSE_FEATURE_SET) == 0); ++} ++ ++bool ACPOMLPythonInterface::setCustomFeature(std::string ModelName, ++ uint64_t FeatureID, ++ double FeatureValue) { ++ auto Find = ModelMap.find(ModelName); ++ if (Find == ModelMap.end()) { ++ LLVM_DEBUG(dbgs() << "ERROR in setCustomFeature: Model " << ModelName ++ << " has not been loaded\n"); ++ return false; ++ } ++ int Index = Find->second->getIndex(FeatureID); ++ sendCommand("SetCustomFeature " + std::to_string(Index) + " " + ++ std::to_string(FeatureValue)); ++ std::string Response = getResponse(); ++ return (Response.find(RESPONSE_FEATURE_SET) == 0); ++} ++ ++bool ACPOMLPythonInterface::setCustomFeature(std::string ModelName, ++ uint64_t FeatureID, ++ float FeatureValue) { ++ auto Find = ModelMap.find(ModelName); ++ if (Find == ModelMap.end()) { ++ LLVM_DEBUG(dbgs() << "ERROR in setCustomFeature: Model " << ModelName ++ << " has not been loaded\n"); ++ return false; ++ } ++ int Index = Find->second->getIndex(FeatureID); ++ sendCommand("SetCustomFeature " + std::to_string(Index) + " " + ++ std::to_string(FeatureValue)); ++ std::string Response = getResponse(); ++ return (Response.find(RESPONSE_FEATURE_SET) == 0); ++} ++ ++bool ACPOMLPythonInterface::setCustomFeature(std::string ModelName, ++ uint64_t FeatureID, ++ bool FeatureValue) { ++ auto Find = ModelMap.find(ModelName); ++ if (Find == ModelMap.end()) { ++ LLVM_DEBUG(dbgs() << "ERROR in setCustomFeature: Model " << ModelName ++ << " has not been loaded\n"); ++ return false; ++ } ++ int Index = Find->second->getIndex(FeatureID); ++ std::string Command = "SetCustomFeature " + std::to_string(Index) + " "; ++ Command += FeatureValue ? "1" : "0"; ++ sendCommand(Command); ++ std::string Response = getResponse(); ++ return (Response.find(RESPONSE_FEATURE_SET) == 0); ++} ++ ++bool ACPOMLPythonInterface::setCustomFeature(std::string ModelName, ++ std::string FeatureName, ++ int FeatureValue) { ++ auto Find = ModelMap.find(ModelName); ++ if (Find == ModelMap.end()) { ++ LLVM_DEBUG(dbgs() << "ERROR in setCustomFeature: Model " << ModelName ++ << " has not been loaded\n"); ++ return false; ++ } ++ int Index = Find->second->getIndex(FeatureName); ++ sendCommand("SetCustomFeature " + std::to_string(Index) + " " + ++ std::to_string(FeatureValue)); ++ std::string Response = getResponse(); ++ return (Response.find(RESPONSE_FEATURE_SET) == 0); ++} ++ ++bool ACPOMLPythonInterface::setCustomFeature(std::string ModelName, ++ std::string FeatureName, ++ int64_t FeatureValue) { ++ auto Find = ModelMap.find(ModelName); ++ if (Find == ModelMap.end()) { ++ LLVM_DEBUG(dbgs() << "ERROR in setCustomFeature: Model " << ModelName ++ << " has not been loaded\n"); ++ return false; ++ } ++ int Index = Find->second->getIndex(FeatureName); ++ sendCommand("SetCustomFeature " + std::to_string(Index) + " " + ++ std::to_string(FeatureValue)); ++ std::string Response = getResponse(); ++ return (Response.find(RESPONSE_FEATURE_SET) == 0); ++} ++ ++bool ACPOMLPythonInterface::setCustomFeature(std::string ModelName, ++ std::string FeatureName, ++ double FeatureValue) { ++ auto Find = ModelMap.find(ModelName); ++ if (Find == ModelMap.end()) { ++ LLVM_DEBUG(dbgs() << "ERROR in setCustomFeature: Model " << ModelName ++ << " has not been loaded\n"); ++ return false; ++ } ++ int Index = Find->second->getIndex(FeatureName); ++ sendCommand("SetCustomFeature " + std::to_string(Index) + " " + ++ std::to_string(FeatureValue)); ++ std::string Response = getResponse(); ++ return (Response.find(RESPONSE_FEATURE_SET) == 0); ++} ++ ++bool ACPOMLPythonInterface::setCustomFeature(std::string ModelName, ++ std::string FeatureName, ++ float FeatureValue) { ++ auto Find = ModelMap.find(ModelName); ++ if (Find == ModelMap.end()) { ++ LLVM_DEBUG(dbgs() << "ERROR in setCustomFeature: Model " << ModelName ++ << " has not been loaded\n"); ++ return false; ++ } ++ int Index = Find->second->getIndex(FeatureName); ++ sendCommand("SetCustomFeature " + std::to_string(Index) + " " + ++ std::to_string(FeatureValue)); ++ std::string Response = getResponse(); ++ return (Response.find(RESPONSE_FEATURE_SET) == 0); ++} ++ ++bool ACPOMLPythonInterface::setCustomFeature(std::string ModelName, ++ std::string FeatureName, ++ bool FeatureValue) { ++ auto Find = ModelMap.find(ModelName); ++ if (Find == ModelMap.end()) { ++ LLVM_DEBUG(dbgs() << "ERROR in setCustomFeature: Model " << ModelName ++ << " has not been loaded\n"); ++ return false; ++ } ++ int Index = Find->second->getIndex(FeatureName); ++ std::string Command = "SetCustomFeature " + std::to_string(Index) + " "; ++ Command += FeatureValue ? "1" : "0"; ++ sendCommand(Command); ++ std::string Response = getResponse(); ++ std::vector Tokens = tokenize(Response); ++ return (Response.find(RESPONSE_FEATURE_SET) == 0); ++} ++ ++bool ACPOMLPythonInterface::initializeFeatures( ++ std::string ModelName, ++ const std::vector> &FeatureValues) { ++ auto Find = ModelMap.find(ModelName); ++ if (Find == ModelMap.end()) { ++ LLVM_DEBUG(dbgs() << "ERROR in initializeFeatures: Model " << ModelName ++ << " has not been loaded\n"); ++ return false; ++ } ++ if (FeatureValues.size() > Find->second->getNumFeatures()) { ++ LLVM_DEBUG(dbgs() << "ERROR in initializeFeatures: Invalid features\n"); ++ return false; ++ } ++ CurrentlyActiveModel = ModelName; ++ std::string Command = "InitializeFeatures " + ModelName; ++ for (const auto &Feature : FeatureValues) { ++ uint64_t FeatureID = Feature.first; ++ std::string FeatureValue = Feature.second; ++ int Index = Find->second->getIndex(FeatureID); ++ Command += " " + std::to_string(Index) + " " + FeatureValue; ++ } ++ sendCommand(Command); ++ std::string Response = getResponse(); ++ return (Response.find(RESPONSE_FEATURES_INITIALIZED) == 0); ++} ++ ++bool ACPOMLPythonInterface::initializeFeatures( ++ std::string ModelName, ++ const std::vector> &FeatureValues) { ++ auto Find = ModelMap.find(ModelName); ++ if (Find == ModelMap.end()) { ++ LLVM_DEBUG(dbgs() << "ERROR in initializeFeatures: Model " << ModelName ++ << " has not been loaded\n"); ++ return false; ++ } ++ if (FeatureValues.size() > Find->second->getNumFeatures()) { ++ LLVM_DEBUG(dbgs() << "ERROR in initializeFeatures: Invalid features\n"); ++ return false; ++ } ++ CurrentlyActiveModel = ModelName; ++ std::string Command = "InitializeFeatures " + ModelName; ++ for (const auto &Feature : FeatureValues) { ++ std::string FeatureName = Feature.first; ++ std::string FeatureValue = Feature.second; ++ int Index = Find->second->getIndex(FeatureName); ++ Command += " " + std::to_string(Index) + " " + FeatureValue; ++ } ++ sendCommand(Command); ++ std::string Response = getResponse(); ++ return (Response.find(RESPONSE_FEATURES_INITIALIZED) == 0); ++} ++ ++bool ACPOMLPythonInterface::setCustomFeatures( ++ std::string ModelName, ++ const std::vector> &FeatureValues) { ++ if (ModelName != CurrentlyActiveModel) { ++ LLVM_DEBUG(dbgs() << "ERROR in setCustomFeatures: Model " << ModelName ++ << " has not been loaded or is not active\n"); ++ return false; ++ } ++ auto Find = ModelMap.find(ModelName); ++ if (FeatureValues.size() > Find->second->getNumFeatures()) { ++ LLVM_DEBUG(dbgs() << "ERROR in setCustomFeatures: Invalid features\n"); ++ return false; ++ } ++ std::string Command = "SetCustomFeatures"; ++ for (const auto &Feature : FeatureValues) { ++ uint64_t FeatureID = Feature.first; ++ std::string FeatureValue = Feature.second; ++ int Index = Find->second->getIndex(FeatureID); ++ Command += " " + std::to_string(Index) + " " + FeatureValue; ++ } ++ sendCommand(Command); ++ std::string Response = getResponse(); ++ return (Response.find(RESPONSE_FEATURES_SET) == 0); ++} ++ ++bool ACPOMLPythonInterface::setCustomFeatures( ++ std::string ModelName, ++ const std::vector> &FeatureValues) { ++ if (ModelName != CurrentlyActiveModel) { ++ LLVM_DEBUG(dbgs() << "ERROR in setCustomFeatures: Model " << ModelName ++ << " has not been loaded or is not active\n"); ++ return false; ++ } ++ auto Find = ModelMap.find(ModelName); ++ if (FeatureValues.size() > Find->second->getNumFeatures()) { ++ LLVM_DEBUG(dbgs() << "ERROR in setCustomFeatures: Invalid features\n"); ++ return false; ++ } ++ std::string Command = "SetCustomFeatures"; ++ for (const auto &Feature : FeatureValues) { ++ std::string FeatureName = Feature.first; ++ std::string FeatureValue = Feature.second; ++ int Index = Find->second->getIndex(FeatureName); ++ Command += " " + std::to_string(Index) + " " + FeatureValue; ++ } ++ sendCommand(Command); ++ std::string Response = getResponse(); ++ return (Response.find(RESPONSE_FEATURES_SET) == 0); ++} ++ ++bool ACPOMLPythonInterface::runModel(std::string ModelName) { ++ if (ModelName != CurrentlyActiveModel) { ++ LLVM_DEBUG(dbgs() << "ERROR in runModel: Model " << ModelName ++ << " is not active\n"); ++ return false; ++ } ++ sendCommand("RunModel"); ++ std::string Response = getResponse(); ++ return (Response.find(RESPONSE_COMPLETED) == 0); ++} ++ ++std::string ACPOMLPythonInterface::getOutputType(std::string ModelName, ++ std::string OutputName) { ++ auto Find = ModelMap.find(ModelName); ++ assert(Find != ModelMap.end()); ++ return Find->second->getOutputType(OutputName); ++} ++ ++int ACPOMLPythonInterface::getModelResultI(std::string OutputName) { ++ auto Find = ModelMap.find(CurrentlyActiveModel); ++ assert(Find->second->checkOutputExists(OutputName)); ++ sendCommand("GetModelOutput " + OutputName); ++ std::string Response = getResponse(); ++ std::vector Tokens = tokenize(Response); ++ assert(Tokens.size() == 3); ++ assert(Tokens[0] == OutputName); ++ int Result = std::stoi(Tokens[2]); ++ return Result; ++} ++ ++int64_t ACPOMLPythonInterface::getModelResultI64(std::string OutputName) { ++ auto Find = ModelMap.find(CurrentlyActiveModel); ++ assert(Find->second->checkOutputExists(OutputName)); ++ sendCommand("GetModelOutput " + OutputName); ++ std::string Response = getResponse(); ++ std::vector Tokens = tokenize(Response); ++ assert(Tokens.size() == 3); ++ assert(Tokens[0] == OutputName); ++ int64_t Result = std::stol(Tokens[2]); ++ return Result; ++} ++ ++float ACPOMLPythonInterface::getModelResultF(std::string OutputName) { ++ auto Find = ModelMap.find(CurrentlyActiveModel); ++ assert(Find->second->checkOutputExists(OutputName)); ++ sendCommand("GetModelOutput " + OutputName); ++ std::string Response = getResponse(); ++ std::vector Tokens = tokenize(Response); ++ assert(Tokens.size() == 3); ++ assert(Tokens[0] == OutputName); ++ float Result = std::stof(Tokens[2]); ++ return Result; ++} ++ ++double ACPOMLPythonInterface::getModelResultD(std::string OutputName) { ++ auto Find = ModelMap.find(CurrentlyActiveModel); ++ assert(Find->second->checkOutputExists(OutputName)); ++ sendCommand("GetModelOutput " + OutputName); ++ std::string Response = getResponse(); ++ std::vector Tokens = tokenize(Response); ++ assert(Tokens.size() == 3); ++ assert(Tokens[0] == OutputName); ++ double Result = std::stod(Tokens[2]); ++ return Result; ++} ++ ++bool ACPOMLPythonInterface::getModelResultB(std::string OutputName) { ++ auto Find = ModelMap.find(CurrentlyActiveModel); ++ assert(Find->second->checkOutputExists(OutputName)); ++ sendCommand("GetModelOutput " + OutputName); ++ std::string Response = getResponse(); ++ std::vector Tokens = tokenize(Response); ++ assert(Tokens.size() == 3); ++ assert(Tokens[0] == OutputName); ++ return (Tokens[2] == "1"); ++} ++ ++int ACPOMLPythonInterface::getStatus() { ++ sendCommand("GetStatus"); ++ std::string Response = getResponse(); ++ return Response.find(RESPONSE_ACTIVE) == 0; ++} ++ ++bool ACPOMLPythonInterface::releaseModel(std::string ModelName) { ++ sendCommand("ReleaseModel " + ModelName); ++ std::string Response = getResponse(); ++ ModelMap.erase(ModelName); ++ CurrentlyActiveModel = ""; ++ return true; ++} ++ ++bool ACPOMLPythonInterface::closeMLInterface() { ++ sendCommand("CloseMLInterface"); ++ std::string Response = getResponse(); ++ return true; ++} ++ ++void ACPOMLPythonInterface::sendCommand(const std::string &Command) { ++ fprintf(PipeOut,"%s\n", Command.c_str()); ++ fflush(PipeOut); ++ usleep(1); ++} ++ ++void ACPOMLPythonInterface::sendCommand( ++ const std::vector &Features) { ++ for (auto I = Features.begin(); I != Features.end(); I++) { ++ fprintf(PipeOut,"%s\n", I->c_str()); ++ fflush(PipeOut); ++ usleep(1); ++ } ++} ++ ++std::string ACPOMLPythonInterface::getResponse() { ++ std::string Response = ""; ++ char Letter = getc(PipeIn); ++ while (Letter != '\n') { ++ if (feof(PipeIn)) ++ assert(false && "ACPO pipeline is closed unexpectively."); ++ ++ Response += Letter; ++ Letter = getc(PipeIn); ++ } ++ Response += '\n'; ++ if (Response.substr(0, 5) == RESPONSE_ERROR) { ++ LLVM_DEBUG(dbgs() << Response); ++ assert(false && "MLInterface reutrned error"); ++ } ++ return Response; ++} ++ ++std::vector ++ACPOMLPythonInterface::tokenize(const std::string &Line) { ++ std::vector Result; ++ std::string Temp = Line; ++ auto Loc = Temp.find(","); ++ while (Loc != std::string::npos) { ++ std::string Sub = Temp.substr(0, Loc); ++ Result.push_back(Sub); ++ Temp = Temp.substr(Loc + 1); ++ Loc = Temp.find(","); ++ } ++ if (Temp.length() > 0) ++ Result.push_back(Temp); ++ ++ return Result; ++} ++ ++std::shared_ptr llvm::createPersistentPythonMLIF() { ++ if (PersistentMLIF == nullptr) { ++ PersistentMLIF = std::make_shared(); ++ ++ if (!PersistentMLIF->isInitialized()) ++ PersistentMLIF = nullptr; ++ } ++ return PersistentMLIF; ++} ++ ++ACPOMLCPPInterface::ACPOMLCPPInterface() { setInitialized(true); } ++ ++ACPOMLCPPInterface::~ACPOMLCPPInterface() {} ++ ++uint64_t ACPOMLCPPInterface::assignID() { ++ NextID++; ++ return NextID - 1; ++} ++ ++bool ACPOMLCPPInterface::loadModel(std::string ModelSpecFile) { ++ std::string ModelName = readModelParam(ModelSpecFile, "ModelName"); ++ // Check if the model is already in the dictionary ++ if (RunnerMap.find(ModelName) != RunnerMap.end()) { ++ LLVM_DEBUG(dbgs() << "loadModel: the compiled model '" << ModelName ++ << "' has already been loaded\n"); ++ return true; ++ } ++ std::vector> Features{}; ++ readFeatures(ModelSpecFile, Features); ++ std::vector> Outputs{}; ++ readOutputs(ModelSpecFile, Outputs); ++ ++ LLVM_DEBUG(llvm::dbgs() << "Loading compiled model with name " << ModelName ++ << "\n"); ++ ++ auto CreatorFunctionIterator = CreateModelRunnerMap.find(ModelName); ++ if (CreatorFunctionIterator == CreateModelRunnerMap.end()) { ++ LLVM_DEBUG(llvm::dbgs() ++ << ("Could not find compiled model class for model '" + ++ ModelName + "'\n")); ++ return false; ++ } ++ ++ auto CreatorFunction = CreatorFunctionIterator->second; ++ ++ std::string OutputKey = readModelParam(ModelSpecFile, "OutputKey"); ++ auto ModelRunner = CreatorFunction(Features, OutputKey); ++ ++ registerModel(ModelName, Features.size()); ++ RunnerMap.insert(std::make_pair(ModelName, std::move(ModelRunner))); ++ auto ModelPtr = ModelMap.find(ModelName)->second; ++ for (size_t I = 0; I < Features.size(); I++) { ++ if (!registerFeature(ModelName, Features[I].first, I)) { ++ return false; ++ } ++ if (!ModelPtr->registerInput(Features[I].first, Features[I].second)) { ++ return false; ++ } ++ } ++ ++ ModelPtr->setNumOutputs(Outputs.size()); ++ for (size_t I = 0; I < Outputs.size(); I++) { ++ if (!registerOutput(ModelName, Outputs[I].first, Outputs[I].second)) { ++ return false; ++ } ++ } ++ ++ LLVM_DEBUG(llvm::dbgs() << "Model " << ModelName ++ << " was successfully loaded\n"); ++ ++ // We do not need to set signature here because it is already given to make ++ // the precompiled model ++ return true; ++} ++ ++bool ACPOMLCPPInterface::registerModel(std::string ModelName, int NumFeatures) { ++ auto Find = ModelMap.find(ModelName); ++ if (Find != ModelMap.end()) { ++ LLVM_DEBUG(dbgs() << "registerModel: Model " << ModelName ++ << " already exists\n"); ++ return false; ++ } ++ std::shared_ptr NewModel = std::make_shared(NumFeatures); ++ ModelMap.insert(std::make_pair(ModelName, NewModel)); ++ return true; ++} ++ ++bool ACPOMLCPPInterface::registerModel(std::string ModelName, int NumFeatures, ++ int NumOutputs) { ++ auto Find = ModelMap.find(ModelName); ++ if (Find != ModelMap.end()) { ++ LLVM_DEBUG(dbgs() << "registerModel: Model " << ModelName ++ << " already exists\n"); ++ return false; ++ } ++ std::shared_ptr NewModel = ++ std::make_shared(NumFeatures, NumOutputs); ++ ModelMap.insert(std::make_pair(ModelName, NewModel)); ++ return true; ++} ++ ++bool ACPOMLCPPInterface::registerFeature(std::string ModelName, ++ std::string FeatureName, int Index) { ++ auto Find = ModelMap.find(ModelName); ++ assert(Find != ModelMap.end()); ++ if (Find == ModelMap.end()) { ++ LLVM_DEBUG(dbgs() << "ERROR in registerFeature: Model " << ModelName ++ << " has not been loaded\n"); ++ return false; ++ } ++ uint64_t ID = assignID(); ++ return Find->second->registerFeature(FeatureName, ID, Index); ++} ++ ++bool ACPOMLCPPInterface::registerOutput(std::string ModelName, ++ std::string OutputName, ++ std::string OutputType) { ++ auto Find = ModelMap.find(ModelName); ++ if (Find == ModelMap.end()) { ++ LLVM_DEBUG(dbgs() << "ERROR in registerOutput: Model " << ModelName ++ << " has not been loaded\n"); ++ return false; ++ } ++ return Find->second->registerOutput(OutputName, OutputType); ++} ++ ++int ACPOMLCPPInterface::getNumLoadedModels() { return ModelMap.size(); } ++ ++bool ACPOMLCPPInterface::defineInputIR(std::string Filename) { return false; } ++ ++bool ACPOMLCPPInterface::setCustomFeature(std::string ModelName, ++ uint64_t FeatureID, ++ int FeatureValue) { ++ LLVM_DEBUG( ++ dbgs() ++ << "ACPOMLCPPInterface: setting custom feature of type int in model " ++ << ModelName << "\n"); ++ auto Find = ModelMap.find(ModelName); ++ if (Find == ModelMap.end()) { ++ LLVM_DEBUG(dbgs() << "ERROR in setCustomFeature: Model " << ModelName ++ << " has not been loaded\n"); ++ return false; ++ } ++ int Index = Find->second->getIndex(FeatureID); ++ std::shared_ptr Runner = ++ RunnerMap.find(ModelName)->second; ++ return Runner->setCustomFeature(Index, FeatureValue); ++} ++ ++bool ACPOMLCPPInterface::setCustomFeature(std::string ModelName, ++ uint64_t FeatureID, ++ int64_t FeatureValue) { ++ LLVM_DEBUG( ++ dbgs() ++ << "ACPOMLCPPInterface: setting custom feature of type double in model " ++ << ModelName << "\n"); ++ auto Find = ModelMap.find(ModelName); ++ if (Find == ModelMap.end()) { ++ LLVM_DEBUG(dbgs() << "ERROR in setCustomFeature: Model " << ModelName ++ << " has not been loaded\n"); ++ return false; ++ } ++ int Index = Find->second->getIndex(FeatureID); ++ std::shared_ptr Runner = ++ RunnerMap.find(ModelName)->second; ++ return Runner->setCustomFeature(Index, FeatureValue); ++} ++ ++bool ACPOMLCPPInterface::setCustomFeature(std::string ModelName, ++ uint64_t FeatureID, ++ double FeatureValue) { ++ LLVM_DEBUG( ++ dbgs() ++ << "ACPOMLCPPInterface: setting custom feature of type double in model " ++ << ModelName << "\n"); ++ auto Find = ModelMap.find(ModelName); ++ if (Find == ModelMap.end()) { ++ LLVM_DEBUG(dbgs() << "ERROR in setCustomFeature: Model " << ModelName ++ << " has not been loaded\n"); ++ return false; ++ } ++ int Index = Find->second->getIndex(FeatureID); ++ std::shared_ptr Runner = ++ RunnerMap.find(ModelName)->second; ++ return Runner->setCustomFeature(Index, FeatureValue); ++} ++ ++bool ACPOMLCPPInterface::setCustomFeature(std::string ModelName, ++ uint64_t FeatureID, ++ float FeatureValue) { ++ LLVM_DEBUG( ++ dbgs() ++ << "ACPOMLCPPInterface: setting custom feature of type float in model " ++ << ModelName << "\n"); ++ auto Find = ModelMap.find(ModelName); ++ if (Find == ModelMap.end()) { ++ LLVM_DEBUG(dbgs() << "ERROR in setCustomFeature: Model " << ModelName ++ << " has not been loaded\n"); ++ return false; ++ } ++ int Index = Find->second->getIndex(FeatureID); ++ std::shared_ptr Runner = ++ RunnerMap.find(ModelName)->second; ++ return Runner->setCustomFeature(Index, FeatureValue); ++} ++ ++bool ACPOMLCPPInterface::setCustomFeature(std::string ModelName, ++ uint64_t FeatureID, ++ bool FeatureValue) { ++ LLVM_DEBUG( ++ dbgs() ++ << "ACPOMLCPPInterface: setting custom feature of type bool in model " ++ << ModelName << "\n"); ++ auto Find = ModelMap.find(ModelName); ++ if (Find == ModelMap.end()) { ++ LLVM_DEBUG(dbgs() << "ERROR in setCustomFeature: Model " << ModelName ++ << " has not been loaded\n"); ++ return false; ++ } ++ int Index = Find->second->getIndex(FeatureID); ++ std::shared_ptr Runner = ++ RunnerMap.find(ModelName)->second; ++ return Runner->setCustomFeature(Index, FeatureValue); ++} ++ ++bool ACPOMLCPPInterface::setCustomFeature(std::string ModelName, ++ std::string FeatureName, ++ int FeatureValue) { ++ LLVM_DEBUG( ++ dbgs() ++ << "ACPOMLCPPInterface: setting custom feature of type int in model " ++ << ModelName << "\n"); ++ auto Find = ModelMap.find(ModelName); ++ if (Find == ModelMap.end()) { ++ LLVM_DEBUG(dbgs() << "ERROR in setCustomFeature: Model " << ModelName ++ << " has not been loaded\n"); ++ return false; ++ } ++ int Index = Find->second->getIndex(FeatureName); ++ std::shared_ptr Runner = ++ RunnerMap.find(ModelName)->second; ++ return Runner->setCustomFeature(Index, FeatureValue); ++} ++ ++bool ACPOMLCPPInterface::setCustomFeature(std::string ModelName, ++ std::string FeatureName, ++ int64_t FeatureValue) { ++ LLVM_DEBUG( ++ dbgs() ++ << "ACPOMLCPPInterface: setting custom feature of type int64 in model " ++ << ModelName << "\n"); ++ auto Find = ModelMap.find(ModelName); ++ if (Find == ModelMap.end()) { ++ LLVM_DEBUG(dbgs() << "ERROR in setCustomFeature: Model " << ModelName ++ << " has not been loaded\n"); ++ return false; ++ } ++ int Index = Find->second->getIndex(FeatureName); ++ std::shared_ptr Runner = ++ RunnerMap.find(ModelName)->second; ++ return Runner->setCustomFeature(Index, FeatureValue); ++} ++ ++bool ACPOMLCPPInterface::setCustomFeature(std::string ModelName, ++ std::string FeatureName, ++ double FeatureValue) { ++ LLVM_DEBUG( ++ dbgs() ++ << "ACPOMLCPPInterface: setting custom feature of type double in model " ++ << ModelName << "\n"); ++ auto Find = ModelMap.find(ModelName); ++ if (Find == ModelMap.end()) { ++ LLVM_DEBUG(dbgs() << "ERROR in setCustomFeature: Model " << ModelName ++ << " has not been loaded\n"); ++ return false; ++ } ++ int Index = Find->second->getIndex(FeatureName); ++ std::shared_ptr Runner = ++ RunnerMap.find(ModelName)->second; ++ return Runner->setCustomFeature(Index, FeatureValue); ++} ++ ++bool ACPOMLCPPInterface::setCustomFeature(std::string ModelName, ++ std::string FeatureName, ++ float FeatureValue) { ++ LLVM_DEBUG( ++ dbgs() ++ << "ACPOMLCPPInterface: setting custom feature of type float in model " ++ << ModelName << "\n"); ++ auto Find = ModelMap.find(ModelName); ++ if (Find == ModelMap.end()) { ++ LLVM_DEBUG(dbgs() << "ERROR in setCustomFeature: Model " << ModelName ++ << " has not been loaded\n"); ++ return false; ++ } ++ int Index = Find->second->getIndex(FeatureName); ++ std::shared_ptr Runner = ++ RunnerMap.find(ModelName)->second; ++ return Runner->setCustomFeature(Index, FeatureValue); ++} ++ ++bool ACPOMLCPPInterface::setCustomFeature(std::string ModelName, ++ std::string FeatureName, ++ bool FeatureValue) { ++ LLVM_DEBUG( ++ dbgs() ++ << "ACPOMLCPPInterface: setting custom feature of type bool in model " ++ << ModelName << "\n"); ++ auto Find = ModelMap.find(ModelName); ++ if (Find == ModelMap.end()) { ++ LLVM_DEBUG(dbgs() << "ERROR in setCustomFeature: Model " << ModelName ++ << " has not been loaded\n"); ++ return false; ++ } ++ int Index = Find->second->getIndex(FeatureName); ++ std::shared_ptr Runner = ++ RunnerMap.find(ModelName)->second; ++ return Runner->setCustomFeature(Index, FeatureValue); ++} ++ ++bool ACPOMLCPPInterface::initializeFeatures( ++ std::string ModelName, ++ const std::vector> &FeatureValues) { ++ LLVM_DEBUG(dbgs() << "Initializing features for model " << ModelName ++ << " using feature IDs\n"); ++ auto Find = ModelMap.find(ModelName); ++ if (Find == ModelMap.end()) { ++ LLVM_DEBUG(dbgs() << "ERROR in initializeFeatures: Model " << ModelName ++ << " has not been loaded\n"); ++ return false; ++ } ++ if (FeatureValues.size() > Find->second->getNumFeatures()) { ++ LLVM_DEBUG(dbgs() << "ERROR in initializeFeatures: Invalid features\n"); ++ return false; ++ } ++ CurrentlyActiveModel = ModelName; ++ for (const auto &Feature : FeatureValues) { ++ uint64_t FeatureID = Feature.first; ++ std::string FeatureValue = Feature.second; ++ ++ std::string FeatureType = ++ getInputType(ModelName, Find->second->getName(FeatureID)); ++ if (FeatureType == "int64") { ++ int64_t Value = std::stoi(FeatureValue); ++ setCustomFeature(ModelName, FeatureID, Value); ++ } else if (FeatureType == "int32") { ++ int32_t Value = std::stoi(FeatureValue); ++ setCustomFeature(ModelName, FeatureID, Value); ++ } else if (FeatureType == "int") { ++ int Value = std::stoi(FeatureValue); ++ setCustomFeature(ModelName, FeatureID, Value); ++ } else if (FeatureType == "float64") { ++ double Value = std::stod(FeatureValue); ++ setCustomFeature(ModelName, FeatureID, Value); ++ } else if (FeatureType == "float32") { ++ float Value = std::stof(FeatureValue); ++ setCustomFeature(ModelName, FeatureID, Value); ++ } else { ++ LLVM_DEBUG(dbgs() << "ERROR in initializeFeatures: Invalid feature type " ++ << FeatureType << "\n"); ++ return false; ++ } ++ } ++ return true; ++} ++ ++bool ACPOMLCPPInterface::initializeFeatures( ++ std::string ModelName, ++ const std::vector> &FeatureValues) { ++ auto Find = ModelMap.find(ModelName); ++ LLVM_DEBUG(dbgs() << "Initializing features for model " << ModelName ++ << " using feature names\n"); ++ if (Find == ModelMap.end()) { ++ LLVM_DEBUG(dbgs() << "ERROR in initializeFeatures: Model " << ModelName ++ << " has not been loaded\n"); ++ return false; ++ } ++ if (FeatureValues.size() > Find->second->getNumFeatures()) { ++ LLVM_DEBUG(dbgs() << "ERROR in initializeFeatures: Invalid features\n"); ++ return false; ++ } ++ CurrentlyActiveModel = ModelName; ++ for (const auto &Feature : FeatureValues) { ++ std::string FeatureName = Feature.first; ++ std::string FeatureValue = Feature.second; ++ ++ std::string FeatureType = getInputType(ModelName, FeatureName); ++ if (FeatureType == "int64") { ++ int64_t Value = std::stol(FeatureValue); ++ setCustomFeature(ModelName, FeatureName, Value); ++ } else if (FeatureType == "int32") { ++ int32_t Value = std::stoi(FeatureValue); ++ setCustomFeature(ModelName, FeatureName, Value); ++ } else if (FeatureType == "int") { ++ int Value = std::stoi(FeatureValue); ++ setCustomFeature(ModelName, FeatureName, Value); ++ } else if (FeatureType == "float64") { ++ double Value = std::stod(FeatureValue); ++ setCustomFeature(ModelName, FeatureName, Value); ++ } else if (FeatureType == "float32") { ++ float Value = std::stof(FeatureValue); ++ setCustomFeature(ModelName, FeatureName, Value); ++ } else { ++ LLVM_DEBUG(dbgs() << "ERROR in initializeFeatures: Invalid feature type " ++ << FeatureType << "\n"); ++ return false; ++ } ++ } ++ return true; ++} ++ ++bool ACPOMLCPPInterface::setCustomFeatures( ++ std::string ModelName, ++ const std::vector> &FeatureValues) { ++ if (ModelName != CurrentlyActiveModel) { ++ LLVM_DEBUG(dbgs() << "ERROR in setCustomFeatures: Model " << ModelName ++ << " has not been loaded or is not active\n"); ++ return false; ++ } ++ auto Find = ModelMap.find(ModelName); ++ if (FeatureValues.size() > Find->second->getNumFeatures()) { ++ LLVM_DEBUG(dbgs() << "ERROR in setCustomFeatures: Invalid features\n"); ++ return false; ++ } ++ std::string Command = "SetCustomFeatures"; ++ for (const auto &Feature : FeatureValues) { ++ uint64_t FeatureID = Feature.first; ++ std::string FeatureValue = Feature.second; ++ ++ std::string FeatureType = ++ getInputType(ModelName, Find->second->getName(FeatureID)); ++ if (FeatureType == "int64") { ++ int64_t Value = std::stol(FeatureValue); ++ setCustomFeature(ModelName, FeatureID, Value); ++ } else if (FeatureType == "int32") { ++ int32_t Value = std::stoi(FeatureValue); ++ setCustomFeature(ModelName, FeatureID, Value); ++ } else if (FeatureType == "int") { ++ int Value = std::stoi(FeatureValue); ++ setCustomFeature(ModelName, FeatureID, Value); ++ } else if (FeatureType == "float64") { ++ double Value = std::stod(FeatureValue); ++ setCustomFeature(ModelName, FeatureID, Value); ++ } else if (FeatureType == "float32") { ++ float Value = std::stof(FeatureValue); ++ setCustomFeature(ModelName, FeatureID, Value); ++ } else { ++ LLVM_DEBUG(dbgs() << "ERROR in setCustomFeatures: Invalid feature type " ++ << FeatureType << "\n"); ++ return false; ++ } ++ } ++ return true; ++} ++ ++bool ACPOMLCPPInterface::setCustomFeatures( ++ std::string ModelName, ++ const std::vector> &FeatureValues) { ++ if (ModelName != CurrentlyActiveModel) { ++ LLVM_DEBUG(dbgs() << "ERROR in setCustomFeatures: Model " << ModelName ++ << " has not been loaded or is not active\n"); ++ return false; ++ } ++ auto Find = ModelMap.find(ModelName); ++ if (FeatureValues.size() > Find->second->getNumFeatures()) { ++ LLVM_DEBUG(dbgs() << "ERROR in setCustomFeatures: Invalid features\n"); ++ return false; ++ } ++ std::string Command = "SetCustomFeatures"; ++ for (const auto &Feature : FeatureValues) { ++ std::string FeatureName = Feature.first; ++ std::string FeatureValueStr = Feature.second; ++ ++ std::string FeatureType = getInputType(ModelName, FeatureName); ++ if (FeatureType == "int64") { ++ int64_t FeatureValue = std::stoi(FeatureValueStr); ++ setCustomFeature(ModelName, FeatureName, FeatureValue); ++ } else if (FeatureType == "int32") { ++ int32_t FeatureValue = std::stoi(FeatureValueStr); ++ setCustomFeature(ModelName, FeatureName, FeatureValue); ++ } else if (FeatureType == "int") { ++ int FeatureValue = std::stoi(FeatureValueStr); ++ setCustomFeature(ModelName, FeatureName, FeatureValue); ++ } else if (FeatureType == "float64") { ++ double FeatureValue = std::stod(FeatureValueStr); ++ setCustomFeature(ModelName, FeatureName, FeatureValue); ++ } else if (FeatureType == "float32") { ++ float FeatureValue = std::stof(FeatureValueStr); ++ setCustomFeature(ModelName, FeatureName, FeatureValue); ++ } else { ++ LLVM_DEBUG(dbgs() << "ERROR in setCustomFeatures: Invalid feature type " ++ << FeatureType << "\n"); ++ return false; ++ } ++ } ++ return true; ++} ++ ++bool ACPOMLCPPInterface::runModel(std::string ModelName) { ++ if (ModelName != CurrentlyActiveModel) { ++ LLVM_DEBUG(dbgs() << "ERROR in runModel: Model " << ModelName ++ << " is not active\n"); ++ return false; ++ } ++ std::shared_ptr Runner = ++ RunnerMap.find(CurrentlyActiveModel)->second; ++ return Runner->runModel(); ++} ++ ++std::string ACPOMLCPPInterface::getInputType(std::string ModelName, ++ std::string InputName) { ++ auto Find = ModelMap.find(ModelName); ++ assert(Find != ModelMap.end()); ++ return Find->second->getInputType(InputName); ++} ++ ++std::string ACPOMLCPPInterface::getOutputType(std::string ModelName, ++ std::string OutputName) { ++ auto Find = ModelMap.find(ModelName); ++ assert(Find != ModelMap.end()); ++ return Find->second->getOutputType(OutputName); ++} ++ ++int ACPOMLCPPInterface::getModelResultI(std::string OutputName) { ++ std::shared_ptr Runner = ++ RunnerMap.find(CurrentlyActiveModel)->second; ++ return Runner->getModelResultI(OutputName); ++} ++ ++int64_t ACPOMLCPPInterface::getModelResultI64(std::string OutputName) { ++ std::shared_ptr Runner = ++ RunnerMap.find(CurrentlyActiveModel)->second; ++ return Runner->getModelResultI64(OutputName); ++} ++ ++float ACPOMLCPPInterface::getModelResultF(std::string OutputName) { ++ std::shared_ptr Runner = ++ RunnerMap.find(CurrentlyActiveModel)->second; ++ return Runner->getModelResultF(OutputName); ++} ++ ++double ACPOMLCPPInterface::getModelResultD(std::string OutputName) { ++ std::shared_ptr Runner = ++ RunnerMap.find(CurrentlyActiveModel)->second; ++ return Runner->getModelResultD(OutputName); ++} ++ ++bool ACPOMLCPPInterface::getModelResultB(std::string OutputName) { ++ std::shared_ptr Runner = ++ RunnerMap.find(CurrentlyActiveModel)->second; ++ return Runner->getModelResultB(OutputName); ++} ++ ++int ACPOMLCPPInterface::getStatus() { return 1; } ++ ++bool ACPOMLCPPInterface::releaseModel(std::string ModelName) { ++ ModelMap.erase(ModelName); ++ RunnerMap.erase(ModelName); ++ CurrentlyActiveModel = ""; ++ return true; ++} ++ ++bool ACPOMLCPPInterface::closeMLInterface() { return true; } ++ ++std::string ACPOMLCPPInterface::readModelParam(std::string FilePath, ++ std::string Param) { ++ std::optional Env = llvm::sys::Process::GetEnv(ACPO_ENV_VAR_DIR); ++ if (!Env || *Env == "") { ++ std::optional LLVMDIROpt = ++ llvm::sys::Process::GetEnv("LLVM_DIR"); ++ if (LLVMDIROpt) { ++ Env = *LLVMDIROpt + "/acpo/"; ++ } else { ++ return ""; ++ } ++ } ++ ++ FilePath = *Env + "/" + FilePath; ++ ++ std::ifstream FileStream{FilePath}; ++ ++ std::string Line; ++ while (std::getline(FileStream, Line)) { ++ if (Line.rfind(Param, 0) == 0) { ++ return Line.substr(Param.size() + 1); ++ } ++ } ++ return ""; ++} ++ ++void ACPOMLCPPInterface::readFeatures( ++ std::string FilePath, ++ std::vector> &Features) { ++ std::string Line = readModelParam(FilePath, "Features"); ++ while (!Line.empty()) { ++ // This reads the features, assuming each feature is written as ++ // {feature_name, feature_type} ++ size_t LeftBracket = Line.find("{"); ++ size_t Comma = Line.find(",", LeftBracket); ++ size_t Space = Line.find(" ", Comma); ++ size_t RightBracket = Line.find("}", Space); ++ if (LeftBracket == Line.size() || Comma == Line.size() || ++ Space == Line.size() || RightBracket == Line.size()) { ++ break; ++ } ++ std::string Feature = Line.substr(LeftBracket + 1, Comma - LeftBracket - 1); ++ std::string Type = Line.substr(Space + 1, RightBracket - Space - 1); ++ ++ Features.emplace_back(std::make_pair(Feature, Type)); ++ int oldLength = Line.size(); ++ Line = Line.substr(RightBracket + 1); ++ int newLength = Line.size(); ++ if (oldLength == newLength) ++ break; ++ } ++} ++ ++void ACPOMLCPPInterface::readOutputs( ++ std::string FilePath, ++ std::vector> &Outputs) { ++ std::string Line = readModelParam(FilePath, "Outputs"); ++ while (!Line.empty()) { ++ // This reads the features, assuming each feature is written as ++ // {feature_name, feature_type} ++ size_t LeftBracket = Line.find("{"); ++ size_t Comma = Line.find(",", LeftBracket); ++ size_t Space = Line.find(" ", Comma); ++ size_t RightBracket = Line.find("}", Space); ++ if (LeftBracket == Line.size() || Comma == Line.size() || ++ Space == Line.size() || RightBracket == Line.size()) { ++ break; ++ } ++ std::string Output = Line.substr(LeftBracket + 1, Comma - LeftBracket - 1); ++ std::string Type = Line.substr(Space + 1, RightBracket - Space - 1); ++ ++ Outputs.emplace_back(std::make_pair(Output, Type)); ++ int oldLength = Line.size(); ++ Line = Line.substr(RightBracket + 1); ++ int newLength = Line.size(); ++ if (oldLength == newLength) ++ break; ++ } ++} ++ ++std::shared_ptr llvm::createPersistentCompiledMLIF() { ++ if (PersistentMLIF == nullptr) { ++ PersistentMLIF = std::make_shared(); ++ if (!PersistentMLIF->isInitialized()) ++ PersistentMLIF = nullptr; ++ } ++ return PersistentMLIF; ++} ++ ++#ifdef LLVM_HAVE_TF_AOT_FICOMPILEDMODEL ++std::unique_ptr ++createFI(std::vector> Inputs, ++ StringRef Decision) { ++ // Context does not ever seem to be used in the model runner, ++ // so for now just create an empty context object ++ LLVMContext Ctx; ++ return std::make_unique(Ctx, Inputs, Decision); ++} ++#endif ++ ++// Generate map using ifdefs for now, in the future we could have this ++// automatically populate using macros ++const std::unordered_map ++ ACPOMLCPPInterface::CreateModelRunnerMap = { ++#ifdef LLVM_HAVE_TF_AOT_FICOMPILEDMODEL ++ {"FI", createFI}, ++#endif ++}; +diff --git a/llvm/lib/Analysis/ACPOModel.cpp b/llvm/lib/Analysis/ACPOModel.cpp +new file mode 100644 +index 000000000000..2d0dae733943 +--- /dev/null ++++ b/llvm/lib/Analysis/ACPOModel.cpp +@@ -0,0 +1,63 @@ ++//===- ACPOModel.cpp - AI-Enabled Continuous Program Optimization ---------===// ++// ++// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. ++// See https://llvm.org/LICENSE.txt for license information. ++// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ++// ++//===----------------------------------------------------------------------===// ++// ++// This file implements the interface between ACPO and ML-guided optimizations. ++// It delegates decision making to inference with a pre-trained model. ++// ++//===----------------------------------------------------------------------===// ++ ++#include "llvm/Analysis/ACPOModel.h" ++#include "llvm/Analysis/LoopInfo.h" ++#include "llvm/Analysis/OptimizationRemarkEmitter.h" ++#include "llvm/Support/Debug.h" ++#include ++ ++using namespace llvm; ++ ++#define DEBUG_TYPE "acpo" ++ ++ACPOAdvice::ACPOAdvice(std::unique_ptr &ResultFormat) { ++ assert(ResultFormat != nullptr); ++ for (auto &Entry : ResultFormat->getFieldMap()) { ++ reserveField(Entry.first, Entry.second.T); ++ } ++} ++ ++void ACPOModel::prepareModelInput() {} ++ ++bool ACPOModel::runModel(std::unique_ptr &Result) { return true; } ++ ++void ACPOModel::addRequiredResultField(std::string name, Type::TypeID &ID) { ++ ResultFormat->reserveField(name, ID); ++} ++ ++std::unique_ptr ACPOModel::getAdvice() { ++ if (ShouldUseML) ++ return getAdviceML(); ++ else ++ return getAdviceNoML(); ++} ++ ++std::unique_ptr ACPOModel::getAdviceML() { ++ // This needs to be filled with a mechanism to invoke a model selected ++ // using the ModelRunner. ++ sendCustomFeatures(); ++ prepareModelInput(); ++ std::unique_ptr Result = ++ std::make_unique(ResultFormat); ++ ++ if (runModel(Result)) ++ return Result; ++ else ++ return nullptr; ++} ++ ++void ACPOModel::addFeature(int64_t ID, Constant *Val) { ++ assert(CustomFeatureMap.find(ID) == CustomFeatureMap.end()); ++ CustomFeatureMap[ID] = Val; ++} +diff --git a/llvm/lib/Analysis/CMakeLists.txt b/llvm/lib/Analysis/CMakeLists.txt +index 9c6a70f0221f..961b5037dd48 100644 +--- a/llvm/lib/Analysis/CMakeLists.txt ++++ b/llvm/lib/Analysis/CMakeLists.txt +@@ -4,6 +4,30 @@ if (DEFINED LLVM_HAVE_TF_AOT OR LLVM_HAVE_TFLITE) + + set(LLVM_INLINER_MODEL_CURRENT_URL "" CACHE STRING "URL to download the LLVM inliner model") + ++ if (ACPO_AOT) ++ foreach (model_name model_path model_signature IN ZIP_LISTS LLVM_ACPO_MODEL_NAMES LLVM_ACPO_MODEL_PATHS LLVM_ACPO_MODEL_SIGNATURES) ++ set(fname ${model_name}CompiledModel) ++ string(TOUPPER ${fname} fname_allcaps) ++ if (LLVM_ACPO_OVERRIDE) ++ string(TOUPPER ${LLVM_ACPO_OVERRIDE_ARCH} arch_allcaps) ++ set(LLVM_OVERRIDE_MODEL_HEADER_${fname_allcaps} ++ ${LLVM_ACPO_OVERRIDE_PATH}/${fname}-${arch_allcaps}.h) ++ set(LLVM_OVERRIDE_MODEL_OBJECT_${fname_allcaps} ++ ${LLVM_ACPO_OVERRIDE_PATH}/${fname}-${arch_allcaps}.o) ++ endif() ++ ++ tf_find_and_compile( ++ ${model_path} ++ ${LLVM_INLINER_MODEL_CURRENT_URL} ++ ${LLVM_INLINER_MODEL_PATH_DEFAULT} ++ "" ++ serve ++ "${model_signature}" ++ "${fname}" ++ "llvm::${fname}" ++ ) ++ endforeach() ++ endif() + if (DEFINED LLVM_HAVE_TF_AOT) + tf_find_and_compile( + ${LLVM_INLINER_MODEL_PATH} +@@ -24,6 +48,10 @@ if (DEFINED LLVM_HAVE_TF_AOT OR LLVM_HAVE_TFLITE) + endif() + + add_llvm_component_library(LLVMAnalysis ++ ACPOCollectFeatures.cpp ++ ACPOFIModel.cpp ++ ACPOMLInterface.cpp ++ ACPOModel.cpp + AliasAnalysis.cpp + AliasAnalysisEvaluator.cpp + AliasSetTracker.cpp +@@ -41,6 +69,7 @@ add_llvm_component_library(LLVMAnalysis + CGSCCPassManager.cpp + CallGraph.cpp + CallGraphSCCPass.cpp ++ CallHeight.cpp + CallPrinter.cpp + CaptureTracking.cpp + CmpInstAnalysis.cpp +@@ -59,6 +88,8 @@ add_llvm_component_library(LLVMAnalysis + DomPrinter.cpp + DomTreeUpdater.cpp + DominanceFrontier.cpp ++ DumpCallsite.cpp ++ DumpFeature.cpp + FunctionPropertiesAnalysis.cpp + GlobalsModRef.cpp + GuardUtils.cpp +@@ -100,6 +131,7 @@ add_llvm_component_library(LLVMAnalysis + MemoryProfileInfo.cpp + MemorySSA.cpp + MemorySSAUpdater.cpp ++ ModelDataCollector.cpp + ModelUnderTrainingRunner.cpp + ModuleDebugInfoPrinter.cpp + ModuleSummaryAnalysis.cpp +diff --git a/llvm/lib/Analysis/CallHeight.cpp b/llvm/lib/Analysis/CallHeight.cpp +new file mode 100644 +index 000000000000..f7b88cbdff05 +--- /dev/null ++++ b/llvm/lib/Analysis/CallHeight.cpp +@@ -0,0 +1,89 @@ ++//===- CallHeight.cpp - CallHeight implementation ------------------------===// ++// ++// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. ++// See https://llvm.org/LICENSE.txt for license information. ++// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ++// ++//===----------------------------------------------------------------------===// ++// ++// This file implements getting the call height of functions in a module. ++// ++//===----------------------------------------------------------------------===// ++#include "llvm/Analysis/CallHeight.h" ++#include "llvm/ADT/SCCIterator.h" ++#include "llvm/Analysis/CallGraph.h" ++#include "llvm/IR/InstIterator.h" ++#include "llvm/InitializePasses.h" ++ ++using namespace llvm; ++ ++#define DEBUG_TYPE "call-height" ++ ++// Adapted from MLInlineAdvisor.cpp ++CallBase *getInlinableCallSite(Instruction &I) { ++ if (auto *CS = dyn_cast(&I)) { ++ if (Function *Callee = CS->getCalledFunction()) ++ if (!Callee->isDeclaration()) { ++ return CS; ++ } ++ } ++ return nullptr; ++} ++ ++unsigned CallHeight::getLevel(Function &F) { return (*Levels)[&F]; } ++ ++CallHeight::CallHeight(Module &M) ++ : Levels(std::make_unique>()) { ++ // Adapted from MLInlineAdvisor.cpp ++ CallGraph CG = CallGraph(M); ++ ++ for (auto I = scc_begin(&CG); !I.isAtEnd(); ++I) { ++ const std::vector &CGNodes = *I; ++ unsigned Level = 0; ++ for (auto *CGNode : CGNodes) { ++ Function *F = CGNode->getFunction(); ++ if (!F || F->isDeclaration()) ++ continue; ++ for (auto &I : instructions(F)) { ++ if (auto *CS = getInlinableCallSite(I)) { ++ auto *Called = CS->getCalledFunction(); ++ auto Pos = Levels->find(Called); ++ // In bottom up traversal, an inlinable callee is either in the ++ // same SCC, or to a function in a visited SCC. So not finding its ++ // level means we haven't visited it yet, meaning it's in this SCC. ++ if (Pos == Levels->end()) ++ continue; ++ Level = std::max(Level, Pos->second + 1); ++ } ++ } ++ } ++ for (auto *CGNode : CGNodes) { ++ Function *F = CGNode->getFunction(); ++ if (F && !F->isDeclaration()) ++ (*Levels)[F] = Level; ++ } ++ } ++} ++ ++AnalysisKey CallHeightAnalysis::Key; ++ ++CallHeight CallHeightAnalysis::run(Module &M, ModuleAnalysisManager &MAM) { ++ return CallHeight(M); ++} ++ ++bool CallHeightAnalysisWrapper::runOnModule(Module &M) { ++ Result.reset(new CallHeight(M)); ++ return false; ++} ++ ++void CallHeightAnalysisWrapper::getAnalysisUsage(AnalysisUsage &AU) const { ++ AU.setPreservesAll(); ++} ++ ++char CallHeightAnalysisWrapper::ID = 0; ++INITIALIZE_PASS(CallHeightAnalysisWrapper, DEBUG_TYPE, "Call Height Analysis", ++ false, true) ++ ++Pass *llvm::createCallHeightAnalysisWrapper() { ++ return new CallHeightAnalysisWrapper(); ++} +diff --git a/llvm/lib/Analysis/DumpCallsite.cpp b/llvm/lib/Analysis/DumpCallsite.cpp +new file mode 100644 +index 000000000000..d49885a372f2 +--- /dev/null ++++ b/llvm/lib/Analysis/DumpCallsite.cpp +@@ -0,0 +1,82 @@ ++//===- DumpCallsite.cpp - DumpCallsite implementation --------------------===// ++// ++// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. ++// See https://llvm.org/LICENSE.txt for license information. ++// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ++// ++//===----------------------------------------------------------------------===// ++// ++// This file implements the ability to dump all callsites in a given function. ++// ++//===----------------------------------------------------------------------===// ++#include "llvm/Analysis/DumpCallsite.h" ++#include "llvm/IR/Function.h" ++#include "llvm/IR/InstIterator.h" ++#include "llvm/IR/Instructions.h" ++#include "llvm/IR/LegacyPassManager.h" ++#include "llvm/InitializePasses.h" ++#include "llvm/Pass.h" ++#include "llvm/Support/CommandLine.h" ++ ++using namespace llvm; ++ ++static cl::opt ++ IncludeDeclaration("include-declaration", cl::Hidden, ++ cl::desc("Also dump declaration in dump-callsite pass")); ++ ++namespace { ++ ++// Implementation of actual DumpCallsite ++class DumpCallsite { ++public: ++ void run(Function &F); ++}; ++ ++// Wrapper for legacy PM ++class DumpCallsiteLegacy : public FunctionPass { ++public: ++ static char ID; ++ DumpCallsiteLegacy() : FunctionPass(ID) {} ++ ++ bool runOnFunction(Function &F) override; ++}; ++ ++void DumpCallsite::run(Function &F) { ++ outs() << F.getName(); ++ // Get all callees from 'call' inst ++ for (auto &I : instructions(F)) { ++ // Is a call inst ++ if (auto *CS = dyn_cast(&I)) { ++ // callee is present ++ if (Function *Callee = CS->getCalledFunction()) { ++ // Not intrinsic ++ if (!Callee->isIntrinsic()) { ++ // decide whether to dump declaration ++ if (!Callee->isDeclaration() || IncludeDeclaration) { ++ outs() << " " << Callee->getName(); ++ } ++ } ++ } ++ } ++ } ++ outs() << "\n"; ++} ++ ++bool DumpCallsiteLegacy::runOnFunction(Function &F) { ++ DumpCallsite Impl; ++ Impl.run(F); ++ return false; ++} ++ ++} // namespace ++ ++char DumpCallsiteLegacy::ID = 0; ++INITIALIZE_PASS(DumpCallsiteLegacy, "dump-callsite", "Dump Callsite", false, ++ false) ++ ++PreservedAnalyses DumpCallsitePass::run(Function &F, ++ FunctionAnalysisManager &FAM) { ++ DumpCallsite Impl; ++ Impl.run(F); ++ return PreservedAnalyses::all(); ++} +diff --git a/llvm/lib/Analysis/DumpFeature.cpp b/llvm/lib/Analysis/DumpFeature.cpp +new file mode 100644 +index 000000000000..81756226c2fd +--- /dev/null ++++ b/llvm/lib/Analysis/DumpFeature.cpp +@@ -0,0 +1,575 @@ ++//===- DumpFeature.cpp - DumpFeature implementation -----------------------===// ++// ++// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. ++// See https://llvm.org/LICENSE.txt for license information. ++// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ++// ++//===----------------------------------------------------------------------===// ++// ++// This file implements dumping features for functions in an scc. ++// ++//===----------------------------------------------------------------------===// ++#include "llvm/Analysis/DumpFeature.h" ++#include "llvm/ADT/SCCIterator.h" ++#include "llvm/Analysis/CallHeight.h" ++#include "llvm/Analysis/TargetLibraryInfo.h" ++#include "llvm/IR/BasicBlock.h" ++#include "llvm/IR/Function.h" ++#include "llvm/IR/Instructions.h" ++#include "llvm/IR/PassManager.h" ++#include "llvm/InitializePasses.h" ++#include "llvm/MC/MCAsmLayout.h" ++#include "llvm/Support/Casting.h" ++#include "llvm/Support/CommandLine.h" ++#include "llvm/Support/FileSystem.h" ++#include "llvm/Support/raw_ostream.h" ++ ++#include ++#include ++#include ++ ++using namespace llvm; ++ ++bool EnableFeatureDump; ++static cl::opt EnableFeatureDumpFlag( ++ "enable-feature-dump", cl::location(EnableFeatureDump), cl::init(false), ++ cl::Hidden, cl::ZeroOrMore, cl::desc("Enable Feature Dump")); ++ ++static cl::opt ++ CheckPairHisto("check-pair-histo", cl::Hidden, ++ cl::desc("Dump instruction pairs in the histogram")); ++ ++static cl::opt Verbose("dump-verbose", cl::Hidden, ++ cl::desc("Dump as human readable format")); ++ ++static llvm::cl::opt ++ OutFile("feature-output", llvm::cl::desc("File for outputting features"), ++ llvm::cl::init("features.csv")); ++ ++namespace { ++unsigned getMaxInstructionID() { ++#define LAST_OTHER_INST(NR) return NR; ++#include "llvm/IR/Instruction.def" ++} ++ ++// This is a point in time - we determined including these pairs of ++// consecutive instructions (in the IR layout available at inline time) as ++// features improves the model performance. We want to move away from manual ++// feature selection. ++// The array is given in opcode pairs rather than labels because 1) labels ++// weren't readily available, and 2) the successions were hand - extracted. ++// ++// This array must be sorted. ++static const std::array, 137> ++ ImportantInstructionSuccessions{ ++ {{1, 1}, {1, 4}, {1, 5}, {1, 7}, {1, 8}, {1, 9}, {1, 11}, ++ {1, 12}, {1, 13}, {1, 14}, {1, 18}, {1, 20}, {1, 22}, {1, 24}, ++ {1, 25}, {1, 26}, {1, 27}, {1, 28}, {1, 29}, {1, 30}, {1, 31}, ++ {1, 32}, {1, 33}, {1, 34}, {1, 39}, {1, 40}, {1, 42}, {1, 45}, ++ {2, 1}, {2, 2}, {2, 13}, {2, 28}, {2, 29}, {2, 32}, {2, 33}, ++ {2, 34}, {2, 38}, {2, 48}, {2, 49}, {2, 53}, {2, 55}, {2, 56}, ++ {13, 2}, {13, 13}, {13, 26}, {13, 33}, {13, 34}, {13, 56}, {15, 27}, ++ {28, 2}, {28, 48}, {28, 53}, {29, 2}, {29, 33}, {29, 56}, {31, 31}, ++ {31, 33}, {31, 34}, {31, 49}, {32, 1}, {32, 2}, {32, 13}, {32, 15}, ++ {32, 28}, {32, 29}, {32, 32}, {32, 33}, {32, 34}, {32, 39}, {32, 40}, ++ {32, 48}, {32, 49}, {32, 53}, {32, 56}, {33, 1}, {33, 2}, {33, 32}, ++ {33, 33}, {33, 34}, {33, 49}, {33, 53}, {33, 56}, {34, 1}, {34, 2}, ++ {34, 32}, {34, 33}, {34, 34}, {34, 49}, {34, 53}, {34, 56}, {38, 34}, ++ {39, 57}, {40, 34}, {47, 15}, {47, 49}, {48, 2}, {48, 34}, {48, 56}, ++ {49, 1}, {49, 2}, {49, 28}, {49, 32}, {49, 33}, {49, 34}, {49, 39}, ++ {49, 49}, {49, 56}, {53, 1}, {53, 2}, {53, 28}, {53, 34}, {53, 53}, ++ {53, 57}, {55, 1}, {55, 28}, {55, 34}, {55, 53}, {55, 55}, {55, 56}, ++ {56, 1}, {56, 2}, {56, 7}, {56, 13}, {56, 32}, {56, 33}, {56, 34}, ++ {56, 49}, {56, 53}, {56, 56}, {56, 64}, {57, 34}, {57, 56}, {57, 57}, ++ {64, 1}, {64, 64}, {65, 1}, {65, 65}}}; ++ ++size_t getSize(Function &F, TargetTransformInfo &TTI) { ++ size_t SumOfAllInstCost = 0; ++ for (const auto &BB : F) ++ for (const auto &I : BB) { ++ std::optional cost = ++ TTI.getInstructionCost( ++ &I, TargetTransformInfo::TargetCostKind::TCK_CodeSize) ++ .getValue(); ++ if (cost.has_value()) ++ SumOfAllInstCost += cost.value(); ++ } ++ return SumOfAllInstCost; ++} ++ ++unsigned getMaxDominatorTreeDepth(const Function &F, ++ const DominatorTree &Tree) { ++ unsigned MaxBBDepth = 0; ++ for (const auto &BB : F) ++ if (const auto *TN = Tree.getNode(&BB)) ++ MaxBBDepth = std::max(MaxBBDepth, TN->getLevel()); ++ ++ return MaxBBDepth; ++} ++ ++// get valid call uses and valid call uses in loop counts. ++std::pair ++getValidCallUsesAndInLoopCounts(Function &F, ++ FunctionAnalysisManager *FAM = nullptr) { ++ unsigned CallUses = 0; ++ unsigned CallUsesInLoop = 0; ++ ++ for (User *U : F.users()) { ++ if (CallBase *CB = dyn_cast(U)) { ++ ++CallUses; ++ BasicBlock *BB = CB->getParent(); ++ Function *FUser = CB->getCaller(); ++ auto &LI = ++ FAM->getResult(*FUser); ++ if (LI.getLoopFor(BB) != nullptr) { ++ ++CallUsesInLoop; ++ } ++ } ++ } ++ return std::make_pair(CallUses, CallUsesInLoop); ++} ++} // namespace ++ ++// We have: 9 calculated features (the features here); 1 feature for each ++// instruction opcode; and 1 feature for each manually-identified sequence. ++// For the latter 2, we build a histogram: we count the number of ++// occurrences of each instruction opcode or succession of instructions, ++// respectively. ++// Note that instruction opcodes start from 1. For convenience, we also have ++// an always 0 feature for the '0' opcode, hence the extra 1. ++const size_t ACPOFIExtendedFeatures::FunctionFeatures::FeatureCount = ++ ImportantInstructionSuccessions.size() + getMaxInstructionID() + 1 + ++ static_cast( ++ ACPOFIExtendedFeatures::NamedFloatFeatureIndex::NumNamedFloatFeatures) + ++ static_cast( ++ ACPOFIExtendedFeatures::NamedFeatureIndex::NumNamedFeatures); ++ ++void ACPOFIExtendedFeatures::updateLoopRelatedFeatures(Function &F, ++ LoopInfo &LI, ++ FunctionFeatures &FF) { ++ uint64_t LoopNum = std::distance(LI.begin(), LI.end()); ++ ++ uint64_t LoopInstrCount = 0; ++ uint64_t BlockWithMulSuccNum = 0; ++ uint64_t LoopLevelSum = 0; ++ for (auto &L : LI) { ++ LoopLevelSum += static_cast(L->getLoopDepth()); ++ FF[NamedFeatureIndex::MaxLoopDepth] = ++ std::max(FF[NamedFeatureIndex::MaxLoopDepth], ++ static_cast(L->getLoopDepth())); ++ for (const BasicBlock *BB : L->getBlocks()) { ++ unsigned SuccCount = std::distance(succ_begin(BB), succ_end(BB)); ++ if (SuccCount > 1) ++ BlockWithMulSuccNum++; ++ LoopInstrCount += std::distance(BB->instructionsWithoutDebug().begin(), ++ BB->instructionsWithoutDebug().end()); ++ } ++ } ++ ++ FF[NamedFeatureIndex::Loops] = LoopNum; ++ if (LoopNum != 0) { ++ uint64_t q = LoopInstrCount / LoopNum; ++ FF[NamedFloatFeatureIndex::InstrPerLoop] = ++ q + ((float)(LoopInstrCount - q * LoopNum)) / LoopNum; ++ q = BlockWithMulSuccNum / LoopNum; ++ FF[NamedFloatFeatureIndex::BlockWithMultipleSuccecorsPerLoop] = ++ q + ((float)(BlockWithMulSuccNum - q * LoopNum)) / LoopNum; ++ q = LoopLevelSum / LoopNum; ++ FF[NamedFloatFeatureIndex::AvgNestedLoopLevel] = ++ q + ((float)(LoopLevelSum - q * LoopNum)) / LoopNum; ++ } ++} ++ ++void ACPOFIExtendedFeatures::updateBBLoopCallsiteBFFeatures( ++ Function &F, FunctionFeatures &FF, LoopInfo &LI, ++ FunctionAnalysisManager *FAM) { ++ // Initializations before looping ++ unsigned NumCallsiteInLoop = 0; ++ unsigned NumCallsite = 0; ++ uint64_t MaxCallsiteBlockFreq = 0; ++ uint64_t InstrNum = 0; ++ uint64_t SuccNum = 0; ++ uint64_t VecNum = 0; ++ uint64_t BlockNum = F.size(); ++ auto getPairIndex = [](size_t a, size_t b) { ++ auto I = llvm::find(ImportantInstructionSuccessions, std::make_pair(a, b)); ++ if (I == ImportantInstructionSuccessions.end()) ++ return -1; ++ return static_cast( ++ std::distance(ImportantInstructionSuccessions.begin(), I)); ++ }; ++ int StartID = 0; ++ int LastID = StartID; ++ ++ // We don't want debug calls, because they'd just add noise. ++ // Sum number of instructions and successors on the way ++ for (auto &BB : F) { ++ SuccNum += std::distance(succ_begin(&BB), succ_end(&BB)); ++ for (auto &I : BB.instructionsWithoutDebug()) { ++ if (CallBase *CB = dyn_cast(&I)) { ++ Function *Callee = CB->getCalledFunction(); ++ if (Callee && !Callee->isIntrinsic()) { ++ ++NumCallsite; ++ if (!Callee->isDeclaration()) { ++ // Check all the functions that was called and get the max block ++ // frequency. ++ uint64_t EntryFreq = ++ FAM->getResult(*Callee) ++ .getEntryFreq(); ++ MaxCallsiteBlockFreq = std::max(EntryFreq, MaxCallsiteBlockFreq); ++ } ++ ++ if (Callee != nullptr) { ++ // Collect the number of callsites that were invoked with a pointer ++ // argument. ++ for (auto arg = Callee->arg_begin(); arg != Callee->arg_end(); ++ arg++) ++ if (isa(arg->getType())) { ++ FF[NamedFeatureIndex::PtrCallee]++; ++ break; ++ } ++ } ++ ++ // Collect the number of callsites that returns a pointer type. ++ if (isa(CB->getType())) { ++ FF[NamedFeatureIndex::CallReturnPtr]++; ++ } ++ ++ // Check if the given function is recursive. ++ if (&F == Callee) { ++ FF[NamedFeatureIndex::IsRecursive] = 1; ++ } ++ ++ BasicBlock *BB = CB->getParent(); ++ // if we found a loop for the BB that Call is in, we do +1 ++ if (LI.getLoopFor(BB) != nullptr) { ++ ++NumCallsiteInLoop; ++ } ++ } ++ } ++ ++ auto ID = I.getOpcode(); ++ ++FF.InstructionHistogram[ID]; ++ int PairIndex = getPairIndex(LastID, ID); ++ if (PairIndex >= 0) ++ ++FF.InstructionPairHistogram[PairIndex]; ++ LastID = ID; ++ InstrNum++; ++ unsigned NumOp = I.getNumOperands(); ++ ++ // If instruction contains vector operand, consider it as a vector ++ // instruction ++ for (unsigned i = 0; i < NumOp; i++) { ++ if (isa(I.getOperand(i)->getType())) { ++ VecNum++; ++ break; ++ } ++ } ++ ++ // If this is a conditional branch, check if it uses an argument ++ if (const auto II = dyn_cast(&I)) ++ if (II->isConditional()) { ++ FF[NamedFeatureIndex::ConditionalBranch]++; ++ // find the instruction where the condition is defined. ++ if (auto def = dyn_cast(II->getCondition())) { ++ // For all operands of def check if isa (operand) then ++ // increment CBwithArg. ++ bool found = false; ++ for (unsigned i = 0; i < def->getNumOperands(); i++) { ++ if (isa(def->getOperand(i))) { ++ FF[NamedFeatureIndex::CBwithArg]++; ++ found = true; ++ break; ++ } ++ } ++ if (found) ++ break; ++ } ++ } ++ } ++ } ++ ++ FF[NamedFloatFeatureIndex::AvgVecInstr] = (float)VecNum / InstrNum; ++ FF[NamedFeatureIndex::Blocks] = BlockNum; ++ if (BlockNum > 0) { ++ uint64_t q = InstrNum / BlockNum; ++ FF[NamedFloatFeatureIndex::InstructionPerBlock] = ++ q + ((float)(InstrNum - q * BlockNum)) / BlockNum; ++ q = SuccNum / BlockNum; ++ FF[NamedFloatFeatureIndex::SuccessorPerBlock] = ++ q + ((float)(SuccNum - q * BlockNum)) / BlockNum; ++ } ++ ++ FF[NamedFeatureIndex::MaxCallsiteBlockFreq] = MaxCallsiteBlockFreq; ++ FF[NamedFeatureIndex::NumCallsiteInLoop] = NumCallsiteInLoop; ++ FF[NamedFeatureIndex::Calls] = NumCallsite; ++} ++ ++ACPOFIExtendedFeatures::FunctionFeatures ++ACPOFIExtendedFeatures::getFunctionFeatures( ++ Function &F, DominatorTree &DomTree, TargetTransformInfo &TTI, LoopInfo &LI, ++ FunctionAnalysisManager *FAM, ++ bool ValidSize, bool ValidLoop, bool ValidTree) { ++ assert(llvm::is_sorted(ImportantInstructionSuccessions) && ++ "expected function features are sorted"); ++ ++ FunctionFeatures FF; ++ size_t InstrCount = getMaxInstructionID() + 1; ++ FF.InstructionHistogram.resize(InstrCount); ++ FF.InstructionPairHistogram.resize(ImportantInstructionSuccessions.size()); ++ ++ // check all the argument to see if there is a pointer type ++ for (auto arg = F.arg_begin(); arg != F.arg_end(); arg++) { ++ if (isa(arg->getType())) { ++ FF[NamedFeatureIndex::PtrArgs]++; ++ } ++ } ++ ++ std::pair ValidCallAndInLoopCounts = ++ getValidCallUsesAndInLoopCounts(F, FAM); ++ if (!ValidSize) ++ FF[NamedFeatureIndex::InitialSize] = getSize(F, TTI); ++ FF[NamedFeatureIndex::IsLocal] = F.hasLocalLinkage(); ++ FF[NamedFeatureIndex::IsLinkOnceODR] = F.hasLinkOnceODRLinkage(); ++ FF[NamedFeatureIndex::IsLinkOnce] = F.hasLinkOnceLinkage(); ++ if (!ValidTree) ++ FF[NamedFeatureIndex::MaxDomTreeLevel] = ++ getMaxDominatorTreeDepth(F, DomTree); ++ FF[NamedFeatureIndex::CallUsage] = ValidCallAndInLoopCounts.first; ++ FF[NamedFeatureIndex::NumOfCallUsesInLoop] = ValidCallAndInLoopCounts.second; ++ FF[NamedFeatureIndex::EntryBlockFreq] = ++ FAM->getResult(F) ++ .getEntryFreq(); ++ ACPOFIExtendedFeatures::updateBBLoopCallsiteBFFeatures(F, FF, LI, FAM); ++ if (!ValidLoop) ++ ACPOFIExtendedFeatures::updateLoopRelatedFeatures(F, LI, FF); ++ return FF; ++} ++ ++static int getCallHeight(Module &M, CallHeight *CH, Function *F) { ++ if (CH == nullptr) { ++ // If we don't have cached result (for ex, running with opt) ++ // We re-calculate the function level ++ // Or using the old pass manager ++ CallHeight CH = CallHeight(M); ++ return CH.getLevel(*F); ++ } ++ return CH->getLevel(*F); ++} ++ ++void dumpInstructionPairs(raw_fd_ostream &OS) { ++ for (size_t i = 0; i < ImportantInstructionSuccessions.size(); i++) { ++ std::pair pair = ImportantInstructionSuccessions[i]; ++ OS << "{" << Instruction::getOpcodeName(pair.first) << ", " ++ << Instruction::getOpcodeName(pair.second) << "} "; ++ } ++ OS << "\n"; ++} ++ ++void dumpFunctionFeatures(raw_fd_ostream &OS, ++ ACPOFIExtendedFeatures::FunctionFeatures &FF, ++ Function &F, bool Verbose) { ++ if (Verbose) { ++ OS << "Function Name: " << F.getName() << "\n"; ++ OS << "FeatureCount: " << FF.FeatureCount << "\n"; ++ OS << "\nAverage instructions per basic block: " ++ << FF[ACPOFIExtendedFeatures::NamedFloatFeatureIndex:: ++ InstructionPerBlock] ++ << "\nAverage number of successors per block: " ++ << FF[ACPOFIExtendedFeatures::NamedFloatFeatureIndex::SuccessorPerBlock] ++ << "\nAverage number of vector instructions per instruction: " ++ << FF[ACPOFIExtendedFeatures::NamedFloatFeatureIndex::AvgVecInstr] ++ << "\nAverage nest level per loop: " ++ << FF[ACPOFIExtendedFeatures::NamedFloatFeatureIndex::AvgNestedLoopLevel] ++ << "\nAverage instructions per loop: " ++ << FF[ACPOFIExtendedFeatures::NamedFloatFeatureIndex::InstrPerLoop] ++ << "\nAverage blocks with multiple succssors per loop: " ++ << FF[ACPOFIExtendedFeatures::NamedFloatFeatureIndex:: ++ BlockWithMultipleSuccecorsPerLoop] ++ << "\nInitial Size: " ++ << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::InitialSize] << "\n" ++ << "Blocks: " << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::Blocks] ++ << "\n" ++ << "Calls (Number of callsites): " ++ << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::Calls] << "\n" ++ << "IsLocal: " << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::IsLocal] ++ << "\n" ++ << "IsLinkOnceODR: " ++ << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::IsLinkOnceODR] << "\n" ++ << "IsLinkOnce: " ++ << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::IsLinkOnce] << "\n" ++ << "Loops: " << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::Loops] ++ << "\n" ++ << "MaxLoopDepth: " ++ << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::MaxLoopDepth] << "\n" ++ << "MaxDomTreeLevel: " ++ << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::MaxDomTreeLevel] ++ << "\nPointer arguments of this caller: " ++ << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::PtrArgs] ++ << "\nCallees with pointer arguments: " ++ << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::PtrCallee] ++ << "\nCallees that return a pointer: " ++ << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::CallReturnPtr] ++ << "\nConditional Branches: " ++ << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::ConditionalBranch] ++ << "\nConditional Branches that depends on an argument: " ++ << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::CBwithArg] ++ << "\nCaller Height of the current function: " ++ << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::CallerHeight] ++ << "\nNumber of explict calls to this function: " ++ << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::CallUsage] ++ << "\nIs recursive: " ++ << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::IsRecursive] ++ << "\nNumber of callsites that are inside loop in this function: " ++ << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::NumCallsiteInLoop] ++ << "\nNumber of explict calls to this function that are in loop: " ++ << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::NumOfCallUsesInLoop] ++ << "\nBlock Frequency for the first block of this function: " ++ << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::EntryBlockFreq] ++ << "\nMaximum of all callsites' entry Block Frequency: " ++ << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::MaxCallsiteBlockFreq] ++ << "\n"; ++ OS << "InstructionHistogram: "; ++ OS << "Size: " << FF.InstructionHistogram.size() << "\n"; ++ for (size_t i = 0; i < FF.InstructionHistogram.size(); i++) { ++ OS << FF.InstructionHistogram[i] << " "; ++ } ++ OS << "\n"; ++ OS << "InstructionPairHistogram: "; ++ OS << "Size: " << FF.InstructionPairHistogram.size() << "\n"; ++ for (size_t i = 0; i < FF.InstructionPairHistogram.size(); i++) { ++ OS << FF.InstructionPairHistogram[i] << " "; ++ } ++ OS << "\n\n"; ++ } else { ++ OS << F.getName() << " " ++ << FF[ACPOFIExtendedFeatures::NamedFloatFeatureIndex:: ++ InstructionPerBlock] ++ << " " ++ << FF[ACPOFIExtendedFeatures::NamedFloatFeatureIndex::SuccessorPerBlock] ++ << " " << FF[ACPOFIExtendedFeatures::NamedFloatFeatureIndex::AvgVecInstr] ++ << " " ++ << FF[ACPOFIExtendedFeatures::NamedFloatFeatureIndex::AvgNestedLoopLevel] ++ << " " ++ << FF[ACPOFIExtendedFeatures::NamedFloatFeatureIndex::InstrPerLoop] ++ << " " ++ << FF[ACPOFIExtendedFeatures::NamedFloatFeatureIndex:: ++ BlockWithMultipleSuccecorsPerLoop] ++ << " " << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::InitialSize] ++ << " " << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::Blocks] << " " ++ << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::Calls] << " " ++ << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::IsLocal] << " " ++ << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::IsLinkOnceODR] << " " ++ << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::IsLinkOnce] << " " ++ << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::Loops] << " " ++ << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::MaxLoopDepth] << " " ++ << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::MaxDomTreeLevel] << " " ++ << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::PtrArgs] << " " ++ << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::PtrCallee] << " " ++ << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::CallReturnPtr] << " " ++ << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::ConditionalBranch] ++ << " " << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::CBwithArg] << " " ++ << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::CallerHeight] << " " ++ << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::CallUsage] << " " ++ << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::IsRecursive] << " " ++ << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::NumCallsiteInLoop] ++ << " " ++ << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::NumOfCallUsesInLoop] ++ << " " << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::EntryBlockFreq] ++ << " " ++ << FF[ACPOFIExtendedFeatures::NamedFeatureIndex::MaxCallsiteBlockFreq] ++ << " "; ++ ++ for (size_t i = 0; i < FF.InstructionHistogram.size(); i++) { ++ OS << FF.InstructionHistogram[i] << " "; ++ } ++ for (size_t i = 0; i < FF.InstructionPairHistogram.size(); i++) { ++ OS << FF.InstructionPairHistogram[i] << " "; ++ } ++ OS << "\n"; ++ } ++} ++ ++void runAndDump(raw_fd_ostream &OS, Function *F, DominatorTree &DomTree, ++ TargetTransformInfo &TTI, LoopInfo &LI, Module &M, ++ CallHeight *CH, FunctionAnalysisManager *FAM = nullptr) { ++ struct ACPOFIExtendedFeatures::FunctionFeatures FF = ++ ACPOFIExtendedFeatures::getFunctionFeatures(*F, DomTree, TTI, LI, FAM); ++ // Get the call height feature ++ FF[ACPOFIExtendedFeatures::NamedFeatureIndex::CallerHeight] = ++ getCallHeight(M, CH, F); ++ dumpFunctionFeatures(OS, FF, *F, Verbose); ++} ++ ++std::unique_ptr setUpOS() { ++ // Check ACPO/llvm-project issue #112 ++ std::error_code FileErr; ++ std::unique_ptr OS( ++ new raw_fd_ostream(OutFile.c_str(), FileErr, llvm::sys::fs::OF_Append)); ++ ++ if (FileErr) { ++ llvm::errs() << "Error opening info file " << OutFile.c_str() << ": " ++ << FileErr.message() << "\n"; ++ return nullptr; ++ } ++ ++ if (CheckPairHisto) { ++ dumpInstructionPairs(*OS); ++ return nullptr; ++ } ++ ++ return OS; ++} ++ ++PreservedAnalyses DumpFeaturePass::run(LazyCallGraph::SCC &C, ++ CGSCCAnalysisManager &AM, ++ LazyCallGraph &CG, ++ CGSCCUpdateResult &UR) { ++ std::unique_ptr OS = setUpOS(); ++ if (!OS) ++ return PreservedAnalyses::all(); ++ ++ FunctionAnalysisManager &FAM = ++ AM.getResult(C, CG).getManager(); ++ ++ const auto &MAMProxy = AM.getResult(C, CG); ++ Module &M = *C.begin()->getFunction().getParent(); ++ CallHeight *CH = MAMProxy.getCachedResult(M); ++ for (LazyCallGraph::Node &N : C) { ++ Function *F = &N.getFunction(); ++ if (F->empty()) { ++ continue; ++ } ++ ++ auto &DomTree = FAM.getResult(*F); ++ auto &TTI = FAM.getResult(*F); ++ auto &LI = FAM.getResult(*F); ++ ++ runAndDump(*OS, F, DomTree, TTI, LI, M, CH, &FAM); ++ } ++ return PreservedAnalyses::all(); ++} ++ ++ACPOFIExtendedFeatures::NamedFeatureIndex & ++llvm::operator++(ACPOFIExtendedFeatures::NamedFeatureIndex &n) { ++ return n = static_cast((int)n + 1); ++} ++ ++ACPOFIExtendedFeatures::NamedFeatureIndex ++operator++(ACPOFIExtendedFeatures::NamedFeatureIndex &n, int) { ++ ACPOFIExtendedFeatures::NamedFeatureIndex res = n; ++ ++n; ++ return res; ++} ++ ++ACPOFIExtendedFeatures::NamedFloatFeatureIndex & ++llvm::operator++(ACPOFIExtendedFeatures::NamedFloatFeatureIndex &n) { ++ return n = static_cast((int)n + 1); ++} ++ ++ACPOFIExtendedFeatures::NamedFloatFeatureIndex ++operator++(ACPOFIExtendedFeatures::NamedFloatFeatureIndex &n, int) { ++ ACPOFIExtendedFeatures::NamedFloatFeatureIndex res = n; ++ ++n; ++ return res; ++} +diff --git a/llvm/lib/Analysis/ModelDataCollector.cpp b/llvm/lib/Analysis/ModelDataCollector.cpp +new file mode 100644 +index 000000000000..5d599bff25a4 +--- /dev/null ++++ b/llvm/lib/Analysis/ModelDataCollector.cpp +@@ -0,0 +1,350 @@ ++//===- ModelDataCollector.cpp - Data collector for ML model --------------===// ++// ++// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. ++// See https://llvm.org/LICENSE.txt for license information. ++// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ++// ++//===----------------------------------------------------------------------===// ++// ++// This file implements the collection and dumping of data for the ML models ++// ++//===----------------------------------------------------------------------===// ++ ++#if defined(ENABLE_ACPO) ++#include "llvm/Analysis/LoopInfo.h" ++#include "llvm/Analysis/ModelDataCollector.h" ++#include "llvm/Demangle/Demangle.h" ++#include "llvm/IR/Function.h" ++#include "llvm/Support/CommandLine.h" ++#include "llvm/Support/FileSystem.h" ++#include "llvm/Support/Path.h" ++ ++using namespace llvm; ++ ++#define DEBUG_TYPE "model-data-collector" ++ ++// Defined in 'lib/IR/AsmWriter.cpp' ++extern cl::opt UnnamedVariablePrefix; ++ ++static cl::opt IRFileDirectory( ++ "IR-file-directory", cl::Hidden, ++ cl::desc("Name of a directory to store IR files.")); ++ ++cl::opt ++ ACPOModelFile("acpo-dump-file", cl::init("-"), cl::Hidden, ++ cl::desc("Name of a file to store feature data in.")); ++ ++std::string ModelDataCollector::getDumpOptionAsString(DumpOption DO) { ++ switch (DO) { ++ case DumpOption::loop: ++ return "loop"; ++ case DumpOption::function: ++ return "function"; ++ case DumpOption::before: ++ return "before"; ++ case DumpOption::after: ++ return "after"; ++ default: ++ return ""; ++ } ++} ++ ++std::vector> ModelDataCollector::getFeatures() { ++ return Features; ++} ++ ++StringMap ModelDataCollector::getIRFileNameMap() { ++ return IRFileNames; ++} ++ ++std::string ModelDataCollector::getOutputFileName() { return OutputFileName; } ++ ++bool ModelDataCollector::isEmptyOutputFile() { ++ if (OutputFileName.empty()) ++ return false; ++ ++ if (!sys::fs::exists(OutputFileName)) ++ return true; ++ ++ uint64_t Size; ++ std::error_code EC = sys::fs::file_size(OutputFileName, Size); ++ if (EC) { ++ llvm::errs() << "Cannot get file size: " << EC.message() << "\n"; ++ assert(false && "Cannot get file size."); ++ } ++ ++ if (Size == 0) ++ return true; ++ ++ return false; ++} ++ ++void ModelDataCollector::collectFeatures(Loop *L, const std::string &ModuleName, ++ const std::string &FuncName, const std::string &LoopName) { ++} ++ ++void ModelDataCollector::collectFeatures() { ++ for (auto &FeatureCollectInfo : FeatureCollectInfos) { ++ ACPOCollectFeatures::FeatureValueMap FeatureMap; ++ ++ if (FeatureCollectInfo->FeaturesInfo.get()) { ++ FeatureMap = FeatureCollectInfo->FeatureCollector->getFeaturesPair( ++ *FeatureCollectInfo->FeaturesInfo.get()); ++ } else if (FeatureCollectInfo->RegisteredScopes.get()) { ++ FeatureCollectInfo->FeatureCollector->setGlobalFeatureInfo( ++ *FeatureCollectInfo->GlobalInfo.get()); ++ FeatureMap = FeatureCollectInfo->FeatureCollector->getFeaturesPair( ++ *FeatureCollectInfo->RegisteredScopes.get()); ++ } else if (FeatureCollectInfo->RegisteredGroupIDs.get()) { ++ FeatureCollectInfo->FeatureCollector->setGlobalFeatureInfo( ++ *FeatureCollectInfo->GlobalInfo.get()); ++ FeatureMap = FeatureCollectInfo->FeatureCollector->getFeaturesPair( ++ *FeatureCollectInfo->RegisteredGroupIDs.get()); ++ } else { ++ outs() << "No Features are collected, since the given " ++ "FeatureCollectInfo is invalid.\n"; ++ return; ++ } ++ for (auto const &[key, val] : FeatureMap) { ++ std::string FeatureName; ++ ++ if (FeatureCollectInfo->Prefix != "") ++ FeatureName += FeatureCollectInfo->Prefix + "_"; ++ ++ FeatureName += ACPOCollectFeatures::getFeatureName(key); ++ ++ if (FeatureCollectInfo->Postfix != "") ++ FeatureName += "_" + FeatureCollectInfo->Postfix; ++ ++ Features.insert(Features.end(), {std::make_pair(FeatureName, val)}); ++ } ++ } ++} ++ ++void ModelDataCollector::registerFeature(ACPOCollectFeatures::FeaturesInfo Info, ++ std::string Pre, std::string Post) { ++ std::unique_ptr tmp = ++ std::make_unique(); ++ tmp->FeaturesInfo.reset(new ACPOCollectFeatures::FeaturesInfo{Info}); ++ tmp->FeatureCollector.reset(new ACPOCollectFeatures{}); ++ tmp->Prefix = Pre; ++ tmp->Postfix = Post; ++ ++ FeatureCollectInfos.push_back(std::move(tmp)); ++} ++ ++void ModelDataCollector::registerFeature( ++ ACPOCollectFeatures::Scopes ScopeVec, ++ ACPOCollectFeatures::FeatureInfo GlobalInfo, std::string Pre, ++ std::string Post) { ++ std::unique_ptr tmp = ++ std::make_unique(); ++ tmp->RegisteredScopes.reset(new ACPOCollectFeatures::Scopes{ScopeVec}); ++ tmp->FeatureCollector.reset(new ACPOCollectFeatures{}); ++ tmp->GlobalInfo.reset(new ACPOCollectFeatures::FeatureInfo{GlobalInfo}); ++ tmp->Prefix = Pre; ++ tmp->Postfix = Post; ++ ++ FeatureCollectInfos.push_back(std::move(tmp)); ++} ++ ++void ModelDataCollector::registerFeature( ++ ACPOCollectFeatures::GroupIDs GroupIDVec, ++ ACPOCollectFeatures::FeatureInfo GlobalInfo, std::string Pre, ++ std::string Post) { ++ std::unique_ptr tmp = ++ std::make_unique(); ++ tmp->RegisteredGroupIDs.reset(new ACPOCollectFeatures::GroupIDs{GroupIDVec}); ++ tmp->FeatureCollector.reset(new ACPOCollectFeatures{}); ++ tmp->GlobalInfo.reset(new ACPOCollectFeatures::FeatureInfo{GlobalInfo}); ++ tmp->Prefix = Pre; ++ tmp->Postfix = Post; ++ ++ FeatureCollectInfos.push_back(std::move(tmp)); ++} ++ ++void ModelDataCollector::resetRegisteredFeatures() { ++ FeatureCollectInfos.clear(); ++ Features.clear(); ++} ++ ++std::string ModelDataCollector::demangleName(const std::string &Name) { ++ ItaniumPartialDemangler D; ++ if (!D.partialDemangle(Name.c_str())) ++ return D.getFunctionBaseName(nullptr, nullptr); ++ ++ return Name; ++} ++ ++void ModelDataCollector::setFeatures( ++ std::vector> NewFeatures) { ++ Features = NewFeatures; ++} ++ ++void ModelDataCollector::addFeatures( ++ std::vector> NewFeatures) { ++ Features.insert(Features.end(), NewFeatures.begin(), NewFeatures.end()); ++} ++ ++void ModelDataCollector::setIRFileNameMap(StringMap IRFileNameMap) { ++ IRFileNames = IRFileNameMap; ++} ++ ++void ModelDataCollector::printRow(bool printHeader) { ++ // Print the IR file names first ++ for (const auto &P : IRFileNames) { ++ if (printHeader) ++ Out << P.getKey(); ++ else ++ Out << P.getValue(); ++ ++ Out << ","; ++ } ++ ++ for (unsigned I = 0, E = Features.size(); I != E; ++I ) { ++ // First value does not get a comma ++ if (I) ++ Out << ","; ++ ++ if (printHeader) ++ Out << Features.at(I).first; ++ else ++ Out << Features.at(I).second; ++ } ++ ++ Out << "\n"; ++} ++ ++/*std::string ModelDataCollector::generateIRFileName(autotuning::CodeRegion CR) { ++ // File name = source_location + pass_name + coderegion_type + hash, ++ // where source_location = file_name + func_name + loop_name ++ // + line_number + column_number ++ std::string IRFileName = ++ sys::path::filename(StringRef(CR.getFileName())).str() + "_" ++ + demangleName(CR.getFuncName()) + "_" ++ + CR.getName() + "_" ++ + std::to_string(CR.getSourceLoc().SourceLine) + "_" ++ + std::to_string(CR.getSourceLoc().SourceColumn) + "_" ++ + CR.getPassName() + "_" ++ + CR.getTypeAsString() + "_" ++ + std::to_string(CR.getHash()) + ".ll"; ++ return IRFileName; ++}*/ ++ ++std::string ModelDataCollector::getIRFileName(StringRef Key) { ++ if (IRFileNames.count(Key)) ++ return IRFileNames.find(Key)->second; ++ ++ return "None"; ++} ++ ++std::unique_ptr ++ModelDataCollector::createFile(const Twine &FilePath, ++ const Twine &FileName, ++ std::error_code &EC) { ++ if (std::error_code EC = sys::fs::create_directories(FilePath)) ++ errs() << "Error creating directory: " << FilePath << ": " ++ << EC.message() << "\n"; ++ ++ return std::make_unique((FilePath + "/" + FileName).str(), EC); ++} ++ ++void ModelDataCollector::createIRFileForLoop(Loop *L, const Twine &IRFilePath, ++ const Twine &IRFileName, ++ bool OverwriteIRFile) { ++ if (!OverwriteIRFile && sys::fs::exists(IRFilePath + "/" + IRFileName)) ++ return; ++ ++ // Write IR to file ++ std::error_code EC; ++ auto OS = createFile(IRFilePath, Twine(IRFileName), EC); ++ if (EC) { ++ errs() << "Error creating loop IR file: " << IRFileName << ": " ++ << EC.message() << "\n"; ++ return; ++ } ++ ++ // Print loop wrapped in function if -unnamed-var-prefix is set by user ++ if (UnnamedVariablePrefix.getNumOccurrences() > 0) { ++ SmallVector ExitBlocks; ++ L->getExitBlocks(ExitBlocks); ++ // May need to move this code out of Loop data structure in LLVM. Will see. ++ L->printWithFunctionWrapper(*OS, L->getHeader()->getParent(), ++ L->getBlocks(), L->getHeader(), ExitBlocks, ++ /*AAW*/ nullptr, ++ /*ShouldPreserveUseListOrder*/ false, ++ /*IsForDebug*/ false); ++ } else { ++ L->print(*OS, /*Depth*/ 0, /*Verbose*/ true); ++ } ++} ++ ++void ModelDataCollector::createIRFileForFunction(Function *F, ++ const Twine &IRFilePath, ++ const Twine &IRFileName, ++ bool OverwriteIRFile) { ++ if (!OverwriteIRFile && sys::fs::exists(IRFilePath + "/" + IRFileName)) ++ return; ++ ++ // Write IR to file ++ std::error_code EC; ++ auto OS = createFile(IRFilePath, Twine(IRFileName), EC); ++ if (EC) { ++ errs() << "Error creating function IR file: " << IRFileName << ": " ++ << EC.message() << "\n"; ++ return; ++ } ++ // May need to investigate this print function change. ++ F->print(*OS, /*AAW*/ nullptr, /*ShouldPreserveUseListOrder*/ false, ++ /*IsForDebug*/ false); ++} ++ ++void ModelDataCollector::writeIR(Loop *L, Function *F, ++ std::string NewIRFileName, ++ std::string PassName, ++ DumpOption DumpBeforeOrAfter, bool PrintLoop, ++ bool PrintFunction, bool OverwriteIRFile) { ++ // Create base directory first ++ SmallString<256> IRFilePath; ++ if (IRFileDirectory.getNumOccurrences() > 0) { ++ // Third priority is the directory specified by ++ // the -IR-file-directory option ++ Twine BaseDir(IRFileDirectory); ++ BaseDir.toVector(IRFilePath); ++ } else { ++ // No directory specified ++ return; ++ } ++ ++ if (getDumpOptionAsString(DumpBeforeOrAfter).empty()) ++ return; ++ ++ // Create sub-directories to store corresponding IR files. ++ // Directory name = before/after + pass_name + coderegion_type ++ std::string SubDir = getDumpOptionAsString(DumpBeforeOrAfter) ++ + "_" + PassName; ++ if (L && PrintLoop) { ++ createIRFileForLoop(L, ++ Twine(IRFilePath) + "/" + SubDir + "_" + ++ getDumpOptionAsString(DumpOption::loop), ++ Twine(NewIRFileName), OverwriteIRFile); ++ // Add IR file name for summary data file ++ IRFileNames.insert(std::pair ( ++ getDumpOptionAsString(DumpBeforeOrAfter) ++ + getDumpOptionAsString(DumpOption::loop), ++ NewIRFileName)); ++ } ++ ++ if (F && PrintFunction) { ++ createIRFileForFunction(F, ++ Twine(IRFilePath) + "/" + SubDir + "_" + ++ getDumpOptionAsString(DumpOption::function), ++ Twine(NewIRFileName), OverwriteIRFile); ++ // Add IR file name for summary data file ++ IRFileNames.insert(std::pair ( ++ getDumpOptionAsString(DumpBeforeOrAfter) ++ + getDumpOptionAsString(DumpOption::function), ++ NewIRFileName)); ++ } ++} ++#endif // ENABLE_ACPO +diff --git a/llvm/lib/CodeGen/CMakeLists.txt b/llvm/lib/CodeGen/CMakeLists.txt +index 9029dc7bb3d9..579074408b55 100644 +--- a/llvm/lib/CodeGen/CMakeLists.txt ++++ b/llvm/lib/CodeGen/CMakeLists.txt +@@ -1,4 +1,4 @@ +-if (DEFINED LLVM_HAVE_TF_AOT OR LLVM_HAVE_TFLITE) ++if ((DEFINED LLVM_HAVE_TF_AOT OR DEFINED LLVM_HAVE_TF_API) AND (NOT ACPO_AOT)) + include(TensorFlowCompile) + set(LLVM_RAEVICT_MODEL_PATH_DEFAULT "models/regalloc-eviction") + +diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp +index af77e6c2dc4d..a02c603a14a5 100644 +--- a/llvm/lib/IR/AsmWriter.cpp ++++ b/llvm/lib/IR/AsmWriter.cpp +@@ -86,8 +86,16 @@ + #include + #include + ++#include "llvm/ADT/StringSet.h" ++#include "llvm/Analysis/LoopInfo.h" ++#include "llvm/Support/CommandLine.h" ++ + using namespace llvm; + ++cl::opt UnnamedVariablePrefix( ++ "unnamed-var-prefix", cl::Hidden, ++ cl::desc("Specify the prefix added to unnamed variables"), cl::init("")); ++ + // Make virtual table appear in this compilation unit. + AssemblyAnnotationWriter::~AssemblyAnnotationWriter() = default; + +@@ -2487,9 +2495,11 @@ static void WriteAsOperandInternal(raw_ostream &Out, const Value *V, + Slot = -1; + } + +- if (Slot != -1) +- Out << Prefix << Slot; +- else ++ if (Slot != -1) { ++ // By default, UnnamedVariablePrefix is empty so it matches original behaviour ++ // unless specified. ++ Out << Prefix << UnnamedVariablePrefix << Slot; ++ } else + Out << ""; + } + +@@ -2602,12 +2612,13 @@ public: + void writeAllAttributeGroups(); + + void printTypeIdentities(); +-#if defined(ENABLE_AUTOTUNER) ++#if defined(ENABLE_AUTOTUNER) || defined(ENABLE_ACPO) + void printGlobal(const GlobalVariable *GV, bool PrintDeclarationOnly = false); + void printAlias(const GlobalAlias *GA); + void printIFunc(const GlobalIFunc *GI); + void printComdat(const Comdat *C); +- void printRequisiteDeclarations(const Function *F); ++ void printRequisiteDeclarations(const Function *F, ++ std::vector LoopBlocks = {}); + void printFunction(const Function *F, bool PrintCompleteIR = false, + bool PrintDeclarationOnly = false); + #else +@@ -2616,9 +2627,15 @@ public: + void printIFunc(const GlobalIFunc *GI); + void printComdat(const Comdat *C); + void printFunction(const Function *F); ++#endif ++#if defined(ENABLE_ACPO) ++ void printLoopWithFunctionWrapper(Function *F, ++ std::vector LoopBlocks, ++ BasicBlock *Header, ++ SmallVector ExitBlocks); + #endif + void printArgument(const Argument *FA, AttributeSet Attrs); +- void printBasicBlock(const BasicBlock *BB); ++ void printBasicBlock(const BasicBlock *BB, bool PrintLabelOnly = false); + void printInstructionLine(const Instruction &I); + void printInstruction(const Instruction &I); + +@@ -3603,7 +3620,7 @@ static void maybePrintComdat(formatted_raw_ostream &Out, + Out << ')'; + } + +-#if defined(ENABLE_AUTOTUNER) ++#if defined(ENABLE_AUTOTUNER) || defined(ENABLE_ACPO) + void AssemblyWriter::printGlobal(const GlobalVariable *GV, + bool PrintDeclarationOnly) { + if (GV->isMaterializable() && !PrintDeclarationOnly) +@@ -3617,7 +3634,7 @@ void AssemblyWriter::printGlobal(const GlobalVariable *GV) { + WriteAsOperandInternal(Out, GV, WriterCtx); + Out << " = "; + +-#if defined(ENABLE_AUTOTUNER) ++#if defined(ENABLE_AUTOTUNER) || defined(ENABLE_ACPO) + if ((!GV->hasInitializer() || PrintDeclarationOnly) && + GV->hasExternalLinkage()) + #else +@@ -3640,7 +3657,7 @@ void AssemblyWriter::printGlobal(const GlobalVariable *GV) { + Out << (GV->isConstant() ? "constant " : "global "); + TypePrinter.print(GV->getValueType(), Out); + +-#if defined(ENABLE_AUTOTUNER) ++#if defined(ENABLE_AUTOTUNER) || defined(ENABLE_ACPO) + if (GV->hasInitializer() && !PrintDeclarationOnly) { + #else + if (GV->hasInitializer()) { +@@ -3794,21 +3811,34 @@ void AssemblyWriter::printTypeIdentities() { + } + } + +-#if defined(ENABLE_AUTOTUNER) ++#if defined(ENABLE_AUTOTUNER) || defined(ENABLE_ACPO) + /// printRequisiteDeclarations - Print the declarations of type identities, + /// global variables, functions, and function attribute groups of a function. +-void AssemblyWriter::printRequisiteDeclarations(const Function *F) { ++void AssemblyWriter::printRequisiteDeclarations( ++ const Function *F, std::vector LoopBlocks) { + // walk through instructions and collect global variables & functions + SmallPtrSet GVs; + SmallPtrSet Functions; +- for (const BasicBlock &BB : *F) { +- for (const Instruction &I : BB) { ++ std::vector BasicBlocks; ++ if (!LoopBlocks.empty()) { ++ for (BasicBlock *BB : LoopBlocks) ++ BasicBlocks.push_back(BB); ++ } else { ++ for (const BasicBlock &BB : *F) ++ BasicBlocks.push_back(const_cast(&BB)); ++ } ++ ++ for (const BasicBlock *BB : BasicBlocks) { ++ for (const Instruction &I : *BB) { + // Check for function + if (const auto *CI = dyn_cast(&I)) { + Function *func = CI->getCalledFunction(); + if (func) + Functions.insert(func); + } ++ if (const InvokeInst *II = dyn_cast(&I)) ++ if (Function *func = dyn_cast(II->getCalledOperand())) ++ Functions.insert(func); + // Check for global variables + for (const Use &U : I.operands()) { + if (GlobalVariable *gv = dyn_cast(U)) +@@ -3823,6 +3853,16 @@ void AssemblyWriter::printRequisiteDeclarations(const Function *F) { + GVs.insert(gv); + } + } ++ // Check for ConstantExpr BitCast ++ if (const auto *CstExpr = dyn_cast(U)) ++ if (CstExpr->isCast()) ++ for (const Use &UU : CstExpr->operands()) { ++ if (GlobalVariable *gv = dyn_cast(UU)) ++ GVs.insert(gv); ++ else if (const Function *func = ++ dyn_cast(CstExpr->stripPointerCasts())) ++ Functions.insert(const_cast(func)); ++ } + } + } + } +@@ -3842,7 +3882,7 @@ void AssemblyWriter::printRequisiteDeclarations(const Function *F) { + // modify property if needed + if (!(*GVit)->hasAvailableExternallyLinkage() && + !((*GVit)->getName() == "llvm.global_ctors") && +- (*GVit)->hasLocalLinkage()) { ++ ((*GVit)->hasLocalLinkage() || (*GVit)->hasCommonLinkage())) { + (*GVit)->setLinkage(GlobalValue::ExternalLinkage); + (*GVit)->setVisibility(GlobalValue::HiddenVisibility); + } +@@ -3860,8 +3900,14 @@ void AssemblyWriter::printRequisiteDeclarations(const Function *F) { + // print functions + for (auto FuncIt = Functions.begin(), et = Functions.end(); FuncIt != et; + ++FuncIt) { ++ if (!LoopBlocks.empty() && *FuncIt == F) ++ continue; + Out << '\n'; ++ GlobalValue::LinkageTypes SavedLinkage = (*FuncIt)->getLinkage(); ++ // Function declarations can only have external or extern_weak linkage ++ (*FuncIt)->setLinkage(GlobalValue::ExternalLinkage); + printFunction(*FuncIt, false, true); ++ (*FuncIt)->setLinkage(SavedLinkage); + } + + // Write attribute groups. +@@ -3873,7 +3919,8 @@ void AssemblyWriter::printRequisiteDeclarations(const Function *F) { + } + + /// printFunction - Print all aspects of a function. +-void AssemblyWriter::printFunction(const Function *F, bool PrintCompleteIR, ++void AssemblyWriter::printFunction(const Function *F, ++ bool PrintCompleteIR, + bool PrintDeclarationOnly) { + if (PrintCompleteIR && !PrintDeclarationOnly) { + printRequisiteDeclarations(F); +@@ -3887,6 +3934,9 @@ void AssemblyWriter::printFunction(const Function *F, bool PrintCompleteIR, + void AssemblyWriter::printFunction(const Function *F) { + if (AnnotationWriter) AnnotationWriter->emitFunctionAnnot(F, Out); + ++ if (AnnotationWriter) ++ AnnotationWriter->emitFunctionAnnot(F, Out); ++ + if (F->isMaterializable()) + Out << "; Materializable\n"; + #endif +@@ -3907,7 +3957,7 @@ void AssemblyWriter::printFunction(const Function *F) { + Out << "; Function Attrs: " << AttrStr << '\n'; + } + +-#if defined(ENABLE_AUTOTUNER) ++#if defined(ENABLE_AUTOTUNER) || defined(ENABLE_ACPO) + if (!PrintDeclarationOnly) + Machine.incorporateFunction(F); + +@@ -3952,7 +4002,7 @@ void AssemblyWriter::printFunction(const Function *F) { + Out << '('; + + // Loop over the arguments, printing them... +-#if defined(ENABLE_AUTOTUNER) ++#if defined(ENABLE_AUTOTUNER) || defined(ENABLE_ACPO) + if ((F->isDeclaration() && !IsForDebug) || PrintDeclarationOnly) { + #else + if (F->isDeclaration() && !IsForDebug) { +@@ -4027,7 +4077,7 @@ void AssemblyWriter::printFunction(const Function *F) { + writeOperand(F->getPersonalityFn(), /*PrintType=*/true); + } + +-#if defined(ENABLE_AUTOTUNER) ++#if defined(ENABLE_AUTOTUNER) || defined(ENABLE_ACPO) + if (F->isDeclaration() || PrintDeclarationOnly) { + #else + if (F->isDeclaration()) { +@@ -4049,16 +4099,102 @@ void AssemblyWriter::printFunction(const Function *F) { + Out << "}\n"; + } + +-#if defined(ENABLE_AUTOTUNER) ++#if defined(ENABLE_AUTOTUNER) || defined(ENABLE_ACPO) + // Output metadata + if (!Machine.mdn_empty() && PrintCompleteIR && !PrintDeclarationOnly) { + Out << '\n'; + writeAllMDNodes(); + } +-#endif ++ if (!PrintDeclarationOnly) ++ Machine.purgeFunction(); ++#else + Machine.purgeFunction(); ++#endif + } + ++#if defined(ENABLE_ACPO) ++/// printLoopWithFunctionWrapper - print out a loop wrapped in a dummy ++/// function. All global/local variables, functions and metadata that ++/// are referenced inside the loop are printed out. Loop predecessors ++/// and loop exit blocks are also included. ++void AssemblyWriter::printLoopWithFunctionWrapper( ++ Function *F, std::vector LoopBlocks, BasicBlock *Header, ++ SmallVector ExitBlocks) { ++ printRequisiteDeclarations(F, LoopBlocks); ++ ++ // Output the dummy function ++ bool IsFirstArgument = true; ++ Out << "define void "; ++ std::string FunctionName = "foo"; ++ PrintLLVMName(Out, FunctionName, GlobalPrefix); ++ Out << '('; ++ for (const Argument &Arg : F->args()) { ++ if (IsFirstArgument) // Add commas if there are more than one ++ IsFirstArgument = false; ++ else ++ Out << ", "; ++ Out << &Arg; ++ } ++ ++ // All local variables referenced in this loop but are not declared here ++ // are printed next in the argument list ++ SmallPtrSet AddedVariables; ++ for (const BasicBlock *BB : LoopBlocks) ++ for (const Instruction &I : *BB) ++ for (unsigned i = 0, e = I.getNumOperands(); i != e; ++i) { ++ Value *Op = I.getOperand(i); ++ // Print out the operand in the function argument list ++ // if it is an instruction that is not contained in this loop ++ if (const Instruction *II = dyn_cast_or_null(Op)) ++ if (!AddedVariables.contains(II) && ++ std::find(LoopBlocks.begin(), LoopBlocks.end(), ++ II->getParent()) != LoopBlocks.end()) { ++ AddedVariables.insert(II); ++ if (IsFirstArgument) ++ IsFirstArgument = false; ++ else ++ Out << ", "; ++ ++ writeOperand(Op, true); ++ } ++ } ++ ++ Out << ") {\n"; ++ ++ // Output loop predecessors ++ // Each predecessor only needs to have an unconditional 'br' instruction ++ // that branches to the loop header ++ for (const BasicBlock *Pred : children>(Header)) ++ // If the block is not in the loop ++ if (std::find(LoopBlocks.begin(), LoopBlocks.end(), Pred) != ++ LoopBlocks.end()) { ++ printBasicBlock(Pred, true); ++ Out << " br label %"; ++ PrintLLVMName(Out, Header->getName(), LabelPrefix); ++ Out << "\n"; ++ } ++ ++ // Output all of the loop's basic blocks ++ for (const BasicBlock *BB : LoopBlocks) ++ printBasicBlock(BB); ++ ++ // Output loop exit blocks ++ // Each exit block only needs a 'ret' instruction ++ for (const BasicBlock *Succ : ExitBlocks) { ++ printBasicBlock(Succ, true); ++ Out << " ret void\n"; ++ } ++ ++ Out << "}\n"; ++ ++ // Output metadata ++ if (!Machine.mdn_empty()) { ++ Out << '\n'; ++ writeAllMDNodes(); ++ } ++} ++#endif ++ + /// printArgument - This member is called for every argument that is passed into + /// the function. Simply print it out + void AssemblyWriter::printArgument(const Argument *Arg, AttributeSet Attrs) { +@@ -4078,13 +4214,17 @@ void AssemblyWriter::printArgument(const Argument *Arg, AttributeSet Attrs) { + } else { + int Slot = Machine.getLocalSlot(Arg); + assert(Slot != -1 && "expect argument in function here"); +- Out << " %" << Slot; ++ // By default, UnnamedVariablePrefix is empty so it matches original behaviour ++ // unless specified. ++ Out << " %" << UnnamedVariablePrefix << Slot; + } + } + + /// printBasicBlock - This member is called for each basic block in a method. +-void AssemblyWriter::printBasicBlock(const BasicBlock *BB) { +- bool IsEntryBlock = BB->getParent() && BB->isEntryBlock(); ++void AssemblyWriter::printBasicBlock(const BasicBlock *BB, ++ bool PrintLabelOnly) { ++ assert(BB && BB->getParent() && "block without parent!"); ++ bool IsEntryBlock = BB == &BB->getParent()->getEntryBlock(); + if (BB->hasName()) { // Print out the label if it exists... + Out << "\n"; + PrintLLVMName(Out, BB->getName(), LabelPrefix); +@@ -4092,12 +4232,19 @@ void AssemblyWriter::printBasicBlock(const BasicBlock *BB) { + } else if (!IsEntryBlock) { + Out << "\n"; + int Slot = Machine.getLocalSlot(BB); +- if (Slot != -1) +- Out << Slot << ":"; +- else ++ if (Slot != -1) { ++ // By default, UnnamedVariablePrefix is empty so it matches original behaviour ++ // unless specified. ++ Out << UnnamedVariablePrefix << Slot << ":"; ++ } else + Out << ":"; + } + ++ if (PrintLabelOnly) { ++ Out << "\n"; ++ return; ++ } ++ + if (!IsEntryBlock) { + // Output predecessors for the block. + Out.PadToColumn(50); +@@ -4191,8 +4338,11 @@ void AssemblyWriter::printInstruction(const Instruction &I) { + int SlotNum = Machine.getLocalSlot(&I); + if (SlotNum == -1) + Out << " = "; +- else +- Out << '%' << SlotNum << " = "; ++ else { ++ // By default, UnnamedVariablePrefix is empty so it matches original behaviour ++ // unless specified. ++ Out << '%' << UnnamedVariablePrefix << SlotNum << " = "; ++ } + } + + if (const CallInst *CI = dyn_cast(&I)) { +@@ -4762,6 +4912,20 @@ void BasicBlock::print(raw_ostream &ROS, AssemblyAnnotationWriter *AAW, + W.printBasicBlock(this); + } + ++#if defined(ENABLE_ACPO) ++void Loop::printWithFunctionWrapper( ++ raw_ostream &ROS, Function *F, ArrayRef LoopBlocks, ++ BasicBlock *Header, SmallVector ExitBlocks, ++ AssemblyAnnotationWriter *AAW, bool ShouldPreserveUseListOrder, ++ bool IsForDebug) const { ++ SlotTracker SlotTable(F); ++ formatted_raw_ostream OS(ROS); ++ AssemblyWriter W(OS, SlotTable, F->getParent(), AAW, IsForDebug, ++ ShouldPreserveUseListOrder); ++ W.printLoopWithFunctionWrapper(F, LoopBlocks, Header, ExitBlocks); ++} ++#endif ++ + void Module::print(raw_ostream &ROS, AssemblyAnnotationWriter *AAW, + bool ShouldPreserveUseListOrder, bool IsForDebug) const { + SlotTracker SlotTable(this); +diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp +index a3ccbc6d258f..e2fe3322aef4 100644 +--- a/llvm/lib/Passes/PassBuilder.cpp ++++ b/llvm/lib/Passes/PassBuilder.cpp +@@ -267,6 +267,12 @@ + #include "llvm/Transforms/Scalar/AutoTuningCompile.h" + #endif + ++#if defined(ENABLE_ACPO) ++#include "llvm/Analysis/CallHeight.h" ++#include "llvm/Analysis/DumpCallsite.h" ++#include "llvm/Analysis/DumpFeature.h" ++#endif ++ + using namespace llvm; + + static const Regex DefaultAliasRegex( +diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp +index 8009e011833c..de89f5393ba2 100644 +--- a/llvm/lib/Passes/PassBuilderPipelines.cpp ++++ b/llvm/lib/Passes/PassBuilderPipelines.cpp +@@ -138,6 +138,12 @@ + #include "llvm/Transforms/Scalar/AutoTuningCompile.h" + #endif + ++#if defined(ENABLE_ACPO) ++#include "llvm/Analysis/CallHeight.h" ++#include "llvm/Analysis/DumpCallsite.h" ++#include "llvm/Analysis/DumpFeature.h" ++#endif ++ + using namespace llvm; + + static cl::opt UseInlineAdvisor( +@@ -894,6 +900,14 @@ PassBuilder::buildInlinerPipeline(OptimizationLevel Level, + // make a lot of sense and we should revisit the core CGSCC structure. + CGSCCPassManager &MainCGPipeline = MIWP.getPM(); + ++#if defined(ENABLE_ACPO) ++ if (EnableFeatureDump) { ++ // Add CallHeight analysis for dump feature ++ MIWP.addModulePass(RequireAnalysisPass()); ++ MainCGPipeline.addPass(DumpFeaturePass()); ++ } ++#endif ++ + // Note: historically, the PruneEH pass was run first to deduce nounwind and + // generally clean up exception handling overhead. It isn't clear this is + // valuable as the inliner doesn't currently care whether it is inlining an +diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def +index 45a539f14b93..6ef0d6791ff2 100644 +--- a/llvm/lib/Passes/PassRegistry.def ++++ b/llvm/lib/Passes/PassRegistry.def +@@ -33,6 +33,10 @@ MODULE_ANALYSIS("ir-similarity", IRSimilarityAnalysis()) + MODULE_ANALYSIS("autotuning-dump", AutotuningDumpAnalysis()) + #endif + ++#if defined(ENABLE_ACPO) ++MODULE_ANALYSIS("call-height", CallHeightAnalysis()) ++#endif ++ + #ifndef MODULE_ALIAS_ANALYSIS + #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS) \ + MODULE_ANALYSIS(NAME, CREATE_PASS) +@@ -215,6 +219,9 @@ CGSCC_PASS("invalidate", InvalidateAllAnalysesPass()) + CGSCC_PASS("attributor-cgscc", AttributorCGSCCPass()) + CGSCC_PASS("openmp-opt-cgscc", OpenMPOptCGSCCPass()) + CGSCC_PASS("no-op-cgscc", NoOpCGSCCPass()) ++#if defined(ENABLE_ACPO) ++CGSCC_PASS("dump-feature", DumpFeaturePass()) ++#endif + #undef CGSCC_PASS + + #ifndef CGSCC_PASS_WITH_PARAMS +@@ -325,6 +332,9 @@ FUNCTION_PASS("view-dom", DomViewer()) + FUNCTION_PASS("view-dom-only", DomOnlyViewer()) + FUNCTION_PASS("view-post-dom", PostDomViewer()) + FUNCTION_PASS("view-post-dom-only", PostDomOnlyViewer()) ++#if defined(ENABLE_ACPO) ++FUNCTION_PASS("dump-callsite", DumpCallsitePass()) ++#endif + FUNCTION_PASS("fix-irreducible", FixIrreduciblePass()) + FUNCTION_PASS("flattencfg", FlattenCFGPass()) + FUNCTION_PASS("make-guards-explicit", MakeGuardsExplicitPass()) +-- +2.38.1.windows.1 + diff --git a/0032-ACPO-Introduce-MLInliner-using-ACPO-infrastructure.patch b/0032-ACPO-Introduce-MLInliner-using-ACPO-infrastructure.patch new file mode 100644 index 0000000000000000000000000000000000000000..31ac7a998d11ef180a087cb05547f276e1770943 --- /dev/null +++ b/0032-ACPO-Introduce-MLInliner-using-ACPO-infrastructure.patch @@ -0,0 +1,1748 @@ +From 35db41f1fc006aa06fb012ec942d17c93bf0f8d5 Mon Sep 17 00:00:00 2001 +From: Amir Ashouri +Date: Thu, 22 Aug 2024 23:58:33 -0400 +Subject: [PATCH] [ACPO] Introduce MLInliner using ACPO infrastructure + +This change adds ML model to the inliner for performance optimization. +--- + .../llvm/Analysis/ACPOCollectFeatures.h | 3 + + llvm/include/llvm/Analysis/ACPOFIModel.h | 144 +++++++++ + llvm/include/llvm/Analysis/ACPOMLInterface.h | 4 +- + llvm/include/llvm/Analysis/ACPOModel.h | 2 + + llvm/include/llvm/Analysis/ACPOModelRunner.h | 2 + + llvm/include/llvm/Analysis/AOTModelRunner.h | 2 + + llvm/include/llvm/Analysis/CallHeight.h | 2 + + llvm/include/llvm/Analysis/DumpCallsite.h | 2 + + llvm/include/llvm/Analysis/DumpFeature.h | 2 + + llvm/include/llvm/Analysis/FIModelRunner.h | 277 ++++++++++++++++++ + llvm/include/llvm/Analysis/InlineAdvisor.h | 30 ++ + .../llvm/Analysis/InlineModelFeatureMaps.h | 30 ++ + llvm/include/llvm/Analysis/MLInlineAdvisor.h | 5 +- + llvm/include/llvm/InitializePasses.h | 6 + + llvm/include/llvm/Transforms/IPO.h | 4 + + llvm/include/llvm/Transforms/IPO/Inliner.h | 9 + + llvm/lib/Analysis/ACPOCollectFeatures.cpp | 2 + + llvm/lib/Analysis/ACPOFIModel.cpp | 243 +++++++++++++++ + llvm/lib/Analysis/ACPOMLInterface.cpp | 2 + + llvm/lib/Analysis/ACPOModel.cpp | 2 + + llvm/lib/Analysis/CallHeight.cpp | 3 + + llvm/lib/Analysis/DumpCallsite.cpp | 2 + + llvm/lib/Analysis/DumpFeature.cpp | 3 + + llvm/lib/Analysis/InlineAdvisor.cpp | 115 ++++++++ + llvm/lib/Analysis/MLInlineAdvisor.cpp | 5 + + llvm/lib/IR/AsmWriter.cpp | 35 ++- + llvm/lib/Transforms/IPO/Inliner.cpp | 219 +++++++++++++- + llvm/tools/opt/opt.cpp | 3 + + 28 files changed, 1151 insertions(+), 7 deletions(-) + create mode 100644 llvm/include/llvm/Analysis/ACPOFIModel.h + create mode 100644 llvm/include/llvm/Analysis/FIModelRunner.h + create mode 100644 llvm/lib/Analysis/ACPOFIModel.cpp + +diff --git a/llvm/include/llvm/Analysis/ACPOCollectFeatures.h b/llvm/include/llvm/Analysis/ACPOCollectFeatures.h +index ec62b559542d..8b266b3bc756 100644 +--- a/llvm/include/llvm/Analysis/ACPOCollectFeatures.h ++++ b/llvm/include/llvm/Analysis/ACPOCollectFeatures.h +@@ -10,6 +10,8 @@ + // collected on a given ACPOModel class from all available features. + // + //===----------------------------------------------------------------------===// ++ ++#if defined(ENABLE_ACPO) + #ifndef LLVM_ANALYSIS_ACPOCOLLECTFEATURES_H + #define LLVM_ANALYSIS_ACPOCOLLECTFEATURES_H + #include "llvm/Analysis/InlineAdvisor.h" +@@ -294,3 +296,4 @@ operator++(ACPOCollectFeatures::FeatureIndex &, int); + + } // namespace llvm + #endif // LLVM_ANALYSIS_ACPOCOLLECTFEATURES_H ++#endif // ENABLE_ACPO +diff --git a/llvm/include/llvm/Analysis/ACPOFIModel.h b/llvm/include/llvm/Analysis/ACPOFIModel.h +new file mode 100644 +index 000000000000..8753dd3d7c63 +--- /dev/null ++++ b/llvm/include/llvm/Analysis/ACPOFIModel.h +@@ -0,0 +1,144 @@ ++//===- ACPOFIModel.h - AI-Enabled Continuous Program Optimization ---------===// ++// ++// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. ++// See https://llvm.org/LICENSE.txt for license information. ++// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ++// ++//===----------------------------------------------------------------------===// ++ ++#if defined(ENABLE_ACPO) ++#ifndef LLVM_ANALYSIS_ACPOFIMODEL_H ++#define LLVM_ANALYSIS_ACPOFIMODEL_H ++ ++#include "llvm/Analysis/ACPOModel.h" ++#include "llvm/Analysis/DumpFeature.h" ++#include "llvm/Analysis/FunctionPropertiesAnalysis.h" ++#include "llvm/Analysis/InlineAdvisor.h" ++#include "llvm/Analysis/InlineSizeEstimatorAnalysis.h" ++#include "llvm/Analysis/LoopInfo.h" ++#include "llvm/Analysis/TargetLibraryInfo.h" ++#include "llvm/Analysis/TargetTransformInfo.h" ++ ++#include ++ ++namespace llvm { ++ ++//class ACPOmodel; ++ ++class ACPOFIModel : public ACPOModel { ++public: ++ ACPOFIModel(CallBase *CB, InlineAdvisor *IA, OptimizationRemarkEmitter *ORE, ++ bool OnlyMandatory, bool UseML = true); ++ ++ ~ACPOFIModel(); ++ ++ void setMLCustomFeatures( ++ std::vector> FeatureValues); ++ ++ void sendCustomFeatures() override; ++ ++ InlineAdvisor *getNotACPOAdvisor(); ++ ++ // Recorder's to micmic the behavior for default InlineAdvice. ++ // If the model is turned off or was decided to fall back to ++ // default inline advisor then we need to make sure the advice returned ++ // is properly recorded. Or else there would be an error. ++ void recordUnattemptedInlining(); ++ ++ void recordInlining(); ++ ++ void recordUnsuccessfulInlining(InlineResult &IR); ++ ++ void recordInliningWithCalleeDeleted(); ++ ++ // Interface for IRToPerf Cache system. ++ struct FunctionFeaturesCache { ++ using FunctionSizeMap = DenseMap; ++ using FunctionFloatMap = DenseMap; ++ ++ std::array( ++ ACPOFIExtendedFeatures::NamedFeatureIndex::NumNamedFeatures)> ++ NamedFeatures; ++ std::array( ++ ACPOFIExtendedFeatures::NamedFloatFeatureIndex:: ++ NumNamedFloatFeatures)> ++ NamedFloatFeatures; ++ ++ FunctionSizeMap &operator[](ACPOFIExtendedFeatures::NamedFeatureIndex Pos) { ++ return NamedFeatures[static_cast(Pos)]; ++ } ++ FunctionFloatMap & ++ operator[](ACPOFIExtendedFeatures::NamedFloatFeatureIndex Pos) { ++ return NamedFloatFeatures[static_cast(Pos)]; ++ } ++ }; ++ ++ struct FunctionAnalysisMap { ++ DenseMap DomCache; ++ DenseMap LICache; ++ DenseMap TTICache; ++ }; ++ ++ // Invalidation mechanisms ++ static void invalidateCache(CallBase *CB); ++ ++ static void invalidateCache(const Function *F); ++ ++ static void clearCache(); ++ ++ // Getters/setters for the cache system. ++ static std::optional ++ getCachedSize(const Function *F, ++ ACPOFIExtendedFeatures::NamedFeatureIndex idx); ++ ++ static std::optional ++ getCachedFloat(const Function *F, ++ ACPOFIExtendedFeatures::NamedFloatFeatureIndex idx); ++ ++ static void insertSizeCache(const Function *F, ++ ACPOFIExtendedFeatures::NamedFeatureIndex idx, ++ size_t val); ++ ++ static void ++ insertFloatCache(const Function *F, ++ ACPOFIExtendedFeatures::NamedFloatFeatureIndex idx, ++ float val); ++ ++ static const DominatorTree *getDomCachedAnalysis(const Function *F); ++ ++ static const LoopInfo *getLICachedAnalysis(const Function *F); ++ ++ static const TargetTransformInfo *getTTICachedAnalysis(const Function *F); ++ ++ static void insertAnalysisCache(const Function *F, const DominatorTree *Tree); ++ ++ static void insertAnalysisCache(const Function *F, const LoopInfo *LI); ++ ++ static void insertAnalysisCache(const Function *F, ++ const TargetTransformInfo *TTI); ++ ++protected: ++ // Interface to run the MLInference/default advisor and get advice from the ++ // model/default advisor ++ virtual std::unique_ptr getAdviceML() override; ++ ++ virtual std::unique_ptr getAdviceNoML() override; ++ ++private: ++ static FunctionFeaturesCache FeatureCache; ++ static FunctionAnalysisMap FunctionAnalysisCache; ++ CallBase *CurrentCB = nullptr; ++ InlineAdvisor *NotACPOAdvisor = nullptr; ++ bool ShouldInline = false; ++ bool OnlyMandatory = false; ++ std::unique_ptr NotACPOAdvice = nullptr; ++ std::vector> CustomFeatureValues; ++}; ++ ++} // end namespace llvm ++ ++#endif // LLVM_ANALYSIS_ACPOFIMODEL_H ++ ++#endif // ENABLE_ACPO +diff --git a/llvm/include/llvm/Analysis/ACPOMLInterface.h b/llvm/include/llvm/Analysis/ACPOMLInterface.h +index 996f27ee32ba..fbc8a46b3d9a 100644 +--- a/llvm/include/llvm/Analysis/ACPOMLInterface.h ++++ b/llvm/include/llvm/Analysis/ACPOMLInterface.h +@@ -4,10 +4,9 @@ + // See https://llvm.org/LICENSE.txt for license information. + // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + // +-// Copyright (C) 2021-2022. Huawei Technologies Co., Ltd. All rights reserved. +-// + //===----------------------------------------------------------------------===// + ++#if defined(ENABLE_ACPO) + #ifndef LLVM_ANALYSIS_ACPOML_INTERFACE_H + #define LLVM_ANALYSIS_ACPOML_INTERFACE_H + +@@ -480,3 +479,4 @@ std::shared_ptr createPersistentCompiledMLIF(); + } // namespace llvm + + #endif // LLVM_ANALYSIS_ACPOML_INTERFACE_H ++#endif // ENABLE_ACPO +diff --git a/llvm/include/llvm/Analysis/ACPOModel.h b/llvm/include/llvm/Analysis/ACPOModel.h +index 34dbc0fdb8bf..d61ac00efaaf 100644 +--- a/llvm/include/llvm/Analysis/ACPOModel.h ++++ b/llvm/include/llvm/Analysis/ACPOModel.h +@@ -6,6 +6,7 @@ + // + //===----------------------------------------------------------------------===// + ++#if defined(ENABLE_ACPO) + #ifndef LLVM_ANALYSIS_ACPOMODEL_H + #define LLVM_ANALYSIS_ACPOMODEL_H + +@@ -120,3 +121,4 @@ private: + } // namespace llvm + + #endif // LLVM_ANALYSIS_ACPOMODEL_H ++#endif // ENABLE_ACPO +diff --git a/llvm/include/llvm/Analysis/ACPOModelRunner.h b/llvm/include/llvm/Analysis/ACPOModelRunner.h +index 819e17f71103..044f3af15bbe 100644 +--- a/llvm/include/llvm/Analysis/ACPOModelRunner.h ++++ b/llvm/include/llvm/Analysis/ACPOModelRunner.h +@@ -6,6 +6,7 @@ + // + //===----------------------------------------------------------------------===// + ++#if defined(ENABLE_ACPO) + #ifndef LLVM_ANALYSIS_ACPOMODEL_H + #define LLVM_ANALYSIS_ACPOMODEL_H + +@@ -37,3 +38,4 @@ protected: + } // namespace llvm + + #endif // LLVM_ANALYSIS_ACPOMODEL_H ++#endif // ENABLE_ACPO +diff --git a/llvm/include/llvm/Analysis/AOTModelRunner.h b/llvm/include/llvm/Analysis/AOTModelRunner.h +index abc6258c4f09..fe19a33a5a08 100644 +--- a/llvm/include/llvm/Analysis/AOTModelRunner.h ++++ b/llvm/include/llvm/Analysis/AOTModelRunner.h +@@ -6,6 +6,7 @@ + // + //===----------------------------------------------------------------------===// + ++#if defined(ENABLE_ACPO) + #ifndef LLVM_ANALYSIS_AOTMODEL_H + #define LLVM_ANALYSIS_AOTMODEL_H + +@@ -201,3 +202,4 @@ private: + } // namespace llvm + + #endif // LLVM_ANALYSIS_AOTMODEL_H ++#endif // ENABLE_ACPO +diff --git a/llvm/include/llvm/Analysis/CallHeight.h b/llvm/include/llvm/Analysis/CallHeight.h +index c1251081f525..84e94075ea39 100644 +--- a/llvm/include/llvm/Analysis/CallHeight.h ++++ b/llvm/include/llvm/Analysis/CallHeight.h +@@ -10,6 +10,7 @@ + // + //===----------------------------------------------------------------------===// + ++#if defined(ENABLE_ACPO) + #ifndef LLVM_ANALYSIS_CALLHEIGHT + #define LLVM_ANALYSIS_CALLHEIGHT + +@@ -70,3 +71,4 @@ Pass *createCallHeightAnalysisWrapper(); + } // namespace llvm + + #endif ++#endif // ENABLE_ACPO +diff --git a/llvm/include/llvm/Analysis/DumpCallsite.h b/llvm/include/llvm/Analysis/DumpCallsite.h +index 9f80fe1cb985..02238521580b 100644 +--- a/llvm/include/llvm/Analysis/DumpCallsite.h ++++ b/llvm/include/llvm/Analysis/DumpCallsite.h +@@ -10,6 +10,7 @@ + // + //===----------------------------------------------------------------------===// + ++#if defined(ENABLE_ACPO) + #ifndef LLVM_ANALYSIS_DUMPCALLSITE + #define LLVM_ANALYSIS_DUMPCALLSITE + +@@ -25,3 +26,4 @@ public: + } // namespace llvm + + #endif ++#endif // ENABLE_ACPO +diff --git a/llvm/include/llvm/Analysis/DumpFeature.h b/llvm/include/llvm/Analysis/DumpFeature.h +index 226e06cf5600..67ca36b106cb 100644 +--- a/llvm/include/llvm/Analysis/DumpFeature.h ++++ b/llvm/include/llvm/Analysis/DumpFeature.h +@@ -10,6 +10,7 @@ + // + //===----------------------------------------------------------------------===// + ++#if defined(ENABLE_ACPO) + #ifndef LLVM_ANALYSIS_DUMPFEATURE + #define LLVM_ANALYSIS_DUMPFEATURE + +@@ -192,3 +193,4 @@ operator++(ACPOFIExtendedFeatures::NamedFloatFeatureIndex &n, int); + } // namespace llvm + + #endif ++#endif // ENABLE_ACPO +diff --git a/llvm/include/llvm/Analysis/FIModelRunner.h b/llvm/include/llvm/Analysis/FIModelRunner.h +new file mode 100644 +index 000000000000..3685220aa074 +--- /dev/null ++++ b/llvm/include/llvm/Analysis/FIModelRunner.h +@@ -0,0 +1,277 @@ ++//===- FIModelRunner.h - AI-Enabled Continuous Program Optimization -------===// ++// ++// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. ++// See https://llvm.org/LICENSE.txt for license information. ++// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ++// ++//===----------------------------------------------------------------------===// ++ ++#if defined(ENABLE_ACPO) ++#ifdef LLVM_HAVE_TF_AOT_FICOMPILEDMODEL ++ ++#ifndef LLVM_ANALYSIS_FIMODELRUNNER_H ++#define LLVM_ANALYSIS_FIMODELRUNNER_H ++ ++#include "llvm/Analysis/AOTModelRunner.h" ++#include "llvm/Analysis/FICompiledModel.h" ++ ++namespace llvm { ++ ++class FIModelRunner : public AOTModelRunner { ++ std::vector Means = { ++ 0.40009943697174110699421589742996729910373687744141, ++ 0.0, ++ 47.2218788212687599070704891346395015716552734375, ++ 0.0, ++ 0.07675459224122871404460966004990041255950927734375, ++ 5816.8243862454482950852252542972564697265625, ++ 1333.68016232413765465025790035724639892578125, ++ 321.9700210967629345759632997214794158935546875, ++ 0.94076781467098458122677584469784051179885864257812, ++ 0.0, ++ 0.0, ++ 24.57427538666200916850357316434383392333984375, ++ 0.72785175828753412297089653293369337916374206542969, ++ 22.362582136282401990001744707114994525909423828125, ++ 2.3236404681600126842511144786840304732322692871094, ++ 219.476437468925951179699040949344635009765625, ++ 123.872156304169635632206336595118045806884765625, ++ 759.6211988873809559663641266524791717529296875, ++ 3.5118047810371009198604497214546427130699157714844, ++ 0.0, ++ 14.689125089022963877027905255090445280075073242188, ++ 0.2720138674263292699606608948670327663421630859375, ++ 97.33707789677367827607668004930019378662109375, ++ 5.4576519437240493815011177503038197755813598632812, ++ 222416123463299168.0, ++ 697004967939498496.0, ++ 6.2712796684314486839184610289521515369415283203125, ++ 1.4856427516360068974421437815180979669094085693359, ++ 0.0041427067953076499376430241738944459939375519752502, ++ 0.72785175828753412297089653293369337916374206542969, ++ 552.7808652140716958456323482096195220947265625, ++ 62.5524652090595196796130039729177951812744140625, ++ 385.68509386043888298445381224155426025390625, ++ 92.9494483935554143272383953444659709930419921875, ++ 24.2728066757145342080548289231956005096435546875, ++ 0.90531987798814816947867711860453709959983825683594, ++ 0.0, ++ 0.0, ++ 2.9322753597871509256833633116912096738815307617188, ++ 0.49584111584407208894731411419343203306198120117188, ++ 7.9963853317029256473347231803927570581436157226562, ++ 1.4571144465795025091381376114441081881523132324219, ++ 15.557169540036818844441768305841833353042602539062, ++ 9.6481678066085265754736610688269138336181640625, ++ 50.98738225453177363988288561813533306121826171875, ++ 1.3425469302194332765765238946187309920787811279297, ++ 0.0, ++ 839.271140434566405019722878932952880859375, ++ 0.16440693908813608370422798543586395680904388427734, ++ 2.8829196844891762374629706755513325333595275878906, ++ 132.0555906421747067724936641752719879150390625, ++ 92791372484119440.0, ++ 166968642875823456.0, ++ 5.5557876796248262252220229129306972026824951171875, ++ 1.1750766644405326033506753446999937295913696289062, ++ 0.0042161570432282073628282859090177225880324840545654, ++ 0.49584111584407208894731411419343203306198120117188, ++ 41.15953665944181949498670292086899280548095703125, ++ 5.14903426051142787400749512016773223876953125, ++ 2.0527687821658449074391228350577875971794128417969, ++ 0.52614251736787642776960183255141600966453552246094, ++ 0.74523979091361081117383946548216044902801513671875, ++ 222.345100041656024814074044115841388702392578125, ++ 7.4997648449992606600744693423621356487274169921875, ++ 0.0, ++ 78.5584998454695693226312869228422641754150390625, ++ 0.0, ++ 10.409640011287439875786731136031448841094970703125, ++ 8.4653112780338357623577394406311213970184326171875, ++ 1.3630927585697201198655648113344796001911163330078, ++ 566.7381985783200661899172700941562652587890625, ++ 0.0, ++ 1.2066945269353257508271326514659449458122253417969, ++ 55.41075531786237462483768467791378498077392578125, ++ 0.51243634018194272883306439325679093599319458007812, ++ 1.1147556403606606600931172579294070601463317871094, ++ -31.471868743197301654390685143880546092987060546875, ++ 0.0, ++ 0.030368588666872708276001091576290491502732038497925, ++ 0.58478345583789081985059965518303215503692626953125, ++ 0.00034937314395517275094141251834400918596656993031502, ++ -0.23764092503258577027125397762574721127748489379883, ++ -62.20223330063559075142620713450014591217041015625, ++ 5.8952014942420616350204909394960850477218627929688, ++ 3339.09353794057960840291343629360198974609375, ++ 0.71960117711874660439974604742019437253475189208984, ++ -49.2720273048549444183663581497967243194580078125, ++ 27818.32155766672440222464501857757568359375, ++ 91.64824843118020680776680819690227508544921875, ++ 106.3296335613216996307528461329638957977294921875, ++ 469.83727273948858282892615534365177154541015625, ++ 0.30689743210739195422576131022651679813861846923828, ++ 1071.964175815315911677316762506961822509765625, ++ 1363.988766309679022015188820660114288330078125, ++ 14.079536139964256236112305487040430307388305664062, ++ 63165365211952664.0, ++ 0.38502264206721403816402471420587971806526184082031, ++ 0.015573979763232508391479491649533883901312947273254, ++ 0.13859363872129429329227434664062457159161567687988, ++ 0.0}; ++ ++ std::vector Scales = { ++ 0.48991823553184549178141082848014775663614273071289, ++ 1.0, ++ 19.2517211876445770712962257675826549530029296875, ++ 1.0, ++ 0.26620166192402217042456413764739409089088439941406, ++ 13580.447773648038491955958306789398193359375, ++ 3192.7079136089387247920967638492584228515625, ++ 633.0586155859824657454737462103366851806640625, ++ 0.23605875020885080939336830851971171796321868896484, ++ 1.0, ++ 1.0, ++ 101.565906032925312274528550915420055389404296875, ++ 0.44506581113952026207414292002795264124870300292969, ++ 25.4451961539476627649492002092301845550537109375, ++ 1.8819488669919737233726664271671324968338012695312, ++ 399.4446922340151786556816659867763519287109375, ++ 253.61174866766344848656444810330867767333984375, ++ 1934.51814232197148157865740358829498291015625, ++ 9.2671206485376131922748754732310771942138671875, ++ 1.0, ++ 101.7363052307218964642743230797350406646728515625, ++ 0.44499699252253444026194983962341211736202239990234, ++ 241.819662633324895750774885527789592742919921875, ++ 41.0624051346520815286567085422575473785400390625, ++ 1810657384453411584.0, ++ 2590019375355715584.0, ++ 18.6007475145233769353581010363996028900146484375, ++ 0.30589376767499054654564361044322140514850616455078, ++ 0.021661308027730186848147653222440567333251237869263, ++ 0.44506581113952026207414292002795264124870300292969, ++ 2210.9835111177717408281750977039337158203125, ++ 252.28469071093292086516157723963260650634765625, ++ 1479.28580699818076027440838515758514404296875, ++ 358.2883493183543350824038498103618621826171875, ++ 86.4399992258764626740230596624314785003662109375, ++ 0.29277260204409949473358665272826328873634338378906, ++ 1.0, ++ 1.0, ++ 11.300678128510535103146139590535312891006469726562, ++ 0.49998270338340455865022704529110342264175415039062, ++ 9.4889928089799600030573856201954185962677001953125, ++ 1.0885854822898506366612991769216023385524749755859, ++ 53.20529981175358358314042561687529087066650390625, ++ 36.65171139901388386306280153803527355194091796875, ++ 214.68561782216193023486994206905364990234375, ++ 2.8728217196022858281878598063485696911811828613281, ++ 1.0, ++ 1653.1016242378727838513441383838653564453125, ++ 0.37064443536603375317639574859640561044216156005859, ++ 20.0905336391907667348277755081653594970703125, ++ 288.66579115116110187955200672149658203125, ++ 967784087203564544.0, ++ 986920622098821248.0, ++ 17.499765511468584833210115903057157993316650390625, ++ 0.57797196338014200645005757905892096459865570068359, ++ 0.028955889395889600895772630906321865040808916091919, ++ 0.49998270338340455865022704529110342264175415039062, ++ 319.19585661999855119574931450188159942626953125, ++ 38.6813101625874224964718450792133808135986328125, ++ 39.62777871280881214488545083440840244293212890625, ++ 5.0871202966110988796799574629403650760650634765625, ++ 0.69504605038799238680979897253564558923244476318359, ++ 673.3477042973012203219695948064327239990234375, ++ 56.94168682747444876213194220326840877532958984375, ++ 1.0, ++ 261.01902251155337353338836692273616790771484375, ++ 1.0, ++ 85.0611943221388884239786420948803424835205078125, ++ 53.12927927294536090130350203253328800201416015625, ++ 21.829518414441992035790462978184223175048828125, ++ 1898.72146183866834689979441463947296142578125, ++ 1.0, ++ 9.7285926829767870316345579340122640132904052734375, ++ 174.40267892003106453557847999036312103271484375, ++ 0.98364895900708060327843895720434375107288360595703, ++ 1.1152676652901183373955973365809768438339233398438, ++ 18.12268289087599981712628505192697048187255859375, ++ 1.0, ++ 0.1715993516574435828747624555035145021975040435791, ++ 0.49275933843630442821037718204024713486433029174805, ++ 0.031531692879025040310292382628176710568368434906006, ++ 23.13033056510358420609918539412319660186767578125, ++ 210.58233961820729973624111153185367584228515625, ++ 5.1604155410259560099461850768420845270156860351562, ++ 2053.87275307550726211047731339931488037109375, ++ 1.0834136602451556186110792623367160558700561523438, ++ 3840.080091990574146620929241180419921875, ++ 13192.047960544839952490292489528656005859375, ++ 348.088713237990532434196211397647857666015625, ++ 439.96013885313283253708505071699619293212890625, ++ 897.3433304220051240918110124766826629638671875, ++ 0.69288480487588777201324319321429356932640075683594, ++ 2894.596744865002619917504489421844482421875, ++ 3788.94162413956064483500085771083831787109375, ++ 94.549943427633166947998688556253910064697265625, ++ 649339661894085888.0, ++ 0.48660066498392295919472871901234611868858337402344, ++ 0.12382015553845396316212656984134810045361518859863, ++ 0.50791641118256847242662388453027233481407165527344, ++ 1.0}; ++ ++public: ++ FIModelRunner(LLVMContext &Ctx, ++ std::vector> Features, ++ StringRef DecisionName) ++ : AOTModelRunner( ++ Ctx, ++ {{"input_1", "float32[" + std::to_string(Features.size()) + "]"}}, ++ DecisionName) {} ++ ++ // Features for this model are only floats so we only need to override the ++ // float method to handle feature scaling and the input type ++ bool setCustomFeature(int FeatureIndex, float FeatureValue) override { ++ // Scale the feature according to the constant mean and scale value ++ // Feature scaling is done to create a standard normal distribution: ++ // subtract mean, then divide by standard deviation ("scale") ++ float ScaledValue = ++ (FeatureValue - Means[FeatureIndex]) / Scales[FeatureIndex]; ++ // Assuming the Buffer at index 0 is for feature input of shape: ++ // (Feature.size()) ++ float *Location = getTensor(0) + FeatureIndex; ++ *Location = ScaledValue; ++ return true; ++ } ++ ++ // Outputs for this model are only int so we only need to override this ++ // method ++ int getModelResultI(std::string OutputName) override { ++ if (OutputName == "FI-ShouldInline") { ++ int Classes[] = {0, 1}; ++ void *ResultUntyped = CompiledModel->result_data(0); ++ float *Result = reinterpret_cast(ResultUntyped); ++ float Max = Result[0]; ++ int MaxClass = 0; ++ for (size_t I = 0; I < sizeof(Classes) / sizeof(int); ++I) { ++ if (Result[I] > Max) { ++ Max = Result[I]; ++ MaxClass = I; ++ } ++ } ++ ++ return Classes[MaxClass]; ++ } ++ assert(false && "ModelRunner received invalid result name"); ++ } ++}; ++ ++} // namespace llvm ++ ++#endif // LLVM_ANALYSIS_FIMODELRUNNER_H ++ ++#endif // LLVM_HAVE_TF_AOT_FICOMPILEDMODEL ++ ++#endif // ENABLE_ACPO +diff --git a/llvm/include/llvm/Analysis/InlineAdvisor.h b/llvm/include/llvm/Analysis/InlineAdvisor.h +index 53c018d15cd7..adf36a385725 100644 +--- a/llvm/include/llvm/Analysis/InlineAdvisor.h ++++ b/llvm/include/llvm/Analysis/InlineAdvisor.h +@@ -200,6 +200,22 @@ public: + return AnnotatedInlinePassName.c_str(); + } + ++#if defined(ENABLE_ACPO) ++ /// Helper functions used by getFeatures to retrieve certain information ++ ///{ ++ CallBase *getInlinableCS(Instruction &I); ++ int64_t getLocalCalls(Function &F); ++ unsigned getCallLoopLevel(CallBase &CB) const; ++ uint64_t getCalleeBlockFreq(CallBase &CB) const; ++ unsigned getCallSiteHeight(CallBase *CB); ++ ///} ++ ++ // Allow ACPO infrastructure to replicate Advisor behaviour ++ virtual bool isForcedToStop() const { return false; } ++ bool neverInline(CallBase &CB) const; ++ bool isCSInlinable(CallBase &CB) const; ++#endif ++ + protected: + InlineAdvisor(Module &M, FunctionAnalysisManager &FAM, + std::optional IC = std::nullopt); +@@ -213,6 +229,15 @@ protected: + const std::string AnnotatedInlinePassName; + std::unique_ptr ImportedFunctionsStats; + ++#if defined(ENABLE_ACPO) ++ /// Map a function to its callheight ++ std::map FunctionLevels; ++ // used by getORE() for legacy PM ++ static std::unique_ptr ORE; ++ ++ friend class ACPOCollectFeatures; ++#endif ++ + enum class MandatoryInliningKind { NotMandatory, Always, Never }; + + static MandatoryInliningKind getMandatoryKind(CallBase &CB, +@@ -389,6 +414,11 @@ void emitInlinedIntoBasedOnCost(OptimizationRemarkEmitter &ORE, DebugLoc DLoc, + bool ForProfileContext = false, + const char *PassName = nullptr); + ++#if defined(ENABLE_ACPO) ++/// get call site location as string. ++std::string getCallSiteLocation(DebugLoc DLoc); ++#endif ++ + /// Add location info to ORE message. + void addLocationToRemarks(OptimizationRemark &Remark, DebugLoc DLoc); + +diff --git a/llvm/include/llvm/Analysis/InlineModelFeatureMaps.h b/llvm/include/llvm/Analysis/InlineModelFeatureMaps.h +index 77ae60059ce9..e7ece46342fd 100644 +--- a/llvm/include/llvm/Analysis/InlineModelFeatureMaps.h ++++ b/llvm/include/llvm/Analysis/InlineModelFeatureMaps.h +@@ -72,6 +72,36 @@ enum class InlineCostFeatureIndex : size_t { + + NumberOfFeatures + }; ++ ++#if defined(ENABLE_ACPO) ++const std::map InlineCostFeatureIndexToName = { ++ { InlineCostFeatureIndex::sroa_savings, "sroa_savings" }, ++ { InlineCostFeatureIndex::sroa_losses, "sroa_losses" }, ++ { InlineCostFeatureIndex::load_elimination, "load_elimination" }, ++ { InlineCostFeatureIndex::call_penalty, "call_penalty" }, ++ { InlineCostFeatureIndex::call_argument_setup, "call_argument_setup" }, ++ { InlineCostFeatureIndex::load_relative_intrinsic, "load_relative_intrinsic" }, ++ { InlineCostFeatureIndex::lowered_call_arg_setup, "lowered_call_arg_setup" }, ++ { InlineCostFeatureIndex::indirect_call_penalty, "indirect_call_penalty" }, ++ { InlineCostFeatureIndex::jump_table_penalty, "jump_table_penalty" }, ++ { InlineCostFeatureIndex::case_cluster_penalty, "case_cluster_penalty" }, ++ { InlineCostFeatureIndex::switch_penalty, "switch_penalty" }, ++ { InlineCostFeatureIndex::unsimplified_common_instructions, "unsimplified_common_instructions" }, ++ { InlineCostFeatureIndex::num_loops, "num_loops" }, ++ { InlineCostFeatureIndex::dead_blocks, "dead_blocks" }, ++ { InlineCostFeatureIndex::simplified_instructions, "simplified_instructions" }, ++ { InlineCostFeatureIndex::constant_args, "constant_args" }, ++ { InlineCostFeatureIndex::constant_offset_ptr_args, "constant_offset_ptr_args" }, ++ { InlineCostFeatureIndex::callsite_cost, "callsite_cost" }, ++ { InlineCostFeatureIndex::cold_cc_penalty, "cold_cc_penalty" }, ++ { InlineCostFeatureIndex::last_call_to_static_bonus, "last_call_to_static_bonus" }, ++ { InlineCostFeatureIndex::is_multiple_blocks, "is_multiple_blocks" }, ++ { InlineCostFeatureIndex::nested_inlines, "nested_inlines" }, ++ { InlineCostFeatureIndex::nested_inline_cost_estimate, "nested_inline_cost_estimate" }, ++ { InlineCostFeatureIndex::threshold, "threshold" } ++}; ++#endif ++ + // clang-format on + + using InlineCostFeatures = +diff --git a/llvm/include/llvm/Analysis/MLInlineAdvisor.h b/llvm/include/llvm/Analysis/MLInlineAdvisor.h +index f58862e53352..e302b0a979a5 100644 +--- a/llvm/include/llvm/Analysis/MLInlineAdvisor.h ++++ b/llvm/include/llvm/Analysis/MLInlineAdvisor.h +@@ -41,8 +41,11 @@ public: + } + void onSuccessfulInlining(const MLInlineAdvice &Advice, + bool CalleeWasDeleted); +- ++#if defined(ENABLE_ACPO) ++ bool isForcedToStop() const override { return ForceStop; } ++#else + bool isForcedToStop() const { return ForceStop; } ++#endif + int64_t getLocalCalls(Function &F); + const MLModelRunner &getModelRunner() const { return *ModelRunner.get(); } + FunctionPropertiesInfo &getCachedFPI(Function &) const; +diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h +index 7fdb5db67c16..89160cfd17d1 100644 +--- a/llvm/include/llvm/InitializePasses.h ++++ b/llvm/include/llvm/InitializePasses.h +@@ -100,7 +100,9 @@ void initializeDomPrinterWrapperPassPass(PassRegistry &); + void initializeDomViewerWrapperPassPass(PassRegistry &); + void initializeDominanceFrontierWrapperPassPass(PassRegistry&); + void initializeDominatorTreeWrapperPassPass(PassRegistry&); ++#if defined(ENABLE_ACPO) + void initializeDumpCallsiteLegacyPass(PassRegistry &); ++#endif + void initializeDwarfEHPrepareLegacyPassPass(PassRegistry &); + void initializeEarlyCSELegacyPassPass(PassRegistry&); + void initializeEarlyCSEMemSSALegacyPassPass(PassRegistry&); +@@ -134,7 +136,9 @@ void initializeGlobalsAAWrapperPassPass(PassRegistry&); + void initializeGuardWideningLegacyPassPass(PassRegistry&); + void initializeHardwareLoopsLegacyPass(PassRegistry&); + void initializeMIRProfileLoaderPassPass(PassRegistry &); ++#if defined(ENABLE_ACPO) + void initializeInlineAdvisorAnalysisWrapperPass(PassRegistry &); ++#endif + void initializeIRSimilarityIdentifierWrapperPassPass(PassRegistry&); + void initializeIRTranslatorPass(PassRegistry&); + void initializeIVUsersWrapperPassPass(PassRegistry&); +@@ -152,11 +156,13 @@ void initializeInterleavedLoadCombinePass(PassRegistry &); + void initializeIntervalPartitionPass(PassRegistry&); + void initializeJMCInstrumenterPass(PassRegistry&); + void initializeKCFIPass(PassRegistry &); ++#if defined(ENABLE_ACPO) + void initializeLegacyFAMPass(PassRegistry &); + void initializeLegacyFunctionPropertiesAnalysisPass(PassRegistry &); + void initializeLegacyInlinerPassPass(PassRegistry &); + void initializeLegacyInlineSizeEstimatorAnalysisPass(PassRegistry &); + void initializeLegacyModuleInlinerWrapperPassPass(PassRegistry &); ++#endif + void initializeLCSSAVerificationPassPass(PassRegistry&); + void initializeLCSSAWrapperPassPass(PassRegistry&); + void initializeLazyBlockFrequencyInfoPassPass(PassRegistry&); +diff --git a/llvm/include/llvm/Transforms/IPO.h b/llvm/include/llvm/Transforms/IPO.h +index 4995b000c454..6905acb261fe 100644 +--- a/llvm/include/llvm/Transforms/IPO.h ++++ b/llvm/include/llvm/Transforms/IPO.h +@@ -18,6 +18,10 @@ + #include + #include + ++#if defined(ENABLE_ACPO) ++#include "llvm/Analysis/InlineAdvisor.h" ++#endif ++ + namespace llvm { + + class ModulePass; +diff --git a/llvm/include/llvm/Transforms/IPO/Inliner.h b/llvm/include/llvm/Transforms/IPO/Inliner.h +index 401aa2d3a0cc..46a3468c927c 100644 +--- a/llvm/include/llvm/Transforms/IPO/Inliner.h ++++ b/llvm/include/llvm/Transforms/IPO/Inliner.h +@@ -16,6 +16,15 @@ + #include "llvm/Analysis/Utils/ImportedFunctionsInliningStatistics.h" + #include "llvm/IR/PassManager.h" + ++#if defined(ENABLE_ACPO) ++#include "llvm/ADT/STLExtras.h" ++#include "llvm/Analysis/BasicAliasAnalysis.h" ++#include "llvm/IR/LegacyPassManager.h" ++#include "llvm/IR/Module.h" ++#include "llvm/Pass.h" ++#include ++#endif ++ + namespace llvm { + + /// The inliner pass for the new pass manager. +diff --git a/llvm/lib/Analysis/ACPOCollectFeatures.cpp b/llvm/lib/Analysis/ACPOCollectFeatures.cpp +index f9de26483c76..daa924f2cb3b 100644 +--- a/llvm/lib/Analysis/ACPOCollectFeatures.cpp ++++ b/llvm/lib/Analysis/ACPOCollectFeatures.cpp +@@ -10,6 +10,7 @@ + // + //===----------------------------------------------------------------------===// + ++#if defined(ENABLE_ACPO) + #include "llvm/Analysis/ACPOCollectFeatures.h" + #include "llvm/ADT/SCCIterator.h" + // The ACPOFIModel.h currently contains only the cache system for +@@ -1256,3 +1257,4 @@ operator++(ACPOCollectFeatures::FeatureIndex &N, int) { + } + + } // namespace llvm ++#endif // ENABLE_ACPO +diff --git a/llvm/lib/Analysis/ACPOFIModel.cpp b/llvm/lib/Analysis/ACPOFIModel.cpp +new file mode 100644 +index 000000000000..d9a647ec1012 +--- /dev/null ++++ b/llvm/lib/Analysis/ACPOFIModel.cpp +@@ -0,0 +1,243 @@ ++//===- ACPOFIModel.cpp - AI-Enabled Continuous Program Optimization -------===// ++// ++// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. ++// See https://llvm.org/LICENSE.txt for license information. ++// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ++// ++//===----------------------------------------------------------------------===// ++// ++// This file implements the interface between ACPO and ML-guided optimizations. ++// It delegates decision making to inference with a pre-trained model. ++// ++//===----------------------------------------------------------------------===// ++ ++#if defined(ENABLE_ACPO) ++#include "llvm/Analysis/ACPOFIModel.h" ++#include "llvm/Support/CommandLine.h" ++#include "llvm/Support/Process.h" ++ ++using namespace llvm; ++ ++#define DEBUG_TYPE "acpo" ++#define ACPO_ENV_VAR_DIR "ACPO_DIR" ++ ++cl::opt ++ EnableACPOFI("enable-acpo-fi", cl::init(false), cl::Hidden, ++ cl::desc("Leverage ACPO ML model to decide inlining.")); ++ ++cl::opt ++ EnableAOTFI("enable-acpo-fi-aot", cl::init(false), cl::Hidden, ++ cl::desc("Leverage AOT ML model to decide inlining.")); ++ ++ACPOFIModel::ACPOFIModel(CallBase *CB, InlineAdvisor *IA, ++ OptimizationRemarkEmitter *ORE, bool OnlyMandatory, ++ bool UseML) ++ : ACPOModel(ORE, UseML), CurrentCB(CB), NotACPOAdvisor(IA), ++ OnlyMandatory(OnlyMandatory) { ++ Function *Caller = CB->getCaller(); ++ LLVMContext *Context = &(Caller->getContext()); ++ setContextPtr(Context); ++ if (EnableACPOFI) ++ // ACPO Python support ++ setMLIF(createPersistentPythonMLIF()); ++ else if (EnableAOTFI) ++ // ACPO AOT support ++ setMLIF(createPersistentCompiledMLIF()); ++} ++ ++ACPOFIModel::~ACPOFIModel() {} ++ ++void ACPOFIModel::setMLCustomFeatures( ++ std::vector> FeatureValues) { ++ CustomFeatureValues = FeatureValues; ++} ++ ++void ACPOFIModel::sendCustomFeatures() { ++ // Get an ACPOMLInterface to communicate with the Python side ++ std::shared_ptr MLIF = getMLIF(); ++ MLIF->initializeFeatures("FI", CustomFeatureValues); ++} ++ ++void ACPOFIModel::recordUnattemptedInlining() { ++ if (NotACPOAdvice) ++ NotACPOAdvice->recordUnattemptedInlining(); ++} ++ ++void ACPOFIModel::recordInlining() { ++ if (NotACPOAdvice) ++ NotACPOAdvice->recordInlining(); ++} ++ ++void ACPOFIModel::recordUnsuccessfulInlining(InlineResult &IR) { ++ if (NotACPOAdvice) ++ NotACPOAdvice->recordUnsuccessfulInlining(IR); ++} ++ ++void ACPOFIModel::recordInliningWithCalleeDeleted() { ++ if (NotACPOAdvice) ++ NotACPOAdvice->recordInliningWithCalleeDeleted(); ++} ++ ++void ACPOFIModel::invalidateCache(CallBase *CB) { ++ if (CB) { ++ invalidateCache(CB->getCaller()); ++ } ++} ++ ++InlineAdvisor *ACPOFIModel::getNotACPOAdvisor() { return NotACPOAdvisor; } ++ ++void ACPOFIModel::invalidateCache(const Function *F) { ++ for (ACPOFIExtendedFeatures::NamedFeatureIndex feature = ++ ACPOFIExtendedFeatures::NamedFeatureIndex(0); ++ feature != ACPOFIExtendedFeatures::NamedFeatureIndex::NumNamedFeatures; ++ ++feature) { ++ FeatureCache[feature].erase(F); ++ } ++ for (ACPOFIExtendedFeatures::NamedFloatFeatureIndex feature = ++ ACPOFIExtendedFeatures::NamedFloatFeatureIndex(0); ++ feature != ++ ACPOFIExtendedFeatures::NamedFloatFeatureIndex::NumNamedFloatFeatures; ++ ++feature) { ++ FeatureCache[feature].erase(F); ++ } ++ FunctionAnalysisCache.DomCache.erase(F); ++ FunctionAnalysisCache.LICache.erase(F); ++ FunctionAnalysisCache.TTICache.erase(F); ++} ++ ++void ACPOFIModel::clearCache() { ++ for (ACPOFIExtendedFeatures::NamedFeatureIndex feature = ++ ACPOFIExtendedFeatures::NamedFeatureIndex(0); ++ feature != ACPOFIExtendedFeatures::NamedFeatureIndex::NumNamedFeatures; ++ ++feature) { ++ FeatureCache[feature].clear(); ++ } ++ for (ACPOFIExtendedFeatures::NamedFloatFeatureIndex feature = ++ ACPOFIExtendedFeatures::NamedFloatFeatureIndex(0); ++ feature != ++ ACPOFIExtendedFeatures::NamedFloatFeatureIndex::NumNamedFloatFeatures; ++ ++feature) { ++ FeatureCache[feature].clear(); ++ } ++ FunctionAnalysisCache.DomCache.clear(); ++ FunctionAnalysisCache.LICache.clear(); ++ FunctionAnalysisCache.TTICache.clear(); ++} ++ ++std::optional ++ACPOFIModel::getCachedSize(const Function *F, ++ ACPOFIExtendedFeatures::NamedFeatureIndex idx) { ++ auto it = FeatureCache[idx].find(F); ++ return it != FeatureCache[idx].end() ? std::optional(it->second) ++ : std::nullopt; ++} ++ ++std::optional ACPOFIModel::getCachedFloat( ++ const Function *F, ACPOFIExtendedFeatures::NamedFloatFeatureIndex idx) { ++ auto it = FeatureCache[idx].find(F); ++ return it != FeatureCache[idx].end() ? std::optional(it->second) ++ : std::nullopt; ++} ++ ++void ACPOFIModel::insertSizeCache(const Function *F, ++ ACPOFIExtendedFeatures::NamedFeatureIndex idx, ++ size_t val) { ++ FeatureCache[idx].insert(std::make_pair(F, val)); ++} ++ ++void ACPOFIModel::insertFloatCache( ++ const Function *F, ACPOFIExtendedFeatures::NamedFloatFeatureIndex idx, ++ float val) { ++ FeatureCache[idx].insert(std::make_pair(F, val)); ++} ++ ++const DominatorTree *ACPOFIModel::getDomCachedAnalysis(const Function *F) { ++ auto it = FunctionAnalysisCache.DomCache.find(F); ++ return it != FunctionAnalysisCache.DomCache.end() ? it->second : nullptr; ++} ++ ++const LoopInfo *ACPOFIModel::getLICachedAnalysis(const Function *F) { ++ auto it = FunctionAnalysisCache.LICache.find(F); ++ return it != FunctionAnalysisCache.LICache.end() ? it->second : nullptr; ++} ++ ++const TargetTransformInfo * ++ACPOFIModel::getTTICachedAnalysis(const Function *F) { ++ auto it = FunctionAnalysisCache.TTICache.find(F); ++ return it != FunctionAnalysisCache.TTICache.end() ? it->second : nullptr; ++} ++ ++void ACPOFIModel::insertAnalysisCache(const Function *F, ++ const DominatorTree *Tree) { ++ FunctionAnalysisCache.DomCache.insert(std::make_pair(F, Tree)); ++} ++ ++void ACPOFIModel::insertAnalysisCache(const Function *F, const LoopInfo *LI) { ++ FunctionAnalysisCache.LICache.insert(std::make_pair(F, LI)); ++} ++ ++void ACPOFIModel::insertAnalysisCache(const Function *F, ++ const TargetTransformInfo *TTI) { ++ FunctionAnalysisCache.TTICache.insert(std::make_pair(F, TTI)); ++} ++ ++std::unique_ptr ACPOFIModel::getAdviceML() { ++ std::shared_ptr MLIF = getMLIF(); ++ // Generate result. ++ std::unique_ptr Advice = std::make_unique(); ++ // handle mandatory case, forcestop, never inline or not inlinable cases ++ if (OnlyMandatory) ++ return getAdviceNoML(); ++ if (NotACPOAdvisor->neverInline(*CurrentCB) || ++ !NotACPOAdvisor->isCSInlinable(*CurrentCB)) { ++ Advice->addField("FI-ShouldInline", ++ ConstantInt::get(Type::getInt64Ty(*(getContextPtr())), ++ (int64_t) false)); ++ NotACPOAdvice = nullptr; ++ return Advice; ++ } ++ std::optional Env = llvm::sys::Process::GetEnv(ACPO_ENV_VAR_DIR); ++ if (!Env || *Env == "") { ++ std::optional LLVMDIROpt = ++ llvm::sys::Process::GetEnv("LLVM_DIR"); ++ if (!LLVMDIROpt) { ++ outs() << "ACPO_DIR not found. " ++ << "Did you export ACPO_DIR to $LLVM_DIR/acpo ?\n" ++ << "Falling back to default advisor. \n"; ++ return getAdviceNoML(); ++ } ++ } ++ assert(MLIF != nullptr); ++ if (!MLIF->loadModel("model-fi.acpo")) { ++ outs() << "Model not loaded correctly. \n"; ++ return getAdviceNoML(); ++ } ++ if (!MLIF->initializeFeatures("FI", CustomFeatureValues)) { ++ outs() << "Features not initialized correctly. \n"; ++ return getAdviceNoML(); ++ } ++ bool ModelRunOK = MLIF->runModel("FI"); ++ assert(ModelRunOK); ++ ShouldInline = MLIF->getModelResultI("FI-ShouldInline"); ++ assert(getContextPtr() != nullptr); ++ Advice->addField("FI-ShouldInline", ++ ConstantInt::get(Type::getInt64Ty(*(getContextPtr())), ++ (int64_t)ShouldInline)); ++ return Advice; ++} ++ ++std::unique_ptr ACPOFIModel::getAdviceNoML() { ++ // Use the advisor used by default inlining ++ std::unique_ptr Advice = std::make_unique(); ++ assert(getContextPtr() != nullptr); ++ NotACPOAdvice = NotACPOAdvisor->getAdvice(*CurrentCB, OnlyMandatory); ++ bool ShouldInline = NotACPOAdvice->isInliningRecommended(); ++ Advice->addField("FI-ShouldInline", ++ ConstantInt::get(Type::getInt64Ty(*(getContextPtr())), ++ (int64_t)ShouldInline)); ++ return Advice; ++} ++ ++ACPOFIModel::FunctionFeaturesCache ACPOFIModel::FeatureCache; ++ACPOFIModel::FunctionAnalysisMap ACPOFIModel::FunctionAnalysisCache; ++#endif // ENABLE_ACPO +diff --git a/llvm/lib/Analysis/ACPOMLInterface.cpp b/llvm/lib/Analysis/ACPOMLInterface.cpp +index 271dcfe7d851..f48eb46638e3 100644 +--- a/llvm/lib/Analysis/ACPOMLInterface.cpp ++++ b/llvm/lib/Analysis/ACPOMLInterface.cpp +@@ -10,6 +10,7 @@ + // + //===----------------------------------------------------------------------===// + ++#if defined(ENABLE_ACPO) + #include "llvm/Analysis/ACPOMLInterface.h" + #include "llvm/Analysis/ACPOModelRunner.h" + #include "llvm/Analysis/FIModelRunner.h" +@@ -1403,3 +1404,4 @@ const std::unordered_map ++ ++#include "llvm/ADT/SCCIterator.h" ++#include "llvm/ADT/StringRef.h" ++#include "llvm/Analysis/ACPOFIModel.h" ++#include "llvm/Analysis/DumpFeature.h" ++#include "llvm/Analysis/FunctionPropertiesAnalysis.h" ++#include "llvm/Analysis/InlineModelFeatureMaps.h" ++#include "llvm/Analysis/InlineSizeEstimatorAnalysis.h" ++#include "llvm/Analysis/LoopInfo.h" ++#include "llvm/IR/InstIterator.h" ++#include "llvm/IR/Instructions.h" ++#include "llvm/InitializePasses.h" ++#include "llvm/Transforms/IPO/Inliner.h" ++#include ++#endif ++ + using namespace llvm; + #define DEBUG_TYPE "inline" + #ifdef LLVM_HAVE_TF_AOT_INLINERSIZEMODEL +@@ -537,6 +556,46 @@ void llvm::emitInlinedIntoBasedOnCost( + PassName); + } + ++#if defined(ENABLE_ACPO) ++CallBase *InlineAdvisor::getInlinableCS(Instruction &I) { ++ if (auto *CS = dyn_cast(&I)) ++ if (Function *Callee = CS->getCalledFunction()) { ++ if (!Callee->isDeclaration()) { ++ return CS; ++ } ++ } ++ return nullptr; ++} ++ ++// TODO: We can make this faster on large programs by applying ++// this patch from MLGO f46dd19b480496d2ba0a57d12935882e530f2b93. ++// This patch incrementally computes FunctionPropertiesInfo ++// instead of recomputing. ++int64_t InlineAdvisor::getLocalCalls(Function &F) { ++ return FAM.getResult(F) ++ .DirectCallsToDefinedFunctions; ++} ++ ++unsigned InlineAdvisor::getCallLoopLevel(CallBase &CB) const { ++ Function *F = CB.getCaller(); ++ BasicBlock *BB = CB.getParent(); ++ LoopInfo &LI = FAM.getResult(*F); ++ return LI.getLoopDepth(BB); ++} ++ ++uint64_t InlineAdvisor::getCalleeBlockFreq(CallBase &CB) const { ++ Function *F = CB.getCaller(); ++ BasicBlock *BB = CB.getParent(); ++ BlockFrequencyInfo &BFI = FAM.getResult(*F); ++ return BFI.getBlockFreq(BB).getFrequency(); ++} ++ ++unsigned InlineAdvisor::getCallSiteHeight(CallBase *CB) { ++ Function *Caller = CB->getCaller(); ++ return FunctionLevels[Caller]; ++} ++#endif ++ + InlineAdvisor::InlineAdvisor(Module &M, FunctionAnalysisManager &FAM, + std::optional IC) + : M(M), FAM(FAM), IC(IC), +@@ -548,6 +607,35 @@ InlineAdvisor::InlineAdvisor(Module &M, FunctionAnalysisManager &FAM, + std::make_unique(); + ImportedFunctionsStats->setModuleInfo(M); + } ++#if defined(ENABLE_ACPO) ++ std::unique_ptr CG(std::make_unique(M)); ++ for (auto I = scc_begin(CG.get()); !I.isAtEnd(); ++I) { ++ const std::vector &CGNodes = *I; ++ unsigned Level = 0; ++ for (auto *CGNode : CGNodes) { ++ Function *F = CGNode->getFunction(); ++ if (!F || F->isDeclaration()) ++ continue; ++ for (auto &I : instructions(F)) { ++ if (auto *CS = getInlinableCS(I)) { ++ auto *Called = CS->getCalledFunction(); ++ auto Pos = FunctionLevels.find(Called); ++ // In bottom up traversal, an inlinable callee is either in the ++ // same SCC, or to a function in a visited SCC. So not finding its ++ // level means we haven't visited it yet, meaning it's in this SCC. ++ if (Pos == FunctionLevels.end()) ++ continue; ++ Level = std::max(Level, Pos->second + 1); ++ } ++ } ++ } ++ for (auto *CGNode : CGNodes) { ++ Function *F = CGNode->getFunction(); ++ if (F && !F->isDeclaration()) ++ FunctionLevels[F] = Level; ++ } ++ } ++#endif + } + + InlineAdvisor::~InlineAdvisor() { +@@ -639,6 +727,33 @@ std::unique_ptr InlineAdvisor::getAdvice(CallBase &CB, + return getMandatoryAdvice(CB, Advice); + } + ++#if defined(ENABLE_ACPO) ++bool InlineAdvisor::neverInline(CallBase &CB) const { ++ auto &Caller = *CB.getCaller(); ++ auto &Callee = *CB.getCalledFunction(); ++ auto &ORE = FAM.getResult(Caller); ++ auto MandatoryKind = InlineAdvisor::getMandatoryKind(CB, FAM, ORE); ++ // If this is a "never inline" case, there won't be any changes to internal ++ // state we need to track, so we can just return the base InlineAdvice, ++ // which will do nothing interesting. Same thing if this is a recursive ++ // case. ++ return MandatoryKind == InlineAdvisor::MandatoryInliningKind::Never || ++ &Caller == &Callee; ++} ++ ++bool InlineAdvisor::isCSInlinable(CallBase &CB) const { ++ auto &Callee = *CB.getCalledFunction(); ++ ++ auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & { ++ return FAM.getResult(F); ++ }; ++ auto &TIR = FAM.getResult(Callee); ++ auto IsCallSiteInlinable = ++ llvm::getInliningCostEstimate(CB, TIR, GetAssumptionCache); ++ return !!IsCallSiteInlinable; ++} ++#endif ++ + OptimizationRemarkEmitter &InlineAdvisor::getCallerORE(CallBase &CB) { + return FAM.getResult(*CB.getCaller()); + } +diff --git a/llvm/lib/Analysis/MLInlineAdvisor.cpp b/llvm/lib/Analysis/MLInlineAdvisor.cpp +index 0660a9993b6d..c7ea2eb8ffe9 100644 +--- a/llvm/lib/Analysis/MLInlineAdvisor.cpp ++++ b/llvm/lib/Analysis/MLInlineAdvisor.cpp +@@ -323,6 +323,11 @@ std::unique_ptr MLInlineAdvisor::getAdviceImpl(CallBase &CB) { + auto &Caller = *CB.getCaller(); + auto &Callee = *CB.getCalledFunction(); + ++#if defined(ENABLE_ACPO) ++ LLVM_DEBUG(dbgs() << "Advice on call: " << Caller.getName() << " to " ++ << Callee.getName() << "\n"); ++#endif ++ + auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & { + return FAM.getResult(F); + }; +diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp +index a02c603a14a5..370b248c2b85 100644 +--- a/llvm/lib/IR/AsmWriter.cpp ++++ b/llvm/lib/IR/AsmWriter.cpp +@@ -86,15 +86,19 @@ + #include + #include + ++#if defined(ENABLE_ACPO) + #include "llvm/ADT/StringSet.h" + #include "llvm/Analysis/LoopInfo.h" + #include "llvm/Support/CommandLine.h" ++#endif + + using namespace llvm; + ++#if defined(ENABLE_ACPO) + cl::opt UnnamedVariablePrefix( + "unnamed-var-prefix", cl::Hidden, + cl::desc("Specify the prefix added to unnamed variables"), cl::init("")); ++#endif + + // Make virtual table appear in this compilation unit. + AssemblyAnnotationWriter::~AssemblyAnnotationWriter() = default; +@@ -2494,12 +2498,17 @@ static void WriteAsOperandInternal(raw_ostream &Out, const Value *V, + } else { + Slot = -1; + } +- ++#if defined(ENABLE_ACPO) + if (Slot != -1) { + // By default, UnnamedVariablePrefix is empty so it matches original behaviour + // unless specified. + Out << Prefix << UnnamedVariablePrefix << Slot; + } else ++#else ++ if (Slot != -1) ++ Out << Prefix << Slot; ++ else ++#endif + Out << ""; + } + +@@ -2635,7 +2644,11 @@ public: + SmallVector ExitBlocks); + #endif + void printArgument(const Argument *FA, AttributeSet Attrs); ++#if defined(ENABLE_ACPO) + void printBasicBlock(const BasicBlock *BB, bool PrintLabelOnly = false); ++#else ++ void printBasicBlock(const BasicBlock *BB); ++#endif + void printInstructionLine(const Instruction &I); + void printInstruction(const Instruction &I); + +@@ -4214,17 +4227,27 @@ void AssemblyWriter::printArgument(const Argument *Arg, AttributeSet Attrs) { + } else { + int Slot = Machine.getLocalSlot(Arg); + assert(Slot != -1 && "expect argument in function here"); ++#if defined(ENABLE_ACPO) + // By default, UnnamedVariablePrefix is empty so it matches original behaviour + // unless specified. + Out << " %" << UnnamedVariablePrefix << Slot; ++#else ++ Out << " %" << Slot; ++#endif + } + } + ++ + /// printBasicBlock - This member is called for each basic block in a method. ++#if defined(ENABLE_ACPO) + void AssemblyWriter::printBasicBlock(const BasicBlock *BB, + bool PrintLabelOnly) { + assert(BB && BB->getParent() && "block without parent!"); + bool IsEntryBlock = BB == &BB->getParent()->getEntryBlock(); ++#else ++void AssemblyWriter::printBasicBlock(const BasicBlock *BB) { ++ bool IsEntryBlock = BB == &BB->getParent()->getEntryBlock(); ++#endif + if (BB->hasName()) { // Print out the label if it exists... + Out << "\n"; + PrintLLVMName(Out, BB->getName(), LabelPrefix); +@@ -4233,17 +4256,23 @@ void AssemblyWriter::printBasicBlock(const BasicBlock *BB, + Out << "\n"; + int Slot = Machine.getLocalSlot(BB); + if (Slot != -1) { ++#if defined(ENABLE_ACPO) + // By default, UnnamedVariablePrefix is empty so it matches original behaviour + // unless specified. + Out << UnnamedVariablePrefix << Slot << ":"; ++#else ++ Out << Slot << ":"; ++#endif + } else + Out << ":"; + } + ++#if defined(ENABLE_ACPO) + if (PrintLabelOnly) { + Out << "\n"; + return; + } ++#endif + + if (!IsEntryBlock) { + // Output predecessors for the block. +@@ -4339,9 +4368,13 @@ void AssemblyWriter::printInstruction(const Instruction &I) { + if (SlotNum == -1) + Out << " = "; + else { ++#if defined(ENABLE_ACPO) + // By default, UnnamedVariablePrefix is empty so it matches original behaviour + // unless specified. + Out << '%' << UnnamedVariablePrefix << SlotNum << " = "; ++#else ++ Out << '%' << SlotNum << " = "; ++#endif + } + } + +diff --git a/llvm/lib/Transforms/IPO/Inliner.cpp b/llvm/lib/Transforms/IPO/Inliner.cpp +index 802667819c44..3a0a2494c36a 100644 +--- a/llvm/lib/Transforms/IPO/Inliner.cpp ++++ b/llvm/lib/Transforms/IPO/Inliner.cpp +@@ -64,10 +64,21 @@ + #include + #include + #include +-#if defined(ENABLE_AUTOTUNER) ++#if defined(ENABLE_AUTOTUNER) || defined(ENABLE_ACPO) + #include "llvm/AutoTuner/AutoTuning.h" + #endif + ++#if defined(ENABLE_ACPO) ++#include "llvm/Analysis/ACPOFIModel.h" ++#include "llvm/Analysis/ModelDataCollector.h" ++#include "llvm/IR/IRPrintingPasses.h" ++#include "llvm/Support/FormattedStream.h" ++#include "llvm/Support/raw_ostream.h" ++#include "llvm/Transforms/IPO.h" ++#include "llvm/Transforms/Utils/CallGraphUpdater.h" ++#include ++#endif ++ + using namespace llvm; + + #define DEBUG_TYPE "inline" +@@ -149,6 +160,124 @@ static cl::opt CGSCCInlineReplayFormat( + ":. (default)")), + cl::desc("How cgscc inline replay file is formatted"), cl::Hidden); + ++#if defined(ENABLE_ACPO) ++static cl::opt ++ ACPOVerboseFI("acpo-verbose-fi", cl::init(false), cl::Hidden, ++ cl::desc("Print ACPO invocation messages for FI.")); ++ ++static cl::opt FeatureDump("enable-fi-feature-dump", cl::init(false)); ++ ++// Defined in 'lib/Analysis/ACPOFIModel.cpp' ++extern cl::opt EnableACPOFI; ++extern cl::opt EnableAOTFI; ++// In "llvm/lib/Analysis/ModelDataCollector.cpp" ++extern cl::opt ACPOModelFile; ++ ++namespace { ++/// Class for collecting inlining model data ++class ModelDataFICollector : public ModelDataCollector { ++public: ++ ModelDataFICollector(formatted_raw_ostream &OS, bool OnlyMandatory, ++ std::string OutputFileName) ++ : ModelDataCollector(OS, OutputFileName), OnlyMandatory(OnlyMandatory) {} ++ ++ void collectFeatures(CallBase *CB, InlineAdvisor *IA, ++ FunctionAnalysisManager *FAM) { ++ bool MandatoryOnly = getOnlyMandatory(); ++ resetRegisteredFeatures(); ++ ACPOCollectFeatures::FeaturesInfo CallerFeatures{ ++ {ACPOCollectFeatures::FeatureIndex::BasicBlockCount, ++ /* ACPOCollectFeatures::Scope::Function, */ ++ /* ACPOCollectFeatures::GroupID::FPIRelated, */ ++ {FAM, nullptr}, ++ {CB->getCaller(), nullptr, nullptr, nullptr, nullptr}, ++ {MandatoryOnly, IA}}}; ++ ACPOCollectFeatures::FeaturesInfo CalleeFeatures{ ++ {ACPOCollectFeatures::FeatureIndex::BasicBlockCount, ++ /* ACPOCollectFeatures::Scope::Function, */ ++ /* ACPOCollectFeatures::GroupID::FPIRelated, */ ++ {FAM, nullptr}, ++ {CB->getCalledFunction(), nullptr, nullptr, nullptr, nullptr}, ++ {MandatoryOnly, IA}}}; ++ BasicBlock *GlobalBB = CB->getParent(); ++ Function *GlobalF = GlobalBB->getParent(); ++ Module *GlobalM = GlobalF->getParent(); ++ ACPOCollectFeatures::FeatureInfo GlobalFeatureInfo{ ++ ACPOCollectFeatures::FeatureIndex::NumOfFeatures, ++ {FAM, nullptr}, ++ {GlobalF, CB, GlobalBB, GlobalM, nullptr}, ++ {MandatoryOnly, IA}}; ++ ACPOCollectFeatures::FeatureInfo CallerInfo{ ++ ACPOCollectFeatures::FeatureIndex::NumOfFeatures, ++ {FAM, nullptr}, ++ {CB->getCaller(), CB, GlobalBB, GlobalM, nullptr}, ++ {MandatoryOnly, IA}}; ++ ACPOCollectFeatures::FeatureInfo CalleeInfo{ ++ ACPOCollectFeatures::FeatureIndex::NumOfFeatures, ++ {FAM, nullptr}, ++ {CB->getCalledFunction(), CB, GlobalBB, GlobalM, nullptr}, ++ {MandatoryOnly, IA}}; ++ ++ registerFeature({ACPOCollectFeatures::Scope::Function}, CalleeInfo, ++ "callee"); ++ registerFeature({ACPOCollectFeatures::Scope::Function}, CallerInfo, ++ "caller"); ++ registerFeature({ACPOCollectFeatures::Scope::CallSite}, GlobalFeatureInfo); ++ registerFeature({ACPOCollectFeatures::Scope::Module}, GlobalFeatureInfo); ++ ModelDataCollector::collectFeatures(); ++ } ++ bool getOnlyMandatory() { return OnlyMandatory; } ++ ++private: ++ bool OnlyMandatory = false; ++}; ++ ++llvm::SmallDenseSet, 4> ++ InlinedInternalEdges = ++ llvm::SmallDenseSet, 4>(); ++} // end anonymous namespace ++ ++/// helper function for getting advice with acpo infrastructure ++bool getACPOAdvice(CallBase *CB, std::unique_ptr &FI, ++ ModelDataFICollector *MDC, InlineAdvisor *Advisor, ++ FunctionAnalysisManager *FAM) { ++ bool ShouldInline = false; ++ // ------------------------------------------------------------------------ ++ // Begin ACPO invocation ++ if ((EnableACPOFI || EnableAOTFI) && !MDC->getOnlyMandatory() && ++ !Advisor->neverInline(*CB) && Advisor->isCSInlinable(*CB)) { ++ if (ACPOVerboseFI) { ++ errs() << "--- ACPOModel is activated\n"; ++ } ++ MDC->collectFeatures(CB, Advisor, FAM); ++ std::vector> Features = ++ MDC->getFeatures(); ++ FI->setMLCustomFeatures(Features); ++ } ++ std::unique_ptr Advice = FI->getAdvice(); ++ Constant *Val = Advice->getField("FI-ShouldInline"); ++ assert(Val != nullptr); ++ assert(isa(Val)); ++ ConstantInt *ACPOInline = dyn_cast(Val); ++ ShouldInline = ACPOInline->getSExtValue(); ++ if ((EnableACPOFI || EnableAOTFI) && ACPOVerboseFI) { ++ errs() << "ACPOModel's inline prediction: " << ShouldInline << "\n"; ++ } ++ if (FeatureDump) { ++ MDC->collectFeatures(CB, Advisor, FAM); ++ std::vector> Features = ++ MDC->getFeatures(); ++ if (MDC->isEmptyOutputFile()) { ++ MDC->printRow(true); ++ } ++ MDC->printRow(); ++ } ++ return ShouldInline; ++ // End ACPO Invocation ++ // --------------------------------------------------------------------- ++} ++#endif ++ + /// Return true if the specified inline history ID + /// indicates an inline history that includes the specified function. + static bool inlineHistoryIncludes( +@@ -205,6 +334,14 @@ InlinerPass::getAdvisor(const ModuleAnalysisManagerCGSCCProxy::Result &MAM, + PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC, + CGSCCAnalysisManager &AM, LazyCallGraph &CG, + CGSCCUpdateResult &UR) { ++#if defined(ENABLE_ACPO) ++ if (EnableACPOFI || EnableAOTFI) { ++ // Need to clear the cache at the beggining of the inliner pass, since during ++ // optimization we may have transofrmed the code which invalidated the cache. ++ ACPOFIModel::clearCache(); ++ } ++#endif ++ + const auto &MAMProxy = + AM.getResult(InitialC, CG); + bool Changed = false; +@@ -221,6 +358,10 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC, + Advisor.onPassEntry(&InitialC); + + auto AdvisorOnExit = make_scope_exit([&] { Advisor.onPassExit(&InitialC); }); ++#if defined(ENABLE_ACPO) ++ if (EnableACPOFI || EnableAOTFI) ++ ACPOCollectFeatures::clearFunctionLevel(); ++#endif + + // We use a single common worklist for calls across the entire SCC. We + // process these in-order and append new calls introduced during inlining to +@@ -377,8 +518,51 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC, + continue; + } + +- std::unique_ptr Advice = +- Advisor.getAdvice(*CB, OnlyMandatory); ++ std::unique_ptr Advice = nullptr; ++ #if defined(ENABLE_ACPO) ++ std::unique_ptr FI = nullptr; ++ if (EnableACPOFI || EnableAOTFI) { ++ auto &ORE = ++ FAM.getResult(*CB->getCaller()); ++ FI = std::make_unique( ++ CB, &Advisor, &ORE, OnlyMandatory, EnableACPOFI || EnableAOTFI); ++ std::error_code EC; ++ raw_fd_ostream RawOS(ACPOModelFile.getValue(), EC, sys::fs::CD_OpenAlways, ++ sys::fs::FA_Write, sys::fs::OF_Append); ++ if (EC) ++ errs() << "Could not create/open training data file (Falling back to " ++ "debug mode): " ++ << EC.message() << "\n"; ++ ++ formatted_raw_ostream OS(RawOS); ++ ModelDataFICollector MDC(OS, OnlyMandatory, ACPOModelFile); ++ if (EnableACPOFI) ++ LLVM_DEBUG(dbgs() << "ACPO Python ML infra is activated" << "\n"); ++ else if (EnableAOTFI) ++ LLVM_DEBUG(dbgs() << "ACPO AOT C++ ML infra is activated" << "\n"); ++ bool ShouldInline = getACPOAdvice(CB, FI, &MDC, &Advisor, &FAM); ++ ++ // Check whether we want to inline this callsite. ++ if (!ShouldInline) { ++ FI->recordUnattemptedInlining(); ++ continue; ++ } else { ++ ACPOFIModel::invalidateCache(CB); ++ } ++ } else { ++ Advice = Advisor.getAdvice(*CB, OnlyMandatory); ++ ++ // Check whether we want to inline this callsite. ++ if (!Advice) ++ continue; ++ ++ if (!Advice->isInliningRecommended()) { ++ Advice->recordUnattemptedInlining(); ++ continue; ++ } ++ } ++#else ++ Advice = Advisor.getAdvice(*CB, OnlyMandatory); + + // Check whether we want to inline this callsite. + if (!Advice) +@@ -388,6 +572,7 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC, + Advice->recordUnattemptedInlining(); + continue; + } ++#endif + + int CBCostMult = + getStringFnAttrAsInt( +@@ -396,6 +581,13 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC, + + // Setup the data structure used to plumb customization into the + // `InlineFunction` routine. ++#if defined(ENABLE_ACPO) ++ if ((EnableACPOFI || EnableAOTFI) && ACPOVerboseFI) { ++ Function &F2 = *CB->getCaller(); ++ LLVM_DEBUG(dbgs() << "check: " << F2.getName() << ", " ++ << Callee.getName() << "\n"); ++ } ++#endif + InlineFunctionInfo IFI( + GetAssumptionCache, PSI, + &FAM.getResult(*(CB->getCaller())), +@@ -405,7 +597,14 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC, + InlineFunction(*CB, IFI, /*MergeAttributes=*/true, + &FAM.getResult(*CB->getCaller())); + if (!IR.isSuccess()) { ++#if defined(ENABLE_ACPO) ++ if (EnableACPOFI || EnableAOTFI) ++ FI->recordUnsuccessfulInlining(IR); ++ else ++ Advice->recordUnsuccessfulInlining(IR); ++#else + Advice->recordUnsuccessfulInlining(IR); ++#endif + continue; + } + +@@ -494,10 +693,24 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC, + DeadFunctionsInComdats.push_back(&Callee); + } + } ++#if defined(ENABLE_ACPO) ++ if (EnableACPOFI || EnableAOTFI) { ++ if (CalleeWasDeleted) ++ FI->recordInliningWithCalleeDeleted(); ++ else ++ FI->recordInlining(); ++ } else { ++ if (CalleeWasDeleted) ++ Advice->recordInliningWithCalleeDeleted(); ++ else ++ Advice->recordInlining(); ++ } ++#else + if (CalleeWasDeleted) + Advice->recordInliningWithCalleeDeleted(); + else + Advice->recordInlining(); ++#endif + } + + // Back the call index up by one to put us in a good position to go around +diff --git a/llvm/tools/opt/opt.cpp b/llvm/tools/opt/opt.cpp +index 1401352647cd..671a33309a1b 100644 +--- a/llvm/tools/opt/opt.cpp ++++ b/llvm/tools/opt/opt.cpp +@@ -430,6 +430,9 @@ int main(int argc, char **argv) { + initializeTransformUtils(Registry); + initializeInstCombine(Registry); + initializeTarget(Registry); ++#if defined(ENABLE_ACPO) ++ initializeDumpCallsiteLegacyPass(Registry); ++#endif + // For codegen passes, only passes that do IR to IR transformation are + // supported. + initializeExpandLargeDivRemLegacyPassPass(Registry); +-- +2.38.1.windows.1 + diff --git a/0033-Find-Python3-in-default-env-PATH-for-ACPO.patch b/0033-Find-Python3-in-default-env-PATH-for-ACPO.patch new file mode 100644 index 0000000000000000000000000000000000000000..6c93c96c4f6db06035ef825dcf64d15a59f282da --- /dev/null +++ b/0033-Find-Python3-in-default-env-PATH-for-ACPO.patch @@ -0,0 +1,34 @@ +From d4cfa4fd4496735ea45afcd2b0cfb3607cadd1c9 Mon Sep 17 00:00:00 2001 +From: yinrun +Date: Thu, 17 Oct 2024 18:47:40 +0800 +Subject: [PATCH] Find Python3 in default env PATH for ACPO + +Enable the use of user python version, avoid the wrong version of python without AI infra. +--- + llvm/lib/Analysis/ACPOMLInterface.cpp | 10 +++++++++- + 1 file changed, 9 insertions(+), 1 deletion(-) + +diff --git a/llvm/lib/Analysis/ACPOMLInterface.cpp b/llvm/lib/Analysis/ACPOMLInterface.cpp +index f48eb46638e3..7d84bd5112d6 100644 +--- a/llvm/lib/Analysis/ACPOMLInterface.cpp ++++ b/llvm/lib/Analysis/ACPOMLInterface.cpp +@@ -146,7 +146,15 @@ ACPOMLPythonInterface::ACPOMLPythonInterface() : NextID{0} { + } + + int32_t PID = (int32_t) llvm::sys::Process::getProcessId(); +- std::string ExecPython = "/usr/bin/python3"; ++ std::string ExecPython; ++ llvm::ErrorOr Res = llvm::sys::findProgramByName("python3"); ++ if (std::error_code EC = Res.getError()) { ++ LLVM_DEBUG(dbgs() << "python3 could not be found, error_code " << EC.value() << "\n"); ++ return; ++ } else { ++ ExecPython = Res.get(); ++ LLVM_DEBUG(dbgs() << "python3 version found in " << ExecPython << "\n"); ++ } + std::string + PythonScript = *Env + "/" + std::string(ACPO_ML_PYTHON_INTERFACE_PY); + std::string PIDStr = std::to_string(PID); +-- +2.38.1.windows.1 + diff --git a/llvm.spec b/llvm.spec index 03528f36689dd4d165b943857f4b6a88dffc2444..fabd4ed7c98a365bd315475feb8b2ee786d62e54 100644 --- a/llvm.spec +++ b/llvm.spec @@ -5,6 +5,7 @@ %bcond_with classic_flang %bcond_with toolchain_clang %bcond_without bisheng_autotuner +%bcond_without ACPO %if %{with toolchain_clang} %global toolchain clang @@ -46,7 +47,7 @@ Name: %{pkg_name} Version: %{maj_ver}.%{min_ver}.%{patch_ver} -Release: 26 +Release: 28 Summary: The Low Level Virtual Machine License: NCSA @@ -87,6 +88,9 @@ Patch27: 0027-AArch64-Delete-hip09-macro.patch Patch28: 0028-backport-Clang-Fix-crash-with-fzero-call-used-regs.patch Patch29: 0029-SimplifyLibCalls-Merge-sqrt-into-the-power-of-exp-79.patch Patch30: 0030-LICM-Solve-runtime-error-caused-by-the-signal-functi.patch +Patch31: 0031-ACPO-ACPO-Infrastructure.patch +Patch32: 0032-ACPO-Introduce-MLInliner-using-ACPO-infrastructure.patch +Patch33: 0033-Find-Python3-in-default-env-PATH-for-ACPO.patch BuildRequires: binutils-devel BuildRequires: cmake @@ -202,6 +206,12 @@ pathfix.py -i %{__python3} -pn \ utils/update_cc_test_checks.py %build +%if %{with ACPO} + echo "enable ACPO" + export CFLAGS="-Wp,-DENABLE_ACPO ${CFLAGS}" + export CXXFLAGS="-Wp,-DENABLE_ACPO ${CXXFLAGS}" +%endif + %cmake -G Ninja \ -DBUILD_SHARED_LIBS:BOOL=OFF \ -DLLVM_PARALLEL_LINK_JOBS=%{max_link_jobs} \ @@ -380,6 +390,12 @@ LD_LIBRARY_PATH=%{buildroot}/%{install_libdir} %{__ninja} check-all -C %{__cmake %{install_includedir}/llvm-gmock %changelog +* Wed Nov 20 2024 eastb233 - 17.0.6-28 +- Find Python3 in default env PATH for ACPO + +* Wed Nov 20 2024 eastb233 - 17.0.6-27 +- ACPO Infrastructure for ML integration into LLVM compiler + * Wed Nov 20 2024 eastb233 - 17.0.6-26 - [LICM] Solve runtime error caused by the signal function.