zhiics commented on a change in pull request #5915: URL: https://github.com/apache/incubator-tvm/pull/5915#discussion_r456060646
########## File path: src/relay/backend/contrib/arm_compute_lib/codegen_acl.h ########## @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file src/relay/backend/contrib/arm_compute_lib/codegen_acl.h + * \brief The Relay -> ACL JSON schema compiler. + */ + +#ifndef TVM_RELAY_BACKEND_CONTRIB_ARM_COMPUTE_LIB_CODEGEN_ACL_H_ +#define TVM_RELAY_BACKEND_CONTRIB_ARM_COMPUTE_LIB_CODEGEN_ACL_H_ + +#include <tvm/relay/expr_functor.h> + +#include <map> +#include <memory> +#include <string> +#include <utility> +#include <vector> + +#include "../../../../runtime/contrib/json/json_node.h" +#include "../codegen_json/codegen_json.h" + +namespace tvm { +namespace relay { +namespace contrib { +namespace arm_compute_lib { + +/*! + * \brief Generates an ACLModule from a relay expression. This "compilation" + * does not require ACL since the actual conversion using ACL APIs is + * deferred until creation of the runtime. This step simply serializes the + * relay program into a JSON string. + */ +class ACLJSONSerializer : public backend::contrib::JSONSerializer { + using JSONGraphNode = tvm::runtime::json::JSONGraphNode; + using JSONGraphNodeEntry = tvm::runtime::json::JSONGraphNodeEntry; + + public: + ACLJSONSerializer(const std::string& symbol, const Expr& expr) : JSONSerializer(symbol, expr) {} + + std::vector<JSONGraphNodeEntry> VisitExpr_(const CallNode* cn) override; + + private: + /*! + * \brief Create a JSON representation of an operator. + * + * \param call The call to be represented. + * \return A JSON representation of a specific operator. + */ + std::shared_ptr<JSONGraphNode> CreateOpJSONNode(const CallNode* cn); + std::shared_ptr<JSONGraphNode> CreateCompositeConvJSONNode(const CallNode* cn); +}; + +/*! + * \brief Pre-process a module containing functions ready for ACL codegen. + * + * For now we enforce OHWI kernel layout and fold the transforms away. + * + * \param mod The module to be pre-processed. + * \return The processed module. + */ +IRModule PreProcessModule(const IRModule& mod); Review comment: I think we don't need this. It is only internally used in the codegen.cc ########## File path: src/relay/backend/contrib/arm_compute_lib/codegen.cc ########## @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file src/relay/backend/contrib/arm_compute_lib/codegen_acl.cc + * \brief Implementation of the Relay -> ACL JSON serializer. + */ +#include <tvm/ir/module.h> +#include <tvm/relay/attrs/nn.h> +#include <tvm/relay/type.h> + +#include "../../utils.h" +#include "codegen_acl.h" + +namespace tvm { +namespace relay { +namespace contrib { +namespace arm_compute_lib { + +using JSONGraphNode = tvm::runtime::json::JSONGraphNode; +using JSONGraphNodeEntry = tvm::runtime::json::JSONGraphNodeEntry; + +std::vector<JSONGraphNodeEntry> ACLJSONSerializer::VisitExpr_(const CallNode* cn) { + Expr expr = GetRef<Expr>(cn); + std::string name; + std::shared_ptr<JSONGraphNode> json_node; + + if (cn->op.as<OpNode>()) { + json_node = CreateOpJSONNode(cn); Review comment: I think we can just do `return JSONSerializer::VisiExpr_(cn);` here ########## File path: src/runtime/contrib/arm_compute_lib/acl_runtime.cc ########## @@ -0,0 +1,310 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file src/runtime/contrib/arm_compute_lib/acl_runtime.cc + * \brief A simple JSON runtime for Arm Compute Library. + */ + +#include <tvm/runtime/ndarray.h> +#include <tvm/runtime/registry.h> + +#include "../../file_util.h" +#include "../json/json_node.h" +#include "../json/json_runtime.h" + +#ifdef TVM_GRAPH_RUNTIME_ARM_COMPUTE_LIB +#include <arm_compute/core/Types.h> +#include <arm_compute/runtime/NEON/functions/NEConvolutionLayer.h> +#include <arm_compute/runtime/NEON/functions/NEPoolingLayer.h> +#include <arm_compute/runtime/NEON/functions/NEReshapeLayer.h> + +#include "acl_allocator.h" +#include "acl_utils.h" +#endif + +namespace tvm { +namespace runtime { +namespace contrib { + +using namespace tvm::runtime::json; + +#ifdef TVM_GRAPH_RUNTIME_ARM_COMPUTE_LIB +using namespace arm_compute_lib; +#endif + +class ACLRuntime : public JSONRuntimeBase { + public: + /*! + * \brief The ACL runtime module. Deserialize the provided functions + * on creation and store in the layer cache. + * + * \param symbol_name The name of the function. + * \param graph_json serialized JSON representation of a sub-graph. + * \param const_names The names of each constant in the sub-graph. + */ + explicit ACLRuntime(const std::string& symbol_name, const std::string& graph_json, + const Array<String>& const_names) + : JSONRuntimeBase(symbol_name, graph_json, const_names) {} + + /*! + * \brief The type key of the module. + * + * \return module type key. + */ + const char* type_key() const override { return "arm_compute_lib"; } + + /*! + * \brief Initialize runtime. Create ACL layer from JSON + * representation. + * + * \param consts The constant params from compiled model. + */ + void Init(const Array<NDArray>& consts) override { + CHECK_EQ(consts.size(), const_idx_.size()) + << "The number of input constants must match the number of required."; + SetupConstants(consts); + BuildEngine(); + } + + /*! + * \brief Get the JSON generated by codegen. + * + * \param format the format to return (only JSON for the time being) + * \return A string of JSON. + */ + std::string GetSource(const std::string& format) override { Review comment: This one could be moved to base class and it looks to me we don't need to check the format, but instead just return graph_json_ ########## File path: src/runtime/contrib/arm_compute_lib/acl_allocator.cc ########## @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file src/runtime/contrib/arm_compute_lib/acl_allocator.cc + * \brief ACL Allocator implementation that requests memory from TVM. + */ + +#include "acl_allocator.h" + +namespace tvm { +namespace runtime { +namespace contrib { +namespace arm_compute_lib { + +void* ACLAllocator::allocate(size_t size, size_t alignment) { + CHECK_GT(size, 0) << "Cannot allocate size less than or equal to zero"; + return this->device_api_->AllocWorkspace(this->ctx_, size, {}); +} + +void ACLAllocator::free(void* ptr) { this->device_api_->FreeWorkspace(this->ctx_, ptr); } + +std::unique_ptr<arm_compute::IMemoryRegion> ACLAllocator::make_region(size_t size, Review comment: MakeRegion ########## File path: src/runtime/contrib/arm_compute_lib/acl_allocator.cc ########## @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file src/runtime/contrib/arm_compute_lib/acl_allocator.cc + * \brief ACL Allocator implementation that requests memory from TVM. + */ + +#include "acl_allocator.h" + +namespace tvm { +namespace runtime { +namespace contrib { +namespace arm_compute_lib { + +void* ACLAllocator::allocate(size_t size, size_t alignment) { + CHECK_GT(size, 0) << "Cannot allocate size less than or equal to zero"; + return this->device_api_->AllocWorkspace(this->ctx_, size, {}); +} + +void ACLAllocator::free(void* ptr) { this->device_api_->FreeWorkspace(this->ctx_, ptr); } + +std::unique_ptr<arm_compute::IMemoryRegion> ACLAllocator::make_region(size_t size, + size_t alignment) { + return arm_compute::support::cpp14::make_unique<ACLMemoryRegion>(size, alignment); +} + +ACLMemoryRegion::ACLMemoryRegion(size_t size, size_t alignment) + : IMemoryRegion(size), ptr_(nullptr) { + if (size != 0) { + this->ptr_ = this->device_api_->AllocDataSpace(this->ctx_, size, alignment, {}); + } +} + +ACLMemoryRegion::ACLMemoryRegion(void* ptr, size_t size) + : IMemoryRegion(size), ptr_(nullptr), is_subregion_(true) { + if (size != 0) { + this->ptr_ = ptr; + } +} + +ACLMemoryRegion::~ACLMemoryRegion() { + if (this->ptr_ != nullptr && !is_subregion_) { + this->device_api_->FreeDataSpace(this->ctx_, this->ptr_); + } +} + +std::unique_ptr<arm_compute::IMemoryRegion> ACLMemoryRegion::extract_subregion(size_t offset, Review comment: ExtractSubRegion ########## File path: src/relay/backend/contrib/arm_compute_lib/codegen_acl.h ########## @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file src/relay/backend/contrib/arm_compute_lib/codegen_acl.h + * \brief The Relay -> ACL JSON schema compiler. + */ + +#ifndef TVM_RELAY_BACKEND_CONTRIB_ARM_COMPUTE_LIB_CODEGEN_ACL_H_ +#define TVM_RELAY_BACKEND_CONTRIB_ARM_COMPUTE_LIB_CODEGEN_ACL_H_ + +#include <tvm/relay/expr_functor.h> + +#include <map> +#include <memory> +#include <string> +#include <utility> +#include <vector> + +#include "../../../../runtime/contrib/json/json_node.h" +#include "../codegen_json/codegen_json.h" + +namespace tvm { +namespace relay { +namespace contrib { +namespace arm_compute_lib { + +/*! + * \brief Generates an ACLModule from a relay expression. This "compilation" + * does not require ACL since the actual conversion using ACL APIs is + * deferred until creation of the runtime. This step simply serializes the + * relay program into a JSON string. + */ +class ACLJSONSerializer : public backend::contrib::JSONSerializer { + using JSONGraphNode = tvm::runtime::json::JSONGraphNode; + using JSONGraphNodeEntry = tvm::runtime::json::JSONGraphNodeEntry; + + public: + ACLJSONSerializer(const std::string& symbol, const Expr& expr) : JSONSerializer(symbol, expr) {} + + std::vector<JSONGraphNodeEntry> VisitExpr_(const CallNode* cn) override; + + private: + /*! + * \brief Create a JSON representation of an operator. + * + * \param call The call to be represented. + * \return A JSON representation of a specific operator. + */ + std::shared_ptr<JSONGraphNode> CreateOpJSONNode(const CallNode* cn); + std::shared_ptr<JSONGraphNode> CreateCompositeConvJSONNode(const CallNode* cn); +}; + +/*! + * \brief Pre-process a module containing functions ready for ACL codegen. + * + * For now we enforce OHWI kernel layout and fold the transforms away. + * + * \param mod The module to be pre-processed. + * \return The processed module. + */ +IRModule PreProcessModule(const IRModule& mod); + +/*! + * \brief Create a runtime module for ACL. + * + * This consists of a series of "serialized functions" which each represent a + * sub-graph to be computed by ACL and will each be executed independently from + * one another. Each function consists of serialized JSON describing the sub-graph + * and serialized constant tensors. + * + * \note The ACL runtime module only supports a single operator per + * sub-graph currently. + * + * \param ref The ext_func Relay expression/module to be executed using extern ops. + * \return A runtime module. + */ +runtime::Module ACLCompiler(const ObjectRef& ref); Review comment: We don't needs this as well. I believe we don't even needs this file. It looks to me only the header is needed. We can put it in the cc file since no one else is using it. ########## File path: src/runtime/contrib/arm_compute_lib/acl_allocator.h ########## @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file src/runtime/contrib/arm_compute_lib/acl_allocator.h + * \brief ACL Allocator implementation that requests memory from TVM. + */ + +#ifndef TVM_RUNTIME_CONTRIB_ARM_COMPUTE_LIB_ACL_ALLOCATOR_H_ +#define TVM_RUNTIME_CONTRIB_ARM_COMPUTE_LIB_ACL_ALLOCATOR_H_ + +#include <arm_compute/runtime/IAllocator.h> +#include <arm_compute/runtime/IMemoryRegion.h> +#include <arm_compute/runtime/MemoryRegion.h> +#include <tvm/runtime/data_type.h> +#include <tvm/runtime/device_api.h> +#include <tvm/runtime/registry.h> + +#include <memory> + +namespace tvm { +namespace runtime { +namespace contrib { +namespace arm_compute_lib { + +/*! + * \brief Override ACL memory allocator and replace with TVM workspace based allocation. + */ +class ACLAllocator : public arm_compute::IAllocator { + public: + ACLAllocator() = default; + + /*! + * \brief Allocate bytes to ACL runtime. + * + * Specific implementation requests memory from TVM using their device api. + * + * \param size Size to allocate. + * \param alignment Alignment that the returned pointer should comply with. + * \return A pointer to the allocated memory. + */ + void* allocate(size_t size, size_t alignment) override; + + /*! + * \brief Free memory from ACL runtime. + * + * \param ptr Pointer to workspace to free. + */ + void free(void* ptr) override; + + /*! + * \brief Create self-managed memory region. + * + * \param size Size of the memory region. + * \param alignment Alignment of the memory region. + * \return The memory region object. + */ + std::unique_ptr<arm_compute::IMemoryRegion> make_region(size_t size, size_t alignment) override; Review comment: MakeRegion ########## File path: src/runtime/contrib/arm_compute_lib/acl_allocator.h ########## @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file src/runtime/contrib/arm_compute_lib/acl_allocator.h + * \brief ACL Allocator implementation that requests memory from TVM. + */ + +#ifndef TVM_RUNTIME_CONTRIB_ARM_COMPUTE_LIB_ACL_ALLOCATOR_H_ +#define TVM_RUNTIME_CONTRIB_ARM_COMPUTE_LIB_ACL_ALLOCATOR_H_ + +#include <arm_compute/runtime/IAllocator.h> +#include <arm_compute/runtime/IMemoryRegion.h> +#include <arm_compute/runtime/MemoryRegion.h> +#include <tvm/runtime/data_type.h> +#include <tvm/runtime/device_api.h> +#include <tvm/runtime/registry.h> + +#include <memory> + +namespace tvm { +namespace runtime { +namespace contrib { +namespace arm_compute_lib { + +/*! + * \brief Override ACL memory allocator and replace with TVM workspace based allocation. + */ +class ACLAllocator : public arm_compute::IAllocator { + public: + ACLAllocator() = default; + + /*! + * \brief Allocate bytes to ACL runtime. + * + * Specific implementation requests memory from TVM using their device api. + * + * \param size Size to allocate. + * \param alignment Alignment that the returned pointer should comply with. + * \return A pointer to the allocated memory. + */ + void* allocate(size_t size, size_t alignment) override; + + /*! + * \brief Free memory from ACL runtime. + * + * \param ptr Pointer to workspace to free. + */ + void free(void* ptr) override; + + /*! + * \brief Create self-managed memory region. + * + * \param size Size of the memory region. + * \param alignment Alignment of the memory region. + * \return The memory region object. + */ + std::unique_ptr<arm_compute::IMemoryRegion> make_region(size_t size, size_t alignment) override; + + private: + /*! \brief Always allocate data in the context of the current CPU. */ + const TVMContext ctx_{kDLCPU, 0}; + /*! \brief Device API which allows requests for memory from TVM. */ + runtime::DeviceAPI* device_api_ = runtime::DeviceAPI::Get(ctx_); +}; + +/*! + * \brief Memory region that can request TVM memory for ACL to use. + */ +class ACLMemoryRegion : public arm_compute::IMemoryRegion { + public: + ACLMemoryRegion(size_t size, size_t alignment); + ACLMemoryRegion(void* ptr, size_t size); + + ~ACLMemoryRegion() override; + + /*! \brief Prevent instances of this class from being copied (As this class contains + * pointers). */ + ACLMemoryRegion(const ACLMemoryRegion&) = delete; + /*! \brief Default move constructor. */ + ACLMemoryRegion(ACLMemoryRegion&&) = default; + /*! \brief Prevent instances of this class from being copied (As this class + * contains pointers) */ + ACLMemoryRegion& operator=(const ACLMemoryRegion&) = delete; + /*! Default move assignment operator. */ + ACLMemoryRegion& operator=(ACLMemoryRegion&&) = default; + + void* buffer() override { return this->ptr_; } Review comment: Buffer ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
