tvm-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From GitBox <...@apache.org>
Subject [GitHub] [incubator-tvm] comaniac commented on a change in pull request #5770: [BYOC][runtime] Separate code and metadata for CSourceModule
Date Wed, 17 Jun 2020 17:38:06 GMT

comaniac commented on a change in pull request #5770:
URL: https://github.com/apache/incubator-tvm/pull/5770#discussion_r441694863



##########
File path: src/relay/backend/contrib/codegen_c/codegen.cc
##########
@@ -76,43 +77,31 @@ class CodegenC : public MemoizedExprTranslator<std::vector<Output>>,
public Code
   }
 
   std::vector<Output> VisitExpr_(const ConstantNode* cn) final {
-    // Note this is for demonstration purpose. ConstantNode doesn't necessarily
-    // belong to calls. We need to revisit this when tuples come into play.
-
     std::ostringstream decl_stream;
     std::ostringstream buf_stream;
 
     Output output;
-    output.name = "const_" + std::to_string(const_idx_++);
-
-    runtime::NDArray array = cn->data;
-    const auto& shape = array.Shape();
-
-    // Get the number of elements.
-    int64_t num_elems = 1;
-    for (auto i : shape) num_elems *= i;
-
+    // Get const: static_cast<float*>(dnnl_0_consts[0]->data)

Review comment:
       s/dnnl_0/gcc_0/

##########
File path: src/relay/backend/compile_engine.cc
##########
@@ -38,6 +38,7 @@
 #include <tvm/te/schedule.h>
 #include <tvm/te/schedule_pass.h>
 
+#include <cstdio>

Review comment:
       Remove this line?

##########
File path: src/relay/backend/contrib/codegen_c/codegen_c.h
##########
@@ -190,7 +209,12 @@ class CodegenCBase {
    */
   std::string JitImpl(const std::string& ext_func_id, const Array<Var>& args,
                       const std::vector<std::string>& buf_decl,
-                      const std::vector<std::string>& body, const std::vector<Output>&
outs) {
+                      const std::vector<std::string>& body, const std::string&
const_arr,

Review comment:
       `const_arr` seems a bit confusion. Maybe `const_arr_name` or something like that. (ditto
in some other places)

##########
File path: src/runtime/metadata_module.cc
##########
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file src/runtime/metadata_module.cc
+ * \brief A wrapper for initializing imported modules using metadata. This
+ * module is intended to be used by various runtime in the TVM stack, i.e.
+ * graph runtime, relay VM, AOT runtime, and various user defined runtimes. It
+ * paves the way to separate the code and metedata, which makes compilation
+ * and/or interpretation more convenient. In addition, the clear separation of
+ * code and metadata significantly reduces the efforts for handling external
+ * codegen and runtimes.
+ */
+#include <tvm/node/container.h>
+#include <tvm/runtime/ndarray.h>
+#include <tvm/runtime/packed_func.h>
+#include <tvm/runtime/registry.h>
+
+#include <cstdint>
+#include <sstream>
+
+#include "meta_data.h"
+
+namespace tvm {
+namespace runtime {
+
+/*!
+ * \brief The metadata module is designed to manage initialization of the
+ * imported submodules.
+ */
+class MetadataModuleNode : public ModuleNode {
+ public:
+  MetadataModuleNode(const std::unordered_map<std::string, NDArray>& metadata,
+                     const std::unordered_map<std::string, std::vector<std::string>>&
sym_vars)
+      : metadata_(metadata), sym_vars_(sym_vars) {}
+
+  PackedFunc GetFunction(const std::string& name, const ObjectPtr<Object>&
sptr_to_self) final {
+    // Initialize and memoize the module.
+    // Usually, we have some warmup runs. The module initialization should be
+    // done at this stage. Therefore, runtime overhead is not a concern.
+    if (initialized_.count(name) == 0) {
+      this->InitSubModule(name);
+      initialized_[name] = true;
+    }
+
+    // Run the module.
+    // Normally we would only have a limited number of submodules. The runtime
+    // symobl lookup overhead should be minimal.
+    CHECK(!this->imports().empty());
+    for (Module it : this->imports()) {
+      PackedFunc pf = it.GetFunction(name);
+      if (pf != nullptr) return pf;
+    }
+    return PackedFunc(nullptr);
+  }
+
+  const char* type_key() const { return "metadata"; }
+
+  /*!
+   * \brief Get the list of metadata that is required by the given module.
+   * \param symbol The symbol that is being queried.
+   * \return The list of needed NDArray.
+   */
+  Array<NDArray> GetRequiredMetadata(const std::string& symbol) {
+    Array<NDArray> ret;
+    CHECK_GT(sym_vars_.count(symbol), 0U) << "Not symbol is recorded for " <<
symbol;
+    std::vector<std::string> vars = sym_vars_[symbol];
+    for (const auto& it : vars) {
+      CHECK_GT(metadata_.count(it), 0U) << "Found not recorded constant variable: "
<< it;
+      ret.push_back(metadata_[it]);
+    }
+    return ret;
+  }
+
+  /*!
+   * \brief Initialize each imported module.
+   * \param symobl The symbol used for initializing a module. It is also used
+   * for runtime lookup.
+   *
+   * \note  A module could be like the following:
+   *  MetadataModuleNode (contains all the metadata)
+   *    - CSourceModule
+   *    - JSON runtime module
+   *
+   *  The initializer iterates through the imported modules and intilizes the
+   *  found module accordingly by passing the needed metadata into it.
+   */
+  void InitSubModule(const std::string& symbol) {
+    PackedFunc init(nullptr);
+    for (Module it : this->imports()) {
+      // Get the initialization function from the imported modules.
+      std::string init_name = "__init_" + symbol;
+      init = it.GetFunction(init_name, false);
+      if (init != nullptr) {
+        auto md = GetRequiredMetadata(symbol);
+        // Initialize the module with metadata.
+        init(md);
+        break;
+      }
+    }
+  }
+
+  void SaveToBinary(dmlc::Stream* stream) final {
+    std::vector<std::string> variables;
+    std::vector<NDArray> metadata;
+    for (const auto& it : metadata_) {
+      String var_name = it.first;
+      variables.push_back(var_name);
+      metadata.push_back(it.second);
+    }
+
+    // Save all variables in the function.
+    stream->Write(variables);
+    // Save all constant data.
+    uint64_t sz = static_cast<uint64_t>(metadata.size());
+    stream->Write(sz);
+    for (uint64_t i = 0; i < sz; i++) {
+      metadata[i].Save(stream);
+    }
+
+    // Save the symbol to list of required constant variables mapping
+    std::vector<std::string> symbols;
+    std::vector<std::vector<std::string>> const_vars;
+    for (const auto& it : sym_vars_) {
+      symbols.push_back(it.first);
+      const_vars.push_back(it.second);
+    }
+
+    stream->Write(symbols);
+    sz = static_cast<uint64_t>(sym_vars_.size());
+    stream->Write(sz);
+    for (uint64_t i = 0; i < sz; i++) {
+      stream->Write(const_vars[i]);
+    }
+  }
+
+  static Module LoadFromBinary(void* strm) {
+    dmlc::Stream* stream = static_cast<dmlc::Stream*>(strm);
+
+    // Load the variables.
+    std::vector<std::string> variables;
+    CHECK(stream->Read(&variables)) << "Loading variables failed";
+    uint64_t sz;
+    CHECK(stream->Read(&sz, sizeof(sz))) << "Loading medata size failed";
+    CHECK_EQ(static_cast<size_t>(sz), variables.size())

Review comment:
       I think we can keep this cast to make `size_t m_size = static_cast<size_t>(sz)`
so that the rest loops in this function can all use `size_t i`.

##########
File path: src/runtime/metadata_module.cc
##########
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file src/runtime/metadata_module.cc
+ * \brief A wrapper for initializing imported modules using metadata. This
+ * module is intended to be used by various runtime in the TVM stack, i.e.
+ * graph runtime, relay VM, AOT runtime, and various user defined runtimes. It
+ * paves the way to separate the code and metedata, which makes compilation
+ * and/or interpretation more convenient. In addition, the clear separation of
+ * code and metadata significantly reduces the efforts for handling external
+ * codegen and runtimes.
+ */
+#include <tvm/node/container.h>
+#include <tvm/runtime/ndarray.h>
+#include <tvm/runtime/packed_func.h>
+#include <tvm/runtime/registry.h>
+
+#include <cstdint>
+#include <sstream>
+
+#include "meta_data.h"
+
+namespace tvm {
+namespace runtime {
+
+/*!
+ * \brief The metadata module is designed to manage initialization of the
+ * imported submodules.
+ */
+class MetadataModuleNode : public ModuleNode {
+ public:
+  MetadataModuleNode(const std::unordered_map<std::string, NDArray>& metadata,
+                     const std::unordered_map<std::string, std::vector<std::string>>&
sym_vars)
+      : metadata_(metadata), sym_vars_(sym_vars) {}
+
+  PackedFunc GetFunction(const std::string& name, const ObjectPtr<Object>&
sptr_to_self) final {
+    // Initialize and memoize the module.
+    // Usually, we have some warmup runs. The module initialization should be
+    // done at this stage. Therefore, runtime overhead is not a concern.
+    if (initialized_.count(name) == 0) {
+      this->InitSubModule(name);
+      initialized_[name] = true;
+    }
+
+    // Run the module.
+    // Normally we would only have a limited number of submodules. The runtime
+    // symobl lookup overhead should be minimal.
+    CHECK(!this->imports().empty());
+    for (Module it : this->imports()) {
+      PackedFunc pf = it.GetFunction(name);
+      if (pf != nullptr) return pf;
+    }
+    return PackedFunc(nullptr);
+  }
+
+  const char* type_key() const { return "metadata"; }
+
+  /*!
+   * \brief Get the list of metadata that is required by the given module.
+   * \param symbol The symbol that is being queried.
+   * \return The list of needed NDArray.
+   */
+  Array<NDArray> GetRequiredMetadata(const std::string& symbol) {
+    Array<NDArray> ret;
+    CHECK_GT(sym_vars_.count(symbol), 0U) << "Not symbol is recorded for " <<
symbol;
+    std::vector<std::string> vars = sym_vars_[symbol];
+    for (const auto& it : vars) {
+      CHECK_GT(metadata_.count(it), 0U) << "Found not recorded constant variable: "
<< it;
+      ret.push_back(metadata_[it]);
+    }
+    return ret;
+  }
+
+  /*!
+   * \brief Initialize each imported module.
+   * \param symobl The symbol used for initializing a module. It is also used
+   * for runtime lookup.
+   *
+   * \note  A module could be like the following:
+   *  MetadataModuleNode (contains all the metadata)
+   *    - CSourceModule
+   *    - JSON runtime module
+   *
+   *  The initializer iterates through the imported modules and intilizes the
+   *  found module accordingly by passing the needed metadata into it.
+   */
+  void InitSubModule(const std::string& symbol) {
+    PackedFunc init(nullptr);
+    for (Module it : this->imports()) {
+      // Get the initialization function from the imported modules.
+      std::string init_name = "__init_" + symbol;
+      init = it.GetFunction(init_name, false);
+      if (init != nullptr) {
+        auto md = GetRequiredMetadata(symbol);
+        // Initialize the module with metadata.
+        init(md);
+        break;
+      }
+    }
+  }
+
+  void SaveToBinary(dmlc::Stream* stream) final {
+    std::vector<std::string> variables;
+    std::vector<NDArray> metadata;
+    for (const auto& it : metadata_) {
+      String var_name = it.first;
+      variables.push_back(var_name);
+      metadata.push_back(it.second);
+    }
+
+    // Save all variables in the function.
+    stream->Write(variables);
+    // Save all constant data.
+    uint64_t sz = static_cast<uint64_t>(metadata.size());
+    stream->Write(sz);
+    for (uint64_t i = 0; i < sz; i++) {
+      metadata[i].Save(stream);
+    }
+
+    // Save the symbol to list of required constant variables mapping
+    std::vector<std::string> symbols;
+    std::vector<std::vector<std::string>> const_vars;
+    for (const auto& it : sym_vars_) {
+      symbols.push_back(it.first);
+      const_vars.push_back(it.second);
+    }
+
+    stream->Write(symbols);
+    sz = static_cast<uint64_t>(sym_vars_.size());
+    stream->Write(sz);
+    for (uint64_t i = 0; i < sz; i++) {
+      stream->Write(const_vars[i]);
+    }
+  }
+
+  static Module LoadFromBinary(void* strm) {
+    dmlc::Stream* stream = static_cast<dmlc::Stream*>(strm);
+
+    // Load the variables.
+    std::vector<std::string> variables;
+    CHECK(stream->Read(&variables)) << "Loading variables failed";
+    uint64_t sz;
+    CHECK(stream->Read(&sz, sizeof(sz))) << "Loading medata size failed";
+    CHECK_EQ(static_cast<size_t>(sz), variables.size())
+        << "The number of variables and ndarray counts must match";
+    // Load the list of ndarray.
+    std::vector<NDArray> arrays;
+    for (uint64_t i = 0; i < sz; i++) {
+      NDArray temp;
+      temp.Load(stream);
+      arrays.push_back(temp);
+    }
+
+    std::unordered_map<std::string, NDArray> metadata;
+    for (size_t i = 0; i < variables.size(); i++) {
+      CHECK_EQ(metadata.count(variables[i]), 0U);
+      metadata[variables[i]] = arrays[i];
+    }
+
+    // Load the symbol to list of required constant variables mapping
+    std::vector<std::string> symbols;
+    CHECK(stream->Read(&symbols)) << "Loading symbols failed";
+    CHECK(stream->Read(&sz, sizeof(sz))) << "Loading number of symbols failed";
+    CHECK_EQ(static_cast<size_t>(sz), symbols.size());
+    std::vector<std::vector<std::string>> const_vars;
+    for (uint64_t i = 0; i < sz; i++) {
+      std::vector<std::string> vars;
+      CHECK(stream->Read(&vars)) << "Loading const variables failed";
+      const_vars.push_back(vars);
+    }
+
+    std::unordered_map<std::string, std::vector<std::string>> sym_vars;
+    for (uint64_t i = 0; i < sz; i++) {
+      sym_vars[symbols[i]] = const_vars[i];
+    }
+
+    auto n = make_object<MetadataModuleNode>(metadata, sym_vars);
+    return Module(n);
+  }
+
+ private:
+  /*!
+   * \brief Record if a module is initialized. It is needed by imported
+   * modules using execution engine.
+   */
+  std::unordered_map<std::string, bool> initialized_;

Review comment:
       `unordered_set` fits better for this case, but I suppose `unordered_map` will be replaced
with `runtime::Map` in the future so it should be fine to keep it.

##########
File path: src/runtime/metadata_module.cc
##########
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file src/runtime/metadata_module.cc
+ * \brief A wrapper for initializing imported modules using metadata. This
+ * module is intended to be used by various runtime in the TVM stack, i.e.
+ * graph runtime, relay VM, AOT runtime, and various user defined runtimes. It
+ * paves the way to separate the code and metedata, which makes compilation
+ * and/or interpretation more convenient. In addition, the clear separation of
+ * code and metadata significantly reduces the efforts for handling external
+ * codegen and runtimes.
+ */
+#include <tvm/node/container.h>
+#include <tvm/runtime/ndarray.h>
+#include <tvm/runtime/packed_func.h>
+#include <tvm/runtime/registry.h>
+
+#include <cstdint>
+#include <sstream>
+
+#include "meta_data.h"
+
+namespace tvm {
+namespace runtime {
+
+/*!
+ * \brief The metadata module is designed to manage initialization of the
+ * imported submodules.
+ */
+class MetadataModuleNode : public ModuleNode {
+ public:
+  MetadataModuleNode(const std::unordered_map<std::string, NDArray>& metadata,
+                     const std::unordered_map<std::string, std::vector<std::string>>&
sym_vars)
+      : metadata_(metadata), sym_vars_(sym_vars) {}
+
+  PackedFunc GetFunction(const std::string& name, const ObjectPtr<Object>&
sptr_to_self) final {
+    // Initialize and memoize the module.
+    // Usually, we have some warmup runs. The module initialization should be
+    // done at this stage. Therefore, runtime overhead is not a concern.
+    if (initialized_.count(name) == 0) {
+      this->InitSubModule(name);
+      initialized_[name] = true;
+    }
+
+    // Run the module.
+    // Normally we would only have a limited number of submodules. The runtime
+    // symobl lookup overhead should be minimal.
+    CHECK(!this->imports().empty());
+    for (Module it : this->imports()) {
+      PackedFunc pf = it.GetFunction(name);
+      if (pf != nullptr) return pf;
+    }
+    return PackedFunc(nullptr);
+  }
+
+  const char* type_key() const { return "metadata"; }
+
+  /*!
+   * \brief Get the list of metadata that is required by the given module.
+   * \param symbol The symbol that is being queried.
+   * \return The list of needed NDArray.
+   */
+  Array<NDArray> GetRequiredMetadata(const std::string& symbol) {
+    Array<NDArray> ret;
+    CHECK_GT(sym_vars_.count(symbol), 0U) << "Not symbol is recorded for " <<
symbol;
+    std::vector<std::string> vars = sym_vars_[symbol];
+    for (const auto& it : vars) {
+      CHECK_GT(metadata_.count(it), 0U) << "Found not recorded constant variable: "
<< it;
+      ret.push_back(metadata_[it]);
+    }
+    return ret;
+  }
+
+  /*!
+   * \brief Initialize each imported module.
+   * \param symobl The symbol used for initializing a module. It is also used
+   * for runtime lookup.
+   *
+   * \note  A module could be like the following:
+   *  MetadataModuleNode (contains all the metadata)
+   *    - CSourceModule
+   *    - JSON runtime module
+   *
+   *  The initializer iterates through the imported modules and intilizes the
+   *  found module accordingly by passing the needed metadata into it.
+   */
+  void InitSubModule(const std::string& symbol) {
+    PackedFunc init(nullptr);
+    for (Module it : this->imports()) {
+      // Get the initialization function from the imported modules.
+      std::string init_name = "__init_" + symbol;
+      init = it.GetFunction(init_name, false);
+      if (init != nullptr) {
+        auto md = GetRequiredMetadata(symbol);
+        // Initialize the module with metadata.
+        init(md);
+        break;
+      }
+    }
+  }
+
+  void SaveToBinary(dmlc::Stream* stream) final {
+    std::vector<std::string> variables;
+    std::vector<NDArray> metadata;
+    for (const auto& it : metadata_) {
+      String var_name = it.first;
+      variables.push_back(var_name);
+      metadata.push_back(it.second);
+    }
+
+    // Save all variables in the function.
+    stream->Write(variables);
+    // Save all constant data.
+    uint64_t sz = static_cast<uint64_t>(metadata.size());
+    stream->Write(sz);
+    for (uint64_t i = 0; i < sz; i++) {
+      metadata[i].Save(stream);
+    }
+
+    // Save the symbol to list of required constant variables mapping
+    std::vector<std::string> symbols;
+    std::vector<std::vector<std::string>> const_vars;
+    for (const auto& it : sym_vars_) {
+      symbols.push_back(it.first);
+      const_vars.push_back(it.second);
+    }
+
+    stream->Write(symbols);
+    sz = static_cast<uint64_t>(sym_vars_.size());
+    stream->Write(sz);
+    for (uint64_t i = 0; i < sz; i++) {
+      stream->Write(const_vars[i]);
+    }
+  }
+
+  static Module LoadFromBinary(void* strm) {
+    dmlc::Stream* stream = static_cast<dmlc::Stream*>(strm);
+
+    // Load the variables.
+    std::vector<std::string> variables;
+    CHECK(stream->Read(&variables)) << "Loading variables failed";
+    uint64_t sz;
+    CHECK(stream->Read(&sz, sizeof(sz))) << "Loading medata size failed";

Review comment:
       s/medata/metadata/

##########
File path: src/runtime/metadata_module.cc
##########
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file src/runtime/metadata_module.cc
+ * \brief A wrapper for initializing imported modules using metadata. This
+ * module is intended to be used by various runtime in the TVM stack, i.e.
+ * graph runtime, relay VM, AOT runtime, and various user defined runtimes. It
+ * paves the way to separate the code and metedata, which makes compilation
+ * and/or interpretation more convenient. In addition, the clear separation of
+ * code and metadata significantly reduces the efforts for handling external
+ * codegen and runtimes.
+ */
+#include <tvm/node/container.h>
+#include <tvm/runtime/ndarray.h>
+#include <tvm/runtime/packed_func.h>
+#include <tvm/runtime/registry.h>
+
+#include <cstdint>
+#include <sstream>
+
+#include "meta_data.h"
+
+namespace tvm {
+namespace runtime {
+
+/*!
+ * \brief The metadata module is designed to manage initialization of the
+ * imported submodules.
+ */
+class MetadataModuleNode : public ModuleNode {
+ public:
+  MetadataModuleNode(const std::unordered_map<std::string, NDArray>& metadata,
+                     const std::unordered_map<std::string, std::vector<std::string>>&
sym_vars)
+      : metadata_(metadata), sym_vars_(sym_vars) {}
+
+  PackedFunc GetFunction(const std::string& name, const ObjectPtr<Object>&
sptr_to_self) final {
+    // Initialize and memoize the module.
+    // Usually, we have some warmup runs. The module initialization should be
+    // done at this stage. Therefore, runtime overhead is not a concern.
+    if (initialized_.count(name) == 0) {
+      this->InitSubModule(name);
+      initialized_[name] = true;
+    }
+
+    // Run the module.
+    // Normally we would only have a limited number of submodules. The runtime
+    // symobl lookup overhead should be minimal.
+    CHECK(!this->imports().empty());
+    for (Module it : this->imports()) {
+      PackedFunc pf = it.GetFunction(name);
+      if (pf != nullptr) return pf;
+    }
+    return PackedFunc(nullptr);
+  }
+
+  const char* type_key() const { return "metadata"; }
+
+  /*!
+   * \brief Get the list of metadata that is required by the given module.
+   * \param symbol The symbol that is being queried.
+   * \return The list of needed NDArray.
+   */
+  Array<NDArray> GetRequiredMetadata(const std::string& symbol) {
+    Array<NDArray> ret;
+    CHECK_GT(sym_vars_.count(symbol), 0U) << "Not symbol is recorded for " <<
symbol;
+    std::vector<std::string> vars = sym_vars_[symbol];
+    for (const auto& it : vars) {
+      CHECK_GT(metadata_.count(it), 0U) << "Found not recorded constant variable: "
<< it;
+      ret.push_back(metadata_[it]);
+    }
+    return ret;
+  }
+
+  /*!
+   * \brief Initialize each imported module.
+   * \param symobl The symbol used for initializing a module. It is also used
+   * for runtime lookup.
+   *
+   * \note  A module could be like the following:
+   *  MetadataModuleNode (contains all the metadata)
+   *    - CSourceModule
+   *    - JSON runtime module
+   *
+   *  The initializer iterates through the imported modules and intilizes the
+   *  found module accordingly by passing the needed metadata into it.
+   */
+  void InitSubModule(const std::string& symbol) {
+    PackedFunc init(nullptr);
+    for (Module it : this->imports()) {
+      // Get the initialization function from the imported modules.
+      std::string init_name = "__init_" + symbol;
+      init = it.GetFunction(init_name, false);
+      if (init != nullptr) {
+        auto md = GetRequiredMetadata(symbol);
+        // Initialize the module with metadata.
+        init(md);
+        break;
+      }
+    }

Review comment:
       Better to have a CHECK to make sure it was initialized correctly.

##########
File path: src/runtime/metadata_module.cc
##########
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file src/runtime/metadata_module.cc
+ * \brief A wrapper for initializing imported modules using metadata. This
+ * module is intended to be used by various runtime in the TVM stack, i.e.
+ * graph runtime, relay VM, AOT runtime, and various user defined runtimes. It
+ * paves the way to separate the code and metedata, which makes compilation
+ * and/or interpretation more convenient. In addition, the clear separation of
+ * code and metadata significantly reduces the efforts for handling external
+ * codegen and runtimes.
+ */
+#include <tvm/node/container.h>
+#include <tvm/runtime/ndarray.h>
+#include <tvm/runtime/packed_func.h>
+#include <tvm/runtime/registry.h>
+
+#include <cstdint>
+#include <sstream>
+
+#include "meta_data.h"
+
+namespace tvm {
+namespace runtime {
+
+/*!
+ * \brief The metadata module is designed to manage initialization of the
+ * imported submodules.
+ */
+class MetadataModuleNode : public ModuleNode {
+ public:
+  MetadataModuleNode(const std::unordered_map<std::string, NDArray>& metadata,
+                     const std::unordered_map<std::string, std::vector<std::string>>&
sym_vars)
+      : metadata_(metadata), sym_vars_(sym_vars) {}
+
+  PackedFunc GetFunction(const std::string& name, const ObjectPtr<Object>&
sptr_to_self) final {
+    // Initialize and memoize the module.
+    // Usually, we have some warmup runs. The module initialization should be
+    // done at this stage. Therefore, runtime overhead is not a concern.
+    if (initialized_.count(name) == 0) {
+      this->InitSubModule(name);
+      initialized_[name] = true;
+    }
+
+    // Run the module.
+    // Normally we would only have a limited number of submodules. The runtime
+    // symobl lookup overhead should be minimal.
+    CHECK(!this->imports().empty());
+    for (Module it : this->imports()) {
+      PackedFunc pf = it.GetFunction(name);
+      if (pf != nullptr) return pf;
+    }
+    return PackedFunc(nullptr);
+  }
+
+  const char* type_key() const { return "metadata"; }
+
+  /*!
+   * \brief Get the list of metadata that is required by the given module.
+   * \param symbol The symbol that is being queried.
+   * \return The list of needed NDArray.
+   */
+  Array<NDArray> GetRequiredMetadata(const std::string& symbol) {
+    Array<NDArray> ret;
+    CHECK_GT(sym_vars_.count(symbol), 0U) << "Not symbol is recorded for " <<
symbol;

Review comment:
       s/Not/No/

##########
File path: src/relay/backend/contrib/codegen_c/codegen.cc
##########
@@ -76,43 +77,31 @@ class CodegenC : public MemoizedExprTranslator<std::vector<Output>>,
public Code
   }
 
   std::vector<Output> VisitExpr_(const ConstantNode* cn) final {
-    // Note this is for demonstration purpose. ConstantNode doesn't necessarily
-    // belong to calls. We need to revisit this when tuples come into play.
-
     std::ostringstream decl_stream;
     std::ostringstream buf_stream;
 
     Output output;
-    output.name = "const_" + std::to_string(const_idx_++);
-
-    runtime::NDArray array = cn->data;
-    const auto& shape = array.Shape();
-
-    // Get the number of elements.
-    int64_t num_elems = 1;
-    for (auto i : shape) num_elems *= i;
-
+    // Get const: static_cast<float*>(dnnl_0_consts[0]->data)
+    output.name = "static_cast<float*>(" + ext_func_id_ + "_consts[" + std::to_string(const_idx_)
+
+                  "]->data)";
     const auto* type_node = cn->checked_type().as<TensorTypeNode>();
     CHECK(type_node);
     const auto& dtype = GetDtypeString(type_node);
-    // Define a const buffer: float const_0[64] = {1.0, 2.0, ...};
-    //
-    // Technically, you may need: static float* const_0 = (float*)malloc(4 * 64)
-    // to avoid possible stack overflow.
-    buf_stream << dtype << " " << output.name << "[" << num_elems
<< "] = {";
-    if (dtype == "float") {
-      float* p_flt = static_cast<float*>(array->data);
-      for (int64_t i = 0; i < num_elems - 1; i++) buf_stream << p_flt[i] <<
", ";
-      if (num_elems) buf_stream << p_flt[num_elems - 1];
-    } else if (dtype == "int") {
-      int* p_flt = static_cast<int*>(array->data);
-      for (int64_t i = 0; i < num_elems - 1; i++) buf_stream << p_flt[i] <<
", ";
-      if (num_elems) buf_stream << p_flt[num_elems - 1];
-    } else {
-      LOG(FATAL) << "Only float and int are supported for now.";
+
+    // Generate the global variable for needed ndarrays
+    if (const_array_.empty()) {
+      const_array_ = "Array<NDArray> " + ext_func_id_ + "_consts;";
+      std::ostringstream buf_stream;
+      buf_stream << "CHECK(!" << ext_func_id_
+                 << "_consts.empty()) << \"C source module hasn't been initialized.\";\n";
+      ext_func_body_.insert(ext_func_body_.begin(), buf_stream.str());
     }
-    buf_stream << "};";
-    ext_func_body.insert(ext_func_body.begin(), buf_stream.str());
+
+    CHECK(dtype == "float" || dtype == "int") << "Only float and int are supported
for now.";
+    output.dtype = dtype;
+
+    std::string const_var_name = ext_func_id_ + "_const_" + std::to_string(const_idx_++);

Review comment:
       As `_const_` is now a keyword for runtime to take actions, we may generate `const_var_name`
in the base class to make sure users won't violate miss it.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



Mime
View raw message