Mailing-List: contact dev-help@quickstep.incubator.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@quickstep.incubator.apache.org
From: zuyu <git@git.apache.org>
To: dev@quickstep.incubator.apache.org
Reply-To: dev@quickstep.incubator.apache.org
References: <git-pr-291-incubator-quickstep@git.apache.org>
In-Reply-To: <git-pr-291-incubator-quickstep@git.apache.org>
Subject: [GitHub] incubator-quickstep pull request #291: Add "COPY TO" operator for exporting ...
Content-Type: text/plain
Message-Id: <20170830200402.0889DF55A0@git1-us-west.apache.org>
Date: Wed, 30 Aug 2017 20:04:01 +0000 (UTC)
archived-at: Wed, 30 Aug 2017 20:04:12 -0000

Github user zuyu commented on a diff in the pull request:

    https://github.com/apache/incubator-quickstep/pull/291#discussion_r136162772
  
    --- Diff: query_optimizer/resolver/Resolver.cpp ---
    @@ -418,27 +455,157 @@ L::LogicalPtr Resolver::resolve(const ParseStatement &parse_query) {
     }
     
     L::LogicalPtr Resolver::resolveCopyFrom(
    -    const ParseStatementCopyFrom &copy_from_statement) {
    -  // Default parameters.
    -  std::string column_delimiter_ = "\t";
    -  bool escape_strings_ = true;
    +    const ParseStatementCopy &copy_from_statement) {
    +  DCHECK(copy_from_statement.getCopyDirection() == ParseStatementCopy::kFrom);
    +  const PtrList<ParseKeyValue> *params = copy_from_statement.params();
     
    -  const ParseCopyFromParams *params = copy_from_statement.params();
    +  BulkIOFormat file_format = BulkIOFormat::kText;
       if (params != nullptr) {
    -    if (params->delimiter != nullptr) {
    -      column_delimiter_ = params->delimiter->value();
    -      if (column_delimiter_.size() != 1) {
    -        THROW_SQL_ERROR_AT(params->delimiter)
    -            << "DELIMITER is not a single character";
    +    for (const ParseKeyValue &param : *params) {
    +      const std::string &key = ToLower(param.key()->value());
    +      if (key == "format") {
    +        const ParseString *parse_format = GetKeyValueString(param);
    +        const std::string format = ToLower(parse_format->value());
    +        // TODO(jianqiao): Support other bulk load formats such as CSV.
    +        if (format != "text") {
    +          THROW_SQL_ERROR_AT(parse_format) << "Unsupported file format: " << format;
    +        }
    +        // Update file_format when other formats get supported.
    +        break;
    +      }
    +    }
    +  }
    +
    +  std::unique_ptr<BulkIOConfiguration> options =
    +      std::make_unique<BulkIOConfiguration>(file_format);
    +  if (params != nullptr) {
    +    for (const ParseKeyValue &param : *params) {
    +      const std::string &key = ToLower(param.key()->value());
    +      if (key == "delimiter") {
    +        const ParseString *parse_delimiter = GetKeyValueString(param);
    +        const std::string &delimiter = parse_delimiter->value();
    +        if (delimiter.size() != 1) {
    +          THROW_SQL_ERROR_AT(parse_delimiter)
    +              << "DELIMITER is not a single character";
    +        }
    +        options->setDelimiter(delimiter.front());
    +      } else if (key == "escape_strings") {
    +        options->setEscapeStrings(GetKeyValueBool(param));
    +      } else if (key != "format") {
    +        THROW_SQL_ERROR_AT(&param) << "Unsupported copy option: " << key;
           }
         }
    -    escape_strings_ = params->escape_strings;
       }
     
       return L::CopyFrom::Create(resolveRelationName(copy_from_statement.relation_name()),
    -                             copy_from_statement.source_filename()->value(),
    -                             column_delimiter_[0],
    -                             escape_strings_);
    +                             copy_from_statement.file_name()->value(),
    +                             BulkIOConfigurationPtr(options.release()));
    +}
    +
    +L::LogicalPtr Resolver::resolveCopyTo(
    +    const ParseStatementCopy &copy_to_statement) {
    +  DCHECK(copy_to_statement.getCopyDirection() == ParseStatementCopy::kTo);
    +  const PtrList<ParseKeyValue> *params = copy_to_statement.params();
    +
    +  // Check if copy format is explicitly specified.
    +  BulkIOFormat file_format = BulkIOFormat::kText;
    +  bool format_specified = false;
    +  if (params != nullptr) {
    +    for (const ParseKeyValue &param : *params) {
    +      const std::string &key = ToLower(param.key()->value());
    +      if (key == "format") {
    +        const ParseString *parse_format = GetKeyValueString(param);
    +        const std::string format = ToLower(parse_format->value());
    +        if (format == "csv") {
    +          file_format = BulkIOFormat::kCSV;
    +        } else if (format == "text") {
    +          file_format = BulkIOFormat::kText;
    +        } else {
    +          THROW_SQL_ERROR_AT(parse_format) << "Unsupported file format: " << format;
    +        }
    +        format_specified = true;
    +        break;
    +      }
    +    }
    +  }
    +
    +  const std::string &file_name = copy_to_statement.file_name()->value();
    +  if (file_name.length() <= 1) {
    +    THROW_SQL_ERROR_AT(copy_to_statement.file_name())
    +        << "File name can not be empty";
    +  }
    +
    +  // Infer copy format from file name extension.
    +  if (!format_specified) {
    +    if (file_name.length() > 4) {
    +      if (ToLower(file_name.substr(file_name.length() - 4)) == ".csv") {
    +        file_format = BulkIOFormat::kCSV;
    +      }
    +    }
    +  }
    +
    +  // Resolve the copy options.
    +  std::unique_ptr<BulkIOConfiguration> options =
    +      std::make_unique<BulkIOConfiguration>(file_format);
    +  if (params != nullptr) {
    +    for (const ParseKeyValue &param : *params) {
    +      const std::string &key = ToLower(param.key()->value());
    +      if (key == "delimiter") {
    +        const ParseString *parse_delimiter = GetKeyValueString(param);
    +        const std::string &delimiter = parse_delimiter->value();
    +        if (delimiter.size() != 1) {
    --- End diff --
    
    Should we change to `1u` to avoid unsign-vs-sign-compare warning?


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---