Mailing-List: contact issues-help@hivemall.incubator.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@hivemall.incubator.apache.org
From: myui <git@git.apache.org>
To: issues@hivemall.incubator.apache.org
Reply-To: issues@hivemall.incubator.apache.org
References: <git-pr-118-incubator-hivemall@git.apache.org>
In-Reply-To: <git-pr-118-incubator-hivemall@git.apache.org>
Subject: [GitHub] incubator-hivemall pull request #118: [HIVEMALL-146] Yet another UDF to gene...
Content-Type: text/plain
Message-Id: <20171003064441.A8E58F5A0C@git1-us-west.apache.org>
Date: Tue,  3 Oct 2017 06:44:41 +0000 (UTC)
archived-at: Tue, 03 Oct 2017 06:44:47 -0000

Github user myui commented on a diff in the pull request:

    https://github.com/apache/incubator-hivemall/pull/118#discussion_r142323566
  
    --- Diff: core/src/main/java/hivemall/tools/text/WordNgramsUDF.java ---
    @@ -0,0 +1,85 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements.  See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership.  The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License.  You may obtain a copy of the License at
    + *
    + *   http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing,
    + * software distributed under the License is distributed on an
    + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
    + * KIND, either express or implied.  See the License for the
    + * specific language governing permissions and limitations
    + * under the License.
    + */
    +package hivemall.tools.text;
    +
    +import org.apache.hadoop.hive.ql.exec.Description;
    +import org.apache.hadoop.hive.ql.exec.UDF;
    +import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
    +import org.apache.hadoop.hive.ql.metadata.HiveException;
    +import org.apache.hadoop.hive.ql.udf.UDFType;
    +import org.apache.hadoop.io.Text;
    +
    +import javax.annotation.Nonnegative;
    +import javax.annotation.Nonnull;
    +import javax.annotation.Nullable;
    +
    +import java.util.ArrayList;
    +import java.util.List;
    +
    +@Description(name = "word_ngrams", value = "_FUNC_(array<string> words, int minSize, int maxSize])"
    +        + " - Returns list of n-grams for given words, where `minSize <= n <= maxSize`")
    +@UDFType(deterministic = true, stateful = false)
    +public final class WordNgramsUDF extends UDF {
    +
    +    @Nullable
    +    public List<Text> evaluate(@Nullable final List<Text> words, final int minSize,
    +            final int maxSize) throws HiveException {
    +        if (words == null) {
    +            return null;
    +        }
    +        if (minSize <= 0) {
    +            throw new UDFArgumentException("`minSize` must be greater than zero: " + minSize);
    +        }
    +        if (minSize > maxSize) {
    +            throw new UDFArgumentException("`maxSize` must be greater than or equal to `minSize`: "
    +                    + maxSize);
    +        }
    +        return getNgrams(words, minSize, maxSize);
    +    }
    +
    +    @Nonnull
    +    private static List<Text> getNgrams(@Nonnull final List<Text> words,
    +            @Nonnegative final int minSize, @Nonnegative final int maxSize) throws HiveException {
    +        final List<Text> ngrams = new ArrayList<Text>();
    +        for (int i = 0, numWords = words.size(); i < numWords; i++) {
    +            for (int ngramSize = minSize; ngramSize <= maxSize; ngramSize++) {
    +                final int end = i + ngramSize;
    +                if (end > numWords) { // exceeds the final element
    +                    continue;
    +                }
    +
    +                final StringBuilder ngram = new StringBuilder();
    +                for (int j = i; j < end; j++) {
    +                    final Text word = words.get(j);
    +                    if (word == null) {
    +                        throw new HiveException(
    --- End diff --
    
    UDFArgumentException is more appropriate.


---