drill-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From paul-rogers <...@git.apache.org>
Subject [GitHub] drill pull request #574: DRILL-4726: Dynamic UDFs support
Date Mon, 26 Sep 2016 21:53:22 GMT
Github user paul-rogers commented on a diff in the pull request:

    https://github.com/apache/drill/pull/574#discussion_r80549839
  
    --- Diff: exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
---
    @@ -301,29 +323,120 @@ private ScanResult scan(ClassLoader classLoader, Path path, URL[]
urls) throws I
             return RunTimeScan.dynamicPackageScan(drillConfig, Sets.newHashSet(urls));
           }
         }
    -    throw new FunctionValidationException(String.format("Marker file %s is missing in
%s.",
    +    throw new JarValidationException(String.format("Marker file %s is missing in %s",
             CommonConstants.DRILL_JAR_MARKER_FILE_RESOURCE_PATHNAME, path.getName()));
       }
     
    -  private static String getUdfDir() {
    -    return Preconditions.checkNotNull(System.getenv("DRILL_UDF_DIR"), "DRILL_UDF_DIR
variable is not set");
    +  /**
    +   * Return list of jars that are missing in local function registry
    +   * but present in remote function registry.
    +   *
    +   * @param remoteFunctionRegistry remote function registry
    +   * @param localFunctionRegistry local function registry
    +   * @return list of missing jars
    +   */
    +  private List<String> getMissingJars(RemoteFunctionRegistry remoteFunctionRegistry,
    +                                      LocalFunctionRegistry localFunctionRegistry) {
    +    List<Jar> remoteJars = remoteFunctionRegistry.getRegistry().getJarList();
    +    List<String> localJars = localFunctionRegistry.getAllJarNames();
    +    List<String> missingJars = Lists.newArrayList();
    +    for (Jar jar : remoteJars) {
    +      if (!localJars.contains(jar.getName())) {
    +        missingJars.add(jar.getName());
    +      }
    +    }
    +    return missingJars;
    +  }
    +
    +  /**
    +   * Creates local udf directory, if it doesn't exist.
    +   * Checks if local is a directory and if current application has write rights on it.
    +   * Attempts to clean up local idf directory in case jars were left after previous drillbit
run.
    +   *
    +   * @return path to local udf directory
    +   */
    +  private Path getLocalUdfDir() {
    +    String confDir = getConfDir();
    +    File udfDir = new File(confDir, "udf");
    +    String udfPath = udfDir.getPath();
    +    udfDir.mkdirs();
    +    Preconditions.checkState(udfDir.exists(), "Local udf directory [%s] must exist",
udfPath);
    +    Preconditions.checkState(udfDir.isDirectory(), "Local udf directory [%s] must be
a directory", udfPath);
    +    Preconditions.checkState(udfDir.canWrite(), "Local udf directory [%s] must be writable
for application user", udfPath);
    +    try {
    +      FileUtils.cleanDirectory(udfDir);
    +    } catch (IOException e) {
    +      throw new DrillRuntimeException("Error during local udf directory clean up", e);
    +    }
    +    return new Path(udfDir.toURI());
    +  }
    +
    +  /**
    +   * First tries to get drill conf directory value from system properties,
    +   * if value is missing, checks environment properties.
    +   * Throws exception is value is null.
    +   * @return drill conf dir path
    +   */
    +  private String getConfDir() {
    +    String drillConfDir = "DRILL_CONF_DIR";
    +    String value = System.getProperty(drillConfDir);
    +    if (value == null) {
    +      value = Preconditions.checkNotNull(System.getenv(drillConfDir), "%s variable is
not set", drillConfDir);
    +    }
    +    return value;
    +  }
    +
    +  /**
    +   * Copies jar from remote udf area to local udf area with numeric suffix,
    +   * in order to achieve uniqueness for each locally copied jar.
    +   * Ex: DrillUDF-1.0.jar -> DrillUDF-1.0_12200255588.jar
    +   *
    +   * @param jarName jar name to be copied
    +   * @param remoteFunctionRegistry remote function registry
    +   * @return local path to jar that was copied
    +   * @throws IOException in case of problems during jar coping process
    +   */
    +  private Path copyJarToLocal(String jarName, RemoteFunctionRegistry remoteFunctionRegistry)
throws IOException {
    +    String generatedName = String.format(generated_jar_name_pattern,
    +        Files.getNameWithoutExtension(jarName), System.nanoTime(), Files.getFileExtension(jarName));
    +    Path registryArea = remoteFunctionRegistry.getRegistryArea();
    +    FileSystem fs = remoteFunctionRegistry.getFs();
    +    Path remoteJar = new Path(registryArea, jarName);
    +    Path localJar = new Path(localUdfDir, generatedName);
    +    try {
    +      fs.copyToLocalFile(remoteJar, localJar);
    +    } catch (IOException e) {
    +      String message = String.format("Error during jar [%s] coping from [%s] to [%s]",
    +          jarName, registryArea.toUri().getPath(), localUdfDir.toUri().getPath());
    +      throw new IOException(message, e);
    +    }
    +    return localJar;
       }
     
       /**
        * Fires when jar name is submitted for unregistration.
        * Will unregister all functions associated with the jar name
    -   * and delete binary and source associated with the jar from local DRILL_UDF_DIR.
    +   * and delete binary and source associated with the jar from local udf directory
    +   * according to pattern jar name + {@link #jar_suffix_pattern}.
        */
       public class UnregistrationListener implements TransientStoreListener {
     
         @Override
         public void onChange(TransientStoreEvent event) {
    -        String jarName = (String) event.getValue();
    -        String sourceName = JarUtil.getSourceName(jarName);
    -        String localDir = getUdfDir();
    -        FileUtils.deleteQuietly(new File(localDir, jarName));
    -        FileUtils.deleteQuietly(new File(localDir, sourceName));
    -        drillFuncRegistry.unregister(jarName);
    +      String jarName = (String) event.getValue();
    +      localFunctionRegistry.unregister(jarName);
    --- End diff --
    
    Is this reliable? Above, we append a unique number to the local jar name. Here, we unregister
the "raw" jar name.
    
    * Does the registry have both names? The remote name (given here) and the local name (computed
above)?
    * How do we handle ambiguities? Especially if we do rapid-fire create/remove/create of
the same jar name?
    
    Should we leverage ZK to get a unique name? Maybe jar name + timestamp that the JAR was
registered? That way, the (jar name, timestamp) pair will be universally shared across Drillbits,
between ZK and Drillbit, etc. The local name above would be ${base-name}-${timestamp}.jar,
and we could easily recreate that name here in receiving events.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

Mime
View raw message