ignite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From akuznet...@apache.org
Subject [10/13] incubator-ignite git commit: # Deleted IGFS load tests as they are no longer used.
Date Tue, 10 Mar 2015 10:14:04 GMT
# Deleted IGFS load tests as they are no longer used.


Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/59f008dc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/59f008dc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/59f008dc

Branch: refs/heads/ignite-329
Commit: 59f008dca7ec41de9b5997837f365b75207224a0
Parents: ffcf04c
Author: vozerov-gridgain <vozerov@gridgain.com>
Authored: Tue Mar 10 11:52:57 2015 +0300
Committer: vozerov-gridgain <vozerov@gridgain.com>
Committed: Tue Mar 10 11:52:57 2015 +0300

----------------------------------------------------------------------
 .../java/org/apache/ignite/igfs/IgfsLoad.java   | 549 -------------------
 1 file changed, 549 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/59f008dc/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsLoad.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsLoad.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsLoad.java
deleted file mode 100644
index 6a85bd0..0000000
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsLoad.java
+++ /dev/null
@@ -1,549 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.igfs;
-
-import org.apache.hadoop.conf.*;
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.permission.*;
-import org.apache.ignite.*;
-import org.apache.ignite.internal.util.typedef.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-
-import java.io.*;
-import java.util.*;
-import java.util.concurrent.*;
-
-/**
- * Hadoop file system load application.
- * <p>
- * Command line arguments:
- * <ul>
- * <li>-u {url} file system URL</li>
- * <li>-hadoopCfg {cfg} Hadoop configuration</li>
- * <li>-f {num} files number</li>
- * <li>-r {num} reads number</li>
- * <li>-w {num} writes number</li>
- * <li>-d {num} deletes number</li>
- * <li>-delay {delay} delay between operations in milliseconds</li>
- * <li>-t {num} threads number</li>
- * <li>-minSize {min size} min file size in bytes</li>
- * <li>-maxSize {max size} max file size in bytes</li>
- * <li>-startNode {true|false} if 'true' then starts node before execution</li>
- * <li>-nodeCfg {cfg} configuration for started node</li>
- * <li>-primaryOnly {true|false} if 'true' then creates files only in directory named
'primary' </li>
- * </ul>
- * Note: IGFS logging is disabled by default, to enable logging it is necessary to set flag
- * 'fs.igfs.<name>.log.enabled' in Hadoop configuration file. By default log files
are created in the
- * directory 'work/igfs/log', this path can be changed in Hadoop configuration file using
property
- * 'fs.igfs.<name>.log.dir'.
- */
-public class IgfsLoad {
-    /** */
-    private static final String DFLT_URL = "igfs:///";
-
-    /** */
-    private static final int DFLT_MIN_FILE_SIZE = 100 * 1024;
-
-    /** */
-    private static final int DFLT_MAX_FILE_SIZE = 1024 * 1024;
-
-    /** */
-    private static final int DFLT_FILES_NUMBER = 1000;
-
-    /** */
-    private static final int DFLT_READS_NUMBER = 2000;
-
-    /** */
-    private static final int DFLT_WRITES_NUMBER = 2000;
-
-    /** */
-    private static final int DFLT_DELETES_NUMBER = 100;
-
-    /** */
-    private static final int DFLT_THREADS_NUMBER = 2;
-
-    /** */
-    private static final boolean DFLT_START_NODE = true;
-
-    /** */
-    private static final boolean DFLT_PRIMARY_ONLY = false;
-
-    /** */
-    private static final String DFLT_NODE_CFG = "config/hadoop/default-config.xml";
-
-    /** */
-    private static final long DFLT_DELAY = 5;
-
-    /** */
-    private static final String DFLT_HADOOP_CFG = "examples/config/filesystem/core-site.xml";
-
-    /** */
-    private static final int CREATE_BUF_SIZE = 100 * 1024;
-
-    /** */
-    private static final String DIR_PRIMARY_MODE = "primary";
-
-    /** */
-    private static final String DIR_PROXY_MODE = "proxy";
-
-    /** */
-    private static final String DIR_DUAL_SYNC_MODE = "dual_sync";
-
-    /** */
-    private static final String DIR_DUAL_ASYNC_MODE = "dual_async";
-
-    /**
-     * Main method.
-     *
-     * @param args Command line arguments.
-     * @throws Exception If error occurs.
-     */
-    public static void main(String[] args) throws Exception {
-        String url = DFLT_URL;
-
-        int filesNum = DFLT_FILES_NUMBER;
-
-        int minFileSize = DFLT_MIN_FILE_SIZE;
-
-        int maxFileSize = DFLT_MAX_FILE_SIZE;
-
-        int readsNum = DFLT_READS_NUMBER;
-
-        int writesNum = DFLT_WRITES_NUMBER;
-
-        int deletesNum = DFLT_DELETES_NUMBER;
-
-        int threadsNum = DFLT_THREADS_NUMBER;
-
-        long delay = DFLT_DELAY;
-
-        String nodeCfg = DFLT_NODE_CFG;
-
-        String hadoopCfg = DFLT_HADOOP_CFG;
-
-        boolean startNode = DFLT_START_NODE;
-
-        boolean primaryOnly = DFLT_PRIMARY_ONLY;
-
-        for (int i = 0; i < args.length; i++) {
-            String arg = args[i];
-
-            switch (arg) {
-                case "-u":
-                    url = args[++i];
-
-                    break;
-
-                case "-hadoopCfg":
-                    hadoopCfg= args[++i];
-
-                    break;
-
-                case "-f":
-                    filesNum = Integer.parseInt(args[++i]);
-
-                    break;
-
-                case "-r":
-                    readsNum = Integer.parseInt(args[++i]);
-
-                    break;
-
-                case "-w":
-                    writesNum = Integer.parseInt(args[++i]);
-
-                    break;
-
-                case "-minSize":
-                    minFileSize = Integer.parseInt(args[++i]);
-
-                    break;
-
-                case "-maxSize":
-                    maxFileSize = Integer.parseInt(args[++i]);
-
-                    break;
-
-                case "-d":
-                    deletesNum = Integer.parseInt(args[++i]);
-
-                    break;
-
-                case "-t":
-                    threadsNum = Integer.parseInt(args[++i]);
-
-                    break;
-
-                case "-delay":
-                    delay = Long.parseLong(args[++i]);
-
-                    break;
-
-                case "-startNode":
-                    startNode = Boolean.parseBoolean(args[++i]);
-
-                    break;
-
-                case "-nodeCfg":
-                    nodeCfg= args[++i];
-
-                    break;
-
-                case "-primaryOnly":
-                    primaryOnly = Boolean.parseBoolean(args[++i]);
-
-                    break;
-            }
-        }
-
-        X.println("File system URL: " + url);
-        X.println("Hadoop configuration: " + hadoopCfg);
-        X.println("Primary mode only: " + primaryOnly);
-        X.println("Files number: " + filesNum);
-        X.println("Reads number: " + readsNum);
-        X.println("Writes number: " + writesNum);
-        X.println("Deletes number: " + deletesNum);
-        X.println("Min file size: " + minFileSize);
-        X.println("Max file size: " + maxFileSize);
-        X.println("Threads number: " + threadsNum);
-        X.println("Delay: " + delay);
-
-        if (minFileSize > maxFileSize)
-            throw new IllegalArgumentException();
-
-        Ignite ignite = null;
-
-        if (startNode) {
-            X.println("Starting node using configuration: " + nodeCfg);
-
-            ignite = G.start(U.resolveIgniteUrl(nodeCfg));
-        }
-
-        try {
-            new IgfsLoad().runLoad(url, hadoopCfg, primaryOnly, threadsNum, filesNum, readsNum,
writesNum,
-                deletesNum, minFileSize, maxFileSize, delay);
-        }
-        finally {
-            if (ignite != null)
-                G.stop(true);
-        }
-    }
-
-    /**
-     * Executes read/write/delete operations.
-     *
-     * @param url File system url.
-     * @param hadoopCfg Hadoop configuration.
-     * @param primaryOnly If {@code true} then creates files only on directory named 'primary'.
-     * @param threads Threads number.
-     * @param files Files number.
-     * @param reads Reads number.
-     * @param writes Writes number.
-     * @param deletes Deletes number.
-     * @param minSize Min file size.
-     * @param maxSize Max file size.
-     * @param delay Delay between operations.
-     * @throws Exception If some file system operation failed.
-     */
-    @SuppressWarnings("IfMayBeConditional")
-    public void runLoad(String url, String hadoopCfg, final boolean primaryOnly, int threads,
int files,
-        final int reads, final int writes, final int deletes, final int minSize, final int
maxSize, final long delay)
-        throws Exception {
-        Path fsPath = new Path(url);
-
-        Configuration cfg = new Configuration(true);
-
-        cfg.addResource(U.resolveIgniteUrl(hadoopCfg));
-
-        final FileSystem fs = FileSystem.get(fsPath.toUri(), cfg);
-
-        Path workDir = new Path(fsPath, "/fsload");
-
-        fs.delete(workDir, true);
-
-        fs.mkdirs(workDir, FsPermission.getDefault());
-
-        final Path[] dirs;
-
-        if (primaryOnly)
-            dirs = new Path[]{mkdir(fs, workDir, DIR_PRIMARY_MODE)};
-        else
-            dirs = new Path[]{mkdir(fs, workDir, DIR_PRIMARY_MODE), mkdir(fs, workDir, DIR_PROXY_MODE),
-                mkdir(fs, workDir, DIR_DUAL_SYNC_MODE), mkdir(fs, workDir, DIR_DUAL_ASYNC_MODE)};
-
-        try {
-            ExecutorService exec = Executors.newFixedThreadPool(threads);
-
-            Collection<Future<?>> futs = new ArrayList<>(threads);
-
-            for (int i = 0; i < threads; i++) {
-                final int filesPerThread;
-
-                if (i == 0 && files % threads != 0)
-                    filesPerThread = files / threads + files % threads;
-                else
-                    filesPerThread = files / threads;
-
-                futs.add(exec.submit(new Callable<Void>() {
-                    @Override public Void call() throws Exception {
-                        runLoad(fs, dirs, filesPerThread, reads, writes, deletes, minSize,
maxSize, delay);
-
-                        return null;
-                    }
-                }));
-            }
-
-            exec.shutdown();
-
-            for (Future<?> fut : futs) {
-                try {
-                    fut.get();
-                }
-                catch (ExecutionException e) {
-                    X.error("Error during execution: " + e);
-
-                    e.getCause().printStackTrace();
-                }
-            }
-        }
-        finally {
-            try {
-                fs.delete(workDir, true);
-            }
-            catch (IOException ignored) {
-                // Ignore.
-            }
-        }
-    }
-
-    /**
-     * Executes read/write/delete operations.
-     *
-     * @param fs File system.
-     * @param dirs Directories where files should be created.
-     * @param filesNum Files number.
-     * @param reads Reads number.
-     * @param writes Writes number.
-     * @param deletes Deletes number.
-     * @param minSize Min file size.
-     * @param maxSize Max file size.
-     * @param delay Delay between operations.
-     * @throws Exception If some file system operation failed.
-     */
-    private void runLoad(FileSystem fs, Path[] dirs, int filesNum, int reads, int writes,
int deletes,
-        int minSize, int maxSize, long delay) throws Exception {
-        Random random = random();
-
-        List<T2<Path, Integer>> files = new ArrayList<>(filesNum);
-
-        for (int i = 0; i < filesNum; i++) {
-            int size = maxSize == minSize ? minSize : minSize + random.nextInt(maxSize -
minSize);
-
-            Path file = new Path(dirs[i % dirs.length], "file-" + UUID.randomUUID());
-
-            createFile(fs, file, size, CREATE_BUF_SIZE);
-
-            files.add(new T2<>(file, size));
-        }
-
-        List<Path> toDel = new ArrayList<>(deletes);
-
-        for (int i = 0; i < deletes; i++) {
-            int size = maxSize == minSize ? minSize : minSize + random.nextInt(maxSize -
minSize);
-
-            Path file = new Path(dirs[i % dirs.length], "file-to-delete-" + UUID.randomUUID());
-
-            createFile(fs, file, size, CREATE_BUF_SIZE);
-
-            toDel.add(file);
-        }
-
-        while (reads > 0 || writes > 0 || deletes > 0) {
-            if (reads > 0) {
-                reads--;
-
-                T2<Path, Integer> file = files.get(reads % files.size());
-
-                readFull(fs, file.get1(), CREATE_BUF_SIZE);
-
-                int fileSize = file.get2();
-
-                readRandom(fs, file.get1(), fileSize, random.nextInt(fileSize) + 1);
-            }
-
-            if (writes > 0) {
-                writes--;
-
-                T2<Path, Integer> file = files.get(writes % files.size());
-
-                overwriteFile(fs, file.get1(), file.get2(), CREATE_BUF_SIZE);
-
-                appendToFile(fs, file.get1(), random.nextInt(CREATE_BUF_SIZE) + 1);
-            }
-
-            if (deletes > 0) {
-                deletes--;
-
-                deleteFile(fs, toDel.get(deletes));
-            }
-
-            U.sleep(delay);
-        }
-    }
-
-    /**
-     * Creates file.
-     *
-     * @param fs File system.
-     * @param file File path.
-     * @param fileSize File size.
-     * @param bufSize Write buffer size.
-     * @throws IOException If operation failed.
-     */
-    private static void createFile(FileSystem fs, Path file, int fileSize, int bufSize) throws
IOException {
-        create(fs, file, fileSize, bufSize, false);
-    }
-
-    /**
-     * Overwrites file.
-     *
-     * @param fs File system.
-     * @param file File path.
-     * @param fileSize File size.
-     * @param bufSize Write buffer size.
-     * @throws IOException If operation failed.
-     */
-    private static void overwriteFile(FileSystem fs, Path file, int fileSize, int bufSize)
throws IOException {
-        create(fs, file, fileSize, bufSize, true);
-    }
-
-    /**
-     * Appends to file.
-     *
-     * @param fs File system.
-     * @param file File path.
-     * @param appendSize Append size.
-     * @throws IOException If operation failed.
-     */
-    private static void appendToFile(FileSystem fs, Path file, int appendSize) throws IOException
{
-        try (FSDataOutputStream out = fs.append(file)) {
-            out.write(new byte[appendSize]);
-        }
-    }
-
-    /**
-     * Reads whole file.
-     *
-     * @param fs File system.
-     * @param file File path.
-     * @param bufSize Read buffer size.
-     * @throws IOException If operation failed.
-     */
-    @SuppressWarnings("StatementWithEmptyBody")
-    private static void readFull(FileSystem fs, Path file, int bufSize) throws IOException
{
-        try (FSDataInputStream in = fs.open(file)) {
-            byte[] readBuf = new byte[bufSize];
-
-            while (in.read(readBuf) > 0) {
-                // No-op.
-            }
-        }
-    }
-
-    /**
-     * Deletes file.
-     *
-     * @param fs File system.
-     * @param path File path.
-     * @throws IOException If operation failed.
-     */
-    private static void deleteFile(FileSystem fs, Path path) throws IOException {
-        fs.delete(path, false);
-    }
-
-    /**
-     * Reads from random position.
-     *
-     * @param fs File system.
-     * @param path File path.
-     * @param fileSize File size.
-     * @param readSize Read size.
-     * @throws IOException If operation failed.
-     */
-    private static void readRandom(FileSystem fs, Path path, int fileSize, int readSize)
throws IOException {
-        byte[] readBuf = new byte[readSize];
-
-        try (FSDataInputStream in = fs.open(path)) {
-            in.seek(random().nextInt(fileSize));
-
-            in.read(readBuf);
-        }
-    }
-
-    /**
-     * Creates file.
-     *
-     * @param fs File system.
-     * @param file File path.
-     * @param fileSize File size.
-     * @param bufSize Buffer size.
-     * @param overwrite Overwrite flag.
-     * @throws IOException If operation failed.
-     */
-    private static void create(FileSystem fs, Path file, int fileSize, int bufSize, boolean
overwrite)
-        throws IOException {
-        try (FSDataOutputStream out = fs.create(file, overwrite)) {
-            int size = 0;
-
-            byte[] buf = new byte[bufSize];
-
-            while (size < fileSize) {
-                int len = Math.min(fileSize - size, bufSize);
-
-                out.write(buf, 0, len);
-
-                size += len;
-            }
-        }
-    }
-
-    /**
-     * Creates directory in the given parent directory.
-     *
-     * @param fs File system.
-     * @param parentDir Parent directory.
-     * @param dirName Directory name.
-     * @return Path for created directory.
-     * @throws IOException If operation failed.
-     */
-    private static Path mkdir(FileSystem fs, Path parentDir, String dirName) throws IOException
{
-        Path path = new Path(parentDir, dirName);
-
-        fs.mkdirs(path, FsPermission.getDefault());
-
-        return path;
-    }
-
-    /**
-     * @return Thread local random.
-     */
-    private static Random random() {
-        return ThreadLocalRandom.current();
-    }
-}


Mime
View raw message