hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From su...@apache.org
Subject [16/51] [abbrv] hadoop git commit: HADOOP-12875. [Azure Data Lake] Support for contract test and unit test cases. Contributed by Vishwajeet Dusane.
Date Wed, 22 Jun 2016 00:26:14 GMT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9e71382/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestAdlRead.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestAdlRead.java
b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestAdlRead.java
new file mode 100644
index 0000000..60904d0
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestAdlRead.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdfs.web;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.adl.TestADLResponseData;
+import org.apache.hadoop.fs.common.AdlMockWebServer;
+import org.apache.hadoop.fs.common.TestDataForRead;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Random;
+
+/**
+ * This class is responsible for stress positional reads vs number of network
+ * calls required by to fetch the amount of data. Test does ensure the data
+ * integrity and order of the data is maintained. This tests are meant to test
+ * BufferManager.java and BatchByteArrayInputStream implementation.
+ */
+@RunWith(Parameterized.class)
+public class TestAdlRead extends AdlMockWebServer {
+
+  // Keeping timeout of 1 hour to ensure the test does complete and should
+  // not terminate due to high backend latency.
+  @Rule
+  public Timeout globalTimeout = new Timeout(60 * 60000);
+  private TestDataForRead testData;
+
+  public TestAdlRead(TestDataForRead testData) {
+    this.testData = testData;
+    getConf().set("adl.feature.override.readahead.max.buffersize", "8192");
+    getConf().set("adl.feature.override.readahead.max.concurrent.connection",
+        "1");
+  }
+
+  @Parameterized.Parameters(name = "{index}")
+  public static Collection testDataForReadOperation() {
+    return Arrays.asList(new Object[][] {
+
+        //--------------------------
+        // Test Data
+        //--------------------------
+        {new TestDataForRead("Hello World".getBytes(), 3, 1000, true)},
+        {new TestDataForRead(
+            ("the problem you appear to be wrestling with is that this doesn't "
+                + "display very well. ").getBytes(), 3, 1000, true)},
+        {new TestDataForRead(
+            ("Chinese Indonesians (Indonesian: Orang Tionghoa-Indonesia; "
+                + "Chinese: "
+                + "trad ???????, simp ???????, pin Y�nd�n�x?y� Hu�r�n), are "
+                + "Indonesians descended from various Chinese ethnic groups, "
+                + "particularly Han.").getBytes(), 3, 1000, true)},
+        {new TestDataForRead(
+            TestADLResponseData.getRandomByteArrayData(5 * 1024), 3, 1000,
+            true)}, {new TestDataForRead(
+        TestADLResponseData.getRandomByteArrayData(1 * 1024), 3, 50, true)},
+        {new TestDataForRead(
+            TestADLResponseData.getRandomByteArrayData(8 * 1024), 3, 10, true)},
+        {new TestDataForRead(
+            TestADLResponseData.getRandomByteArrayData(32 * 1024), 6, 10,
+            false)}, {new TestDataForRead(
+        TestADLResponseData.getRandomByteArrayData(48 * 1024), 8, 10, false)}});
+  }
+
+  @After
+  @Before
+  public void cleanReadBuffer() {
+    BufferManager.getInstance().clear();
+  }
+
+  @Test
+  public void testEntireBytes() throws IOException, InterruptedException {
+    getMockServer().setDispatcher(testData.getDispatcher());
+    FSDataInputStream in = getMockAdlFileSystem().open(new Path("/test"));
+    byte[] expectedData = new byte[testData.getActualData().length];
+    Assert.assertEquals(in.read(expectedData), expectedData.length);
+    Assert.assertArrayEquals(expectedData, testData.getActualData());
+    in.close();
+    if (testData.isCheckOfNoOfCalls()) {
+      Assert.assertEquals(testData.getExpectedNoNetworkCall(),
+          getMockServer().getRequestCount());
+    }
+  }
+
+  @Test
+  public void testSeekOperation() throws IOException, InterruptedException {
+    getMockServer().setDispatcher(testData.getDispatcher());
+    FSDataInputStream in = getMockAdlFileSystem().open(new Path("/test"));
+    Random random = new Random();
+    for (int i = 0; i < 1000; ++i) {
+      int position = random.nextInt(testData.getActualData().length);
+      in.seek(position);
+      Assert.assertEquals(in.getPos(), position);
+      Assert.assertEquals(in.read(), testData.getActualData()[position] & 0xFF);
+    }
+    in.close();
+    if (testData.isCheckOfNoOfCalls()) {
+      Assert.assertEquals(testData.getExpectedNoNetworkCall(),
+          getMockServer().getRequestCount());
+    }
+  }
+
+  @Test
+  public void testReadServerCalls() throws IOException, InterruptedException {
+    getMockServer().setDispatcher(testData.getDispatcher());
+    FSDataInputStream in = getMockAdlFileSystem().open(new Path("/test"));
+    byte[] expectedData = new byte[testData.getActualData().length];
+    in.readFully(expectedData);
+    Assert.assertArrayEquals(expectedData, testData.getActualData());
+    Assert.assertEquals(testData.getExpectedNoNetworkCall(),
+        getMockServer().getRequestCount());
+    in.close();
+  }
+
+  @Test
+  public void testReadFully() throws IOException, InterruptedException {
+    getMockServer().setDispatcher(testData.getDispatcher());
+    FSDataInputStream in = getMockAdlFileSystem().open(new Path("/test"));
+    byte[] expectedData = new byte[testData.getActualData().length];
+    in.readFully(expectedData);
+    Assert.assertArrayEquals(expectedData, testData.getActualData());
+
+    in.readFully(0, expectedData);
+    Assert.assertArrayEquals(expectedData, testData.getActualData());
+
+    in.seek(0);
+    in.readFully(expectedData, 0, expectedData.length);
+    Assert.assertArrayEquals(expectedData, testData.getActualData());
+    in.close();
+
+    if (testData.isCheckOfNoOfCalls()) {
+      Assert.assertEquals(testData.getExpectedNoNetworkCall(),
+          getMockServer().getRequestCount());
+    }
+  }
+
+  @Test
+  public void testRandomPositionalReadUsingReadFully()
+      throws IOException, InterruptedException {
+    getMockServer().setDispatcher(testData.getDispatcher());
+    FSDataInputStream in = getMockAdlFileSystem().open(new Path("/test"));
+    ByteArrayInputStream actualData = new ByteArrayInputStream(
+        testData.getActualData());
+    Random random = new Random();
+    for (int i = 0; i < testData.getIntensityOfTest(); ++i) {
+      int offset = random.nextInt(testData.getActualData().length);
+      int length = testData.getActualData().length - offset;
+      byte[] expectedData = new byte[length];
+      byte[] actualDataSubset = new byte[length];
+      actualData.reset();
+      actualData.skip(offset);
+      actualData.read(actualDataSubset, 0, length);
+
+      in.readFully(offset, expectedData, 0, length);
+      Assert.assertArrayEquals(expectedData, actualDataSubset);
+    }
+
+    for (int i = 0; i < testData.getIntensityOfTest(); ++i) {
+      int offset = random.nextInt(testData.getActualData().length);
+      int length = random.nextInt(testData.getActualData().length - offset);
+      byte[] expectedData = new byte[length];
+      byte[] actualDataSubset = new byte[length];
+      actualData.reset();
+      actualData.skip(offset);
+      actualData.read(actualDataSubset, 0, length);
+
+      in.readFully(offset, expectedData, 0, length);
+      Assert.assertArrayEquals(expectedData, actualDataSubset);
+    }
+
+    in.close();
+    if (testData.isCheckOfNoOfCalls()) {
+      Assert.assertEquals(testData.getExpectedNoNetworkCall(),
+          getMockServer().getRequestCount());
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9e71382/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestConcurrentDataReadOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestConcurrentDataReadOperations.java
b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestConcurrentDataReadOperations.java
new file mode 100644
index 0000000..1194336
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestConcurrentDataReadOperations.java
@@ -0,0 +1,306 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdfs.web;
+
+import com.squareup.okhttp.mockwebserver.Dispatcher;
+import com.squareup.okhttp.mockwebserver.MockResponse;
+import com.squareup.okhttp.mockwebserver.RecordedRequest;
+import okio.Buffer;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.adl.TestADLResponseData;
+import org.apache.hadoop.fs.common.AdlMockWebServer;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Random;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * This class is responsible for testing multiple threads trying to access same
+ * or multiple files from the offset. This tests are meant to test
+ * BufferManager.java and BatchByteArrayInputStream implementation.
+ */
+@RunWith(Parameterized.class)
+public class TestConcurrentDataReadOperations extends AdlMockWebServer {
+
+  private static FSDataInputStream commonHandle = null;
+  private static Object lock = new Object();
+  private int concurrencyLevel;
+
+  public TestConcurrentDataReadOperations(int concurrencyLevel) {
+    this.concurrencyLevel = concurrencyLevel;
+    getConf().set("adl.feature.override.readahead.max.buffersize", "102400");
+    getConf().set("adl.feature.override.readahead.max.concurrent.connection",
+        "1");
+  }
+
+  @Parameterized.Parameters(name = "{index}")
+  public static Collection testDataNumberOfConcurrentRun() {
+    return Arrays.asList(new Object[][] {{1}, {2}, {3}, {4}, {5}});
+  }
+
+  public static byte[] getRandomByteArrayData(int size) {
+    byte[] b = new byte[size];
+    Random rand = new Random();
+    rand.nextBytes(b);
+    return b;
+  }
+
+  private void setDispatcher(final ArrayList<CreateTestData> testData) {
+    getMockServer().setDispatcher(new Dispatcher() {
+      @Override
+      public MockResponse dispatch(RecordedRequest recordedRequest)
+          throws InterruptedException {
+        if (recordedRequest.getPath().equals("/refresh")) {
+          return AdlMockWebServer.getTokenResponse();
+        }
+
+        CreateTestData currentRequest = null;
+        for (CreateTestData local : testData) {
+          if (recordedRequest.getPath().contains(local.path.toString())) {
+            currentRequest = local;
+            break;
+          }
+        }
+
+        if (currentRequest == null) {
+          new MockResponse().setBody("Request data not found")
+              .setResponseCode(501);
+        }
+
+        if (recordedRequest.getRequestLine().contains("op=GETFILESTATUS")) {
+          return new MockResponse().setResponseCode(200).setBody(
+              TestADLResponseData
+                  .getGetFileStatusJSONResponse(currentRequest.data.length));
+        }
+
+        if (recordedRequest.getRequestLine().contains("op=OPEN")) {
+          String request = recordedRequest.getRequestLine();
+          int offset = 0;
+          int byteCount = 0;
+
+          Pattern pattern = Pattern.compile("offset=([0-9]+)");
+          Matcher matcher = pattern.matcher(request);
+          if (matcher.find()) {
+            System.out.println(matcher.group(1));
+            offset = Integer.parseInt(matcher.group(1));
+          }
+
+          pattern = Pattern.compile("length=([0-9]+)");
+          matcher = pattern.matcher(request);
+          if (matcher.find()) {
+            System.out.println(matcher.group(1));
+            byteCount = Integer.parseInt(matcher.group(1));
+          }
+
+          Buffer buf = new Buffer();
+          buf.write(currentRequest.data, offset, byteCount);
+          return new MockResponse().setResponseCode(200)
+              .setChunkedBody(buf, 4 * 1024 * 1024);
+        }
+
+        return new MockResponse().setBody("NOT SUPPORTED").setResponseCode(501);
+      }
+    });
+  }
+
+  @Before
+  public void resetHandle() {
+    commonHandle = null;
+  }
+
+  @Test
+  public void testParallelReadOnDifferentStreams()
+      throws IOException, InterruptedException, ExecutionException {
+
+    ArrayList<CreateTestData> createTestData = new ArrayList<CreateTestData>();
+
+    Random random = new Random();
+
+    for (int i = 0; i < concurrencyLevel; i++) {
+      CreateTestData testData = new CreateTestData();
+      testData
+          .set(new Path("/test/concurrentRead/" + UUID.randomUUID().toString()),
+              getRandomByteArrayData(random.nextInt(1 * 1024 * 1024)));
+      createTestData.add(testData);
+    }
+
+    setDispatcher(createTestData);
+
+    ArrayList<ReadTestData> readTestData = new ArrayList<ReadTestData>();
+    for (CreateTestData local : createTestData) {
+      ReadTestData localReadData = new ReadTestData();
+      localReadData.set(local.path, local.data, 0);
+      readTestData.add(localReadData);
+    }
+
+    runReadTest(readTestData, false);
+  }
+
+  @Test
+  public void testParallelReadOnSameStreams()
+      throws IOException, InterruptedException, ExecutionException {
+    ArrayList<CreateTestData> createTestData = new ArrayList<CreateTestData>();
+
+    Random random = new Random();
+
+    for (int i = 0; i < 1; i++) {
+      CreateTestData testData = new CreateTestData();
+      testData
+          .set(new Path("/test/concurrentRead/" + UUID.randomUUID().toString()),
+              getRandomByteArrayData(1024 * 1024));
+      createTestData.add(testData);
+    }
+
+    setDispatcher(createTestData);
+
+    ArrayList<ReadTestData> readTestData = new ArrayList<ReadTestData>();
+    ByteArrayInputStream buffered = new ByteArrayInputStream(
+        createTestData.get(0).data);
+
+    ReadTestData readInitially = new ReadTestData();
+    byte[] initialData = new byte[1024 * 1024];
+    buffered.read(initialData);
+
+    readInitially.set(createTestData.get(0).path, initialData, 0);
+    readTestData.add(readInitially);
+    runReadTest(readTestData, false);
+
+    readTestData.clear();
+
+    for (int i = 0; i < concurrencyLevel * 5; i++) {
+      ReadTestData localReadData = new ReadTestData();
+      int offset = random.nextInt((1024 * 1024)-1);
+      int length = 1024 * 1024 - offset;
+      byte[] expectedData = new byte[length];
+      buffered.reset();
+      buffered.skip(offset);
+      buffered.read(expectedData);
+      localReadData.set(createTestData.get(0).path, expectedData, offset);
+      readTestData.add(localReadData);
+    }
+
+    runReadTest(readTestData, true);
+  }
+
+  void runReadTest(ArrayList<ReadTestData> testData, boolean useSameStream)
+      throws InterruptedException, ExecutionException {
+
+    ExecutorService executor = Executors.newFixedThreadPool(testData.size());
+    Future[] subtasks = new Future[testData.size()];
+
+    for (int i = 0; i < testData.size(); i++) {
+      subtasks[i] = executor.submit(
+          new ReadConcurrentRunnable(testData.get(i).data, testData.get(i).path,
+              testData.get(i).offset, useSameStream));
+    }
+
+    executor.shutdown();
+
+    // wait until all tasks are finished
+    executor.awaitTermination(120, TimeUnit.SECONDS);
+
+    for (int i = 0; i < testData.size(); ++i) {
+      Assert.assertTrue((Boolean) subtasks[i].get());
+    }
+  }
+
+  class ReadTestData {
+    private Path path;
+    private byte[] data;
+    private int offset;
+
+    public void set(Path filePath, byte[] dataToBeRead, int fromOffset) {
+      this.path = filePath;
+      this.data = dataToBeRead;
+      this.offset = fromOffset;
+    }
+  }
+
+  class CreateTestData {
+    private Path path;
+    private byte[] data;
+
+    public void set(Path filePath, byte[] dataToBeWritten) {
+      this.path = filePath;
+      this.data = dataToBeWritten;
+    }
+  }
+
+  class ReadConcurrentRunnable implements Callable<Boolean> {
+    private Path path;
+    private int offset;
+    private byte[] expectedData;
+    private boolean useSameStream;
+
+    public ReadConcurrentRunnable(byte[] expectedData, Path path, int offset,
+        boolean useSameStream) {
+      this.path = path;
+      this.offset = offset;
+      this.expectedData = expectedData;
+      this.useSameStream = useSameStream;
+    }
+
+    public Boolean call() throws IOException {
+      try {
+        FSDataInputStream in;
+        if (useSameStream) {
+          synchronized (lock) {
+            if (commonHandle == null) {
+              commonHandle = getMockAdlFileSystem().open(path);
+            }
+            in = commonHandle;
+          }
+        } else {
+          in = getMockAdlFileSystem().open(path);
+        }
+
+        byte[] actualData = new byte[expectedData.length];
+        in.readFully(offset, actualData);
+        Assert.assertArrayEquals("Path :" + path.toString() + " did not match.",
+            expectedData, actualData);
+        if (!useSameStream) {
+          in.close();
+        }
+      } catch (IOException e) {
+        e.printStackTrace();
+        return false;
+      }
+      return true;
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9e71382/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestConfigurationSetting.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestConfigurationSetting.java
b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestConfigurationSetting.java
new file mode 100644
index 0000000..9a54ec2
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestConfigurationSetting.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdfs.web;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.adl.TestableAdlFileSystem;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
+import org.apache.hadoop.hdfs.web.oauth2.ConfCredentialBasedAccessTokenProvider;
+import org.apache.hadoop.hdfs.web.oauth2.CredentialBasedAccessTokenProvider;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+/**
+ * This class is responsible for testing adl file system required configuration
+ * and feature set keys.
+ */
+public class TestConfigurationSetting {
+
+  @Test
+  public void testAllConfiguration() throws URISyntaxException, IOException {
+    TestableAdlFileSystem fs = new TestableAdlFileSystem();
+    Configuration conf = new Configuration();
+    conf.set(HdfsClientConfigKeys.OAUTH_REFRESH_URL_KEY,
+        "http://localhost:1111/refresh");
+    conf.set(CredentialBasedAccessTokenProvider.OAUTH_CREDENTIAL_KEY,
+        "credential");
+    conf.set(HdfsClientConfigKeys.OAUTH_CLIENT_ID_KEY, "MY_CLIENTID");
+    conf.set(HdfsClientConfigKeys.ACCESS_TOKEN_PROVIDER_KEY,
+        ConfCredentialBasedAccessTokenProvider.class.getName());
+    conf.set(HdfsClientConfigKeys.DFS_WEBHDFS_OAUTH_ENABLED_KEY, "true");
+
+    URI uri = new URI("adl://localhost:1234");
+    fs.initialize(uri, conf);
+
+    // Default setting check
+    Assert.assertEquals(true, fs.isFeatureRedirectOff());
+    Assert.assertEquals(true, fs.isFeatureGetBlockLocationLocallyBundled());
+    Assert.assertEquals(true, fs.isFeatureConcurrentReadWithReadAhead());
+    Assert.assertEquals(false, fs.isOverrideOwnerFeatureOn());
+    Assert.assertEquals(8 * 1024 * 1024, fs.getMaxBufferSize());
+    Assert.assertEquals(2, fs.getMaxConcurrentConnection());
+
+    fs.close();
+
+    // Configuration toggle check
+    conf.set("adl.feature.override.redirection.off", "false");
+    fs.initialize(uri, conf);
+    Assert.assertEquals(false, fs.isFeatureRedirectOff());
+    fs.close();
+    conf.set("adl.feature.override.redirection.off", "true");
+    fs.initialize(uri, conf);
+    Assert.assertEquals(true, fs.isFeatureRedirectOff());
+    fs.close();
+
+    conf.set("adl.feature.override.getblocklocation.locally.bundled", "false");
+    fs.initialize(uri, conf);
+    Assert.assertEquals(false, fs.isFeatureGetBlockLocationLocallyBundled());
+    fs.close();
+    conf.set("adl.feature.override.getblocklocation.locally.bundled", "true");
+    fs.initialize(uri, conf);
+    Assert.assertEquals(true, fs.isFeatureGetBlockLocationLocallyBundled());
+    fs.close();
+
+    conf.set("adl.feature.override.readahead", "false");
+    fs.initialize(uri, conf);
+    Assert.assertEquals(false, fs.isFeatureConcurrentReadWithReadAhead());
+    fs.close();
+    conf.set("adl.feature.override.readahead", "true");
+    fs.initialize(uri, conf);
+    Assert.assertEquals(true, fs.isFeatureConcurrentReadWithReadAhead());
+    fs.close();
+
+    conf.set("adl.feature.override.readahead.max.buffersize", "101");
+    fs.initialize(uri, conf);
+    Assert.assertEquals(101, fs.getMaxBufferSize());
+    fs.close();
+    conf.set("adl.feature.override.readahead.max.buffersize", "12134565");
+    fs.initialize(uri, conf);
+    Assert.assertEquals(12134565, fs.getMaxBufferSize());
+    fs.close();
+
+    conf.set("adl.debug.override.localuserasfileowner", "true");
+    fs.initialize(uri, conf);
+    Assert.assertEquals(true, fs.isOverrideOwnerFeatureOn());
+    fs.close();
+    conf.set("adl.debug.override.localuserasfileowner", "false");
+    fs.initialize(uri, conf);
+    Assert.assertEquals(false, fs.isOverrideOwnerFeatureOn());
+    fs.close();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9e71382/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestSplitSizeCalculation.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestSplitSizeCalculation.java
b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestSplitSizeCalculation.java
new file mode 100644
index 0000000..42a4701
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestSplitSizeCalculation.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdfs.web;
+
+import com.squareup.okhttp.mockwebserver.MockResponse;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.adl.TestADLResponseData;
+import org.apache.hadoop.fs.common.AdlMockWebServer;
+import org.apache.hadoop.hdfs.web.PrivateAzureDataLakeFileSystem.BatchByteArrayInputStream;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.net.URL;
+
+/**
+ * This class is responsible for testing split size calculation during
+ * read ahead buffer initiation based on the data size and configuration
+ * initialization.
+ */
+public class TestSplitSizeCalculation extends AdlMockWebServer {
+
+  @Test
+  public void testSplitSizeCalculations()
+      throws URISyntaxException, IOException {
+
+    getMockServer().enqueue(new MockResponse().setResponseCode(200).setBody(
+        TestADLResponseData.getGetFileStatusJSONResponse(128 * 1024 * 1024)));
+    getMockServer().enqueue(new MockResponse().setResponseCode(200).setBody(
+        TestADLResponseData.getGetFileStatusJSONResponse(128 * 1024 * 1024)));
+    getMockServer().enqueue(new MockResponse().setResponseCode(200).setBody(
+        TestADLResponseData.getGetFileStatusJSONResponse(128 * 1024 * 1024)));
+    getMockServer().enqueue(new MockResponse().setResponseCode(200).setBody(
+        TestADLResponseData.getGetFileStatusJSONResponse(128 * 1024 * 1024)));
+
+    URL url = getMockServer().getUrl("");
+
+    BatchByteArrayInputStream stream = getMockAdlFileSystem()
+        .new BatchByteArrayInputStream(url,
+        new Path("/test1/test2"), 16 * 1024 * 1024, 4);
+    Assert.assertEquals(1, stream.getSplitSize(1 * 1024 * 1024));
+    Assert.assertEquals(1, stream.getSplitSize(2 * 1024 * 1024));
+    Assert.assertEquals(1, stream.getSplitSize(3 * 1024 * 1024));
+    Assert.assertEquals(1, stream.getSplitSize(4 * 1024 * 1024));
+    Assert.assertEquals(1, stream.getSplitSize(5 * 1024 * 1024));
+    Assert.assertEquals(1, stream.getSplitSize(6 * 1024 * 1024));
+    Assert.assertEquals(1, stream.getSplitSize(7 * 1024 * 1024));
+    Assert.assertEquals(2, stream.getSplitSize(8 * 1024 * 1024));
+    Assert.assertEquals(4, stream.getSplitSize(16 * 1024 * 1024));
+    Assert.assertEquals(3, stream.getSplitSize(12 * 1024 * 1024));
+    Assert.assertEquals(4, stream.getSplitSize(102 * 1024 * 1024));
+    Assert.assertEquals(1, stream.getSplitSize(102));
+    stream.close();
+
+    stream = getMockAdlFileSystem().new BatchByteArrayInputStream(url,
+        new Path("/test1/test2"), 4 * 1024 * 1024, 4);
+    Assert.assertEquals(1, stream.getSplitSize(1 * 1024 * 1024));
+    Assert.assertEquals(1, stream.getSplitSize(2 * 1024 * 1024));
+    Assert.assertEquals(1, stream.getSplitSize(3 * 1024 * 1024));
+    Assert.assertEquals(1, stream.getSplitSize(4 * 1024 * 1024));
+    Assert.assertEquals(1, stream.getSplitSize(5 * 1024 * 1024));
+    Assert.assertEquals(1, stream.getSplitSize(8 * 1024 * 1024));
+    Assert.assertEquals(1, stream.getSplitSize(5 * 1024 * 1024));
+    Assert.assertEquals(1, stream.getSplitSize(6 * 1024 * 1024));
+    Assert.assertEquals(1, stream.getSplitSize(7 * 1024 * 1024));
+    Assert.assertEquals(1, stream.getSplitSize(16 * 1024 * 1024));
+    Assert.assertEquals(1, stream.getSplitSize(12 * 1024 * 1024));
+    Assert.assertEquals(1, stream.getSplitSize(102 * 1024 * 1024));
+    Assert.assertEquals(1, stream.getSplitSize(102));
+    stream.close();
+
+    stream = getMockAdlFileSystem().new BatchByteArrayInputStream(url,
+        new Path("/test1/test2"), 16 * 1024 * 1024, 2);
+    Assert.assertEquals(1, stream.getSplitSize(1 * 1024 * 1024));
+    Assert.assertEquals(1, stream.getSplitSize(2 * 1024 * 1024));
+    Assert.assertEquals(1, stream.getSplitSize(3 * 1024 * 1024));
+    Assert.assertEquals(1, stream.getSplitSize(4 * 1024 * 1024));
+    Assert.assertEquals(1, stream.getSplitSize(5 * 1024 * 1024));
+    Assert.assertEquals(1, stream.getSplitSize(5 * 1024 * 1024));
+    Assert.assertEquals(1, stream.getSplitSize(6 * 1024 * 1024));
+    Assert.assertEquals(1, stream.getSplitSize(7 * 1024 * 1024));
+    Assert.assertEquals(2, stream.getSplitSize(8 * 1024 * 1024));
+    Assert.assertEquals(2, stream.getSplitSize(16 * 1024 * 1024));
+    Assert.assertEquals(2, stream.getSplitSize(12 * 1024 * 1024));
+    Assert.assertEquals(2, stream.getSplitSize(102 * 1024 * 1024));
+    Assert.assertEquals(1, stream.getSplitSize(102));
+    stream.close();
+
+    stream = getMockAdlFileSystem().new BatchByteArrayInputStream(url,
+        new Path("/test1/test2"), 8 * 1024 * 1024, 2);
+    Assert.assertEquals(1, stream.getSplitSize(1 * 1024 * 1024));
+    Assert.assertEquals(1, stream.getSplitSize(2 * 1024 * 1024));
+    Assert.assertEquals(1, stream.getSplitSize(3 * 1024 * 1024));
+    Assert.assertEquals(1, stream.getSplitSize(4 * 1024 * 1024));
+    Assert.assertEquals(1, stream.getSplitSize(5 * 1024 * 1024));
+    Assert.assertEquals(1, stream.getSplitSize(6 * 1024 * 1024));
+    Assert.assertEquals(1, stream.getSplitSize(7 * 1024 * 1024));
+    Assert.assertEquals(2, stream.getSplitSize(8 * 1024 * 1024));
+    Assert.assertEquals(2, stream.getSplitSize(16 * 1024 * 1024));
+    Assert.assertEquals(2, stream.getSplitSize(12 * 1024 * 1024));
+    Assert.assertEquals(2, stream.getSplitSize(102 * 1024 * 1024));
+    Assert.assertEquals(1, stream.getSplitSize(102));
+    stream.close();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9e71382/hadoop-tools/hadoop-azure-datalake/src/test/resources/adls.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/resources/adls.xml b/hadoop-tools/hadoop-azure-datalake/src/test/resources/adls.xml
new file mode 100644
index 0000000..f72de16
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/resources/adls.xml
@@ -0,0 +1,139 @@
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+<configuration>
+  <property>
+    <name>fs.contract.test.root-tests-enabled</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.test.supports-concat</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.rename-returns-false-if-source-missing</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.test.random-seek-count</name>
+    <value>10</value>
+  </property>
+
+  <property>
+    <name>fs.contract.is-case-sensitive</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.rename-returns-true-if-dest-exists</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.rename-returns-true-if-source-missing</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.rename-creates-dest-dirs</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.rename-remove-dest-if-empty-dir</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.supports-settimes</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.supports-append</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.supports-atomic-directory-delete</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.supports-atomic-rename</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.supports-block-locality</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.supports-concat</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.supports-seek</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.supports-seek-on-closed-file</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.rejects-seek-past-eof</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.supports-available-on-closed-file</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.supports-strict-exceptions</name>
+    <value>false</value>
+  </property>
+
+  <property>
+    <name>fs.contract.supports-unix-permissions</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.rename-overwrites-dest</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.supports-append</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.azure.enable.append.support</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.supports-getfilestatus</name>
+    <value>true</value>
+  </property>
+</configuration>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9e71382/hadoop-tools/hadoop-azure-datalake/src/test/resources/contract-test-options.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/resources/contract-test-options.xml
b/hadoop-tools/hadoop-azure-datalake/src/test/resources/contract-test-options.xml
new file mode 100644
index 0000000..160fba4
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/resources/contract-test-options.xml
@@ -0,0 +1,57 @@
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+<configuration>
+  <property>
+    <name>dfs.webhdfs.oauth2.refresh.token.expires.ms.since.epoch</name>
+    <value>0</value>
+  </property>
+  <property>
+    <name>dfs.webhdfs.oauth2.credential</name>
+    <value>bearer.and.refresh.token</value>
+  </property>
+  <property>
+    <name>dfs.webhdfs.oauth2.refresh.url</name>
+    <value>https://login.windows.net/common/oauth2/token/</value>
+  </property>
+  <property>
+    <name>dfs.webhdfs.oauth2.access.token.provider</name>
+    <value>
+      org.apache.hadoop.fs.adl.oauth2.CachedRefreshTokenBasedAccessTokenProvider
+    </value>
+  </property>
+  <property>
+    <name>dfs.webhdfs.oauth2.enabled</name>
+    <value>true</value>
+  </property>
+  <!--USER INPUT REQUIRED-->
+  <property>
+    <name>dfs.webhdfs.oauth2.client.id</name>
+    <value>ADD CLIENT ID</value>
+  </property>
+  <!--USER INPUT REQUIRED-->
+  <property>
+    <name>dfs.webhdfs.oauth2.refresh.token</name>
+    <value>ADD REFRESH TOKEN</value>
+  </property>
+  <!--USER INPUT REQUIRED-->
+  <property>
+    <name>fs.defaultFS</name>
+    <value>adl://urAdlAccountGoesHere.azuredatalakestore.net:443/</value>
+  </property>
+  <!--USER INPUT REQUIRED-->
+  <property>
+    <name>dfs.adl.test.contract.enable</name>
+    <value>false</value>
+  </property>
+</configuration>
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message