From common-commits-return-88442-archive-asf-public=cust-asf.ponee.io@hadoop.apache.org Sun Sep 23 05:24:08 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id AF8AB180677 for ; Sun, 23 Sep 2018 05:24:06 +0200 (CEST) Received: (qmail 99468 invoked by uid 500); 23 Sep 2018 03:24:03 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 96138 invoked by uid 99); 23 Sep 2018 03:24:01 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 23 Sep 2018 03:24:01 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 98386E0514; Sun, 23 Sep 2018 03:23:59 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit From: mackrorysd@apache.org To: common-commits@hadoop.apache.org Date: Sun, 23 Sep 2018 03:24:04 -0000 Message-Id: In-Reply-To: <1537a6731926494e80f1b0e7a58d1509@git.apache.org> References: <1537a6731926494e80f1b0e7a58d1509@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [07/45] hadoop git commit: HADOOP-15407. HADOOP-15540. Support Windows Azure Storage - Blob file system "ABFS" in Hadoop: Core Commit. http://git-wip-us.apache.org/repos/asf/hadoop/blob/f044deed/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java new file mode 100644 index 0000000..6059766 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java @@ -0,0 +1,132 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs; + +import java.io.FileNotFoundException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import org.junit.Assert; +import org.junit.Test; + +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; + +/** + * Test listStatus operation. + */ +public class ITestAzureBlobFileSystemListStatus extends DependencyInjectedTest { + private static final int TEST_FILES_NUMBER = 6000; + public ITestAzureBlobFileSystemListStatus() { + super(); + } + + @Test + public void testListPath() throws Exception { + final AzureBlobFileSystem fs = this.getFileSystem(); + final List tasks = new ArrayList<>(); + + ExecutorService es = Executors.newFixedThreadPool(10); + for (int i = 0; i < TEST_FILES_NUMBER; i++) { + final Path fileName = new Path("/test" + i); + Callable callable = new Callable() { + @Override + public Void call() throws Exception { + fs.create(fileName); + return null; + } + }; + + tasks.add(es.submit(callable)); + } + + for (Future task : tasks) { + task.get(); + } + + es.shutdownNow(); + FileStatus[] files = fs.listStatus(new Path("/")); + Assert.assertEquals(files.length, TEST_FILES_NUMBER + 1 /* user directory */); + } + + @Test + public void testListFileVsListDir() throws Exception { + final AzureBlobFileSystem fs = this.getFileSystem(); + fs.create(new Path("/testFile")); + + FileStatus[] testFiles = fs.listStatus(new Path("/testFile")); + Assert.assertEquals(testFiles.length, 1); + Assert.assertFalse(testFiles[0].isDirectory()); + } + + @Test + public void testListFileVsListDir2() throws Exception { + final AzureBlobFileSystem fs = this.getFileSystem(); + fs.mkdirs(new Path("/testFolder")); + fs.mkdirs(new Path("/testFolder/testFolder2")); + fs.mkdirs(new Path("/testFolder/testFolder2/testFolder3")); + fs.create(new Path("/testFolder/testFolder2/testFolder3/testFile")); + + FileStatus[] testFiles = fs.listStatus(new Path("/testFolder/testFolder2/testFolder3/testFile")); + Assert.assertEquals(testFiles.length, 1); + Assert.assertEquals(testFiles[0].getPath(), new Path(this.getTestUrl(), + "/testFolder/testFolder2/testFolder3/testFile")); + Assert.assertFalse(testFiles[0].isDirectory()); + } + + @Test(expected = FileNotFoundException.class) + public void testListNonExistentDir() throws Exception { + final AzureBlobFileSystem fs = this.getFileSystem(); + fs.listStatus(new Path("/testFile/")); + } + + @Test + public void testListFiles() throws Exception { + final AzureBlobFileSystem fs = this.getFileSystem(); + fs.mkdirs(new Path("/test")); + + FileStatus[] fileStatuses = fs.listStatus(new Path("/")); + assertEquals(fileStatuses.length, 2); + + fs.mkdirs(new Path("/test/sub")); + fileStatuses = fs.listStatus(new Path("/test")); + assertEquals(fileStatuses.length, 1); + assertEquals(fileStatuses[0].getPath().getName(), "sub"); + assertTrue(fileStatuses[0].isDirectory()); + assertEquals(fileStatuses[0].getLen(), 0); + + fs.create(new Path("/test/f")); + fileStatuses = fs.listStatus(new Path("/test")); + assertEquals(fileStatuses.length, 2); + assertEquals(fileStatuses[0].getPath().getName(), "f"); + assertFalse(fileStatuses[0].isDirectory()); + assertEquals(fileStatuses[0].getLen(), 0); + assertEquals(fileStatuses[1].getPath().getName(), "sub"); + assertTrue(fileStatuses[1].isDirectory()); + assertEquals(fileStatuses[1].getLen(), 0); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/f044deed/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemMkDir.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemMkDir.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemMkDir.java new file mode 100644 index 0000000..b61908c --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemMkDir.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs; + +import java.util.concurrent.Callable; + +import org.junit.Test; + +import org.apache.hadoop.fs.FileAlreadyExistsException; +import org.apache.hadoop.fs.Path; + +import static org.apache.hadoop.test.LambdaTestUtils.intercept; +import static org.junit.Assert.assertTrue; + +/** + * Test mkdir operation. + */ +public class ITestAzureBlobFileSystemMkDir extends DependencyInjectedTest { + public ITestAzureBlobFileSystemMkDir() { + super(); + } + + @Test + public void testCreateDirWithExistingDir() throws Exception { + final AzureBlobFileSystem fs = this.getFileSystem(); + assertTrue(fs.mkdirs(new Path("testFolder"))); + assertTrue(fs.mkdirs(new Path("testFolder"))); + } + + @Test(expected = FileAlreadyExistsException.class) + public void createDirectoryUnderFile() throws Exception { + final AzureBlobFileSystem fs = this.getFileSystem(); + fs.create(new Path("testFile")); + fs.mkdirs(new Path("testFile/TestDirectory")); + } + + @Test + public void testCreateDirectoryOverExistingFiles() throws Exception { + final AzureBlobFileSystem fs = this.getFileSystem(); + fs.create(new Path("/testPath")); + FileAlreadyExistsException ex = intercept( + FileAlreadyExistsException.class, + new Callable() { + @Override + public Void call() throws Exception { + fs.mkdirs(new Path("/testPath")); + return null; + } + }); + + assertTrue(ex instanceof FileAlreadyExistsException); + + fs.create(new Path("/testPath1/file1")); + ex = intercept( + FileAlreadyExistsException.class, + new Callable() { + @Override + public Void call() throws Exception { + fs.mkdirs(new Path("/testPath1/file1")); + return null; + } + }); + + assertTrue(ex instanceof FileAlreadyExistsException); + } + + @Test + public void testCreateRoot() throws Exception { + final AzureBlobFileSystem fs = this.getFileSystem(); + assertTrue(fs.mkdirs(new Path("/"))); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/f044deed/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOpen.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOpen.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOpen.java new file mode 100644 index 0000000..fef7f47 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOpen.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs; + +import java.io.FileNotFoundException; + +import org.junit.Test; + +import org.apache.hadoop.fs.Path; + +/** + * Test open operation. + */ +public class ITestAzureBlobFileSystemOpen extends DependencyInjectedTest { + public ITestAzureBlobFileSystemOpen() throws Exception { + super(); + } + + @Test(expected = FileNotFoundException.class) + public void testOpenDirectory() throws Exception { + final AzureBlobFileSystem fs = this.getFileSystem(); + fs.mkdirs(new Path("testFolder")); + fs.open(new Path("testFolder")); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/f044deed/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java new file mode 100644 index 0000000..9477587 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java @@ -0,0 +1,582 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.azurebfs; + + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FSExceptionMessages; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.azure.NativeAzureFileSystem; +import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.junit.Test; + +import java.io.EOFException; +import java.io.IOException; +import java.util.Random; +import java.util.concurrent.Callable; + +import static org.apache.hadoop.test.LambdaTestUtils.intercept; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertArrayEquals; + + +/** + * Test random read operation. + */ +public class ITestAzureBlobFileSystemRandomRead extends DependencyInjectedTest { + private static final int KILOBYTE = 1024; + private static final int MEGABYTE = KILOBYTE * KILOBYTE; + private static final long TEST_FILE_SIZE = 8 * MEGABYTE; + private static final int MAX_ELAPSEDTIMEMS = 20; + private static final int SEQUENTIAL_READ_BUFFER_SIZE = 16 * KILOBYTE; + private static final int CREATE_BUFFER_SIZE = 26 * KILOBYTE; + + private static final int SEEK_POSITION_ONE = 2* KILOBYTE; + private static final int SEEK_POSITION_TWO = 5 * KILOBYTE; + private static final int SEEK_POSITION_THREE = 10 * KILOBYTE; + private static final int SEEK_POSITION_FOUR = 4100 * KILOBYTE; + + private static final Path TEST_FILE_PATH = new Path( + "/TestRandomRead.txt"); + private static final String WASB = "WASB"; + private static final String ABFS = "ABFS"; + private static long testFileLength = 0; + + public ITestAzureBlobFileSystemRandomRead() throws Exception { + super(); + } + + @Test + public void testBasicRead() throws Exception { + assumeHugeFileExists(); + + try (FSDataInputStream inputStream = this.getFileSystem().open(TEST_FILE_PATH)) { + byte[] buffer = new byte[3 * MEGABYTE]; + + // forward seek and read a kilobyte into first kilobyte of bufferV2 + inputStream.seek(5 * MEGABYTE); + int numBytesRead = inputStream.read(buffer, 0, KILOBYTE); + assertEquals(KILOBYTE, numBytesRead); + + int len = MEGABYTE; + int offset = buffer.length - len; + + // reverse seek and read a megabyte into last megabyte of bufferV1 + inputStream.seek(3 * MEGABYTE); + numBytesRead = inputStream.read(buffer, offset, len); + assertEquals(len, numBytesRead); + } + } + + /** + * Validates the implementation of random read in ABFS + * @throws IOException + */ + @Test + public void testRandomRead() throws Exception { + assumeHugeFileExists(); + try ( + FSDataInputStream inputStreamV1 + = this.getFileSystem().open(TEST_FILE_PATH); + FSDataInputStream inputStreamV2 + = this.getWasbFileSystem().open(TEST_FILE_PATH); + ) { + final int bufferSize = 4 * KILOBYTE; + byte[] bufferV1 = new byte[bufferSize]; + byte[] bufferV2 = new byte[bufferV1.length]; + + verifyConsistentReads(inputStreamV1, inputStreamV2, bufferV1, bufferV2); + + inputStreamV1.seek(0); + inputStreamV2.seek(0); + + verifyConsistentReads(inputStreamV1, inputStreamV2, bufferV1, bufferV2); + + verifyConsistentReads(inputStreamV1, inputStreamV2, bufferV1, bufferV2); + + inputStreamV1.seek(SEEK_POSITION_ONE); + inputStreamV2.seek(SEEK_POSITION_ONE); + + inputStreamV1.seek(0); + inputStreamV2.seek(0); + + verifyConsistentReads(inputStreamV1, inputStreamV2, bufferV1, bufferV2); + + verifyConsistentReads(inputStreamV1, inputStreamV2, bufferV1, bufferV2); + + verifyConsistentReads(inputStreamV1, inputStreamV2, bufferV1, bufferV2); + + inputStreamV1.seek(SEEK_POSITION_TWO); + inputStreamV2.seek(SEEK_POSITION_TWO); + + verifyConsistentReads(inputStreamV1, inputStreamV2, bufferV1, bufferV2); + + inputStreamV1.seek(SEEK_POSITION_THREE); + inputStreamV2.seek(SEEK_POSITION_THREE); + + verifyConsistentReads(inputStreamV1, inputStreamV2, bufferV1, bufferV2); + + verifyConsistentReads(inputStreamV1, inputStreamV2, bufferV1, bufferV2); + + inputStreamV1.seek(SEEK_POSITION_FOUR); + inputStreamV2.seek(SEEK_POSITION_FOUR); + + verifyConsistentReads(inputStreamV1, inputStreamV2, bufferV1, bufferV2); + } + } + + /** + * Validates the implementation of Seekable.seekToNewSource + * @throws IOException + */ + @Test + public void testSeekToNewSource() throws Exception { + assumeHugeFileExists(); + try (FSDataInputStream inputStream = this.getFileSystem().open(TEST_FILE_PATH)) { + assertFalse(inputStream.seekToNewSource(0)); + } + } + + /** + * Validates the implementation of InputStream.skip and ensures there is no + * network I/O for AbfsInputStream + * @throws Exception + */ + @Test + public void testSkipBounds() throws Exception { + assumeHugeFileExists(); + try (FSDataInputStream inputStream = this.getFileSystem().open(TEST_FILE_PATH)) { + ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer(); + + long skipped = inputStream.skip(-1); + assertEquals(0, skipped); + + skipped = inputStream.skip(0); + assertEquals(0, skipped); + + assertTrue(testFileLength > 0); + + skipped = inputStream.skip(testFileLength); + assertEquals(testFileLength, skipped); + + intercept(EOFException.class, + new Callable() { + @Override + public Long call() throws Exception { + return inputStream.skip(1); + } + } + ); + long elapsedTimeMs = timer.elapsedTimeMs(); + assertTrue( + String.format( + "There should not be any network I/O (elapsedTimeMs=%1$d).", + elapsedTimeMs), + elapsedTimeMs < MAX_ELAPSEDTIMEMS); + } + } + + /** + * Validates the implementation of Seekable.seek and ensures there is no + * network I/O for forward seek. + * @throws Exception + */ + @Test + public void testValidateSeekBounds() throws Exception { + assumeHugeFileExists(); + try (FSDataInputStream inputStream = this.getFileSystem().open(TEST_FILE_PATH)) { + ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer(); + + inputStream.seek(0); + assertEquals(0, inputStream.getPos()); + + intercept(EOFException.class, + FSExceptionMessages.NEGATIVE_SEEK, + new Callable() { + @Override + public FSDataInputStream call() throws Exception { + inputStream.seek(-1); + return inputStream; + } + } + ); + + assertTrue("Test file length only " + testFileLength, testFileLength > 0); + inputStream.seek(testFileLength); + assertEquals(testFileLength, inputStream.getPos()); + + intercept(EOFException.class, + FSExceptionMessages.CANNOT_SEEK_PAST_EOF, + new Callable() { + @Override + public FSDataInputStream call() throws Exception { + inputStream.seek(testFileLength + 1); + return inputStream; + } + } + ); + + long elapsedTimeMs = timer.elapsedTimeMs(); + assertTrue( + String.format( + "There should not be any network I/O (elapsedTimeMs=%1$d).", + elapsedTimeMs), + elapsedTimeMs < MAX_ELAPSEDTIMEMS); + } + } + + /** + * Validates the implementation of Seekable.seek, Seekable.getPos, + * and InputStream.available. + * @throws Exception + */ + @Test + public void testSeekAndAvailableAndPosition() throws Exception { + assumeHugeFileExists(); + try (FSDataInputStream inputStream = this.getFileSystem().open(TEST_FILE_PATH)) { + byte[] expected1 = {(byte) 'a', (byte) 'b', (byte) 'c'}; + byte[] expected2 = {(byte) 'd', (byte) 'e', (byte) 'f'}; + byte[] expected3 = {(byte) 'b', (byte) 'c', (byte) 'd'}; + byte[] expected4 = {(byte) 'g', (byte) 'h', (byte) 'i'}; + byte[] buffer = new byte[3]; + + int bytesRead = inputStream.read(buffer); + assertEquals(buffer.length, bytesRead); + assertArrayEquals(expected1, buffer); + assertEquals(buffer.length, inputStream.getPos()); + assertEquals(testFileLength - inputStream.getPos(), + inputStream.available()); + + bytesRead = inputStream.read(buffer); + assertEquals(buffer.length, bytesRead); + assertArrayEquals(expected2, buffer); + assertEquals(2 * buffer.length, inputStream.getPos()); + assertEquals(testFileLength - inputStream.getPos(), + inputStream.available()); + + // reverse seek + int seekPos = 0; + inputStream.seek(seekPos); + + bytesRead = inputStream.read(buffer); + assertEquals(buffer.length, bytesRead); + assertArrayEquals(expected1, buffer); + assertEquals(buffer.length + seekPos, inputStream.getPos()); + assertEquals(testFileLength - inputStream.getPos(), + inputStream.available()); + + // reverse seek + seekPos = 1; + inputStream.seek(seekPos); + + bytesRead = inputStream.read(buffer); + assertEquals(buffer.length, bytesRead); + assertArrayEquals(expected3, buffer); + assertEquals(buffer.length + seekPos, inputStream.getPos()); + assertEquals(testFileLength - inputStream.getPos(), + inputStream.available()); + + // forward seek + seekPos = 6; + inputStream.seek(seekPos); + + bytesRead = inputStream.read(buffer); + assertEquals(buffer.length, bytesRead); + assertArrayEquals(expected4, buffer); + assertEquals(buffer.length + seekPos, inputStream.getPos()); + assertEquals(testFileLength - inputStream.getPos(), + inputStream.available()); + } + } + + /** + * Validates the implementation of InputStream.skip, Seekable.getPos, + * and InputStream.available. + * @throws IOException + */ + @Test + public void testSkipAndAvailableAndPosition() throws Exception { + assumeHugeFileExists(); + try (FSDataInputStream inputStream = this.getFileSystem().open(TEST_FILE_PATH)) { + byte[] expected1 = {(byte) 'a', (byte) 'b', (byte) 'c'}; + byte[] expected2 = {(byte) 'd', (byte) 'e', (byte) 'f'}; + byte[] expected3 = {(byte) 'b', (byte) 'c', (byte) 'd'}; + byte[] expected4 = {(byte) 'g', (byte) 'h', (byte) 'i'}; + + assertEquals(testFileLength, inputStream.available()); + assertEquals(0, inputStream.getPos()); + + int n = 3; + long skipped = inputStream.skip(n); + + assertEquals(skipped, inputStream.getPos()); + assertEquals(testFileLength - inputStream.getPos(), + inputStream.available()); + assertEquals(skipped, n); + + byte[] buffer = new byte[3]; + int bytesRead = inputStream.read(buffer); + assertEquals(buffer.length, bytesRead); + assertArrayEquals(expected2, buffer); + assertEquals(buffer.length + skipped, inputStream.getPos()); + assertEquals(testFileLength - inputStream.getPos(), + inputStream.available()); + + // does skip still work after seek? + int seekPos = 1; + inputStream.seek(seekPos); + + bytesRead = inputStream.read(buffer); + assertEquals(buffer.length, bytesRead); + assertArrayEquals(expected3, buffer); + assertEquals(buffer.length + seekPos, inputStream.getPos()); + assertEquals(testFileLength - inputStream.getPos(), + inputStream.available()); + + long currentPosition = inputStream.getPos(); + n = 2; + skipped = inputStream.skip(n); + + assertEquals(currentPosition + skipped, inputStream.getPos()); + assertEquals(testFileLength - inputStream.getPos(), + inputStream.available()); + assertEquals(skipped, n); + + bytesRead = inputStream.read(buffer); + assertEquals(buffer.length, bytesRead); + assertArrayEquals(expected4, buffer); + assertEquals(buffer.length + skipped + currentPosition, + inputStream.getPos()); + assertEquals(testFileLength - inputStream.getPos(), + inputStream.available()); + } + } + + /** + * Ensures parity in the performance of sequential read after reverse seek for + * abfs of the AbfsInputStream. + * @throws IOException + */ + @Test + public void testSequentialReadAfterReverseSeekPerformance() + throws Exception { + assumeHugeFileExists(); + final int maxAttempts = 10; + final double maxAcceptableRatio = 1.01; + double beforeSeekElapsedMs = 0, afterSeekElapsedMs = 0; + double ratio = Double.MAX_VALUE; + for (int i = 0; i < maxAttempts && ratio >= maxAcceptableRatio; i++) { + beforeSeekElapsedMs = sequentialRead(ABFS, + this.getFileSystem(), false); + afterSeekElapsedMs = sequentialRead(ABFS, + this.getFileSystem(), true); + ratio = afterSeekElapsedMs / beforeSeekElapsedMs; + System.out.println((String.format( + "beforeSeekElapsedMs=%1$d, afterSeekElapsedMs=%2$d, ratio=%3$.2f", + (long) beforeSeekElapsedMs, + (long) afterSeekElapsedMs, + ratio))); + } + assertTrue(String.format( + "Performance of ABFS stream after reverse seek is not acceptable:" + + " beforeSeekElapsedMs=%1$d, afterSeekElapsedMs=%2$d," + + " ratio=%3$.2f", + (long) beforeSeekElapsedMs, + (long) afterSeekElapsedMs, + ratio), + ratio < maxAcceptableRatio); + } + + @Test + public void testRandomReadPerformance() throws Exception { + createTestFile(); + assumeHugeFileExists(); + + final AzureBlobFileSystem abFs = this.getFileSystem(); + final NativeAzureFileSystem wasbFs = this.getWasbFileSystem(); + + final int maxAttempts = 10; + final double maxAcceptableRatio = 1.025; + double v1ElapsedMs = 0, v2ElapsedMs = 0; + double ratio = Double.MAX_VALUE; + for (int i = 0; i < maxAttempts && ratio >= maxAcceptableRatio; i++) { + v1ElapsedMs = randomRead(1, wasbFs); + v2ElapsedMs = randomRead(2, abFs); + + ratio = v2ElapsedMs / v1ElapsedMs; + + System.out.println(String.format( + "v1ElapsedMs=%1$d, v2ElapsedMs=%2$d, ratio=%3$.2f", + (long) v1ElapsedMs, + (long) v2ElapsedMs, + ratio)); + } + assertTrue(String.format( + "Performance of version 2 is not acceptable: v1ElapsedMs=%1$d," + + " v2ElapsedMs=%2$d, ratio=%3$.2f", + (long) v1ElapsedMs, + (long) v2ElapsedMs, + ratio), + ratio < maxAcceptableRatio); + } + + + private long sequentialRead(String version, + FileSystem fs, + boolean afterReverseSeek) throws IOException { + byte[] buffer = new byte[SEQUENTIAL_READ_BUFFER_SIZE]; + long totalBytesRead = 0; + long bytesRead = 0; + + try(FSDataInputStream inputStream = fs.open(TEST_FILE_PATH)) { + if (afterReverseSeek) { + while (bytesRead > 0 && totalBytesRead < 4 * MEGABYTE) { + bytesRead = inputStream.read(buffer); + totalBytesRead += bytesRead; + } + totalBytesRead = 0; + inputStream.seek(0); + } + + ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer(); + while ((bytesRead = inputStream.read(buffer)) > 0) { + totalBytesRead += bytesRead; + } + long elapsedTimeMs = timer.elapsedTimeMs(); + + System.out.println(String.format( + "v%1$s: bytesRead=%2$d, elapsedMs=%3$d, Mbps=%4$.2f," + + " afterReverseSeek=%5$s", + version, + totalBytesRead, + elapsedTimeMs, + toMbps(totalBytesRead, elapsedTimeMs), + afterReverseSeek)); + + assertEquals(testFileLength, totalBytesRead); + inputStream.close(); + return elapsedTimeMs; + } + } + + private long randomRead(int version, FileSystem fs) throws Exception { + assumeHugeFileExists(); + final long minBytesToRead = 2 * MEGABYTE; + Random random = new Random(); + byte[] buffer = new byte[8 * KILOBYTE]; + long totalBytesRead = 0; + long bytesRead = 0; + try(FSDataInputStream inputStream = fs.open(TEST_FILE_PATH)) { + ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer(); + do { + bytesRead = inputStream.read(buffer); + totalBytesRead += bytesRead; + inputStream.seek(random.nextInt( + (int) (TEST_FILE_SIZE - buffer.length))); + } while (bytesRead > 0 && totalBytesRead < minBytesToRead); + long elapsedTimeMs = timer.elapsedTimeMs(); + inputStream.close(); + System.out.println(String.format( + "v%1$d: totalBytesRead=%2$d, elapsedTimeMs=%3$d, Mbps=%4$.2f", + version, + totalBytesRead, + elapsedTimeMs, + toMbps(totalBytesRead, elapsedTimeMs))); + assertTrue(minBytesToRead <= totalBytesRead); + return elapsedTimeMs; + } + } + + /** + * Calculate megabits per second from the specified values for bytes and + * milliseconds. + * @param bytes The number of bytes. + * @param milliseconds The number of milliseconds. + * @return The number of megabits per second. + */ + private static double toMbps(long bytes, long milliseconds) { + return bytes / 1000.0 * 8 / milliseconds; + } + + private void createTestFile() throws Exception { + FileSystem fs = this.getWasbFileSystem(); + + if (fs.exists(TEST_FILE_PATH)) { + FileStatus status = fs.getFileStatus(TEST_FILE_PATH); + if (status.getLen() >= TEST_FILE_SIZE) { + return; + } + } + + byte[] buffer = new byte[CREATE_BUFFER_SIZE]; + char character = 'a'; + for (int i = 0; i < buffer.length; i++) { + buffer[i] = (byte) character; + character = (character == 'z') ? 'a' : (char) ((int) character + 1); + } + + System.out.println(("Creating test file {} of size: {} " + TEST_FILE_PATH + + TEST_FILE_SIZE)); + ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer(); + + try(FSDataOutputStream outputStream = fs.create(TEST_FILE_PATH)) { + int bytesWritten = 0; + while (bytesWritten < TEST_FILE_SIZE) { + outputStream.write(buffer); + bytesWritten += buffer.length; + } + System.out.println("Closing stream {}" + outputStream); + ContractTestUtils.NanoTimer closeTimer + = new ContractTestUtils.NanoTimer(); + outputStream.close(); + closeTimer.end("time to close() output stream"); + } + timer.end("time to write %d KB", TEST_FILE_SIZE / 1024); + testFileLength = fs.getFileStatus(TEST_FILE_PATH).getLen(); + + } + + private void assumeHugeFileExists() throws Exception{ + createTestFile(); + FileSystem fs = this.getFileSystem(); + ContractTestUtils.assertPathExists(this.getFileSystem(), "huge file not created", TEST_FILE_PATH); + FileStatus status = fs.getFileStatus(TEST_FILE_PATH); + ContractTestUtils.assertIsFile(TEST_FILE_PATH, status); + assertTrue("File " + TEST_FILE_PATH + " is empty", status.getLen() > 0); + } + + private void verifyConsistentReads(FSDataInputStream inputStreamV1, + FSDataInputStream inputStreamV2, + byte[] bufferV1, + byte[] bufferV2) throws IOException { + int size = bufferV1.length; + final int numBytesReadV1 = inputStreamV1.read(bufferV1, 0, size); + assertEquals("Bytes read from wasb stream", size, numBytesReadV1); + + final int numBytesReadV2 = inputStreamV2.read(bufferV2, 0, size); + assertEquals("Bytes read from abfs stream", size, numBytesReadV2); + + assertArrayEquals("Mismatch in read data", bufferV1, bufferV2); + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/f044deed/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java new file mode 100644 index 0000000..a0e648c --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java @@ -0,0 +1,152 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs; + +import java.io.FileNotFoundException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import org.junit.Assert; +import org.junit.Test; + +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +/** + * Test rename operation. + */ +public class ITestAzureBlobFileSystemRename extends DependencyInjectedTest { + public ITestAzureBlobFileSystemRename() { + super(); + } + + @Test(expected = FileNotFoundException.class) + public void testEnsureFileIsRenamed() throws Exception { + final AzureBlobFileSystem fs = this.getFileSystem(); + fs.create(new Path("testfile")); + fs.rename(new Path("testfile"), new Path("testfile2")); + + FileStatus fileStatus = fs.getFileStatus(new Path("testfile2")); + assertNotNull(fileStatus); + + fs.getFileStatus(new Path("testfile")); + } + + @Test + public void testRenameFile() throws Exception { + final AzureBlobFileSystem fs = this.getFileSystem(); + fs.mkdirs(new Path("/testSrc")); + fs.create(new Path("/testSrc/file1")); + + fs.rename(new Path("/testSrc"), new Path("/testDst")); + FileStatus[] fileStatus = fs.listStatus(new Path("/testDst")); + assertNotNull(fileStatus); + } + + @Test + public void testRenameFileUsingUnicode() throws Exception { + final AzureBlobFileSystem fs = this.getFileSystem(); + //known issue: ListStatus operation to folders/files whose name contains '?' will fail + //This is because Auto rest client didn't encode '?' in the uri query parameters + String[] folders1 = new String[]{"/%2c%26", "/ÖáΠ⇒", "/A +B", "/A~`!@#$%^&*()-_+={};:'>,,, tasks = new ArrayList<>(); + + ExecutorService es = Executors.newFixedThreadPool(10); + for (int i = 0; i < 1000; i++) { + final Path fileName = new Path("/test/" + i); + Callable callable = new Callable() { + @Override + public Void call() throws Exception { + fs.create(fileName); + return null; + } + }; + + tasks.add(es.submit(callable)); + } + + for (Future task : tasks) { + task.get(); + } + + es.shutdownNow(); + fs.rename(new Path("/test"), new Path("/renamedDir")); + + FileStatus[] files = fs.listStatus(new Path("/renamedDir")); + Assert.assertEquals(files.length, 1000); + fs.getFileStatus(new Path("/test")); + } + + @Test + public void testRenameRoot() throws Exception { + final AzureBlobFileSystem fs = this.getFileSystem(); + boolean renamed = fs.rename(new Path("/"), new Path("/ddd")); + assertFalse(renamed); + + renamed = fs.rename(new Path(fs.getUri().toString() + "/"), new Path(fs.getUri().toString() + "/s")); + assertFalse(renamed); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/f044deed/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemInitialization.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemInitialization.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemInitialization.java new file mode 100644 index 0000000..aa30a85 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemInitialization.java @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs; + +import java.net.URI; + +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes; +import org.apache.hadoop.fs.azurebfs.contracts.services.AbfsHttpService; +import org.apache.hadoop.fs.azurebfs.services.AbfsServiceProviderImpl; + +import static org.mockito.Matchers.anyObject; +import static org.mockito.Mockito.doReturn; + +/** + * Test AzureBlobFileSystem initialization. + */ +public class ITestFileSystemInitialization extends DependencyInjectedTest { + public ITestFileSystemInitialization() { + super(); + + this.getMockServiceInjector().removeProvider(AbfsHttpService.class); + this.getMockServiceInjector().replaceInstance(AbfsHttpService.class, Mockito.mock(AbfsHttpService.class)); + } + + @Test + public void ensureAzureBlobFileSystemIsInitialized() throws Exception { + doReturn(new FileStatus(0, true, 0, 0, 0, new Path("/blah"))) + .when(AbfsServiceProviderImpl.instance().get(AbfsHttpService.class)) + .getFileStatus((AzureBlobFileSystem) anyObject(), (Path) anyObject()); + + final FileSystem fs = FileSystem.get(this.getConfiguration()); + final String accountName = this.getAccountName(); + final String filesystem = this.getFileSystemName(); + + Assert.assertEquals(fs.getUri(), new URI(FileSystemUriSchemes.ABFS_SCHEME, filesystem + "@" + accountName, null, null, null)); + Assert.assertNotNull(fs.getWorkingDirectory()); + } + + @Test + public void ensureSecureAzureBlobFileSystemIsInitialized() throws Exception { + doReturn(new FileStatus(0, true, 0, 0, 0, new Path("/blah"))) + .when(AbfsServiceProviderImpl.instance().get(AbfsHttpService.class)) + .getFileStatus((AzureBlobFileSystem) anyObject(), (Path) anyObject()); + + final String accountName = this.getAccountName(); + final String filesystem = this.getFileSystemName(); + final URI defaultUri = new URI(FileSystemUriSchemes.ABFS_SECURE_SCHEME, filesystem + "@" + accountName, null, null, null); + this.getConfiguration().set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri.toString()); + + final FileSystem fs = FileSystem.get(this.getConfiguration()); + Assert.assertEquals(fs.getUri(), new URI(FileSystemUriSchemes.ABFS_SECURE_SCHEME, filesystem + "@" + accountName, null, null, null)); + Assert.assertNotNull(fs.getWorkingDirectory()); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/f044deed/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemRegistration.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemRegistration.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemRegistration.java new file mode 100644 index 0000000..a55599b --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemRegistration.java @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs; + +import java.net.URI; + +import org.apache.hadoop.fs.azurebfs.services.AbfsServiceProviderImpl; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +import org.apache.hadoop.fs.AbstractFileSystem; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.FileContext; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes; +import org.apache.hadoop.fs.azurebfs.contracts.services.AbfsHttpService; + +import static org.mockito.Matchers.anyObject; +import static org.mockito.Mockito.doReturn; + +/** + * Test AzureBlobFileSystem registration. + */ +public class ITestFileSystemRegistration extends DependencyInjectedTest { + public ITestFileSystemRegistration() throws Exception { + super(); + + this.getMockServiceInjector().removeProvider(AbfsHttpService.class); + this.getMockServiceInjector().replaceInstance(AbfsHttpService.class, Mockito.mock(AbfsHttpService.class)); + } + + @Test + public void ensureAzureBlobFileSystemIsDefaultFileSystem() throws Exception { + doReturn(new FileStatus(0, true, 0, 0, 0, new Path("/blah"))) + .when(AbfsServiceProviderImpl.instance().get(AbfsHttpService.class)) + .getFileStatus((AzureBlobFileSystem) anyObject(), (Path) anyObject()); + + FileSystem fs = FileSystem.get(this.getConfiguration()); + Assert.assertTrue(fs instanceof AzureBlobFileSystem); + + AbstractFileSystem afs = FileContext.getFileContext(this.getConfiguration()).getDefaultFileSystem(); + Assert.assertTrue(afs instanceof Abfs); + } + + @Test + public void ensureSecureAzureBlobFileSystemIsDefaultFileSystem() throws Exception { + doReturn(new FileStatus(0, true, 0, 0, 0, new Path("/blah"))) + .when(AbfsServiceProviderImpl.instance().get(AbfsHttpService.class)) + .getFileStatus((AzureBlobFileSystem) anyObject(), (Path) anyObject()); + + final String accountName = this.getAccountName(); + final String filesystem = this.getFileSystemName(); + + final URI defaultUri = new URI(FileSystemUriSchemes.ABFS_SECURE_SCHEME, filesystem + "@" + accountName, null, null, null); + this.getConfiguration().set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri.toString()); + + FileSystem fs = FileSystem.get(this.getConfiguration()); + Assert.assertTrue(fs instanceof SecureAzureBlobFileSystem); + + AbstractFileSystem afs = FileContext.getFileContext(this.getConfiguration()).getDefaultFileSystem(); + Assert.assertTrue(afs instanceof Abfss); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/f044deed/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java new file mode 100644 index 0000000..7010e74 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java @@ -0,0 +1,202 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.azurebfs; + +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.azure.NativeAzureFileSystem; + +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStreamReader; + +import static junit.framework.TestCase.assertEquals; +import static junit.framework.TestCase.assertFalse; +import static junit.framework.TestCase.assertTrue; + +/** + * Test compatibility between ABFS client and WASB client. + */ +public class ITestWasbAbfsCompatibility extends DependencyInjectedTest { + private static final String WASB_TEST_CONTEXT = "wasb test file"; + private static final String ABFS_TEST_CONTEXT = "abfs test file"; + private static final String TEST_CONTEXT = "THIS IS FOR TEST"; + + public ITestWasbAbfsCompatibility() throws Exception { + super(); + + Assume.assumeFalse(this.isEmulator()); + } + + @Test + public void testListFileStatus() throws Exception { + // crate file using abfs + AzureBlobFileSystem fs = this.getFileSystem(); + NativeAzureFileSystem wasb = this.getWasbFileSystem(); + + Path path1 = new Path("/testfiles/~12/!008/3/abFsTestfile"); + FSDataOutputStream abfsStream = fs.create(path1, true); + abfsStream.write(ABFS_TEST_CONTEXT.getBytes()); + abfsStream.flush(); + abfsStream.hsync(); + abfsStream.close(); + + // create file using wasb + Path path2 = new Path("/testfiles/~12/!008/3/nativeFsTestfile"); + System.out.println(wasb.getUri()); + FSDataOutputStream nativeFsStream = wasb.create(path2, true); + nativeFsStream.write(WASB_TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + nativeFsStream.close(); + // list file using abfs and wasb + FileStatus[] abfsFileStatus = fs.listStatus(new Path("/testfiles/~12/!008/3/")); + FileStatus[] nativeFsFileStatus = wasb.listStatus(new Path("/testfiles/~12/!008/3/")); + + assertEquals(2, abfsFileStatus.length); + assertEquals(2, nativeFsFileStatus.length); + } + + @Test + public void testReadFile() throws Exception { + boolean[] createFileWithAbfs = new boolean[]{false, true, false, true}; + boolean[] readFileWithAbfs = new boolean[]{false, true, true, false}; + + AzureBlobFileSystem abfs = this.getFileSystem(); + NativeAzureFileSystem wasb = this.getWasbFileSystem(); + + FileSystem fs; + BufferedReader br = null; + for (int i = 0; i< 4; i++) { + try { + Path path = new Path("/testfiles/~12/!008/testfile" + i); + if (createFileWithAbfs[i]) { + fs = abfs; + } else { + fs = wasb; + } + + // Write + FSDataOutputStream nativeFsStream = fs.create(path, true); + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + nativeFsStream.close(); + + // Check file status + assertEquals(true, fs.exists(path)); + assertEquals(false, fs.getFileStatus(path).isDirectory()); + + // Read + if (readFileWithAbfs[i]) { + fs = abfs; + } else { + fs = wasb; + } + FSDataInputStream inputStream = fs.open(path); + br = new BufferedReader(new InputStreamReader(fs.open(path))); + String line = br.readLine(); + assertEquals(TEST_CONTEXT, line); + + // Remove file + fs.delete(path, true); + assertFalse(fs.exists(path)); + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (br != null) { + br.close(); + } + } + } + } + + @Test + public void testDir() throws Exception { + boolean[] createDirWithAbfs = new boolean[]{false, true, false, true}; + boolean[] readDirWithAbfs = new boolean[]{false, true, true, false}; + + AzureBlobFileSystem abfs = this.getFileSystem(); + NativeAzureFileSystem wasb = this.getWasbFileSystem(); + + FileSystem fs; + for (int i = 0; i < 4; i++) { + Path path = new Path("/testDir/t" + i); + //create + if (createDirWithAbfs[i]) { + fs = abfs; + } else { + fs = wasb; + } + assertTrue(fs.mkdirs(path)); + //check + assertTrue(fs.exists(path)); + //read + if (readDirWithAbfs[i]) { + fs = abfs; + } else { + fs = wasb; + } + assertTrue(fs.exists(path)); + FileStatus dirStatus = fs.getFileStatus(path); + assertTrue(dirStatus.isDirectory()); + fs.delete(path, true); + assertFalse(fs.exists(path)); + } + } + + + @Test + public void testUrlConversion(){ + String abfsUrl = "abfs://abcde-1111-1111-1111-1111@xxxx.dfs.xxx.xxx.xxxx.xxxx"; + String wabsUrl = "wasb://abcde-1111-1111-1111-1111@xxxx.blob.xxx.xxx.xxxx.xxxx"; + Assert.assertEquals(abfsUrl, wasbUrlToAbfsUrl(wabsUrl)); + Assert.assertEquals(wabsUrl, abfsUrlToWasbUrl(abfsUrl)); + } + + @Test + public void testSetWorkingDirectory() throws Exception { + //create folders + AzureBlobFileSystem abfs = this.getFileSystem(); + NativeAzureFileSystem wasb = this.getWasbFileSystem(); + + assertTrue(abfs.mkdirs(new Path("/d1/d2/d3/d4"))); + + //set working directory to path1 + Path path1 = new Path("/d1/d2"); + wasb.setWorkingDirectory(path1); + abfs.setWorkingDirectory(path1); + assertEquals(path1, wasb.getWorkingDirectory()); + assertEquals(path1, abfs.getWorkingDirectory()); + + //set working directory to path2 + Path path2 = new Path("d3/d4"); + wasb.setWorkingDirectory(path2); + abfs.setWorkingDirectory(path2); + + Path path3 = new Path("/d1/d2/d3/d4"); + assertEquals(path3, wasb.getWorkingDirectory()); + assertEquals(path3, abfs.getWorkingDirectory()); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/f044deed/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java new file mode 100644 index 0000000..4b44765 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs.constants; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * Responsible to keep all the Azure Blob File System configurations keys in Hadoop configuration file. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public final class TestConfigurationKeys { + public static final String FS_AZURE_TEST_ACCOUNT_NAME = "fs.azure.test.account.name"; + public static final String FS_AZURE_TEST_ACCOUNT_KEY_PREFIX = "fs.azure.test.account.key."; + public static final String FS_AZURE_TEST_HOST_NAME = "fs.azure.test.host.name"; + public static final String FS_AZURE_TEST_HOST_PORT = "fs.azure.test.host.port"; + public static final String FS_AZURE_CONTRACT_TEST_URI = "fs.contract.test.fs.abfs"; + + private TestConfigurationKeys() {} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/f044deed/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/package-info.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/package-info.java new file mode 100644 index 0000000..109f887 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +package org.apache.hadoop.fs.azurebfs.constants; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/f044deed/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/DependencyInjectedContractTest.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/DependencyInjectedContractTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/DependencyInjectedContractTest.java new file mode 100644 index 0000000..5fc81ce --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/DependencyInjectedContractTest.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs.contract; + +import java.net.URI; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.azurebfs.DependencyInjectedTest; +import org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes; +import org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys; + +/** + * Dependency inject for ABFS contract tests. + */ +public class DependencyInjectedContractTest extends DependencyInjectedTest { + private final URI testUri; + + public DependencyInjectedContractTest(final boolean secure) throws Exception { + this(secure, true); + } + + public DependencyInjectedContractTest(final boolean secure, final boolean useExistedFileSystem) throws Exception{ + super(secure); + if (useExistedFileSystem) { + Configuration configuration = getConfiguration(); + String testUrl = configuration.get(TestConfigurationKeys.FS_AZURE_CONTRACT_TEST_URI); + + if (secure) { + testUrl = testUrl.replaceFirst(FileSystemUriSchemes.ABFS_SCHEME, FileSystemUriSchemes.ABFS_SECURE_SCHEME); + } + updateTestUrl(testUrl); + + this.testUri = new URI(testUrl); + //Get container for contract tests + configuration.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, this.testUri.toString()); + String[] splitAuthority = this.testUri.getAuthority().split("\\@"); + updateFileSystemName(splitAuthority[0]); + } else { + this.testUri = new URI(super.getTestUrl()); + } + } + + public Configuration getConfiguration() { + return super.getConfiguration(); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/f044deed/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContract.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContract.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContract.java new file mode 100644 index 0000000..7f7a09a --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContract.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes; +import org.apache.hadoop.fs.azurebfs.utils.UriUtils; +import org.apache.hadoop.fs.contract.AbstractBondedFSContract; + +/** + * Azure BlobFileSystem Contract. Test paths are created using any maven fork + * identifier, if defined. This guarantees paths unique to tests + * running in parallel. + */ +public class ITestAbfsFileSystemContract extends AbstractBondedFSContract { + + public static final String CONTRACT_XML = "abfs.xml"; + private final boolean isSecure; + + protected ITestAbfsFileSystemContract(final Configuration conf, boolean secure) { + super(conf); + //insert the base features + addConfResource(CONTRACT_XML); + this.isSecure = secure; + } + + @Override + public String getScheme() { + return isSecure ? FileSystemUriSchemes.ABFS_SECURE_SCHEME : FileSystemUriSchemes.ABFS_SCHEME; + } + + @Override + public Path getTestPath() { + Path path = new Path(UriUtils.generateUniqueTestPath()); + return path; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/f044deed/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractAppend.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractAppend.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractAppend.java new file mode 100644 index 0000000..d4cca14 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractAppend.java @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs.contract; + +import java.util.Arrays; + +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractAppendTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +import static org.apache.hadoop.fs.contract.ContractTestUtils.skip; + +/** + * Contract test for open operation. + */ +@RunWith(Parameterized.class) +public class ITestAbfsFileSystemContractAppend extends AbstractContractAppendTest { + @Parameterized.Parameters(name = "SecureMode={0}") + public static Iterable secure() { + return Arrays.asList(new Object[][] { {true}, {false} }); + } + + private final boolean isSecure; + private final DependencyInjectedContractTest dependencyInjectedContractTest; + + public ITestAbfsFileSystemContractAppend(final boolean secure) throws Exception { + this.isSecure = secure; + dependencyInjectedContractTest = new DependencyInjectedContractTest(this.isSecure); + } + + @Override + public void setup() throws Exception { + dependencyInjectedContractTest.initialize(); + super.setup(); + } + + @Override + protected Configuration createConfiguration() { + return this.dependencyInjectedContractTest.getConfiguration(); + } + + @Override + protected AbstractFSContract createContract(final Configuration conf) { + return new ITestAbfsFileSystemContract(conf, this.isSecure); + } + + @Override + public void testRenameFileBeingAppended() throws Throwable { + skip("Skipping as renaming an opened file is not supported"); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/f044deed/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractConcat.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractConcat.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractConcat.java new file mode 100644 index 0000000..4f724e2 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractConcat.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.azurebfs.contract; + +import java.util.Arrays; + +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractConcatTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * Contract test for concat operation. + */ +@RunWith(Parameterized.class) +public class ITestAbfsFileSystemContractConcat extends AbstractContractConcatTest{ + @Parameterized.Parameters(name = "SecureMode={0}") + public static Iterable secure() { + return Arrays.asList(new Object[][] { {true}, {false} }); + } + + private final boolean isSecure; + private final DependencyInjectedContractTest dependencyInjectedContractTest; + + public ITestAbfsFileSystemContractConcat(final boolean secure) throws Exception { + this.isSecure = secure; + dependencyInjectedContractTest = new DependencyInjectedContractTest(isSecure); + } + + @Override + public void setup() throws Exception { + dependencyInjectedContractTest.initialize(); + super.setup(); + } + + @Override + protected Configuration createConfiguration() { + return this.dependencyInjectedContractTest.getConfiguration(); + } + + @Override + protected AbstractFSContract createContract(final Configuration conf) { + return new ITestAbfsFileSystemContract(conf, this.isSecure); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/f044deed/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractCreate.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractCreate.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractCreate.java new file mode 100644 index 0000000..16b959f --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractCreate.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs.contract; + +import java.util.Arrays; + +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractCreateTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * Contract test for create operation. + */ +@RunWith(Parameterized.class) +public class ITestAbfsFileSystemContractCreate extends AbstractContractCreateTest{ + @Parameterized.Parameters(name = "SecureMode={0}") + public static Iterable secure() { + return Arrays.asList(new Object[][] { {true}, {false} }); + } + + private final boolean isSecure; + private final DependencyInjectedContractTest dependencyInjectedContractTest; + + public ITestAbfsFileSystemContractCreate(final boolean secure) throws Exception { + this.isSecure = secure; + dependencyInjectedContractTest = new DependencyInjectedContractTest(this.isSecure); + } + + @Override + public void setup() throws Exception { + dependencyInjectedContractTest.initialize(); + super.setup(); + } + + @Override + protected Configuration createConfiguration() { + return this.dependencyInjectedContractTest.getConfiguration(); + } + + @Override + protected AbstractFSContract createContract(final Configuration conf) { + return new ITestAbfsFileSystemContract(conf, this.isSecure); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/f044deed/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractDelete.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractDelete.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractDelete.java new file mode 100644 index 0000000..fabd3273 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractDelete.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs.contract; + +import java.util.Arrays; + +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractDeleteTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * Contract test for delete operation. + */ +@RunWith(Parameterized.class) +public class ITestAbfsFileSystemContractDelete extends AbstractContractDeleteTest { + @Parameterized.Parameters(name = "SecureMode={0}") + public static Iterable secure() { + return Arrays.asList(new Object[][] { {true}, {false} }); + } + + private final boolean isSecure; + private final DependencyInjectedContractTest dependencyInjectedContractTest; + + public ITestAbfsFileSystemContractDelete(final boolean secure) throws Exception { + this.isSecure = secure; + dependencyInjectedContractTest = new DependencyInjectedContractTest(isSecure); + } + + @Override + public void setup() throws Exception { + dependencyInjectedContractTest.initialize(); + super.setup(); + } + + @Override + protected Configuration createConfiguration() { + return this.dependencyInjectedContractTest.getConfiguration(); + } + + @Override + protected AbstractFSContract createContract(final Configuration conf) { + return new ITestAbfsFileSystemContract(conf, this.isSecure); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/f044deed/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractDistCp.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractDistCp.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractDistCp.java new file mode 100644 index 0000000..a1360e4 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractDistCp.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.tools.contract.AbstractContractDistCpTest; + +/** + * Contract test for distCp operation. + */ +public class ITestAbfsFileSystemContractDistCp extends AbstractContractDistCpTest { + private final DependencyInjectedContractTest dependencyInjectedContractTest; + + public ITestAbfsFileSystemContractDistCp() throws Exception { + dependencyInjectedContractTest = new DependencyInjectedContractTest(false); + } + + @Override + public void setup() throws Exception { + dependencyInjectedContractTest.initialize(); + super.setup(); + } + + @Override + protected ITestAbfsFileSystemContract createContract(Configuration conf) { + return new ITestAbfsFileSystemContract(conf, false); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/f044deed/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractGetFileStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractGetFileStatus.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractGetFileStatus.java new file mode 100644 index 0000000..5bb41ad --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractGetFileStatus.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.azurebfs.contract; + +import java.util.Arrays; + +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractGetFileStatusTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * Contract test for getFileStatus operation. + */ +@RunWith(Parameterized.class) +public class ITestAbfsFileSystemContractGetFileStatus extends AbstractContractGetFileStatusTest { + @Parameterized.Parameters(name = "SecureMode={0}") + public static Iterable secure() { + return Arrays.asList(new Object[][] { {true}, {false} }); + } + + private final boolean isSecure; + private final DependencyInjectedContractTest dependencyInjectedContractTest; + + public ITestAbfsFileSystemContractGetFileStatus(final boolean secure) throws Exception { + this.isSecure = secure; + dependencyInjectedContractTest = new DependencyInjectedContractTest(isSecure); + } + + @Override + public void setup() throws Exception { + dependencyInjectedContractTest.initialize(); + super.setup(); + } + + @Override + protected Configuration createConfiguration() { + return this.dependencyInjectedContractTest.getConfiguration(); + } + + @Override + protected AbstractFSContract createContract(final Configuration conf) { + return new ITestAbfsFileSystemContract(conf, this.isSecure); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/f044deed/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractMkdir.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractMkdir.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractMkdir.java new file mode 100644 index 0000000..9d732d5 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractMkdir.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs.contract; + +import java.util.Arrays; + +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractMkdirTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * Contract test for mkdir operation. + */ +@RunWith(Parameterized.class) +public class ITestAbfsFileSystemContractMkdir extends AbstractContractMkdirTest { + @Parameterized.Parameters(name = "SecureMode={0}") + public static Iterable secure() { + return Arrays.asList(new Object[][] { {true}, {false} }); + } + + private final boolean isSecure; + private final DependencyInjectedContractTest dependencyInjectedContractTest; + + public ITestAbfsFileSystemContractMkdir(final boolean secure) throws Exception { + this.isSecure = secure; + dependencyInjectedContractTest = new DependencyInjectedContractTest(secure); + } + + @Override + public void setup() throws Exception { + dependencyInjectedContractTest.initialize(); + super.setup(); + } + + @Override + protected Configuration createConfiguration() { + return this.dependencyInjectedContractTest.getConfiguration(); + } + + @Override + protected AbstractFSContract createContract(final Configuration conf) { + return new ITestAbfsFileSystemContract(conf, this.isSecure); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/f044deed/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractOpen.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractOpen.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractOpen.java new file mode 100644 index 0000000..a71149b --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractOpen.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs.contract; + +import java.util.Arrays; + +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractOpenTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * Contract test for open operation. + */ +@RunWith(Parameterized.class) +public class ITestAbfsFileSystemContractOpen extends AbstractContractOpenTest { + @Parameterized.Parameters(name = "SecureMode={0}") + public static Iterable secure() { + return Arrays.asList(new Object[][] { {true}, {false} }); + } + + private final boolean isSecure; + private final DependencyInjectedContractTest dependencyInjectedContractTest; + + public ITestAbfsFileSystemContractOpen(final boolean secure) throws Exception { + this.isSecure = secure; + dependencyInjectedContractTest = new DependencyInjectedContractTest(this.isSecure); + } + + @Override + public void setup() throws Exception { + dependencyInjectedContractTest.initialize(); + super.setup(); + } + + @Override + protected Configuration createConfiguration() { + return this.dependencyInjectedContractTest.getConfiguration(); + } + + @Override + protected AbstractFSContract createContract(final Configuration conf) { + return new ITestAbfsFileSystemContract(conf, this.isSecure); + } +} \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org