hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From xu...@apache.org
Subject svn commit: r1667547 - in /hive/trunk: itests/src/test/resources/ ql/src/java/org/apache/hadoop/hive/ql/parse/ ql/src/test/queries/clientpositive/ ql/src/test/results/clientnegative/ ql/src/test/results/clientpositive/encrypted/
Date Wed, 18 Mar 2015 13:36:01 GMT
Author: xuefu
Date: Wed Mar 18 13:36:00 2015
New Revision: 1667547

URL: http://svn.apache.org/r1667547
Log:
HIVE-9991: Cannot do a SELECT on external tables that are on S3 due to Encryption error (Sergio
via Xuefu)

Added:
    hive/trunk/ql/src/test/queries/clientpositive/encryption_unencrypted_nonhdfs_external_tables.q
    hive/trunk/ql/src/test/results/clientpositive/encrypted/encryption_unencrypted_nonhdfs_external_tables.q.out
Modified:
    hive/trunk/itests/src/test/resources/testconfiguration.properties
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
    hive/trunk/ql/src/test/results/clientnegative/fs_default_name2.q.out

Modified: hive/trunk/itests/src/test/resources/testconfiguration.properties
URL: http://svn.apache.org/viewvc/hive/trunk/itests/src/test/resources/testconfiguration.properties?rev=1667547&r1=1667546&r2=1667547&view=diff
==============================================================================
--- hive/trunk/itests/src/test/resources/testconfiguration.properties (original)
+++ hive/trunk/itests/src/test/resources/testconfiguration.properties Wed Mar 18 13:36:00
2015
@@ -314,7 +314,8 @@ encrypted.query.files=encryption_join_un
   encryption_join_with_different_encryption_keys.q,\
   encryption_select_read_only_encrypted_tbl.q,\
   encryption_select_read_only_unencrypted_tbl.q,\
-  encryption_load_data_to_encrypted_tables.q
+  encryption_load_data_to_encrypted_tables.q, \
+  encryption_unencrypted_nonhdfs_external_tables.q
 
 beeline.positive.exclude=add_part_exist.q,\
   alter1.q,\

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1667547&r1=1667546&r2=1667547&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Wed Mar 18
13:36:00 2015
@@ -1856,7 +1856,7 @@ public class SemanticAnalyzer extends Ba
           return true;
         }
       } catch (Exception e) {
-        throw new HiveException("Unable to determine if " + path + "is encrypted: " + e,
e);
+        throw new HiveException("Unable to determine if " + path + " is encrypted: " + e,
e);
       }
     }
 
@@ -1919,7 +1919,7 @@ public class SemanticAnalyzer extends Ba
    * Gets the strongest encrypted table path.
    *
    * @param qb The QB object that contains a list of all table locations.
-   * @return The strongest encrypted path
+   * @return The strongest encrypted path. It may return NULL if there are not tables encrypted,
or are not HDFS tables.
    * @throws HiveException if an error occurred attempting to compare the encryption strength
    */
   private Path getStrongestEncryptedTablePath(QB qb) throws HiveException {
@@ -1932,17 +1932,14 @@ public class SemanticAnalyzer extends Ba
       if (tab != null) {
         Path tablePath = tab.getDataLocation();
         if (tablePath != null) {
-          try {
-            if (strongestPath == null) {
-              strongestPath = tablePath;
-            } else if ("hdfs".equals(tablePath.toUri().getScheme())
-                && isPathEncrypted(tablePath)
-                && comparePathKeyStrength(tablePath, strongestPath) > 0)
-            {
-              strongestPath = tablePath;
+          if ("hdfs".equalsIgnoreCase(tablePath.toUri().getScheme())) {
+            if (isPathEncrypted(tablePath)) {
+              if (strongestPath == null) {
+                strongestPath = tablePath;
+              } else if (comparePathKeyStrength(tablePath, strongestPath) > 0) {
+                strongestPath = tablePath;
+              }
             }
-          } catch (HiveException e) {
-            throw new HiveException("Unable to find the most secure table path: " + e, e);
           }
         }
       }
@@ -1966,22 +1963,19 @@ public class SemanticAnalyzer extends Ba
   private Path getStagingDirectoryPathname(QB qb) throws HiveException {
     Path stagingPath = null, tablePath;
 
-    // Looks for the most encrypted table location (if there is one)
+    // Looks for the most encrypted table location
+    // It may return null if there are not tables encrypted, or are not part of HDFS
     tablePath = getStrongestEncryptedTablePath(qb);
-    if (tablePath != null && isPathEncrypted(tablePath)) {
-      // Only HDFS paths can be checked for encryption
-      if ("hdfs".equals(tablePath.toUri().getScheme())) {
-        if (isPathReadOnly(tablePath)) {
-          Path tmpPath = ctx.getMRTmpPath();
-          if (comparePathKeyStrength(tablePath, tmpPath) < 0) {
-            throw new HiveException("Read-only encrypted tables cannot be read " +
-                "if the scratch directory is not encrypted (or encryption is weak)");
-          } else {
-            stagingPath = tmpPath;
-          }
+    if (tablePath != null) {
+      // At this point, tablePath is part of HDFS and it is encrypted
+      if (isPathReadOnly(tablePath)) {
+        Path tmpPath = ctx.getMRTmpPath();
+        if (comparePathKeyStrength(tablePath, tmpPath) < 0) {
+          throw new HiveException("Read-only encrypted tables cannot be read " +
+              "if the scratch directory is not encrypted (or encryption is weak)");
+        } else {
+          stagingPath = tmpPath;
         }
-      } else {
-        LOG.debug("Encryption is not applicable to table path " + tablePath.toString());
       }
 
       if (stagingPath == null) {

Added: hive/trunk/ql/src/test/queries/clientpositive/encryption_unencrypted_nonhdfs_external_tables.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/encryption_unencrypted_nonhdfs_external_tables.q?rev=1667547&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/encryption_unencrypted_nonhdfs_external_tables.q
(added)
+++ hive/trunk/ql/src/test/queries/clientpositive/encryption_unencrypted_nonhdfs_external_tables.q
Wed Mar 18 13:36:00 2015
@@ -0,0 +1,13 @@
+-- This test does not test encrypted data, but it makes sure that external tables out of
HDFS can
+-- be queried due to internal encryption functions;
+
+DROP TABLE mydata;
+
+CREATE EXTERNAL TABLE mydata (key STRING, value STRING) ROW FORMAT DELIMITED FIELDS TERMINATED
BY ' '
+LOCATION 'pfile://${system:test.tmp.dir}/external_mydata';
+
+LOAD DATA LOCAL INPATH '../../data/files/kv1.txt' OVERWRITE INTO TABLE mydata;
+
+SELECT * from mydata;
+
+DROP TABLE mydata;
\ No newline at end of file

Modified: hive/trunk/ql/src/test/results/clientnegative/fs_default_name2.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/fs_default_name2.q.out?rev=1667547&r1=1667546&r2=1667547&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/fs_default_name2.q.out (original)
+++ hive/trunk/ql/src/test/results/clientnegative/fs_default_name2.q.out Wed Mar 18 13:36:00
2015
@@ -1 +1 @@
-FAILED: SemanticException java.lang.IllegalArgumentException: Illegal character in scheme
name at index 0: 'http://www.example.com
+FAILED: IllegalArgumentException Illegal character in scheme name at index 0: 'http://www.example.com

Added: hive/trunk/ql/src/test/results/clientpositive/encrypted/encryption_unencrypted_nonhdfs_external_tables.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/encrypted/encryption_unencrypted_nonhdfs_external_tables.q.out?rev=1667547&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/encrypted/encryption_unencrypted_nonhdfs_external_tables.q.out
(added)
+++ hive/trunk/ql/src/test/results/clientpositive/encrypted/encryption_unencrypted_nonhdfs_external_tables.q.out
Wed Mar 18 13:36:00 2015
@@ -0,0 +1,546 @@
+PREHOOK: query: -- This test does not test encrypted data, but it makes sure that external
tables out of HDFS can
+-- be queried due to internal encryption functions;
+
+DROP TABLE mydata
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: -- This test does not test encrypted data, but it makes sure that external
tables out of HDFS can
+-- be queried due to internal encryption functions;
+
+DROP TABLE mydata
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE EXTERNAL TABLE mydata (key STRING, value STRING) ROW FORMAT DELIMITED
FIELDS TERMINATED BY ' '
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@mydata
+POSTHOOK: query: CREATE EXTERNAL TABLE mydata (key STRING, value STRING) ROW FORMAT DELIMITED
FIELDS TERMINATED BY ' '
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@mydata
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/kv1.txt' OVERWRITE INTO TABLE mydata
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@mydata
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/kv1.txt' OVERWRITE INTO TABLE mydata
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@mydata
+PREHOOK: query: SELECT * from mydata
+PREHOOK: type: QUERY
+PREHOOK: Input: default@mydata
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * from mydata
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@mydata
+#### A masked pattern was here ####
+238val_238	NULL
+86val_86	NULL
+311val_311	NULL
+27val_27	NULL
+165val_165	NULL
+409val_409	NULL
+255val_255	NULL
+278val_278	NULL
+98val_98	NULL
+484val_484	NULL
+265val_265	NULL
+193val_193	NULL
+401val_401	NULL
+150val_150	NULL
+273val_273	NULL
+224val_224	NULL
+369val_369	NULL
+66val_66	NULL
+128val_128	NULL
+213val_213	NULL
+146val_146	NULL
+406val_406	NULL
+429val_429	NULL
+374val_374	NULL
+152val_152	NULL
+469val_469	NULL
+145val_145	NULL
+495val_495	NULL
+37val_37	NULL
+327val_327	NULL
+281val_281	NULL
+277val_277	NULL
+209val_209	NULL
+15val_15	NULL
+82val_82	NULL
+403val_403	NULL
+166val_166	NULL
+417val_417	NULL
+430val_430	NULL
+252val_252	NULL
+292val_292	NULL
+219val_219	NULL
+287val_287	NULL
+153val_153	NULL
+193val_193	NULL
+338val_338	NULL
+446val_446	NULL
+459val_459	NULL
+394val_394	NULL
+237val_237	NULL
+482val_482	NULL
+174val_174	NULL
+413val_413	NULL
+494val_494	NULL
+207val_207	NULL
+199val_199	NULL
+466val_466	NULL
+208val_208	NULL
+174val_174	NULL
+399val_399	NULL
+396val_396	NULL
+247val_247	NULL
+417val_417	NULL
+489val_489	NULL
+162val_162	NULL
+377val_377	NULL
+397val_397	NULL
+309val_309	NULL
+365val_365	NULL
+266val_266	NULL
+439val_439	NULL
+342val_342	NULL
+367val_367	NULL
+325val_325	NULL
+167val_167	NULL
+195val_195	NULL
+475val_475	NULL
+17val_17	NULL
+113val_113	NULL
+155val_155	NULL
+203val_203	NULL
+339val_339	NULL
+0val_0	NULL
+455val_455	NULL
+128val_128	NULL
+311val_311	NULL
+316val_316	NULL
+57val_57	NULL
+302val_302	NULL
+205val_205	NULL
+149val_149	NULL
+438val_438	NULL
+345val_345	NULL
+129val_129	NULL
+170val_170	NULL
+20val_20	NULL
+489val_489	NULL
+157val_157	NULL
+378val_378	NULL
+221val_221	NULL
+92val_92	NULL
+111val_111	NULL
+47val_47	NULL
+72val_72	NULL
+4val_4	NULL
+280val_280	NULL
+35val_35	NULL
+427val_427	NULL
+277val_277	NULL
+208val_208	NULL
+356val_356	NULL
+399val_399	NULL
+169val_169	NULL
+382val_382	NULL
+498val_498	NULL
+125val_125	NULL
+386val_386	NULL
+437val_437	NULL
+469val_469	NULL
+192val_192	NULL
+286val_286	NULL
+187val_187	NULL
+176val_176	NULL
+54val_54	NULL
+459val_459	NULL
+51val_51	NULL
+138val_138	NULL
+103val_103	NULL
+239val_239	NULL
+213val_213	NULL
+216val_216	NULL
+430val_430	NULL
+278val_278	NULL
+176val_176	NULL
+289val_289	NULL
+221val_221	NULL
+65val_65	NULL
+318val_318	NULL
+332val_332	NULL
+311val_311	NULL
+275val_275	NULL
+137val_137	NULL
+241val_241	NULL
+83val_83	NULL
+333val_333	NULL
+180val_180	NULL
+284val_284	NULL
+12val_12	NULL
+230val_230	NULL
+181val_181	NULL
+67val_67	NULL
+260val_260	NULL
+404val_404	NULL
+384val_384	NULL
+489val_489	NULL
+353val_353	NULL
+373val_373	NULL
+272val_272	NULL
+138val_138	NULL
+217val_217	NULL
+84val_84	NULL
+348val_348	NULL
+466val_466	NULL
+58val_58	NULL
+8val_8	NULL
+411val_411	NULL
+230val_230	NULL
+208val_208	NULL
+348val_348	NULL
+24val_24	NULL
+463val_463	NULL
+431val_431	NULL
+179val_179	NULL
+172val_172	NULL
+42val_42	NULL
+129val_129	NULL
+158val_158	NULL
+119val_119	NULL
+496val_496	NULL
+0val_0	NULL
+322val_322	NULL
+197val_197	NULL
+468val_468	NULL
+393val_393	NULL
+454val_454	NULL
+100val_100	NULL
+298val_298	NULL
+199val_199	NULL
+191val_191	NULL
+418val_418	NULL
+96val_96	NULL
+26val_26	NULL
+165val_165	NULL
+327val_327	NULL
+230val_230	NULL
+205val_205	NULL
+120val_120	NULL
+131val_131	NULL
+51val_51	NULL
+404val_404	NULL
+43val_43	NULL
+436val_436	NULL
+156val_156	NULL
+469val_469	NULL
+468val_468	NULL
+308val_308	NULL
+95val_95	NULL
+196val_196	NULL
+288val_288	NULL
+481val_481	NULL
+457val_457	NULL
+98val_98	NULL
+282val_282	NULL
+197val_197	NULL
+187val_187	NULL
+318val_318	NULL
+318val_318	NULL
+409val_409	NULL
+470val_470	NULL
+137val_137	NULL
+369val_369	NULL
+316val_316	NULL
+169val_169	NULL
+413val_413	NULL
+85val_85	NULL
+77val_77	NULL
+0val_0	NULL
+490val_490	NULL
+87val_87	NULL
+364val_364	NULL
+179val_179	NULL
+118val_118	NULL
+134val_134	NULL
+395val_395	NULL
+282val_282	NULL
+138val_138	NULL
+238val_238	NULL
+419val_419	NULL
+15val_15	NULL
+118val_118	NULL
+72val_72	NULL
+90val_90	NULL
+307val_307	NULL
+19val_19	NULL
+435val_435	NULL
+10val_10	NULL
+277val_277	NULL
+273val_273	NULL
+306val_306	NULL
+224val_224	NULL
+309val_309	NULL
+389val_389	NULL
+327val_327	NULL
+242val_242	NULL
+369val_369	NULL
+392val_392	NULL
+272val_272	NULL
+331val_331	NULL
+401val_401	NULL
+242val_242	NULL
+452val_452	NULL
+177val_177	NULL
+226val_226	NULL
+5val_5	NULL
+497val_497	NULL
+402val_402	NULL
+396val_396	NULL
+317val_317	NULL
+395val_395	NULL
+58val_58	NULL
+35val_35	NULL
+336val_336	NULL
+95val_95	NULL
+11val_11	NULL
+168val_168	NULL
+34val_34	NULL
+229val_229	NULL
+233val_233	NULL
+143val_143	NULL
+472val_472	NULL
+322val_322	NULL
+498val_498	NULL
+160val_160	NULL
+195val_195	NULL
+42val_42	NULL
+321val_321	NULL
+430val_430	NULL
+119val_119	NULL
+489val_489	NULL
+458val_458	NULL
+78val_78	NULL
+76val_76	NULL
+41val_41	NULL
+223val_223	NULL
+492val_492	NULL
+149val_149	NULL
+449val_449	NULL
+218val_218	NULL
+228val_228	NULL
+138val_138	NULL
+453val_453	NULL
+30val_30	NULL
+209val_209	NULL
+64val_64	NULL
+468val_468	NULL
+76val_76	NULL
+74val_74	NULL
+342val_342	NULL
+69val_69	NULL
+230val_230	NULL
+33val_33	NULL
+368val_368	NULL
+103val_103	NULL
+296val_296	NULL
+113val_113	NULL
+216val_216	NULL
+367val_367	NULL
+344val_344	NULL
+167val_167	NULL
+274val_274	NULL
+219val_219	NULL
+239val_239	NULL
+485val_485	NULL
+116val_116	NULL
+223val_223	NULL
+256val_256	NULL
+263val_263	NULL
+70val_70	NULL
+487val_487	NULL
+480val_480	NULL
+401val_401	NULL
+288val_288	NULL
+191val_191	NULL
+5val_5	NULL
+244val_244	NULL
+438val_438	NULL
+128val_128	NULL
+467val_467	NULL
+432val_432	NULL
+202val_202	NULL
+316val_316	NULL
+229val_229	NULL
+469val_469	NULL
+463val_463	NULL
+280val_280	NULL
+2val_2	NULL
+35val_35	NULL
+283val_283	NULL
+331val_331	NULL
+235val_235	NULL
+80val_80	NULL
+44val_44	NULL
+193val_193	NULL
+321val_321	NULL
+335val_335	NULL
+104val_104	NULL
+466val_466	NULL
+366val_366	NULL
+175val_175	NULL
+403val_403	NULL
+483val_483	NULL
+53val_53	NULL
+105val_105	NULL
+257val_257	NULL
+406val_406	NULL
+409val_409	NULL
+190val_190	NULL
+406val_406	NULL
+401val_401	NULL
+114val_114	NULL
+258val_258	NULL
+90val_90	NULL
+203val_203	NULL
+262val_262	NULL
+348val_348	NULL
+424val_424	NULL
+12val_12	NULL
+396val_396	NULL
+201val_201	NULL
+217val_217	NULL
+164val_164	NULL
+431val_431	NULL
+454val_454	NULL
+478val_478	NULL
+298val_298	NULL
+125val_125	NULL
+431val_431	NULL
+164val_164	NULL
+424val_424	NULL
+187val_187	NULL
+382val_382	NULL
+5val_5	NULL
+70val_70	NULL
+397val_397	NULL
+480val_480	NULL
+291val_291	NULL
+24val_24	NULL
+351val_351	NULL
+255val_255	NULL
+104val_104	NULL
+70val_70	NULL
+163val_163	NULL
+438val_438	NULL
+119val_119	NULL
+414val_414	NULL
+200val_200	NULL
+491val_491	NULL
+237val_237	NULL
+439val_439	NULL
+360val_360	NULL
+248val_248	NULL
+479val_479	NULL
+305val_305	NULL
+417val_417	NULL
+199val_199	NULL
+444val_444	NULL
+120val_120	NULL
+429val_429	NULL
+169val_169	NULL
+443val_443	NULL
+323val_323	NULL
+325val_325	NULL
+277val_277	NULL
+230val_230	NULL
+478val_478	NULL
+178val_178	NULL
+468val_468	NULL
+310val_310	NULL
+317val_317	NULL
+333val_333	NULL
+493val_493	NULL
+460val_460	NULL
+207val_207	NULL
+249val_249	NULL
+265val_265	NULL
+480val_480	NULL
+83val_83	NULL
+136val_136	NULL
+353val_353	NULL
+172val_172	NULL
+214val_214	NULL
+462val_462	NULL
+233val_233	NULL
+406val_406	NULL
+133val_133	NULL
+175val_175	NULL
+189val_189	NULL
+454val_454	NULL
+375val_375	NULL
+401val_401	NULL
+421val_421	NULL
+407val_407	NULL
+384val_384	NULL
+256val_256	NULL
+26val_26	NULL
+134val_134	NULL
+67val_67	NULL
+384val_384	NULL
+379val_379	NULL
+18val_18	NULL
+462val_462	NULL
+492val_492	NULL
+100val_100	NULL
+298val_298	NULL
+9val_9	NULL
+341val_341	NULL
+498val_498	NULL
+146val_146	NULL
+458val_458	NULL
+362val_362	NULL
+186val_186	NULL
+285val_285	NULL
+348val_348	NULL
+167val_167	NULL
+18val_18	NULL
+273val_273	NULL
+183val_183	NULL
+281val_281	NULL
+344val_344	NULL
+97val_97	NULL
+469val_469	NULL
+315val_315	NULL
+84val_84	NULL
+28val_28	NULL
+37val_37	NULL
+448val_448	NULL
+152val_152	NULL
+348val_348	NULL
+307val_307	NULL
+194val_194	NULL
+414val_414	NULL
+477val_477	NULL
+222val_222	NULL
+126val_126	NULL
+90val_90	NULL
+169val_169	NULL
+403val_403	NULL
+400val_400	NULL
+200val_200	NULL
+97val_97	NULL
+PREHOOK: query: DROP TABLE mydata
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@mydata
+PREHOOK: Output: default@mydata
+POSTHOOK: query: DROP TABLE mydata
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@mydata
+POSTHOOK: Output: default@mydata



Mime
View raw message