hadoop-common-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Matthew Paduano (JIRA)" <j...@apache.org>
Subject [jira] [Updated] (HADOOP-12563) Updated utility to create/modify token files
Date Wed, 27 Apr 2016 01:19:13 GMT

     [ https://issues.apache.org/jira/browse/HADOOP-12563?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]

Matthew Paduano updated HADOOP-12563:
-------------------------------------
    Attachment: HADOOP-12563.15.patch

diff of patch 14,15

{code}
218c218
< +      readProto(in);
---
> +      readProtos(in);
333c333
< +  public void readProto(DataInput in) throws IOException {
---
> +  public void readProtos(DataInput in) throws IOException {



<  public class TestCredentials {
...
< +  @Test
< +  public void testBasicReadWriteProtoEmpty()
< +      throws IOException, NoSuchAlgorithmException {
< +    String testname ="testBasicReadWriteProtoEmpty";
< +    Credentials ts = new Credentials();
< +    writeCredentialsProto(ts, testname);
< +    Credentials ts2 = readCredentialsProto(testname);
< +    assertEquals("test empty tokens", 0, ts2.numberOfTokens());
< +    assertEquals("test empty keys", 0, ts2.numberOfSecretKeys());
< +  }
< +
< +  @Test
< +  public void testBasicReadWriteProto()
< +      throws IOException, NoSuchAlgorithmException {
< +    String testname ="testBasicReadWriteProto";
< +    Text tok1 = new Text("token1");
< +    Text tok2 = new Text("token2");
< +    Text key1 = new Text("key1");
< +    Credentials ts = generateCredentials(tok1, tok2, key1);
< +    writeCredentialsProto(ts, testname);
< +    Credentials ts2 = readCredentialsProto(testname);
< +    assertCredentials(testname, tok1, key1, ts, ts2);
< +    assertCredentials(testname, tok2, key1, ts, ts2);
< +  }
< +
< +  @Test
< +  public void testBasicReadWriteStreamEmpty()
< +      throws IOException, NoSuchAlgorithmException {
< +    String testname ="testBasicReadWriteStreamEmpty";
< +    Credentials ts = new Credentials();
< +    writeCredentialsStream(ts, testname);
< +    Credentials ts2 = readCredentialsStream(testname);
< +    assertEquals("test empty tokens", 0, ts2.numberOfTokens());
< +    assertEquals("test empty keys", 0, ts2.numberOfSecretKeys());
< +  }
< +
< +  @Test
< +  public void testBasicReadWriteStream()
< +      throws IOException, NoSuchAlgorithmException {
< +    String testname ="testBasicReadWriteStream";
< +    Text tok1 = new Text("token1");
< +    Text tok2 = new Text("token2");
< +    Text key1 = new Text("key1");
< +    Credentials ts = generateCredentials(tok1, tok2, key1);
< +    writeCredentialsStream(ts, testname);
< +    Credentials ts2 = readCredentialsStream(testname);
< +    assertCredentials(testname, tok1, key1, ts, ts2);
< +    assertCredentials(testname, tok2, key1, ts, ts2);
< +  }
< +
< +  @Test
< +  /**
< +   * Verify the suitability of read/writeProto for use with Writable interfac
< +   * This test uses only empty credentials.
< +   */
< +  public void testWritablePropertiesEmpty()
< +      throws IOException, NoSuchAlgorithmException {
< +    String testname ="testWritablePropertiesEmpty";
< +    Credentials ts = new Credentials();
< +    Credentials ts2 = new Credentials();
< +    writeCredentialsProtos(ts, ts2, testname);
< +    List<Credentials> clist = readCredentialsProtos(testname);
< +    assertEquals("test empty tokens 0", 0, clist.get(0).numberOfTokens());
< +    assertEquals("test empty keys 0", 0, clist.get(0).numberOfSecretKeys());
< +    assertEquals("test empty tokens 1", 0, clist.get(1).numberOfTokens());
< +    assertEquals("test empty keys 1", 0, clist.get(1).numberOfSecretKeys());
< +  }
< +
< +  @Test
< +  /**
< +   * Verify the suitability of read/writeProto for use with Writable interfac
< +   */
< +  public void testWritableProperties()
< +      throws IOException, NoSuchAlgorithmException {
< +    String testname ="testWritableProperties";
< +    Text tok1 = new Text("token1");
< +    Text tok2 = new Text("token2");
< +    Text key1 = new Text("key1");
< +    Credentials ts = generateCredentials(tok1, tok2, key1);
< +    Text tok3 = new Text("token3");
< +    Text key2 = new Text("key2");
< +    Credentials ts2 = generateCredentials(tok1, tok3, key2);
< +    writeCredentialsProtos(ts, ts2, testname);
< +    List<Credentials> clist = readCredentialsProtos(testname);
< +    assertCredentials(testname, tok1, key1, ts, clist.get(0));
< +    assertCredentials(testname, tok2, key1, ts, clist.get(0));
< +    assertCredentials(testname, tok1, key2, ts2, clist.get(1));
< +    assertCredentials(testname, tok3, key2, ts2, clist.get(1));
< +  }
< +
< +  private Credentials generateCredentials(Text t1, Text t2, Text t3)
< +      throws NoSuchAlgorithmException {
< +    Text kind = new Text("TESTTOK");
< +    byte[] id1 = {0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72}
< +    byte[] pass1 = {0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64};
< +    byte[] id2 = {0x68, 0x63, 0x64, 0x6d, 0x73, 0x68, 0x65, 0x68, 0x64, 0x71}
< +    byte[] pass2 = {0x6f, 0x60, 0x72, 0x72, 0x76, 0x6e, 0x71, 0x63};
< +    Credentials ts = new Credentials();
< +    generateToken(ts, id1, pass1, kind, t1);
< +    generateToken(ts, id2, pass2, kind, t2);
< +    generateKey(ts, t3);
< +    return ts;
< +  }
< +
< +  private void assertCredentials(String tag, Text alias, Text keykey,
< +                                 Credentials a, Credentials b) {
< +    assertEquals(tag + ": test token count", a.numberOfTokens(),
< +                                             b.numberOfTokens());
< +    assertEquals(tag + ": test service", a.getToken(alias).getService(),
< +                                         b.getToken(alias).getService());
< +    assertEquals(tag + ": test kind", a.getToken(alias).getKind(),
< +                                      b.getToken(alias).getKind());
< +    assertTrue(tag + ": test password",
< +        Arrays.equals(a.getToken(alias).getPassword(),
< +                      b.getToken(alias).getPassword()));
< +    assertTrue(tag + ": test identifier",
< +        Arrays.equals(a.getToken(alias).getIdentifier(),
< +                      b.getToken(alias).getIdentifier()));
< +    assertEquals(tag + ": test number of keys", a.numberOfSecretKeys(),
< +                                                b.numberOfSecretKeys());
< +    assertTrue(tag + ":test key values", Arrays.equals(a.getSecretKey(keykey)
< +                                                       b.getSecretKey(keykey)
< +  }
< +
< +  private void writeCredentialsStream(Credentials creds, String filename)
< +      throws IOException, FileNotFoundException {
< +    DataOutputStream dos = new DataOutputStream(
< +        new FileOutputStream(new File(tmpDir, filename)));
< +    creds.writeTokenStorageToStream(dos);
< +  }
< +
< +  private Credentials readCredentialsStream(String filename)
< +      throws IOException, FileNotFoundException {
< +    Credentials creds = new Credentials();
< +    DataInputStream dis = new DataInputStream(
< +        new FileInputStream(new File(tmpDir, filename)));
< +    creds.readTokenStorageStream(dis);
< +    return creds;
< +  }
< +
< +  private void writeCredentialsProto(Credentials creds, String filename)
< +      throws IOException, FileNotFoundException {
< +    DataOutputStream dos = new DataOutputStream(
< +        new FileOutputStream(new File(tmpDir, filename)));
< +    creds.writeProto(dos);
< +  }
< +
< +  private Credentials readCredentialsProto(String filename)
< +      throws IOException, FileNotFoundException {
< +    Credentials creds = new Credentials();
< +    DataInputStream dis = new DataInputStream(
< +        new FileInputStream(new File(tmpDir, filename)));
< +    creds.readProto(dis);
< +    return creds;
< +  }
< +
< +  private void writeCredentialsProtos(Credentials c1, Credentials c2,
< +      String filename) throws IOException, FileNotFoundException {
< +    DataOutputStream dos = new DataOutputStream(
< +        new FileOutputStream(new File(tmpDir, filename)));
< +    c1.writeProto(dos);
< +    c2.writeProto(dos);
< +  }
< +
< +  private List<Credentials> readCredentialsProtos(String filename)
< +      throws IOException, FileNotFoundException {
< +    Credentials c1 = new Credentials();
< +    Credentials c2 = new Credentials();
< +    DataInputStream dis = new DataInputStream(
< +        new FileInputStream(new File(tmpDir, filename)));
< +    c1.readProto(dis);
< +    c2.readProto(dis);
< +    List<Credentials> r = new ArrayList<Credentials>(2);
< +    r.add(0, c1);
< +    r.add(1, c2);
< +    return r;
< +  }
< +
< +  private <T extends TokenIdentifier> void generateToken(
< +      Credentials creds, byte[] ident, byte[] pass, Text kind, Text service) 
< +    Token<T> token = new Token(ident, pass, kind, service);
< +    creds.addToken(service, token);
< +  }
< +
< +  private void generateKey(Credentials creds, Text alias)
< +      throws NoSuchAlgorithmException {
< +    final KeyGenerator kg = KeyGenerator.getInstance(DEFAULT_HMAC_ALGORITHM);
< +    Key key = kg.generateKey();
< +    creds.addSecretKey(alias, key.getEncoded());
< +  }
< +

{code}

> Updated utility to create/modify token files
> --------------------------------------------
>
>                 Key: HADOOP-12563
>                 URL: https://issues.apache.org/jira/browse/HADOOP-12563
>             Project: Hadoop Common
>          Issue Type: New Feature
>    Affects Versions: 3.0.0
>            Reporter: Allen Wittenauer
>            Assignee: Matthew Paduano
>             Fix For: 3.0.0
>
>         Attachments: HADOOP-12563.01.patch, HADOOP-12563.02.patch, HADOOP-12563.03.patch,
HADOOP-12563.04.patch, HADOOP-12563.05.patch, HADOOP-12563.06.patch, HADOOP-12563.07.patch,
HADOOP-12563.07.patch, HADOOP-12563.08.patch, HADOOP-12563.09.patch, HADOOP-12563.10.patch,
HADOOP-12563.11.patch, HADOOP-12563.12.patch, HADOOP-12563.13.patch, HADOOP-12563.14.patch,
HADOOP-12563.15.patch, dtutil-test-out, example_dtutil_commands_and_output.txt, generalized_token_case.pdf
>
>
> hdfs fetchdt is missing some critical features and is geared almost exclusively towards
HDFS operations.  Additionally, the token files that are created use Java serializations which
are hard/impossible to deal with in other languages. It should be replaced with a better utility
in common that can read/write protobuf-based token files, has enough flexibility to be used
with other services, and offers key functionality such as append and rename. The old version
file format should still be supported for backward compatibility, but will be effectively
deprecated.
> A follow-on JIRA will deprecrate fetchdt.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

Mime
View raw message