flink-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tzulitai <...@git.apache.org>
Subject [GitHub] flink pull request #3562: [FLINK-6018] Add tests for KryoSerializer restore ...
Date Sat, 18 Mar 2017 08:48:25 GMT
Github user tzulitai commented on a diff in the pull request:

    https://github.com/apache/flink/pull/3562#discussion_r106776349
  
    --- Diff: flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendTestBase.java
---
    @@ -162,22 +176,422 @@ protected CheckpointStreamFactory createStreamFactory() throws
Exception {
     	}
     
     	@Test
    +	public void testBackendUsesRegisteredKryoDefaultSerializer() throws Exception {
    +		CheckpointStreamFactory streamFactory = createStreamFactory();
    +		Environment env = new DummyEnvironment("test", 1, 0);
    +		AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE,
env);
    +
    +		// cast because our test serializer is not typed to TestPojo
    +		env.getExecutionConfig()
    +				.addDefaultKryoSerializer(TestPojo.class, (Class) ExceptionThrowingTestSerializer.class);
    +
    +		TypeInformation<TestPojo> pojoType = new GenericTypeInfo<>(TestPojo.class);
    +
    +		// make sure that we are in fact using the KryoSerializer
    +		assertTrue(pojoType.createSerializer(env.getExecutionConfig()) instanceof KryoSerializer);
    +
    +		pojoType.createSerializer(env.getExecutionConfig());
    +
    +		ValueStateDescriptor<TestPojo> kvId = new ValueStateDescriptor<>("id",
pojoType);
    +
    +		ValueState<TestPojo> state = backend.getPartitionedState(VoidNamespace.INSTANCE,
VoidNamespaceSerializer.INSTANCE, kvId);
    +
    +		// backends that eagerly serializer will fail when updating state, others will
    +		// fail only when performing the snapshot
    +		int numExceptions = 0;
    +
    +		backend.setCurrentKey(1);
    +
    +		try {
    +			state.update(new TestPojo("u1", 1));
    +		} catch (ExpectedKryoTestException e) {
    +			numExceptions++;
    +		} catch (RuntimeException e) {
    +			if (e.getCause() instanceof ExpectedKryoTestException) {
    +				numExceptions++;
    +			} else {
    +				throw e;
    +			}
    +		}
    +
    +		try {
    +			runSnapshot(backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forFullCheckpoint()));
    +		} catch (ExpectedKryoTestException e) {
    +			numExceptions++;
    +		}
    +
    +		assertEquals("Didn't see the expected Kryo exception.", 1, numExceptions);
    +	}
    +
    +	@Test
    +	public void testBackendUsesRegisteredKryoDefaultSerializerUsingGetOrCreate() throws
Exception {
    +		CheckpointStreamFactory streamFactory = createStreamFactory();
    +		Environment env = new DummyEnvironment("test", 1, 0);
    +		AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE,
env);
    +
    +		// cast because our test serializer is not typed to TestPojo
    +		env.getExecutionConfig()
    +				.addDefaultKryoSerializer(TestPojo.class, (Class) ExceptionThrowingTestSerializer.class);
    +
    +		TypeInformation<TestPojo> pojoType = new GenericTypeInfo<>(TestPojo.class);
    +
    +		// make sure that we are in fact using the KryoSerializer
    +		assertTrue(pojoType.createSerializer(env.getExecutionConfig()) instanceof KryoSerializer);
    +
    +		pojoType.createSerializer(env.getExecutionConfig());
    +
    +		ValueStateDescriptor<TestPojo> kvId = new ValueStateDescriptor<>("id",
pojoType);
    +
    +		ValueState<TestPojo> state = backend.getOrCreateKeyedState(VoidNamespaceSerializer.INSTANCE,
kvId);
    +		assert state instanceof InternalValueState;
    +		((InternalValueState) state).setCurrentNamespace(VoidNamespace.INSTANCE);
    +
    +		// backends that eagerly serializer will fail when updating state, others will
    +		// fail only when performing the snapshot
    +		int numExceptions = 0;
    +
    +		backend.setCurrentKey(1);
    +
    +		try {
    +			state.update(new TestPojo("u1", 1));
    +		} catch (ExpectedKryoTestException e) {
    +			numExceptions++;
    +		} catch (RuntimeException e) {
    +			if (e.getCause() instanceof ExpectedKryoTestException) {
    +				numExceptions++;
    +			} else {
    +				throw e;
    +			}
    +		}
    +
    +		try {
    +			runSnapshot(backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forFullCheckpoint()));
    +		} catch (ExpectedKryoTestException e) {
    +			numExceptions++;
    +		}
    +
    +		assertEquals("Didn't see the expected Kryo exception.", 1, numExceptions);
    +	}
    +
    +	@Test
    +	public void testBackendUsesRegisteredKryoSerializer() throws Exception {
    +		CheckpointStreamFactory streamFactory = createStreamFactory();
    +		Environment env = new DummyEnvironment("test", 1, 0);
    +		AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE,
env);
    +
    +		env.getExecutionConfig()
    +				.registerTypeWithKryoSerializer(TestPojo.class, ExceptionThrowingTestSerializer.class);
    +
    +		TypeInformation<TestPojo> pojoType = new GenericTypeInfo<>(TestPojo.class);
    +
    +		// make sure that we are in fact using the KryoSerializer
    +		assertTrue(pojoType.createSerializer(env.getExecutionConfig()) instanceof KryoSerializer);
    +
    +		pojoType.createSerializer(env.getExecutionConfig());
    +
    +		ValueStateDescriptor<TestPojo> kvId = new ValueStateDescriptor<>("id",
pojoType);
    +
    +		ValueState<TestPojo> state = backend.getPartitionedState(VoidNamespace.INSTANCE,
VoidNamespaceSerializer.INSTANCE, kvId);
    +
    +		// backends that eagerly serializer will fail when updating state, others will
    +		// fail only when performing the snapshot
    +		int numExceptions = 0;
    +
    +		backend.setCurrentKey(1);
    +
    +		try {
    +			state.update(new TestPojo("u1", 1));
    +		} catch (ExpectedKryoTestException e) {
    +			numExceptions++;
    +		} catch (RuntimeException e) {
    +			if (e.getCause() instanceof ExpectedKryoTestException) {
    +				numExceptions++;
    +			} else {
    +				throw e;
    +			}
    +		}
    +
    +		try {
    +			runSnapshot(backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forFullCheckpoint()));
    +		} catch (ExpectedKryoTestException e) {
    +			numExceptions++;
    +		}
    +
    +		assertEquals("Didn't see the expected Kryo exception.", 1, numExceptions);
    +	}
    +
    +	@Test
    +	public void testBackendUsesRegisteredKryoSerializerUsingGetOrCreate() throws Exception
{
    +		CheckpointStreamFactory streamFactory = createStreamFactory();
    +		Environment env = new DummyEnvironment("test", 1, 0);
    +		AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE,
env);
    +
    +		env.getExecutionConfig()
    +				.registerTypeWithKryoSerializer(TestPojo.class, ExceptionThrowingTestSerializer.class);
    +
    +		TypeInformation<TestPojo> pojoType = new GenericTypeInfo<>(TestPojo.class);
    +
    +		// make sure that we are in fact using the KryoSerializer
    +		assertTrue(pojoType.createSerializer(env.getExecutionConfig()) instanceof KryoSerializer);
    +
    +		pojoType.createSerializer(env.getExecutionConfig());
    +
    +		ValueStateDescriptor<TestPojo> kvId = new ValueStateDescriptor<>("id",
pojoType);
    +
    +		ValueState<TestPojo> state = backend.getOrCreateKeyedState(VoidNamespaceSerializer.INSTANCE,
kvId);
    +		assert state instanceof InternalValueState;
    --- End diff --
    
    `assertTrue`


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

Mime
View raw message