flink-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tzulitai <...@git.apache.org>
Subject [GitHub] flink pull request #3562: [FLINK-6018] Add tests for KryoSerializer restore ...
Date Sat, 18 Mar 2017 08:48:44 GMT
Github user tzulitai commented on a diff in the pull request:

    https://github.com/apache/flink/pull/3562#discussion_r106776658
  
    --- Diff: flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendTestBase.java
---
    @@ -162,22 +176,422 @@ protected CheckpointStreamFactory createStreamFactory() throws
Exception {
     	}
     
     	@Test
    +	public void testBackendUsesRegisteredKryoDefaultSerializer() throws Exception {
    +		CheckpointStreamFactory streamFactory = createStreamFactory();
    +		Environment env = new DummyEnvironment("test", 1, 0);
    +		AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE,
env);
    +
    +		// cast because our test serializer is not typed to TestPojo
    +		env.getExecutionConfig()
    +				.addDefaultKryoSerializer(TestPojo.class, (Class) ExceptionThrowingTestSerializer.class);
    +
    +		TypeInformation<TestPojo> pojoType = new GenericTypeInfo<>(TestPojo.class);
    +
    +		// make sure that we are in fact using the KryoSerializer
    +		assertTrue(pojoType.createSerializer(env.getExecutionConfig()) instanceof KryoSerializer);
    +
    +		pojoType.createSerializer(env.getExecutionConfig());
    +
    +		ValueStateDescriptor<TestPojo> kvId = new ValueStateDescriptor<>("id",
pojoType);
    +
    +		ValueState<TestPojo> state = backend.getPartitionedState(VoidNamespace.INSTANCE,
VoidNamespaceSerializer.INSTANCE, kvId);
    +
    +		// backends that eagerly serializer will fail when updating state, others will
    +		// fail only when performing the snapshot
    +		int numExceptions = 0;
    +
    +		backend.setCurrentKey(1);
    +
    +		try {
    +			state.update(new TestPojo("u1", 1));
    +		} catch (ExpectedKryoTestException e) {
    +			numExceptions++;
    +		} catch (RuntimeException e) {
    +			if (e.getCause() instanceof ExpectedKryoTestException) {
    +				numExceptions++;
    +			} else {
    +				throw e;
    +			}
    +		}
    +
    +		try {
    +			runSnapshot(backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forFullCheckpoint()));
    +		} catch (ExpectedKryoTestException e) {
    +			numExceptions++;
    +		}
    +
    +		assertEquals("Didn't see the expected Kryo exception.", 1, numExceptions);
    +	}
    +
    +	@Test
    +	public void testBackendUsesRegisteredKryoDefaultSerializerUsingGetOrCreate() throws
Exception {
    +		CheckpointStreamFactory streamFactory = createStreamFactory();
    +		Environment env = new DummyEnvironment("test", 1, 0);
    +		AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE,
env);
    +
    +		// cast because our test serializer is not typed to TestPojo
    +		env.getExecutionConfig()
    +				.addDefaultKryoSerializer(TestPojo.class, (Class) ExceptionThrowingTestSerializer.class);
    +
    +		TypeInformation<TestPojo> pojoType = new GenericTypeInfo<>(TestPojo.class);
    +
    +		// make sure that we are in fact using the KryoSerializer
    +		assertTrue(pojoType.createSerializer(env.getExecutionConfig()) instanceof KryoSerializer);
    +
    +		pojoType.createSerializer(env.getExecutionConfig());
    +
    +		ValueStateDescriptor<TestPojo> kvId = new ValueStateDescriptor<>("id",
pojoType);
    +
    +		ValueState<TestPojo> state = backend.getOrCreateKeyedState(VoidNamespaceSerializer.INSTANCE,
kvId);
    +		assert state instanceof InternalValueState;
    +		((InternalValueState) state).setCurrentNamespace(VoidNamespace.INSTANCE);
    +
    +		// backends that eagerly serializer will fail when updating state, others will
    +		// fail only when performing the snapshot
    +		int numExceptions = 0;
    +
    +		backend.setCurrentKey(1);
    +
    +		try {
    +			state.update(new TestPojo("u1", 1));
    +		} catch (ExpectedKryoTestException e) {
    +			numExceptions++;
    +		} catch (RuntimeException e) {
    +			if (e.getCause() instanceof ExpectedKryoTestException) {
    +				numExceptions++;
    +			} else {
    +				throw e;
    +			}
    +		}
    +
    +		try {
    +			runSnapshot(backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forFullCheckpoint()));
    +		} catch (ExpectedKryoTestException e) {
    +			numExceptions++;
    +		}
    +
    +		assertEquals("Didn't see the expected Kryo exception.", 1, numExceptions);
    +	}
    +
    +	@Test
    +	public void testBackendUsesRegisteredKryoSerializer() throws Exception {
    +		CheckpointStreamFactory streamFactory = createStreamFactory();
    +		Environment env = new DummyEnvironment("test", 1, 0);
    +		AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE,
env);
    +
    +		env.getExecutionConfig()
    +				.registerTypeWithKryoSerializer(TestPojo.class, ExceptionThrowingTestSerializer.class);
    +
    +		TypeInformation<TestPojo> pojoType = new GenericTypeInfo<>(TestPojo.class);
    +
    +		// make sure that we are in fact using the KryoSerializer
    +		assertTrue(pojoType.createSerializer(env.getExecutionConfig()) instanceof KryoSerializer);
    +
    +		pojoType.createSerializer(env.getExecutionConfig());
    +
    +		ValueStateDescriptor<TestPojo> kvId = new ValueStateDescriptor<>("id",
pojoType);
    +
    +		ValueState<TestPojo> state = backend.getPartitionedState(VoidNamespace.INSTANCE,
VoidNamespaceSerializer.INSTANCE, kvId);
    +
    +		// backends that eagerly serializer will fail when updating state, others will
    +		// fail only when performing the snapshot
    +		int numExceptions = 0;
    +
    +		backend.setCurrentKey(1);
    +
    +		try {
    +			state.update(new TestPojo("u1", 1));
    +		} catch (ExpectedKryoTestException e) {
    +			numExceptions++;
    +		} catch (RuntimeException e) {
    +			if (e.getCause() instanceof ExpectedKryoTestException) {
    +				numExceptions++;
    +			} else {
    +				throw e;
    +			}
    +		}
    +
    +		try {
    +			runSnapshot(backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forFullCheckpoint()));
    +		} catch (ExpectedKryoTestException e) {
    +			numExceptions++;
    +		}
    +
    +		assertEquals("Didn't see the expected Kryo exception.", 1, numExceptions);
    +	}
    +
    +	@Test
    +	public void testBackendUsesRegisteredKryoSerializerUsingGetOrCreate() throws Exception
{
    +		CheckpointStreamFactory streamFactory = createStreamFactory();
    +		Environment env = new DummyEnvironment("test", 1, 0);
    +		AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE,
env);
    +
    +		env.getExecutionConfig()
    +				.registerTypeWithKryoSerializer(TestPojo.class, ExceptionThrowingTestSerializer.class);
    +
    +		TypeInformation<TestPojo> pojoType = new GenericTypeInfo<>(TestPojo.class);
    +
    +		// make sure that we are in fact using the KryoSerializer
    +		assertTrue(pojoType.createSerializer(env.getExecutionConfig()) instanceof KryoSerializer);
    +
    +		pojoType.createSerializer(env.getExecutionConfig());
    +
    +		ValueStateDescriptor<TestPojo> kvId = new ValueStateDescriptor<>("id",
pojoType);
    +
    +		ValueState<TestPojo> state = backend.getOrCreateKeyedState(VoidNamespaceSerializer.INSTANCE,
kvId);
    +		assert state instanceof InternalValueState;
    +		((InternalValueState) state).setCurrentNamespace(VoidNamespace.INSTANCE);
    +
    +		// backends that eagerly serializer will fail when updating state, others will
    +		// fail only when performing the snapshot
    +		int numExceptions = 0;
    +
    +		backend.setCurrentKey(1);
    +
    +		try {
    +			state.update(new TestPojo("u1", 1));
    +		} catch (ExpectedKryoTestException e) {
    +			numExceptions++;
    +		} catch (RuntimeException e) {
    +			if (e.getCause() instanceof ExpectedKryoTestException) {
    +				numExceptions++;
    +			} else {
    +				throw e;
    +			}
    +		}
    +
    +		try {
    +			runSnapshot(backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forFullCheckpoint()));
    +		} catch (ExpectedKryoTestException e) {
    +			numExceptions++;
    +		}
    +
    +		assertEquals("Didn't see the expected Kryo exception.", 1, numExceptions);
    +	}
    +
    +
    +	/**
    +	 * Verify that we can restore a snapshot that was done with without registered types
    +	 * after registering types.
    +	 */
    +	@Test
    +	public void testKryoRegisteringRestoreResilienceWithRegisteredType() throws Exception
{
    +		CheckpointStreamFactory streamFactory = createStreamFactory();
    +		Environment env = new DummyEnvironment("test", 1, 0);
    +		AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE,
env);
    +
    +		TypeInformation<TestPojo> pojoType = new GenericTypeInfo<>(TestPojo.class);
    +
    +		// make sure that we are in fact using the KryoSerializer
    +		assertTrue(pojoType.createSerializer(env.getExecutionConfig()) instanceof KryoSerializer);
    +
    +		pojoType.createSerializer(env.getExecutionConfig());
    +
    +		ValueStateDescriptor<TestPojo> kvId = new ValueStateDescriptor<>("id",
pojoType);
    +
    +		ValueState<TestPojo> state = backend.getPartitionedState(VoidNamespace.INSTANCE,
VoidNamespaceSerializer.INSTANCE, kvId);
    +
    +		// make some more modifications
    +		backend.setCurrentKey(1);
    +		state.update(new TestPojo("u1", 1));
    +
    +		backend.setCurrentKey(2);
    +		state.update(new TestPojo("u2", 2));
    +
    +		KeyGroupsStateHandle snapshot = runSnapshot(backend.snapshot(
    +				682375462378L,
    +				2,
    +				streamFactory,
    +				CheckpointOptions.forFullCheckpoint()));
    +
    +		backend.dispose();
    +
    +		env.getExecutionConfig().registerKryoType(TestPojo.class);
    +		
    +		backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot, env);
    +
    +		snapshot.discardState();
    +
    +		state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE,
kvId);
    +		backend.setCurrentKey(1);
    +		assertEquals(state.value(), new TestPojo("u1", 1));
    +
    +		backend.setCurrentKey(2);
    +		assertEquals(state.value(), new TestPojo("u2", 2));
    +
    +		backend.dispose();
    +	}
    +
    +	/**
    +	 * Verify that we can restore a snapshot that was done with without registered default
    --- End diff --
    
    "with without" redundancy --> "without", I think ;-)?



---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

Mime
View raw message