incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From twilli...@apache.org
Subject git commit: An idea for a better way to deal with integration tests - allowing them to individually fail - using ExternalResource. We'd need some convention for tablenames that are unique for a test - I'm reckoning the method name itself. The backpress
Date Mon, 13 Jan 2014 01:40:44 GMT
Updated Branches:
  refs/heads/int-test-idea [created] 527de7df8


An idea for a better way to deal with integration tests - allowing them to individually fail
- using ExternalResource.  We'd need some convention for tablenames that are unique for a
test - I'm reckoning the method name itself.  The backpressure test fails with this since
I'm not sure how to fix it but you'll get the idea.  It's not fully fleshed out obviously,
but an idea for discussion.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/527de7df
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/527de7df
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/527de7df

Branch: refs/heads/int-test-idea
Commit: 527de7df8944550e587cb737d8dc5ce355d618cd
Parents: 85ebe0a
Author: twilliams <twilliams@apache.org>
Authored: Sun Jan 12 20:37:31 2014 -0500
Committer: twilliams <twilliams@apache.org>
Committed: Sun Jan 12 20:37:31 2014 -0500

----------------------------------------------------------------------
 .../blur/thrift/BlurIntegrationSuite.java       | 589 +++++++++++++++++++
 1 file changed, 589 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/527de7df/blur-core/src/test/java/org/apache/blur/thrift/BlurIntegrationSuite.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/thrift/BlurIntegrationSuite.java b/blur-core/src/test/java/org/apache/blur/thrift/BlurIntegrationSuite.java
new file mode 100644
index 0000000..a06960e
--- /dev/null
+++ b/blur-core/src/test/java/org/apache/blur/thrift/BlurIntegrationSuite.java
@@ -0,0 +1,589 @@
+package org.apache.blur.thrift;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryMXBean;
+import java.lang.management.MemoryUsage;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.blur.MiniCluster;
+import org.apache.blur.TestType;
+import org.apache.blur.analysis.FieldManager;
+import org.apache.blur.manager.IndexManager;
+import org.apache.blur.server.TableContext;
+import org.apache.blur.thirdparty.thrift_0_9_0.TException;
+import org.apache.blur.thrift.generated.Blur;
+import org.apache.blur.thrift.generated.Blur.Iface;
+import org.apache.blur.thrift.generated.BlurException;
+import org.apache.blur.thrift.generated.BlurQuery;
+import org.apache.blur.thrift.generated.BlurResult;
+import org.apache.blur.thrift.generated.BlurResults;
+import org.apache.blur.thrift.generated.ColumnDefinition;
+import org.apache.blur.thrift.generated.ErrorType;
+import org.apache.blur.thrift.generated.Facet;
+import org.apache.blur.thrift.generated.FetchResult;
+import org.apache.blur.thrift.generated.Query;
+import org.apache.blur.thrift.generated.RecordMutation;
+import org.apache.blur.thrift.generated.RowMutation;
+import org.apache.blur.thrift.generated.Schema;
+import org.apache.blur.thrift.generated.Selector;
+import org.apache.blur.thrift.generated.TableDescriptor;
+import org.apache.blur.thrift.util.BlurThriftHelper;
+import org.apache.blur.utils.GCWatcher;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.zookeeper.KeeperException;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.Rule;
+import org.junit.rules.ExternalResource;
+import org.junit.rules.TemporaryFolder;
+
+public class BlurIntegrationSuite {
+	private static final String TBL_1000 = "table1000";
+	
+	private static final File TMPDIR = new File(System.getProperty(
+			"blur.tmp.dir", "./target/tmp_BlurClusterTest"));
+	private static MiniCluster miniCluster;
+
+	private int numberOfDocs = 1000;
+
+	@Rule
+	public ExternalResource blurCluster = new ExternalResource() {
+
+		@Override
+		protected void before() throws Throwable {
+			GCWatcher.init(0.60);
+			
+			LocalFileSystem localFS = FileSystem.getLocal(new Configuration());
+			File testDirectory = new File(TMPDIR, "blur-cluster-test")
+					.getAbsoluteFile();
+			testDirectory.mkdirs();
+
+			Path directory = new Path(testDirectory.getPath());
+			FsPermission dirPermissions = localFS.getFileStatus(directory)
+					.getPermission();
+			FsAction userAction = dirPermissions.getUserAction();
+			FsAction groupAction = dirPermissions.getGroupAction();
+			FsAction otherAction = dirPermissions.getOtherAction();
+
+			StringBuilder builder = new StringBuilder();
+			builder.append(userAction.ordinal());
+			builder.append(groupAction.ordinal());
+			builder.append(otherAction.ordinal());
+			String dirPermissionNum = builder.toString();
+			System.setProperty("dfs.datanode.data.dir.perm", dirPermissionNum);
+			testDirectory.delete();
+			miniCluster = new MiniCluster();
+			miniCluster.startBlurCluster(
+					new File(testDirectory, "cluster").getAbsolutePath(), 2, 3,
+					true);
+			newTable(TBL_1000, 5);
+			loadTestData(TBL_1000, numberOfDocs);
+		}
+
+		@Override
+		protected void after() {
+			miniCluster.shutdownBlurCluster();
+		}
+	};
+
+	private static Iface getClient() {
+		return BlurClient.getClient(miniCluster.getControllerConnectionStr());
+	}
+
+	private void newTable(String tableName, int shardCount)
+			throws BlurException, TException, IOException {
+		Blur.Iface client = getClient();
+		TableDescriptor tableDescriptor = new TableDescriptor();
+		tableDescriptor.setName(tableName);
+		tableDescriptor.setShardCount(shardCount);
+		tableDescriptor.setTableUri(miniCluster.getFileSystemUri().toString()
+				+ "/blur/" + tableName);
+		client.createTable(tableDescriptor);
+	}
+	
+	private static void loadTestData(String tableName, int docCount) throws Exception {
+		Iface client = getClient();
+		int maxFacetValue = 100;
+		List<RowMutation> mutations = new ArrayList<RowMutation>();
+		Random random = new Random(1);
+		for (int i = 0; i < docCount; i++) {
+			String rowId = UUID.randomUUID().toString();
+			RecordMutation mutation = BlurThriftHelper.newRecordMutation(
+					"test",
+					rowId,
+					BlurThriftHelper.newColumn("test", "value"),
+					BlurThriftHelper.newColumn("facet",
+							Integer.toString(random.nextInt(maxFacetValue))));
+			RowMutation rowMutation = BlurThriftHelper.newRowMutation(tableName,
+					rowId, mutation);
+			rowMutation.setWaitToBeVisible(true);
+			mutations.add(rowMutation);
+		}
+		long s = System.nanoTime();
+		client.mutateBatch(mutations);
+		long e = System.nanoTime();
+		System.out.println("mutateBatch took [" + (e - s) / 1000000.0 + "]");
+	}
+
+	@Test
+	public void simpleCreateTable() throws BlurException, TException,
+			IOException {
+		newTable("simpleCreateTable", 1);
+
+		Blur.Iface client = getClient();
+
+		List<String> tableList = client.tableList();
+		assertTrue("We should get our table[" + tableList + "]",
+				tableList.contains("simpleCreateTable"));
+	}
+
+	@Test
+	public void testLoadTable() throws BlurException, TException,
+			InterruptedException, IOException {
+		Iface client = getClient();
+		
+		BlurQuery blurQueryRow = new BlurQuery();
+		Query queryRow = new Query();
+		queryRow.setQuery("test.test:value");
+		blurQueryRow.setQuery(queryRow);
+		blurQueryRow.setUseCacheIfPresent(false);
+		blurQueryRow.setCacheResult(false);
+		BlurResults resultsRow = client.query(TBL_1000, blurQueryRow);
+		assertRowResults(resultsRow);
+		assertEquals(numberOfDocs, resultsRow.getTotalResults());
+
+		BlurQuery blurQueryRecord = new BlurQuery();
+		Query queryRecord = new Query();
+		queryRecord.rowQuery = false;
+		queryRecord.setQuery("test.test:value");
+		blurQueryRecord.setQuery(queryRecord);
+		BlurResults resultsRecord = client.query(TBL_1000, blurQueryRecord);
+		assertRecordResults(resultsRecord);
+		assertEquals(numberOfDocs, resultsRecord.getTotalResults());
+
+		Schema schema = client.schema(TBL_1000);
+		assertFalse(schema.getFamilies().isEmpty());
+	}
+
+	@Ignore
+	@Test
+	public void testForEmptySchema() throws BlurException, TException,
+			IOException {
+		Blur.Iface client = getClient();
+		Schema schema = client.schema(TBL_1000);
+		Map<String, Map<String, ColumnDefinition>> families = schema
+				.getFamilies();
+		assertTrue(!families.isEmpty());
+		int size = families.size();
+		System.out.println(size);
+
+		TableContext tableContext = TableContext
+				.create(client.describe(TBL_1000));
+		FieldManager fieldManager = tableContext.getFieldManager();
+
+		assertTrue(fieldManager.addColumnDefinition("test-family",
+				"test-column", null, false, "string", null));
+
+		TableContext.clear();
+		Schema newschema = client.schema(TBL_1000);
+		Map<String, Map<String, ColumnDefinition>> newfamilies = newschema
+				.getFamilies();
+		assertTrue(!newfamilies.isEmpty());
+		int newsize = newfamilies.size();
+		assertEquals(size + 1, newsize);
+	}
+
+	@Test
+	public void testCreateTableWithCustomType() throws IOException,
+			BlurException, TException {
+		Blur.Iface client = getClient();
+		TableDescriptor tableDescriptor = new TableDescriptor();
+		tableDescriptor.setName("test_type");
+		tableDescriptor.setShardCount(1);
+		tableDescriptor.setTableUri(miniCluster.getFileSystemUri().toString()
+				+ "/blur/test_type");
+		client.createTable(tableDescriptor);
+		List<String> tableList = client.tableList();
+		assertTrue(tableList.contains("test_type"));
+
+		client.disableTable("test_type");
+
+		client.enableTable("test_type");
+
+		TableDescriptor describe = client.describe("test_type");
+		Map<String, String> tableProperties = describe.getTableProperties();
+		assertEquals(TestType.class.getName(),
+				tableProperties.get("blur.fieldtype.customtype1"));
+	}
+
+	@Test
+	public void testQueryWithSelector() throws BlurException, TException, IOException {
+		Iface client = getClient();
+		BlurQuery blurQueryRow = new BlurQuery();
+		Query queryRow = new Query();
+		queryRow.setQuery("test.test:value");
+		blurQueryRow.setQuery(queryRow);
+		blurQueryRow.setUseCacheIfPresent(false);
+		blurQueryRow.setCacheResult(false);
+		blurQueryRow.setSelector(new Selector());
+
+		BlurResults resultsRow = client.query(TBL_1000, blurQueryRow);
+		// assertRowResults(resultsRow);
+		assertEquals(numberOfDocs, resultsRow.getTotalResults());
+
+		for (BlurResult blurResult : resultsRow.getResults()) {
+			System.out.println(blurResult);
+		}
+
+	}
+
+	@Test
+	public void testQueryWithFacets() throws BlurException, TException, IOException {
+		Iface client = getClient();
+		BlurQuery blurQueryRow = new BlurQuery();
+		Query queryRow = new Query();
+		// queryRow.setQuery("test.test:value");
+		queryRow.setQuery("*");
+		blurQueryRow.setQuery(queryRow);
+		blurQueryRow.setUseCacheIfPresent(false);
+		blurQueryRow.setCacheResult(false);
+		blurQueryRow.setSelector(new Selector());
+		for (int i = 0; i < 250; i++) {
+			blurQueryRow.addToFacets(new Facet("test.facet:" + i,
+					Long.MAX_VALUE));
+		}
+
+		BlurResults resultsRow = client.query(TBL_1000, blurQueryRow);
+		// assertRowResults(resultsRow);
+		assertEquals(numberOfDocs, resultsRow.getTotalResults());
+
+		System.out.println(resultsRow.getFacetCounts());
+
+		System.out.println();
+
+	}
+
+	@Test
+	public void testBatchFetch() throws BlurException, TException, IOException {
+
+		final Iface client = getClient();
+		List<String> terms = client.terms(TBL_1000, null, "rowid", "",
+				(short) 100);
+
+		List<Selector> selectors = new ArrayList<Selector>();
+		for (String s : terms) {
+			Selector selector = new Selector();
+			selector.setRowId(s);
+			selectors.add(selector);
+		}
+
+		List<FetchResult> fetchRowBatch = client.fetchRowBatch(TBL_1000,
+				selectors);
+		assertEquals(100, fetchRowBatch.size());
+
+		int i = 0;
+		for (FetchResult fetchResult : fetchRowBatch) {
+			assertEquals(terms.get(i), fetchResult.getRowResult().getRow()
+					.getId());
+			i++;
+		}
+
+	}
+
+	@Test
+	public void testQueryCancel() throws BlurException, TException,
+			InterruptedException {
+		// This will make each collect in the collectors pause 250 ms per
+		// collect
+		// call
+		IndexManager.DEBUG_RUN_SLOW.set(true);
+
+		final Iface client = getClient();
+		final BlurQuery blurQueryRow = new BlurQuery();
+		Query queryRow = new Query();
+		queryRow.setQuery("test.test:value");
+		blurQueryRow.setQuery(queryRow);
+		blurQueryRow.setUseCacheIfPresent(false);
+		blurQueryRow.setCacheResult(false);
+		blurQueryRow.setUuid("1234");
+
+		final AtomicReference<BlurException> error = new AtomicReference<BlurException>();
+		final AtomicBoolean fail = new AtomicBoolean();
+
+		new Thread(new Runnable() {
+			@Override
+			public void run() {
+				try {
+					// This call will take several seconds to execute.
+					client.query(TBL_1000, blurQueryRow);
+					fail.set(true);
+				} catch (BlurException e) {
+					error.set(e);
+				} catch (TException e) {
+					e.printStackTrace();
+					fail.set(true);
+				}
+			}
+		}).start();
+		Thread.sleep(500);
+		client.cancelQuery(TBL_1000, blurQueryRow.getUuid());
+		BlurException blurException = pollForError(error, 10, TimeUnit.SECONDS,
+				null, fail, -1);
+		if (fail.get()) {
+			fail("Unknown error, failing test.");
+		}
+		assertEquals(ErrorType.QUERY_CANCEL, blurException.getErrorType());
+	}
+
+	@Test
+	public void testBackPressureViaQuery() throws BlurException, TException,
+			InterruptedException {
+		// This will make each collect in the collectors pause 250 ms per
+		// collect
+		// call
+		IndexManager.DEBUG_RUN_SLOW.set(true);
+		runBackPressureViaQuery();
+		Thread.sleep(1000);
+		System.gc();
+		System.gc();
+		Thread.sleep(1000);
+	}
+
+	private void runBackPressureViaQuery() throws InterruptedException {
+		final Iface client = getClient();
+		final BlurQuery blurQueryRow = new BlurQuery();
+		Query queryRow = new Query();
+		queryRow.setQuery("test.test:value");
+		blurQueryRow.setQuery(queryRow);
+		blurQueryRow.setUseCacheIfPresent(false);
+		blurQueryRow.setCacheResult(false);
+		blurQueryRow.setUuid("1234");
+
+		final AtomicReference<BlurException> error = new AtomicReference<BlurException>();
+		final AtomicBoolean fail = new AtomicBoolean();
+
+		System.gc();
+		System.gc();
+		MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean();
+		MemoryUsage usage = memoryMXBean.getHeapMemoryUsage();
+		long max = usage.getMax();
+		System.out.println("Max Heap [" + max + "]");
+		long used = usage.getUsed();
+		System.out.println("Used Heap [" + used + "]");
+		long limit = (long) (max * 0.80);
+		System.out.println("Limit Heap [" + limit + "]");
+		long difference = limit - used;
+		int sizeToAllocate = (int) ((int) difference * 0.50);
+		System.out.println("Allocating [" + sizeToAllocate + "] Heap ["
+				+ getHeapSize() + "] Max [" + getMaxHeapSize() + "]");
+
+		byte[] bufferToFillHeap = new byte[sizeToAllocate];
+		new Thread(new Runnable() {
+			@Override
+			public void run() {
+				try {
+					// This call will take several seconds to execute.
+					client.query(TBL_1000, blurQueryRow);
+					fail.set(true);
+				} catch (BlurException e) {
+					System.out.println("-------------------");
+					System.out.println("-------------------");
+					System.out.println("-------------------");
+					e.printStackTrace();
+					System.out.println("-------------------");
+					System.out.println("-------------------");
+					System.out.println("-------------------");
+					error.set(e);
+				} catch (TException e) {
+					e.printStackTrace();
+					fail.set(true);
+				}
+			}
+		}).start();
+		Thread.sleep(500);
+		List<byte[]> bufferToPutGcWatcherOverLimitList = new ArrayList<byte[]>();
+		BlurException blurException = pollForError(error, 120,
+				TimeUnit.SECONDS, bufferToPutGcWatcherOverLimitList, fail,
+				(int) (difference / 7));
+		if (fail.get()) {
+			fail("Unknown error, failing test.");
+		}
+		System.out.println(bufferToFillHeap.hashCode());
+		System.out.println(bufferToPutGcWatcherOverLimitList.hashCode());
+		assertEquals(ErrorType.BACK_PRESSURE, blurException.getErrorType() );
+		bufferToPutGcWatcherOverLimitList.clear();
+		bufferToPutGcWatcherOverLimitList = null;
+		bufferToFillHeap = null;
+	}
+
+	private BlurException pollForError(AtomicReference<BlurException> error,
+			long period, TimeUnit timeUnit,
+			List<byte[]> bufferToPutGcWatcherOverLimitList, AtomicBoolean fail,
+			int sizeToAllocate) throws InterruptedException {
+		long s = System.nanoTime();
+		long totalTime = timeUnit.toNanos(period) + s;
+		if (bufferToPutGcWatcherOverLimitList != null) {
+			System.out.println("Allocating [" + sizeToAllocate + "] Heap ["
+					+ getHeapSize() + "] Max [" + getMaxHeapSize() + "]");
+			bufferToPutGcWatcherOverLimitList.add(new byte[sizeToAllocate]);
+		}
+		while (totalTime > System.nanoTime()) {
+			if (fail.get()) {
+				fail("The query failed.");
+			}
+			BlurException blurException = error.get();
+			if (blurException != null) {
+				return blurException;
+			}
+			Thread.sleep(100);
+			if (bufferToPutGcWatcherOverLimitList != null) {
+				if (getHeapSize() < (getMaxHeapSize() * 0.8)) {
+					System.out.println("Allocating [" + sizeToAllocate
+							+ "] Heap [" + getHeapSize() + "] Max ["
+							+ getMaxHeapSize() + "]");
+					bufferToPutGcWatcherOverLimitList
+							.add(new byte[sizeToAllocate]);
+				} else {
+					System.gc();
+					System.gc();
+					System.out.println("Already allocated enough Heap ["
+							+ getHeapSize() + "] Max [" + getMaxHeapSize()
+							+ "]");
+				}
+			}
+		}
+		return null;
+	}
+
+	private long getHeapSize() {
+		return ManagementFactory.getMemoryMXBean().getHeapMemoryUsage()
+				.getUsed();
+	}
+
+	private long getMaxHeapSize() {
+		return ManagementFactory.getMemoryMXBean().getHeapMemoryUsage()
+				.getMax();
+	}
+
+	@Test
+	public void testTestShardFailover() throws BlurException, TException,
+			InterruptedException, IOException, KeeperException {
+
+		Iface client = getClient();
+		BlurQuery blurQuery = new BlurQuery();
+		blurQuery.setUseCacheIfPresent(false);
+		Query query = new Query();
+		query.setQuery("test.test:value");
+		blurQuery.setQuery(query);
+		BlurResults results1 = client.query(TBL_1000, blurQuery);
+		assertEquals(numberOfDocs, results1.getTotalResults());
+		assertRowResults(results1);
+
+		miniCluster.killShardServer(1);
+
+		// make sure the WAL syncs
+		Thread.sleep(TimeUnit.SECONDS.toMillis(1));
+
+		// This should block until shards have failed over
+		client.shardServerLayout(TBL_1000);
+
+		assertEquals(numberOfDocs, client.query(TBL_1000, blurQuery)
+				.getTotalResults());
+
+	}
+
+	@Test
+	public void testTermsList() throws BlurException, TException {
+		Iface client = getClient();
+		List<String> terms = client.terms(TBL_1000, "test", "test", null,
+				(short) 10);
+		List<String> list = new ArrayList<String>();
+		list.add("value");
+		assertEquals(list, terms);
+	}
+
+	private void assertRowResults(BlurResults results) {
+		for (BlurResult result : results.getResults()) {
+			assertNull(result.locationId);
+			assertNull(result.fetchResult.recordResult);
+			assertNull(result.fetchResult.rowResult.row.records);
+			assertNotNull(result.fetchResult.rowResult.row.id);
+		}
+	}
+
+	private void assertRecordResults(BlurResults results) {
+		for (BlurResult result : results.getResults()) {
+			assertNull(result.locationId);
+			assertNotNull(result.fetchResult.recordResult);
+			assertNotNull(result.fetchResult.recordResult.rowid);
+			assertNotNull(result.fetchResult.recordResult.record.recordId);
+			assertNotNull(result.fetchResult.recordResult.record.family);
+			assertNull("Not null ["
+					+ result.fetchResult.recordResult.record.columns + "]",
+					result.fetchResult.recordResult.record.columns);
+			assertNull(result.fetchResult.rowResult);
+		}
+	}
+
+	@Test
+	public void testCreateDisableAndRemoveTable() throws IOException,
+			BlurException, TException {
+		Iface client = getClient();
+		String tableName = UUID.randomUUID().toString();
+		TableDescriptor tableDescriptor = new TableDescriptor();
+		tableDescriptor.setName(tableName);
+		tableDescriptor.setShardCount(5);
+		tableDescriptor.setTableUri(miniCluster.getFileSystemUri().toString()
+				+ "/blur/" + tableName);
+
+		for (int i = 0; i < 3; i++) {
+			client.createTable(tableDescriptor);
+			client.disableTable(tableName);
+			client.removeTable(tableName, true);
+		}
+
+		assertFalse(client.tableList().contains(tableName));
+
+	}
+}


Mime
View raw message