asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From buyin...@apache.org
Subject [16/28] asterixdb git commit: Introduce IStorageComponentProvider
Date Thu, 02 Feb 2017 18:24:25 GMT
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeDataflowHelperFactoryProvider.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeDataflowHelperFactoryProvider.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeDataflowHelperFactoryProvider.java
new file mode 100644
index 0000000..0560bd0
--- /dev/null
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeDataflowHelperFactoryProvider.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.metadata.declared;
+
+import java.util.Map;
+
+import org.apache.asterix.common.context.AsterixVirtualBufferCacheProvider;
+import org.apache.asterix.common.context.IStorageComponentProvider;
+import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.external.indexing.IndexingConstants;
+import org.apache.asterix.metadata.api.IIndexDataflowHelperFactoryProvider;
+import org.apache.asterix.metadata.entities.Dataset;
+import org.apache.asterix.metadata.entities.Index;
+import org.apache.asterix.metadata.utils.ExternalDatasetsRegistry;
+import org.apache.asterix.metadata.utils.IndexUtil;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
+import org.apache.hyracks.storage.am.lsm.btree.dataflow.ExternalBTreeDataflowHelperFactory;
+import org.apache.hyracks.storage.am.lsm.btree.dataflow.ExternalBTreeWithBuddyDataflowHelperFactory;
+import org.apache.hyracks.storage.am.lsm.btree.dataflow.LSMBTreeDataflowHelperFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
+
+public class BTreeDataflowHelperFactoryProvider implements IIndexDataflowHelperFactoryProvider
{
+
+    public static final BTreeDataflowHelperFactoryProvider INSTANCE = new BTreeDataflowHelperFactoryProvider();
+
+    private BTreeDataflowHelperFactoryProvider() {
+    }
+
+    public static String externalFileIndexName(Dataset dataset) {
+        return dataset.getDatasetName().concat(IndexingConstants.EXTERNAL_FILE_INDEX_NAME_SUFFIX);
+    }
+
+    @Override
+    public IIndexDataflowHelperFactory getIndexDataflowHelperFactory(MetadataProvider mdProvider,
Dataset dataset,
+            Index index, ARecordType recordType, ARecordType metaType, ILSMMergePolicyFactory
mergePolicyFactory,
+            Map<String, String> mergePolicyProperties, ITypeTraits[] filterTypeTraits,
+            IBinaryComparatorFactory[] filterCmpFactories) throws AlgebricksException {
+        int[] filterFields = IndexUtil.getFilterFields(dataset, index, filterTypeTraits);
+        int[] btreeFields = IndexUtil.getBtreeFieldsIfFiltered(dataset, index);
+        IStorageComponentProvider storageComponentProvider = mdProvider.getStorageComponentProvider();
+        switch (dataset.getDatasetType()) {
+            case EXTERNAL:
+                return index.getIndexName().equals(externalFileIndexName(dataset))
+                        ? new ExternalBTreeDataflowHelperFactory(mergePolicyFactory, mergePolicyProperties,
+                                dataset.getIndexOperationTrackerFactory(index),
+                                storageComponentProvider.getIoOperationSchedulerProvider(),
+                                dataset.getIoOperationCallbackFactory(index),
+                                mdProvider.getStorageProperties().getBloomFilterFalsePositiveRate(),
+                                ExternalDatasetsRegistry.INSTANCE.getAndLockDatasetVersion(dataset,
mdProvider),
+                                !dataset.getDatasetDetails().isTemp())
+                        : new ExternalBTreeWithBuddyDataflowHelperFactory(mergePolicyFactory,
mergePolicyProperties,
+                                dataset.getIndexOperationTrackerFactory(index),
+                                storageComponentProvider.getIoOperationSchedulerProvider(),
+                                dataset.getIoOperationCallbackFactory(index),
+                                mdProvider.getStorageProperties().getBloomFilterFalsePositiveRate(),
+                                new int[] { index.getKeyFieldNames().size() },
+                                ExternalDatasetsRegistry.INSTANCE.getAndLockDatasetVersion(dataset,
mdProvider),
+                                !dataset.getDatasetDetails().isTemp());
+            case INTERNAL:
+                return new LSMBTreeDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()),
+                        mergePolicyFactory, mergePolicyProperties, dataset.getIndexOperationTrackerFactory(index),
+                        storageComponentProvider.getIoOperationSchedulerProvider(),
+                        dataset.getIoOperationCallbackFactory(index),
+                        mdProvider.getStorageProperties().getBloomFilterFalsePositiveRate(),
index.isPrimaryIndex(),
+                        filterTypeTraits, filterCmpFactories, btreeFields, filterFields,
+                        !dataset.getDatasetDetails().isTemp());
+            default:
+                throw new CompilationException(ErrorCode.COMPILATION_UNKNOWN_DATASET_TYPE,
+                        dataset.getDatasetType().toString());
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
index 973651f..68c7e22 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
@@ -28,7 +28,7 @@ import org.apache.asterix.metadata.entities.Dataset;
 import org.apache.asterix.metadata.entities.ExternalDatasetDetails;
 import org.apache.asterix.metadata.entities.Index;
 import org.apache.asterix.metadata.entities.InternalDatasetDetails;
-import org.apache.asterix.metadata.utils.KeyFieldTypeUtils;
+import org.apache.asterix.metadata.utils.KeyFieldTypeUtil;
 import org.apache.asterix.om.types.ARecordType;
 import org.apache.asterix.om.types.IAType;
 import org.apache.asterix.runtime.formats.NonTaggedDataFormat;
@@ -73,7 +73,7 @@ public class DatasetDataSource extends DataSource {
         ARecordType recordType = (ARecordType) itemType;
         ARecordType metaRecordType = (ARecordType) metaItemType;
         List<IAType> partitioningKeyTypes =
-                KeyFieldTypeUtils.getPartitioningKeyTypes(internalDatasetDetails, recordType,
metaRecordType);
+                KeyFieldTypeUtil.getPartitioningKeyTypes(internalDatasetDetails, recordType,
metaRecordType);
         int n = partitioningKeyTypes.size();
         schemaTypes = metaItemType == null ? new IAType[n + 1] : new IAType[n + 2];
         for (int keyIndex = 0; keyIndex < n; ++keyIndex) {
@@ -106,7 +106,7 @@ public class DatasetDataSource extends DataSource {
 
                 ExternalDatasetDetails edd = (ExternalDatasetDetails) externalDataset.getDatasetDetails();
                 IAdapterFactory adapterFactory = metadataProvider.getConfiguredAdapterFactory(externalDataset,
-                        edd.getAdapter(), edd.getProperties(), (ARecordType) itemType, false,
null, null);
+                        edd.getAdapter(), edd.getProperties(), (ARecordType) itemType, null);
                 return metadataProvider.buildExternalDatasetDataScannerRuntime(jobSpec, itemType,
adapterFactory,
                         NonTaggedDataFormat.INSTANCE);
             case INTERNAL:
@@ -118,9 +118,9 @@ public class DatasetDataSource extends DataSource {
 
                 int[] minFilterFieldIndexes = createFilterIndexes(minFilterVars, opSchema);
                 int[] maxFilterFieldIndexes = createFilterIndexes(maxFilterVars, opSchema);
-                return metadataProvider.buildBtreeRuntime(jobSpec, scanVariables, opSchema,
typeEnv, context, true,
+                return metadataProvider.buildBtreeRuntime(jobSpec, opSchema, typeEnv, context,
true,
                         false, ((DatasetDataSource) dataSource).getDataset(), primaryIndex.getIndexName(),
null, null,
-                        true, true, implConfig, minFilterFieldIndexes, maxFilterFieldIndexes);
+                        true, true, minFilterFieldIndexes, maxFilterFieldIndexes);
             default:
                 throw new AlgebricksException("Unknown datasource type");
         }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java
index 703f8c8..0d3d06d 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java
@@ -33,7 +33,7 @@ import org.apache.asterix.metadata.feeds.BuiltinFeedPolicies;
 import org.apache.asterix.om.types.ARecordType;
 import org.apache.asterix.om.types.IAType;
 import org.apache.asterix.runtime.formats.NonTaggedDataFormat;
-import org.apache.asterix.runtime.util.ClusterStateManager;
+import org.apache.asterix.runtime.utils.ClusterStateManager;
 import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
 import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
 import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/LoadableDataSource.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/LoadableDataSource.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/LoadableDataSource.java
index 5b5844c..cdf699d 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/LoadableDataSource.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/LoadableDataSource.java
@@ -27,7 +27,7 @@ import org.apache.asterix.common.exceptions.AsterixException;
 import org.apache.asterix.external.api.IAdapterFactory;
 import org.apache.asterix.metadata.entities.Dataset;
 import org.apache.asterix.metadata.entities.InternalDatasetDetails;
-import org.apache.asterix.metadata.utils.DatasetUtils;
+import org.apache.asterix.metadata.utils.DatasetUtil;
 import org.apache.asterix.om.types.ARecordType;
 import org.apache.asterix.om.types.ATypeTag;
 import org.apache.asterix.om.types.IAType;
@@ -60,14 +60,14 @@ public class LoadableDataSource extends DataSource {
         this.targetDataset = targetDataset;
         this.adapter = adapter;
         this.adapterProperties = properties;
-        partitioningKeys = DatasetUtils.getPartitioningKeys(targetDataset);
+        partitioningKeys = DatasetUtil.getPartitioningKeys(targetDataset);
         ARecordType recType = (ARecordType) itemType;
         isPKAutoGenerated = ((InternalDatasetDetails) targetDataset.getDatasetDetails()).isAutogenerated();
         if (isPKAutoGenerated) {
             // Since the key is auto-generated, we need to use another
             // record type (possibly nested) which has all fields except the PK
             try {
-                recType = getStrippedPKType(new LinkedList<String>(partitioningKeys.get(0)),
recType);
+                recType = getStrippedPKType(new LinkedList<>(partitioningKeys.get(0)),
recType);
             } catch (AsterixException e) {
                 throw new AlgebricksException(e);
             }
@@ -81,7 +81,7 @@ public class LoadableDataSource extends DataSource {
         List<IAType> fieldTypes = new LinkedList<>();
         int j = 0;
         for (int i = 0; i < recType.getFieldNames().length; i++) {
-            IAType fieldType = null;
+            IAType fieldType;
             if (partitioningKeys.get(0).equals(recType.getFieldNames()[j])) {
                 if (recType.getFieldTypes()[j].getTypeTag() == ATypeTag.RECORD) {
                     if (j != 0) {
@@ -132,15 +132,10 @@ public class LoadableDataSource extends DataSource {
             List<LogicalVariable> minFilterVars, List<LogicalVariable> maxFilterVars,
IOperatorSchema opSchema,
             IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec,
Object implConfig)
             throws AlgebricksException {
-        // This is a load into dataset operation
         LoadableDataSource alds = (LoadableDataSource) dataSource;
-        List<List<String>> aldsPartitioningKeys = alds.getPartitioningKeys();
-        boolean isAldsPKAutoGenerated = ((InternalDatasetDetails) alds.getTargetDataset().getDatasetDetails())
-                .isAutogenerated();
         ARecordType itemType = (ARecordType) alds.getLoadedType();
         IAdapterFactory adapterFactory = metadataProvider.getConfiguredAdapterFactory(alds.getTargetDataset(),
-                alds.getAdapter(), alds.getAdapterProperties(), itemType, isAldsPKAutoGenerated,
aldsPartitioningKeys,
-                null);
+                alds.getAdapter(), alds.getAdapterProperties(), itemType, null);
         RecordDescriptor rDesc = JobGenHelper.mkRecordDescriptor(typeEnv, opSchema, context);
         return metadataProvider.buildLoadableDatasetScan(jobSpec, adapterFactory, rDesc);
     }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataManagerUtil.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataManagerUtil.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataManagerUtil.java
index 536a688..f1a90c7 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataManagerUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataManagerUtil.java
@@ -35,7 +35,7 @@ import org.apache.asterix.metadata.entities.NodeGroup;
 import org.apache.asterix.metadata.utils.MetadataConstants;
 import org.apache.asterix.om.types.ARecordType;
 import org.apache.asterix.om.types.IAType;
-import org.apache.asterix.runtime.util.ClusterStateManager;
+import org.apache.asterix.runtime.utils.ClusterStateManager;
 import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
 import org.apache.hyracks.algebricks.core.algebra.properties.DefaultNodeGroupDomain;
 import org.apache.hyracks.algebricks.core.algebra.properties.INodeDomain;


Mime
View raw message