phoenix-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From GitBox <...@apache.org>
Subject [GitHub] twdsilva commented on a change in pull request #428: PHOENIX-374: Enable access to dynamic columns in * or cf.* selection
Date Mon, 04 Feb 2019 23:49:07 GMT
twdsilva commented on a change in pull request #428: PHOENIX-374: Enable access to dynamic
columns in * or cf.* selection
URL: https://github.com/apache/phoenix/pull/428#discussion_r253674762
 
 

 ##########
 File path: phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ScanRegionObserver.java
 ##########
 @@ -76,6 +97,87 @@ public static void serializeIntoScan(Scan scan, int thresholdBytes, int
limit, L
         }
     }
 
+    @Override
+    public void preBatchMutate(ObserverContext<RegionCoprocessorEnvironment> c,
+            MiniBatchOperationInProgress<Mutation> miniBatchOp) throws IOException
{
+        try {
+            preBatchMutateWithExceptions(miniBatchOp);
+        } catch(Throwable t) {
+            // Wrap all exceptions in an IOException to prevent region server crashes
+            throw ServerUtil.createIOException("Unable to Put cells corresponding to dynamic"
+
+                    "column metadata for " +
+                    c.getEnvironment().getRegion().getRegionInfo().getTable().getNameAsString(),
t);
+        }
+    }
+
+    /**
+     * In case we are supporting exposing dynamic columns for wildcard queries, which is
based on
+     * the client-side config
+     * {@link org.apache.phoenix.query.QueryServices#WILDCARD_QUERY_DYNAMIC_COLS_ATTRIB},
+     * we previously set attributes on the Put mutations where the key is the column family
and
+     * the value is the serialized list of dynamic columns.
+     * Here we iterate over all Put mutations and add metadata for the list of dynamic columns
for
+     * each column family in its own cell under reserved qualifiers. See PHOENIX-374
+     * @param miniBatchOp batch of mutations getting applied to region
+     * @throws IOException If an I/O error occurs when parsing protobuf
+     */
+    private void preBatchMutateWithExceptions(MiniBatchOperationInProgress<Mutation>
miniBatchOp)
+    throws IOException {
+        for (int i = 0; i < miniBatchOp.size(); i++) {
+            Mutation m = miniBatchOp.getOperation(i);
+            // There is at max 1 extra Put (for dynamic column shadow cells) per original
Put
+            Put dynColShadowCellsPut = null;
+            if (m instanceof Put && !Bytes.equals(m.getAttribute(
+                    NO_DYNAMIC_COLUMN_METADATA_STORED_FOR_MUTATION), TRUE_BYTES)) {
+                NavigableMap<byte[], List<Cell>> famCellMap = m.getFamilyCellMap();
+                for (byte[] fam : famCellMap.keySet()) {
+                    byte[] serializedDynColsList = m.getAttribute(Bytes.toString(fam));
+                    if (serializedDynColsList == null) {
+                        // There are no dynamic columns for this column family
+                        continue;
+                    }
+                    List<PTableProtos.PColumn> dynColsInThisFam = DynamicColumnMetaDataProtos.
+                            DynamicColumnMetaData.parseFrom(serializedDynColsList)
+                            .getDynamicColumnsList();
+                    if (dynColShadowCellsPut == null && !dynColsInThisFam.isEmpty())
{
+                        dynColShadowCellsPut = new Put(m.getRow());
+                    }
+                    for (PTableProtos.PColumn dynColProto : dynColsInThisFam) {
+                        // Add a column for this dynamic column to the metadata Put operation
+                        dynColShadowCellsPut.addColumn(fam,
+                                getQualifierForDynamicColumnMetaDataCell(dynColProto),
+                                dynColProto.toByteArray());
+                    }
+                }
+            }
+            if (dynColShadowCellsPut != null) {
+                miniBatchOp.addOperationsFromCP(i, new Mutation[]{dynColShadowCellsPut});
+            }
+        }
+    }
+
+    /**
+     * We store the metadata for each dynamic cell in a separate cell in the same column
family.
+     * The column qualifier for this cell is:
+     * {@link ScanRegionObserver#DYN_COLS_METADATA_CELL_QUALIFIER} concatenated with the
+     * qualifier of the actual dynamic column
+     * @param dynColProto Protobuf representation of the dynamic column PColumn
+     * @return Final qualifier for the metadata cell
+     * @throws IOException If an I/O error occurs when parsing the byte array output stream
+     */
+    private static byte[] getQualifierForDynamicColumnMetaDataCell(PTableProtos.PColumn dynColProto)
+    throws IOException {
+        PColumn dynCol = PColumnImpl.createFromProto(dynColProto);
+        ByteArrayOutputStream qual = new ByteArrayOutputStream();
+        qual.write(DYN_COLS_METADATA_CELL_QUALIFIER);
+        qual.write(dynCol.getColumnQualifierBytes());
+        if (LOG.isDebugEnabled()) {
 
 Review comment:
   This might end up being a lot of logs,  change it to the TRACE level.

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services

Mime
View raw message