hbase-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Jithender Boreddy <jithen1...@gmail.com>
Subject Re: Getting intermittent errors while insertind data into HBase
Date Mon, 25 May 2015 11:39:41 GMT
Hi,
Thanks for the replies. Sorry for late response from my side as I was on
vacation.

My HBase version: HBase 0.98.6-cdh5.3.1

We are interacting with HBase from our rest webservices which we developed
using Spring Jersey. Hence Spring and DAO.

Below I have pasted the code where we are trying to create the connection
objects with some overridden parameters. Also placed the service code flow
where we try to create sensor readings. If you go through the code you
might be seeing primary and seconday cluster connection objects and that is
because we are using HBase replication and hence there are primary and
secondary connections in the code and secondary is used when primary fails.
But in our case primary cluster is healthy and accessible.

We are overriding few default configurations as below while creating
connection. Will this has any side affect?
HBASE.CLIENT.PAUSE=50
HBASE.CLIENT.RETRIES.NUMBER=1
ZOOKEEPER.RECOVERY.RETRY=3
ZOOKEEPER.RECOVERY.RETRY.INTERVALMILL=1000

*and the connection object is created like below. We are intializing the
connections while tomcat startup.*

    static {

        rowKeyDelimiter =
PropUtils.getProperty(ADConstants.ROWKEY_DELIMITER);

        try {
            SensorDataDAO.primaryClusterConnection = getConnection(true);
        } catch (IOException e) {
            ExceptionUtils.logException("", "", "", e);
        }

        try {
            backupClusterConnection = getConnection(false);
        } catch (IOException e) {
            ExceptionUtils.logException("", "", "", e);
        }
    }

 private static HConnection getConnection(boolean connectToPrimary) throws
IOException {
        Configuration conf = HBaseConfiguration.create();
        if (connectToPrimary) {

conf.addResource(PropUtils.getProperty(ADConstants.PRIMARY_CLUSTER_HBASE_SITE_FILE_NAME));

conf.addResource(PropUtils.getProperty(ADConstants.PRIMARY_CLUSTER_CORE_SITE_FILE_NAME));

conf.addResource(PropUtils.getProperty(ADConstants.PRIMARY_CLUSTER_HDFS_SITE_FILE_NAME));

            conf.set(ADConstants.HBASE_CLIENT_PAUSE.toLowerCase(),
                     PropUtils.getProperty(ADConstants.HBASE_CLIENT_PAUSE));

            conf.set(ADConstants.HBASE_CLIENT_RETRIES_NUMBER.toLowerCase(),

 PropUtils.getProperty(ADConstants.HBASE_CLIENT_RETRIES_NUMBER));

            conf.set(ADConstants.ZOOKEEPER_RECOVERY_RETRY.toLowerCase(),

 PropUtils.getProperty(ADConstants.ZOOKEEPER_RECOVERY_RETRY));

conf.set(ADConstants.ZOOKEEPER_RECOVERY_RETRY_INTERVALMILL.toLowerCase(),

 PropUtils.getProperty(ADConstants.ZOOKEEPER_RECOVERY_RETRY_INTERVALMILL));

        } else {

conf.addResource(PropUtils.getProperty(ADConstants.BACKUP_CLUSTER_HBASE_SITE_FILE_NAME));

conf.addResource(PropUtils.getProperty(ADConstants.BACKUP_CLUSTER_CORE_SITE_FILE_NAME));

conf.addResource(PropUtils.getProperty(ADConstants.BACKUP_CLUSTER_HDFS_SITE_FILE_NAME));

            conf.set(ADConstants.HBASE_CLIENT_PAUSE.toLowerCase(),
                     PropUtils.getProperty(ADConstants.HBASE_CLIENT_PAUSE));

            conf.set(ADConstants.HBASE_CLIENT_RETRIES_NUMBER.toLowerCase(),

 PropUtils.getProperty(ADConstants.HBASE_CLIENT_RETRIES_NUMBER));

            conf.set(ADConstants.ZOOKEEPER_RECOVERY_RETRY.toLowerCase(),

 PropUtils.getProperty(ADConstants.ZOOKEEPER_RECOVERY_RETRY));

conf.set(ADConstants.ZOOKEEPER_RECOVERY_RETRY_INTERVALMILL.toLowerCase(),

 PropUtils.getProperty(ADConstants.ZOOKEEPER_RECOVERY_RETRY_INTERVALMILL));
        }

        return HConnectionManager.createConnection(conf);
    }

*Service Class: Below is our service class where it starts serving the
client request.*
/**
 * Service Class to handle all the REST WebService requests for the Reading.

@Path("/reading")
public class ReadingService extends SensorDataService {

    public ReadingProcessor getProcessor() {
        return (ReadingProcessor) getContext().getBean("readingProcessor");
    }

    /**
     * WebService EndPoint to create the Reading. If reading is created
     * successfully , reading Information is returned. Supported request
     * formats: XML,JSON, Protobuf
     *
     * @param format
     * @param authorizationHeader
     * @param uri
     * @param readingCreate
     * @param xForwardedProto
     * @return
     * @throws ADException
     */
    @Timer
    @POST
    @Path("/v1/create.{format: (protobuf|xml|json)}")
    @Consumes({ MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON })
    @Produces({ MediaType.APPLICATION_XML,
               MediaType.APPLICATION_JSON,
               ADConstants.APPLICATION_PROTOBUF_MEDIA_TYPE })
    public Response createReading(@PathParam("format") String format,
                                  @HeaderParam("authorization") String
authorizationHeader,
                                  @Context UriInfo uri,
                                  ReadingCreateRequestV1 readingCreate,
                                  @HeaderParam("X-Forwarded-Proto") String
xForwardedProto,
                                  @HeaderParam("userId") String userId,
                                  @HeaderParam("requestGUID") String
requestGUID) throws ADException {

        String operationType = "create-readings-using-JAXB-object";
        try {
            this.validateCreateRequestPayload(readingCreate);
            String readings =
this.buildReadingString(readingCreate.getReadingCreateList());
            Object obj = this.createReadings(format,
                                             readingCreate.getCompanyId(),
                                             readingCreate.getSiteId(),
                                             readingCreate.getBuildingId(),
                                             readings,
                                             uri,
                                             operationType,
                                             requestGUID,
                                             userId);
            Format responseFormat = Format.validateFormat(format,
operationType, requestGUID, userId);
            return
Response.status(OK_RESPONSE).entity(obj).type(responseFormat.toString()).build();
        } catch (ADException ade) {
            throw ade;
        } catch (Exception e) {
            LogUtils.error(operationType,
                           requestGUID,
                           userId,
                           ERROR_OCCURED,
                           new String[] { e.getMessage(),
ExceptionUtils.getStackTrace(e) });
            throw new ADException(INTERNAL_ERROR);
        }
    }

    private void validateCreateRequestPayload(ReadingCreateRequestV1
request) {
        if (request == null) {
            throw new ADException(ADConstants.INVALID_PAYLOAD);
        }
        if (request.getReadingCreateList() == null ||
request.getReadingCreateList().size() == 0) {
            throw new ADException(ADConstants.INVALID_PAYLOAD);
        }
    }


    /**
     * Method to build the reading string used to create the reading object.
     *
     * @param readingList
     * @return
     */
    private String buildReadingString(List<ReadingCreateInfoV1>
readingCreateList) {
        StringBuffer readingString = new StringBuffer();
        for (int i = 0; i < readingCreateList.size(); i++) {
            ReadingCreateInfoV1 reading = readingCreateList.get(i);
            readingString.append(reading.getPointBasId())
                         .append("\t")
                         .append(reading.getDatetimeUtc())
                         .append("\t")
                         .append(reading.getValue());
            if (i != readingCreateList.size() - 1) {
                readingString.append(System.getProperty("line.separator"));
            }
        }
        return readingString.toString();
    }

    private Object createReadings(String format,
                                  String companyId,
                                  String siteId,
                                  String buildingId,
                                  String readings,
                                  UriInfo uri,
                                  String operationType,
                                  String requestGUID,
                                  String userId) throws ADException,
UnsupportedEncodingException {
        String[] params = { uri.getRequestUri().toString(),
                           "company ID : " + companyId + ", site ID : " +
siteId + ", building ID : "
                                           + buildingId + "\nreadings : \n"
+ readings + "\n",
                           format };
        LogUtils.info(operationType,
                      requestGUID,
                      userId,
                      PropUtils.getProperty("REQUEST.PARAMS.READING",
params));

        format = checkFormat(format, operationType, requestGUID, userId);

        LogUtils.debug(operationType, requestGUID, userId, "building ID :"
+ buildingId);
        LogUtils.debug(operationType, requestGUID, userId, "site ID : " +
siteId);
        LogUtils.debug(operationType, requestGUID, userId, "company ID : "
+ companyId);

        // check for mandatory parameters
        buildingId = validateParam(buildingId, "building_id",
operationType, requestGUID, userId);
        siteId = validateParam(siteId, "site_id", operationType,
requestGUID, userId);
        companyId = validateParam(companyId, "company_id", operationType,
requestGUID, userId);
        readings = validateParam(readings, "readings", operationType,
requestGUID, userId);

        ReadingProcessor processor = getProcessor();

        // make a call to business class to create Points
        return processor.createReadings(operationType,
                                        requestGUID,
                                        userId,
                                        companyId,
                                        siteId,
                                        buildingId,
                                        readings,
                                        userId,
                                        format);
    }

  }

*ProcessorClass method: *Provided method which was called from service
class. Please look the hightlighted code below from where the error
originated.

@Timer
    public Object createReadings(String operationType,
                                 String requestGUID,
                                 String userId,
                                 String companyId,
                                 String siteId,
                                 String buildingId,
                                 String readings,
                                 String requestingUser,
                                 String format) {

        UserDetails userDetails = getUserDetails(operationType,
                                                 requestGUID,
                                                 userId,
                                                 companyId,
                                                 requestingUser);

        checkParentEntities(operationType, requestGUID, userId, companyId,
siteId, buildingId, userDetails);

        // Check the user Privileges
        if (!userDetails.getPrivilege().contains(Privilege.CREATE)) {
            String stackTrace =
ExceptionUtils.getStackTrace(Thread.currentThread().getStackTrace());
            LogUtils.error(operationType,
                           requestGUID,
                           userId,
                           CREATE_NOT_ALLOWED_ERROR,
                           new String[] { requestingUser, "Readings",
companyId, stackTrace });
            throw new ADException(CREATE_NOT_ALLOWED, new String[] {
requestingUser, "Readings", companyId });
        }

        // Getting the array of Column Names defined in the properties
files.
        columnNames = HBASE_READING_COLUMNS.split(HBASE_COLUMNS_DELIMITER,
-1);

        Map<String, String> pointBasIdMap = new HashMap<String, String>();
        List<Put> hbaseRecordList = new ArrayList<Put>();

        int successRecords = 0;
        int failedRecords = 0;
        pointBasIdMap = pointDAO.getPointBASIDs(operationType,
                                                requestGUID,
                                                userId,
                                                companyId,
                                                siteId,
                                                buildingId,
                                                userDetails.getUserRole());
        // List of points to be updated
        List<Put> updatePointRecordList = new ArrayList<Put>();
        Map<String, String> batchPointIdMap = new HashMap<String, String>();

        // Parse the Reading data String to fetch logical records.
        List<String> recordList = parseSensorDataRows(readings);
        for (String record : recordList) {
            String columnValues[] = null;
            try {
                // Check if Record is valid for Entry
                columnValues = parseSensorDataColumns(record);
                validateRecord(columnValues, columnNames, record);
                // Validate Reading specific attributes
                validateReadingAttributes(columnValues, record);
                String pointBasId = columnValues[0];
                String recordTimeStamp = columnValues[1];
                String recordValue = columnValues[2];

                // Check if Point Exists
                String pointId = pointBasIdMap.get(pointBasId);
                if (pointId == null) {
                    String[] params = new String[] { companyId, siteId,
buildingId, pointBasId };
                    throw new
NotFoundException(ADConstants.POINT_NOT_FOUND, params);
                }

                // Check for the update of LatestReadingTimestamp column in
                // points table
                checkForUpdate(operationType,
                               requestGUID,
                               userId,
                               companyId,
                               siteId,
                               buildingId,
                               pointId,
                               recordTimeStamp,
                               batchPointIdMap);

                String rowKey = readingDAO.getRowKey(operationType,
                                                     requestGUID,
                                                     userId,
                                                     companyId,
                                                     siteId,
                                                     buildingId,
                                                     pointId,
                                                     recordTimeStamp);
                hbaseRecordList.add(buildHBaseRecord(rowKey, pointBasId,
recordTimeStamp, recordValue));
                successRecords++;// Increment Success Record count.

            } catch (BadRecordException e) {
                failedRecords++;// Increment Failed Record count.
                LogUtils.error(operationType, requestGUID, userId, e);
            } catch (NotFoundException e) {
                failedRecords++;
                LogUtils.error(operationType, requestGUID, userId, "record
not found : " + record);
                LogUtils.error(operationType, requestGUID, userId, e);
            }
        }
        // Insert Readings in HBase. *Calls SensorDataDAO  class
insertRecords method.*
        readingDAO.insertRecords(operationType, requestGUID, userId,
hbaseRecordList);

        Iterator<Entry<String, String>> iterator =
batchPointIdMap.entrySet().iterator();
        while (iterator.hasNext()) {
            Entry<String, String> pointIdMap = iterator.next();
            Put putPoint = new Put(Bytes.toBytes(pointIdMap.getKey()));

            putPoint.add(PointDAO.COLUMN_FAMILY_CF,
                         PointDAO.Columns.COLUMN_LAST_READING_TS.getName(),
                         Bytes.toBytes(pointIdMap.getValue()));
            updatePointRecordList.add(putPoint);
        }
        // Update Last reading time stamp for Points in Hbase. *Calls
SensorDataDAO
 class insertRecords method.*
       * pointDAO.insertRecords(operationType, requestGUID, userId,
updatePointRecordList); *

        // Build the response Object
        ProtobufReadingCreateResponse.Builder protobufResponse =
ProtobufReadingCreateResponse.newBuilder();
        ReadingCreateResponseV1 readingCreateResponseV1 = new
ReadingCreateResponseV1();
        if (successRecords > 0) {

readingCreateResponseV1.setSuccessfullyCreatedCount(String.valueOf(successRecords));

protobufResponse.setSuccessfullyCreatedCount(String.valueOf(successRecords));
        }
        if (failedRecords > 0) {

readingCreateResponseV1.setFailedCount(String.valueOf(failedRecords));

protobufResponse.setFailedCount(String.valueOf(failedRecords)).build();
        }
        if (format.equalsIgnoreCase(MediaType.APPLICATION_XML)
            || format.equalsIgnoreCase(MediaType.APPLICATION_JSON) ||
format.equalsIgnoreCase("json")
            || format.equalsIgnoreCase("xml")) {
            return readingCreateResponseV1;
        } else {
            return protobufResponse.build();
        }
    }

*SensorDataDAO class methods:This class was extended by both PointDAO and
ReadingDAO classes and hence insertRecords calls in above method are
directed to below method.*
 /**
     * Put (or insert) a list of rows
     */
    @Timer
    public void insertRecords(String operationType, String requestGUID,
String userId, List<Put> putList) throws ADException {

        boolean recordsInsertedInPrimaryCluster =
this.insertRecords(operationType,

 requestGUID,
                                                                     userId,

 SensorDataDAO.primaryClusterConnection,

 putList,
                                                                     false);
        if (recordsInsertedInPrimaryCluster == false) {
            LogUtils.info(operationType,
                          requestGUID,
                          userId,
                          "Failed to insert records from primary cluster.
Inserting rows from backup cluster.");
            this.insertRecords(operationType, requestGUID, userId,
backupClusterConnection, putList, true);
        }
    }


   private boolean insertRecords(String operationType,
                                  String requestGUID,
                                  String userId,
                                  HConnection connection,
                                  List<Put> putList,
                                  boolean throwException) throws
ADException {
        boolean recordsInserted = false;
        try {
            hTableInterface = connection.getTable(getTableName());
            hTableInterface.put(putList);
            recordsInserted = true;
        } catch (IOException e) {
            recordsInserted = false;
            ExceptionUtils.logException(operationType, requestGUID, userId,
HBASE_INSERT_ERROR_ERROR, e);
            if (throwException) {
                throw new ADException(ADConstants.HBASE_INSERT_ERROR);
            }
        } finally {
            closeConnection(operationType, requestGUID, userId, null);
        }
        return recordsInserted;
    }

*I think I provided sufficient information if not please let me know. Could
you please let us know what is going wrong. *

*Also below is the complete stack trace.*

2015-05-25 00:09:41 privateLog [DEBUG]  8674ecf718ace2c285a755afcfcf
     Execution time for "void
com.autodesk.dao.SensorDataDAO.insertRecords(String,String,String,List)" is
"9" milliseconds   com.autodesk.utils.aspects.TimerAspect  log
2015-05-25 00:09:41 privateLog [ERROR] create-readings-using-JAXB-object
     8674da549dd8382812a34df5da0d          Error occured-  Error Message:
"null", Stack Trace:
"java.util.LinkedList$ListItr.checkForComodification(LinkedList.java:953)
                java.util.LinkedList$ListItr.remove(LinkedList.java:919)

org.apache.hadoop.hbase.client.AsyncProcess.submit(AsyncProcess.java:319)

org.apache.hadoop.hbase.client.HTable.backgroundFlushCommits(HTable.java:965)

org.apache.hadoop.hbase.client.HTable.flushCommits(HTable.java:1281)
                org.apache.hadoop.hbase.client.HTable.put(HTable.java:925)
                com.autodesk.dao.SensorDataDAO.insertRecords(Unknown Source)
                com.autodesk.dao.SensorDataDAO.insertRecords(Unknown Source)

com.autodesk.dao.SensorDataDAO$$FastClassByCGLIB$$36f4c9d9.invoke(<generated>)
                net.sf.cglib.proxy.MethodProxy.invoke(MethodProxy.java:191)

org.springframework.aop.framework.Cglib2AopProxy$CglibMethodInvocation.invokeJoinpoint(Cglib2AopProxy.java:688)

org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150)

org.springframework.aop.aspectj.MethodInvocationProceedingJoinPoint.proceed(MethodInvocationProceedingJoinPoint.java:80)
                com.autodesk.utils.aspects.TimerAspect.log(Unknown Source)
                sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source)

sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
  java.lang.reflect.Method.invoke(Method.java:606)

org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethodWithGivenArgs(AbstractAspectJAdvice.java:621)

org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethod(AbstractAspectJAdvice.java:610)

org.springframework.aop.aspectj.AspectJAroundAdvice.invoke(AspectJAroundAdvice.java:65)

org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)

org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:89)

org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)

org.springframework.aop.framework.Cglib2AopProxy$DynamicAdvisedInterceptor.intercept(Cglib2AopProxy.java:621)

com.autodesk.dao.ReadingDAO$$EnhancerByCGLIB$$fa7dd7e1.insertRecords(<generated>)

com.autodesk.business.ReadingProcessor.createReadings(Unknown Source)

com.autodesk.business.ReadingProcessor$$FastClassByCGLIB$$f8b249db.invoke(<generated>)
                net.sf.cglib.proxy.MethodProxy.invoke(MethodProxy.java:191)

org.springframework.aop.framework.Cglib2AopProxy$CglibMethodInvocation.invokeJoinpoint(Cglib2AopProxy.java:688)

org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150)

org.springframework.aop.aspectj.MethodInvocationProceedingJoinPoint.proceed(MethodInvocationProceedingJoinPoint.java:80)
                com.autodesk.utils.aspects.TimerAspect.log(Unknown Source)
                sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source)

sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
                java.lang.reflect.Method.invoke(Method.java:606)

org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethodWithGivenArgs(AbstractAspectJAdvice.java:621)

org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethod(AbstractAspectJAdvice.java:610)

org.springframework.aop.aspectj.AspectJAroundAdvice.invoke(AspectJAroundAdvice.java:65)

org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)

org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:89)

org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)

org.springframework.aop.framework.Cglib2AopProxy$DynamicAdvisedInterceptor.intercept(Cglib2AopProxy.java:621)

com.autodesk.business.ReadingProcessor$$EnhancerByCGLIB$$703b6fe1.createReadings(<generated>)
                com.autodesk.services.ReadingService.createReadings(Unknown
Source)
                com.autodesk.services.ReadingService.createReading(Unknown
Source)

com.autodesk.services.ReadingService$$FastClassByCGLIB$$230421e0.invoke(<generated>)
                net.sf.cglib.proxy.MethodProxy.invoke(MethodProxy.java:191)

org.springframework.aop.framework.Cglib2AopProxy$CglibMethodInvocation.invokeJoinpoint(Cglib2AopProxy.java:688)

org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150)

org.springframework.aop.aspectj.MethodInvocationProceedingJoinPoint.proceed(MethodInvocationProceedingJoinPoint.java:80)
                com.autodesk.utils.aspects.TimerAspect.log(Unknown Source)
                sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source)

sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
                java.lang.reflect.Method.invoke(Method.java:606)

org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethodWithGivenArgs(AbstractAspectJAdvice.java:621)

org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethod(AbstractAspectJAdvice.java:610)

org.springframework.aop.aspectj.AspectJAroundAdvice.invoke(AspectJAroundAdvice.java:65)

org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)

org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:89)

org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)

org.springframework.aop.framework.Cglib2AopProxy$DynamicAdvisedInterceptor.intercept(Cglib2AopProxy.java:621)

com.autodesk.services.ReadingService$$EnhancerByCGLIB$$69808f18.createReading(<generated>)
                sun.reflect.GeneratedMethodAccessor69.invoke(Unknown Source)

sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
                java.lang.reflect.Method.invoke(Method.java:606)

com.sun.jersey.spi.container.JavaMethodInvokerFactory$1.invoke(JavaMethodInvokerFactory.java:60)

 com.sun.jersey.server.impl.model.method.dispatch.AbstractResourceMethodDispatchProvider$ResponseOutInvoker._dispatch(AbstractResourceMethodDispatchProvider.java:205)

com.sun.jersey.server.impl.model.method.dispatch.ResourceJavaMethodDispatcher.dispatch(ResourceJavaMethodDispatcher.java:75)

com.sun.jersey.server.impl.uri.rules.HttpMethodRule.accept(HttpMethodRule.java:288)

com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147)

com.sun.jersey.server.impl.uri.rules.ResourceClassRule.accept(ResourceClassRule.java:108)

com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147)

com.sun.jersey.server.impl.uri.rules.RootResourceClassesRule.accept(RootResourceClassesRule.java:84)

com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1469)

com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1400)

com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1349)

com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1339)

com.sun.jersey.spi.container.servlet.WebComponent.service(WebComponent.java:416)

com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:537)

com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:699)
                javax.servlet.http.HttpServlet.service(HttpServlet.java:722)

org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:305)

org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210)

org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:224)

org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:169)

org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:472)

org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:168)

org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:98)

org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:927)

org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:118)

org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:407)

org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:987)

org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:579)

org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:309)

java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)

java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
                java.lang.Thread.run(Thread.java:724)"
 com.autodesk.utils.LogUtils     error
2015-05-25 00:09:41 privateLog [DEBUG]  8674ecf718ace2c285a755afcfcf
     Execution time for "void
com.autodesk.dao.SensorDataDAO.insertRecords(String,String,String,List)" is
"6" milliseconds   com.autodesk.utils.aspects.TimerAspect  log


On Thu, May 21, 2015 at 6:16 PM, Ted Yu <yuzhihong@gmail.com> wrote:

> bq. Stack Trace: "java.util.LinkedList$ListItr.checkForComodification(
> LinkedList.java:953)
>
> What was the error prior to the above line ?
>
> Please let us know the hbase release you're using.
>
> Cheers
>
> On Thu, May 21, 2015 at 2:08 AM, Serega Sheypak <serega.sheypak@gmail.com>
> wrote:
>
> > Maybe you share HTable instance across several threads?
> > Can you share the code:
> > 1. HTable initialization
> > 2. Htable.put(something)
> >
> > 2015-05-21 11:57 GMT+03:00 Jithender Boreddy <jithen1010@gmail.com>:
> >
> > > Hi,
> > >
> > > I am inserting data from my java application into two HBase tables
> > > back to back. And I am running my application sequentially as part of
> > > stress testing. I am getting strange error intermittently. It is
> > > passing many times but failing by throwing below error few times.
> > >
> > > Can someone point me to the correct direction here by letting me know
> > > what going wrong ?
> > >
> > > Pasted below partial stack trace:
> > > Stack Trace:
> > >
> "java.util.LinkedList$ListItr.checkForComodification(LinkedList.java:953)
> > >
>  java.util.LinkedList$ListItr.remove(LinkedList.java:919)
> > >
> > >
> org.apache.hadoop.hbase.client.AsyncProcess.submit(AsyncProcess.java:319)
> > >
> > >
> >
> org.apache.hadoop.hbase.client.HTable.backgroundFlushCommits(HTable.java:965)
> > >
> > > org.apache.hadoop.hbase.client.HTable.flushCommits(HTable.java:1281)
> > >
> >  org.apache.hadoop.hbase.client.HTable.put(HTable.java:925)
> > >                 com.autodesk.dao.SensorDataDAO.insertRecords(Unknown
> > > Source)
> > >                 com.autodesk.dao.SensorDataDAO.insertRecords(Unknown
> > > Source)
> > >
> > >
> > >
> >
> com.autodesk.dao.SensorDataDAO$$FastClassByCGLIB$$36f4c9d9.invoke(<generated>)
> > >
> >  net.sf.cglib.proxy.MethodProxy.invoke(MethodProxy.java:191)
> > >
> > >
> > >
> >
> org.springframework.aop.framework.Cglib2AopProxy$CglibMethodInvocation.invokeJoinpoint(Cglib2AopProxy.java:688)
> > >
> > >
> > >
> >
> org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150)
> > >
> > >
> > >
> >
> org.springframework.aop.aspectj.MethodInvocationProceedingJoinPoint.proceed(MethodInvocationProceedingJoinPoint.java:80)
> > >                 com.autodesk.utils.aspects.TimerAspect.log(Unknown
> > Source)
> > >                 sun.reflect.GeneratedMethodAccessor38.invoke(Unknown
> > > Source)
> > >
> > >
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> > >                 java.lang.reflect.Method.invoke(Method.java:606)
> > >
> > >
> > >
> >
> org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethodWithGivenArgs(AbstractAspectJAdvice.java:621)
> > >
> > >
> > >
> >
> org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethod(AbstractAspectJAdvice.java:610)
> > >
> > >
> > >
> >
> org.springframework.aop.aspectj.AspectJAroundAdvice.invoke(AspectJAroundAdvice.java:65)
> > >
> > >
> > >
> >
> org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)
> > >
> > >
> > >
> >
> org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:89)
> > >
> > >
> > >
> >
> org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)
> > >
> > >
> > >
> >
> org.springframework.aop.framework.Cglib2AopProxy$DynamicAdvisedInterceptor.intercept(Cglib2AopProxy.java:621)
> > >
> > >
> > >
> >
> com.autodesk.dao.ReadingDAO$$EnhancerByCGLIB$$fa7dd7e1.insertRecords(<generated>)
> > >
> > > com.autodesk.business.ReadingProcessor.createReadings(Unknown Source)
> > >
> >
>

Mime
  • Unnamed multipart/alternative (inline, None, 0 bytes)
View raw message