FAQ
Author: sershe
Date: Fri Feb 20 20:51:47 2015
New Revision: 1661217

URL: http://svn.apache.org/r1661217
Log:
HIVE-9730p1 : LLAP: make sure logging is never called when not needed (in LLAP proper)

Added:
     hive/branches/llap/ql/src/java/org/apache/hadoop/hive/llap/LogLevels.java
Modified:
     hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java
     hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java
     hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataProducer.java
     hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/InStream.java
     hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/LlapUtils.java

Modified: hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java?rev=1661217&r1=1661216&r2=1661217&view=diff
==============================================================================
--- hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java (original)
+++ hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java Fri Feb 20 20:51:47 2015
@@ -23,8 +23,6 @@ import java.io.IOException;
  import java.util.LinkedList;
  import java.util.List;

-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
  import org.apache.hadoop.hive.llap.Consumer;
  import org.apache.hadoop.hive.llap.ConsumerFeedback;
  import org.apache.hadoop.hive.llap.DebugUtils;
@@ -48,7 +46,6 @@ import org.apache.hadoop.mapred.Reporter

  public class LlapInputFormat
    implements InputFormat<NullWritable, VectorizedRowBatch>, VectorizedInputFormatInterface {
- private static final Log LOG = LogFactory.getLog(LlapInputFormat.class);
    private final LlapIoImpl llapIo;
    private InputFormat sourceInputFormat;

@@ -65,7 +62,7 @@ public class LlapInputFormat
        InputSplit split, JobConf job, Reporter reporter) throws IOException {
      boolean isVectorMode = Utilities.isVectorMode(job);
      if (!isVectorMode) {
- LOG.error("No llap in non-vectorized mode");
+ LlapIoImpl.LOG.error("No llap in non-vectorized mode");
        throw new UnsupportedOperationException("No llap in non-vectorized mode");
      }
      FileSplit fileSplit = (FileSplit)split;
@@ -160,7 +157,7 @@ public class LlapInputFormat
          // We are waiting for next block. Either we will get it, or be told we are done.
          boolean doLogBlocking = DebugUtils.isTraceMttEnabled() && isNothingToReport();
          if (doLogBlocking) {
- LlapIoImpl.LOG.info("next will block"); // TODO: separate log objects
+ LlapIoImpl.LOG.info("next will block");
          }
          while (isNothingToReport()) {
            pendingData.wait(100);
@@ -202,7 +199,7 @@ public class LlapInputFormat

      @Override
      public void close() throws IOException {
- if (DebugUtils.isTraceEnabled()) {
+ if (DebugUtils.isTraceMttEnabled()) {
          LlapIoImpl.LOG.info("close called; closed " + isClosed + ", done " + isDone
              + ", err " + pendingError + ", pending " + pendingData.size());
        }
@@ -220,8 +217,8 @@ public class LlapInputFormat

      @Override
      public void setDone() {
- if (DebugUtils.isTraceEnabled()) {
- LlapIoImpl.LOG.info("setDone called; cclosed " + isClosed
+ if (DebugUtils.isTraceMttEnabled()) {
+ LlapIoImpl.LOG.info("setDone called; closed " + isClosed
            + ", done " + isDone + ", err " + pendingError + ", pending " + pendingData.size());
        }
        synchronized (pendingData) {
@@ -232,7 +229,7 @@ public class LlapInputFormat

      @Override
      public void consumeData(ColumnVectorBatch data) {
- if (DebugUtils.isTraceEnabled()) {
+ if (DebugUtils.isTraceMttEnabled()) {
          LlapIoImpl.LOG.info("consume called; closed " + isClosed + ", done " + isDone
              + ", err " + pendingError + ", pending " + pendingData.size());
        }

Modified: hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java?rev=1661217&r1=1661216&r2=1661217&view=diff
==============================================================================
--- hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java (original)
+++ hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java Fri Feb 20 20:51:47 2015
@@ -27,6 +27,7 @@ import org.apache.commons.logging.Log;
  import org.apache.commons.logging.LogFactory;
  import org.apache.hadoop.conf.Configuration;
  import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.llap.LogLevels;
  import org.apache.hadoop.hive.llap.cache.Allocator;
  import org.apache.hadoop.hive.llap.cache.BuddyAllocator;
  import org.apache.hadoop.hive.llap.cache.Cache;
@@ -48,6 +49,7 @@ import org.apache.hadoop.mapred.InputSpl

  public class LlapIoImpl implements LlapIo<VectorizedRowBatch> {
    public static final Log LOG = LogFactory.getLog(LlapIoImpl.class);
+ public static final LogLevels LOGL = new LogLevels(LOG);

    private final OrcColumnVectorProducer cvp;
    private final OrcEncodedDataProducer edp;

Modified: hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataProducer.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataProducer.java?rev=1661217&r1=1661216&r2=1661217&view=diff
==============================================================================
--- hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataProducer.java (original)
+++ hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataProducer.java Fri Feb 20 20:51:47 2015
@@ -114,7 +114,9 @@ public class OrcEncodedDataProducer impl

      @Override
      public Void call() throws IOException {
- LlapIoImpl.LOG.info("Processing split for " + internedFilePath);
+ if (LlapIoImpl.LOGL.isInfoEnabled()) {
+ LlapIoImpl.LOG.info("Processing split for " + internedFilePath);
+ }
        if (isStopped) return null;
        orcReader = null;
        // 1. Get FILE metadata from cache, or create the reader and read it.
@@ -457,7 +459,7 @@ public class OrcEncodedDataProducer impl
        long offset = split.getStart(), maxOffset = offset + split.getLength();
        stripeIxFrom = -1;
        int stripeIxTo = -1;
- if (LlapIoImpl.LOG.isDebugEnabled()) {
+ if (LlapIoImpl.LOGL.isDebugEnabled()) {
          String tmp = "FileSplit {" + split.getStart() + ", " + split.getLength() + "}; stripes ";
          for (StripeInformation stripe : stripes) {
            tmp += "{" + stripe.getOffset() + ", " + stripe.getLength() + "}, ";

Added: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/llap/LogLevels.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/llap/LogLevels.java?rev=1661217&view=auto
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/llap/LogLevels.java (added)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/llap/LogLevels.java Fri Feb 20 20:51:47 2015
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional debugrmation
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap;
+
+import org.apache.commons.logging.Log;
+
+public class LogLevels {
+ private final boolean isT, isD, isI, isW, isE;
+
+ public LogLevels(Log log) {
+ isT = log.isTraceEnabled();
+ isD = log.isDebugEnabled();
+ isI = log.isInfoEnabled();
+ isW = log.isWarnEnabled();
+ isE = log.isErrorEnabled();
+ }
+
+ public boolean isTraceEnabled() {
+ return isT;
+ }
+
+ public boolean isDebugEnabled() {
+ return isD;
+ }
+
+ public boolean isInfoEnabled() {
+ return isI;
+ }
+
+ public boolean isWarnEnabled() {
+ return isW;
+ }
+
+ public boolean isErrorEnabled() {
+ return isE;
+ }
+}

Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/InStream.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/InStream.java?rev=1661217&r1=1661216&r2=1661217&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/InStream.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/InStream.java Fri Feb 20 20:51:47 2015
@@ -28,6 +28,7 @@ import org.apache.commons.logging.Log;
  import org.apache.commons.logging.LogFactory;
  import org.apache.hadoop.hive.common.DiskRange;
  import org.apache.hadoop.hive.llap.DebugUtils;
+import org.apache.hadoop.hive.llap.LogLevels;
  import org.apache.hadoop.hive.llap.io.api.EncodedColumnBatch.StreamBuffer;
  import org.apache.hadoop.hive.llap.io.api.cache.LlapMemoryBuffer;
  import org.apache.hadoop.hive.llap.io.api.cache.LowLevelCache;
@@ -40,6 +41,8 @@ import com.google.common.annotations.Vis
  public abstract class InStream extends InputStream {

    private static final Log LOG = LogFactory.getLog(InStream.class);
+ private static final LogLevels LOGL = new LogLevels(LOG);
+
    protected final String fileName;
    protected final String name;
    protected final long length;
@@ -463,7 +466,7 @@ public abstract class InStream extends I
    public abstract void seek(PositionProvider index) throws IOException;

    private static void logEmptySeek(String name) {
- if (LOG.isWarnEnabled()) {
+ if (LOGL.isWarnEnabled()) {
        LOG.warn("Attempting seek into empty stream (" + name + ") Skipping stream.");
      }
    }

Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/LlapUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/LlapUtils.java?rev=1661217&r1=1661216&r2=1661217&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/LlapUtils.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/LlapUtils.java Fri Feb 20 20:51:47 2015
@@ -19,12 +19,9 @@ package org.apache.hadoop.hive.ql.io.orc

  import java.io.IOException;

-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
  import org.apache.hadoop.hive.ql.io.orc.BitFieldReader;

  public class LlapUtils {
- public static final Log LOG = LogFactory.getLog(LlapUtils.class);
    public static final int DOUBLE_GROUP_SIZE = 64; // just happens to be equal to bitmask size

    /** Helper for readPresentStream. */

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedFeb 20, '15 at 8:51p
activeFeb 20, '15 at 8:51p
posts1
users1
websitehive.apache.org

1 user in discussion

Sershe: 1 post

People

Translate

site design / logo © 2021 Grokbase