FAQ
Author: jssarma
Date: Tue Jun 16 00:49:34 2009
New Revision: 785039

URL: http://svn.apache.org/viewvc?rev=785039&view=rev
Log:
Hive-557 - catch outofmemory exceptions in mapper/reducers

Modified:
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecReducer.java

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java?rev=785039&r1=785038&r2=785039&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java Tue Jun 16 00:49:34 2009
@@ -124,10 +124,15 @@
}

rp = reporter;
- } catch (Exception e) {
+ } catch (Throwable e) {
abort = true;
e.printStackTrace();
- throw new RuntimeException ("Map operator initialization failed", e);
+ if (e instanceof OutOfMemoryError) {
+ // Don't create a new object if we are already out of memory
+ throw (OutOfMemoryError) e;
+ } else {
+ throw new RuntimeException ("Map operator initialization failed", e);
+ }
}
}

@@ -137,10 +142,15 @@
else
// Since there is no concept of a group, we don't invoke startGroup/endGroup for a mapper
mo.process((Writable)value);
- } catch (Exception e) {
+ } catch (Throwable e) {
abort = true;
e.printStackTrace();
- throw new RuntimeException ("Map operator process failed", e);
+ if (e instanceof OutOfMemoryError) {
+ // Don't create a new object if we are already out of memory
+ throw (OutOfMemoryError) e;
+ } else {
+ throw new RuntimeException (e.getMessage(), e);
+ }
}
}

@@ -151,10 +161,15 @@
l4j.trace("Close called no row");
mo.initialize(jc, null, null);
rp = null;
- } catch (Exception e) {
+ } catch (Throwable e) {
abort = true;
e.printStackTrace();
- throw new RuntimeException ("Map operator close failed during initialize", e);
+ if (e instanceof OutOfMemoryError) {
+ // Don't create a new object if we are already out of memory
+ throw (OutOfMemoryError) e;
+ } else {
+ throw new RuntimeException ("Map operator close failed during initialize", e);
+ }
}
}


Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecReducer.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecReducer.java?rev=785039&r1=785038&r2=785039&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecReducer.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecReducer.java Tue Jun 16 00:49:34 2009
@@ -118,10 +118,15 @@
reducer.setOutputCollector(oc);
reducer.initialize(jc, reporter, rowObjectInspector);
rp = reporter;
- } catch (Exception e) {
+ } catch (Throwable e) {
abort = true;
e.printStackTrace();
- throw new RuntimeException ("Reduce operator process failed", e);
+ if (e instanceof OutOfMemoryError) {
+ // Don't create a new object if we are already out of memory
+ throw (OutOfMemoryError) e;
+ } else {
+ throw new RuntimeException ("Reduce operator initialization failed");
+ }
}
}

@@ -175,9 +180,14 @@
reducer.process(row, rowObjectInspector[tag.get()], tag.get());
}

- } catch (Exception e) {
+ } catch (Throwable e) {
abort = true;
- throw new IOException (e);
+ if (e instanceof OutOfMemoryError) {
+ // Don't create a new object if we are already out of memory
+ throw (OutOfMemoryError) e;
+ } else {
+ throw new IOException (e);
+ }
}
}

@@ -198,10 +208,15 @@
l4j.trace("Close called no row");
reducer.initialize(jc, null, rowObjectInspector);
rp = null;
- } catch (Exception e) {
+ } catch (Throwable e) {
abort = true;
e.printStackTrace();
- throw new RuntimeException ("Reduce operator close failed during initialize", e);
+ if (e instanceof OutOfMemoryError) {
+ // Don't create a new object if we are already out of memory
+ throw (OutOfMemoryError) e;
+ } else {
+ throw new RuntimeException ("Reduce operator close failed during initialize", e);
+ }
}
}

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
posts ‹ prev | 1 of 1 | next ›
Discussion Overview
groupcommits @
categorieshive, hadoop
postedJun 16, '09 at 12:49a
activeJun 16, '09 at 12:49a
posts1
users1
websitehive.apache.org

1 user in discussion

Jssarma: 1 post

People

Translate

site design / logo © 2021 Grokbase