FAQ
Hi Guys,

I'm copying a big file (9GB) to the hdfs using the command line interface
zcat wpc_ALL_200910.log.gz | dfs -copyFromLocal -
/user/cdh-hadoop/mscdata/wpc_ALL_200910.log
I'm getting the exceptions below.

Is this a issue or a misconfiguration?
Can anyone help-me!?

Thanks!

zcat wpc_ALL_200909.log.gz | dfs -copyFromLocal -
/user/cdh-hadoop/mscdata/wpc_ALL_200909.log
11/01/18 22:49:33 WARN hdfs.DFSClient: DFSOutputStream ResponseProcessor
exception for block blk_456158034633756525_1026java.io.IOException: Bad
response 1 for block blk_456158034633756525_1026 from datanode
150.164.3.194:50010
at
org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$ResponseProcessor.run(DFSClient.java:2454)

11/01/18 22:49:33 WARN hdfs.DFSClient: DataStreamer Exception:
java.io.IOException: Connection reset by peer
at sun.nio.ch.FileDispatcher.write0(Native Method)
at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:29)
at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:100)
at sun.nio.ch.IOUtil.write(IOUtil.java:71)
at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:334)
at
org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:55)
at
org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:142)
at
org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:146)
at
org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:107)
at java.io.BufferedOutputStream.write(BufferedOutputStream.java:105)
at java.io.DataOutputStream.write(DataOutputStream.java:90)
at
org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:2314)

11/01/18 22:49:33 WARN hdfs.DFSClient: Error Recovery for block
blk_456158034633756525_1026 bad datanode[1] 150.164.3.194:50010
11/01/18 22:49:33 WARN hdfs.DFSClient: Error Recovery for block
blk_456158034633756525_1026 in pipeline 150.164.3.136:50010,
150.164.3.194:50010, 150.164.3.198:50010: bad datanode 150.164.3.194:50010
11/01/18 22:49:38 WARN hdfs.DFSClient: DFSOutputStream ResponseProcessor
exception for block blk_1982092232090555893_1027java.io.IOException: Bad
response 1 for block blk_1982092232090555893_1027 from datanode
150.164.3.194:50010
at
org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$ResponseProcessor.run(DFSClient.java:2454)

11/01/18 22:49:38 WARN hdfs.DFSClient: DataStreamer Exception:
java.io.IOException: Connection reset by peer
at sun.nio.ch.FileDispatcher.write0(Native Method)
at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:29)
at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:100)
at sun.nio.ch.IOUtil.write(IOUtil.java:71)
at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:334)
at
org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:55)
at
org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:142)
at
org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:146)
at
org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:107)
at java.io.BufferedOutputStream.write(BufferedOutputStream.java:105)
at java.io.DataOutputStream.write(DataOutputStream.java:90)
at
org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:2314)

11/01/18 22:49:38 WARN hdfs.DFSClient: Error Recovery for block
blk_1982092232090555893_1027 bad datanode[2] 150.164.3.194:50010
11/01/18 22:49:38 WARN hdfs.DFSClient: Error Recovery for block
blk_1982092232090555893_1027 in pipeline 150.164.3.136:50010,
150.164.3.195:50010, 150.164.3.194:50010: bad datanode 150.164.3.194:50010
11/01/18 22:51:06 WARN hdfs.DFSClient: DataStreamer Exception:
java.io.IOException: Connection reset by peer
at sun.nio.ch.FileDispatcher.write0(Native Method)
at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:29)
at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:100)
at sun.nio.ch.IOUtil.write(IOUtil.java:71)
at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:334)
at
org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:55)
at
org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:142)
at
org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:146)
at
org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:107)
at java.io.BufferedOutputStream.write(BufferedOutputStream.java:105)
at java.io.DataOutputStream.write(DataOutputStream.java:90)
at
org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:2314)

11/01/18 22:51:06 WARN hdfs.DFSClient: Error Recovery for block
blk_-4070972757932012056_1028 bad datanode[0] 150.164.3.136:50010
11/01/18 22:51:06 WARN hdfs.DFSClient: Error Recovery for block
blk_-4070972757932012056_1028 in pipeline 150.164.3.136:50010,
150.164.3.195:50010, 150.164.3.198:50010: bad datanode 150.164.3.136:50010
11/01/18 22:51:06 WARN hdfs.DFSClient: DFSOutputStream ResponseProcessor
exception for block blk_-4070972757932012056_1029java.io.EOFException
at java.io.DataInputStream.readFully(DataInputStream.java:180)
at java.io.DataInputStream.readLong(DataInputStream.java:399)
at
org.apache.hadoop.hdfs.protocol.DataTransferProtocol$PipelineAck.readFields(DataTransferProtocol.java:119)
at
org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$ResponseProcessor.run(DFSClient.java:2424)

11/01/18 22:51:06 WARN hdfs.DFSClient: DataStreamer Exception:
java.io.IOException: Connection reset by peer
at sun.nio.ch.FileDispatcher.write0(Native Method)
at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:29)
at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:100)
at sun.nio.ch.IOUtil.write(IOUtil.java:71)
at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:334)
at
org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:55)
at
org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:142)
at
org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:146)
at
org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:107)
at java.io.BufferedOutputStream.write(BufferedOutputStream.java:105)
at java.io.DataOutputStream.write(DataOutputStream.java:90)
at
org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:2314)

11/01/18 22:51:06 WARN hdfs.DFSClient: Error Recovery for block
blk_-4070972757932012056_1029 bad datanode[0] 150.164.3.195:50010
11/01/18 22:51:06 WARN hdfs.DFSClient: Error Recovery for block
blk_-4070972757932012056_1029 in pipeline 150.164.3.195:50010,
150.164.3.198:50010: bad datanode 150.164.3.195:50010
11/01/18 22:51:06 WARN hdfs.DFSClient: DFSOutputStream ResponseProcessor
exception for block blk_-4070972757932012056_1030java.io.IOException:
Connection reset by peer
at sun.nio.ch.FileDispatcher.read0(Native Method)
at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:21)
at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:237)
at sun.nio.ch.IOUtil.read(IOUtil.java:210)
at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:236)
at
org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:55)
at
org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:142)
at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:155)
at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:128)
at java.io.DataInputStream.readFully(DataInputStream.java:178)
at java.io.DataInputStream.readLong(DataInputStream.java:399)
at
org.apache.hadoop.hdfs.protocol.DataTransferProtocol$PipelineAck.readFields(DataTransferProtocol.java:119)
at
org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$ResponseProcessor.run(DFSClient.java:2424)

11/01/18 22:51:06 WARN hdfs.DFSClient: DataStreamer Exception:
java.io.IOException: Broken pipe
at sun.nio.ch.FileDispatcher.write0(Native Method)
at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:29)
at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:100)
at sun.nio.ch.IOUtil.write(IOUtil.java:71)
at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:334)
at
org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:55)
at
org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:142)
at
org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:146)
at
org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:107)
at java.io.BufferedOutputStream.write(BufferedOutputStream.java:105)
at java.io.DataOutputStream.write(DataOutputStream.java:90)
at
org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:2314)

11/01/18 22:51:06 WARN hdfs.DFSClient: Error Recovery for block
blk_-4070972757932012056_1030 bad datanode[0] 150.164.3.198:50010
copyFromLocal: All datanodes 150.164.3.198:50010 are bad. Aborting...
11/01/18 22:51:06 ERROR hdfs.DFSClient: Exception closing file
/user/cdh-hadoop/mscdata/wpc_ALL_200909.log : java.io.IOException: All
datanodes 150.164.3.198:50010 are bad. Aborting...
java.io.IOException: All datanodes 150.164.3.198:50010 are bad. Aborting...
at
org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.processDatanodeError(DFSClient.java:2556)
at
org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.access$1600(DFSClient.java:2102)
at
org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:2265)


--
*Charles Ferreira Gonçalves *
http://homepages.dcc.ufmg.br/~charles/
UFMG - ICEx - Dcc
Cel.: 55 31 87741485
Tel.: 55 31 34741485
Lab.: 55 31 34095840

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
posts ‹ prev | 1 of 1 | next ›
Discussion Overview
grouphdfs-user @
categorieshadoop
postedJan 19, '11 at 1:06a
activeJan 19, '11 at 1:06a
posts1
users1
websitehadoop.apache.org...
irc#hadoop

1 user in discussion

Charles Gonçalves: 1 post

People

Translate

site design / logo © 2022 Grokbase