local host is: "node5/172.16.100.115"; destination host is: "node5":9000;_local host is: "node1.itcast.cn/192.168.88.161"; d

作者：菜鸟追梦旅行 | 2024-03-18 20:05:02

踩

local host is: "node1.itcast.cn/192.168.88.161"; destination host is: "node1

17/08/09 15:13:32 INFO spark.SparkContext: Created broadcast 0 from textFile at WordCount.scala:15
Exception in thread "main" java.io.IOException: Failed on local exception: com.google.protobuf.InvalidProtocolBufferException: Protocol message end-group tag did not match expected tag.; Host Details : local host is: "node5/172.16.100.115"; destination host is: "node5":9000;
at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:772)
at org.apache.hadoop.ipc.Client.call(Client.java:1476)
at org.apache.hadoop.ipc.Client.call(Client.java:1409)
at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230)
at com.sun.proxy.$Proxy13.getFileInfo(Unknown Source)
at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getFileInfo(ClientNamenodeProtocolTranslatorPB.java:771)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:256)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:104)
at com.sun.proxy.$Proxy14.getFileInfo(Unknown Source)
at org.apache.hadoop.hdfs.DFSClient.getFileInfo(DFSClient.java:2123)
at org.apache.hadoop.hdfs.DistributedFileSystem$20.doCall(DistributedFileSystem.java:1253)
at org.apache.hadoop.hdfs.DistributedFileSystem$20.doCall(DistributedFileSystem.java:1249)
at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81)
at org.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:1249)
at org.apache.hadoop.fs.FileSystem.exists(FileSystem.java:1417)
at org.apache.hadoop.mapred.FileOutputFormat.checkOutputSpecs(FileOutputFormat.java:130)
at org.apache.spark.rdd.PairRDDFunctions

a n o n f u n $ s a v e A s H a d o o p D a t a s e t $ 1. a p p l y $ m c V $ s p (P a i r R D D F u n c t i o n s . s c a l a : 1177) a t o r g . a p a c h e . s p a r k . r d d . P a i r R D D F u n c t i o n s

$anonfun$saveAsHadoopDataset$1.apply$mcV$sp(PairRDDFunctions.scala:1177) at org.apache.spark.rdd.PairRDDFunctions$ anonfun$saveAsHadoopDataset$1.apply(PairRDDFunctions.scala:1154)
at org.apache.spark.rdd.PairRDDFunctions

a n o n f u n $ s a v e A s H a d o o p D a t a s e t $ 1. a p p l y (P a i r R D D F u n c t i o n s . s c a l a : 1154) a t o r g . a p a c h e . s p a r k . r d d . R D D O p e r a t i o n S c o p e $ . w i t h S c o p e (R D D O p e r a t i o n S c o p e . s c a l a : 150) a t o r g . a p a c h e . s p a r k . r d d . R D D O p e r a t i o n S c o p e $ . w i t h S c o p e (R D D O p e r a t i o n S c o p e . s c a l a : 111) a t o r g . a p a c h e . s p a r k . r d d . R D D . w i t h S c o p e (R D D . s c a l a : 316) a t o r g . a p a c h e . s p a r k . r d d . P a i r R D D F u n c t i o n s . s a v e A s H a d o o p D a t a s e t (P a i r R D D F u n c t i o n s . s c a l a : 1154) a t o r g . a p a c h e . s p a r k . r d d . P a i r R D D F u n c t i o n s

$anonfun$saveAsHadoopDataset$1.apply(PairRDDFunctions.scala:1154) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:150) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:111) at org.apache.spark.rdd.RDD.withScope(RDD.scala:316) at org.apache.spark.rdd.PairRDDFunctions.saveAsHadoopDataset(PairRDDFunctions.scala:1154) at org.apache.spark.rdd.PairRDDFunctions$ anonfun$saveAsHadoopFile$4.apply$mcV$sp(PairRDDFunctions.scala:1060)
at org.apache.spark.rdd.PairRDDFunctions

a n o n f u n $ s a v e A s H a d o o p F i l e $ 4. a p p l y (P a i r R D D F u n c t i o n s . s c a l a : 1026) a t o r g . a p a c h e . s p a r k . r d d . P a i r R D D F u n c t i o n s

$anonfun$saveAsHadoopFile$4.apply(PairRDDFunctions.scala:1026) at org.apache.spark.rdd.PairRDDFunctions$ anonfun$saveAsHadoopFile$4.apply(PairRDDFunctions.scala:1026)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:150)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:111)
at org.apache.spark.rdd.RDD.withScope(RDD.scala:316)
at org.apache.spark.rdd.PairRDDFunctions.saveAsHadoopFile(PairRDDFunctions.scala:1026)
at org.apache.spark.rdd.PairRDDFunctions

a n o n f u n $ s a v e A s H a d o o p F i l e $ 1. a p p l y $ m c V $ s p (P a i r R D D F u n c t i o n s . s c a l a : 952) a t o r g . a p a c h e . s p a r k . r d d . P a i r R D D F u n c t i o n s

$anonfun$saveAsHadoopFile$1.apply$mcV$sp(PairRDDFunctions.scala:952) at org.apache.spark.rdd.PairRDDFunctions$ anonfun$saveAsHadoopFile$1.apply(PairRDDFunctions.scala:952)
at org.apache.spark.rdd.PairRDDFunctions

a n o n f u n $ s a v e A s H a d o o p F i l e $ 1. a p p l y (P a i r R D D F u n c t i o n s . s c a l a : 952) a t o r g . a p a c h e . s p a r k . r d d . R D D O p e r a t i o n S c o p e $ . w i t h S c o p e (R D D O p e r a t i o n S c o p e . s c a l a : 150) a t o r g . a p a c h e . s p a r k . r d d . R D D O p e r a t i o n S c o p e $ . w i t h S c o p e (R D D O p e r a t i o n S c o p e . s c a l a : 111) a t o r g . a p a c h e . s p a r k . r d d . R D D . w i t h S c o p e (R D D . s c a l a : 316) a t o r g . a p a c h e . s p a r k . r d d . P a i r R D D F u n c t i o n s . s a v e A s H a d o o p F i l e (P a i r R D D F u n c t i o n s . s c a l a : 951) a t o r g . a p a c h e . s p a r k . r d d . R D D

$anonfun$saveAsHadoopFile$1.apply(PairRDDFunctions.scala:952) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:150) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:111) at org.apache.spark.rdd.RDD.withScope(RDD.scala:316) at org.apache.spark.rdd.PairRDDFunctions.saveAsHadoopFile(PairRDDFunctions.scala:951) at org.apache.spark.rdd.RDD$ anonfun$saveAsTextFile$1.apply$mcV$sp(RDD.scala:1457)
at org.apache.spark.rdd.RDD

a n o n f u n $ s a v e A s T e x t F i l e $ 1. a p p l y (R D D . s c a l a : 1436) a t o r g . a p a c h e . s p a r k . r d d . R D D

$anonfun$saveAsTextFile$1.apply(RDD.scala:1436) at org.apache.spark.rdd.RDD$ anonfun$saveAsTextFile$1.apply(RDD.scala:1436)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:150)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:111)
at org.apache.spark.rdd.RDD.withScope(RDD.scala:316)
at org.apache.spark.rdd.RDD.saveAsTextFile(RDD.scala:1436)
at cn.atide.WordCount$.main(WordCount.scala:15)
at cn.atide.WordCount.main(WordCount.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:729)
at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:181)
at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:206)
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:121)
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
Caused by: com.google.protobuf.InvalidProtocolBufferException: Protocol message end-group tag did not match expected tag.
at com.google.protobuf.InvalidProtocolBufferException.invalidEndTag(InvalidProtocolBufferException.java:94)
at com.google.protobuf.CodedInputStream.checkLastTagWas(CodedInputStream.java:124)
at com.google.protobuf.AbstractParser.parsePartialFrom(AbstractParser.java:202)
at com.google.protobuf.AbstractParser.parsePartialDelimitedFrom(AbstractParser.java:241)
at com.google.protobuf.AbstractParser.parseDelimitedFrom(AbstractParser.java:253)
at com.google.protobuf.AbstractParser.parseDelimitedFrom(AbstractParser.java:259)
at com.google.protobuf.AbstractParser.parseDelimitedFrom(AbstractParser.java:49)
at org.apache.hadoop.ipc.protobuf.RpcHeaderProtos$RpcResponseHeaderProto.parseDelimitedFrom(RpcHeaderProtos.java:3167)
at org.apache.hadoop.ipc.Client$Connection.receiveRpcResponse(Client.java:1082)

at org.apache.hadoop.ipc.Client$Connection.run(Client.java:976)

异常处理：端口错误

检查cdh-manager中spark master 与 hdfs 的端口

声明：本文内容由网友自发贡献，不代表【wpsshop博客】立场，版权归原作者所有，本站不承担相应法律责任。如您发现有侵权的内容，请联系我们。转载请注明出处：https://www.wpsshop.cn/w/菜鸟追梦旅行/article/detail/265470