加入收藏 | 设为首页 | 会员中心 | 我要投稿 李大同 （https://www.lidatong.com.cn/）- 科技、建站、经验、云计算、5G、大数据,站长网!
当前位置： 首页 > 综合聚焦 > 服务器 > 安全 > 正文

scala – Apache spark MultilayerPerceptronClassifier因ArrayI

发布时间：2020-12-16 18:18:57 所属栏目：安全 来源：网络整理
导读：我正在使用此代码来尝试预测： import org.apache.spark.sql.functions.colimport org.apache.spark.Loggingimport org.apache.spark.graphx._import org.apache.spark.{ SparkConf,SparkContext }import org.apache.spark.SparkContext._import org.apache.
我正在使用此代码来尝试预测： 

import org.apache.spark.sql.functions.col
import org.apache.spark.Logging
import org.apache.spark.graphx._
import org.apache.spark.{ SparkConf,SparkContext }
import org.apache.spark.SparkContext._
import org.apache.spark.sql.SQLContext._
import org.apache.log4j.Logger
import org.apache.log4j.Level
import org.apache.spark.sql.functions.col
import org.apache.spark.ml.feature.VectorAssembler

object NN extends App {

Logger.getLogger("org").setLevel(Level.OFF)
Logger.getLogger("akka").setLevel(Level.OFF)

val sc = new SparkContext(new SparkConf().setMaster("local[2]")
.setAppName("cs"))

val sqlContext = new org.apache.spark.sql.SQLContext(sc)
import sqlContext.implicits._

val df = sc.parallelize(Seq(

("3","1","1"),("2","3","3"),("3",("0","0")))

.toDF("label","feature1","feature2")

val numeric = df
.select(df.columns.map(c => col(c).cast("double").alias(c)): _*)

val assembler = new VectorAssembler()
.setInputCols(Array("feature1","feature2"))
.setOutputCol("features")

val data = assembler.transform(numeric)

import org.apache.spark.ml.classification.MultilayerPerceptronClassifier

val layers = Array[Int](2,3,5,4) // Note 2 neurons in the input layer
val trainer = new MultilayerPerceptronClassifier()
.setLayers(layers)
.setBlockSize(128)
.setSeed(1234L)
.setMaxIter(100)

val model = trainer.fit(data)
model.transform(data).show


}

对于数据帧(df),如果我使用
(“4”,“1”,“1”)而不是(“3”,“1”)我收到错误：

Java HotSpot(TM) 64-Bit Server VM warning: ignoring option MaxPermSize=256m; support was removed in 8.0
[info] Set current project to spark-applications1458853926-master (in build file:/C:/Users/Desktop/spark-applications1458853926-master/)
[info] Compiling 1 Scala source to C:UsersDesktopspark-applications1458853926-mastertargetscala-2.11classes...
[info] Running NN
Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties
16/04/06 12:42:11 INFO Remoting: Starting remoting
16/04/06 12:42:11 INFO Remoting: Remoting started; listening on addresses :[akka.tcp://sparkDriverActorSystem@10.95.132.202:64056]
[error] (run-main-0) org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 0.0 failed 1 times,most recent failure: Lost task 0.0 in stage 0.0 (TID 0,localhost): java.lang.ArrayIndexOutOfBoundsException: 4
[error]         at org.apache.spark.ml.classification.LabelConverter$.encodeLabeledPoint(MultilayerPerceptronClassifier.scala:85)
[error]         at org.apache.spark.ml.classification.MultilayerPerceptronClassifier$$anonfun$2.apply(MultilayerPerceptronClassifier.scala:165)
[error]         at org.apache.spark.ml.classification.MultilayerPerceptronClassifier$$anonfun$2.apply(MultilayerPerceptronClassifier.scala:165)
[error]         at scala.collection.Iterator$$anon$11.next(Iterator.scala:370)
[error]         at scala.collection.Iterator$GroupedIterator.takeDestructively(Iterator.scala:934)
[error]         at scala.collection.Iterator$GroupedIterator.go(Iterator.scala:949)
[error]         at scala.collection.Iterator$GroupedIterator.fill(Iterator.scala:986)
[error]         at scala.collection.Iterator$GroupedIterator.hasNext(Iterator.scala:990)
[error]         at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:369)
[error]         at org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1595)
[error]         at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1143)
[error]         at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1143)
[error]         at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1858)
[error]         at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1858)
[error]         at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66)
[error]         at org.apache.spark.scheduler.Task.run(Task.scala:89)
[error]         at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213)
[error]         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[error]         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[error]         at java.lang.Thread.run(Thread.java:745)
[error]
[error] Driver stacktrace:
org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 0.0 failed 1 times,localhost): java.lang.ArrayIndexOutOfBoundsException: 4
        at org.apache.spark.ml.classification.LabelConverter$.encodeLabeledPoint(MultilayerPerceptronClassifier.scala:85)
        at org.apache.spark.ml.classification.MultilayerPerceptronClassifier$$anonfun$2.apply(MultilayerPerceptronClassifier.scala:165)
        at org.apache.spark.ml.classification.MultilayerPerceptronClassifier$$anonfun$2.apply(MultilayerPerceptronClassifier.scala:165)
        at scala.collection.Iterator$$anon$11.next(Iterator.scala:370)
        at scala.collection.Iterator$GroupedIterator.takeDestructively(Iterator.scala:934)
        at scala.collection.Iterator$GroupedIterator.go(Iterator.scala:949)
        at scala.collection.Iterator$GroupedIterator.fill(Iterator.scala:986)
        at scala.collection.Iterator$GroupedIterator.hasNext(Iterator.scala:990)
        at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:369)
        at org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1595)
        at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1143)
        at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1143)
        at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1858)
        at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1858)
        at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66)
        at org.apache.spark.scheduler.Task.run(Task.scala:89)
        at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)

Driver stacktrace:
        at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1431)
        at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1419)
        at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1418)
        at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
        at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)
        at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1418)
        at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:799)
        at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:799)
        at scala.Option.foreach(Option.scala:257)
        at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:799)
        at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1640)
        at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1599)
        at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1588)
        at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48)
        at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:620)
        at org.apache.spark.SparkContext.runJob(SparkContext.scala:1832)
        at org.apache.spark.SparkContext.runJob(SparkContext.scala:1845)
        at org.apache.spark.SparkContext.runJob(SparkContext.scala:1858)
        at org.apache.spark.SparkContext.runJob(SparkContext.scala:1929)
        at org.apache.spark.rdd.RDD.count(RDD.scala:1143)
        at org.apache.spark.mllib.optimization.LBFGS$.runLBFGS(LBFGS.scala:170)
        at org.apache.spark.mllib.optimization.LBFGS.optimize(LBFGS.scala:117)
        at org.apache.spark.ml.ann.FeedForwardTrainer.train(Layer.scala:878)
        at org.apache.spark.ml.classification.MultilayerPerceptronClassifier.train(MultilayerPerceptronClassifier.scala:170)
        at org.apache.spark.ml.classification.MultilayerPerceptronClassifier.train(MultilayerPerceptronClassifier.scala:110)
        at org.apache.spark.ml.Predictor.fit(Predictor.scala:90)
        at NN$.delayedEndpoint$NN$1(NN.scala:56)
        at NN$delayedInit$body.apply(NN.scala:15)
        at scala.Function0$class.apply$mcV$sp(Function0.scala:34)
        at scala.runtime.AbstractFunction0.apply$mcV$sp(AbstractFunction0.scala:12)
        at scala.App$$anonfun$main$1.apply(App.scala:76)
        at scala.App$$anonfun$main$1.apply(App.scala:76)
        at scala.collection.immutable.List.foreach(List.scala:381)
        at scala.collection.generic.TraversableForwarder$class.foreach(TraversableForwarder.scala:35)
        at scala.App$class.main(App.scala:76)
        at NN$.main(NN.scala:15)
        at NN.main(NN.scala)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:497)
Caused by: java.lang.ArrayIndexOutOfBoundsException: 4
        at org.apache.spark.ml.classification.LabelConverter$.encodeLabeledPoint(MultilayerPerceptronClassifier.scala:85)
        at org.apache.spark.ml.classification.MultilayerPerceptronClassifier$$anonfun$2.apply(MultilayerPerceptronClassifier.scala:165)
        at org.apache.spark.ml.classification.MultilayerPerceptronClassifier$$anonfun$2.apply(MultilayerPerceptronClassifier.scala:165)
        at scala.collection.Iterator$$anon$11.next(Iterator.scala:370)
        at scala.collection.Iterator$GroupedIterator.takeDestructively(Iterator.scala:934)
        at scala.collection.Iterator$GroupedIterator.go(Iterator.scala:949)
        at scala.collection.Iterator$GroupedIterator.fill(Iterator.scala:986)
        at scala.collection.Iterator$GroupedIterator.hasNext(Iterator.scala:990)
        at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:369)
        at org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1595)
        at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1143)
        at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1143)
        at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1858)
        at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1858)
        at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66)
        at org.apache.spark.scheduler.Task.run(Task.scala:89)
        at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
[trace] Stack trace suppressed: run last compile:run for the full output.
java.lang.RuntimeException: Nonzero exit code: 1
        at scala.sys.package$.error(package.scala:27)
[trace] Stack trace suppressed: run last compile:run for the full output.
[error] (compile:run) Nonzero exit code: 1
[error] Total time: 19 s,completed 06-Apr-2016 12:42:20

为什么我收到ArrayIndexOutOfBoundsException,我没有正确设置我的标签？标签不能取任何价值,因为它们只是标签吗？在这个例子中,它们似乎必须在0-3范围内？

解决方法

输出层使用 one-hot encoding;也就是说,标签“3”转换为(0,1),其中“第三”元素为1,其余为0.当您有4个输出节点且标签为4时,LabelConverter函数(其来源可见 here)将失败. (labelCount为4,标记为Point.label.toInt为4,因此您的错误.) 

val output = Array.fill(labelCount)(0.0)
output(labeledPoint.label.toInt) = 1.0
(labeledPoint.features,Vectors.dense(output))

所以改变这一行：

val layers = Array[Int](2,4) // Note 2 neurons in the input layer

对此：

val layers = Array[Int](2,5) // Note 2 neurons in the input layer and 5 neurons in the output layer

我希望它能奏效.

（编辑：李大同）

【声明】本站内容均来自网络，其相关言论仅代表作者个人观点，不代表本站立场。若无意侵犯到您的权利，请及时与联系站长删除相关内容!
    相关内容
      推荐文章
        站长推荐
        热点阅读

          【免责声明】本站内容转载自互联网，其发布内容言论不代表本站观点，如果其链接、内容的侵犯您的权益，烦请提交相关链接至邮箱bqsm@foxmail.com我们将及时予以处理。

          建议您使用1920×1080分辨率、谷歌浏览器Google Chrome、Microsoft Edge以获得本站的最佳浏览效果

          Copygight © 2008-2022 https://www.lidatong.com.cn/ All Rights Reserved. 李大同