--------------------------------------------------------------------------- Py4JJavaError Traceback (most recent call last) <command-3647723071348962> in <module>() 8 p = Pipeline(stages=[featurizer, lr]) 9 ---> 10 p_model = p.fit(train_df) /databricks/spark/python/pyspark/ml/base.py in fit(self, dataset, params) 62 return self.copy(params)._fit(dataset) 63 else: ---> 64 return self._fit(dataset) 65 else: 66 raise ValueError("Params must be either a param map or a list/tuple of param maps, " /databricks/spark/python/pyspark/ml/pipeline.py in _fit(self, dataset) 104 if isinstance(stage, Transformer): 105 transformers.append(stage) --> 106 dataset = stage.transform(dataset) 107 else: # must be an Estimator 108 model = stage.fit(dataset) /databricks/spark/python/pyspark/ml/base.py in transform(self, dataset, params) 103 return self.copy(params)._transform(dataset) 104 else: --> 105 return self._transform(dataset) 106 else: 107 raise ValueError("Params must be a param map but got %s." % type(params)) /local_disk0/spark-0e2d732e-52e9-4321-9f51-c15c7b795732/userFiles-e82d45e4-e217-4f4a-9c01-29ba4c7399f8/addedFile4685221319697144792databricks_spark_deep_learning_0_1_0_spark2_1_s_2_11-2076e.jar/sparkdl/transformers/named_image.py in _transform(self, dataset) 157 outputCol=self.getOutputCol(), 158 modelName=self.getModelName(), featurize=True) --> 159 return transformer.transform(dataset) 160 161 /databricks/spark/python/pyspark/ml/base.py in transform(self, dataset, params) 103 return self.copy(params)._transform(dataset) 104 else: --> 105 return self._transform(dataset) 106 else: 107 raise ValueError("Params must be a param map but got %s." % type(params)) /local_disk0/spark-0e2d732e-52e9-4321-9f51-c15c7b795732/userFiles-e82d45e4-e217-4f4a-9c01-29ba4c7399f8/addedFile4685221319697144792databricks_spark_deep_learning_0_1_0_spark2_1_s_2_11-2076e.jar/sparkdl/transformers/named_image.py in _transform(self, dataset) 220 outputMode=modelGraphSpec["outputMode"]) 221 resizeUdf = resizeImage(modelGraphSpec["inputTensorSize"]) --> 222 result = tfTransformer.transform(dataset.withColumn(resizedCol, resizeUdf(inputCol))) 223 return result.drop(resizedCol) 224 /databricks/spark/python/pyspark/ml/base.py in transform(self, dataset, params) 103 return self.copy(params)._transform(dataset) 104 else: --> 105 return self._transform(dataset) 106 else: 107 raise ValueError("Params must be a param map but got %s." % type(params)) /local_disk0/spark-0e2d732e-52e9-4321-9f51-c15c7b795732/userFiles-e82d45e4-e217-4f4a-9c01-29ba4c7399f8/addedFile4685221319697144792databricks_spark_deep_learning_0_1_0_spark2_1_s_2_11-2076e.jar/sparkdl/transformers/tf_image.py in _transform(self, dataset) 120 def _transform(self, dataset): 121 graph = self.getGraph() --> 122 composed_graph = self._addReshapeLayers(graph, self._getImageDtype(dataset)) 123 final_graph = self._stripGraph(composed_graph) 124 /local_disk0/spark-0e2d732e-52e9-4321-9f51-c15c7b795732/userFiles-e82d45e4-e217-4f4a-9c01-29ba4c7399f8/addedFile4685221319697144792databricks_spark_deep_learning_0_1_0_spark2_1_s_2_11-2076e.jar/sparkdl/transformers/tf_image.py in _getImageDtype(self, dataset) 159 # This may not be the best way to get the type of image, but it is one way. 160 # Assumes that the dtype for all images is the same in the given dataframe. --> 161 pdf = dataset.select(self.getInputCol()).take(1) 162 img = pdf[0][self.getInputCol()] 163 img_type = sparkModeLookup[img.mode] /databricks/spark/python/pyspark/sql/dataframe.py in take(self, num) 493 [Row(age=2, name=u'Alice'), Row(age=5, name=u'Bob')] 494 """ --> 495 return self.limit(num).collect() 496 497 @since(1.3) /databricks/spark/python/pyspark/sql/dataframe.py in collect(self) 452 # Default path used in OSS Spark / for non-DF-ACL clusters: 453 with SCCallSiteSync(self._sc) as css: --> 454 port = self._jdf.collectToPython() 455 return list(_load_from_socket(port, BatchedSerializer(PickleSerializer()))) 456 /databricks/spark/python/lib/py4j-0.10.4-src.zip/py4j/java_gateway.py in __call__(self, *args) 1131 answer = self.gateway_client.send_command(command) 1132 return_value = get_return_value( -> 1133 answer, self.gateway_client, self.target_id, self.name) 1134 1135 for temp_arg in temp_args: /databricks/spark/python/pyspark/sql/utils.py in deco(*a, **kw) 61 def deco(*a, **kw): 62 try: ---> 63 return f(*a, **kw) 64 except py4j.protocol.Py4JJavaError as e: 65 s = e.java_exception.toString() /databricks/spark/python/lib/py4j-0.10.4-src.zip/py4j/protocol.py in get_return_value(answer, gateway_client, target_id, name) 317 raise Py4JJavaError( 318 "An error occurred while calling {0}{1}{2}.\n". --> 319 format(target_id, ".", name), value) 320 else: 321 raise Py4JError( Py4JJavaError: An error occurred while calling o4608.collectToPython. : org.apache.hadoop.mapreduce.lib.input.InvalidInputException: Input path does not exist: dbfs:/tmp/flower_photos/tulips at org.apache.hadoop.mapreduce.lib.input.FileInputFormat.singleThreadedListStatus(FileInputFormat.java:323) at org.apache.hadoop.mapreduce.lib.input.FileInputFormat.listStatus(FileInputFormat.java:265) at org.apache.spark.input.StreamFileInputFormat.setMinPartitions(PortableDataStream.scala:51) at org.apache.spark.rdd.BinaryFileRDD.getPartitions(BinaryFileRDD.scala:51) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:258) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:256) at scala.Option.getOrElse(Option.scala:121) at org.apache.spark.rdd.RDD.partitions(RDD.scala:256) at org.apache.spark.api.python.PythonRDD.getPartitions(PythonRDD.scala:55) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:258) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:256) at scala.Option.getOrElse(Option.scala:121) at org.apache.spark.rdd.RDD.partitions(RDD.scala:256) at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:258) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:256) at scala.Option.getOrElse(Option.scala:121) at org.apache.spark.rdd.RDD.partitions(RDD.scala:256) at org.apache.spark.ShuffleDependency.<init>(Dependency.scala:91) at org.apache.spark.rdd.ShuffledRDD.getDependencies(ShuffledRDD.scala:87) at org.apache.spark.rdd.RDD$$anonfun$dependencies$2.apply(RDD.scala:242) at org.apache.spark.rdd.RDD$$anonfun$dependencies$2.apply(RDD.scala:240) at scala.Option.getOrElse(Option.scala:121) at org.apache.spark.rdd.RDD.dependencies(RDD.scala:240) at org.apache.spark.rdd.ShuffledRDD.getPreferredLocations(ShuffledRDD.scala:98) at org.apache.spark.rdd.RDD$$anonfun$preferredLocations$2.apply(RDD.scala:283) at org.apache.spark.rdd.RDD$$anonfun$preferredLocations$2.apply(RDD.scala:283) at scala.Option.getOrElse(Option.scala:121) at org.apache.spark.rdd.RDD.preferredLocations(RDD.scala:282) at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$getPreferredLocsInternal(DAGScheduler.scala:1794) at org.apache.spark.scheduler.DAGScheduler.getPreferredLocs(DAGScheduler.scala:1768) at org.apache.spark.SparkContext.getPreferredLocs(SparkContext.scala:1844) at org.apache.spark.rdd.DefaultPartitionCoalescer.currPrefLocs(CoalescedRDD.scala:178) at org.apache.spark.rdd.DefaultPartitionCoalescer$PartitionLocations$$anonfun$getAllPrefLocs$2.apply(CoalescedRDD.scala:196) at org.apache.spark.rdd.DefaultPartitionCoalescer$PartitionLocations$$anonfun$getAllPrefLocs$2.apply(CoalescedRDD.scala:195) at scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33) at scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:186) at org.apache.spark.rdd.DefaultPartitionCoalescer$PartitionLocations.getAllPrefLocs(CoalescedRDD.scala:195) at org.apache.spark.rdd.DefaultPartitionCoalescer$PartitionLocations.<init>(CoalescedRDD.scala:188) at org.apache.spark.rdd.DefaultPartitionCoalescer.coalesce(CoalescedRDD.scala:391) at org.apache.spark.rdd.CoalescedRDD.getPartitions(CoalescedRDD.scala:91) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:258) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:256) at scala.Option.getOrElse(Option.scala:121) at org.apache.spark.rdd.RDD.partitions(RDD.scala:256) at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:258) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:256) at scala.Option.getOrElse(Option.scala:121) at org.apache.spark.rdd.RDD.partitions(RDD.scala:256) at org.apache.spark.api.python.PythonRDD.getPartitions(PythonRDD.scala:55) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:258) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:256) at scala.Option.getOrElse(Option.scala:121) at org.apache.spark.rdd.RDD.partitions(RDD.scala:256) at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:258) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:256) at scala.Option.getOrElse(Option.scala:121) at org.apache.spark.rdd.RDD.partitions(RDD.scala:256) at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:258) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:256) at scala.Option.getOrElse(Option.scala:121) at org.apache.spark.rdd.RDD.partitions(RDD.scala:256) at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:258) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:256) at scala.Option.getOrElse(Option.scala:121) at org.apache.spark.rdd.RDD.partitions(RDD.scala:256) at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:258) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:256) at scala.Option.getOrElse(Option.scala:121) at org.apache.spark.rdd.RDD.partitions(RDD.scala:256) at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:258) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:256) at scala.Option.getOrElse(Option.scala:121) at org.apache.spark.rdd.RDD.partitions(RDD.scala:256) at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:258) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:256) at scala.Option.getOrElse(Option.scala:121) at org.apache.spark.rdd.RDD.partitions(RDD.scala:256) at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:258) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:256) at scala.Option.getOrElse(Option.scala:121) at org.apache.spark.rdd.RDD.partitions(RDD.scala:256) at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:258) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:256) at scala.Option.getOrElse(Option.scala:121) at org.apache.spark.rdd.RDD.partitions(RDD.scala:256) at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:258) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:256) at scala.Option.getOrElse(Option.scala:121) at org.apache.spark.rdd.RDD.partitions(RDD.scala:256) at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:258) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:256) at scala.Option.getOrElse(Option.scala:121) at org.apache.spark.rdd.RDD.partitions(RDD.scala:256) at org.apache.spark.rdd.UnionRDD$$anonfun$1.apply(UnionRDD.scala:84) at org.apache.spark.rdd.UnionRDD$$anonfun$1.apply(UnionRDD.scala:84) at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) at scala.collection.immutable.List.foreach(List.scala:381) at scala.collection.TraversableLike$class.map(TraversableLike.scala:234) at scala.collection.immutable.List.map(List.scala:285) at org.apache.spark.rdd.UnionRDD.getPartitions(UnionRDD.scala:84) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:258) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:256) at scala.Option.getOrElse(Option.scala:121) at org.apache.spark.rdd.RDD.partitions(RDD.scala:256) at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:258) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:256) at scala.Option.getOrElse(Option.scala:121) at org.apache.spark.rdd.RDD.partitions(RDD.scala:256) at org.apache.spark.rdd.RDD.getNumPartitions(RDD.scala:275) at org.apache.spark.sql.execution.collect.Collector.<init>(Collector.scala:141) at org.apache.spark.sql.execution.collect.Collector$.collect(Collector.scala:59) at org.apache.spark.sql.execution.collect.Collector$.collect(Collector.scala:70) at org.apache.spark.sql.execution.CollectLimitExec.executeCollectResult(limit.scala:45) at org.apache.spark.sql.Dataset$$anonfun$collectToPython$1.apply$mcI$sp(Dataset.scala:2938) at org.apache.spark.sql.Dataset$$anonfun$collectToPython$1.apply(Dataset.scala:2936) at org.apache.spark.sql.Dataset$$anonfun$collectToPython$1.apply(Dataset.scala:2936) at org.apache.spark.sql.execution.SQLExecution$.withCustomExecutionEnv(SQLExecution.scala:89) at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:127) at org.apache.spark.sql.Dataset.withNewExecutionId(Dataset.scala:3007) at org.apache.spark.sql.Dataset.collectToPython(Dataset.scala:2936) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380) at py4j.Gateway.invoke(Gateway.java:293) at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) at py4j.commands.CallCommand.execute(CallCommand.java:79) at py4j.GatewayConnection.run(GatewayConnection.java:226) at java.lang.Thread.run(Thread.java:748)