--------------------------------------------------------------------------- PicklingError Traceback (most recent call last) <ipython-input-3-d89ec16938b1> in <module>() 2 return range(value - 1, value + 1) 3 ----> 4 range2 = udf(return_range2, ArrayType(IntegerType())) /databricks/spark/python/pyspark/sql/functions.pyc in udf(f, returnType) 1866 [Row(slen=5), Row(slen=3)] 1867 """ -> 1868 return UserDefinedFunction(f, returnType) 1869 1870 blacklist = ['map', 'since', 'ignore_unicode_prefix'] /databricks/spark/python/pyspark/sql/functions.pyc in __init__(self, func, returnType, name) 1824 self.returnType = returnType 1825 self._broadcast = None -> 1826 self._judf = self._create_judf(name) 1827 1828 def _create_judf(self, name): /databricks/spark/python/pyspark/sql/functions.pyc in _create_judf(self, name) 1829 from pyspark.sql import SparkSession 1830 sc = SparkContext.getOrCreate() -> 1831 wrapped_func = _wrap_function(sc, self.func, self.returnType) 1832 spark = SparkSession.builder.getOrCreate() 1833 jdt = spark._jsparkSession.parseDataType(self.returnType.json()) /databricks/spark/python/pyspark/sql/functions.pyc in _wrap_function(sc, func, returnType) 1809 def _wrap_function(sc, func, returnType): 1810 command = (func, returnType) -> 1811 pickled_command, broadcast_vars, env, includes = _prepare_for_python_RDD(sc, command) 1812 return sc._jvm.PythonFunction(bytearray(pickled_command), env, includes, sc.pythonExec, 1813 sc.pythonVer, broadcast_vars, sc._javaAccumulator) /databricks/spark/python/pyspark/rdd.pyc in _prepare_for_python_RDD(sc, command) 2356 # the serialized command will be compressed by broadcast 2357 ser = CloudPickleSerializer() -> 2358 pickled_command = ser.dumps(command) 2359 if len(pickled_command) > (1 << 20): # 1M 2360 # The broadcast will have same life cycle as created PythonRDD /databricks/spark/python/pyspark/serializers.pyc in dumps(self, obj) 426 427 def dumps(self, obj): --> 428 return cloudpickle.dumps(obj, 2) 429 430 /databricks/spark/python/pyspark/cloudpickle.pyc in dumps(obj, protocol) 665 666 cp = CloudPickler(file,protocol) --> 667 cp.dump(obj) 668 669 return file.getvalue() /databricks/spark/python/pyspark/cloudpickle.pyc in dump(self, obj) 118 msg = "Could not serialize object: " + e.__class__.__name__ + ": " + e.message 119 print_exec(sys.stderr) --> 120 raise pickle.PicklingError(msg) 121 122 PicklingError: Could not serialize object: Py4JError: An error occurred while calling o102.__getnewargs__. Trace: py4j.Py4JException: Method __getnewargs__([]) does not exist at py4j.reflection.ReflectionEngine.getMethod(ReflectionEngine.java:318) at py4j.reflection.ReflectionEngine.getMethod(ReflectionEngine.java:326) at py4j.Gateway.invoke(Gateway.java:272) at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) at py4j.commands.CallCommand.execute(CallCommand.java:79) at py4j.GatewayConnection.run(GatewayConnection.java:214) at java.lang.Thread.run(Thread.java:745)