--------------------------------------------------------------------------- AnalysisException Traceback (most recent call last) <command-697034> in <module>() 2 # Filter for data points that were predicted to be Fraud cases 3 cvPredictions = sqlContext.sql("SELECT * FROM cvPredictions") ----> 4 targetDF = cvPredictions.select("prediction", "total_claim_amount", "incident_severityIndex").filter("prediction = 1") /databricks/spark/python/pyspark/sql/dataframe.py in select(self, *cols) 1041 [Row(name=u'Alice', age=12), Row(name=u'Bob', age=15)] 1042 """ -> 1043 jdf = self._jdf.select(self._jcols(*cols)) 1044 return DataFrame(jdf, self.sql_ctx) 1045 /databricks/spark/python/lib/py4j-0.10.4-src.zip/py4j/java_gateway.py in __call__(self, *args) 1131 answer = self.gateway_client.send_command(command) 1132 return_value = get_return_value( -> 1133 answer, self.gateway_client, self.target_id, self.name) 1134 1135 for temp_arg in temp_args: /databricks/spark/python/pyspark/sql/utils.py in deco(*a, **kw) 67 e.java_exception.getStackTrace())) 68 if s.startswith('org.apache.spark.sql.AnalysisException: '): ---> 69 raise AnalysisException(s.split(': ', 1)[1], stackTrace) 70 if s.startswith('org.apache.spark.sql.catalyst.analysis'): 71 raise AnalysisException(s.split(': ', 1)[1], stackTrace) AnalysisException: u"cannot resolve '`incident_severityIndex`' given input columns: [cvpredictions.features, cvpredictions.insured_occupation, cvpredictions.bodily_injuries, cvpredictions.injury_claim, cvpredictions.probability, cvpredictions.prediction, cvpredictions.police_report_available, cvpredictions.auto_year, cvpredictions.insured_zip, cvpredictions.rawPrediction, cvpredictions.incident_severity, cvpredictions.policy_deductable, cvpredictions.incident_state, cvpredictions.fraud_reported, cvpredictions.vehicle_claim, cvpredictions.number_of_vehicles_involved, cvpredictions.incident_hour_of_the_day, cvpredictions.property_damage, cvpredictions.witnesses, cvpredictions.capital-gains, cvpredictions.incident_location, cvpredictions.policy_state, cvpredictions.capital-loss, cvpredictions.months_as_customer, cvpredictions.auto_model, cvpredictions.umbrella_limit, cvpredictions.age, cvpredictions.policy_annual_premium, cvpredictions.insured_hobbies, cvpredictions.policy_bind_date, cvpredictions.total_claim_amount, cvpredictions.policy_number, cvpredictions.incident_city, cvpredictions.insured_education_level, cvpredictions.property_claim, cvpredictions.incident_date, cvpredictions.auto_make, cvpredictions.insured_sex, cvpredictions.insured_relationship, cvpredictions.collision_type, cvpredictions.incident_type, cvpredictions.policy_csl, cvpredictions.label, cvpredictions.authorities_contacted];;\n'Project [prediction#51872, total_claim_amount#50, 'incident_severityIndex]\n+- Project [label#7245, fraud_reported#57, features#7185, months_as_customer#19, age#20, policy_number#21, policy_bind_date#140, policy_state#23, policy_csl#24, policy_deductable#25, policy_annual_premium#26, umbrella_limit#27, insured_zip#28, insured_sex#29, insured_education_level#30, insured_occupation#31, insured_hobbies#32, insured_relationship#33, capital-gains#34, capital-loss#35, incident_date#181, incident_type#37, collision_type#38, incident_severity#39, ... 21 more fields]\n +- SubqueryAlias cvpredictions\n +- Project [label#7245, fraud_reported#57, features#7185, months_as_customer#19, age#20, policy_number#21, policy_bind_date#140, policy_state#23, policy_csl#24, policy_deductable#25, policy_annual_premium#26, umbrella_limit#27, insured_zip#28, insured_sex#29, insured_education_level#30, insured_occupation#31, insured_hobbies#32, insured_relationship#33, capital-gains#34, capital-loss#35, incident_date#181, incident_type#37, collision_type#38, incident_severity#39, ... 21 more fields]\n +- Project [label#7245, fraud_reported#57, features#7185, months_as_customer#19, age#20, policy_number#21, policy_bind_date#140, policy_state#23, policy_csl#24, policy_deductable#25, policy_annual_premium#26, umbrella_limit#27, insured_zip#28, insured_sex#29, insured_education_level#30, insured_occupation#31, insured_hobbies#32, insured_relationship#33, capital-gains#34, capital-loss#35, incident_date#181, incident_type#37, collision_type#38, incident_severity#39, ... 20 more fields]\n +- Project [label#7245, fraud_reported#57, features#7185, months_as_customer#19, age#20, policy_number#21, policy_bind_date#140, policy_state#23, policy_csl#24, policy_deductable#25, policy_annual_premium#26, umbrella_limit#27, insured_zip#28, insured_sex#29, insured_education_level#30, insured_occupation#31, insured_hobbies#32, insured_relationship#33, capital-gains#34, capital-loss#35, incident_date#181, incident_type#37, collision_type#38, incident_severity#39, ... 19 more fields]\n +- SubqueryAlias testdata\n +- Sample 0.7, 1.0, false, 100\n +- Sort [label#7245 ASC NULLS FIRST, fraud_reported#57 ASC NULLS FIRST, features#7185 ASC NULLS FIRST, months_as_customer#19 ASC NULLS FIRST, age#20 ASC NULLS FIRST, policy_number#21 ASC NULLS FIRST, policy_bind_date#140 ASC NULLS FIRST, policy_state#23 ASC NULLS FIRST, policy_csl#24 ASC NULLS FIRST, policy_deductable#25 ASC NULLS FIRST, policy_annual_premium#26 ASC NULLS FIRST, umbrella_limit#27 ASC NULLS FIRST, insured_zip#28 ASC NULLS FIRST, insured_sex#29 ASC NULLS FIRST, insured_education_level#30 ASC NULLS FIRST, insured_occupation#31 ASC NULLS FIRST, insured_hobbies#32 ASC NULLS FIRST, insured_relationship#33 ASC NULLS FIRST, capital-gains#34 ASC NULLS FIRST, capital-loss#35 ASC NULLS FIRST, incident_date#181 ASC NULLS FIRST, incident_type#37 ASC NULLS FIRST, collision_type#38 ASC NULLS FIRST, incident_severity#39 ASC NULLS FIRST, ... 18 more fields], false\n +- Project [label#7245, fraud_reported#57, features#7185, months_as_customer#19, age#20, policy_number#21, policy_bind_date#140, policy_state#23, policy_csl#24, policy_deductable#25, policy_annual_premium#26, umbrella_limit#27, insured_zip#28, insured_sex#29, insured_education_level#30, insured_occupation#31, insured_hobbies#32, insured_relationship#33, capital-gains#34, capital-loss#35, incident_date#181, incident_type#37, collision_type#38, incident_severity#39, ... 18 more fields]\n +- Project [months_as_customer#19, age#20, policy_number#21, policy_bind_date#140, policy_state#23, policy_csl#24, policy_deductable#25, policy_annual_premium#26, umbrella_limit#27, insured_zip#28, insured_sex#29, insured_education_level#30, insured_occupation#31, insured_hobbies#32, insured_relationship#33, capital-gains#34, capital-loss#35, incident_date#181, incident_type#37, collision_type#38, incident_severity#39, authorities_contacted#40, incident_state#41, incident_city#42, ... 34 more fields]\n +- Project [months_as_customer#19, age#20, policy_number#21, policy_bind_date#140, policy_state#23, policy_csl#24, policy_deductable#25, policy_annual_premium#26, umbrella_limit#27, insured_zip#28, insured_sex#29, insured_education_level#30, insured_occupation#31, insured_hobbies#32, insured_relationship#33, capital-gains#34, capital-loss#35, incident_date#181, incident_type#37, collision_type#38, incident_severity#39, authorities_contacted#40, incident_state#41, incident_city#42, ... 34 more fields]\n +- Project [months_as_customer#19, age#20, policy_number#21, policy_bind_date#140, policy_state#23, policy_csl#24, policy_deductable#25, policy_annual_premium#26, umbrella_limit#27, insured_zip#28, insured_sex#29, insured_education_level#30, insured_occupation#31, insured_hobbies#32, insured_relationship#33, capital-gains#34, capital-loss#35, incident_date#181, incident_type#37, collision_type#38, incident_severity#39, authorities_contacted#40, incident_state#41, incident_city#42, ... 33 more fields]\n +- Project [months_as_customer#19, age#20, policy_number#21, policy_bind_date#140, policy_state#23, policy_csl#24, policy_deductable#25, policy_annual_premium#26, umbrella_limit#27, insured_zip#28, insured_sex#29, insured_education_level#30, insured_occupation#31, insured_hobbies#32, insured_relationship#33, capital-gains#34, capital-loss#35, incident_date#181, incident_type#37, collision_type#38, incident_severity#39, authorities_contacted#40, incident_state#41, incident_city#42, ... 32 more fields]\n +- Project [months_as_customer#19, age#20, policy_number#21, policy_bind_date#140, policy_state#23, policy_csl#24, policy_deductable#25, policy_annual_premium#26, umbrella_limit#27, insured_zip#28, insured_sex#29, insured_education_level#30, insured_occupation#31, insured_hobbies#32, insured_relationship#33, capital-gains#34, capital-loss#35, incident_date#181, incident_type#37, collision_type#38, incident_severity#39, authorities_contacted#40, incident_state#41, incident_city#42, ... 31 more fields]\n +- Project [months_as_customer#19, age#20, policy_number#21, policy_bind_date#140, policy_state#23, policy_csl#24, policy_deductable#25, policy_annual_premium#26, umbrella_limit#27, insured_zip#28, insured_sex#29, insured_education_level#30, insured_occupation#31, insured_hobbies#32, insured_relationship#33, capital-gains#34, capital-loss#35, incident_date#181, incident_type#37, collision_type#38, incident_severity#39, authorities_contacted#40, incident_state#41, incident_city#42, ... 30 more fields]\n +- Project [months_as_customer#19, age#20, policy_number#21, policy_bind_date#140, policy_state#23, policy_csl#24, policy_deductable#25, policy_annual_premium#26, umbrella_limit#27, insured_zip#28, insured_sex#29, insured_education_level#30, insured_occupation#31, insured_hobbies#32, insured_relationship#33, capital-gains#34, capital-loss#35, incident_date#181, incident_type#37, collision_type#38, incident_severity#39, authorities_contacted#40, incident_state#41, incident_city#42, ... 29 more fields]\n +- Project [months_as_customer#19, age#20, policy_number#21, policy_bind_date#140, policy_state#23, policy_csl#24, policy_deductable#25, policy_annual_premium#26, umbrella_limit#27, insured_zip#28, insured_sex#29, insured_education_level#30, insured_occupation#31, insured_hobbies#32, insured_relationship#33, capital-gains#34, capital-loss#35, incident_date#181, incident_type#37, collision_type#38, incident_severity#39, authorities_contacted#40, incident_state#41, incident_city#42, ... 28 more fields]\n +- Project [months_as_customer#19, age#20, policy_number#21, policy_bind_date#140, policy_state#23, policy_csl#24, policy_deductable#25, policy_annual_premium#26, umbrella_limit#27, insured_zip#28, insured_sex#29, insured_education_level#30, insured_occupation#31, insured_hobbies#32, insured_relationship#33, capital-gains#34, capital-loss#35, incident_date#181, incident_type#37, collision_type#38, incident_severity#39, authorities_contacted#40, incident_state#41, incident_city#42, ... 27 more fields]\n +- Project [months_as_customer#19, age#20, policy_number#21, policy_bind_date#140, policy_state#23, policy_csl#24, policy_deductable#25, policy_annual_premium#26, umbrella_limit#27, insured_zip#28, insured_sex#29, insured_education_level#30, insured_occupation#31, insured_hobbies#32, insured_relationship#33, capital-gains#34, capital-loss#35, incident_date#181, incident_type#37, collision_type#38, incident_severity#39, authorities_contacted#40, incident_state#41, incident_city#42, ... 26 more fields]\n +- Project [months_as_customer#19, age#20, policy_number#21, policy_bind_date#140, policy_state#23, policy_csl#24, policy_deductable#25, policy_annual_premium#26, umbrella_limit#27, insured_zip#28, insured_sex#29, insured_education_level#30, insured_occupation#31, insured_hobbies#32, insured_relationship#33, capital-gains#34, capital-loss#35, incident_date#181, incident_type#37, collision_type#38, incident_severity#39, authorities_contacted#40, incident_state#41, incident_city#42, ... 25 more fields]\n +- Project [months_as_customer#19, age#20, policy_number#21, policy_bind_date#140, policy_state#23, policy_csl#24, policy_deductable#25, policy_annual_premium#26, umbrella_limit#27, insured_zip#28, insured_sex#29, insured_education_level#30, insured_occupation#31, insured_hobbies#32, insured_relationship#33, capital-gains#34, capital-loss#35, incident_date#181, incident_type#37, collision_type#38, incident_severity#39, authorities_contacted#40, incident_state#41, incident_city#42, ... 24 more fields]\n +- Project [months_as_customer#19, age#20, policy_number#21, policy_bind_date#140, policy_state#23, policy_csl#24, policy_deductable#25, policy_annual_premium#26, umbrella_limit#27, insured_zip#28, insured_sex#29, insured_education_level#30, insured_occupation#31, insured_hobbies#32, insured_relationship#33, capital-gains#34, capital-loss#35, incident_date#181, incident_type#37, collision_type#38, incident_severity#39, authorities_contacted#40, incident_state#41, incident_city#42, ... 23 more fields]\n +- Project [months_as_customer#19, age#20, policy_number#21, policy_bind_date#140, policy_state#23, policy_csl#24, policy_deductable#25, policy_annual_premium#26, umbrella_limit#27, insured_zip#28, insured_sex#29, insured_education_level#30, insured_occupation#31, insured_hobbies#32, insured_relationship#33, capital-gains#34, capital-loss#35, incident_date#181, incident_type#37, collision_type#38, incident_severity#39, authorities_contacted#40, incident_state#41, incident_city#42, ... 22 more fields]\n +- Project [months_as_customer#19, age#20, policy_number#21, policy_bind_date#140, policy_state#23, policy_csl#24, policy_deductable#25, policy_annual_premium#26, umbrella_limit#27, insured_zip#28, insured_sex#29, insured_education_level#30, insured_occupation#31, insured_hobbies#32, insured_relationship#33, capital-gains#34, capital-loss#35, incident_date#181, incident_type#37, collision_type#38, incident_severity#39, authorities_contacted#40, incident_state#41, incident_city#42, ... 21 more fields]\n +- Project [months_as_customer#19, age#20, policy_number#21, policy_bind_date#140, policy_state#23, policy_csl#24, policy_deductable#25, policy_annual_premium#26, umbrella_limit#27, insured_zip#28, insured_sex#29, insured_education_level#30, insured_occupation#31, insured_hobbies#32, insured_relationship#33, capital-gains#34, capital-loss#35, incident_date#181, incident_type#37, collision_type#38, incident_severity#39, authorities_contacted#40, incident_state#41, incident_city#42, ... 20 more fields]\n +- Project [months_as_customer#19, age#20, policy_number#21, policy_bind_date#140, policy_state#23, policy_csl#24, policy_deductable#25, policy_annual_premium#26, umbrella_limit#27, insured_zip#28, insured_sex#29, insured_education_level#30, insured_occupation#31, insured_hobbies#32, insured_relationship#33, capital-gains#34, capital-loss#35, incident_date#181, incident_type#37, collision_type#38, incident_severity#39, authorities_contacted#40, incident_state#41, incident_city#42, ... 19 more fields]\n +- Project [months_as_customer#19, age#20, policy_number#21, policy_bind_date#140, policy_state#23, policy_csl#24, policy_deductable#25, policy_annual_premium#26, umbrella_limit#27, insured_zip#28, insured_sex#29, insured_education_level#30, insured_occupation#31, insured_hobbies#32, insured_relationship#33, capital-gains#34, capital-loss#35, incident_date#181, incident_type#37, collision_type#38, incident_severity#39, authorities_contacted#40, incident_state#41, incident_city#42, ... 18 more fields]\n +- Project [months_as_customer#19, age#20, policy_number#21, policy_bind_date#140, policy_state#23, policy_csl#24, policy_deductable#25, policy_annual_premium#26, umbrella_limit#27, insured_zip#28, insured_sex#29, insured_education_level#30, insured_occupation#31, insured_hobbies#32, insured_relationship#33, capital-gains#34, capital-loss#35, incident_date#181, incident_type#37, collision_type#38, incident_severity#39, authorities_contacted#40, incident_state#41, incident_city#42, ... 17 more fields]\n +- Project [months_as_customer#19, age#20, policy_number#21, policy_bind_date#140, policy_state#23, policy_csl#24, policy_deductable#25, policy_annual_premium#26, umbrella_limit#27, insured_zip#28, insured_sex#29, insured_education_level#30, insured_occupation#31, insured_hobbies#32, insured_relationship#33, capital-gains#34, capital-loss#35, incident_date#181, incident_type#37, collision_type#38, incident_severity#39, authorities_contacted#40, incident_state#41, incident_city#42, ... 16 more fields]\n +- Project [months_as_customer#19, age#20, policy_number#21, policy_bind_date#140, policy_state#23, policy_csl#24, policy_deductable#25, policy_annual_premium#26, umbrella_limit#27, insured_zip#28, insured_sex#29, insured_education_level#30, insured_occupation#31, insured_hobbies#32, insured_relationship#33, capital-gains#34, capital-loss#35, cast(incident_date#36 as string) AS incident_date#181, incident_type#37, collision_type#38, incident_severity#39, authorities_contacted#40, incident_state#41, incident_city#42, ... 15 more fields]\n +- Project [months_as_customer#19, age#20, policy_number#21, cast(policy_bind_date#22 as string) AS policy_bind_date#140, policy_state#23, policy_csl#24, policy_deductable#25, policy_annual_premium#26, umbrella_limit#27, insured_zip#28, insured_sex#29, insured_education_level#30, insured_occupation#31, insured_hobbies#32, insured_relationship#33, capital-gains#34, capital-loss#35, incident_date#36, incident_type#37, collision_type#38, incident_severity#39, authorities_contacted#40, incident_state#41, incident_city#42, ... 15 more fields]\n +- Project [months_as_customer#19, age#20, policy_number#21, policy_bind_date#22, policy_state#23, policy_csl#24, policy_deductable#25, policy_annual_premium#26, umbrella_limit#27, insured_zip#28, insured_sex#29, insured_education_level#30, insured_occupation#31, insured_hobbies#32, insured_relationship#33, capital-gains#34, capital-loss#35, incident_date#36, incident_type#37, collision_type#38, incident_severity#39, authorities_contacted#40, incident_state#41, incident_city#42, ... 15 more fields]\n +- Relation[months_as_customer#19,age#20,policy_number#21,policy_bind_date#22,policy_state#23,policy_csl#24,policy_deductable#25,policy_annual_premium#26,umbrella_limit#27,insured_zip#28,insured_sex#29,insured_education_level#30,insured_occupation#31,insured_hobbies#32,insured_relationship#33,capital-gains#34,capital-loss#35,incident_date#36,incident_type#37,collision_type#38,incident_severity#39,authorities_contacted#40,incident_state#41,incident_city#42,... 16 more fields] csv\n"