【发布时间】：2022-10-05 16:18:13
【问题描述】：
我有使用databricks sql从增量文件中删除重复记录的要求。以下是我的查询
%sql
delete from delta.`adls_delta_file_path` where code = \'XYZ \'
但它给出了以下错误
com.databricks.backend.common.rpc.DatabricksExceptions$SQLExecutionException: java.util.NoSuchElementException: None.get at scala.None$.get(Option.scala:529) at scala.None$.get(Option.scala:527 ) 在 com.privacera.spark.agent.bV.a(bV.java) 在 com.privacera.spark.agent.bV.a(bV.java) 在 com.privacera.spark.agent.bc.a(bc. java) 在 com.privacera.spark.agent.bc.apply(bc.java) 在 org.apache.spark.sql.catalyst.trees.TreeNode.foreach(TreeNode.scala:252) 在 com.privacera.spark.agent .bV.a(bV.java) 在 com.privacera.spark.base.interceptor.c.b(c.java) 在 com.privacera.spark.base.interceptor.c.a(c.java) 在 com.privacera.spark。 agent.n.a(n.java) at com.privacera.spark.agent.n.apply(n.java) at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$3(RuleExecutor.scala: 221) 在 com.databricks.spark.util.FrameProfiler$.record(FrameProfiler.scala:80) 在 org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$2(RuleExecutor.scala:221) 在scala.collection.LinearSeqOptimized.foldLeft(LinearS eqOptimized.scala:126) 在 scala.collection.LinearSeqOptimized.foldLeft$(LinearSeqOptimized.scala:122) 在 scala.collection.immutable.List.foldLeft(List.scala:89) 在 org.apache.spark.sql.catalyst。 rules.RuleExecutor.$anonfun$execute$1(RuleExecutor.scala:218) at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$1$adapted(RuleExecutor.scala:210) at scala.collection。 immutable.List.foreach(List.scala:392) 在 org.apache.spark.sql.catalyst.rules.RuleExecutor.execute(RuleExecutor.scala:210) 在 org.apache.spark.sql.catalyst.rules.RuleExecutor。 $anonfun$executeAndTrack$1(RuleExecutor.scala:188) at org.apache.spark.sql.catalyst.QueryPlanningTracker$.withTracker(QueryPlanningTracker.scala:109) at org.apache.spark.sql.catalyst.rules.RuleExecutor.executeAndTrack (RuleExecutor.scala:188) at org.apache.spark.sql.execution.QueryExecution.$anonfun$optimizedPlan$1(QueryExecution.scala:112) at com.databricks.spark.util.FrameProfiler$.record(FrameProfiler.scala: 80) 在 org.apache.spark.sql.catal yst.QueryPlanningTracker.measurePhase(QueryPlanningTracker.scala:134) at org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$1(QueryExecution.scala:180) at org.apache.spark.sql.SparkSession.withActive( SparkSession.scala:854) 在 org.apache.spark.sql.execution.QueryExecution.executePhase(QueryExecution.scala:180) 在 org.apache.spark.sql.execution.QueryExecution.optimizedPlan$lzycompute(QueryExecution.scala:109)在 org.apache.spark.sql.execution.QueryExecution.optimizedPlan(QueryExecution.scala:109) 在 org.apache.spark.sql.execution.QueryExecution.assertOptimized(QueryExecution.scala:120) 在 org.apache.spark.sql .execution.QueryExecution.executedPlan$lzycompute(QueryExecution.scala:139) at org.apache.spark.sql.execution.QueryExecution.executedPlan(QueryExecution.scala:136) at org.apache.spark.sql.execution.QueryExecution.$ anonfun$simpleString$2(QueryExecution.scala:199) at org.apache.spark.sql.execution.ExplainUtils$.processPlan(ExplainUtils.scala:115) at org.apache.spark.s ql.execution.QueryExecution.simpleString(QueryExecution.scala:199) at org.apache.spark.sql.execution.QueryExecution.org$apache$spark$sql$execution$QueryExecution$$explainString(QueryExecution.scala:260) at org .apache.spark.sql.execution.QueryExecution.explainStringLocal(QueryExecution.scala:226) 在 org.apache.spark.sql.execution.SQLExecution$.$anonfun$withCustomExecutionEnv$5(SQLExecution.scala:123) 在 org.apache。 spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:273) 在 org.apache.spark.sql.execution.SQLExecution$.$anonfun$withCustomExecutionEnv$1(SQLExecution.scala:104) 在 org.apache.spark。 org.apache.spark.sql.execution.SQLExecution$.withCustomExecutionEnv(SQLExecution.scala:77) 在 org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId( SQLExecution.scala:223) 在 org.apache.spark.sql.Dataset.withAction(Dataset.scala:3823) 在 org.apache.spark.sql.Dataset.(Dataset.scala:235) 在 org.apache.spark。 sql.D ataset$.$anonfun$ofRows$2(Dataset.scala:104) at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:854) at org.apache.spark.sql.Dataset$.ofRows(Dataset. scala:101) 在 org.apache.spark.sql.SparkSession.$anonfun$sql$1(SparkSession.scala:689) 在 org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:854) 在 org.apache .spark.sql.SparkSession.sql(SparkSession.scala:684) at org.apache.spark.sql.SQLContext.sql(SQLContext.scala:694) at com.databricks.backend.daemon.driver.SQLDriverLocal.$anonfun$在 scala.collection 的 scala.collection.immutable.List.foreach(List.scala:392) 的 scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:238) 执行Sql$1(SQLDriverLocal.scala:91)。 TraversableLike.map(TraversableLike.scala:238) at scala.collection.TraversableLike.map$(TraversableLike.scala:231) at scala.collection.immutable.List.map(List.scala:298) at com.databricks.backend。 daemon.driver.SQLDriverLocal.executeSql(SQLDriverLocal.scala:37) 在 com.databricks.backend.daem on.driver.SQLDriverLocal.repl(SQLDriverLocal.scala:145) at com.databricks.backend.daemon.driver.DriverLocal.$anonfun$execute$11(DriverLocal.scala:529) at com.databricks.logging.UsageLogging.$anonfun $withAttributionContext$1(UsageLogging.scala:266) at scala.util.DynamicVariable.withValue(DynamicVariable.scala:62) at com.databricks.logging.UsageLogging.withAttributionContext(UsageLogging.scala:261) at com.databricks.logging.UsageLogging .withAttributionContext$(UsageLogging.scala:258) at com.databricks.backend.daemon.driver.DriverLocal.withAttributionContext(DriverLocal.scala:50) at com.databricks.logging.UsageLogging.withAttributionTags(UsageLogging.scala:305) at com .databricks.logging.UsageLogging.withAttributionTags$(UsageLogging.scala:297) 在 com.databricks.backend.daemon.driver.DriverLocal.withAttributionTags(DriverLocal.scala:50) 在 com.databricks.backend.daemon.driver.DriverLocal。在 com.databricks.backend.daemon.driver.DriverWrapper.$anonfun$tryExecut 处执行（DriverLocal.scala:506）在 com.databricks.backend.daemon.driver.DriverWrapper.tryExecutingCommand(DriverWrapper.scala:603) 在 com.databricks 在 scala.util.Try$.apply(Try.scala:213) 的 ingCommand$1(DriverWrapper.scala:611) 在 com.databricks .backend.daemon.driver.DriverWrapper.executeCommandAndGetError(DriverWrapper.scala:522) at com.databricks.backend.daemon.driver.DriverWrapper.executeCommand(DriverWrapper.scala:557) at com.databricks.backend.daemon.driver.DriverWrapper .runInnerLoop(DriverWrapper.scala:427) 在 com.databricks.backend.daemon.driver.DriverWrapper.runInner(DriverWrapper.scala:370) 在 com.databricks.backend.daemon.driver.DriverWrapper.run(DriverWrapper.scala:221 ) 在 java.lang.Thread.run(Thread.java:748) 在 com.databricks.backend.daemon.driver.SQLDriverLocal.executeSql(SQLDriverLocal.scala:130) 在 com.databricks.backend.daemon.driver.SQLDriverLocal。 repl(SQLDriverLocal.scala:145) at com.databricks.backend.daemon.driver.DriverLocal.$anonfun$execute$11(DriverLocal.scala:529) at com.databricks.logging.UsageLogging.$anonfun $withAttributionContext$1(UsageLogging.scala:266) at scala.util.DynamicVariable.withValue(DynamicVariable.scala:62) at com.databricks.logging.UsageLogging.withAttributionContext(UsageLogging.scala:261) at com.databricks.logging.UsageLogging .withAttributionContext$(UsageLogging.scala:258) at com.databricks.backend.daemon.driver.DriverLocal.withAttributionContext(DriverLocal.scala:50) at com.databricks.logging.UsageLogging.withAttributionTags(UsageLogging.scala:305) at com .databricks.logging.UsageLogging.withAttributionTags$(UsageLogging.scala:297) 在 com.databricks.backend.daemon.driver.DriverLocal.withAttributionTags(DriverLocal.scala:50) 在 com.databricks.backend.daemon.driver.DriverLocal。在 scala.util.Try$.apply(Try.scala:213) 在 com.databricks.backend.daemon.driver.DriverWrapper.$anonfun$tryExecutingCommand$1(DriverWrapper.scala:611) 处执行（DriverLocal.scala:506） com.databricks.backend.daemon.driver.DriverWrapper.tryExecutingCommand(DriverWrapper.scala:603) 在 com.databricks.bac kend.daemon.driver.DriverWrapper.executeCommandAndGetError(DriverWrapper.scala:522) 在 com.databricks.backend.daemon.driver.DriverWrapper.executeCommand(DriverWrapper.scala:557) 在 com.databricks.backend.daemon.driver.DriverWrapper。 runInnerLoop(DriverWrapper.scala:427) 在 com.databricks.backend.daemon.driver.DriverWrapper.runInner(DriverWrapper.scala:370) 在 com.databricks.backend.daemon.driver.DriverWrapper.run(DriverWrapper.scala:221)在 java.lang.Thread.run(Thread.java:748)
这里有任何建议。
-
您能否提供有关 delta 表的其他信息（架构、示例数据）
标签： python-3.x azure-databricks delta-lake databricks-sql