org.apache.spark.sql.catalyst.analysis.NoSuchTableException Java Examples

The following examples show how to use org.apache.spark.sql.catalyst.analysis.NoSuchTableException. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: NBadQueryAndPushDownTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
@Test
public void testPushDownToNonExistentDB() {
    //from tpch database
    try {
        final String sql = "select * from lineitem where l_orderkey = o.o_orderkey and l_commitdate < l_receiptdate";
        KylinConfig.getInstanceFromEnv().setProperty(PUSHDOWN_RUNNER_KEY,
                "org.apache.kylin.query.pushdown.PushDownRunnerSparkImpl");
        pushDownSql(getProject(), sql, 0, 0,
                new SQLException(new NoRealizationFoundException("testPushDownToNonExistentDB")));
    } catch (Exception e) {
        Assert.assertTrue(ExceptionUtils.getRootCause(e) instanceof NoSuchTableException);
        Assert.assertTrue(ExceptionUtils.getRootCauseMessage(e)
                .contains("Table or view 'lineitem' not found in database 'default'"));
    }
}
 
Example #2
Source File: SparkTableUtil.java    From iceberg with Apache License 2.0 5 votes vote down vote up
/**
 * Import files from an existing Spark table to an Iceberg table.
 *
 * The import uses the Spark session to get table metadata. It assumes no
 * operation is going on the original and target table and thus is not
 * thread-safe.
 *
 * @param spark a Spark session
 * @param sourceTableIdent an identifier of the source Spark table
 * @param targetTable an Iceberg table where to import the data
 * @param stagingDir a staging directory to store temporary manifest files
 */
public static void importSparkTable(
    SparkSession spark, TableIdentifier sourceTableIdent, Table targetTable, String stagingDir) {
  SessionCatalog catalog = spark.sessionState().catalog();

  String db = sourceTableIdent.database().nonEmpty() ?
      sourceTableIdent.database().get() :
      catalog.getCurrentDatabase();
  TableIdentifier sourceTableIdentWithDB = new TableIdentifier(sourceTableIdent.table(), Some.apply(db));

  if (!catalog.tableExists(sourceTableIdentWithDB)) {
    throw new org.apache.iceberg.exceptions.NoSuchTableException(
        String.format("Table %s does not exist", sourceTableIdentWithDB));
  }

  try {
    PartitionSpec spec = SparkSchemaUtil.specForTable(spark, sourceTableIdentWithDB.unquotedString());

    if (spec == PartitionSpec.unpartitioned()) {
      importUnpartitionedSparkTable(spark, sourceTableIdentWithDB, targetTable);
    } else {
      List<SparkPartition> sourceTablePartitions = getPartitions(spark, sourceTableIdent);
      importSparkPartitions(spark, sourceTablePartitions, targetTable, spec, stagingDir);
    }
  } catch (AnalysisException e) {
    throw SparkExceptionUtil.toUncheckedException(
        e, "Unable to get partition spec for table: %s", sourceTableIdentWithDB);
  }
}
 
Example #3
Source File: SparkSessionCatalog.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Override
public Table loadTable(Identifier ident) throws NoSuchTableException {
  try {
    return icebergCatalog.loadTable(ident);
  } catch (NoSuchTableException e) {
    return sessionCatalog.loadTable(ident);
  }
}
 
Example #4
Source File: SparkSessionCatalog.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Override
public Table alterTable(Identifier ident, TableChange... changes) throws NoSuchTableException {
  if (icebergCatalog.tableExists(ident)) {
    return icebergCatalog.alterTable(ident, changes);
  } else {
    return sessionCatalog.alterTable(ident, changes);
  }
}
 
Example #5
Source File: SparkSessionCatalog.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Override
public void renameTable(Identifier from, Identifier to) throws NoSuchTableException, TableAlreadyExistsException {
  // rename is not supported by HadoopCatalog. to avoid UnsupportedOperationException for session catalog tables,
  // check table existence first to ensure that the table belongs to the Iceberg catalog.
  if (icebergCatalog.tableExists(from)) {
    icebergCatalog.renameTable(from, to);
  } else {
    sessionCatalog.renameTable(from, to);
  }
}
 
Example #6
Source File: SparkCatalog.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Override
public SparkTable loadTable(Identifier ident) throws NoSuchTableException {
  try {
    return new SparkTable(icebergCatalog.loadTable(buildIdentifier(ident)));
  } catch (org.apache.iceberg.exceptions.NoSuchTableException e) {
    throw new NoSuchTableException(ident);
  }
}
 
Example #7
Source File: SparkCatalog.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Override
public StagedTable stageReplace(Identifier ident, StructType schema, Transform[] transforms,
                                Map<String, String> properties) throws NoSuchTableException {
  Schema icebergSchema = SparkSchemaUtil.convert(schema);
  try {
    return new StagedSparkTable(icebergCatalog.newReplaceTableTransaction(buildIdentifier(ident), icebergSchema,
        Spark3Util.toPartitionSpec(icebergSchema, transforms), properties.get("location"), properties,
        false /* do not create */));
  } catch (org.apache.iceberg.exceptions.NoSuchTableException e) {
    throw new NoSuchTableException(ident);
  }
}
 
Example #8
Source File: SparkCatalog.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Override
public SparkTable alterTable(Identifier ident, TableChange... changes) throws NoSuchTableException {
  SetProperty setLocation = null;
  SetProperty setSnapshotId = null;
  SetProperty pickSnapshotId = null;
  List<TableChange> propertyChanges = Lists.newArrayList();
  List<TableChange> schemaChanges = Lists.newArrayList();

  for (TableChange change : changes) {
    if (change instanceof SetProperty) {
      SetProperty set = (SetProperty) change;
      if (TableCatalog.PROP_LOCATION.equalsIgnoreCase(set.property())) {
        setLocation = set;
      } else if ("current-snapshot-id".equalsIgnoreCase(set.property())) {
        setSnapshotId = set;
      } else if ("cherry-pick-snapshot-id".equalsIgnoreCase(set.property())) {
        pickSnapshotId = set;
      } else {
        propertyChanges.add(set);
      }
    } else if (change instanceof RemoveProperty) {
      propertyChanges.add(change);
    } else if (change instanceof ColumnChange) {
      schemaChanges.add(change);
    } else {
      throw new UnsupportedOperationException("Cannot apply unknown table change: " + change);
    }
  }

  try {
    Table table = icebergCatalog.loadTable(buildIdentifier(ident));
    commitChanges(table, setLocation, setSnapshotId, pickSnapshotId, propertyChanges, schemaChanges);
  } catch (org.apache.iceberg.exceptions.NoSuchTableException e) {
    throw new NoSuchTableException(ident);
  }

  return null;
}
 
Example #9
Source File: SparkCatalog.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Override
public void invalidateTable(Identifier ident) {
  try {
    icebergCatalog.loadTable(buildIdentifier(ident)).refresh();
  } catch (org.apache.iceberg.exceptions.NoSuchTableException ignored) {
    // ignore if the table doesn't exist, it is not cached
  }
}
 
Example #10
Source File: SparkCatalog.java    From iceberg with Apache License 2.0 4 votes vote down vote up
@Override
public boolean dropTable(Identifier ident) {
  try {
    return icebergCatalog.dropTable(buildIdentifier(ident), true);
  } catch (org.apache.iceberg.exceptions.NoSuchTableException e) {
    return false;
  }
}
 
Example #11
Source File: AbstractConceptMaps.java    From bunsen with Apache License 2.0 4 votes vote down vote up
/**
 * Writes mappings to the given tables.
 *
 * <p>Warning: these updates are likely <em>not</em> atomic due to the lack of transactional
 * semantics in the underlying data store. Concurrent users may see previous items
 * removed before new ones are added, or items appear separately than others. This is intended
 * for use in a user-specific sandbox or staging environment.
 *
 * @param mappingsTable name of the table containing the mapping records
 * @param conceptMapTable name of the table containing the concept map metadata
 */
public void writeToTables(String mappingsTable, String conceptMapTable) {

  boolean hasExistingMaps;

  try {

    this.spark.sql("describe table " + conceptMapTable);

    hasExistingMaps = true;

  } catch (Exception describeException) {

    // Checked exceptions when calling into Scala upset the Java compiler,
    // hence the need for this workaround and re-throw to propagate unexpected
    // failures.
    if (describeException instanceof NoSuchTableException) {

      hasExistingMaps = false;

    } else {

      throw new RuntimeException(describeException);
    }
  }

  if (!hasExistingMaps) {

    // No target tables exist, so create and write them. The mappings
    // and ancestors tables are created explicitly to meet our
    // partitioning system.
    createMappingTable(this.spark, mappingsTable, null);

    JavaSparkContext javaContext = new JavaSparkContext(spark.sparkContext());

    // Create a concept map table by writing empty data having the proper schema and properties
    this.spark.createDataFrame(javaContext.emptyRDD(), conceptMapRowConverter.getSchema())
        .withColumn("timestamp", lit(null).cast("timestamp"))
        .write()
        .format("parquet")
        .partitionBy("timestamp")
        .saveAsTable(conceptMapTable);
  }

  Dataset<UrlAndVersion> currentMembers = this.spark
      .sql("SELECT url, version FROM " + conceptMapTable)
      .distinct()
      .as(URL_AND_VERSION_ENCODER);

  if (hasDuplicateUrlAndVersions(currentMembers)) {

    throw new IllegalArgumentException("The given concept maps contains duplicates url and "
        + "versions against concept maps already stored in the table, " + conceptMapTable);
  }

  writeMappingsToTable(this.mappings, mappingsTable);

  this.conceptMaps.write()
      .mode(SaveMode.ErrorIfExists)
      .insertInto(conceptMapTable);
}
 
Example #12
Source File: AbstractValueSets.java    From bunsen with Apache License 2.0 4 votes vote down vote up
/**
 * Writes value sets to the given tables.
 *
 * <p>Warning: these updates are likely <em>not</em> atomic due to the lack of transactional
 * semantics in the underlying data store. Concurrent users may see previous items
 * removed before new ones are added, or items appear separately than others. This is intended
 * for use in a user-specific sandbox or staging environment.
 *
 * @param valuesTable name of the table to which the value records are saved
 * @param valueSetTable name of the table to which the value set metadata is saved
 */
public void writeToTables(String valuesTable, String valueSetTable) {

  boolean hasExistingValueSets;

  try {

    spark.sql("DESCRIBE TABLE " + valueSetTable);

    hasExistingValueSets = true;

  } catch (Exception describeException) {

    // Checked exceptions when calling into Scala upset the Java compiler,
    // hence the need for this workaround and re-throw to propagate unexpected
    // failures.
    if (describeException instanceof NoSuchTableException) {

      hasExistingValueSets = false;

    } else {

      throw new RuntimeException(describeException);
    }
  }

  // If the target tables do not exist, we create them. The values and ancestors tables are
  // created explicitly to meet our partitioning system
  if (!hasExistingValueSets) {

    createValuesTable(spark, valuesTable, null);

    JavaSparkContext sparkContext = new JavaSparkContext(spark.sparkContext());

    // Create a value set table by writing empty data having the proper schema and properties
    spark.createDataFrame(sparkContext.emptyRDD(), valueSetRowConverter.getSchema())
        .withColumn("timestamp", lit(null).cast("timestamp"))
        .write()
        .format("parquet")
        .partitionBy("timestamp")
        .saveAsTable(valueSetTable);

  }

  // Check existing value set URIs and Versions for duplicates among the new members
  Dataset<UrlAndVersion> currentMembers = this.spark.table(valueSetTable)
      .select("url", "version")
      .distinct()
      .as(URL_AND_VERSION_ENCODER);

  if (hasDuplicateUrlAndVersions(currentMembers)) {

    throw new IllegalArgumentException("The given value sets contains duplicate url and versions "
        + "against value sets already stored in the table, " + valueSetTable);
  }

  writeValuesToTable(this.values, valuesTable);

  this.valueSets.write()
      .mode(SaveMode.ErrorIfExists)
      .insertInto(valueSetTable);
}
 
Example #13
Source File: Hierarchies.java    From bunsen with Apache License 2.0 2 votes vote down vote up
/**
 * Writes the ancestors to the given table.
 *
 * <p>Warning: these updates are likely <em>not</em> atomic due to the lack of transactional
 * semantics in the underlying data store. Concurrent users may see previous items
 * removed before new ones are added, or items appear separately than others. This is intended
 * for use in a user-specific sandbox or staging environment.
 *
 * @param ancestorsTable the name of the table to which the ancestors are saved
 */
public void writeToTables(String ancestorsTable) {

  boolean hasExistingAncestors;

  try {

    spark.sql("DESCRIBE TABLE " + ancestorsTable);

    hasExistingAncestors = true;

  } catch (Exception describeException) {

    if (describeException instanceof NoSuchTableException) {

      hasExistingAncestors = false;

    } else {

      throw new RuntimeException(describeException);
    }
  }

  if (!hasExistingAncestors) {

    createAncestorsTable(spark, ancestorsTable, null);
  }

  Dataset<UrlAndVersion> currentMembers = this.spark.table(ancestorsTable)
      .select(col("uri").alias("url"), col("version"))
      .distinct()
      .as(URI_AND_VERSION_ENCODER);

  if (hasDuplicateUriAndVersions(currentMembers)) {

    throw new IllegalArgumentException("The given hierarchies contains duplicate uri and "
        + "versions against ancestors already stored in the table, " + ancestorsTable);
  }

  writeAncestorsToTable(this.ancestors, ancestorsTable);
}