Java Code Examples for org.apache.iceberg.io.InputFile#location()

private static VectorizedRowBatchIterator newOrcIterator(InputFile file,
                                                         TypeDescription readerSchema,
                                                         Long start, Long length,
                                                         Reader orcFileReader, SearchArgument sarg) {
  final Reader.Options options = orcFileReader.options();
  if (start != null) {
    options.range(start, length);
  }
  options.schema(readerSchema);
  options.searchArgument(sarg, new String[]{});

  try {
    return new VectorizedRowBatchIterator(file.location(), readerSchema, orcFileReader.rows(options));
  } catch (IOException ioe) {
    throw new RuntimeIOException(ioe, "Failed to get ORC rows for file: %s", file);
  }
}
 

static List<ManifestFile> read(InputFile manifestList) {
  try (CloseableIterable<ManifestFile> files = Avro.read(manifestList)
      .rename("manifest_file", GenericManifestFile.class.getName())
      .rename("partitions", GenericPartitionFieldSummary.class.getName())
      .rename("r508", GenericPartitionFieldSummary.class.getName())
      .classLoader(GenericManifestFile.class.getClassLoader())
      .project(ManifestFile.schema())
      .reuseContainers(false)
      .build()) {

    return Lists.newLinkedList(files);

  } catch (IOException e) {
    throw new RuntimeIOException(e, "Cannot read manifest list file: %s", manifestList.location());
  }
}
 

GenericManifestFile(InputFile file, int specId) {
  this.avroSchema = AVRO_SCHEMA;
  this.file = file;
  this.manifestPath = file.location();
  this.length = null; // lazily loaded from file
  this.specId = specId;
  this.sequenceNumber = 0;
  this.minSequenceNumber = 0;
  this.snapshotId = null;
  this.addedFilesCount = null;
  this.addedRowsCount = null;
  this.existingFilesCount = null;
  this.existingRowsCount = null;
  this.deletedFilesCount = null;
  this.deletedRowsCount = null;
  this.partitions = null;
  this.fromProjectionPos = null;
}
 

static Metrics fromInputFile(InputFile file, Configuration config) {
  try (Reader orcReader = ORC.newFileReader(file, config)) {
    return buildOrcMetrics(orcReader.getNumberOfRows(), orcReader.getSchema(), orcReader.getStatistics());
  } catch (IOException ioe) {
    throw new RuntimeIOException(ioe, "Failed to open file: %s", file.location());
  }
}
 

private static ParquetFileReader newReader(InputFile file, ParquetReadOptions options) {
  try {
    return ParquetFileReader.open(ParquetIO.file(file), options);
  } catch (IOException e) {
    throw new RuntimeIOException(e, "Failed to open Parquet file: %s", file.location());
  }
}
 

public Builder withInputFile(InputFile file) {
  if (file instanceof HadoopInputFile) {
    return withStatus(((HadoopInputFile) file).getStat());
  }

  this.filePath = file.location();
  this.fileSizeInBytes = file.getLength();
  return this;
}
 

public static DataFile fromInputFile(InputFile file, PartitionData partition, long rowCount) {
  if (file instanceof HadoopInputFile) {
    return fromStat(((HadoopInputFile) file).getStat(), partition, rowCount);
  }

  String location = file.location();
  FileFormat format = FileFormat.fromFileName(location);
  return new GenericDataFile(
      location, format, partition, rowCount, file.getLength());
}
 

public static DataFile fromInputFile(InputFile file, long rowCount) {
  if (file instanceof HadoopInputFile) {
    return fromStat(((HadoopInputFile) file).getStat(), rowCount);
  }

  String location = file.location();
  FileFormat format = FileFormat.fromFileName(location);
  return new GenericDataFile(location, format, rowCount, file.getLength());
}
 

public static DataFile fromEncryptedOutputFile(EncryptedOutputFile encryptedFile, PartitionData partition,
                                              Metrics metrics, List<Long> splitOffsets) {
  EncryptionKeyMetadata keyMetadata = encryptedFile.keyMetadata();
  InputFile file = encryptedFile.encryptingOutputFile().toInputFile();
  if (encryptedFile instanceof HadoopInputFile) {
    return fromStat(((HadoopInputFile) file).getStat(), partition, metrics, keyMetadata, splitOffsets);
  }

  String location = file.location();
  FileFormat format = FileFormat.fromFileName(location);
  return new GenericDataFile(
      location, format, partition, file.getLength(), metrics, keyMetadata.buffer(), splitOffsets);
}
 

public Builder withInputFile(InputFile file) {
  if (file instanceof HadoopInputFile) {
    return withStatus(((HadoopInputFile) file).getStat());
  }

  this.filePath = file.location();
  this.fileSizeInBytes = file.getLength();
  return this;
}
 

static ManifestFile copyAppendManifest(int formatVersion,
                                       InputFile toCopy, Map<Integer, PartitionSpec> specsById,
                                       OutputFile outputFile, long snapshotId,
                                       SnapshotSummary.Builder summaryBuilder) {
  // use metadata that will add the current snapshot's ID for the rewrite
  InheritableMetadata inheritableMetadata = InheritableMetadataFactory.forCopy(snapshotId);
  try (ManifestReader<DataFile> reader =
           new ManifestReader<>(toCopy, specsById, inheritableMetadata, FileType.DATA_FILES)) {
    return copyManifestInternal(
        formatVersion, reader, outputFile, snapshotId, summaryBuilder, ManifestEntry.Status.ADDED);
  } catch (IOException e) {
    throw new RuntimeIOException(e, "Failed to close manifest: %s", toCopy.location());
  }
}
 

static ManifestFile copyRewriteManifest(int formatVersion,
                                        InputFile toCopy, Map<Integer, PartitionSpec> specsById,
                                        OutputFile outputFile, long snapshotId,
                                        SnapshotSummary.Builder summaryBuilder) {
  // for a rewritten manifest all snapshot ids should be set. use empty metadata to throw an exception if it is not
  InheritableMetadata inheritableMetadata = InheritableMetadataFactory.empty();
  try (ManifestReader<DataFile> reader =
           new ManifestReader<>(toCopy, specsById, inheritableMetadata, FileType.DATA_FILES)) {
    return copyManifestInternal(
        formatVersion, reader, outputFile, snapshotId, summaryBuilder, ManifestEntry.Status.EXISTING);
  } catch (IOException e) {
    throw new RuntimeIOException(e, "Failed to close manifest: %s", toCopy.location());
  }
}
 

TableMetadata(InputFile file,
              int formatVersion,
              String uuid,
              String location,
              long lastSequenceNumber,
              long lastUpdatedMillis,
              int lastColumnId,
              Schema schema,
              int defaultSpecId,
              List<PartitionSpec> specs,
              Map<String, String> properties,
              long currentSnapshotId,
              List<Snapshot> snapshots,
              List<HistoryEntry> snapshotLog,
              List<MetadataLogEntry> previousFiles) {
  Preconditions.checkArgument(specs != null && !specs.isEmpty(), "Partition specs cannot be null or empty");
  Preconditions.checkArgument(formatVersion <= SUPPORTED_TABLE_FORMAT_VERSION,
      "Unsupported format version: v%s", formatVersion);
  Preconditions.checkArgument(formatVersion == 1 || uuid != null,
      "UUID is required in format v%s", formatVersion);
  Preconditions.checkArgument(formatVersion > 1 || lastSequenceNumber == 0,
      "Sequence number must be 0 in v1: %s", lastSequenceNumber);

  this.formatVersion = formatVersion;
  this.file = file;
  this.metadataFileLocation = file != null ? file.location() : null;
  this.uuid = uuid;
  this.location = location;
  this.lastSequenceNumber = lastSequenceNumber;
  this.lastUpdatedMillis = lastUpdatedMillis;
  this.lastColumnId = lastColumnId;
  this.schema = schema;
  this.specs = specs;
  this.defaultSpecId = defaultSpecId;
  this.properties = properties;
  this.currentSnapshotId = currentSnapshotId;
  this.snapshots = snapshots;
  this.snapshotLog = snapshotLog;
  this.previousFiles = previousFiles;

  this.snapshotsById = indexAndValidateSnapshots(snapshots, lastSequenceNumber);
  this.specsById = indexSpecs(specs);

  HistoryEntry last = null;
  for (HistoryEntry logEntry : snapshotLog) {
    if (last != null) {
      Preconditions.checkArgument(
          (logEntry.timestampMillis() - last.timestampMillis()) >= -ONE_MINUTE,
          "[BUG] Expected sorted snapshot log entries.");
    }
    last = logEntry;
  }
  if (last != null) {
    Preconditions.checkArgument(
        // commits can happen concurrently from different machines.
        // A tolerance helps us avoid failure for small clock skew
        lastUpdatedMillis - last.timestampMillis() >= -ONE_MINUTE,
        "Invalid update timestamp %s: before last snapshot log entry at %s",
        lastUpdatedMillis, last.timestampMillis());
  }

  MetadataLogEntry previous = null;
  for (MetadataLogEntry metadataEntry : previousFiles) {
    if (previous != null) {
      Preconditions.checkArgument(
          // commits can happen concurrently from different machines.
          // A tolerance helps us avoid failure for small clock skew
          (metadataEntry.timestampMillis() - previous.timestampMillis()) >= -ONE_MINUTE,
          "[BUG] Expected sorted previous metadata log entries.");
    }
    previous = metadataEntry;
  }
    // Make sure that this update's lastUpdatedMillis is > max(previousFile's timestamp)
  if (previous != null) {
    Preconditions.checkArgument(
        // commits can happen concurrently from different machines.
        // A tolerance helps us avoid failure for small clock skew
        lastUpdatedMillis - previous.timestampMillis >= -ONE_MINUTE,
        "Invalid update timestamp %s: before the latest metadata log entry timestamp %s",
        lastUpdatedMillis, previous.timestampMillis);
  }

  Preconditions.checkArgument(
      currentSnapshotId < 0 || snapshotsById.containsKey(currentSnapshotId),
      "Invalid table metadata: Cannot find current version");
}
 

public static DataFile fromManifestList(InputFile manifestList) {
  return new GenericDataFile(manifestList.location(), FileFormat.AVRO, 1, manifestList.getLength());
}