Java Code Examples for org.apache.orc.OrcProto#StripeFooter

The following examples show how to use org.apache.orc.OrcProto#StripeFooter . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: DremioORCRecordUtils.java From dremio-oss with Apache License 2.0

6 votes

/**
 * This function is a copy of original implementation from hive-private repository
 */
@Override
public OrcProto.StripeFooter readStripeFooter(StripeInformation stripe) throws IOException {
  if (file == null) {
    open();
  }
  long offset = stripe.getOffset() + stripe.getIndexLength() + stripe.getDataLength();
  int tailLength = (int) stripe.getFooterLength();

  // read the footer
  ByteBuffer tailBuf = ByteBuffer.allocate(tailLength);
  file.readFully(offset, tailBuf.array(), tailBuf.arrayOffset(), tailLength);
  return OrcProto.StripeFooter.parseFrom(InStream.createCodedInputStream("footer",
    Lists.<DiskRange>newArrayList(new BufferChunk(tailBuf, 0)),
    tailLength, codec, bufferSize));
}

Example 2

Source File: TreeReaderFactory.java From tajo with Apache License 2.0

6 votes

@Override
void startStripe(Map<StreamName, InStream> streams,
                 OrcProto.StripeFooter stripeFooter
) throws IOException {
  // For each stripe, checks the encoding and initializes the appropriate
  // reader
  switch (stripeFooter.getColumnsList().get(columnId).getKind()) {
    case DIRECT:
    case DIRECT_V2:
      reader = new StringDirectTreeReader(columnId);
      break;
    case DICTIONARY:
    case DICTIONARY_V2:
      reader = new StringDictionaryTreeReader(columnId);
      break;
    default:
      throw new IllegalArgumentException("Unsupported encoding " +
          stripeFooter.getColumnsList().get(columnId).getKind());
  }
  reader.startStripe(streams, stripeFooter);
}

Example 3

Source File: TreeReaderFactory.java From tajo with Apache License 2.0

6 votes

@Override
void startStripe(Map<org.apache.orc.impl.StreamName, InStream> streams,
                 OrcProto.StripeFooter stripeFooter
) throws IOException {
  super.startStripe(streams, stripeFooter);

  // read the dictionary blob
  org.apache.orc.impl.StreamName name = new org.apache.orc.impl.StreamName(columnId,
      OrcProto.Stream.Kind.DICTIONARY_DATA);
  InStream in = streams.get(name);
  readDictionaryStream(in);

  // read the lengths
  name = new org.apache.orc.impl.StreamName(columnId, OrcProto.Stream.Kind.LENGTH);
  in = streams.get(name);
  readDictionaryLengthStream(in, stripeFooter.getColumnsList().get(columnId));

  // set up the row reader
  name = new org.apache.orc.impl.StreamName(columnId, OrcProto.Stream.Kind.DATA);
  reader = createIntegerReader(stripeFooter.getColumnsList().get(columnId).getKind(),
      streams.get(name), false, false);
}

Example 4

Source File: TreeReaderFactory.java From tajo with Apache License 2.0

6 votes

@Override
void startStripe(Map<org.apache.orc.impl.StreamName, InStream> streams,
                 OrcProto.StripeFooter stripeFooter
) throws IOException {
  // For each stripe, checks the encoding and initializes the appropriate
  // reader
  switch (stripeFooter.getColumnsList().get(columnId).getKind()) {
    case DIRECT:
    case DIRECT_V2:
      reader = new StringDirectTreeReader(columnId);
      break;
    case DICTIONARY:
    case DICTIONARY_V2:
      reader = new StringDictionaryTreeReader(columnId);
      break;
    default:
      throw new IllegalArgumentException("Unsupported encoding " +
          stripeFooter.getColumnsList().get(columnId).getKind());
  }
  reader.startStripe(streams, stripeFooter);
}

Example 5

Source File: PhysicalWriterImpl.java From flink with Apache License 2.0

5 votes

private void writeStripeFooter(OrcProto.StripeFooter footer, long dataSize,
								long indexSize, OrcProto.StripeInformation.Builder dirEntry) throws IOException {
	footer.writeTo(protobufWriter);
	protobufWriter.flush();
	writer.flush();

	dirEntry.setOffset(stripeStart);
	dirEntry.setFooterLength(out.getPos() - stripeStart - dataSize - indexSize);
}

Example 6

Source File: TreeReaderFactory.java From tajo with Apache License 2.0

5 votes

@Override
void startStripe(Map<org.apache.orc.impl.StreamName, InStream> streams,
                 OrcProto.StripeFooter stripeFooter
) throws IOException {
  super.startStripe(streams, stripeFooter);
  org.apache.orc.impl.StreamName name = new org.apache.orc.impl.StreamName(columnId,
      OrcProto.Stream.Kind.DATA);
  stream = streams.get(name);
  data = new BasicTextReaderShim(stream);

  lengths = createIntegerReader(stripeFooter.getColumnsList().get(columnId).getKind(),
      streams.get(new org.apache.orc.impl.StreamName(columnId, OrcProto.Stream.Kind.LENGTH)),
      false, false);
}

Example 7

Source File: TreeReaderFactory.java From tajo with Apache License 2.0

5 votes

@Override
void startStripe(Map<org.apache.orc.impl.StreamName, InStream> streams,
                 OrcProto.StripeFooter stripeFooter
) throws IOException {
  super.startStripe(streams, stripeFooter);
  org.apache.orc.impl.StreamName name = new org.apache.orc.impl.StreamName(columnId,
      OrcProto.Stream.Kind.DATA);
  reader = createIntegerReader(stripeFooter.getColumnsList().get(columnId).getKind(),
      streams.get(name), true, false);
}

Example 8

Source File: TreeReaderFactory.java From tajo with Apache License 2.0

5 votes

@Override
void startStripe(Map<org.apache.orc.impl.StreamName, InStream> streams,
                 OrcProto.StripeFooter stripeFooter
) throws IOException {
  super.startStripe(streams, stripeFooter);
  data = createIntegerReader(stripeFooter.getColumnsList().get(columnId).getKind(),
      streams.get(new org.apache.orc.impl.StreamName(columnId,
          OrcProto.Stream.Kind.DATA)), true, skipCorrupt);
  nanos = createIntegerReader(stripeFooter.getColumnsList().get(columnId).getKind(),
      streams.get(new org.apache.orc.impl.StreamName(columnId,
          OrcProto.Stream.Kind.SECONDARY)), false, skipCorrupt);
  base_timestamp = getBaseTimestamp(stripeFooter.getWriterTimezone());
}

Example 9

Source File: TreeReaderFactory.java From tajo with Apache License 2.0

5 votes

@Override
void startStripe(Map<org.apache.orc.impl.StreamName, InStream> streams,
                 OrcProto.StripeFooter stripeFooter
) throws IOException {
  super.startStripe(streams, stripeFooter);
  org.apache.orc.impl.StreamName name = new org.apache.orc.impl.StreamName(columnId,
      OrcProto.Stream.Kind.DATA);
  stream = streams.get(name);
  lengths = createIntegerReader(stripeFooter.getColumnsList().get(columnId).getKind(),
      streams.get(new org.apache.orc.impl.StreamName(columnId, OrcProto.Stream.Kind.LENGTH)), false, false);
}

Example 10

Source File: TreeReaderFactory.java From tajo with Apache License 2.0

5 votes

@Override
void startStripe(Map<org.apache.orc.impl.StreamName, InStream> streams,
                 OrcProto.StripeFooter stripeFooter
) throws IOException {
  super.startStripe(streams, stripeFooter);
  org.apache.orc.impl.StreamName name =
      new org.apache.orc.impl.StreamName(columnId,
          OrcProto.Stream.Kind.DATA);
  stream = streams.get(name);
}

Example 11

Source File: TreeReaderFactory.java From tajo with Apache License 2.0

5 votes

@Override
void startStripe(Map<org.apache.orc.impl.StreamName, InStream> streams,
                 OrcProto.StripeFooter stripeFooter
) throws IOException {
  super.startStripe(streams, stripeFooter);
  org.apache.orc.impl.StreamName name = new org.apache.orc.impl.StreamName(columnId,
      OrcProto.Stream.Kind.DATA);
  stream = streams.get(name);
}

Example 12

Source File: TreeReaderFactory.java From tajo with Apache License 2.0

5 votes

@Override
void startStripe(Map<org.apache.orc.impl.StreamName, InStream> streams,
                 OrcProto.StripeFooter stripeFooter
) throws IOException {
  super.startStripe(streams, stripeFooter);
  org.apache.orc.impl.StreamName name = new org.apache.orc.impl.StreamName(columnId,
      OrcProto.Stream.Kind.DATA);
  reader = createIntegerReader(stripeFooter.getColumnsList().get(columnId).getKind(),
      streams.get(name), true, false);
}

Example 13

Source File: TreeReaderFactory.java From tajo with Apache License 2.0

5 votes

@Override
void startStripe(Map<org.apache.orc.impl.StreamName, InStream> streams,
                 OrcProto.StripeFooter stripeFooter
) throws IOException {
  super.startStripe(streams, stripeFooter);
  org.apache.orc.impl.StreamName name = new org.apache.orc.impl.StreamName(columnId,
      OrcProto.Stream.Kind.DATA);
  reader = createIntegerReader(stripeFooter.getColumnsList().get(columnId).getKind(),
      streams.get(name), true, false);
}

Example 14

Source File: TreeReaderFactory.java From tajo with Apache License 2.0

5 votes

@Override
void startStripe(Map<org.apache.orc.impl.StreamName, InStream> streams,
                 OrcProto.StripeFooter stripeFooter
) throws IOException {
  super.startStripe(streams, stripeFooter);
  org.apache.orc.impl.StreamName name = new org.apache.orc.impl.StreamName(columnId,
      OrcProto.Stream.Kind.DATA);
  reader = createIntegerReader(stripeFooter.getColumnsList().get(columnId).getKind(),
      streams.get(name), true, false);
}

Example 15

Source File: TreeReaderFactory.java From tajo with Apache License 2.0

5 votes

@Override
void startStripe(Map<org.apache.orc.impl.StreamName, InStream> streams,
                 OrcProto.StripeFooter stripeFooter
) throws IOException {
  super.startStripe(streams, stripeFooter);
  reader = new RunLengthByteReader(streams.get(new org.apache.orc.impl.StreamName(columnId,
      OrcProto.Stream.Kind.DATA)));
}

Example 16

Source File: TreeReaderFactory.java From tajo with Apache License 2.0

5 votes

@Override
void startStripe(Map<org.apache.orc.impl.StreamName, InStream> streams,
                 OrcProto.StripeFooter stripeFooter
) throws IOException {
  super.startStripe(streams, stripeFooter);
  reader = new BitFieldReader(streams.get(new org.apache.orc.impl.StreamName(columnId,
      OrcProto.Stream.Kind.DATA)), 1);
}

Example 17

Source File: TreeReaderFactory.java From tajo with Apache License 2.0

5 votes

void startStripe(Map<org.apache.orc.impl.StreamName, InStream> streams,
                 OrcProto.StripeFooter stripeFooter
) throws IOException {
  checkEncoding(stripeFooter.getColumnsList().get(columnId));
  InStream in = streams.get(new org.apache.orc.impl.StreamName(columnId,
      OrcProto.Stream.Kind.PRESENT));
  if (in == null) {
    present = null;
    valuePresent = true;
  } else {
    present = new BitFieldReader(in, 1);
  }
}

Example 18

Source File: DremioORCRecordUtils.java From dremio-oss with Apache License 2.0

5 votes

@Override
public OrcProto.StripeFooter readStripeFooter(StripeInformation stripe) throws IOException {
  if (file == null) {
    open();
  }
  long offset = stripe.getOffset() + stripe.getIndexLength() + stripe.getDataLength();
  int tailLength = (int) stripe.getFooterLength();

  // read the footer
  ByteBuffer tailBuf = ByteBuffer.allocate(tailLength);
  file.readFully(offset, tailBuf.array(), tailBuf.arrayOffset(), tailLength);
  return OrcProto.StripeFooter.parseFrom(
    InStream.createCodedInputStream("footer", singleton(
      new BufferChunk(tailBuf, 0)), tailLength, codec, bufferSize));
}

Example 19

Source File: DremioORCRecordUtils.java From dremio-oss with Apache License 2.0

4 votes

/**
 * This function is a copy of original implementation from hive-private repository
 */
@Override
public OrcIndex readRowIndex(StripeInformation stripe,
                             OrcProto.StripeFooter footer,
                             boolean[] included,
                             OrcProto.RowIndex[] indexes,
                             boolean[] sargColumns,
                             OrcProto.BloomFilterIndex[] bloomFilterIndices
) throws IOException {
  if (file == null) {
    open();
  }
  if (footer == null) {
    footer = readStripeFooter(stripe);
  }
  if (indexes == null) {
    indexes = new OrcProto.RowIndex[typeCount];
  }
  if (bloomFilterIndices == null) {
    bloomFilterIndices = new OrcProto.BloomFilterIndex[typeCount];
  }
  long offset = stripe.getOffset();
  List<OrcProto.Stream> streams = footer.getStreamsList();
  for (int i = 0; i < streams.size(); i++) {
    OrcProto.Stream stream = streams.get(i);
    OrcProto.Stream nextStream = null;
    if (i < streams.size() - 1) {
      nextStream = streams.get(i+1);
    }
    int col = stream.getColumn();
    int len = (int) stream.getLength();
    // row index stream and bloom filter are interlaced, check if the sarg column contains bloom
    // filter and combine the io to read row index and bloom filters for that column together
    if (stream.hasKind() && (stream.getKind() == OrcProto.Stream.Kind.ROW_INDEX)) {
      boolean readBloomFilter = false;
      if (sargColumns != null && sargColumns[col] &&
        nextStream.getKind() == OrcProto.Stream.Kind.BLOOM_FILTER) {
        len += nextStream.getLength();
        i += 1;
        readBloomFilter = true;
      }
      if ((included == null || included[col]) && indexes[col] == null) {
        byte[] buffer = new byte[len];
        file.readFully(offset, buffer, 0, buffer.length);
        ByteBuffer bb = ByteBuffer.wrap(buffer);
        ByteBuffer rowIndexBB = bb.duplicate();
        rowIndexBB.position(0);
        rowIndexBB.limit((int)stream.getLength());
        indexes[col] = OrcProto.RowIndex.parseFrom(InStream.create("index",
          Lists.<DiskRange>newArrayList(new BufferChunk(rowIndexBB, 0)), stream.getLength(),
          codec, bufferSize, null));
        if (readBloomFilter) {
          ByteBuffer bloomFilterBB = bb.duplicate();
          bloomFilterBB.position((int)stream.getLength());
          bloomFilterBB.limit(buffer.length);
          bloomFilterIndices[col] = OrcProto.BloomFilterIndex.parseFrom(InStream.create(
            "bloom_filter", Lists.<DiskRange>newArrayList(new BufferChunk(bloomFilterBB, 0)),
            nextStream.getLength(), codec, bufferSize, null));
        }
      }
    }
    offset += len;
  }

  OrcIndex index = new OrcIndex(indexes, bloomFilterIndices);
  return index;
}