Java Code Examples for com.google.common.collect.ImmutableMap#builderWithExpectedSize()

The following examples show how to use com.google.common.collect.ImmutableMap#builderWithExpectedSize() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: PostAnalysisQueryEnvironment.java    From bazel with Apache License 2.0 6 votes vote down vote up
public TopLevelConfigurations(
    Collection<TargetAndConfiguration> topLevelTargetsAndConfigurations) {
  ImmutableMap.Builder<Label, BuildConfiguration> nonNullsBuilder =
      ImmutableMap.builderWithExpectedSize(topLevelTargetsAndConfigurations.size());
  ImmutableList.Builder<Label> nullsBuilder = new ImmutableList.Builder<>();
  for (TargetAndConfiguration targetAndConfiguration : topLevelTargetsAndConfigurations) {
    if (targetAndConfiguration.getConfiguration() == null) {
      nullsBuilder.add(targetAndConfiguration.getLabel());
    } else {
      nonNullsBuilder.put(
          targetAndConfiguration.getLabel(), targetAndConfiguration.getConfiguration());
    }
  }
  nonNulls = nonNullsBuilder.build();
  nonNullConfigs =
      ImmutableSortedSet.copyOf(
          Comparator.comparing(BuildConfiguration::checksum), nonNulls.values());
  nulls = nullsBuilder.build();
}
 
Example 2
Source File: ActionExecutionValue.java    From bazel with Apache License 2.0 6 votes vote down vote up
private static <V> ImmutableMap<Artifact, V> transformMap(
    ImmutableMap<Artifact, V> data,
    Map<OwnerlessArtifactWrapper, Artifact> newArtifactMap,
    BiFunction<Artifact, V, V> transform) {
  if (data.isEmpty()) {
    return data;
  }

  ImmutableMap.Builder<Artifact, V> result = ImmutableMap.builderWithExpectedSize(data.size());
  for (Map.Entry<Artifact, V> entry : data.entrySet()) {
    Artifact artifact = entry.getKey();
    Artifact newArtifact =
        Preconditions.checkNotNull(
            newArtifactMap.get(new OwnerlessArtifactWrapper(artifact)),
            "Output artifact %s from one shared action not present in another's outputs (%s)",
            artifact,
            newArtifactMap);
    result.put(newArtifact, transform.apply(newArtifact, entry.getValue()));
  }
  return result.build();
}
 
Example 3
Source File: PathReferenceRuleWithMultipleOutputs.java    From buck with Apache License 2.0 6 votes vote down vote up
public PathReferenceRuleWithMultipleOutputs(
    BuildTarget buildTarget,
    ProjectFilesystem projectFilesystem,
    Path source,
    ImmutableMap<OutputLabel, ImmutableSet<Path>> outputLabelsToOutputs) {
  super(buildTarget, projectFilesystem, source);
  ImmutableMap.Builder<OutputLabel, ImmutableSortedSet<SourcePath>> builder =
      ImmutableMap.builderWithExpectedSize(1 + outputLabelsToOutputs.size());
  if (!outputLabelsToOutputs.containsKey(OutputLabel.defaultLabel())) {
    builder.put(
        OutputLabel.defaultLabel(),
        source == null
            ? ImmutableSortedSet.of()
            : ImmutableSortedSet.of(ExplicitBuildTargetSourcePath.of(getBuildTarget(), source)));
  }
  for (Map.Entry<OutputLabel, ImmutableSet<Path>> entry : outputLabelsToOutputs.entrySet()) {
    builder.put(
        entry.getKey(),
        entry.getValue().stream()
            .map(path -> ExplicitBuildTargetSourcePath.of(getBuildTarget(), path))
            .collect(ImmutableSortedSet.toImmutableSortedSet(Ordering.natural())));
  }
  outputLabelsToSourcePaths = builder.build();
}
 
Example 4
Source File: TargetCountVerificationParserDecoratorTest.java    From buck with Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() {
  eventBus = BuckEventBusForTests.newInstance();
  capturingConsoleEventListener = new CapturingConsoleEventListener();
  eventBus.register(capturingConsoleEventListener);
  path = Paths.get("/foo/bar");
  parserMock = EasyMock.createMock(ProjectBuildFileParser.class);

  Map<String, Object> retMap1 = new HashMap<>();
  retMap1.put("a", "a");
  retMap1.put("b", "b");
  retMap1.put("c", "c");
  retMap1.put("d", "d");
  retMap1.put("e", "e");

  String[] names = {"a", "b", "c", "d", "e"};
  ImmutableMap.Builder<String, ImmutableMap<String, Object>> builder =
      ImmutableMap.builderWithExpectedSize(names.length);
  for (String name : names) {
    builder.put(name, ImmutableMap.copyOf(retMap1));
  }

  rawTargets = builder.build();
}
 
Example 5
Source File: RemoteExecutionEventListener.java    From buck with Apache License 2.0 6 votes vote down vote up
@Override
public ImmutableMap<String, String> exportFieldsToMap() {
  ImmutableMap.Builder<String, String> retval = ImmutableMap.builderWithExpectedSize(16);

  retval
      .put("cas_downloads_count", Integer.toString(getCasDownloads()))
      .put("cas_downloads_bytes", Long.toString(getCasDownloadSizeBytes()))
      .put("cas_uploads_count", Integer.toString(getCasUploads()))
      .put("cas_uploads_bytes", Long.toString(getCasUploadSizeBytes()))
      .put("localfallback_totally_executed_rules", localFallbackTotalExecutions.toString())
      .put("localfallback_locally_executed_rules", localFallbackLocalExecutions.toString())
      .put(
          "localfallback_locally_successful_executed_rules",
          localFallbackSuccessfulLocalExecutions.toString())
      .put("remote_cpu_time_ms", Long.toString(getRemoteCpuTimeMs()))
      .put("remote_queue_time_ms", Long.toString(getRemoteQueueTimeMs()))
      .put("remote_total_time_ms", Long.toString(getTotalRemoteTimeMs()));

  for (ImmutableMap.Entry<State, Integer> entry : getActionsPerState().entrySet()) {
    retval.put(
        String.format("remote_state_%s_count", entry.getKey().getAbbreviateName()),
        Integer.toString(entry.getValue()));
  }

  return retval.build();
}
 
Example 6
Source File: DataNodeContainerModificationStrategy.java    From yangtools with Eclipse Public License 1.0 6 votes vote down vote up
private Optional<ModificationApplyOperation> appendChild(
        final ImmutableMap<PathArgument, ModificationApplyOperation> initial, final PathArgument identifier,
        final ModificationApplyOperation computed) {

    ImmutableMap<PathArgument, ModificationApplyOperation> previous = initial;
    while (true) {
        // Build up a new map based on observed snapshot and computed child
        final Builder<PathArgument, ModificationApplyOperation> builder = ImmutableMap.builderWithExpectedSize(
            previous.size() + 1);
        builder.putAll(previous);
        builder.put(identifier, computed);
        final ImmutableMap<PathArgument, ModificationApplyOperation> updated = builder.build();

        // Attempt to install the updated map
        if (UPDATER.compareAndSet(this, previous, updated)) {
            return Optional.of(computed);
        }

        // We have raced, acquire a new snapshot, recheck presence and retry if needed
        previous = children;
        final ModificationApplyOperation raced = previous.get(identifier);
        if (raced != null) {
            return Optional.of(raced);
        }
    }
}
 
Example 7
Source File: DefaultClassInfo.java    From buck with Apache License 2.0 6 votes vote down vote up
private ImmutableMap<Field, Boolean> parameterFieldsFromFields(Class<?> clazz) {
  Field[] fields = clazz.getDeclaredFields();
  ImmutableMap.Builder<Field, Boolean> fieldsBuilder =
      ImmutableMap.builderWithExpectedSize(fields.length);
  Field ignoredField = null;
  for (Field field : fields) {
    field.setAccessible(true);
    // static fields can be ignored and volatile fields are used for handling lazy field states.
    if (Modifier.isStatic(field.getModifiers()) || Modifier.isVolatile(field.getModifiers())) {
      ignoredField = field;
      continue;
    }
    boolean isLazy = ignoredField != null && ignoredField.getName().endsWith("LAZY_INIT_BIT");
    fieldsBuilder.put(field, isLazy);
  }
  return fieldsBuilder.build();
}
 
Example 8
Source File: AbstractMagnesiumDataInput.java    From yangtools with Eclipse Public License 1.0 6 votes vote down vote up
private NodeIdentifierWithPredicates readNodeIdentifierWithPredicates(final QName qname, final int size)
        throws IOException {
    if (size == 1) {
        return NodeIdentifierWithPredicates.of(qname, readQName(), readLeafValue());
    } else if (size > 1) {
        final ImmutableMap.Builder<QName, Object> builder = ImmutableMap.builderWithExpectedSize(size);
        for (int i = 0; i < size; ++i) {
            builder.put(readQName(), readLeafValue());
        }
        return NodeIdentifierWithPredicates.of(qname, builder.build());
    } else if (size == 0) {
        return NodeIdentifierWithPredicates.of(qname);
    } else {
        throw new InvalidNormalizedNodeStreamException("Invalid predicate count " + size);
    }
}
 
Example 9
Source File: BuildReport.java    From buck with Apache License 2.0 5 votes vote down vote up
/**
 * Returns a set of paths representing all outputs generated by the given {@code rule}, or null if
 * no outputs are available.
 *
 * <p>For rules that do not provide multiple outputs, the return value is null or a set of one
 * output. For rules with multiple outputs, the rule will provide at least the default output
 * group, so the return value is a set of zero or more outputs. Note that zero outputs in an
 * output group is valid.
 */
@Nullable
private ImmutableMap<OutputLabel, ImmutableSet<Path>> getMultipleOutputPaths(BuildRule rule) {
  if (rule instanceof HasMultipleOutputs) {
    HasMultipleOutputs multipleOutputsRule = (HasMultipleOutputs) rule;
    ProjectFilesystem projectFilesystem = rule.getProjectFilesystem();
    ImmutableSet<OutputLabel> outputLabels = multipleOutputsRule.getOutputLabels();
    ImmutableMap.Builder<OutputLabel, ImmutableSet<Path>> allPathsBuilder =
        ImmutableMap.builderWithExpectedSize(outputLabels.size());
    for (OutputLabel outputLabel : outputLabels) {
      ImmutableSortedSet<SourcePath> sourcePaths =
          multipleOutputsRule.getSourcePathToOutput(outputLabel);
      ImmutableSet.Builder<Path> pathBuilderForLabel =
          ImmutableSet.builderWithExpectedSize(sourcePaths.size());
      for (SourcePath sourcePath : sourcePaths) {
        pathBuilderForLabel.add(
            relativizeSourcePathToProjectRoot(projectFilesystem, sourcePath).getPath());
      }
      allPathsBuilder.put(outputLabel, pathBuilderForLabel.build());
    }
    return allPathsBuilder.build();
  }
  Path output = getRuleOutputPath(rule);
  if (output == null) {
    return null;
  }
  return ImmutableMap.of(OutputLabel.defaultLabel(), ImmutableSet.of(output));
}
 
Example 10
Source File: MainWithoutNailgun.java    From buck with Apache License 2.0 5 votes vote down vote up
private static ImmutableMap<String, String> getClientEnvironment() {
  ImmutableMap<String, String> systemEnv = EnvVariablesProvider.getSystemEnv();
  ImmutableMap.Builder<String, String> builder =
      ImmutableMap.builderWithExpectedSize(systemEnv.size());

  systemEnv.entrySet().stream()
      .filter(
          e ->
              !AnsiEnvironmentChecking.NAILGUN_STDOUT_ISTTY_ENV.equals(e.getKey())
                  && !AnsiEnvironmentChecking.NAILGUN_STDERR_ISTTY_ENV.equals(e.getKey()))
      .forEach(builder::put);
  return builder.build();
}
 
Example 11
Source File: UserDefinedProvider.java    From buck with Apache License 2.0 5 votes vote down vote up
@Override
protected Object call(Object[] args, @Nullable FuncallExpression ast, Environment env) {
  Verify.verify(isExported, "Tried to call a Provider before exporting it");

  ImmutableList<String> fieldNames =
      Objects.requireNonNull(getSignature()).getSignature().getNames();
  Verify.verify(args.length == fieldNames.size());

  ImmutableMap.Builder<String, Object> builder =
      ImmutableMap.builderWithExpectedSize(args.length);
  for (int i = 0; i < fieldNames.size(); i++) {
    builder.put(fieldNames.get(i), args[i]);
  }
  return new UserDefinedProviderInfo(this, builder.build());
}
 
Example 12
Source File: VersionBuckConfig.java    From buck with Apache License 2.0 5 votes vote down vote up
public ImmutableMap<String, VersionUniverse> getVersionUniverses(
    Optional<TargetConfiguration> targetConfiguration) {
  ImmutableSet<String> entries = delegate.getEntriesForSection(UNIVERSES_SECTION).keySet();
  ImmutableMap.Builder<String, VersionUniverse> universes =
      ImmutableMap.builderWithExpectedSize(entries.size());
  for (String name : entries) {
    universes.put(name, getVersionUniverse(name, targetConfiguration));
  }
  return universes.build();
}
 
Example 13
Source File: BuckSkylarkTypes.java    From buck with Apache License 2.0 5 votes vote down vote up
/**
 * Attempt to get a deeply immutable instance of a value passed in from Skylark
 *
 * <p>Note that if mutable objects are passed in (namely {@link SkylarkList} or {@link
 * SkylarkDict}, a copy may be made to get an immutable instance. This may happen in very deeply
 * nested structures, so the runtime is variable based on how mutable the objects are. For the
 * best performance, only immutable structures should be passed in, as that turns into a simple
 * identity function.
 *
 * @param arg A value from the skylark interpreter. This should only be primitive objects, or
 *     {@link com.google.devtools.build.lib.skylarkinterface.SkylarkValue} instances
 * @return
 *     <li>The original object if it was already an immutable {@link
 *         com.google.devtools.build.lib.skylarkinterface.SkylarkValue} or a primitive value
 *     <li>an immutable {@link SkylarkList} if the original object is a {@link SkylarkList} and
 *         all values were immutable or could be made immutable. As above, this may be a copy, or
 *         inner elements may have had to be copied if they were mutable
 *     <li>An immutable {@link SkylarkDict} if the original object is a {@link SkylarkDict} and
 *         all keys and values were immutable, or could be made so. Again, note that copies may be
 *         made in order to make mutable objects immutable
 * @throws MutableObjectException If a nested or top level object was mutable, and could not be
 *     made immutable. This generally only applies to incorrectly implemented native data types
 *     that are exported to Skylark.
 */
public static Object asDeepImmutable(Object arg) throws MutableObjectException {
  // NOTE: For most built in types, this should be reliable. However, if isImmutable is improperly
  // implemented on our custom types (that is, it returns "true" when a sub-object is not actually
  // immutable), this has the potential to allow an immutable SkylarkList/Dict that contains
  // mutable objects. We would rather not pay to iterate and double check for immutability here,
  // so for now, just assume things are implemented correctly. The number of user-accessible
  // custom objects should be vanishingly small.

  // Grab frozen objects and primitives
  if (EvalUtils.isImmutable(arg)) {
    return arg;
  }

  if (arg instanceof SkylarkList) {
    SkylarkList<?> listArg = ((SkylarkList<?>) arg);
    return listArg.stream()
        .filter(a -> !EvalUtils.isImmutable(a))
        .findFirst()
        // if we have a mutable element, like a sub list, try to make it immutable
        .map(
            mutableElement ->
                SkylarkList.createImmutable(
                    Iterables.transform(listArg, element -> asDeepImmutable(element))))
        // Else just copy the list elements into a list with an immutable mutability
        // We can't just freeze the list, as it may be mutated elsewhere, but this at least
        // elides a copy.
        .orElseGet(() -> SkylarkList.createImmutable(listArg));
  } else if (arg instanceof SkylarkDict) {
    SkylarkDict<?, ?> dictArg = (SkylarkDict<?, ?>) arg;
    ImmutableMap.Builder<Object, Object> tempBuilder =
        ImmutableMap.builderWithExpectedSize(dictArg.size());
    for (Map.Entry<?, ?> entry : dictArg.entrySet()) {
      tempBuilder.put(asDeepImmutable(entry.getKey()), asDeepImmutable(entry.getValue()));
    }
    return SkylarkDict.copyOf(null, tempBuilder.build());
  } else {
    throw new MutableObjectException(arg);
  }
}
 
Example 14
Source File: StackdriverJsonLayout.java    From curiostack with MIT License 5 votes vote down vote up
@Override
protected Object wrapLogEvent(LogEvent event) {
  Object wrapped = super.wrapLogEvent(event);

  ImmutableMap.Builder<String, String> additional =
      ImmutableMap.builderWithExpectedSize(additionalFields.length + 1);
  additional.put("severity", stackdriverSeverity(event));

  if (wrapped instanceof LogEventWithAdditionalFields) {
    additional.putAll(((LogEventWithAdditionalFields) wrapped).getAdditionalFields());
  }

  return createLogEventWithAdditionalFields(event, additional.build());
}
 
Example 15
Source File: RequestLoggingContextInjector.java    From curiostack with MIT License 5 votes vote down vote up
@Override
public StringMap injectContextData(List<Property> properties, StringMap reusable) {
  ImmutableMap<String, String> current = RequestLoggingContext.get();
  if (properties == null || properties.isEmpty()) {
    return new JdkMapAdapaterStringMap(current);
  }
  ImmutableMap.Builder<String, String> injected =
      ImmutableMap.builderWithExpectedSize(current.size() + properties.size());
  injected.putAll(current);
  for (Property prop : properties) {
    // This will throw if a property name is already in the context, don't worry about it for now.
    injected.put(prop.getName(), prop.getValue());
  }
  return new JdkMapAdapaterStringMap(injected.build());
}
 
Example 16
Source File: GenruleBuildable.java    From buck with Apache License 2.0 4 votes vote down vote up
public GenruleBuildable(
    BuildTarget buildTarget,
    ProjectFilesystem filesystem,
    SandboxExecutionStrategy sandboxExecutionStrategy,
    SourceSet srcs,
    Optional<Arg> cmd,
    Optional<Arg> bash,
    Optional<Arg> cmdExe,
    Optional<String> type,
    Optional<String> out,
    Optional<ImmutableMap<OutputLabel, ImmutableSet<String>>> outs,
    boolean enableSandboxingInGenrule,
    boolean isCacheable,
    String environmentExpansionSeparator,
    Optional<SandboxProperties> sandboxProperties,
    Optional<GenruleAndroidTools> androidTools,
    boolean executeRemotely) {
  this.buildTarget = buildTarget;
  this.sandboxExecutionStrategy = sandboxExecutionStrategy;
  this.srcs = srcs;
  this.cmd = cmd;
  this.bash = bash;
  this.cmdExe = cmdExe;
  this.type = type;
  this.out = out;
  this.outputLabelsSupplier = MoreSuppliers.memoize(this::getOutputLabelsSupplier);
  this.enableSandboxingInGenrule = enableSandboxingInGenrule;
  this.isCacheable = isCacheable;
  this.environmentExpansionSeparator = environmentExpansionSeparator;
  this.sandboxProperties = sandboxProperties;
  this.isWorkerGenrule = isWorkerGenrule();
  this.androidTools = androidTools;
  this.executeRemotely = executeRemotely;

  Preconditions.checkArgument(
      out.isPresent() ^ outs.isPresent(), "Genrule unexpectedly has both 'out' and 'outs'.");
  if (outs.isPresent()) {
    ImmutableMap<OutputLabel, ImmutableSet<String>> outputs = outs.get();
    ImmutableMap.Builder<OutputLabel, ImmutableSet<String>> outsBuilder =
        ImmutableMap.builderWithExpectedSize(outputs.size() + 1);
    ImmutableMap.Builder<OutputLabel, ImmutableSet<OutputPath>> outputPathsBuilder =
        ImmutableMap.builderWithExpectedSize(outputs.size() + 1);
    for (Map.Entry<OutputLabel, ImmutableSet<String>> outputLabelToOutputs : outputs.entrySet()) {
      OutputLabel outputLabel = outputLabelToOutputs.getKey();
      outsBuilder.put(outputLabel, outputLabelToOutputs.getValue());
      outputPathsBuilder.put(
          outputLabel,
          outputLabelToOutputs.getValue().stream()
              .map(
                  p -> {
                    Path path = Paths.get(p);
                    return new OutputPath(path);
                  })
              .collect(ImmutableSet.toImmutableSet()));
    }
    if (!outputs.containsKey(OutputLabel.defaultLabel())) {
      outsBuilder.put(OutputLabel.defaultLabel(), DEFAULT_OUTS);
      outputPathsBuilder.put(OutputLabel.defaultLabel(), DEFAULT_OUTPUTS);
    }
    this.outs = Optional.of(outsBuilder.build());
    this.outputPaths = Optional.of(outputPathsBuilder.build());
    this.outputPath = Optional.empty();
  } else {
    this.outs = Optional.empty();
    // Sanity check for the output paths.
    this.outputPath = Optional.of(new PublicOutputPath(getLegacyPath(filesystem, out.get())));
    this.outputPaths = Optional.empty();
  }
}
 
Example 17
Source File: SkyFunctionEnvironment.java    From bazel with Apache License 2.0 4 votes vote down vote up
private ImmutableMap<SkyKey, SkyValue> batchPrefetch(
    SkyKey requestor, GroupedList<SkyKey> depKeys, Set<SkyKey> oldDeps, boolean assertDone)
    throws InterruptedException, UndonePreviouslyRequestedDeps {
  QueryableGraph.PrefetchDepsRequest request = null;
  if (PREFETCH_OLD_DEPS) {
    request = new QueryableGraph.PrefetchDepsRequest(requestor, oldDeps, depKeys);
    evaluatorContext.getGraph().prefetchDeps(request);
  }
  Map<SkyKey, ? extends NodeEntry> batchMap =
      evaluatorContext.getBatchValues(
          requestor,
          Reason.PREFETCH,
          (request != null && request.excludedKeys != null)
              ? request.excludedKeys
              : depKeys.getAllElementsAsIterable());
  if (batchMap.size() != depKeys.numElements()) {
    Set<SkyKey> difference = Sets.difference(depKeys.toSet(), batchMap.keySet());
    evaluatorContext
        .getGraphInconsistencyReceiver()
        .noteInconsistencyAndMaybeThrow(
            requestor, difference, Inconsistency.ALREADY_DECLARED_CHILD_MISSING);
    throw new UndonePreviouslyRequestedDeps(ImmutableList.copyOf(difference));
  }
  ImmutableMap.Builder<SkyKey, SkyValue> depValuesBuilder =
      ImmutableMap.builderWithExpectedSize(batchMap.size());
  for (Entry<SkyKey, ? extends NodeEntry> entry : batchMap.entrySet()) {
    SkyValue valueMaybeWithMetadata = entry.getValue().getValueMaybeWithMetadata();
    boolean depDone = valueMaybeWithMetadata != null;
    if (assertDone && !depDone) {
      // A previously requested dep may have transitioned from done to dirty between when the node
      // was read during a previous attempt to build this node and now. Notify the graph
      // inconsistency receiver so that we can crash if that's unexpected.
      evaluatorContext
          .getGraphInconsistencyReceiver()
          .noteInconsistencyAndMaybeThrow(
              skyKey,
              ImmutableList.of(entry.getKey()),
              Inconsistency.BUILDING_PARENT_FOUND_UNDONE_CHILD);
      throw new UndonePreviouslyRequestedDeps(ImmutableList.of(entry.getKey()));
    }
    depValuesBuilder.put(entry.getKey(), !depDone ? NULL_MARKER : valueMaybeWithMetadata);
    if (depDone) {
      maybeUpdateMaxChildVersion(entry.getValue());
    }
  }
  return depValuesBuilder.build();
}
 
Example 18
Source File: IcebergMetadata.java    From presto with Apache License 2.0 4 votes vote down vote up
@Override
public ConnectorOutputTableHandle beginCreateTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, Optional<ConnectorNewTableLayout> layout)
{
    SchemaTableName schemaTableName = tableMetadata.getTable();
    String schemaName = schemaTableName.getSchemaName();
    String tableName = schemaTableName.getTableName();

    Schema schema = toIcebergSchema(tableMetadata.getColumns());

    PartitionSpec partitionSpec = parsePartitionFields(schema, getPartitioning(tableMetadata.getProperties()));

    Database database = metastore.getDatabase(schemaName)
            .orElseThrow(() -> new SchemaNotFoundException(schemaName));

    HdfsContext hdfsContext = new HdfsContext(session, schemaName, tableName);
    HiveIdentity identity = new HiveIdentity(session);
    String targetPath = getTableLocation(tableMetadata.getProperties());
    if (targetPath == null) {
        targetPath = getTableDefaultLocation(database, hdfsContext, hdfsEnvironment, schemaName, tableName).toString();
    }

    TableOperations operations = new HiveTableOperations(metastore, hdfsEnvironment, hdfsContext, identity, schemaName, tableName, session.getUser(), targetPath);
    if (operations.current() != null) {
        throw new TableAlreadyExistsException(schemaTableName);
    }

    ImmutableMap.Builder<String, String> propertiesBuilder = ImmutableMap.builderWithExpectedSize(2);
    FileFormat fileFormat = getFileFormat(tableMetadata.getProperties());
    propertiesBuilder.put(DEFAULT_FILE_FORMAT, fileFormat.toString());
    if (tableMetadata.getComment().isPresent()) {
        propertiesBuilder.put(TABLE_COMMENT, tableMetadata.getComment().get());
    }

    TableMetadata metadata = newTableMetadata(operations, schema, partitionSpec, targetPath, propertiesBuilder.build());

    transaction = createTableTransaction(operations, metadata);

    return new IcebergWritableTableHandle(
            schemaName,
            tableName,
            SchemaParser.toJson(metadata.schema()),
            PartitionSpecParser.toJson(metadata.spec()),
            getColumns(metadata.schema(), typeManager),
            targetPath,
            fileFormat);
}
 
Example 19
Source File: RemoteCache.java    From bazel with Apache License 2.0 4 votes vote down vote up
private void injectRemoteArtifact(
    Artifact output,
    ActionResultMetadata metadata,
    Path execRoot,
    MetadataInjector metadataInjector,
    String actionId)
    throws IOException {
  if (output.isTreeArtifact()) {
    DirectoryMetadata directory =
        metadata.directory(execRoot.getRelative(output.getExecPathString()));
    if (directory == null) {
      // A declared output wasn't created. It might have been an optional output and if not
      // SkyFrame will make sure to fail.
      return;
    }
    if (!directory.symlinks().isEmpty()) {
      throw new IOException(
          "Symlinks in action outputs are not yet supported by "
              + "--experimental_remote_download_outputs=minimal");
    }
    SpecialArtifact parent = (SpecialArtifact) output;
    ImmutableMap.Builder<TreeFileArtifact, FileArtifactValue> childMetadata =
        ImmutableMap.builderWithExpectedSize(directory.files.size());
    for (FileMetadata file : directory.files()) {
      TreeFileArtifact child =
          TreeFileArtifact.createTreeOutput(parent, file.path().relativeTo(parent.getPath()));
      RemoteFileArtifactValue value =
          new RemoteFileArtifactValue(
              DigestUtil.toBinaryDigest(file.digest()),
              file.digest().getSizeBytes(),
              /*locationIndex=*/ 1,
              actionId);
      childMetadata.put(child, value);
    }
    metadataInjector.injectDirectory(parent, childMetadata.build());
  } else {
    FileMetadata outputMetadata = metadata.file(execRoot.getRelative(output.getExecPathString()));
    if (outputMetadata == null) {
      // A declared output wasn't created. It might have been an optional output and if not
      // SkyFrame will make sure to fail.
      return;
    }
    metadataInjector.injectFile(
        output,
        new RemoteFileArtifactValue(
            DigestUtil.toBinaryDigest(outputMetadata.digest()),
            outputMetadata.digest().getSizeBytes(),
            /*locationIndex=*/ 1,
            actionId));
  }
}
 
Example 20
Source File: PythonDslProjectBuildFileParser.java    From buck with Apache License 2.0 4 votes vote down vote up
/** Initialize the parser, starting buck.py. */
private void init() throws IOException {
  try (SimplePerfEvent.Scope scope =
      SimplePerfEvent.scope(buckEventBus, SimplePerfEvent.PerfEventId.of("ParserInit"))) {

    ImmutableMap.Builder<String, String> pythonEnvironmentBuilder =
        ImmutableMap.builderWithExpectedSize(environment.size());
    // Strip out PYTHONPATH. buck.py manually sets this to include only nailgun. We don't want
    // to inject nailgun into the parser's PYTHONPATH, so strip that value out.
    // If we wanted to pass on some environmental PYTHONPATH, we would have to do some actual
    // merging of this and the BuckConfig's python module search path.
    // Also ignore PYTHONHASHSEED environment variable passed by clients since Buck manages it to
    // prevent non-determinism.
    pythonEnvironmentBuilder.putAll(
        Maps.filterKeys(
            environment,
            k -> !PYTHONPATH_ENV_VAR_NAME.equals(k) && !PYTHON_HASH_SEED_ENV_VAR_NAME.equals(k)));
    // set Python hash seed to a fixed number to make parsing reproducible
    pythonEnvironmentBuilder.put(PYTHON_HASH_SEED_ENV_VAR_NAME, PYTHON_HASH_SEED_VALUE);

    if (options.getPythonModuleSearchPath().isPresent()) {
      pythonEnvironmentBuilder.put(
          PYTHONPATH_ENV_VAR_NAME, options.getPythonModuleSearchPath().get());
    }

    ImmutableMap<String, String> pythonEnvironment = pythonEnvironmentBuilder.build();

    ProcessExecutorParams params =
        ProcessExecutorParams.builder()
            .setCommand(buildArgs())
            .setEnvironment(pythonEnvironment)
            .build();

    LOG.debug(
        "Starting buck.py command: %s environment: %s",
        params.getCommand(), params.getEnvironment());
    buckPyProcess = processExecutor.launchProcess(params);
    LOG.debug("Started process %s successfully", buckPyProcess);
    buckPyProcessInput =
        createParserInputStream(
            Objects.requireNonNull(buckPyProcess).getStdout(), processedBytes.isPresent());
    buckPyProcessJsonGenerator = ObjectMappers.createGenerator(buckPyProcess.getStdin());

    // Explicitly use Reader instead of InputStream because in case if InputStream is provided
    // Jackson tries to detect encoding be reading first bytes; because the process may not
    // output anything at this moment yet this can hang indefinitely.
    buckPyProcessJsonParser =
        ObjectMappers.createParser(
            new InputStreamReader(
                Objects.requireNonNull(buckPyProcessInput).getInputStream(), Charsets.UTF_8));

    InputStream stderr = buckPyProcess.getStderr();

    AtomicInteger numberOfLines = new AtomicInteger(0);
    AtomicReference<Path> lastPath = new AtomicReference<Path>();
    InputStreamConsumer stderrConsumer =
        new InputStreamConsumer(
            stderr,
            (InputStreamConsumer.Handler)
                line -> {
                  Path path = currentBuildFile.get();
                  if (!Objects.equals(path, lastPath.get())) {
                    numberOfLines.set(0);
                    lastPath.set(path);
                  }
                  int count = numberOfLines.getAndIncrement();
                  if (count == 0) {
                    buckEventBus.post(
                        ConsoleEvent.warning("WARNING: Output when parsing %s:", path));
                  }
                  buckEventBus.post(ConsoleEvent.warning("| %s", line));
                });
    stderrConsumerTerminationFuture = new FutureTask<>(stderrConsumer);
    stderrConsumerThread =
        Threads.namedThread(
            PythonDslProjectBuildFileParser.class.getSimpleName(),
            stderrConsumerTerminationFuture);
    stderrConsumerThread.start();
  }
}