List of usage examples for com.google.common.collect ImmutableSortedMap.Builder put
V put(K key, V value);
From source file:org.apache.phoenix.schema.PTableImpl.java
private void init(PName tenantId, PName schemaName, PName tableName, PTableType type, PIndexState state, long timeStamp, long sequenceNumber, PName pkName, Integer bucketNum, List<PColumn> columns, PTableStats stats, PName parentSchemaName, PName parentTableName, List<PTable> indexes, boolean isImmutableRows, List<PName> physicalNames, PName defaultFamilyName, String viewExpression, boolean disableWAL, boolean multiTenant, boolean storeNulls, ViewType viewType, Short viewIndexId, IndexType indexType, int baseColumnCount, boolean rowKeyOrderOptimizable) throws SQLException { Preconditions.checkNotNull(schemaName); Preconditions.checkArgument(tenantId == null || tenantId.getBytes().length > 0); // tenantId should be null or not empty int estimatedSize = SizedUtil.OBJECT_SIZE * 2 + 23 * SizedUtil.POINTER_SIZE + 4 * SizedUtil.INT_SIZE + 2 * SizedUtil.LONG_SIZE + 2 * SizedUtil.INT_OBJECT_SIZE + PNameFactory.getEstimatedSize(tenantId) + PNameFactory.getEstimatedSize(schemaName) + PNameFactory.getEstimatedSize(tableName) + PNameFactory.getEstimatedSize(pkName) + PNameFactory.getEstimatedSize(parentTableName) + PNameFactory.getEstimatedSize(defaultFamilyName); this.tenantId = tenantId; this.schemaName = schemaName; this.tableName = tableName; this.name = PNameFactory.newName(SchemaUtil.getTableName(schemaName.getString(), tableName.getString())); this.key = new PTableKey(tenantId, name.getString()); this.type = type; this.state = state; this.timeStamp = timeStamp; this.sequenceNumber = sequenceNumber; this.pkName = pkName; this.isImmutableRows = isImmutableRows; this.defaultFamilyName = defaultFamilyName; this.viewStatement = viewExpression; this.disableWAL = disableWAL; this.multiTenant = multiTenant; this.storeNulls = storeNulls; this.viewType = viewType; this.viewIndexId = viewIndexId; this.indexType = indexType; this.tableStats = stats; this.rowKeyOrderOptimizable = rowKeyOrderOptimizable; List<PColumn> pkColumns; PColumn[] allColumns;/* w w w. j ava 2 s. co m*/ this.columnsByName = ArrayListMultimap.create(columns.size(), 1); int numPKColumns = 0; if (bucketNum != null) { // Add salt column to allColumns and pkColumns, but don't add to // columnsByName, since it should not be addressable via name. allColumns = new PColumn[columns.size() + 1]; allColumns[SALTING_COLUMN.getPosition()] = SALTING_COLUMN; pkColumns = Lists.newArrayListWithExpectedSize(columns.size() + 1); ++numPKColumns; } else { allColumns = new PColumn[columns.size()]; pkColumns = Lists.newArrayListWithExpectedSize(columns.size()); } for (int i = 0; i < columns.size(); i++) { PColumn column = columns.get(i); allColumns[column.getPosition()] = column; PName familyName = column.getFamilyName(); if (familyName == null) { ++numPKColumns; } String columnName = column.getName().getString(); if (columnsByName.put(columnName, column)) { int count = 0; for (PColumn dupColumn : columnsByName.get(columnName)) { if (Objects.equal(familyName, dupColumn.getFamilyName())) { count++; if (count > 1) { throw new ColumnAlreadyExistsException(null, name.getString(), columnName); } } } } } estimatedSize += SizedUtil.sizeOfMap(allColumns.length, SizedUtil.POINTER_SIZE, SizedUtil.sizeOfArrayList(1)); // for multi-map this.bucketNum = bucketNum; this.allColumns = ImmutableList.copyOf(allColumns); estimatedSize += SizedUtil.sizeOfMap(numPKColumns) + SizedUtil.sizeOfMap(allColumns.length); RowKeySchemaBuilder builder = new RowKeySchemaBuilder(numPKColumns); // Two pass so that column order in column families matches overall column order // and to ensure that column family order is constant int maxExpectedSize = allColumns.length - numPKColumns; // Maintain iteration order so that column families are ordered as they are listed Map<PName, List<PColumn>> familyMap = Maps.newLinkedHashMap(); PColumn rowTimestampCol = null; for (PColumn column : allColumns) { PName familyName = column.getFamilyName(); if (familyName == null) { hasColumnsRequiringUpgrade |= (column.getSortOrder() == SortOrder.DESC && (!column.getDataType().isFixedWidth() || column.getDataType() == PChar.INSTANCE || column.getDataType() == PFloat.INSTANCE || column.getDataType() == PDouble.INSTANCE || column.getDataType() == PBinary.INSTANCE)) || (column.getSortOrder() == SortOrder.ASC && column.getDataType() == PBinary.INSTANCE && column.getMaxLength() != null && column.getMaxLength() > 1); pkColumns.add(column); if (column.isRowTimestamp()) { rowTimestampCol = column; } } if (familyName == null) { estimatedSize += column.getEstimatedSize(); // PK columns builder.addField(column, column.isNullable(), column.getSortOrder()); } else { List<PColumn> columnsInFamily = familyMap.get(familyName); if (columnsInFamily == null) { columnsInFamily = Lists.newArrayListWithExpectedSize(maxExpectedSize); familyMap.put(familyName, columnsInFamily); } columnsInFamily.add(column); } } this.pkColumns = ImmutableList.copyOf(pkColumns); if (rowTimestampCol != null) { this.rowTimestampColPos = this.pkColumns.indexOf(rowTimestampCol); } else { this.rowTimestampColPos = -1; } builder.rowKeyOrderOptimizable(this.rowKeyOrderOptimizable()); // after hasDescVarLengthColumns is calculated this.rowKeySchema = builder.build(); estimatedSize += rowKeySchema.getEstimatedSize(); Iterator<Map.Entry<PName, List<PColumn>>> iterator = familyMap.entrySet().iterator(); PColumnFamily[] families = new PColumnFamily[familyMap.size()]; estimatedSize += this.tableStats.getEstimatedSize(); ImmutableMap.Builder<String, PColumnFamily> familyByString = ImmutableMap.builder(); ImmutableSortedMap.Builder<byte[], PColumnFamily> familyByBytes = ImmutableSortedMap .orderedBy(Bytes.BYTES_COMPARATOR); for (int i = 0; i < families.length; i++) { Map.Entry<PName, List<PColumn>> entry = iterator.next(); PColumnFamily family = new PColumnFamilyImpl(entry.getKey(), entry.getValue()); families[i] = family; familyByString.put(family.getName().getString(), family); familyByBytes.put(family.getName().getBytes(), family); estimatedSize += family.getEstimatedSize(); } this.families = ImmutableList.copyOf(families); this.familyByBytes = familyByBytes.build(); this.familyByString = familyByString.build(); estimatedSize += SizedUtil.sizeOfArrayList(families.length); estimatedSize += SizedUtil.sizeOfMap(families.length) * 2; this.indexes = indexes == null ? Collections.<PTable>emptyList() : indexes; for (PTable index : this.indexes) { estimatedSize += index.getEstimatedSize(); } this.parentSchemaName = parentSchemaName; this.parentTableName = parentTableName; this.parentName = parentTableName == null ? null : PNameFactory.newName( SchemaUtil.getTableName(parentSchemaName.getString(), parentTableName.getString())); estimatedSize += PNameFactory.getEstimatedSize(this.parentName); this.physicalNames = physicalNames == null ? ImmutableList.<PName>of() : ImmutableList.copyOf(physicalNames); for (PName name : this.physicalNames) { estimatedSize += name.getEstimatedSize(); } this.estimatedSize = estimatedSize; this.baseColumnCount = baseColumnCount; }
From source file:com.facebook.buck.jvm.java.AbstractResourcesParameters.java
public static ImmutableSortedMap<String, SourcePath> getNamedResources(SourcePathResolver pathResolver, SourcePathRuleFinder ruleFinder, ProjectFilesystem filesystem, ImmutableCollection<SourcePath> resources) { ImmutableSortedMap.Builder<String, SourcePath> builder = ImmutableSortedMap.naturalOrder(); for (SourcePath rawResource : resources) { // If the path to the file defining this rule were: // "first-party/orca/lib-http/tests/com/facebook/orca/BUCK" ////from w ww.jav a 2 s. c o m // And the value of resource were: // "first-party/orca/lib-http/tests/com/facebook/orca/protocol/base/batch_exception1.txt" // // Assuming that `src_roots = tests` were in the [java] section of the .buckconfig file, // then javaPackageAsPath would be: // "com/facebook/orca/protocol/base/" // // And the path that we would want to copy to the classes directory would be: // "com/facebook/orca/protocol/base/batch_exception1.txt" // // Therefore, some path-wrangling is required to produce the correct string. Optional<BuildRule> underlyingRule = ruleFinder.getRule(rawResource); Path relativePathToResource = pathResolver.getRelativePath(rawResource); String resource; if (underlyingRule.isPresent()) { BuildTarget underlyingTarget = underlyingRule.get().getBuildTarget(); if (underlyingRule.get() instanceof HasOutputName) { resource = MorePaths.pathWithUnixSeparators(underlyingTarget.getBasePath() .resolve(((HasOutputName) underlyingRule.get()).getOutputName())); } else { Path genOutputParent = BuildTargetPaths.getGenPath(filesystem, underlyingTarget, "%s") .getParent(); Path scratchOutputParent = BuildTargetPaths.getScratchPath(filesystem, underlyingTarget, "%s") .getParent(); Optional<Path> outputPath = MorePaths.stripPrefix(relativePathToResource, genOutputParent) .map(Optional::of) .orElse(MorePaths.stripPrefix(relativePathToResource, scratchOutputParent)); Preconditions.checkState(outputPath.isPresent(), "%s is used as a resource but does not output to a default output directory", underlyingTarget.getFullyQualifiedName()); resource = MorePaths .pathWithUnixSeparators(underlyingTarget.getBasePath().resolve(outputPath.get())); } } else { resource = MorePaths.pathWithUnixSeparators(relativePathToResource); } builder.put(resource, rawResource); } return builder.build(); }
From source file:com.facebook.buck.features.haskell.HaskellLibraryDescription.java
private HaskellHaddockLibRule requireHaddockLibrary(BuildTarget baseTarget, ProjectFilesystem projectFilesystem, BuildRuleParams baseParams, ActionGraphBuilder graphBuilder, SourcePathResolver pathResolver, SourcePathRuleFinder ruleFinder, HaskellPlatform platform, HaskellLibraryDescriptionArg args) { CxxPlatform cxxPlatform = platform.getCxxPlatform(); CxxDeps allDeps = CxxDeps.builder().addDeps(args.getDeps()).addPlatformDeps(args.getPlatformDeps()).build(); ImmutableSet<BuildRule> deps = allDeps.get(graphBuilder, cxxPlatform); // Collect all Haskell deps ImmutableSet.Builder<SourcePath> haddockInterfaces = ImmutableSet.builder(); ImmutableSortedMap.Builder<String, HaskellPackage> packagesBuilder = ImmutableSortedMap.naturalOrder(); ImmutableSortedMap.Builder<String, HaskellPackage> exposedPackagesBuilder = ImmutableSortedMap .naturalOrder();//from ww w . ja v a2 s . co m // Traverse all deps to pull interfaces new AbstractBreadthFirstTraversal<BuildRule>(deps) { @Override public Iterable<BuildRule> visit(BuildRule rule) { ImmutableSet.Builder<BuildRule> traverse = ImmutableSet.builder(); if (rule instanceof HaskellCompileDep) { HaskellCompileDep haskellCompileDep = (HaskellCompileDep) rule; // Get haddock-interfaces HaskellHaddockInput inp = haskellCompileDep.getHaddockInput(platform); haddockInterfaces.addAll(inp.getInterfaces()); HaskellCompileInput compileInput = haskellCompileDep.getCompileInput(platform, Linker.LinkableDepType.STATIC, args.isEnableProfiling()); boolean firstOrderDep = deps.contains(rule); for (HaskellPackage pkg : compileInput.getPackages()) { if (firstOrderDep) { exposedPackagesBuilder.put(pkg.getInfo().getIdentifier(), pkg); } else { packagesBuilder.put(pkg.getInfo().getIdentifier(), pkg); } } traverse.addAll(haskellCompileDep.getCompileDeps(platform)); } return traverse.build(); } }.start(); Collection<CxxPreprocessorInput> cxxPreprocessorInputs = CxxPreprocessables .getTransitiveCxxPreprocessorInput(cxxPlatform, graphBuilder, deps); ExplicitCxxToolFlags.Builder toolFlagsBuilder = CxxToolFlags.explicitBuilder(); PreprocessorFlags.Builder ppFlagsBuilder = PreprocessorFlags.builder(); toolFlagsBuilder.setPlatformFlags( StringArg.from(CxxSourceTypes.getPlatformPreprocessFlags(cxxPlatform, CxxSource.Type.C))); for (CxxPreprocessorInput input : cxxPreprocessorInputs) { ppFlagsBuilder.addAllIncludes(input.getIncludes()); ppFlagsBuilder.addAllFrameworkPaths(input.getFrameworks()); toolFlagsBuilder.addAllRuleFlags(input.getPreprocessorFlags().get(CxxSource.Type.C)); } ppFlagsBuilder.setOtherFlags(toolFlagsBuilder.build()); return graphBuilder.addToIndex(HaskellHaddockLibRule .from(baseTarget.withAppendedFlavors(Type.HADDOCK.getFlavor(), platform.getFlavor()), projectFilesystem, baseParams, ruleFinder, HaskellSources.from(baseTarget, graphBuilder, pathResolver, ruleFinder, platform, "srcs", args.getSrcs()), platform.getHaddock().resolve(graphBuilder), args.getHaddockFlags(), args.getCompilerFlags(), platform.getLinkerFlags(), haddockInterfaces.build(), packagesBuilder.build(), exposedPackagesBuilder.build(), getPackageInfo(platform, baseTarget), platform, CxxSourceTypes .getPreprocessor(platform.getCxxPlatform(), CxxSource.Type.C).resolve(graphBuilder), ppFlagsBuilder.build())); }
From source file:com.facebook.buck.rules.CachingBuildRuleBuilder.java
ListenableFuture<BuildResult> build() {
final AtomicReference<Long> outputSize = Atomics.newReference();
ListenableFuture<List<BuildResult>> depResults = Futures.immediateFuture(Collections.emptyList());
// If we're performing a deep build, guarantee that all dependencies will *always* get
// materialized locally
if (buildMode == CachingBuildEngine.BuildMode.DEEP
|| buildMode == CachingBuildEngine.BuildMode.POPULATE_FROM_REMOTE_CACHE) {
depResults = buildRuleBuilderDelegate.getDepResults(rule, buildContext, executionContext);
}/*from ww w . java2s .c o m*/
ListenableFuture<BuildResult> buildResult = Futures.transformAsync(depResults,
input -> buildOrFetchFromCache(),
serviceByAdjustingDefaultWeightsTo(CachingBuildEngine.SCHEDULING_MORE_WORK_RESOURCE_AMOUNTS));
// Check immediately (without posting a new task) for a failure so that we can short-circuit
// pending work. Use .catchingAsync() instead of .catching() so that we can propagate unchecked
// exceptions.
buildResult = Futures.catchingAsync(buildResult, Throwable.class, throwable -> {
Preconditions.checkNotNull(throwable);
buildRuleBuilderDelegate.setFirstFailure(throwable);
Throwables.throwIfInstanceOf(throwable, Exception.class);
throw new RuntimeException(throwable);
});
buildResult = Futures.transform(buildResult, (result) -> {
buildRuleBuilderDelegate.markRuleAsUsed(rule, buildContext.getEventBus());
return result;
}, MoreExecutors.directExecutor());
// Setup a callback to handle either the cached or built locally cases.
AsyncFunction<BuildResult, BuildResult> callback = input -> {
// If we weren't successful, exit now.
if (input.getStatus() != BuildRuleStatus.SUCCESS) {
return Futures.immediateFuture(input);
}
try (Scope scope = LeafEvents.scope(buildContext.getEventBus(), "finalizing_build_rule")) {
// We shouldn't see any build fail result at this point.
BuildRuleSuccessType success = Preconditions.checkNotNull(input.getSuccess());
// If we didn't build the rule locally, reload the recorded paths from the build
// metadata.
if (success != BuildRuleSuccessType.BUILT_LOCALLY) {
try {
for (String str : onDiskBuildInfo.getValuesOrThrow(BuildInfo.MetadataKey.RECORDED_PATHS)) {
buildInfoRecorder.recordArtifact(Paths.get(str));
}
} catch (IOException e) {
LOG.error(e, "Failed to read RECORDED_PATHS for %s", rule);
throw e;
}
}
// Try get the output size now that all outputs have been recorded.
if (success == BuildRuleSuccessType.BUILT_LOCALLY) {
outputSize.set(buildInfoRecorder.getOutputSize());
}
// If the success type means the rule has potentially changed it's outputs...
if (success.outputsHaveChanged()) {
// The build has succeeded, whether we've fetched from cache, or built locally.
// So run the post-build steps.
if (rule instanceof HasPostBuildSteps) {
executePostBuildSteps(
((HasPostBuildSteps) rule).getPostBuildSteps(buildContext.getBuildContext()));
}
// Invalidate any cached hashes for the output paths, since we've updated them.
for (Path path : buildInfoRecorder.getRecordedPaths()) {
fileHashCache.invalidate(rule.getProjectFilesystem().resolve(path));
}
}
if (SupportsInputBasedRuleKey.isSupported(rule) && success == BuildRuleSuccessType.BUILT_LOCALLY
&& !buildInfoRecorder.getBuildMetadataFor(BuildInfo.MetadataKey.INPUT_BASED_RULE_KEY)
.isPresent()) {
// Doing this here is probably not strictly necessary, however in the case of
// pipelined rules built locally we will never do an input-based cache check.
// That check would have written the key to metadata, and there are some asserts
// during cache upload that try to ensure they are present.
Optional<RuleKey> inputRuleKey = calculateInputBasedRuleKey(buildContext.getEventBus());
if (inputRuleKey.isPresent()) {
buildInfoRecorder.addBuildMetadata(BuildInfo.MetadataKey.INPUT_BASED_RULE_KEY,
inputRuleKey.get().toString());
}
}
// If this rule uses dep files and we built locally, make sure we store the new dep file
// list and re-calculate the dep file rule key.
if (useDependencyFileRuleKey() && success == BuildRuleSuccessType.BUILT_LOCALLY) {
// Query the rule for the actual inputs it used.
ImmutableList<SourcePath> inputs = ((SupportsDependencyFileRuleKey) rule)
.getInputsAfterBuildingLocally(buildContext.getBuildContext(),
executionContext.getCellPathResolver());
// Record the inputs into our metadata for next time.
// TODO(#9117006): We don't support a way to serlialize `SourcePath`s to the cache,
// so need to use DependencyFileEntry's instead and recover them on deserialization.
ImmutableList<String> inputStrings = inputs.stream()
.map(inputString -> DependencyFileEntry.fromSourcePath(inputString, pathResolver))
.map(MoreFunctions.toJsonFunction()).collect(MoreCollectors.toImmutableList());
buildInfoRecorder.addMetadata(BuildInfo.MetadataKey.DEP_FILE, inputStrings);
// Re-calculate and store the depfile rule key for next time.
Optional<RuleKeyAndInputs> depFileRuleKeyAndInputs = calculateDepFileRuleKey(
Optional.of(inputStrings), /* allowMissingInputs */ false);
if (depFileRuleKeyAndInputs.isPresent()) {
RuleKey depFileRuleKey = depFileRuleKeyAndInputs.get().getRuleKey();
buildInfoRecorder.addBuildMetadata(BuildInfo.MetadataKey.DEP_FILE_RULE_KEY,
depFileRuleKey.toString());
// Push an updated manifest to the cache.
if (useManifestCaching()) {
Optional<RuleKeyAndInputs> manifestKey = calculateManifestKey(
buildContext.getEventBus());
if (manifestKey.isPresent()) {
buildInfoRecorder.addBuildMetadata(BuildInfo.MetadataKey.MANIFEST_KEY,
manifestKey.get().getRuleKey().toString());
updateAndStoreManifest(depFileRuleKeyAndInputs.get().getRuleKey(),
depFileRuleKeyAndInputs.get().getInputs(), manifestKey.get(),
buildContext.getArtifactCache());
}
}
}
}
// If this rule was built locally, grab and record the output hashes in the build
// metadata so that cache hits avoid re-hashing file contents. Since we use output
// hashes for input-based rule keys and for detecting non-determinism, we would spend
// a lot of time re-hashing output paths -- potentially in serialized in a single step.
// So, do the hashing here to distribute the workload across several threads and cache
// the results.
//
// Also, since hashing outputs can potentially be expensive, we avoid doing this for
// rules that are marked as uncacheable. The rationale here is that they are likely not
// cached due to the sheer size which would be costly to hash or builtin non-determinism
// in the rule which somewhat defeats the purpose of logging the hash.
if (success == BuildRuleSuccessType.BUILT_LOCALLY
&& shouldUploadToCache(success, Preconditions.checkNotNull(outputSize.get()))) {
ImmutableSortedMap.Builder<String, String> outputHashes = ImmutableSortedMap.naturalOrder();
for (Path path : buildInfoRecorder.getOutputPaths()) {
outputHashes.put(path.toString(),
fileHashCache.get(rule.getProjectFilesystem().resolve(path)).toString());
}
buildInfoRecorder.addBuildMetadata(BuildInfo.MetadataKey.RECORDED_PATH_HASHES,
outputHashes.build());
}
// If this rule was fetched from cache, seed the file hash cache with the recorded
// output hashes from the build metadata. Since outputs which have been changed have
// already been invalidated above, this is purely a best-effort optimization -- if the
// the output hashes weren't recorded in the cache we do nothing.
if (success != BuildRuleSuccessType.BUILT_LOCALLY && success.outputsHaveChanged()) {
Optional<ImmutableMap<String, String>> hashes = onDiskBuildInfo
.getBuildMap(BuildInfo.MetadataKey.RECORDED_PATH_HASHES);
// We only seed after first verifying the recorded path hashes. This prevents the
// optimization, but is useful to keep in place for a while to verify this optimization
// is causing issues.
if (hashes.isPresent() && verifyRecordedPathHashes(rule.getBuildTarget(),
rule.getProjectFilesystem(), hashes.get())) {
// Seed the cache with the hashes.
for (Map.Entry<String, String> ent : hashes.get().entrySet()) {
Path path = rule.getProjectFilesystem().getPath(ent.getKey());
HashCode hashCode = HashCode.fromString(ent.getValue());
fileHashCache.set(rule.getProjectFilesystem().resolve(path), hashCode);
}
}
}
// Make sure the origin field is filled in.
BuildId buildId = buildContext.getBuildId();
if (success == BuildRuleSuccessType.BUILT_LOCALLY) {
buildInfoRecorder.addBuildMetadata(BuildInfo.MetadataKey.ORIGIN_BUILD_ID, buildId.toString());
} else if (success.outputsHaveChanged()) {
Preconditions.checkState(
buildInfoRecorder.getBuildMetadataFor(BuildInfo.MetadataKey.ORIGIN_BUILD_ID)
.isPresent(),
"Cache hits must populate the %s field (%s)", BuildInfo.MetadataKey.ORIGIN_BUILD_ID,
success);
}
// Make sure that all of the local files have the same values they would as if the
// rule had been built locally.
buildInfoRecorder.addBuildMetadata(BuildInfo.MetadataKey.TARGET, rule.getBuildTarget().toString());
buildInfoRecorder.addMetadata(BuildInfo.MetadataKey.RECORDED_PATHS,
buildInfoRecorder.getRecordedPaths().stream().map(Object::toString)
.collect(MoreCollectors.toImmutableList()));
if (success.shouldWriteRecordedMetadataToDiskAfterBuilding()) {
try {
boolean clearExistingMetadata = success.shouldClearAndOverwriteMetadataOnDisk();
buildInfoRecorder.writeMetadataToDisk(clearExistingMetadata);
} catch (IOException e) {
throw new IOException(String.format("Failed to write metadata to disk for %s.", rule), e);
}
}
// Give the rule a chance to populate its internal data structures now that all of
// the files should be in a valid state.
try {
if (rule instanceof InitializableFromDisk) {
doInitializeFromDisk((InitializableFromDisk<?>) rule);
}
} catch (IOException e) {
throw new IOException(
String.format("Error initializing %s from disk: %s.", rule, e.getMessage()), e);
}
}
return Futures.immediateFuture(input);
};
buildResult = Futures.transformAsync(buildResult, ruleAsyncFunction(buildContext.getEventBus(), callback),
serviceByAdjustingDefaultWeightsTo(CachingBuildEngine.RULE_KEY_COMPUTATION_RESOURCE_AMOUNTS));
buildResult = Futures.catchingAsync(buildResult, Throwable.class, thrown -> {
LOG.debug(thrown, "Building rule [%s] failed.", rule.getBuildTarget());
if (consoleLogBuildFailuresInline) {
buildContext.getEventBus().post(ConsoleEvent.severe(getErrorMessageIncludingBuildRule(thrown)));
}
thrown = maybeAttachBuildRuleNameToException(thrown);
recordFailureAndCleanUp(thrown);
return Futures.immediateFuture(BuildResult.failure(rule, thrown));
});
// Do things that need to happen after either success or failure, but don't block the dependents
// while doing so:
buildRuleBuilderDelegate
.addAsyncCallback(MoreFutures.addListenableCallback(buildResult, new FutureCallback<BuildResult>() {
private void uploadToCache(BuildRuleSuccessType success) {
// Collect up all the rule keys we have index the artifact in the cache with.
Set<RuleKey> ruleKeys = new HashSet<>();
// If the rule key has changed (and is not already in the cache), we need to push
// the artifact to cache using the new key.
ruleKeys.add(ruleKeyFactories.getDefaultRuleKeyFactory().build(rule));
// If the input-based rule key has changed, we need to push the artifact to cache
// using the new key.
if (SupportsInputBasedRuleKey.isSupported(rule)) {
Optional<RuleKey> calculatedRuleKey = calculateInputBasedRuleKey(
buildContext.getEventBus());
Optional<RuleKey> onDiskRuleKey = onDiskBuildInfo
.getRuleKey(BuildInfo.MetadataKey.INPUT_BASED_RULE_KEY);
Optional<RuleKey> metaDataRuleKey = buildInfoRecorder
.getBuildMetadataFor(BuildInfo.MetadataKey.INPUT_BASED_RULE_KEY)
.map(RuleKey::new);
Preconditions.checkState(calculatedRuleKey.equals(onDiskRuleKey),
"%s (%s): %s: invalid on-disk input-based rule key: %s != %s",
rule.getBuildTarget(), rule.getType(), success, calculatedRuleKey,
onDiskRuleKey);
Preconditions.checkState(calculatedRuleKey.equals(metaDataRuleKey),
"%s: %s: invalid meta-data input-based rule key: %s != %s",
rule.getBuildTarget(), success, calculatedRuleKey, metaDataRuleKey);
if (calculatedRuleKey.isPresent()) {
ruleKeys.add(calculatedRuleKey.get());
}
}
// If the manifest-based rule key has changed, we need to push the artifact to cache
// using the new key.
if (useManifestCaching()) {
Optional<RuleKey> onDiskRuleKey = onDiskBuildInfo
.getRuleKey(BuildInfo.MetadataKey.DEP_FILE_RULE_KEY);
Optional<RuleKey> metaDataRuleKey = buildInfoRecorder
.getBuildMetadataFor(BuildInfo.MetadataKey.DEP_FILE_RULE_KEY).map(RuleKey::new);
Preconditions.checkState(onDiskRuleKey.equals(metaDataRuleKey),
"%s: %s: inconsistent meta-data and on-disk dep-file rule key: %s != %s",
rule.getBuildTarget(), success, onDiskRuleKey, metaDataRuleKey);
if (onDiskRuleKey.isPresent()) {
ruleKeys.add(onDiskRuleKey.get());
}
}
// Do the actual upload.
try {
// Verify that the recorded path hashes are accurate.
Optional<String> recordedPathHashes = buildInfoRecorder
.getBuildMetadataFor(BuildInfo.MetadataKey.RECORDED_PATH_HASHES);
if (recordedPathHashes.isPresent() && !verifyRecordedPathHashes(rule.getBuildTarget(),
rule.getProjectFilesystem(), recordedPathHashes.get())) {
return;
}
// Push to cache.
buildInfoRecorder.performUploadToArtifactCache(ImmutableSet.copyOf(ruleKeys),
buildContext.getArtifactCache(), buildContext.getEventBus());
} catch (Throwable t) {
buildContext.getEventBus().post(
ThrowableConsoleEvent.create(t, "Error uploading to cache for %s.", rule));
}
}
private void handleResult(BuildResult input) {
Optional<Long> outputSize = Optional.empty();
Optional<HashCode> outputHash = Optional.empty();
Optional<BuildRuleSuccessType> successType = Optional.empty();
boolean shouldUploadToCache = false;
BuildRuleEvent.Resumed resumedEvent = BuildRuleEvent.resumed(rule, buildRuleDurationTracker,
ruleKeyFactories.getDefaultRuleKeyFactory());
LOG.verbose(resumedEvent.toString());
buildContext.getEventBus().post(resumedEvent);
if (input.getStatus() == BuildRuleStatus.SUCCESS) {
BuildRuleSuccessType success = Preconditions.checkNotNull(input.getSuccess());
successType = Optional.of(success);
// Try get the output size.
if (success == BuildRuleSuccessType.BUILT_LOCALLY
|| success.shouldUploadResultingArtifact()) {
try {
outputSize = Optional.of(buildInfoRecorder.getOutputSize());
} catch (IOException e) {
buildContext.getEventBus().post(ThrowableConsoleEvent.create(e,
"Error getting output size for %s.", rule));
}
}
// Compute it's output hash for logging/tracing purposes, as this artifact will
// be consumed by other builds.
if (outputSize.isPresent() && shouldHashOutputs(success, outputSize.get())) {
try {
outputHash = Optional.of(buildInfoRecorder.getOutputHash(fileHashCache));
} catch (IOException e) {
buildContext.getEventBus().post(ThrowableConsoleEvent.create(e,
"Error getting output hash for %s.", rule));
}
}
// Determine if this is rule is cacheable.
shouldUploadToCache = outputSize.isPresent()
&& shouldUploadToCache(success, outputSize.get());
// Upload it to the cache.
if (shouldUploadToCache) {
uploadToCache(success);
}
}
boolean failureOrBuiltLocally = input.getStatus() == BuildRuleStatus.FAIL
|| input.getSuccess() == BuildRuleSuccessType.BUILT_LOCALLY;
// Log the result to the event bus.
BuildRuleEvent.Finished finished = BuildRuleEvent.finished(resumedEvent, getBuildRuleKeys(),
input.getStatus(), input.getCacheResult(),
onDiskBuildInfo.getBuildValue(BuildInfo.MetadataKey.ORIGIN_BUILD_ID)
.map(BuildId::new),
successType, shouldUploadToCache, outputHash, outputSize,
getBuildRuleDiagnosticData(failureOrBuiltLocally));
LOG.verbose(finished.toString());
buildContext.getEventBus().post(finished);
}
@Override
public void onSuccess(BuildResult input) {
handleResult(input);
// Reset interrupted flag once failure has been recorded.
if (input.getFailure() instanceof InterruptedException) {
Threads.interruptCurrentThread();
}
}
@Override
public void onFailure(@Nonnull Throwable thrown) {
throw new AssertionError("Dead code");
}
}, serviceByAdjustingDefaultWeightsTo(CachingBuildEngine.RULE_KEY_COMPUTATION_RESOURCE_AMOUNTS)));
return buildResult;
}
From source file:org.onosproject.onosjar.ProjectJavadocDescription.java
@Override public <A extends Arg> BuildRule createBuildRule(TargetGraph targetGraph, BuildRuleParams params, BuildRuleResolver resolver, A args) throws NoSuchBuildTargetException { ImmutableSet.Builder<SourcePath> srcs = ImmutableSet.builder(); ImmutableSet.Builder<BuildRule> deps = ImmutableSet.builder(); ImmutableSortedMap.Builder<SourcePath, Path> docfiles = ImmutableSortedMap.naturalOrder(); for (BuildTarget dep : args.deps) { BuildRule rule = resolver.requireRule(dep.withFlavors(JavaLibrary.JAVADOC_JAR)); if (rule instanceof JavadocJar) { JavadocJar jarRule = (JavadocJar) rule; srcs.addAll(jarRule.getSources()); deps.addAll(jarRule.getDeps()); docfiles.putAll(jarRule.getDocFiles()); } else {// w w w . j a v a2 s . c om throw new RuntimeException("rule is not a javalib"); //FIXME } } BuildRuleParams newParams = params.copyWithDeps( Suppliers.ofInstance(FluentIterable.from(deps.build()).toSortedSet(Ordering.<BuildRule>natural())), Suppliers.ofInstance(ImmutableSortedSet.<BuildRule>of())); SourcePathResolver sourceResolver = new SourcePathResolver(resolver); ImmutableList.Builder<SourcePath> auxSources = ImmutableList.builder(); JavadocJar.JavadocArgs.Builder javadocArgs = JavadocJar.JavadocArgs.builder() .addArg("-windowtitle", args.projectTitle).addArg("-doctitle", args.projectTitle) .addArg("-link", "http://docs.oracle.com/javase/8/docs/api") .addArg("-tag", "onos.rsModel:a:\"onos model\""); //FIXME from buckconfig + rule if (args.groups.isPresent()) { for (Pair<String, ImmutableList<String>> pair : args.groups.get()) { javadocArgs.addArg("-group", pair.getFirst(), pair.getSecond()); } } if (args.excludePackages.isPresent() && !args.excludePackages.get().isEmpty()) { javadocArgs.addArg("-exclude", args.excludePackages.get()); } if (args.overview.isPresent()) { javadocArgs.addArg("-overview", sourceResolver.getAbsolutePath(args.overview.get()).toString()); } if (args.javadocFiles.isPresent()) { for (SourcePath path : args.javadocFiles.get()) { docfiles.put(path, JavadocJar.getDocfileWithPath(sourceResolver, path, args.javadocFilesRoot.orNull())); } } JavadocJar javadocJar = new JavadocJar(newParams, sourceResolver, ImmutableSortedSet.copyOf(srcs.build()), docfiles.build(), javadocArgs.build(), /* mavenCoords */ Optional.absent()); return javadocJar; }
From source file:com.facebook.buck.apple.ProjectGenerator.java
private ImmutableSortedMap<Path, SourcePath> convertMapKeysToPaths( ImmutableSortedMap<String, SourcePath> input) { ImmutableSortedMap.Builder<Path, SourcePath> output = ImmutableSortedMap.naturalOrder(); for (Map.Entry<String, SourcePath> entry : input.entrySet()) { output.put(Paths.get(entry.getKey()), entry.getValue()); }//w w w. j ava 2 s . c o m return output.build(); }
From source file:com.facebook.buck.apple.ProjectGenerator.java
private void createHeaderSymlinkTree(Function<SourcePath, Path> pathResolver, Map<Path, SourcePath> contents, Path headerSymlinkTreeRoot) throws IOException { LOG.verbose("Building header symlink tree at %s with contents %s", headerSymlinkTreeRoot, contents); ImmutableSortedMap.Builder<Path, Path> resolvedContentsBuilder = ImmutableSortedMap.naturalOrder(); for (Map.Entry<Path, SourcePath> entry : contents.entrySet()) { Path link = headerSymlinkTreeRoot.resolve(entry.getKey()); Path existing = projectFilesystem.resolve(pathResolver.apply(entry.getValue())); resolvedContentsBuilder.put(link, existing); }//from w ww .j av a 2 s. c om ImmutableSortedMap<Path, Path> resolvedContents = resolvedContentsBuilder.build(); Path headerMapLocation = getHeaderMapLocationFromSymlinkTreeRoot(headerSymlinkTreeRoot); Path hashCodeFilePath = headerSymlinkTreeRoot.resolve(".contents-hash"); Optional<String> currentHashCode = projectFilesystem.readFileIfItExists(hashCodeFilePath); String newHashCode = getHeaderSymlinkTreeHashCode(resolvedContents).toString(); if (Optional.of(newHashCode).equals(currentHashCode)) { LOG.debug("Symlink tree at %s is up to date, not regenerating (key %s).", headerSymlinkTreeRoot, newHashCode); } else { LOG.debug("Updating symlink tree at %s (old key %s, new key %s).", headerSymlinkTreeRoot, currentHashCode, newHashCode); projectFilesystem.deleteRecursivelyIfExists(headerSymlinkTreeRoot); projectFilesystem.mkdirs(headerSymlinkTreeRoot); for (Map.Entry<Path, Path> entry : resolvedContents.entrySet()) { Path link = entry.getKey(); Path existing = entry.getValue(); projectFilesystem.createParentDirs(link); projectFilesystem.createSymLink(link, existing, /* force */ false); } projectFilesystem.writeContentsToPath(newHashCode, hashCodeFilePath); HeaderMap.Builder headerMapBuilder = new HeaderMap.Builder(); for (Map.Entry<Path, SourcePath> entry : contents.entrySet()) { headerMapBuilder.add(entry.getKey().toString(), BuckConstant.BUCK_OUTPUT_PATH.relativize(headerSymlinkTreeRoot).resolve(entry.getKey())); } projectFilesystem.writeBytesToPath(headerMapBuilder.build().getBytes(), headerMapLocation); } headerSymlinkTrees.add(headerSymlinkTreeRoot); }
From source file:com.facebook.buck.apple.project_generator.ProjectGenerator.java
private void createHeaderSymlinkTree(Function<SourcePath, Path> pathResolver, Map<Path, SourcePath> contents, Path headerSymlinkTreeRoot, boolean shouldCreateHeadersSymlinks) throws IOException { LOG.verbose("Building header symlink tree at %s with contents %s", headerSymlinkTreeRoot, contents); ImmutableSortedMap.Builder<Path, Path> resolvedContentsBuilder = ImmutableSortedMap.naturalOrder(); for (Map.Entry<Path, SourcePath> entry : contents.entrySet()) { Path link = headerSymlinkTreeRoot.resolve(entry.getKey()); Path existing = projectFilesystem.resolve(pathResolver.apply(entry.getValue())); resolvedContentsBuilder.put(link, existing); }//from w w w. j a v a 2 s . c o m ImmutableSortedMap<Path, Path> resolvedContents = resolvedContentsBuilder.build(); Path headerMapLocation = getHeaderMapLocationFromSymlinkTreeRoot(headerSymlinkTreeRoot); Path hashCodeFilePath = headerSymlinkTreeRoot.resolve(".contents-hash"); Optional<String> currentHashCode = projectFilesystem.readFileIfItExists(hashCodeFilePath); String newHashCode = getHeaderSymlinkTreeHashCode(resolvedContents, shouldCreateHeadersSymlinks).toString(); if (Optional.of(newHashCode).equals(currentHashCode)) { LOG.debug("Symlink tree at %s is up to date, not regenerating (key %s).", headerSymlinkTreeRoot, newHashCode); } else { LOG.debug("Updating symlink tree at %s (old key %s, new key %s).", headerSymlinkTreeRoot, currentHashCode, newHashCode); projectFilesystem.deleteRecursivelyIfExists(headerSymlinkTreeRoot); projectFilesystem.mkdirs(headerSymlinkTreeRoot); if (shouldCreateHeadersSymlinks) { for (Map.Entry<Path, Path> entry : resolvedContents.entrySet()) { Path link = entry.getKey(); Path existing = entry.getValue(); projectFilesystem.createParentDirs(link); projectFilesystem.createSymLink(link, existing, /* force */ false); } } projectFilesystem.writeContentsToPath(newHashCode, hashCodeFilePath); HeaderMap.Builder headerMapBuilder = new HeaderMap.Builder(); for (Map.Entry<Path, SourcePath> entry : contents.entrySet()) { if (shouldCreateHeadersSymlinks) { headerMapBuilder.add(entry.getKey().toString(), Paths.get("../../").resolve(projectCell.getRoot().getFileName()) .resolve(headerSymlinkTreeRoot).resolve(entry.getKey())); } else { headerMapBuilder.add(entry.getKey().toString(), projectFilesystem.resolve(pathResolver.apply(entry.getValue()))); } } projectFilesystem.writeBytesToPath(headerMapBuilder.build().getBytes(), headerMapLocation); } headerSymlinkTrees.add(headerSymlinkTreeRoot); }
From source file:com.facebook.buck.features.apple.project.ProjectGenerator.java
private ImmutableMap<Path, Path> getSwiftPublicHeaderMapEntriesForTarget( TargetNode<? extends CxxLibraryDescription.CommonArg> node) { boolean hasSwiftVersionArg = getSwiftVersionForTargetNode(node).isPresent(); if (!hasSwiftVersionArg) { return ImmutableMap.of(); }//from w ww. jav a2 s . c o m Optional<TargetNode<AppleNativeTargetDescriptionArg>> maybeAppleNode = TargetNodes.castArg(node, AppleNativeTargetDescriptionArg.class); if (!maybeAppleNode.isPresent()) { return ImmutableMap.of(); } TargetNode<? extends AppleNativeTargetDescriptionArg> appleNode = maybeAppleNode.get(); if (!projGenerationStateCache.targetContainsSwiftSourceCode(appleNode)) { return ImmutableMap.of(); } BuildTarget buildTarget = appleNode.getBuildTarget(); Path headerPrefix = AppleDescriptions.getHeaderPathPrefix(appleNode.getConstructorArg(), buildTarget); Path relativePath = headerPrefix.resolve(getSwiftObjCGeneratedHeaderName(appleNode)); ImmutableSortedMap.Builder<Path, Path> builder = ImmutableSortedMap.naturalOrder(); builder.put(relativePath, getSwiftObjCGeneratedHeaderPath(appleNode, projectFilesystem).toAbsolutePath()); return builder.build(); }
From source file:com.facebook.buck.features.apple.project.ProjectGenerator.java
private Optional<ImmutableSortedMap<String, ImmutableMap<String, String>>> getXcodeBuildConfigurationsForTargetNode( TargetNode<?> targetNode) { Optional<ImmutableSortedMap<String, ImmutableMap<String, String>>> configs = Optional.empty(); Optional<TargetNode<AppleNativeTargetDescriptionArg>> appleTargetNode = TargetNodes.castArg(targetNode, AppleNativeTargetDescriptionArg.class); Optional<TargetNode<HalideLibraryDescriptionArg>> halideTargetNode = TargetNodes.castArg(targetNode, HalideLibraryDescriptionArg.class); if (appleTargetNode.isPresent()) { configs = Optional.of(appleTargetNode.get().getConstructorArg().getConfigs()); } else if (halideTargetNode.isPresent()) { configs = Optional.of(halideTargetNode.get().getConstructorArg().getConfigs()); }/* w w w . ja v a 2s . c o m*/ HashMap<String, HashMap<String, String>> combinedConfig = new HashMap<String, HashMap<String, String>>(); combinedConfig.put(CxxPlatformXcodeConfigGenerator.DEBUG_BUILD_CONFIGURATION_NAME, new HashMap<String, String>(getDefaultDebugBuildConfiguration())); combinedConfig.put(CxxPlatformXcodeConfigGenerator.PROFILE_BUILD_CONFIGURATION_NAME, new HashMap<String, String>()); combinedConfig.put(CxxPlatformXcodeConfigGenerator.RELEASE_BUILD_CONFIGURATION_NAME, new HashMap<String, String>()); if (configs.isPresent() && !configs.get().isEmpty() && !(targetNode.getDescription() instanceof CxxLibraryDescription)) { for (Map.Entry<String, ImmutableMap<String, String>> targetLevelConfig : configs.get().entrySet()) { HashMap<String, String> pendingConfig = new HashMap<String, String>(targetLevelConfig.getValue()); String configTarget = targetLevelConfig.getKey(); if (combinedConfig.containsKey(configTarget)) { combinedConfig.get(configTarget).putAll(pendingConfig); } else { combinedConfig.put(configTarget, pendingConfig); } } } ImmutableSortedMap.Builder<String, ImmutableMap<String, String>> configBuilder = ImmutableSortedMap .naturalOrder(); for (Map.Entry<String, HashMap<String, String>> config : combinedConfig.entrySet()) { configBuilder.put(config.getKey(), ImmutableMap.copyOf(config.getValue())); } return Optional.of(configBuilder.build()); }