/* * This file is part of Bisq. * * Bisq is free software: you can redistribute it and/or modify it * under the terms of the GNU Affero General Public License as published by * the Free Software Foundation, either version 3 of the License, or (at * your option) any later version. * * Bisq is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public * License for more details. * * You should have received a copy of the GNU Affero General Public License * along with Bisq. If not, see <http://www.gnu.org/licenses/>. */ package bisq.network.p2p.storage; import bisq.network.p2p.NodeAddress; import bisq.network.p2p.network.CloseConnectionReason; import bisq.network.p2p.network.Connection; import bisq.network.p2p.network.ConnectionListener; import bisq.network.p2p.network.MessageListener; import bisq.network.p2p.network.NetworkNode; import bisq.network.p2p.peers.BroadcastHandler; import bisq.network.p2p.peers.Broadcaster; import bisq.network.p2p.peers.getdata.messages.GetDataRequest; import bisq.network.p2p.peers.getdata.messages.GetDataResponse; import bisq.network.p2p.peers.getdata.messages.GetUpdatedDataRequest; import bisq.network.p2p.peers.getdata.messages.PreliminaryGetDataRequest; import bisq.network.p2p.storage.messages.AddDataMessage; import bisq.network.p2p.storage.messages.AddOncePayload; import bisq.network.p2p.storage.messages.AddPersistableNetworkPayloadMessage; import bisq.network.p2p.storage.messages.BroadcastMessage; import bisq.network.p2p.storage.messages.RefreshOfferMessage; import bisq.network.p2p.storage.messages.RemoveDataMessage; import bisq.network.p2p.storage.messages.RemoveMailboxDataMessage; import bisq.network.p2p.storage.payload.CapabilityRequiringPayload; import bisq.network.p2p.storage.payload.DateTolerantPayload; import bisq.network.p2p.storage.payload.MailboxStoragePayload; import bisq.network.p2p.storage.payload.PersistableNetworkPayload; import bisq.network.p2p.storage.payload.ProcessOncePersistableNetworkPayload; import bisq.network.p2p.storage.payload.ProtectedMailboxStorageEntry; import bisq.network.p2p.storage.payload.ProtectedStorageEntry; import bisq.network.p2p.storage.payload.ProtectedStoragePayload; import bisq.network.p2p.storage.payload.RequiresOwnerIsOnlinePayload; import bisq.network.p2p.storage.persistence.AppendOnlyDataStoreListener; import bisq.network.p2p.storage.persistence.AppendOnlyDataStoreService; import bisq.network.p2p.storage.persistence.ProtectedDataStoreService; import bisq.network.p2p.storage.persistence.ResourceDataStoreService; import bisq.network.p2p.storage.persistence.SequenceNumberMap; import bisq.common.Timer; import bisq.common.UserThread; import bisq.common.app.Capabilities; import bisq.common.crypto.CryptoException; import bisq.common.crypto.Hash; import bisq.common.crypto.Sig; import bisq.common.proto.network.NetworkEnvelope; import bisq.common.proto.network.NetworkPayload; import bisq.common.proto.persistable.PersistablePayload; import bisq.common.proto.persistable.PersistedDataHost; import bisq.common.storage.Storage; import bisq.common.util.Hex; import bisq.common.util.Tuple2; import bisq.common.util.Utilities; import com.google.protobuf.ByteString; import com.google.inject.name.Named; import javax.inject.Inject; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Maps; import java.security.KeyPair; import java.security.PublicKey; import java.time.Clock; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArraySet; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import java.util.stream.Collectors; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.ToString; import lombok.extern.slf4j.Slf4j; import javax.annotation.Nullable; @Slf4j public class P2PDataStorage implements MessageListener, ConnectionListener, PersistedDataHost { /** * How many days to keep an entry before it is purged. */ @VisibleForTesting public static final int PURGE_AGE_DAYS = 10; @VisibleForTesting public static final int CHECK_TTL_INTERVAL_SEC = 60; private boolean initialRequestApplied = false; private final Broadcaster broadcaster; private final AppendOnlyDataStoreService appendOnlyDataStoreService; private final ProtectedDataStoreService protectedDataStoreService; private final ResourceDataStoreService resourceDataStoreService; @Getter private final Map<ByteArray, ProtectedStorageEntry> map = new ConcurrentHashMap<>(); private final Set<ByteArray> removedAddOncePayloads = new HashSet<>(); private final Set<HashMapChangedListener> hashMapChangedListeners = new CopyOnWriteArraySet<>(); private Timer removeExpiredEntriesTimer; private final Storage<SequenceNumberMap> sequenceNumberMapStorage; @VisibleForTesting final SequenceNumberMap sequenceNumberMap = new SequenceNumberMap(); private final Set<AppendOnlyDataStoreListener> appendOnlyDataStoreListeners = new CopyOnWriteArraySet<>(); private final Clock clock; /// The maximum number of items that must exist in the SequenceNumberMap before it is scheduled for a purge /// which removes entries after PURGE_AGE_DAYS. private final int maxSequenceNumberMapSizeBeforePurge; /////////////////////////////////////////////////////////////////////////////////////////// // Constructor /////////////////////////////////////////////////////////////////////////////////////////// @Inject public P2PDataStorage(NetworkNode networkNode, Broadcaster broadcaster, AppendOnlyDataStoreService appendOnlyDataStoreService, ProtectedDataStoreService protectedDataStoreService, ResourceDataStoreService resourceDataStoreService, Storage<SequenceNumberMap> sequenceNumberMapStorage, Clock clock, @Named("MAX_SEQUENCE_NUMBER_MAP_SIZE_BEFORE_PURGE") int maxSequenceNumberBeforePurge) { this.broadcaster = broadcaster; this.appendOnlyDataStoreService = appendOnlyDataStoreService; this.protectedDataStoreService = protectedDataStoreService; this.resourceDataStoreService = resourceDataStoreService; this.clock = clock; this.maxSequenceNumberMapSizeBeforePurge = maxSequenceNumberBeforePurge; networkNode.addMessageListener(this); networkNode.addConnectionListener(this); this.sequenceNumberMapStorage = sequenceNumberMapStorage; sequenceNumberMapStorage.setNumMaxBackupFiles(5); } @Override public void readPersisted() { SequenceNumberMap persistedSequenceNumberMap = sequenceNumberMapStorage.initAndGetPersisted(sequenceNumberMap, 300); if (persistedSequenceNumberMap != null) sequenceNumberMap.setMap(getPurgedSequenceNumberMap(persistedSequenceNumberMap.getMap())); } // This method is called at startup in a non-user thread. // We should not have any threading issues here as the p2p network is just initializing public synchronized void readFromResources(String postFix) { appendOnlyDataStoreService.readFromResources(postFix); protectedDataStoreService.readFromResources(postFix); resourceDataStoreService.readFromResources(postFix); map.putAll(protectedDataStoreService.getMap()); } /////////////////////////////////////////////////////////////////////////////////////////// // RequestData API /////////////////////////////////////////////////////////////////////////////////////////// /** * Returns a PreliminaryGetDataRequest that can be sent to a peer node to request missing Payload data. */ public PreliminaryGetDataRequest buildPreliminaryGetDataRequest(int nonce) { return new PreliminaryGetDataRequest(nonce, this.getKnownPayloadHashes()); } /** * Returns a GetUpdatedDataRequest that can be sent to a peer node to request missing Payload data. */ public GetUpdatedDataRequest buildGetUpdatedDataRequest(NodeAddress senderNodeAddress, int nonce) { return new GetUpdatedDataRequest(senderNodeAddress, nonce, this.getKnownPayloadHashes()); } /** * Returns the set of known payload hashes. This is used in the GetData path to request missing data from peer nodes */ private Set<byte[]> getKnownPayloadHashes() { // We collect the keys of the PersistableNetworkPayload items so we exclude them in our request. // PersistedStoragePayload items don't get removed, so we don't have an issue with the case that // an object gets removed in between PreliminaryGetDataRequest and the GetUpdatedDataRequest and we would // miss that event if we do not load the full set or use some delta handling. Set<byte[]> excludedKeys = this.appendOnlyDataStoreService.getMap().keySet().stream() .map(e -> e.bytes) .collect(Collectors.toSet()); Set<byte[]> excludedKeysFromPersistedEntryMap = this.map.keySet() .stream() .map(e -> e.bytes) .collect(Collectors.toSet()); excludedKeys.addAll(excludedKeysFromPersistedEntryMap); return excludedKeys; } /** * Generic function that can be used to filter a Map<ByteArray, ProtectedStorageEntry || PersistableNetworkPayload> * by a given set of keys and peer capabilities. */ static private <T extends NetworkPayload> Set<T> filterKnownHashes( Map<ByteArray, T> toFilter, Function<T, ? extends NetworkPayload> objToPayload, Set<ByteArray> knownHashes, Capabilities peerCapabilities, int maxEntries, AtomicBoolean outTruncated) { AtomicInteger limit = new AtomicInteger(maxEntries); Set<T> filteredResults = toFilter.entrySet().stream() .filter(e -> !knownHashes.contains(e.getKey())) .filter(e -> limit.decrementAndGet() >= 0) .map(Map.Entry::getValue) .filter(networkPayload -> shouldTransmitPayloadToPeer(peerCapabilities, objToPayload.apply(networkPayload))) .collect(Collectors.toSet()); if (limit.get() < 0) outTruncated.set(true); return filteredResults; } /** * Returns a GetDataResponse object that contains the Payloads known locally, but not remotely. */ public GetDataResponse buildGetDataResponse( GetDataRequest getDataRequest, int maxEntriesPerType, AtomicBoolean outPersistableNetworkPayloadOutputTruncated, AtomicBoolean outProtectedStorageEntryOutputTruncated, Capabilities peerCapabilities) { Set<P2PDataStorage.ByteArray> excludedKeysAsByteArray = P2PDataStorage.ByteArray.convertBytesSetToByteArraySet(getDataRequest.getExcludedKeys()); Set<PersistableNetworkPayload> filteredPersistableNetworkPayloads = filterKnownHashes( this.appendOnlyDataStoreService.getMap(), Function.identity(), excludedKeysAsByteArray, peerCapabilities, maxEntriesPerType, outPersistableNetworkPayloadOutputTruncated); Set<ProtectedStorageEntry> filteredProtectedStorageEntries = filterKnownHashes( this.map, ProtectedStorageEntry::getProtectedStoragePayload, excludedKeysAsByteArray, peerCapabilities, maxEntriesPerType, outProtectedStorageEntryOutputTruncated); return new GetDataResponse( filteredProtectedStorageEntries, filteredPersistableNetworkPayloads, getDataRequest.getNonce(), getDataRequest instanceof GetUpdatedDataRequest); } /** * Returns true if a Payload should be transmit to a peer given the peer's supported capabilities. */ private static boolean shouldTransmitPayloadToPeer(Capabilities peerCapabilities, NetworkPayload payload) { // Sanity check to ensure this isn't used outside P2PDataStorage if (!(payload instanceof ProtectedStoragePayload || payload instanceof PersistableNetworkPayload)) return false; // If the payload doesn't have a required capability, we should transmit it if (!(payload instanceof CapabilityRequiringPayload)) return true; // Otherwise, only transmit the Payload if the peer supports all capabilities required by the payload boolean shouldTransmit = peerCapabilities.containsAll(((CapabilityRequiringPayload) payload).getRequiredCapabilities()); if (!shouldTransmit) { log.debug("We do not send the message to the peer because they do not support the required capability for that message type.\n" + "storagePayload is: " + Utilities.toTruncatedString(payload)); } return shouldTransmit; } /** * Processes a GetDataResponse message and updates internal state. Does not broadcast updates to the P2P network * or domain listeners. */ public void processGetDataResponse(GetDataResponse getDataResponse, NodeAddress sender) { final Set<ProtectedStorageEntry> dataSet = getDataResponse.getDataSet(); Set<PersistableNetworkPayload> persistableNetworkPayloadSet = getDataResponse.getPersistableNetworkPayloadSet(); long ts2 = System.currentTimeMillis(); dataSet.forEach(e -> { // We don't broadcast here (last param) as we are only connected to the seed node and would be pointless addProtectedStorageEntry(e, sender, null, false); }); log.info("Processing {} protectedStorageEntries took {} ms.", dataSet.size(), this.clock.millis() - ts2); ts2 = this.clock.millis(); persistableNetworkPayloadSet.forEach(e -> { if (e instanceof ProcessOncePersistableNetworkPayload) { // We use an optimized method as many checks are not required in that case to avoid // performance issues. // Processing 82645 items took now 61 ms compared to earlier version where it took ages (> 2min). // Usually we only get about a few hundred or max. a few 1000 items. 82645 is all // trade stats stats and all account age witness data. // We only apply it once from first response if (!initialRequestApplied) { addPersistableNetworkPayloadFromInitialRequest(e); } } else { // We don't broadcast here as we are only connected to the seed node and would be pointless addPersistableNetworkPayload(e, sender, false, false, false); } }); log.info("Processing {} persistableNetworkPayloads took {} ms.", persistableNetworkPayloadSet.size(), this.clock.millis() - ts2); // We only process PersistableNetworkPayloads implementing ProcessOncePersistableNetworkPayload once. It can cause performance // issues and since the data is rarely out of sync it is not worth it to apply them from multiple peers during // startup. initialRequestApplied = true; } /////////////////////////////////////////////////////////////////////////////////////////// // API /////////////////////////////////////////////////////////////////////////////////////////// public void shutDown() { if (removeExpiredEntriesTimer != null) removeExpiredEntriesTimer.stop(); } @VisibleForTesting void removeExpiredEntries() { // The moment when an object becomes expired will not be synchronous in the network and we could // get add network_messages after the object has expired. To avoid repeated additions of already expired // object when we get it sent from new peers, we don’t remove the sequence number from the map. // That way an ADD message for an already expired data will fail because the sequence number // is equal and not larger as expected. ArrayList<Map.Entry<ByteArray, ProtectedStorageEntry>> toRemoveList = map.entrySet().stream() .filter(entry -> entry.getValue().isExpired(this.clock)) .collect(Collectors.toCollection(ArrayList::new)); // Batch processing can cause performance issues, so do all of the removes first, then update the listeners // to let them know about the removes. toRemoveList.forEach(toRemoveItem -> { log.debug("We found an expired data entry. We remove the protectedData:\n\t" + Utilities.toTruncatedString(toRemoveItem.getValue())); }); removeFromMapAndDataStore(toRemoveList); if (sequenceNumberMap.size() > this.maxSequenceNumberMapSizeBeforePurge) sequenceNumberMap.setMap(getPurgedSequenceNumberMap(sequenceNumberMap.getMap())); } public void onBootstrapComplete() { removeExpiredEntriesTimer = UserThread.runPeriodically(this::removeExpiredEntries, CHECK_TTL_INTERVAL_SEC); } public Map<ByteArray, PersistableNetworkPayload> getAppendOnlyDataStoreMap() { return appendOnlyDataStoreService.getMap(); } /////////////////////////////////////////////////////////////////////////////////////////// // MessageListener implementation /////////////////////////////////////////////////////////////////////////////////////////// @Override public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { if (networkEnvelope instanceof BroadcastMessage) { connection.getPeersNodeAddressOptional().ifPresent(peersNodeAddress -> { if (networkEnvelope instanceof AddDataMessage) { addProtectedStorageEntry(((AddDataMessage) networkEnvelope).getProtectedStorageEntry(), peersNodeAddress, null, true); } else if (networkEnvelope instanceof RemoveDataMessage) { remove(((RemoveDataMessage) networkEnvelope).getProtectedStorageEntry(), peersNodeAddress); } else if (networkEnvelope instanceof RemoveMailboxDataMessage) { remove(((RemoveMailboxDataMessage) networkEnvelope).getProtectedMailboxStorageEntry(), peersNodeAddress); } else if (networkEnvelope instanceof RefreshOfferMessage) { refreshTTL((RefreshOfferMessage) networkEnvelope, peersNodeAddress); } else if (networkEnvelope instanceof AddPersistableNetworkPayloadMessage) { addPersistableNetworkPayload(((AddPersistableNetworkPayloadMessage) networkEnvelope).getPersistableNetworkPayload(), peersNodeAddress, true, false, true); } }); } } /////////////////////////////////////////////////////////////////////////////////////////// // ConnectionListener implementation /////////////////////////////////////////////////////////////////////////////////////////// @Override public void onConnection(Connection connection) { } @Override public void onDisconnect(CloseConnectionReason closeConnectionReason, Connection connection) { if (closeConnectionReason.isIntended) return; if (!connection.getPeersNodeAddressOptional().isPresent()) return; NodeAddress peersNodeAddress = connection.getPeersNodeAddressOptional().get(); // Backdate all the eligible payloads based on the node that disconnected map.values().stream() .filter(protectedStorageEntry -> protectedStorageEntry.getProtectedStoragePayload() instanceof RequiresOwnerIsOnlinePayload) .filter(protectedStorageEntry -> ((RequiresOwnerIsOnlinePayload) protectedStorageEntry.getProtectedStoragePayload()).getOwnerNodeAddress().equals(peersNodeAddress)) .forEach(protectedStorageEntry -> { // We only set the data back by half of the TTL and remove the data only if is has // expired after that back dating. // We might get connection drops which are not caused by the node going offline, so // we give more tolerance with that approach, giving the node the chance to // refresh the TTL with a refresh message. // We observed those issues during stress tests, but it might have been caused by the // test set up (many nodes/connections over 1 router) // TODO investigate what causes the disconnections. // Usually the are: SOCKET_TIMEOUT ,TERMINATED (EOFException) log.debug("Backdating {} due to closeConnectionReason={}", protectedStorageEntry, closeConnectionReason); protectedStorageEntry.backDate(); }); } @Override public void onError(Throwable throwable) { } /////////////////////////////////////////////////////////////////////////////////////////// // Client API /////////////////////////////////////////////////////////////////////////////////////////// /** * Adds a PersistableNetworkPayload to the local P2P data storage. If it does not already exist locally, it will * be broadcast to the P2P network. * @param payload PersistableNetworkPayload to add to the network * @param sender local NodeAddress, if available * @param allowReBroadcast <code>true</code> if the PersistableNetworkPayload should be rebroadcast even if it * already exists locally * @return <code>true</code> if the PersistableNetworkPayload passes all validation and exists in the P2PDataStore * on completion */ public boolean addPersistableNetworkPayload(PersistableNetworkPayload payload, @Nullable NodeAddress sender, boolean allowReBroadcast) { return addPersistableNetworkPayload( payload, sender, true, allowReBroadcast, false); } private boolean addPersistableNetworkPayload(PersistableNetworkPayload payload, @Nullable NodeAddress sender, boolean allowBroadcast, boolean reBroadcast, boolean checkDate) { log.trace("addPersistableNetworkPayload payload={}", payload); // Payload hash size does not match expectation for that type of message. if (!payload.verifyHashSize()) { log.warn("addPersistableNetworkPayload failed due to unexpected hash size"); return false; } ByteArray hashAsByteArray = new ByteArray(payload.getHash()); boolean payloadHashAlreadyInStore = getAppendOnlyDataStoreMap().containsKey(hashAsByteArray); // Store already knows about this payload. Ignore it unless the caller specifically requests a republish. if (payloadHashAlreadyInStore && !reBroadcast) { log.trace("addPersistableNetworkPayload failed due to duplicate payload"); return false; } // DateTolerantPayloads are only checked for tolerance from the onMessage handler (checkDate == true). If not in // tolerance, ignore it. if (checkDate && payload instanceof DateTolerantPayload && !((DateTolerantPayload) payload).isDateInTolerance((clock))) { log.warn("addPersistableNetworkPayload failed due to payload time outside tolerance.\n" + "Payload={}; now={}", payload.toString(), new Date()); return false; } // Add the payload and publish the state update to the appendOnlyDataStoreListeners if (!payloadHashAlreadyInStore) { appendOnlyDataStoreService.put(hashAsByteArray, payload); appendOnlyDataStoreListeners.forEach(e -> e.onAdded(payload)); } // Broadcast the payload if requested by caller if (allowBroadcast) broadcaster.broadcast(new AddPersistableNetworkPayloadMessage(payload), sender, null); return true; } // When we receive initial data we skip several checks to improve performance. We requested only missing entries so we // do not need to check again if the item is contained in the map, which is a bit slow as the map can be very large. // Overwriting an entry would be also no issue. We also skip notifying listeners as we get called before the domain // is ready so no listeners are set anyway. We might get called twice from a redundant call later, so listeners // might be added then but as we have the data already added calling them would be irrelevant as well. private void addPersistableNetworkPayloadFromInitialRequest(PersistableNetworkPayload payload) { byte[] hash = payload.getHash(); if (payload.verifyHashSize()) { ByteArray hashAsByteArray = new ByteArray(hash); appendOnlyDataStoreService.put(hashAsByteArray, payload); } else { log.warn("We got a hash exceeding our permitted size"); } } /** * Adds a ProtectedStorageEntry to the local P2P data storage. If it does not already exist locally, it will be * broadcast to the P2P network. * * @param protectedStorageEntry ProtectedStorageEntry to add to the network * @param sender local NodeAddress, if available * @param listener optional listener that can be used to receive events on broadcast * @return <code>true</code> if the ProtectedStorageEntry was added to the local P2P data storage and broadcast */ public boolean addProtectedStorageEntry(ProtectedStorageEntry protectedStorageEntry, @Nullable NodeAddress sender, @Nullable BroadcastHandler.Listener listener) { return addProtectedStorageEntry(protectedStorageEntry, sender, listener, true); } private boolean addProtectedStorageEntry(ProtectedStorageEntry protectedStorageEntry, @Nullable NodeAddress sender, @Nullable BroadcastHandler.Listener listener, boolean allowBroadcast) { ProtectedStoragePayload protectedStoragePayload = protectedStorageEntry.getProtectedStoragePayload(); ByteArray hashOfPayload = get32ByteHashAsByteArray(protectedStoragePayload); if (protectedStoragePayload instanceof AddOncePayload && removedAddOncePayloads.contains(hashOfPayload)) { log.warn("We have already removed that AddOncePayload by a previous removeDataMessage. " + "We ignore that message. ProtectedStoragePayload: {}", protectedStoragePayload.toString()); return false; } // To avoid that expired data get stored and broadcast we check early for expire date. if (protectedStorageEntry.isExpired(clock)) { log.warn("We received an expired protectedStorageEntry from peer {}", sender != null ? sender.getFullAddress() : "sender is null"); log.debug("Expired protectedStorageEntry from peer {}. getCreationTimeStamp={}, protectedStorageEntry={}", sender != null ? sender.getFullAddress() : "sender is null", new Date(protectedStorageEntry.getCreationTimeStamp()), protectedStorageEntry); return false; } ProtectedStorageEntry storedEntry = map.get(hashOfPayload); // If we have seen a more recent operation for this payload and we have a payload locally, ignore it if (storedEntry != null && !hasSequenceNrIncreased(protectedStorageEntry.getSequenceNumber(), hashOfPayload)) { return false; } // We want to allow add operations for equal sequence numbers if we don't have the payload locally. This is // the case for non-persistent Payloads that need to be reconstructed from peer and seed nodes each startup. MapValue sequenceNumberMapValue = sequenceNumberMap.get(hashOfPayload); if (sequenceNumberMapValue != null && protectedStorageEntry.getSequenceNumber() < sequenceNumberMapValue.sequenceNr) { return false; } // Verify the ProtectedStorageEntry is well formed and valid for the add operation if (!protectedStorageEntry.isValidForAddOperation()) return false; // If we have already seen an Entry with the same hash, verify the metadata is equal if (storedEntry != null && !protectedStorageEntry.matchesRelevantPubKey(storedEntry)) return false; // This is an updated entry. Record it and signal listeners. map.put(hashOfPayload, protectedStorageEntry); hashMapChangedListeners.forEach(e -> e.onAdded(Collections.singletonList(protectedStorageEntry))); // Record the updated sequence number and persist it. Higher delay so we can batch more items. sequenceNumberMap.put(hashOfPayload, new MapValue(protectedStorageEntry.getSequenceNumber(), this.clock.millis())); sequenceNumberMapStorage.queueUpForSave(SequenceNumberMap.clone(sequenceNumberMap), 2000); // Optionally, broadcast the add/update depending on the calling environment if (allowBroadcast) broadcaster.broadcast(new AddDataMessage(protectedStorageEntry), sender, listener); // Persist ProtectedStorageEntrys carrying PersistablePayload payloads if (protectedStoragePayload instanceof PersistablePayload) protectedDataStoreService.put(hashOfPayload, protectedStorageEntry); return true; } /** * Updates a local RefreshOffer with TTL changes and broadcasts those changes to the network * * @param refreshTTLMessage refreshTTLMessage containing the update * @param sender local NodeAddress, if available * @return <code>true</code> if the RefreshOffer was successfully updated and changes broadcast */ public boolean refreshTTL(RefreshOfferMessage refreshTTLMessage, @Nullable NodeAddress sender) { ByteArray hashOfPayload = new ByteArray(refreshTTLMessage.getHashOfPayload()); ProtectedStorageEntry storedData = map.get(hashOfPayload); if (storedData == null) { log.debug("We don't have data for that refresh message in our map. That is expected if we missed the data publishing."); return false; } ProtectedStorageEntry storedEntry = map.get(hashOfPayload); ProtectedStorageEntry updatedEntry = new ProtectedStorageEntry( storedEntry.getProtectedStoragePayload(), storedEntry.getOwnerPubKey(), refreshTTLMessage.getSequenceNumber(), refreshTTLMessage.getSignature(), this.clock); // If we have seen a more recent operation for this payload, we ignore the current one if (!hasSequenceNrIncreased(updatedEntry.getSequenceNumber(), hashOfPayload)) return false; // Verify the updated ProtectedStorageEntry is well formed and valid for update if (!updatedEntry.isValidForAddOperation()) return false; // Update the hash map with the updated entry map.put(hashOfPayload, updatedEntry); // Record the latest sequence number and persist it sequenceNumberMap.put(hashOfPayload, new MapValue(updatedEntry.getSequenceNumber(), this.clock.millis())); sequenceNumberMapStorage.queueUpForSave(SequenceNumberMap.clone(sequenceNumberMap), 1000); // Always broadcast refreshes broadcaster.broadcast(refreshTTLMessage, sender, null); return true; } /** * Removes a ProtectedStorageEntry from the local P2P data storage. If it is successful, it will broadcast that * change to the P2P network. * * @param protectedStorageEntry ProtectedStorageEntry to add to the network * @param sender local NodeAddress, if available * @return <code>true</code> if the ProtectedStorageEntry was removed from the local P2P data storage and broadcast */ public boolean remove(ProtectedStorageEntry protectedStorageEntry, @Nullable NodeAddress sender) { ProtectedStoragePayload protectedStoragePayload = protectedStorageEntry.getProtectedStoragePayload(); ByteArray hashOfPayload = get32ByteHashAsByteArray(protectedStoragePayload); // If we have seen a more recent operation for this payload, ignore this one if (!hasSequenceNrIncreased(protectedStorageEntry.getSequenceNumber(), hashOfPayload)) return false; // Verify the ProtectedStorageEntry is well formed and valid for the remove operation if (!protectedStorageEntry.isValidForRemoveOperation()) return false; // If we have already seen an Entry with the same hash, verify the metadata is the same ProtectedStorageEntry storedEntry = map.get(hashOfPayload); if (storedEntry != null && !protectedStorageEntry.matchesRelevantPubKey(storedEntry)) return false; // Record the latest sequence number and persist it sequenceNumberMap.put(hashOfPayload, new MapValue(protectedStorageEntry.getSequenceNumber(), this.clock.millis())); sequenceNumberMapStorage.queueUpForSave(SequenceNumberMap.clone(sequenceNumberMap), 300); // Update that we have seen this AddOncePayload so the next time it is seen it fails verification if (protectedStoragePayload instanceof AddOncePayload) removedAddOncePayloads.add(hashOfPayload); if (storedEntry != null) { // Valid remove entry, do the remove and signal listeners removeFromMapAndDataStore(protectedStorageEntry, hashOfPayload); } /* else { // This means the RemoveData or RemoveMailboxData was seen prior to the AddData. We have already updated // the SequenceNumberMap appropriately so the stale Add will not pass validation, but we still want to // broadcast the remove to peers so they can update their state appropriately } */ printData("after remove"); if (protectedStorageEntry instanceof ProtectedMailboxStorageEntry) { broadcaster.broadcast(new RemoveMailboxDataMessage((ProtectedMailboxStorageEntry) protectedStorageEntry), sender, null); } else { broadcaster.broadcast(new RemoveDataMessage(protectedStorageEntry), sender, null); } return true; } /** * This method must be called only from client code not from network messages! We omit the ownership checks * so we must apply it only if it comes from our trusted application code. It is used from client code which detects * that the domain object violates specific domain rules. * We could make it more generic by adding an Interface with a generic validation method. * * @param protectedStorageEntry The entry to be removed */ public void removeInvalidProtectedStorageEntry(ProtectedStorageEntry protectedStorageEntry) { log.warn("We remove an invalid protectedStorageEntry: {}", protectedStorageEntry); ProtectedStoragePayload protectedStoragePayload = protectedStorageEntry.getProtectedStoragePayload(); ByteArray hashOfPayload = get32ByteHashAsByteArray(protectedStoragePayload); if (!map.containsKey(hashOfPayload)) { return; } removeFromMapAndDataStore(protectedStorageEntry, hashOfPayload); // We do not update the sequence number as that method is only called if we have received an invalid // protectedStorageEntry from a previous add operation. // We do not call maybeAddToRemoveAddOncePayloads to avoid that an invalid object might block a valid object // which we might receive in future (could be potential attack). // We do not broadcast as this is a local operation only to avoid our maps get polluted with invalid objects // and as we do not check for ownership a node would not accept such a procedure if it would come from untrusted // source (network). } public ProtectedStorageEntry getProtectedStorageEntry(ProtectedStoragePayload protectedStoragePayload, KeyPair ownerStoragePubKey) throws CryptoException { ByteArray hashOfData = get32ByteHashAsByteArray(protectedStoragePayload); int sequenceNumber; if (sequenceNumberMap.containsKey(hashOfData)) sequenceNumber = sequenceNumberMap.get(hashOfData).sequenceNr + 1; else sequenceNumber = 1; byte[] hashOfDataAndSeqNr = P2PDataStorage.get32ByteHash(new DataAndSeqNrPair(protectedStoragePayload, sequenceNumber)); byte[] signature = Sig.sign(ownerStoragePubKey.getPrivate(), hashOfDataAndSeqNr); return new ProtectedStorageEntry(protectedStoragePayload, ownerStoragePubKey.getPublic(), sequenceNumber, signature, this.clock); } public RefreshOfferMessage getRefreshTTLMessage(ProtectedStoragePayload protectedStoragePayload, KeyPair ownerStoragePubKey) throws CryptoException { ByteArray hashOfPayload = get32ByteHashAsByteArray(protectedStoragePayload); int sequenceNumber; if (sequenceNumberMap.containsKey(hashOfPayload)) sequenceNumber = sequenceNumberMap.get(hashOfPayload).sequenceNr + 1; else sequenceNumber = 1; byte[] hashOfDataAndSeqNr = P2PDataStorage.get32ByteHash(new DataAndSeqNrPair(protectedStoragePayload, sequenceNumber)); byte[] signature = Sig.sign(ownerStoragePubKey.getPrivate(), hashOfDataAndSeqNr); return new RefreshOfferMessage(hashOfDataAndSeqNr, signature, hashOfPayload.bytes, sequenceNumber); } public ProtectedMailboxStorageEntry getMailboxDataWithSignedSeqNr(MailboxStoragePayload expirableMailboxStoragePayload, KeyPair storageSignaturePubKey, PublicKey receiversPublicKey) throws CryptoException { ByteArray hashOfData = get32ByteHashAsByteArray(expirableMailboxStoragePayload); int sequenceNumber; if (sequenceNumberMap.containsKey(hashOfData)) sequenceNumber = sequenceNumberMap.get(hashOfData).sequenceNr + 1; else sequenceNumber = 1; byte[] hashOfDataAndSeqNr = P2PDataStorage.get32ByteHash(new DataAndSeqNrPair(expirableMailboxStoragePayload, sequenceNumber)); byte[] signature = Sig.sign(storageSignaturePubKey.getPrivate(), hashOfDataAndSeqNr); return new ProtectedMailboxStorageEntry(expirableMailboxStoragePayload, storageSignaturePubKey.getPublic(), sequenceNumber, signature, receiversPublicKey, this.clock); } public void addHashMapChangedListener(HashMapChangedListener hashMapChangedListener) { hashMapChangedListeners.add(hashMapChangedListener); } public void removeHashMapChangedListener(HashMapChangedListener hashMapChangedListener) { hashMapChangedListeners.remove(hashMapChangedListener); } public void addAppendOnlyDataStoreListener(AppendOnlyDataStoreListener listener) { appendOnlyDataStoreListeners.add(listener); } @SuppressWarnings("unused") public void removeAppendOnlyDataStoreListener(AppendOnlyDataStoreListener listener) { appendOnlyDataStoreListeners.remove(listener); } /////////////////////////////////////////////////////////////////////////////////////////// // Private /////////////////////////////////////////////////////////////////////////////////////////// private void removeFromMapAndDataStore(ProtectedStorageEntry protectedStorageEntry, ByteArray hashOfPayload) { removeFromMapAndDataStore(Collections.singletonList(Maps.immutableEntry(hashOfPayload, protectedStorageEntry))); } private void removeFromMapAndDataStore( Collection<Map.Entry<ByteArray, ProtectedStorageEntry>> entriesToRemoveWithPayloadHash) { if (entriesToRemoveWithPayloadHash.isEmpty()) return; ArrayList<ProtectedStorageEntry> entriesForSignal = new ArrayList<>(entriesToRemoveWithPayloadHash.size()); entriesToRemoveWithPayloadHash.forEach(entryToRemoveWithPayloadHash -> { ByteArray hashOfPayload = entryToRemoveWithPayloadHash.getKey(); ProtectedStorageEntry protectedStorageEntry = entryToRemoveWithPayloadHash.getValue(); map.remove(hashOfPayload); entriesForSignal.add(protectedStorageEntry); ProtectedStoragePayload protectedStoragePayload = protectedStorageEntry.getProtectedStoragePayload(); if (protectedStoragePayload instanceof PersistablePayload) { ProtectedStorageEntry previous = protectedDataStoreService.remove(hashOfPayload, protectedStorageEntry); if (previous == null) log.error("We cannot remove the protectedStorageEntry from the persistedEntryMap as it does not exist."); } }); hashMapChangedListeners.forEach(e -> e.onRemoved(entriesForSignal)); } private boolean hasSequenceNrIncreased(int newSequenceNumber, ByteArray hashOfData) { if (sequenceNumberMap.containsKey(hashOfData)) { int storedSequenceNumber = sequenceNumberMap.get(hashOfData).sequenceNr; if (newSequenceNumber > storedSequenceNumber) { log.trace("Sequence number has increased (>). sequenceNumber = " + newSequenceNumber + " / storedSequenceNumber=" + storedSequenceNumber + " / hashOfData=" + hashOfData.toString()); return true; } else if (newSequenceNumber == storedSequenceNumber) { String msg; if (newSequenceNumber == 0) { msg = "Sequence number is equal to the stored one and both are 0." + "That is expected for network_messages which never got updated (mailbox msg)."; } else { msg = "Sequence number is equal to the stored one. sequenceNumber = " + newSequenceNumber + " / storedSequenceNumber=" + storedSequenceNumber; } log.trace(msg); return false; } else { log.debug("Sequence number is invalid. sequenceNumber = " + newSequenceNumber + " / storedSequenceNumber=" + storedSequenceNumber + "\n" + "That can happen if the data owner gets an old delayed data storage message."); return false; } } else { return true; } } public static ByteArray get32ByteHashAsByteArray(NetworkPayload data) { return new ByteArray(P2PDataStorage.get32ByteHash(data)); } // Get a new map with entries older than PURGE_AGE_DAYS purged from the given map. private Map<ByteArray, MapValue> getPurgedSequenceNumberMap(Map<ByteArray, MapValue> persisted) { Map<ByteArray, MapValue> purged = new HashMap<>(); long maxAgeTs = this.clock.millis() - TimeUnit.DAYS.toMillis(PURGE_AGE_DAYS); persisted.forEach((key, value) -> { if (value.timeStamp > maxAgeTs) purged.put(key, value); }); return purged; } private void printData(String info) { if (log.isTraceEnabled()) { StringBuilder sb = new StringBuilder("\n\n------------------------------------------------------------\n"); sb.append("Data set ").append(info).append(" operation"); // We print the items sorted by hash with the payload class name and id List<Tuple2<String, ProtectedStorageEntry>> tempList = map.values().stream() .map(e -> new Tuple2<>(org.bitcoinj.core.Utils.HEX.encode(get32ByteHashAsByteArray(e.getProtectedStoragePayload()).bytes), e)) .sorted(Comparator.comparing(o -> o.first)) .collect(Collectors.toList()); tempList.forEach(e -> { ProtectedStorageEntry storageEntry = e.second; ProtectedStoragePayload protectedStoragePayload = storageEntry.getProtectedStoragePayload(); MapValue mapValue = sequenceNumberMap.get(get32ByteHashAsByteArray(protectedStoragePayload)); sb.append("\n") .append("Hash=") .append(e.first) .append("; Class=") .append(protectedStoragePayload.getClass().getSimpleName()) .append("; SequenceNumbers (Object/Stored)=") .append(storageEntry.getSequenceNumber()) .append(" / ") .append(mapValue != null ? mapValue.sequenceNr : "null") .append("; TimeStamp (Object/Stored)=") .append(storageEntry.getCreationTimeStamp()) .append(" / ") .append(mapValue != null ? mapValue.timeStamp : "null") .append("; Payload=") .append(Utilities.toTruncatedString(protectedStoragePayload)); }); sb.append("\n------------------------------------------------------------\n"); log.trace(sb.toString()); //log.debug("Data set " + info + " operation: size=" + map.values().size()); } } /** * @param data Network payload * @return Hash of data */ public static byte[] get32ByteHash(NetworkPayload data) { return Hash.getSha256Hash(data.toProtoMessage().toByteArray()); } /////////////////////////////////////////////////////////////////////////////////////////// // Static class /////////////////////////////////////////////////////////////////////////////////////////// /** * Used as container for calculating cryptographic hash of data and sequenceNumber. */ @EqualsAndHashCode @ToString public static final class DataAndSeqNrPair implements NetworkPayload { // data are only used for calculating cryptographic hash from both values so they are kept private private final ProtectedStoragePayload protectedStoragePayload; private final int sequenceNumber; public DataAndSeqNrPair(ProtectedStoragePayload protectedStoragePayload, int sequenceNumber) { this.protectedStoragePayload = protectedStoragePayload; this.sequenceNumber = sequenceNumber; } // Used only for calculating hash of byte array from PB object @Override public com.google.protobuf.Message toProtoMessage() { return protobuf.DataAndSeqNrPair.newBuilder() .setPayload((protobuf.StoragePayload) protectedStoragePayload.toProtoMessage()) .setSequenceNumber(sequenceNumber) .build(); } } /** * Used as key object in map for cryptographic hash of stored data as byte[] as primitive data type cannot be * used as key */ @EqualsAndHashCode public static final class ByteArray implements PersistablePayload { // That object is saved to disc. We need to take care of changes to not break deserialization. public final byte[] bytes; @Override public String toString() { return "ByteArray{" + "bytes as Hex=" + Hex.encode(bytes) + '}'; } public ByteArray(byte[] bytes) { this.bytes = bytes; } /////////////////////////////////////////////////////////////////////////////////////////// // Protobuffer /////////////////////////////////////////////////////////////////////////////////////////// @Override public protobuf.ByteArray toProtoMessage() { return protobuf.ByteArray.newBuilder().setBytes(ByteString.copyFrom(bytes)).build(); } public static ByteArray fromProto(protobuf.ByteArray proto) { return new ByteArray(proto.getBytes().toByteArray()); } /////////////////////////////////////////////////////////////////////////////////////////// // Util /////////////////////////////////////////////////////////////////////////////////////////// @SuppressWarnings("unused") public String getHex() { return Utilities.encodeToHex(bytes); } public static Set<P2PDataStorage.ByteArray> convertBytesSetToByteArraySet(Set<byte[]> set) { return set != null ? set.stream() .map(P2PDataStorage.ByteArray::new) .collect(Collectors.toSet()) : new HashSet<>(); } } /** * Used as value in map */ @EqualsAndHashCode @ToString public static final class MapValue implements PersistablePayload { // That object is saved to disc. We need to take care of changes to not break deserialization. final public int sequenceNr; final public long timeStamp; MapValue(int sequenceNr, long timeStamp) { this.sequenceNr = sequenceNr; this.timeStamp = timeStamp; } @Override public protobuf.MapValue toProtoMessage() { return protobuf.MapValue.newBuilder().setSequenceNr(sequenceNr).setTimeStamp(timeStamp).build(); } public static MapValue fromProto(protobuf.MapValue proto) { return new MapValue(proto.getSequenceNr(), proto.getTimeStamp()); } } }