Branch data Line data Source code
1 : : // Copyright (c) 2011-2022 The Bitcoin Core developers
2 : : // Distributed under the MIT software license, see the accompanying
3 : : // file COPYING or http://www.opensource.org/licenses/mit-license.php.
4 : :
5 : : #ifndef BITCOIN_NODE_BLOCKSTORAGE_H
6 : : #define BITCOIN_NODE_BLOCKSTORAGE_H
7 : :
8 : : #include <attributes.h>
9 : : #include <chain.h>
10 : : #include <dbwrapper.h>
11 : : #include <flatfile.h>
12 : : #include <kernel/blockmanager_opts.h>
13 : : #include <kernel/chainparams.h>
14 : : #include <kernel/cs_main.h>
15 : : #include <kernel/messagestartchars.h>
16 : : #include <primitives/block.h>
17 : : #include <streams.h>
18 : : #include <sync.h>
19 : : #include <uint256.h>
20 : : #include <util/fs.h>
21 : : #include <util/hasher.h>
22 : :
23 : : #include <array>
24 : : #include <atomic>
25 : : #include <cstdint>
26 : : #include <functional>
27 : : #include <limits>
28 : : #include <map>
29 : : #include <memory>
30 : : #include <optional>
31 : : #include <set>
32 : : #include <span>
33 : : #include <string>
34 : : #include <unordered_map>
35 : : #include <utility>
36 : : #include <vector>
37 : :
38 : : class BlockValidationState;
39 : : class CBlockUndo;
40 : : class Chainstate;
41 : : class ChainstateManager;
42 : : namespace Consensus {
43 : : struct Params;
44 : : }
45 : : namespace util {
46 : : class SignalInterrupt;
47 : : } // namespace util
48 : :
49 : : namespace kernel {
50 : : class CBlockFileInfo
51 : : {
52 : : public:
53 : : uint32_t nBlocks{}; //!< number of blocks stored in file
54 : : uint32_t nSize{}; //!< number of used bytes of block file
55 : : uint32_t nUndoSize{}; //!< number of used bytes in the undo file
56 : : uint32_t nHeightFirst{}; //!< lowest height of block in file
57 : : uint32_t nHeightLast{}; //!< highest height of block in file
58 : : uint64_t nTimeFirst{}; //!< earliest time of block in file
59 : : uint64_t nTimeLast{}; //!< latest time of block in file
60 : :
61 : 230718 : SERIALIZE_METHODS(CBlockFileInfo, obj)
62 : : {
63 : 132671 : READWRITE(VARINT(obj.nBlocks));
64 : 132658 : READWRITE(VARINT(obj.nSize));
65 : 132648 : READWRITE(VARINT(obj.nUndoSize));
66 : 132641 : READWRITE(VARINT(obj.nHeightFirst));
67 : 132639 : READWRITE(VARINT(obj.nHeightLast));
68 : 132639 : READWRITE(VARINT(obj.nTimeFirst));
69 : 132632 : READWRITE(VARINT(obj.nTimeLast));
70 : 132626 : }
71 : :
72 : 1687 : CBlockFileInfo() = default;
73 : :
74 : : std::string ToString() const;
75 : :
76 : : /** update statistics (does not update nSize) */
77 : 112284 : void AddBlock(unsigned int nHeightIn, uint64_t nTimeIn)
78 : : {
79 [ + + - + ]: 112284 : if (nBlocks == 0 || nHeightFirst > nHeightIn)
80 : 1687 : nHeightFirst = nHeightIn;
81 [ + + - + ]: 112284 : if (nBlocks == 0 || nTimeFirst > nTimeIn)
82 : 1687 : nTimeFirst = nTimeIn;
83 : 112284 : nBlocks++;
84 [ + + ]: 112284 : if (nHeightIn > nHeightLast)
85 : 101972 : nHeightLast = nHeightIn;
86 [ + + ]: 112284 : if (nTimeIn > nTimeLast)
87 : 21647 : nTimeLast = nTimeIn;
88 : 112284 : }
89 : : };
90 : :
91 : : /** Access to the block database (blocks/index/) */
92 : 2167 : class BlockTreeDB : public CDBWrapper
93 : : {
94 : : public:
95 [ + - ]: 2167 : using CDBWrapper::CDBWrapper;
96 : : void WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*>>& fileInfo, int nLastFile, const std::vector<const CBlockIndex*>& blockinfo);
97 : : bool ReadBlockFileInfo(int nFile, CBlockFileInfo& info);
98 : : bool ReadLastBlockFile(int& nFile);
99 : : void WriteReindexing(bool fReindexing);
100 : : void ReadReindexing(bool& fReindexing);
101 : : void WriteFlag(const std::string& name, bool fValue);
102 : : bool ReadFlag(const std::string& name, bool& fValue);
103 : : bool LoadBlockIndexGuts(const Consensus::Params& consensusParams, std::function<CBlockIndex*(const uint256&)> insertBlockIndex, const util::SignalInterrupt& interrupt)
104 : : EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
105 : : };
106 : : } // namespace kernel
107 : :
108 : : namespace node {
109 : : using kernel::CBlockFileInfo;
110 : : using kernel::BlockTreeDB;
111 : :
112 : : /** The pre-allocation chunk size for blk?????.dat files (since 0.8) */
113 : : static const unsigned int BLOCKFILE_CHUNK_SIZE = 0x1000000; // 16 MiB
114 : : /** The pre-allocation chunk size for rev?????.dat files (since 0.8) */
115 : : static const unsigned int UNDOFILE_CHUNK_SIZE = 0x100000; // 1 MiB
116 : : /** The maximum size of a blk?????.dat file (since 0.8) */
117 : : static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB
118 : :
119 : : /** Size of header written by WriteBlock before a serialized CBlock (8 bytes) */
120 : : static constexpr uint32_t STORAGE_HEADER_BYTES{std::tuple_size_v<MessageStartChars> + sizeof(unsigned int)};
121 : :
122 : : /** Total overhead when writing undo data: header (8 bytes) plus checksum (32 bytes) */
123 : : static constexpr uint32_t UNDO_DATA_DISK_OVERHEAD{STORAGE_HEADER_BYTES + uint256::size()};
124 : :
125 : : // Because validation code takes pointers to the map's CBlockIndex objects, if
126 : : // we ever switch to another associative container, we need to either use a
127 : : // container that has stable addressing (true of all std associative
128 : : // containers), or make the key a `std::unique_ptr<CBlockIndex>`
129 : : using BlockMap = std::unordered_map<uint256, CBlockIndex, BlockHasher>;
130 : :
131 : : struct CBlockIndexWorkComparator {
132 : : bool operator()(const CBlockIndex* pa, const CBlockIndex* pb) const;
133 : : };
134 : :
135 : : struct CBlockIndexHeightOnlyComparator {
136 : : /* Only compares the height of two block indices, doesn't try to tie-break */
137 : : bool operator()(const CBlockIndex* pa, const CBlockIndex* pb) const;
138 : : };
139 : :
140 : 0 : struct PruneLockInfo {
141 : : int height_first{std::numeric_limits<int>::max()}; //! Height of earliest block that should be kept and not pruned
142 : : };
143 : :
144 : : enum BlockfileType {
145 : : // Values used as array indexes - do not change carelessly.
146 : : NORMAL = 0,
147 : : ASSUMED = 1,
148 : : NUM_TYPES = 2,
149 : : };
150 : :
151 : : std::ostream& operator<<(std::ostream& os, const BlockfileType& type);
152 : :
153 : : struct BlockfileCursor {
154 : : // The latest blockfile number.
155 : : int file_num{0};
156 : :
157 : : // Track the height of the highest block in file_num whose undo
158 : : // data has been written. Block data is written to block files in download
159 : : // order, but is written to undo files in validation order, which is
160 : : // usually in order by height. To avoid wasting disk space, undo files will
161 : : // be trimmed whenever the corresponding block file is finalized and
162 : : // the height of the highest block written to the block file equals the
163 : : // height of the highest block written to the undo file. This is a
164 : : // heuristic and can sometimes preemptively trim undo files that will write
165 : : // more data later, and sometimes fail to trim undo files that can't have
166 : : // more data written later.
167 : : int undo_height{0};
168 : : };
169 : :
170 : : std::ostream& operator<<(std::ostream& os, const BlockfileCursor& cursor);
171 : :
172 : :
173 : : /**
174 : : * Maintains a tree of blocks (stored in `m_block_index`) which is consulted
175 : : * to determine where the most-work tip is.
176 : : *
177 : : * This data is used mostly in `Chainstate` - information about, e.g.,
178 : : * candidate tips is not maintained here.
179 : : */
180 : : class BlockManager
181 : : {
182 : : friend Chainstate;
183 : : friend ChainstateManager;
184 : :
185 : : private:
186 [ + - - + : 220141 : const CChainParams& GetParams() const { return m_opts.chainparams; }
+ - ]
187 [ + - - + : 14033 : const Consensus::Params& GetConsensus() const { return m_opts.chainparams.GetConsensus(); }
- - + - ]
188 : : /**
189 : : * Load the blocktree off disk and into memory. Populate certain metadata
190 : : * per index entry (nStatus, nChainWork, nTimeMax, etc.) as well as peripheral
191 : : * collections like m_dirty_blockindex.
192 : : */
193 : : bool LoadBlockIndex(const std::optional<uint256>& snapshot_blockhash)
194 : : EXCLUSIVE_LOCKS_REQUIRED(cs_main);
195 : :
196 : : /** Return false if block file or undo file flushing fails. */
197 : : [[nodiscard]] bool FlushBlockFile(int blockfile_num, bool fFinalize, bool finalize_undo);
198 : :
199 : : /** Return false if undo file flushing fails. */
200 : : [[nodiscard]] bool FlushUndoFile(int block_file, bool finalize = false);
201 : :
202 : : /**
203 : : * Helper function performing various preparations before a block can be saved to disk:
204 : : * Returns the correct position for the block to be saved, which may be in the current or a new
205 : : * block file depending on nAddSize. May flush the previous blockfile to disk if full, updates
206 : : * blockfile info, and checks if there is enough disk space to save the block.
207 : : *
208 : : * The nAddSize argument passed to this function should include not just the size of the serialized CBlock, but also the size of
209 : : * separator fields (STORAGE_HEADER_BYTES).
210 : : */
211 : : [[nodiscard]] FlatFilePos FindNextBlockPos(unsigned int nAddSize, unsigned int nHeight, uint64_t nTime);
212 : : [[nodiscard]] bool FlushChainstateBlockFile(int tip_height);
213 : : bool FindUndoPos(BlockValidationState& state, int nFile, FlatFilePos& pos, unsigned int nAddSize);
214 : :
215 : : AutoFile OpenUndoFile(const FlatFilePos& pos, bool fReadOnly = false) const;
216 : :
217 : : /* Calculate the block/rev files to delete based on height specified by user with RPC command pruneblockchain */
218 : : void FindFilesToPruneManual(
219 : : std::set<int>& setFilesToPrune,
220 : : int nManualPruneHeight,
221 : : const Chainstate& chain,
222 : : ChainstateManager& chainman);
223 : :
224 : : /**
225 : : * Prune block and undo files (blk???.dat and rev???.dat) so that the disk space used is less than a user-defined target.
226 : : * The user sets the target (in MB) on the command line or in config file. This will be run on startup and whenever new
227 : : * space is allocated in a block or undo file, staying below the target. Changing back to unpruned requires a reindex
228 : : * (which in this case means the blockchain must be re-downloaded.)
229 : : *
230 : : * Pruning functions are called from FlushStateToDisk when the m_check_for_pruning flag has been set.
231 : : * Block and undo files are deleted in lock-step (when blk00003.dat is deleted, so is rev00003.dat.)
232 : : * Pruning cannot take place until the longest chain is at least a certain length (CChainParams::nPruneAfterHeight).
233 : : * Pruning will never delete a block within a defined distance (currently 288) from the active chain's tip.
234 : : * The block index is updated by unsetting HAVE_DATA and HAVE_UNDO for any blocks that were stored in the deleted files.
235 : : * A db flag records the fact that at least some block files have been pruned.
236 : : *
237 : : * @param[out] setFilesToPrune The set of file indices that can be unlinked will be returned
238 : : * @param last_prune The last height we're able to prune, according to the prune locks
239 : : */
240 : : void FindFilesToPrune(
241 : : std::set<int>& setFilesToPrune,
242 : : int last_prune,
243 : : const Chainstate& chain,
244 : : ChainstateManager& chainman);
245 : :
246 : : RecursiveMutex cs_LastBlockFile;
247 : : std::vector<CBlockFileInfo> m_blockfile_info;
248 : :
249 : : //! Since assumedvalid chainstates may be syncing a range of the chain that is very
250 : : //! far away from the normal/background validation process, we should segment blockfiles
251 : : //! for assumed chainstates. Otherwise, we might have wildly different height ranges
252 : : //! mixed into the same block files, which would impair our ability to prune
253 : : //! effectively.
254 : : //!
255 : : //! This data structure maintains separate blockfile number cursors for each
256 : : //! BlockfileType. The ASSUMED state is initialized, when necessary, in FindNextBlockPos().
257 : : //!
258 : : //! The first element is the NORMAL cursor, second is ASSUMED.
259 : : std::array<std::optional<BlockfileCursor>, BlockfileType::NUM_TYPES>
260 : : m_blockfile_cursors GUARDED_BY(cs_LastBlockFile) = {
261 : : BlockfileCursor{},
262 : : std::nullopt,
263 : : };
264 : 97672 : int MaxBlockfileNum() const EXCLUSIVE_LOCKS_REQUIRED(cs_LastBlockFile)
265 : : {
266 : 97672 : static const BlockfileCursor empty_cursor;
267 [ + - ]: 97672 : const auto& normal = m_blockfile_cursors[BlockfileType::NORMAL].value_or(empty_cursor);
268 [ - + ]: 97672 : const auto& assumed = m_blockfile_cursors[BlockfileType::ASSUMED].value_or(empty_cursor);
269 [ + - ]: 97672 : return std::max(normal.file_num, assumed.file_num);
270 : : }
271 : :
272 : : /** Global flag to indicate we should check to see if there are
273 : : * block/undo files that should be deleted. Set on startup
274 : : * or if we allocate more file space when we're in prune mode
275 : : */
276 : : bool m_check_for_pruning = false;
277 : :
278 : : const bool m_prune_mode;
279 : :
280 : : const Obfuscation m_obfuscation;
281 : :
282 : : /** Dirty block index entries. */
283 : : std::set<CBlockIndex*> m_dirty_blockindex;
284 : :
285 : : /** Dirty block file entries. */
286 : : std::set<int> m_dirty_fileinfo;
287 : :
288 : : /**
289 : : * Map from external index name to oldest block that must not be pruned.
290 : : *
291 : : * @note Internally, only blocks at height (height_first - PRUNE_LOCK_BUFFER - 1) and
292 : : * below will be pruned, but callers should avoid assuming any particular buffer size.
293 : : */
294 : : std::unordered_map<std::string, PruneLockInfo> m_prune_locks GUARDED_BY(::cs_main);
295 : :
296 : : BlockfileType BlockfileTypeForHeight(int height);
297 : :
298 : : const kernel::BlockManagerOpts m_opts;
299 : :
300 : : const FlatFileSeq m_block_file_seq;
301 : : const FlatFileSeq m_undo_file_seq;
302 : :
303 : : public:
304 : : using Options = kernel::BlockManagerOpts;
305 : :
306 : : explicit BlockManager(const util::SignalInterrupt& interrupt, Options opts);
307 : :
308 : : const util::SignalInterrupt& m_interrupt;
309 : : std::atomic<bool> m_importing{false};
310 : :
311 : : /**
312 : : * Whether all blockfiles have been added to the block tree database.
313 : : * Normally true, but set to false when a reindex is requested and the
314 : : * database is wiped. The value is persisted in the database across restarts
315 : : * and will be false until reindexing completes.
316 : : */
317 : : std::atomic_bool m_blockfiles_indexed{true};
318 : :
319 : : BlockMap m_block_index GUARDED_BY(cs_main);
320 : :
321 : : /**
322 : : * The height of the base block of an assumeutxo snapshot, if one is in use.
323 : : *
324 : : * This controls how blockfiles are segmented by chainstate type to avoid
325 : : * comingling different height regions of the chain when an assumedvalid chainstate
326 : : * is in use. If heights are drastically different in the same blockfile, pruning
327 : : * suffers.
328 : : *
329 : : * This is set during ActivateSnapshot() or upon LoadBlockIndex() if a snapshot
330 : : * had been previously loaded. After the snapshot is validated, this is unset to
331 : : * restore normal LoadBlockIndex behavior.
332 : : */
333 : : std::optional<int> m_snapshot_height;
334 : :
335 : : std::vector<CBlockIndex*> GetAllBlockIndices() EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
336 : :
337 : : /**
338 : : * All pairs A->B, where A (or one of its ancestors) misses transactions, but B has transactions.
339 : : * Pruned nodes may have entries where B is missing data.
340 : : */
341 : : std::multimap<CBlockIndex*, CBlockIndex*> m_blocks_unlinked;
342 : :
343 : : std::unique_ptr<BlockTreeDB> m_block_tree_db GUARDED_BY(::cs_main);
344 : :
345 : : void WriteBlockIndexDB() EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
346 : : bool LoadBlockIndexDB(const std::optional<uint256>& snapshot_blockhash)
347 : : EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
348 : :
349 : : /**
350 : : * Remove any pruned block & undo files that are still on disk.
351 : : * This could happen on some systems if the file was still being read while unlinked,
352 : : * or if we crash before unlinking.
353 : : */
354 : : void ScanAndUnlinkAlreadyPrunedFiles() EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
355 : :
356 : : CBlockIndex* AddToBlockIndex(const CBlockHeader& block, CBlockIndex*& best_header) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
357 : : /** Create a new block index entry for a given block hash */
358 : : CBlockIndex* InsertBlockIndex(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
359 : :
360 : : //! Mark one block file as pruned (modify associated database entries)
361 : : void PruneOneBlockFile(const int fileNumber) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
362 : :
363 : : CBlockIndex* LookupBlockIndex(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
364 : : const CBlockIndex* LookupBlockIndex(const uint256& hash) const EXCLUSIVE_LOCKS_REQUIRED(cs_main);
365 : :
366 : : /** Get block file info entry for one block file */
367 : : CBlockFileInfo* GetBlockFileInfo(size_t n);
368 : :
369 : : bool WriteBlockUndo(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex& block)
370 : : EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
371 : :
372 : : /** Store block on disk and update block file statistics.
373 : : *
374 : : * @param[in] block the block to be stored
375 : : * @param[in] nHeight the height of the block
376 : : *
377 : : * @returns in case of success, the position to which the block was written to
378 : : * in case of an error, an empty FlatFilePos
379 : : */
380 : : FlatFilePos WriteBlock(const CBlock& block, int nHeight);
381 : :
382 : : /** Update blockfile info while processing a block during reindex. The block must be available on disk.
383 : : *
384 : : * @param[in] block the block being processed
385 : : * @param[in] nHeight the height of the block
386 : : * @param[in] pos the position of the serialized CBlock on disk
387 : : */
388 : : void UpdateBlockInfo(const CBlock& block, unsigned int nHeight, const FlatFilePos& pos);
389 : :
390 : : /** Whether running in -prune mode. */
391 [ + + ][ # # : 1647324 : [[nodiscard]] bool IsPruneMode() const { return m_prune_mode; }
# # # # #
# ][ - + -
+ - + #
# ][ - - +
- - + + -
- + ]
392 : :
393 : : /** Attempt to stay below this number of bytes of block files. */
394 [ - + ]: 1687 : [[nodiscard]] uint64_t GetPruneTarget() const { return m_opts.prune_target; }
[ # # # # ]
395 : : static constexpr auto PRUNE_TARGET_MANUAL{std::numeric_limits<uint64_t>::max()};
396 : :
397 [ + - - + ]: 755603 : [[nodiscard]] bool LoadingBlocks() const { return m_importing || !m_blockfiles_indexed; }
398 : :
399 : : /** Calculate the amount of disk space the block & undo files currently use */
400 : : uint64_t CalculateCurrentUsage();
401 : :
402 : : //! Check if all blocks in the [upper_block, lower_block] range have data available.
403 : : //! The caller is responsible for ensuring that lower_block is an ancestor of upper_block
404 : : //! (part of the same chain).
405 : : bool CheckBlockDataAvailability(const CBlockIndex& upper_block, const CBlockIndex& lower_block) EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
406 : :
407 : : /**
408 : : * @brief Returns the earliest block with specified `status_mask` flags set after
409 : : * the latest block _not_ having those flags.
410 : : *
411 : : * This function starts from `upper_block`, which must have all
412 : : * `status_mask` flags set, and iterates backwards through its ancestors. It
413 : : * continues as long as each block has all `status_mask` flags set, until
414 : : * reaching the oldest ancestor or `lower_block`.
415 : : *
416 : : * @pre `upper_block` must have all `status_mask` flags set.
417 : : * @pre `lower_block` must be null or an ancestor of `upper_block`
418 : : *
419 : : * @param upper_block The starting block for the search, which must have all
420 : : * `status_mask` flags set.
421 : : * @param status_mask Bitmask specifying required status flags.
422 : : * @param lower_block The earliest possible block to return. If null, the
423 : : * search can extend to the genesis block.
424 : : *
425 : : * @return A reference to the earliest block between `upper_block`
426 : : * and `lower_block`, inclusive, such that every block between the
427 : : * returned block and `upper_block` has `status_mask` flags set.
428 : : */
429 : : const CBlockIndex& GetFirstBlock(
430 : : const CBlockIndex& upper_block LIFETIMEBOUND,
431 : : uint32_t status_mask,
432 : : const CBlockIndex* lower_block LIFETIMEBOUND = nullptr
433 : : ) const EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
434 : :
435 : : /** True if any block files have ever been pruned. */
436 : : bool m_have_pruned = false;
437 : :
438 : : //! Check whether the block associated with this index entry is pruned or not.
439 : : bool IsBlockPruned(const CBlockIndex& block) const EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
440 : :
441 : : //! Create or update a prune lock identified by its name
442 : : void UpdatePruneLock(const std::string& name, const PruneLockInfo& lock_info) EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
443 : :
444 : : /** Open a block file (blk?????.dat) */
445 : : AutoFile OpenBlockFile(const FlatFilePos& pos, bool fReadOnly) const;
446 : :
447 : : /** Translation to a filesystem path */
448 : : fs::path GetBlockPosFilename(const FlatFilePos& pos) const;
449 : :
450 : : /**
451 : : * Actually unlink the specified files
452 : : */
453 : : void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune) const;
454 : :
455 : : /** Functions for disk access for blocks */
456 : : bool ReadBlock(CBlock& block, const FlatFilePos& pos, const std::optional<uint256>& expected_hash) const;
457 : : bool ReadBlock(CBlock& block, const CBlockIndex& index) const;
458 : : bool ReadRawBlock(std::vector<std::byte>& block, const FlatFilePos& pos) const;
459 : :
460 : : bool ReadBlockUndo(CBlockUndo& blockundo, const CBlockIndex& index) const;
461 : :
462 : : void CleanupBlockRevFiles() const;
463 : : };
464 : :
465 : : // Calls ActivateBestChain() even if no blocks are imported.
466 : : void ImportBlocks(ChainstateManager& chainman, std::span<const fs::path> import_paths);
467 : : } // namespace node
468 : :
469 : : #endif // BITCOIN_NODE_BLOCKSTORAGE_H
|
