diff --git a/cmake/dependencies.cmake b/cmake/dependencies.cmake index 11c568c3ea..69b2a92caf 100644 --- a/cmake/dependencies.cmake +++ b/cmake/dependencies.cmake @@ -5,9 +5,6 @@ hunter_add_package(GTest) find_package(GTest CONFIG REQUIRED) -hunter_add_package(libarchive) -find_package(libarchive CONFIG REQUIRED) - # https://docs.hunter.sh/en/latest/packages/pkg/Boost.html hunter_add_package(Boost COMPONENTS date_time filesystem iostreams random program_options thread) find_package(Boost CONFIG REQUIRED date_time filesystem iostreams random program_options thread) @@ -72,5 +69,8 @@ find_package(RapidJSON CONFIG REQUIRED) hunter_add_package(jwt-cpp) find_package(jwt-cpp CONFIG REQUIRED) +hunter_add_package(libarchive) +find_package(libarchive CONFIG REQUIRED) + hunter_add_package(prometheus-cpp) find_package(prometheus-cpp CONFIG REQUIRED) diff --git a/core/common/put_in_function.hpp b/core/common/put_in_function.hpp new file mode 100644 index 0000000000..ad69415c1f --- /dev/null +++ b/core/common/put_in_function.hpp @@ -0,0 +1,25 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +namespace fc::common { + + /** + * Class to put move-only lambdas in std::function interface + */ + template + class PutInFunction { + public: + explicit PutInFunction(T &&function) + : move_function_{std::make_shared(std::move(function))} {}; + + template + auto operator()(Args &&...args) const { + return move_function_->operator()(std::forward(args)...); + } + + private: + std::shared_ptr move_function_; + }; +} // namespace fc::common diff --git a/core/common/vector_cow.hpp b/core/common/vector_cow.hpp new file mode 100644 index 0000000000..773dbe6da8 --- /dev/null +++ b/core/common/vector_cow.hpp @@ -0,0 +1,69 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include + +namespace fc { + + template + class VectorCoW { + public: + using VectorType = std::vector; + using SpanType = gsl::span; + + VectorCoW() = default; + // NOLINTNEXTLINE(google-explicit-constructor) + VectorCoW(VectorType &&vector) : variant_{std::move(vector)} {} + VectorCoW(const VectorType &vector) = delete; + // NOLINTNEXTLINE(google-explicit-constructor) + VectorCoW(const SpanType &span) : variant_{span} {} + VectorCoW(const VectorCoW &) = delete; + VectorCoW(VectorCoW &&) = default; + VectorCoW &operator=(const VectorCoW &) = delete; + VectorCoW &operator=(VectorCoW &&) = default; + ~VectorCoW() = default; + + bool owned() const { + return variant_.index() == 1; + } + + SpanType span() const { + if (!owned()) { + return std::get(variant_); + } + return SpanType{std::get(variant_)}; + } + + size_t size() const { + return span().size(); + } + + bool empty() const { + return span().empty(); + } + + // get mutable container reference, copy once if span + VectorType &mut() { + if (!owned()) { + const auto span = std::get(variant_); + variant_.template emplace(span.begin(), span.end()); + } + return std::get(variant_); + } + + // move container away, copy once if span + VectorType into() { + auto vector{std::move(mut())}; + variant_.template emplace(); + return vector; + } + + private: + std::variant variant_; + }; +} // namespace fc diff --git a/core/markets/storage/provider/impl/provider_impl.cpp b/core/markets/storage/provider/impl/provider_impl.cpp index 568a30c449..f433bb708a 100644 --- a/core/markets/storage/provider/impl/provider_impl.cpp +++ b/core/markets/storage/provider/impl/provider_impl.cpp @@ -681,10 +681,7 @@ namespace fc::markets::storage::provider { deal_context->deal->is_fast_retrieval}); FSM_HALT_ON_ERROR( maybe_piece_location, "Unable to locate piece", deal_context); - FSM_HALT_ON_ERROR( - recordPieceInfo(deal_context->deal, maybe_piece_location.value()), - "Record piece failed", - deal_context); + deal_context->maybe_piece_location = maybe_piece_location.value(); // TODO(a.chernyshov): add piece retry FSM_SEND(deal_context, ProviderEvent::ProviderEventDealHandedOff); } @@ -700,6 +697,17 @@ namespace fc::markets::storage::provider { } FSM_HANDLE_DEFINITION(StorageProviderImpl::onProviderEventDealActivated) { + if (not deal_context->maybe_piece_location.has_value()) { + (deal_context)->deal->message = "Unknown piece location"; + FSM_SEND((deal_context), ProviderEvent::ProviderEventFailed); + return; + } + + FSM_HALT_ON_ERROR( + recordPieceInfo(deal_context->deal, + deal_context->maybe_piece_location.value()), + "Record piece failed", + deal_context); // TODO(a.chernyshov): wait expiration } diff --git a/core/markets/storage/provider/impl/provider_impl.hpp b/core/markets/storage/provider/impl/provider_impl.hpp index 0bebf00cff..e93152bec2 100644 --- a/core/markets/storage/provider/impl/provider_impl.hpp +++ b/core/markets/storage/provider/impl/provider_impl.hpp @@ -90,6 +90,7 @@ namespace fc::markets::storage::provider { struct DealContext { std::shared_ptr deal; std::string protocol; + boost::optional maybe_piece_location{}; }; using ProviderTransition = @@ -116,7 +117,6 @@ namespace fc::markets::storage::provider { if (request) { if (auto asker{stored_ask.lock()}) { if (auto ask{asker->getAsk(request.value().miner)}) { - return stream->write(AskResponseType{{ask.value()}}, [stream](auto) { stream->close(); }); } diff --git a/core/miner/storage_fsm/impl/sealing_impl.cpp b/core/miner/storage_fsm/impl/sealing_impl.cpp index dedcd9df73..bcf18f66df 100644 --- a/core/miner/storage_fsm/impl/sealing_impl.cpp +++ b/core/miner/storage_fsm/impl/sealing_impl.cpp @@ -251,12 +251,13 @@ namespace fc::mining { auto sector_ref = minerSector(seal_proof_type, piece_location.sector); if (sector_and_padding.padding.size != 0) { - OUTCOME_TRY( - sealer_->addPieceSync(sector_ref, - unsealed_sector.piece_sizes, - sector_and_padding.padding.size.unpadded(), - PieceData::makeNull(), - kDealSectorPriority)); + OUTCOME_TRY(sealer_->addPieceSync( + sector_ref, + VectorCoW(gsl::span( + unsealed_sector.piece_sizes)), + sector_and_padding.padding.size.unpadded(), + PieceData::makeNull(), + kDealSectorPriority)); unsealed_sector.stored += sector_and_padding.padding.size; @@ -283,7 +284,9 @@ namespace fc::mining { logger_->info("Add piece to sector {}", piece_location.sector); OUTCOME_TRY(piece_info, sealer_->addPieceSync(sector_ref, - unsealed_sector.piece_sizes, + VectorCoW( + gsl::span( + unsealed_sector.piece_sizes)), size, std::move(piece_data), kDealSectorPriority)); @@ -972,7 +975,7 @@ namespace fc::mining { sealer_->addPiece( sector, - existing_piece_sizes, + VectorCoW(std::move(existing_piece_sizes)), filler, PieceData::makeNull(), [fill = std::move(result), cb](const auto &maybe_error) -> void { diff --git a/core/sector_storage/impl/local_worker.cpp b/core/sector_storage/impl/local_worker.cpp index 1ae299c4c7..ffad76ff43 100644 --- a/core/sector_storage/impl/local_worker.cpp +++ b/core/sector_storage/impl/local_worker.cpp @@ -18,11 +18,13 @@ #include #include #include "api/storage_miner/storage_api.hpp" +#include "common/put_in_function.hpp" #include "primitives/rle_bitset/runs_utils.hpp" #include "primitives/sector_file/sector_file.hpp" #include "sector_storage/stores/store_error.hpp" namespace fc::sector_storage { + using common::PutInFunction; using primitives::piece::PaddedByteIndex; using primitives::piece::PaddedPieceSize; using primitives::sector_file::SectorFile; @@ -270,20 +272,22 @@ namespace fc::sector_storage { outcome::result LocalWorker::addPiece( const SectorRef §or, - gsl::span piece_sizes, + VectorCoW piece_sizes, const UnpaddedPieceSize &new_piece_size, proofs::PieceData piece_data) { return asyncCall( sector, return_->ReturnAddPiece, - [=, piece_data{std::make_shared(std::move(piece_data))}]( - Self self) -> outcome::result { + PutInFunction([=, + exist_sizes = std::move(piece_sizes), + data = std::move(piece_data)]( + const Self &self) -> outcome::result { OUTCOME_TRY(max_size, primitives::sector::getSectorSize(sector.proof_type)); UnpaddedPieceSize offset; - for (const auto &piece_size : piece_sizes) { + for (const auto &piece_size : exist_sizes.span()) { offset += piece_size; } @@ -299,7 +303,7 @@ namespace fc::sector_storage { } }); - if (piece_sizes.empty()) { + if (exist_sizes.empty()) { OUTCOME_TRYA(acquire_response, self->acquireSector(sector, SectorFileType::FTNone, @@ -322,13 +326,13 @@ namespace fc::sector_storage { } OUTCOME_TRY(piece_info, - staged_file->write(*piece_data, + staged_file->write(data, offset.padded(), new_piece_size.padded(), sector.proof_type)); return piece_info.value(); - }); + })); } outcome::result LocalWorker::sealPreCommit1( @@ -461,11 +465,12 @@ namespace fc::sector_storage { } outcome::result LocalWorker::finalizeSector( - const SectorRef §or, const gsl::span &keep_unsealed) { + const SectorRef §or, std::vector keep_unsealed) { return asyncCall( sector, return_->ReturnFinalizeSector, - [=](Self self) -> outcome::result { + [=, keep_unsealed{std::move(keep_unsealed)}]( + Self self) -> outcome::result { OUTCOME_TRY(size, primitives::sector::getSectorSize(sector.proof_type)); { @@ -796,17 +801,16 @@ namespace fc::sector_storage { } }); - OUTCOME_TRY(self->proofs_->unsealRange( - sector.proof_type, - response.paths.cache, - sealed, - PieceData(fds[1]), - sector.id.sector, - sector.id.miner, - randomness, - unsealed_cid, - primitives::piece::paddedIndex(range.offset), - range.size)); + OUTCOME_TRY(self->proofs_->unsealRange(sector.proof_type, + response.paths.cache, + sealed, + PieceData(fds[1]), + sector.id.sector, + sector.id.miner, + randomness, + unsealed_cid, + range.offset, + range.size)); } for (auto &t : threads) { diff --git a/core/sector_storage/impl/local_worker.hpp b/core/sector_storage/impl/local_worker.hpp index d65c95811e..b17eb71782 100644 --- a/core/sector_storage/impl/local_worker.hpp +++ b/core/sector_storage/impl/local_worker.hpp @@ -29,11 +29,10 @@ namespace fc::sector_storage { std::shared_ptr proofs = std::make_shared()); - outcome::result addPiece( - const SectorRef §or, - gsl::span piece_sizes, - const UnpaddedPieceSize &new_piece_size, - PieceData piece_data) override; + outcome::result addPiece(const SectorRef §or, + VectorCoW piece_sizes, + const UnpaddedPieceSize &new_piece_size, + PieceData piece_data) override; outcome::result sealPreCommit1( const SectorRef §or, @@ -54,8 +53,7 @@ namespace fc::sector_storage { const SectorRef §or, const Commit1Output &commit_1_output) override; outcome::result finalizeSector( - const SectorRef §or, - const gsl::span &keep_unsealed) override; + const SectorRef §or, std::vector keep_unsealed) override; outcome::result replicaUpdate( const SectorRef §or, const std::vector &pieces) override; diff --git a/core/sector_storage/impl/manager_impl.cpp b/core/sector_storage/impl/manager_impl.cpp index 43b768f106..9050cd2a4f 100644 --- a/core/sector_storage/impl/manager_impl.cpp +++ b/core/sector_storage/impl/manager_impl.cpp @@ -17,6 +17,7 @@ #include "api/storage_miner/return_api.hpp" #include "codec/json/json.hpp" #include "common/outcome_fmt.hpp" +#include "common/put_in_function.hpp" #include "sector_storage/impl/allocate_selector.hpp" #include "sector_storage/impl/existing_selector.hpp" #include "sector_storage/impl/local_worker.hpp" @@ -25,6 +26,7 @@ #include "sector_storage/stores/store_error.hpp" namespace fc::sector_storage { + using common::PutInFunction; using primitives::sector::SectorInfo; using primitives::sector::toSectorInfo; using primitives::sector_file::SectorFileType; @@ -700,7 +702,7 @@ namespace fc::sector_storage { void ManagerImpl::finalizeSector( const SectorRef §or, - const gsl::span &keep_unsealed, + std::vector keep_unsealed, const std::function)> &cb, uint64_t priority) { OUTCOME_CB(auto lock, @@ -726,7 +728,7 @@ namespace fc::sector_storage { [cb, index{index_}, unsealed, - keep_unsealed, + need_unsealed{not keep_unsealed.empty()}, scheduler{scheduler_}, sector, lock{std::move(lock)}, @@ -740,10 +742,8 @@ namespace fc::sector_storage { | SectorFileType::FTCache), PathType::kStorage); - auto moveUnsealed = unsealed; - if (keep_unsealed.empty()) { - moveUnsealed = SectorFileType::FTNone; - } + const auto moveUnsealed = + need_unsealed ? unsealed : SectorFileType::FTNone; OUTCOME_CB1(scheduler->schedule( sector, @@ -787,9 +787,9 @@ namespace fc::sector_storage { | SectorFileType::FTCache | unsealed), PathType::kSealing, AcquireMode::kMove), - [sector, keep_unsealed]( + [sector, keep_unsealed{std::move(keep_unsealed)}]( const std::shared_ptr &worker) -> outcome::result { - return worker->finalizeSector(sector, keep_unsealed); + return worker->finalizeSector(sector, std::move(keep_unsealed)); }, callbackWrapper(std::move(next_cb)), priority, @@ -922,7 +922,7 @@ namespace fc::sector_storage { void ManagerImpl::addPiece( const SectorRef §or, - gsl::span piece_sizes, + VectorCoW piece_sizes, const UnpaddedPieceSize &new_piece_size, proofs::PieceData piece_data, const std::function)> &cb, @@ -946,15 +946,16 @@ namespace fc::sector_storage { primitives::kTTAddPiece, selector, schedNothing(), - [sector, - piece_sizes, - new_piece_size, - data = std::make_shared(std::move(piece_data)), - lock = std::move(lock)]( - const std::shared_ptr &worker) -> outcome::result { + PutInFunction([sector, + exist_sizes = std::move(piece_sizes), + new_piece_size, + data = std::move(piece_data), + lock = std::move(lock)]( + const std::shared_ptr &worker) mutable + -> outcome::result { return worker->addPiece( - sector, piece_sizes, new_piece_size, std::move(*data)); - }, + sector, std::move(exist_sizes), new_piece_size, std::move(data)); + }), callbackWrapper(cb), priority, boost::none)); @@ -962,7 +963,7 @@ namespace fc::sector_storage { outcome::result ManagerImpl::addPieceSync( const SectorRef §or, - gsl::span piece_sizes, + VectorCoW piece_sizes, const UnpaddedPieceSize &new_piece_size, proofs::PieceData piece_data, uint64_t priority) { @@ -970,7 +971,7 @@ namespace fc::sector_storage { addPiece( sector, - piece_sizes, + std::move(piece_sizes), new_piece_size, std::move(piece_data), [&wait_result](const outcome::result &maybe_pi) -> void { diff --git a/core/sector_storage/impl/manager_impl.hpp b/core/sector_storage/impl/manager_impl.hpp index b98401e75b..4f5b9cf027 100644 --- a/core/sector_storage/impl/manager_impl.hpp +++ b/core/sector_storage/impl/manager_impl.hpp @@ -37,15 +37,14 @@ namespace fc::sector_storage { const std::shared_ptr &proofs = std::make_shared()); - void readPiece( - PieceData output, - const SectorRef §or, - UnpaddedByteIndex offset, - const UnpaddedPieceSize &size, - const SealRandomness &randomness, - const CID &cid, - const std::function)> &cb, - uint64_t priority) override; + void readPiece(PieceData output, + const SectorRef §or, + UnpaddedByteIndex offset, + const UnpaddedPieceSize &size, + const SealRandomness &randomness, + const CID &cid, + const std::function)> &cb, + uint64_t priority) override; void sealPreCommit1( const SectorRef §or, @@ -75,7 +74,7 @@ namespace fc::sector_storage { uint64_t priority) override; void finalizeSector(const SectorRef §or, - const gsl::span &keep_unsealed, + std::vector keep_unsealed, const std::function)> &cb, uint64_t priority) override; @@ -103,7 +102,7 @@ namespace fc::sector_storage { uint64_t priority) override; void addPiece(const SectorRef §or, - gsl::span piece_sizes, + VectorCoW piece_sizes, const UnpaddedPieceSize &new_piece_size, proofs::PieceData piece_data, const std::function)> &cb, @@ -111,7 +110,7 @@ namespace fc::sector_storage { outcome::result addPieceSync( const SectorRef §or, - gsl::span piece_sizes, + VectorCoW piece_sizes, const UnpaddedPieceSize &new_piece_size, proofs::PieceData piece_data, uint64_t priority) override; diff --git a/core/sector_storage/impl/remote_worker.cpp b/core/sector_storage/impl/remote_worker.cpp index a5d0e9063a..4430ca7230 100644 --- a/core/sector_storage/impl/remote_worker.cpp +++ b/core/sector_storage/impl/remote_worker.cpp @@ -136,7 +136,7 @@ namespace fc::sector_storage { outcome::result RemoteWorker::addPiece( const SectorRef §or, - gsl::span piece_sizes, + VectorCoW piece_sizes, const UnpaddedPieceSize &new_piece_size, PieceData piece_data) { MetaPieceData meta_data = @@ -163,11 +163,7 @@ namespace fc::sector_storage { } }); } - return api_.AddPiece( - sector, - std::vector(piece_sizes.begin(), piece_sizes.end()), - new_piece_size, - meta_data); + return api_.AddPiece(sector, piece_sizes.into(), new_piece_size, meta_data); } outcome::result RemoteWorker::sealPreCommit1( @@ -197,9 +193,8 @@ namespace fc::sector_storage { } outcome::result RemoteWorker::finalizeSector( - const SectorRef §or, const gsl::span &keep_unsealed) { - return api_.FinalizeSector( - sector, std::vector(keep_unsealed.begin(), keep_unsealed.end())); + const SectorRef §or, std::vector keep_unsealed) { + return api_.FinalizeSector(sector, keep_unsealed); } outcome::result RemoteWorker::replicaUpdate( diff --git a/core/sector_storage/impl/remote_worker.hpp b/core/sector_storage/impl/remote_worker.hpp index 8cda77388c..c93fe2e6d5 100644 --- a/core/sector_storage/impl/remote_worker.hpp +++ b/core/sector_storage/impl/remote_worker.hpp @@ -24,11 +24,10 @@ namespace fc::sector_storage { const std::shared_ptr &api, const std::string &address); - outcome::result addPiece( - const SectorRef §or, - gsl::span piece_sizes, - const UnpaddedPieceSize &new_piece_size, - PieceData piece_data) override; + outcome::result addPiece(const SectorRef §or, + VectorCoW piece_sizes, + const UnpaddedPieceSize &new_piece_size, + PieceData piece_data) override; outcome::result sealPreCommit1( const SectorRef §or, @@ -49,8 +48,7 @@ namespace fc::sector_storage { const SectorRef §or, const Commit1Output &commit_1_output) override; outcome::result finalizeSector( - const SectorRef §or, - const gsl::span &keep_unsealed) override; + const SectorRef §or, std::vector keep_unsealed) override; outcome::result replicaUpdate( const SectorRef §or, const std::vector &pieces) override; diff --git a/core/sector_storage/manager.hpp b/core/sector_storage/manager.hpp index 742ee51a20..ff2701547c 100644 --- a/core/sector_storage/manager.hpp +++ b/core/sector_storage/manager.hpp @@ -32,15 +32,14 @@ namespace fc::sector_storage { virtual std::shared_ptr getProofEngine() const = 0; - virtual void readPiece( - PieceData output, - const SectorRef §or, - UnpaddedByteIndex offset, - const UnpaddedPieceSize &size, - const SealRandomness &randomness, - const CID &cid, - const std::function)> &cb, - uint64_t priority) = 0; + virtual void readPiece(PieceData output, + const SectorRef §or, + UnpaddedByteIndex offset, + const UnpaddedPieceSize &size, + const SealRandomness &randomness, + const CID &cid, + const std::function)> &cb, + uint64_t priority) = 0; // Sealer virtual void sealPreCommit1( @@ -73,7 +72,7 @@ namespace fc::sector_storage { virtual void finalizeSector( const SectorRef §or, - const gsl::span &keep_unsealed, + std::vector keep_unsealed, const std::function)> &cb, uint64_t priority) = 0; @@ -107,7 +106,7 @@ namespace fc::sector_storage { // Storage virtual void addPiece( const SectorRef §or, - gsl::span piece_sizes, + VectorCoW piece_sizes, const UnpaddedPieceSize &new_piece_size, proofs::PieceData piece_data, const std::function)> &cb, @@ -115,7 +114,7 @@ namespace fc::sector_storage { virtual outcome::result addPieceSync( const SectorRef §or, - gsl::span piece_sizes, + VectorCoW piece_sizes, const UnpaddedPieceSize &new_piece_size, proofs::PieceData piece_data, uint64_t priority) = 0; diff --git a/core/sector_storage/worker.hpp b/core/sector_storage/worker.hpp index f1b005c441..e9d8b00b4b 100644 --- a/core/sector_storage/worker.hpp +++ b/core/sector_storage/worker.hpp @@ -6,6 +6,7 @@ #pragma once #include "common/outcome.hpp" +#include "common/vector_cow.hpp" #include "primitives/piece/piece.hpp" #include "primitives/piece/piece_data.hpp" #include "primitives/seal_tasks/task.hpp" @@ -68,7 +69,7 @@ namespace fc::sector_storage { virtual outcome::result addPiece( const SectorRef §or, - gsl::span piece_sizes, + VectorCoW piece_sizes, const UnpaddedPieceSize &new_piece_size, PieceData piece_data) = 0; @@ -92,8 +93,7 @@ namespace fc::sector_storage { const SectorRef §or, const Commit1Output &commit_1_output) = 0; virtual outcome::result finalizeSector( - const SectorRef §or, - const gsl::span &keep_unsealed) = 0; + const SectorRef §or, std::vector keep_unsealed) = 0; virtual outcome::result replicaUpdate( const SectorRef §or, const std::vector &pieces) = 0; diff --git a/core/storage/piece/impl/piece_storage_impl.cpp b/core/storage/piece/impl/piece_storage_impl.cpp index 6a2b1b8a6e..524d254d8c 100644 --- a/core/storage/piece/impl/piece_storage_impl.cpp +++ b/core/storage/piece/impl/piece_storage_impl.cpp @@ -101,9 +101,11 @@ namespace fc::storage::piece { outcome::result PieceStorageImpl::hasPieceInfo( CID payload_cid, const boost::optional &piece_cid) const { auto piece_info_res = getPieceInfoFromCid(payload_cid, piece_cid); - if (piece_info_res.has_error() - && piece_info_res.error() == PieceStorageError::kPieceNotFound) { - return false; + if (piece_info_res.has_error()) { + if (piece_info_res.error() == PieceStorageError::kPieceNotFound) { + return false; + } + return piece_info_res.error(); } return !piece_info_res.value().deals.empty(); } diff --git a/test/core/miner/sealing_test.cpp b/test/core/miner/sealing_test.cpp index 13350749ea..679676a084 100644 --- a/test/core/miner/sealing_test.cpp +++ b/test/core/miner/sealing_test.cpp @@ -54,6 +54,7 @@ namespace fc::mining { using sector_storage::ManagerMock; using storage::InMemoryStorage; using testing::_; + using testing::IsEmpty; using types::kDealSectorPriority; using types::Piece; using types::SectorInfo; @@ -247,11 +248,8 @@ namespace fc::mining { .proof_type = seal_proof_type_, }; EXPECT_CALL(*manager_, - doAddPieceSync(sector_ref, - gsl::span(), - piece_size, - _, - kDealSectorPriority)) + doAddPieceSync( + sector_ref, IsEmpty(), piece_size, _, kDealSectorPriority)) .WillOnce(testing::Return(info)); EXPECT_OUTCOME_TRUE_1( @@ -372,11 +370,8 @@ namespace fc::mining { SectorRef sector_ref{.id = SectorId{.miner = miner_id_, .sector = sector}, .proof_type = seal_proof_type_}; EXPECT_CALL(*manager_, - doAddPieceSync(sector_ref, - gsl::span(), - piece_size, - _, - kDealSectorPriority)) + doAddPieceSync( + sector_ref, IsEmpty(), piece_size, _, kDealSectorPriority)) .WillOnce(testing::Return(info)); EXPECT_OUTCOME_TRUE( @@ -423,11 +418,8 @@ namespace fc::mining { SectorRef sector_ref{.id = SectorId{.miner = miner_id_, .sector = sector}, .proof_type = seal_proof_type_}; EXPECT_CALL(*manager_, - doAddPieceSync(sector_ref, - gsl::span(), - piece_size, - _, - kDealSectorPriority)) + doAddPieceSync( + sector_ref, IsEmpty(), piece_size, _, kDealSectorPriority)) .WillOnce(testing::Return(info)); EXPECT_OUTCOME_TRUE_1( @@ -472,20 +464,15 @@ namespace fc::mining { SectorRef sector_ref{.id = SectorId{.miner = miner_id_, .sector = sector}, .proof_type = seal_proof_type_}; EXPECT_CALL(*manager_, - doAddPieceSync(sector_ref, - gsl::span(), - piece_size, - _, - kDealSectorPriority)) + doAddPieceSync( + sector_ref, IsEmpty(), piece_size, _, kDealSectorPriority)) .WillOnce(testing::Return(info1)); std::vector exist_pieces({piece_size}); - EXPECT_CALL(*manager_, - doAddPieceSync(sector_ref, - gsl::span(exist_pieces), - piece_size, - _, - kDealSectorPriority)) + EXPECT_CALL( + *manager_, + doAddPieceSync( + sector_ref, exist_pieces, piece_size, _, kDealSectorPriority)) .WillOnce(testing::Return(info2)); EXPECT_OUTCOME_TRUE_1( @@ -532,11 +519,8 @@ namespace fc::mining { SectorRef sector_ref{.id = SectorId{.miner = miner_id_, .sector = sector}, .proof_type = seal_proof_type_}; EXPECT_CALL(*manager_, - doAddPieceSync(sector_ref, - gsl::span(), - piece_size, - _, - kDealSectorPriority)) + doAddPieceSync( + sector_ref, IsEmpty(), piece_size, _, kDealSectorPriority)) .WillOnce(testing::Return(info)); EXPECT_OUTCOME_TRUE_1( @@ -603,11 +587,8 @@ namespace fc::mining { SectorRef sector_ref{.id = SectorId{.miner = miner_id_, .sector = sector}, .proof_type = seal_proof_type_}; EXPECT_CALL(*manager_, - doAddPieceSync(sector_ref, - gsl::span(), - piece_size, - _, - kDealSectorPriority)) + doAddPieceSync( + sector_ref, IsEmpty(), piece_size, _, kDealSectorPriority)) .WillOnce(testing::Return(info)); EXPECT_OUTCOME_TRUE_1( @@ -648,11 +629,8 @@ namespace fc::mining { SectorRef sector_ref{.id = sector_id, .proof_type = seal_proof_type_}; EXPECT_CALL(*manager_, - doAddPieceSync(sector_ref, - gsl::span(), - piece_size, - _, - kDealSectorPriority)) + doAddPieceSync( + sector_ref, IsEmpty(), piece_size, _, kDealSectorPriority)) .WillOnce(testing::Return(info)); EXPECT_OUTCOME_TRUE_1( @@ -913,7 +891,7 @@ namespace fc::mining { std::vector exist_pieces = {}; EXPECT_CALL(*manager_, doAddNullPiece(sector_ref, - gsl::span(exist_pieces), + exist_pieces, PaddedPieceSize(sector_size_).unpadded(), _, 0)) diff --git a/test/core/sector_storage/local_worker_test.cpp b/test/core/sector_storage/local_worker_test.cpp index 6a618e2c09..a6e888e856 100644 --- a/test/core/sector_storage/local_worker_test.cpp +++ b/test/core/sector_storage/local_worker_test.cpp @@ -939,7 +939,7 @@ namespace fc::sector_storage { sector_.id.miner, randomness, unsealed_cid, - primitives::piece::paddedIndex(offset), + offset, uint64_t(piece_size))) .WillOnce( testing::Invoke([&](RegisteredSealProof proof_type, @@ -1129,7 +1129,7 @@ namespace fc::sector_storage { sector_.id.miner, randomness, unsealed_cid, - primitives::piece::paddedIndex(offset), + offset, uint64_t(piece_size))) .WillOnce( testing::Invoke([&](RegisteredSealProof proof_type, @@ -1546,15 +1546,16 @@ namespace fc::sector_storage { * @then WorkerErrors::kOutOfBound occurs */ TEST_F(LocalWorkerTest, AddPieceOutOfBound) { - std::vector pieces = {UnpaddedPieceSize(1016), - UnpaddedPieceSize(1016)}; + VectorCoW pieces( + {UnpaddedPieceSize(1016), UnpaddedPieceSize(1016)}); MOCK_API(return_interface_, ReturnAddPiece); - EXPECT_OUTCOME_TRUE( - call_id, - local_worker_->addPiece( - sector_, pieces, UnpaddedPieceSize(127), PieceData("/dev/null"))); + EXPECT_OUTCOME_TRUE(call_id, + local_worker_->addPiece(sector_, + std::move(pieces), + UnpaddedPieceSize(127), + PieceData("/dev/null"))); CallError error; EXPECT_CALL(mock_ReturnAddPiece, Call(call_id, _, Ne(boost::none))) @@ -1770,7 +1771,10 @@ namespace fc::sector_storage { EXPECT_OUTCOME_TRUE( call_id, local_worker_->addPiece( - sector_, pieces, piece_size, PieceData(input_path))); + sector_, + VectorCoW(gsl::span(pieces)), + piece_size, + PieceData(input_path))); PieceInfo info; EXPECT_CALL(mock_ReturnAddPiece, Call(call_id, _, Eq(boost::none))) diff --git a/test/core/sector_storage/manager_test.cpp b/test/core/sector_storage/manager_test.cpp index 4cd2cb9f75..5df0e5b48b 100644 --- a/test/core/sector_storage/manager_test.cpp +++ b/test/core/sector_storage/manager_test.cpp @@ -401,8 +401,7 @@ namespace fc::sector_storage { outcome::success(std::vector()))); CallId call_id{.sector = sector_.id, .id = "some UUID"}; - EXPECT_CALL(*worker_, - finalizeSector(sector_, gsl::span(keep_unsealed))) + EXPECT_CALL(*worker_, finalizeSector(sector_, keep_unsealed)) .WillOnce(testing::Return(outcome::success(call_id))); EXPECT_CALL(*scheduler_, @@ -692,10 +691,8 @@ namespace fc::sector_storage { }; CallId call_id{.sector = sector_.id, .id = "some UUID"}; - EXPECT_CALL( - *worker_, - doAddPiece( - sector_, gsl::span({}), piece_size, _)) + EXPECT_CALL(*worker_, + doAddPiece(sector_, testing::IsEmpty(), piece_size, _)) .WillOnce(testing::Return(outcome::success(call_id))); EXPECT_CALL(*scheduler_, diff --git a/test/testutil/mocks/sector_storage/manager_mock.hpp b/test/testutil/mocks/sector_storage/manager_mock.hpp index 4f083a9b9a..c0dfcf5e0e 100644 --- a/test/testutil/mocks/sector_storage/manager_mock.hpp +++ b/test/testutil/mocks/sector_storage/manager_mock.hpp @@ -72,7 +72,7 @@ namespace fc::sector_storage { MOCK_METHOD4(finalizeSector, void(const SectorRef §or, - const gsl::span &keep_unsealed, + std::vector keep_unsealed, const std::function)> &, uint64_t priority)); @@ -107,7 +107,7 @@ namespace fc::sector_storage { MOCK_METHOD6(doAddPiece, void(const SectorRef §or, - gsl::span piece_sizes, + const std::vector &piece_sizes, const UnpaddedPieceSize &new_piece_size, int, const std::function)> &, @@ -115,24 +115,24 @@ namespace fc::sector_storage { MOCK_METHOD5(doAddNullPiece, void(const SectorRef §or, - gsl::span piece_sizes, + const std::vector &piece_sizes, const UnpaddedPieceSize &new_piece_size, const std::function)> &, uint64_t priority)); void addPiece(const SectorRef §or, - gsl::span piece_sizes, + VectorCoW piece_sizes, const UnpaddedPieceSize &new_piece_size, proofs::PieceData piece_data, const std::function)> &cb, uint64_t priority) override { if (piece_data.isNullData()) { return doAddNullPiece( - sector, piece_sizes, new_piece_size, cb, priority); + sector, piece_sizes.mut(), new_piece_size, cb, priority); } return doAddPiece(sector, - piece_sizes, + piece_sizes.mut(), new_piece_size, piece_data.getFd(), cb, @@ -142,7 +142,7 @@ namespace fc::sector_storage { MOCK_METHOD5(doAddPieceSync, outcome::result( const SectorRef §or, - gsl::span piece_sizes, + const std::vector &piece_sizes, const UnpaddedPieceSize &new_piece_size, int, uint64_t priority)); @@ -150,22 +150,25 @@ namespace fc::sector_storage { MOCK_METHOD4(doAddNullPieceSync, outcome::result( const SectorRef §or, - gsl::span piece_sizes, + const std::vector &piece_sizes, const UnpaddedPieceSize &new_piece_size, uint64_t priority)); outcome::result addPieceSync( const SectorRef §or, - gsl::span piece_sizes, + VectorCoW piece_sizes, const UnpaddedPieceSize &new_piece_size, proofs::PieceData piece_data, uint64_t priority) override { if (piece_data.isNullData()) { return doAddNullPieceSync( - sector, piece_sizes, new_piece_size, priority); + sector, piece_sizes.mut(), new_piece_size, priority); } - return doAddPieceSync( - sector, piece_sizes, new_piece_size, piece_data.getFd(), priority); + return doAddPieceSync(sector, + piece_sizes.mut(), + new_piece_size, + piece_data.getFd(), + priority); } MOCK_METHOD3(generateWinningPoSt, diff --git a/test/testutil/mocks/sector_storage/worker_mock.hpp b/test/testutil/mocks/sector_storage/worker_mock.hpp index c5af9a565a..05da3eda38 100644 --- a/test/testutil/mocks/sector_storage/worker_mock.hpp +++ b/test/testutil/mocks/sector_storage/worker_mock.hpp @@ -85,20 +85,19 @@ namespace fc::sector_storage { MOCK_METHOD2(finalizeSector, outcome::result(const SectorRef &, - const gsl::span &)); + std::vector)); - outcome::result addPiece( - const SectorRef §or, - gsl::span piece_sizes, - const UnpaddedPieceSize &new_piece_size, - PieceData piece_data) override { + outcome::result addPiece(const SectorRef §or, + VectorCoW piece_sizes, + const UnpaddedPieceSize &new_piece_size, + PieceData piece_data) override { return doAddPiece( - sector, piece_sizes, new_piece_size, piece_data.getFd()); + sector, piece_sizes.mut(), new_piece_size, piece_data.getFd()); } MOCK_METHOD4(doAddPiece, outcome::result(const SectorRef &, - gsl::span, + const std::vector &, const UnpaddedPieceSize &, int)); };