Skip to content

Straighten BSC <-> NW interoperation protocol #2696

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Mar 13, 2024
Merged

Straighten BSC <-> NW interoperation protocol #2696

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 29 additions & 58 deletions ydb/core/base/blobstorage.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -786,64 +786,35 @@ struct TEvBlobStorage {
EvBunchOfEvents,

// blobstorage controller interface
// EvControllerReadSchemeString = EvPut + 11 * 512,
// EvControllerReadDataString,
EvControllerRegisterNode = EvPut + 11 * 512 + 2,
EvControllerCreatePDisk,
EvControllerCreateVDiskSlots,
EvControllerCreateGroup,
EvControllerSelectGroups,
EvControllerGetGroup,
EvControllerUpdateDiskStatus,
EvControllerUpdateGroupsUsage, // Not used.
EvControllerConfigRequest,
EvControllerConfigResponse,
EvControllerProposeRequest,
EvControllerProposeResponse,
EvControllerVDiskStatusSubscribeRequest,
EvControllerVDiskStatusReport,
EvControllerGroupStatusRequest,
EvControllerGroupStatusResponse,
EvControllerUpdateGroup,
EvControllerUpdateFaultyDisks,
EvControllerProposeGroupKey,
EvControllerUpdateGroupLatencies, // Not used.
EvControllerUpdateGroupStat,
EvControllerNotifyGroupChange,
EvControllerCommitGroupLatencies,
EvControllerUpdateSelfHealInfo,
EvControllerScrubQueryStartQuantum,
EvControllerScrubQuantumFinished,
EvControllerScrubReportQuantumInProgress,
EvControllerUpdateNodeDrives,
EvControllerGroupDecommittedNotify,
EvControllerGroupDecommittedResponse,
EvControllerGroupMetricsExchange,

// EvControllerReadSchemeStringResult = EvPut + 12 * 512,
// EvControllerReadDataStringResult,
EvControllerNodeServiceSetUpdate = EvPut + 12 * 512 + 2,
EvControllerCreatePDiskResult,
EvControllerCreateVDiskSlotsResult,
EvControllerCreateGroupResult,
EvControllerSelectGroupsResult,
EvRequestControllerInfo,
EvResponseControllerInfo,
EvControllerGroupReconfigureReplace, // Not used.
EvControllerGroupReconfigureReplaceResult, // Not used.
EvControllerGroupReconfigureWipe,
EvControllerGroupReconfigureWipeResult,
EvControllerNodeReport,
EvControllerScrubStartQuantum,

EvControllerMigrationPause,
EvControllerMigrationContinue,
EvControllerMigrationFinished,
EvControllerMigrationBatch,
EvControllerMigrationBatchRequest,
EvControllerMigrationDone,

EvControllerUpdateSystemViews,
EvControllerRegisterNode = 0x10031602,
EvControllerSelectGroups = 0x10031606,
EvControllerGetGroup = 0x10031607,
EvControllerUpdateDiskStatus = 0x10031608,
EvControllerConfigRequest = 0x1003160a,
EvControllerConfigResponse = 0x1003160b,
EvControllerProposeGroupKey = 0x10031614,
EvControllerUpdateGroupStat = 0x10031616,
EvControllerNotifyGroupChange = 0x10031617,
EvControllerCommitGroupLatencies = 0x10031618,
EvControllerUpdateSelfHealInfo = 0x10031619,
EvControllerScrubQueryStartQuantum = 0x1003161a,
EvControllerScrubQuantumFinished = 0x1003161b,
EvControllerScrubReportQuantumInProgress = 0x1003161c,
EvControllerUpdateNodeDrives = 0x1003161d,
EvControllerGroupDecommittedNotify = 0x1003161e,
EvControllerGroupDecommittedResponse = 0x1003161f,
EvControllerGroupMetricsExchange = 0x10031620,

// BSC interface result section
EvControllerNodeServiceSetUpdate = 0x10031802,
EvControllerSelectGroupsResult = 0x10031806,
EvRequestControllerInfo = 0x10031807,
EvResponseControllerInfo = 0x10031808,
EvControllerGroupReconfigureWipe = 0x1003180b,
EvControllerGroupReconfigureWipeResult = 0x1003180c,
EvControllerNodeReport = 0x1003180d,
EvControllerScrubStartQuantum = 0x1003180e,
EvControllerUpdateSystemViews = 0x10031815,

// proxy - node controller interface
EvConfigureProxy = EvPut + 13 * 512,
Expand Down
6 changes: 4 additions & 2 deletions ydb/core/blobstorage/ut_blobstorage/balancing.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -303,9 +303,11 @@ struct TRandomTest {
ui32 pos = random() % Env->Settings.NodeCount;
if (Env.RunningNodes.contains(pos)) {
auto baseConfig = Env->FetchBaseConfig();
const auto& somePDisk = baseConfig.GetPDisk(pos);
const auto& someVSlot = baseConfig.GetVSlot(pos);
Env->Wipe(somePDisk.GetNodeId(), somePDisk.GetPDiskId(), someVSlot.GetVSlotId().GetVSlotId());
const auto& loc = someVSlot.GetVSlotId();
Env->Wipe(loc.GetNodeId(), loc.GetPDiskId(), loc.GetVSlotId(),
TVDiskID(someVSlot.GetGroupId(), someVSlot.GetGroupGeneration(), someVSlot.GetFailRealmIdx(),
someVSlot.GetFailDomainIdx(), someVSlot.GetVDiskIdx()));
Env->Sim(TDuration::Seconds(10));
}
}
Expand Down
2 changes: 1 addition & 1 deletion ydb/core/blobstorage/ut_blobstorage/group_reconfiguration.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -331,7 +331,7 @@ Y_UNIT_TEST_SUITE(GroupReconfiguration) {
Timer.Reset();
auto ev = std::make_unique<TEvBlobStorage::TEvControllerGetGroup>();
ev->Record.AddGroupIDs(GroupId);
NTabletPipe::SendData(SelfId(), ClientId, ev.release(), 0);
NTabletPipe::SendData(SelfId(), ClientId, ev.release(), Max<ui64>());
}
}

Expand Down
19 changes: 11 additions & 8 deletions ydb/core/blobstorage/ut_blobstorage/lib/env.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -764,17 +764,20 @@ struct TEnvironmentSetup {
Sim(TDuration::Seconds(15));
}

void Wipe(ui32 nodeId, ui32 pdiskId, ui32 vslotId) {
const TActorId self = Runtime->AllocateEdgeActor(Settings.ControllerNodeId, __FILE__, __LINE__);
auto ev = std::make_unique<TEvBlobStorage::TEvControllerGroupReconfigureWipe>();
auto& record = ev->Record;
auto *vslot = record.MutableVSlotId();
void Wipe(ui32 nodeId, ui32 pdiskId, ui32 vslotId, const TVDiskID& vdiskId) {
NKikimrBlobStorage::TConfigRequest request;
request.SetIgnoreGroupFailModelChecks(true);
request.SetIgnoreDegradedGroupsChecks(true);
request.SetIgnoreDisintegratedGroupsChecks(true);
auto *cmd = request.AddCommand();
auto *wipe = cmd->MutableWipeVDisk();
auto *vslot = wipe->MutableVSlotId();
vslot->SetNodeId(nodeId);
vslot->SetPDiskId(pdiskId);
vslot->SetVSlotId(vslotId);
Runtime->SendToPipe(TabletId, self, ev.release(), 0, TTestActorSystem::GetPipeConfigWithRetries());
auto response = WaitForEdgeActorEvent<TEvBlobStorage::TEvControllerGroupReconfigureWipeResult>(self);
UNIT_ASSERT_VALUES_EQUAL(response->Get()->Record.GetStatus(), NKikimrProto::OK);
VDiskIDFromVDiskID(vdiskId, wipe->MutableVDiskId());
auto response = Invoke(request);
UNIT_ASSERT_C(response.GetSuccess(), response.GetErrorDescription());
}

void WaitForVDiskToGetRunning(const TVDiskID& vdiskId, TActorId actorId) {
Expand Down
20 changes: 2 additions & 18 deletions ydb/core/blobstorage/ut_blobstorage/mirror3of4.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,26 +65,10 @@ Y_UNIT_TEST_SUITE(Mirror3of4) {
UNIT_ASSERT_VALUES_EQUAL(res->Get()->Status, NKikimrProto::OK);
}
if (i == 500) {
const TActorId self = runtime->AllocateEdgeActor(1);
auto ev = std::make_unique<TEvBlobStorage::TEvControllerGroupReconfigureWipe>();
ev->Record.MutableVSlotId()->SetNodeId(2);
ev->Record.MutableVSlotId()->SetPDiskId(1000);
ev->Record.MutableVSlotId()->SetVSlotId(1000);
runtime->SendToPipe(env.TabletId, self, ev.release(), 0, TTestActorSystem::GetPipeConfigWithRetries());
auto response = env.WaitForEdgeActorEvent<TEvBlobStorage::TEvControllerGroupReconfigureWipeResult>(self);
auto& r = response->Get()->Record;
UNIT_ASSERT_EQUAL(r.GetStatus(), NKikimrProto::OK);
env.Wipe(2, 1000, 1000, TVDiskID(groupId, 1, 0, 1, 0));
}
if (i == 600) {
const TActorId self = runtime->AllocateEdgeActor(1);
auto ev = std::make_unique<TEvBlobStorage::TEvControllerGroupReconfigureWipe>();
ev->Record.MutableVSlotId()->SetNodeId(3);
ev->Record.MutableVSlotId()->SetPDiskId(1000);
ev->Record.MutableVSlotId()->SetVSlotId(1000);
runtime->SendToPipe(env.TabletId, self, ev.release(), 0, TTestActorSystem::GetPipeConfigWithRetries());
auto response = env.WaitForEdgeActorEvent<TEvBlobStorage::TEvControllerGroupReconfigureWipeResult>(self);
auto& r = response->Get()->Record;
UNIT_ASSERT_EQUAL(r.GetStatus(), NKikimrProto::OK);
env.Wipe(3, 1000, 1000, TVDiskID(groupId, 1, 0, 2, 0));
}
}

Expand Down
11 changes: 2 additions & 9 deletions ydb/core/blobstorage/ut_blobstorage/osiris.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,15 +62,8 @@ bool DoTestCase(TBlobStorageGroupType::EErasureSpecies erasure, const std::set<s
for (const auto& vslot : response.GetStatus(0).GetBaseConfig().GetVSlot()) {
const TVDiskID vdiskId(vslot.GetGroupId(), vslot.GetGroupGeneration(), vslot.GetFailRealmIdx(),
vslot.GetFailDomainIdx(), vslot.GetVDiskIdx());

const TActorId sender = env.Runtime->AllocateEdgeActor(1);
auto ev = std::make_unique<TEvBlobStorage::TEvControllerGroupReconfigureWipe>();
auto *slotId = ev->Record.MutableVSlotId();
slotId->CopyFrom(vslot.GetVSlotId());
env.Runtime->SendToPipe(env.TabletId, sender, ev.release(), 0, TTestActorSystem::GetPipeConfigWithRetries());
auto response = env.WaitForEdgeActorEvent<TEvBlobStorage::TEvControllerGroupReconfigureWipeResult>(sender);
Y_ABORT_UNLESS(response->Get()->Record.GetStatus() == NKikimrProto::OK);

const auto& v = vslot.GetVSlotId();
env.Wipe(v.GetNodeId(), v.GetPDiskId(), v.GetVSlotId(), vdiskId);
env.Sim(TDuration::Seconds(30));
}

Expand Down
6 changes: 0 additions & 6 deletions ydb/core/blobstorage/vdisk/common/blobstorage_event_filter.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,12 +54,6 @@ namespace NKikimr {
TEvBlobStorage::EvControllerUpdateDiskStatus,
TEvBlobStorage::EvControllerConfigRequest,
TEvBlobStorage::EvControllerConfigResponse,
TEvBlobStorage::EvControllerVDiskStatusSubscribeRequest,
TEvBlobStorage::EvControllerVDiskStatusReport,
TEvBlobStorage::EvControllerGroupStatusRequest,
TEvBlobStorage::EvControllerGroupStatusResponse,
TEvBlobStorage::EvControllerUpdateGroup,
TEvBlobStorage::EvControllerUpdateFaultyDisks,
TEvBlobStorage::EvControllerUpdateGroupStat,

TEvBlobStorage::EvControllerSelectGroupsResult,
Expand Down
22 changes: 12 additions & 10 deletions ydb/core/mind/bscontroller/config.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,22 +17,21 @@ namespace NKikimr::NBsController {
, State(state)
{}

void Execute(std::deque<std::unique_ptr<IEventHandle>>& outbox) {
void Execute() {
ApplyUpdates();

for (auto &pair : Services) {
const TNodeId &nodeId = pair.first;

if (TNodeInfo *node = Self->FindNode(nodeId); node && node->ConnectedCount) {
auto event = MakeHolder<TEvBlobStorage::TEvControllerNodeServiceSetUpdate>();
if (TNodeInfo *node = Self->FindNode(nodeId); node && node->ConnectedServerId) {
auto event = std::make_unique<TEvBlobStorage::TEvControllerNodeServiceSetUpdate>();
auto& record = event->Record;
pair.second.Swap(&record);
record.SetStatus(NKikimrProto::OK);
record.SetNodeID(nodeId);
record.SetInstanceId(Self->InstanceId);
record.SetAvailDomain(AppData()->DomainsInfo->GetDomain()->DomainUid);
outbox.push_back(std::make_unique<IEventHandle>(MakeBlobStorageNodeWardenID(nodeId),
Self->SelfId(), event.Release()));
State.Outbox.emplace_back(nodeId, std::move(event), 0);
}
}
}
Expand Down Expand Up @@ -448,7 +447,7 @@ namespace NKikimr::NBsController {
}
}

TNodeWardenUpdateNotifier(this, state).Execute(state.Outbox);
TNodeWardenUpdateNotifier(this, state).Execute();

state.CheckConsistency();
state.Commit();
Expand All @@ -460,7 +459,7 @@ namespace NKikimr::NBsController {
}

void TBlobStorageController::CommitSelfHealUpdates(TConfigState& state) {
auto ev = MakeHolder<TEvControllerNotifyGroupChange>();
auto ev = std::make_unique<TEvControllerNotifyGroupChange>();
auto sh = MakeHolder<TEvControllerUpdateSelfHealInfo>();

for (auto&& [base, overlay] : state.Groups.Diff()) {
Expand Down Expand Up @@ -500,7 +499,7 @@ namespace NKikimr::NBsController {
}

if (ev->Created || ev->Deleted) {
state.Outbox.push_back(std::make_unique<IEventHandle>(StatProcessorActorId, SelfId(), ev.Release()));
state.StatProcessorOutbox.push_back(std::move(ev));
}
if (sh->GroupsToUpdate) {
FillInSelfHealGroups(*sh, &state);
Expand Down Expand Up @@ -590,8 +589,11 @@ namespace NKikimr::NBsController {
}

ui64 TBlobStorageController::TConfigState::ApplyConfigUpdates() {
for (auto& msg : Outbox) {
TActivationContext::Send(msg.release());
for (auto& [nodeId, ev, cookie] : Outbox) {
Self.SendToWarden(nodeId, std::move(ev), cookie);
}
for (auto& ev : StatProcessorOutbox) {
Self.SelfId().Send(Self.StatProcessorActorId, ev.release());
}

if (UpdateSelfHealInfoMsg) {
Expand Down
3 changes: 2 additions & 1 deletion ydb/core/mind/bscontroller/config.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,8 @@ namespace NKikimr {
THashSet<TPDiskId> PDisksToRemove;

// outgoing messages
std::deque<std::unique_ptr<IEventHandle>> Outbox;
std::deque<std::tuple<TNodeId, std::unique_ptr<IEventBase>, ui64>> Outbox;
std::deque<std::unique_ptr<IEventBase>> StatProcessorOutbox;
THolder<TEvControllerUpdateSelfHealInfo> UpdateSelfHealInfoMsg;

// deferred callbacks
Expand Down
28 changes: 18 additions & 10 deletions ydb/core/mind/bscontroller/get_group.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@ namespace NKikimr::NBsController {

class TBlobStorageController::TTxGetGroup : public TTransactionBase<TBlobStorageController> {
TEvBlobStorage::TEvControllerGetGroup::TPtr Request;
std::unique_ptr<IEventHandle> Response;
std::unique_ptr<TEvBlobStorage::TEvControllerNodeServiceSetUpdate> Response;
TNodeId NodeId = {};

public:
TTxGetGroup(TEvBlobStorage::TEvControllerGetGroup::TPtr& ev, TBlobStorageController *controller)
Expand All @@ -20,27 +21,34 @@ class TBlobStorageController::TTxGetGroup : public TTransactionBase<TBlobStorage

STLOG(PRI_DEBUG, BS_CONTROLLER, BSCTXGG01, "Handle TEvControllerGetGroup", (Request, Request->Get()->Record));

NodeId = Request->Get()->Record.GetNodeID();

if (Request->Cookie != Max<ui64>() && !Self->ValidateIncomingNodeWardenEvent(*Request)) {
Response = std::make_unique<TEvBlobStorage::TEvControllerNodeServiceSetUpdate>(NKikimrProto::ERROR, NodeId);
return true;
}

const auto& v = Request->Get()->Record.GetGroupIDs();
TSet<ui32> groupIDsToRead(v.begin(), v.end());

const TNodeId nodeId = Request->Get()->Record.GetNodeID();
auto res = std::make_unique<TEvBlobStorage::TEvControllerNodeServiceSetUpdate>(NKikimrProto::OK, nodeId);
Self->ReadGroups(groupIDsToRead, true, res.get(), nodeId);
Response = std::make_unique<TEvBlobStorage::TEvControllerNodeServiceSetUpdate>(NKikimrProto::OK, NodeId);
Self->ReadGroups(groupIDsToRead, true, Response.get(), NodeId);

auto& node = Self->GetNode(nodeId);
auto& node = Self->GetNode(NodeId);
for (TGroupId groupId : v) {
node.GroupsRequested.insert(groupId);
Self->GroupToNode.emplace(groupId, nodeId);
Self->GroupToNode.emplace(groupId, NodeId);
}

Response = std::make_unique<IEventHandle>(nodeId ? MakeBlobStorageNodeWardenID(nodeId) : Request->Sender,
Self->SelfId(), res.release());

return true;
}

void Complete(const TActorContext&) override {
TActivationContext::Send(Response.release());
if (NodeId) {
Self->SendToWarden(NodeId, std::move(Response), Request->Cookie);
} else {
Self->SendInReply(*Request, std::move(Response));
}
}
};

Expand Down
Loading