Skip to content

Commit 7da70ef

Browse files
PhyronnazPhyronnaz
committed
Build 54
🐛 Fix OSX packaging 🐛 Switch voxel save to use 64 bits arrays
1 parent c38b466 commit 7da70ef

File tree

9 files changed

+444
-389
lines changed

9 files changed

+444
-389
lines changed

‎Source/Voxel/Private/VoxelData/VoxelSave.cpp‎

Lines changed: 167 additions & 121 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
// Copyright 2020 Phyronnaz
22

33
#include "VoxelData/VoxelSave.h"
4-
#include "VoxelUtilities/VoxelSerializationUtilities.h"
4+
#include "VoxelUtilities/VoxelSerializationUtilities.inl"
55
#include "VoxelUtilities/VoxelMathUtilities.h"
66
#include "VoxelMessages.h"
7+
#include "Misc/ScopeExit.h"
78

89
DEFINE_VOXEL_MEMORY_STAT(STAT_VoxelUncompressedSavesMemory);
910
DEFINE_VOXEL_MEMORY_STAT(STAT_VoxelCompressedSavesMemory);
@@ -67,10 +68,10 @@ void FVoxelUncompressedWorldSaveImpl::UpdateAllocatedSize() const
6768
{
6869
DEC_VOXEL_MEMORY_STAT_BY(STAT_VoxelUncompressedSavesMemory, AllocatedSize);
6970
AllocatedSize =
70-
Chunks.GetAllocatedSize() +
71-
ValueBuffers.GetAllocatedSize() +
72-
MaterialBuffers.GetAllocatedSize() +
73-
PlaceableItems.GetAllocatedSize();
71+
Chunks64.GetAllocatedSize() +
72+
ValueBuffers64.GetAllocatedSize() +
73+
MaterialBuffers64.GetAllocatedSize() +
74+
PlaceableItems64.GetAllocatedSize();
7475
INC_VOXEL_MEMORY_STAT_BY(STAT_VoxelUncompressedSavesMemory, AllocatedSize);
7576
}
7677

@@ -139,179 +140,224 @@ bool FVoxelUncompressedWorldSaveImpl::Serialize(FArchive& Ar)
139140
uint32 MaterialConfigFlag = GVoxelMaterialConfigFlag;
140141
Ar << MaterialConfigFlag;
141142

142-
// Serialize buffers
143-
if (Version >= FVoxelSaveVersion::StoreMaterialChannelsIndividuallyAndRemoveFoliage)
143+
if (Version >= FVoxelSaveVersion::Use64BitArrays)
144144
{
145145
// Serialize value buffers
146-
FVoxelSerializationUtilities::SerializeValues(Ar, ValueBuffers, ValueConfigFlag, SerializationVersion);
147-
FVoxelSerializationUtilities::SerializeValues(Ar, SingleValues, ValueConfigFlag, SerializationVersion);
146+
FVoxelSerializationUtilities::SerializeValues(Ar, ValueBuffers64, ValueConfigFlag, SerializationVersion);
147+
FVoxelSerializationUtilities::SerializeValues(Ar, SingleValues64, ValueConfigFlag, SerializationVersion);
148148

149149
// Serialize material buffers
150-
FVoxelSerializationUtilities::SerializeMaterials(Ar, MaterialsIndices, MaterialConfigFlag);
151-
MaterialBuffers.BulkSerialize(Ar);
152-
SingleMaterials.BulkSerialize(Ar);
150+
FVoxelSerializationUtilities::SerializeMaterials(Ar, MaterialsIndices64, MaterialConfigFlag);
151+
MaterialBuffers64.BulkSerialize(Ar);
152+
SingleMaterials64.BulkSerialize(Ar);
153153

154154
// Serialize chunks indices
155155
// Note: make sure to not use BulkSerialize as data isn't aligned
156-
Ar << Chunks;
156+
Ar << Chunks64;
157+
158+
// Serialize placeable items
159+
Ar << PlaceableItems64;
157160
}
158161
else
159162
{
160-
TNoGrowArray<FVoxelMaterial> OldMaterialBuffers;
161-
TNoGrowArray<FVoxelMaterial> OldSingleMaterials;
162-
163-
// Serialize value buffers
164-
FVoxelSerializationUtilities::SerializeValues(Ar, ValueBuffers, ValueConfigFlag, SerializationVersion);
163+
TNoGrowArray<FVoxelValue> ValueBuffers;
164+
TNoGrowArray<FVoxelValue> SingleValues;
165165

166-
// Serialize material buffers
167-
FVoxelSerializationUtilities::SerializeMaterials(Ar, OldMaterialBuffers, MaterialConfigFlag, SerializationVersion);
166+
TNoGrowArray<TVoxelMaterialStorage<uint32>> MaterialsIndices;
167+
TNoGrowArray<uint8> MaterialBuffers;
168+
TNoGrowArray<uint8> SingleMaterials;
169+
170+
TNoGrowArray<FVoxelChunkSave> Chunks;
171+
172+
TArray<uint8> PlaceableItems;
168173

169-
// Serialize foliage buffers
170-
if (Version >= FVoxelSaveVersion::FoliagePaint)
174+
ON_SCOPE_EXIT
171175
{
172-
TArray<FVoxelFoliage> FoliageBuffers;
173-
FoliageBuffers.BulkSerialize(Ar);
174-
}
176+
ValueBuffers64 = MoveTemp(ValueBuffers);
177+
SingleValues64 = MoveTemp(SingleValues);
178+
179+
MaterialsIndices64 = MoveTemp(MaterialsIndices);
180+
MaterialBuffers64 = MoveTemp(MaterialBuffers);
181+
SingleMaterials64 = MoveTemp(SingleMaterials);
182+
183+
Chunks64 = MoveTemp(Chunks);
184+
185+
PlaceableItems64 = MoveTemp(PlaceableItems);
186+
};
175187

176-
// Serialize single values buffers
177-
if (Version >= FVoxelSaveVersion::SingleValues)
188+
if (Version >= FVoxelSaveVersion::StoreMaterialChannelsIndividuallyAndRemoveFoliage)
178189
{
190+
// Serialize value buffers
191+
FVoxelSerializationUtilities::SerializeValues(Ar, ValueBuffers, ValueConfigFlag, SerializationVersion);
179192
FVoxelSerializationUtilities::SerializeValues(Ar, SingleValues, ValueConfigFlag, SerializationVersion);
180-
FVoxelSerializationUtilities::SerializeMaterials(Ar, OldSingleMaterials, MaterialConfigFlag, SerializationVersion);
181193

182-
TArray<FVoxelFoliage> SingleFoliage;
183-
SingleFoliage.BulkSerialize(Ar);
184-
}
194+
// Serialize material buffers
195+
FVoxelSerializationUtilities::SerializeMaterials(Ar, MaterialsIndices, MaterialConfigFlag);
196+
MaterialBuffers.BulkSerialize(Ar);
197+
SingleMaterials.BulkSerialize(Ar);
185198

186-
// Serialize chunks indices
187-
struct FVoxelChunkSaveWithSingleMaterial
199+
// Serialize chunks indices
200+
// Note: make sure to not use BulkSerialize as data isn't aligned
201+
Ar << Chunks;
202+
}
203+
else
188204
{
189-
FIntVector Position;
205+
TNoGrowArray<FVoxelMaterial> OldMaterialBuffers;
206+
TNoGrowArray<FVoxelMaterial> OldSingleMaterials;
190207

191-
int32 ValuesIndex = -1;
192-
int32 MaterialsIndex = -1;
208+
// Serialize value buffers
209+
FVoxelSerializationUtilities::SerializeValues(Ar, ValueBuffers, ValueConfigFlag, SerializationVersion);
193210

194-
bool bSingleValue = false;
195-
// Makes life easier when loading legacy files
196-
bool bSingleMaterial_Unused = false;
197-
};
198-
TNoGrowArray<FVoxelChunkSaveWithSingleMaterial> NewChunks;
199-
{
200-
TArray<FVoxelChunkSave32Bits> OldChunks;
201-
if (Version < FVoxelSaveVersion::FoliagePaint)
211+
// Serialize material buffers
212+
FVoxelSerializationUtilities::SerializeMaterials(Ar, OldMaterialBuffers, MaterialConfigFlag, SerializationVersion);
213+
214+
// Serialize foliage buffers
215+
if (Version >= FVoxelSaveVersion::FoliagePaint)
202216
{
203-
TArray<FVoxelChunkSaveWithoutFoliage> ChunksWithoutFoliage;
204-
if (Version == FVoxelSaveVersion::BeforeCustomVersionWasAdded)
205-
{
206-
Ar << ChunksWithoutFoliage;
207-
}
208-
else
209-
{
210-
ChunksWithoutFoliage.BulkSerialize(Ar);
211-
}
212-
OldChunks = TArray<FVoxelChunkSave32Bits>(ChunksWithoutFoliage);
217+
TArray<FVoxelFoliage> FoliageBuffers;
218+
FoliageBuffers.BulkSerialize(Ar);
213219
}
214-
else
220+
221+
// Serialize single values buffers
222+
if (Version >= FVoxelSaveVersion::SingleValues)
215223
{
216-
OldChunks.BulkSerialize(Ar);
224+
FVoxelSerializationUtilities::SerializeValues(Ar, SingleValues, ValueConfigFlag, SerializationVersion);
225+
FVoxelSerializationUtilities::SerializeMaterials(Ar, OldSingleMaterials, MaterialConfigFlag, SerializationVersion);
226+
227+
TArray<FVoxelFoliage> SingleFoliage;
228+
SingleFoliage.BulkSerialize(Ar);
217229
}
218230

219-
NewChunks.Empty(OldChunks.Num());
220-
for (auto& OldChunk : OldChunks)
231+
// Serialize chunks indices
232+
struct FVoxelChunkSaveWithSingleMaterial
221233
{
222-
constexpr int32 SingleValueIndexFlag = 1 << 30;
234+
FIntVector Position;
223235

224-
FVoxelChunkSaveWithSingleMaterial& NewChunk = NewChunks.Emplace_GetRef();
236+
int32 ValuesIndex = -1;
237+
int32 MaterialsIndex = -1;
225238

226-
NewChunk.Position = OldChunk.Position;
227-
228-
if (OldChunk.ValuesIndex != -1)
239+
bool bSingleValue = false;
240+
// Makes life easier when loading legacy files
241+
bool bSingleMaterial_Unused = false;
242+
};
243+
TNoGrowArray<FVoxelChunkSaveWithSingleMaterial> NewChunks;
244+
{
245+
TArray<FVoxelChunkSave32Bits> OldChunks;
246+
if (Version < FVoxelSaveVersion::FoliagePaint)
229247
{
230-
NewChunk.ValuesIndex = OldChunk.ValuesIndex & (~SingleValueIndexFlag);
231-
NewChunk.bSingleValue = OldChunk.ValuesIndex & SingleValueIndexFlag;
248+
TArray<FVoxelChunkSaveWithoutFoliage> ChunksWithoutFoliage;
249+
if (Version == FVoxelSaveVersion::BeforeCustomVersionWasAdded)
250+
{
251+
Ar << ChunksWithoutFoliage;
252+
}
253+
else
254+
{
255+
ChunksWithoutFoliage.BulkSerialize(Ar);
256+
}
257+
OldChunks = TArray<FVoxelChunkSave32Bits>(ChunksWithoutFoliage);
232258
}
233-
if (OldChunk.MaterialsIndex != -1)
259+
else
234260
{
235-
NewChunk.MaterialsIndex = OldChunk.MaterialsIndex & (~SingleValueIndexFlag);
236-
NewChunk.bSingleMaterial_Unused = OldChunk.MaterialsIndex & SingleValueIndexFlag;
261+
OldChunks.BulkSerialize(Ar);
237262
}
238-
}
239-
ensure(NewChunks.GetSlack() == 0);
240-
}
241263

242-
// Fixup material indices, as they are now referencing the MaterialsIndices array and not MaterialBuffers/SingleMaterials
243-
{
244-
check(OldMaterialBuffers.Num() % VOXELS_PER_DATA_CHUNK == 0);
264+
NewChunks.Empty(OldChunks.Num());
265+
for (auto& OldChunk : OldChunks)
266+
{
267+
constexpr int32 SingleValueIndexFlag = 1 << 30;
245268

246-
MaterialsIndices.Empty(OldMaterialBuffers.Num() / VOXELS_PER_DATA_CHUNK + OldSingleMaterials.Num());
247-
MaterialBuffers.Empty(OldMaterialBuffers.Num() * FVoxelMaterial::NumChannels);
248-
SingleMaterials.Empty(OldSingleMaterials.Num() * FVoxelMaterial::NumChannels);
269+
FVoxelChunkSaveWithSingleMaterial& NewChunk = NewChunks.Emplace_GetRef();
249270

250-
Chunks.Empty(NewChunks.Num());
271+
NewChunk.Position = OldChunk.Position;
251272

252-
// Fixup chunks
253-
for (auto& Chunk : NewChunks)
254-
{
255-
if (Chunk.MaterialsIndex != -1)
256-
{
257-
if (Chunk.bSingleMaterial_Unused)
273+
if (OldChunk.ValuesIndex != -1)
274+
{
275+
NewChunk.ValuesIndex = OldChunk.ValuesIndex & (~SingleValueIndexFlag);
276+
NewChunk.bSingleValue = OldChunk.ValuesIndex & SingleValueIndexFlag;
277+
}
278+
if (OldChunk.MaterialsIndex != -1)
258279
{
259-
const int32 OldIndex = Chunk.MaterialsIndex;
280+
NewChunk.MaterialsIndex = OldChunk.MaterialsIndex & (~SingleValueIndexFlag);
281+
NewChunk.bSingleMaterial_Unused = OldChunk.MaterialsIndex & SingleValueIndexFlag;
282+
}
283+
}
284+
ensureVoxelSlow(NewChunks.GetSlack() == 0);
285+
}
286+
287+
// Fixup material indices, as they are now referencing the MaterialsIndices array and not MaterialBuffers/SingleMaterials
288+
{
289+
check(OldMaterialBuffers.Num() % VOXELS_PER_DATA_CHUNK == 0);
260290

261-
Chunk.MaterialsIndex = MaterialsIndices.AddUninitialized(1);
262-
auto& MaterialIndices = MaterialsIndices[Chunk.MaterialsIndex];
291+
MaterialsIndices.Empty(OldMaterialBuffers.Num() / VOXELS_PER_DATA_CHUNK + OldSingleMaterials.Num());
292+
MaterialBuffers.Empty(OldMaterialBuffers.Num() * FVoxelMaterial::NumChannels);
293+
SingleMaterials.Empty(OldSingleMaterials.Num() * FVoxelMaterial::NumChannels);
263294

264-
const FVoxelMaterial& Material = OldSingleMaterials[OldIndex];
295+
Chunks.Empty(NewChunks.Num());
265296

266-
for (int32 Channel = 0; Channel < FVoxelMaterial::NumChannels; Channel++)
267-
{
268-
MaterialIndices.GetRaw(Channel) = SingleMaterials.Add(Material.GetRaw(Channel)) | MaterialIndexSingleValueFlag;
269-
}
270-
}
271-
else
297+
// Fixup chunks
298+
for (auto& Chunk : NewChunks)
299+
{
300+
if (Chunk.MaterialsIndex != -1)
272301
{
273-
check(Chunk.MaterialsIndex % VOXELS_PER_DATA_CHUNK == 0);
302+
if (Chunk.bSingleMaterial_Unused)
303+
{
304+
const int32 OldIndex = Chunk.MaterialsIndex;
274305

275-
const int32 OldIndex = Chunk.MaterialsIndex;
306+
Chunk.MaterialsIndex = MaterialsIndices.AddUninitialized(1);
307+
auto& MaterialIndices = MaterialsIndices[Chunk.MaterialsIndex];
276308

277-
Chunk.MaterialsIndex = MaterialsIndices.AddUninitialized(1);
278-
auto& MaterialIndices = MaterialsIndices[Chunk.MaterialsIndex];
309+
const FVoxelMaterial& Material = OldSingleMaterials[OldIndex];
279310

280-
for (int32 Channel = 0; Channel < FVoxelMaterial::NumChannels; Channel++)
281-
{
282-
MaterialIndices.GetRaw(Channel) = MaterialBuffers.AddUninitialized(VOXELS_PER_DATA_CHUNK);
311+
for (int32 Channel = 0; Channel < FVoxelMaterial::NumChannels; Channel++)
312+
{
313+
MaterialIndices.GetRaw(Channel) = SingleMaterials.Add(Material.GetRaw(Channel)) | MaterialIndexSingleValueFlag;
314+
}
283315
}
284-
285-
for (int32 Index = 0; Index < VOXELS_PER_DATA_CHUNK; Index++)
316+
else
286317
{
287-
const FVoxelMaterial& Material = OldMaterialBuffers[OldIndex + Index];
318+
check(Chunk.MaterialsIndex % VOXELS_PER_DATA_CHUNK == 0);
319+
320+
const int32 OldIndex = Chunk.MaterialsIndex;
321+
322+
Chunk.MaterialsIndex = MaterialsIndices.AddUninitialized(1);
323+
auto& MaterialIndices = MaterialsIndices[Chunk.MaterialsIndex];
288324

289325
for (int32 Channel = 0; Channel < FVoxelMaterial::NumChannels; Channel++)
290326
{
291-
MaterialBuffers[MaterialIndices.GetRaw(Channel) + Index] = Material.GetRaw(Channel);
327+
MaterialIndices.GetRaw(Channel) = MaterialBuffers.AddUninitialized(VOXELS_PER_DATA_CHUNK);
328+
}
329+
330+
for (int32 Index = 0; Index < VOXELS_PER_DATA_CHUNK; Index++)
331+
{
332+
const FVoxelMaterial& Material = OldMaterialBuffers[OldIndex + Index];
333+
334+
for (int32 Channel = 0; Channel < FVoxelMaterial::NumChannels; Channel++)
335+
{
336+
MaterialBuffers[MaterialIndices.GetRaw(Channel) + Index] = Material.GetRaw(Channel);
337+
}
292338
}
293339
}
294340
}
341+
FVoxelChunkSave NewChunk;
342+
NewChunk.Position = Chunk.Position;
343+
NewChunk.ValuesIndex = Chunk.ValuesIndex;
344+
NewChunk.MaterialsIndex = Chunk.MaterialsIndex;
345+
NewChunk.bSingleValue = Chunk.bSingleValue;
346+
Chunks.Add(NewChunk);
295347
}
296-
FVoxelChunkSave NewChunk;
297-
NewChunk.Position = Chunk.Position;
298-
NewChunk.ValuesIndex = Chunk.ValuesIndex;
299-
NewChunk.MaterialsIndex = Chunk.MaterialsIndex;
300-
NewChunk.bSingleValue = Chunk.bSingleValue;
301-
Chunks.Add(NewChunk);
302-
}
303348

304-
ensure(MaterialsIndices.GetSlack() == 0);
305-
ensure(MaterialBuffers.GetSlack() == 0);
306-
ensure(SingleMaterials.GetSlack() == 0);
307-
ensure(Chunks.GetSlack() == 0);
349+
ensureVoxelSlow(MaterialsIndices.GetSlack() == 0);
350+
ensureVoxelSlow(MaterialBuffers.GetSlack() == 0);
351+
ensureVoxelSlow(SingleMaterials.GetSlack() == 0);
352+
ensureVoxelSlow(Chunks.GetSlack() == 0);
353+
}
308354
}
309-
}
310355

311-
// Serialize placeable items
312-
if (Version >= FVoxelSaveVersion::PlaceableItemsInSave)
313-
{
314-
Ar << PlaceableItems;
356+
// Serialize placeable items
357+
if (Version >= FVoxelSaveVersion::PlaceableItemsInSave)
358+
{
359+
Ar << PlaceableItems;
360+
}
315361
}
316362

317363
if (Ar.IsLoading() && Ar.IsError())

0 commit comments

Comments
 (0)