try to fix sync seed again

Author: lucidrains

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "vector-quantize-pytorch"
-version = "1.18.7"
+version = "1.18.8"
 description = "Vector Quantization - Pytorch"
 authors = [
  { name = "Phil Wang", email = "lucidrains@gmail.com" }

vector_quantize_pytorch/residual_fsq.py

@@ -185,7 +185,7 @@ def forward(
  # check if seed is manually passed in
 
  if not exists(rand_quantize_dropout_fixed_seed):
- rand_quantize_dropout_fixed_seed = get_maybe_sync_seed()
+ rand_quantize_dropout_fixed_seed = get_maybe_sync_seed(device)
 
  rand = random.Random(rand_quantize_dropout_fixed_seed)
 
@@ -296,7 +296,7 @@ def forward(
  x,
  return_all_codes = False
  ):
- shape, split_dim = x.shape, self.split_dim
+ shape, split_dim, device = x.shape, self.split_dim, x.device
  assert shape[split_dim] == self.dim
 
  # split the feature dimension into groups
@@ -305,7 +305,7 @@ def forward(
 
  forward_kwargs = dict(
  return_all_codes = return_all_codes,
- rand_quantize_dropout_fixed_seed = get_maybe_sync_seed()
+ rand_quantize_dropout_fixed_seed = get_maybe_sync_seed(device)
  )
 
  # invoke residual vq on each group

vector_quantize_pytorch/residual_lfq.py

@@ -31,8 +31,8 @@ def round_up_multiple(num, mult):
 def is_distributed():
  return dist.is_initialized() and dist.get_world_size() > 1
 
-def get_maybe_sync_seed(max_size = 10_000):
- rand_int = torch.randint(0, max_size, ())
+def get_maybe_sync_seed(device, max_size = 10_000):
+ rand_int = torch.randint(0, max_size, (), device = device)
 
  if is_distributed():
  dist.all_reduce(rand_int)
@@ -162,7 +162,7 @@ def forward(
  # check if seed is manually passed in
 
  if not exists(rand_quantize_dropout_fixed_seed):
- rand_quantize_dropout_fixed_seed = get_maybe_sync_seed()
+ rand_quantize_dropout_fixed_seed = get_maybe_sync_seed(device)
 
  rand = random.Random(rand_quantize_dropout_fixed_seed)
 
@@ -262,7 +262,7 @@ def forward(
  mask = None,
  return_all_codes = False
  ):
- shape, split_dim = x.shape, self.split_dim
+ shape, split_dim, device = x.shape, self.split_dim, x.device
  assert shape[split_dim] == self.dim
 
  # split the feature dimension into groups
@@ -272,7 +272,7 @@ def forward(
  forward_kwargs = dict(
  mask = mask,
  return_all_codes = return_all_codes,
- rand_quantize_dropout_fixed_seed = get_maybe_sync_seed()
+ rand_quantize_dropout_fixed_seed = get_maybe_sync_seed(device)
  )
 
  # invoke residual vq on each group

vector_quantize_pytorch/residual_vq.py

@@ -36,8 +36,8 @@ def round_up_multiple(num, mult):
 def is_distributed():
  return dist.is_initialized() and dist.get_world_size() > 1
 
-def get_maybe_sync_seed(max_size = 10_000):
- rand_int = torch.randint(0, max_size, ())
+def get_maybe_sync_seed(device, max_size = 10_000):
+ rand_int = torch.randint(0, max_size, (), device = device)
 
  if is_distributed():
  dist.all_reduce(rand_int)
@@ -296,7 +296,7 @@ def forward(
  # check if seed is manually passed in
 
  if not exists(rand_quantize_dropout_fixed_seed):
- rand_quantize_dropout_fixed_seed = get_maybe_sync_seed()
+ rand_quantize_dropout_fixed_seed = get_maybe_sync_seed(device)
 
  rand = random.Random(rand_quantize_dropout_fixed_seed)
 
@@ -452,7 +452,7 @@ def forward(
  freeze_codebook = False,
  mask = None,
  ):
- shape, split_dim = x.shape, self.split_dim
+ shape, split_dim, device = x.shape, self.split_dim, x.device
  assert shape[split_dim] == self.dim
 
  # split the feature dimension into groups
@@ -468,7 +468,7 @@ def forward(
  sample_codebook_temp = sample_codebook_temp,
  mask = mask,
  freeze_codebook = freeze_codebook,
- rand_quantize_dropout_fixed_seed = get_maybe_sync_seed()
+ rand_quantize_dropout_fixed_seed = get_maybe_sync_seed(device)
  )
 
  # invoke residual vq on each group