Update log_weights.py

This commit is contained in:
Deepanshu Gupta 2023-04-09 11:11:43 +05:30 committed by GitHub
parent e3173f83cc
commit 8fb8f2329e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

View File

@ -74,7 +74,7 @@ def log_ebc_norms(
: min(sample_size, emb_weight_tensor.shape[0])
]
# WARNING: .cpu() transfer executes malloc that may be the cause of memory leaks
# Change sample_size if the user observe frequent OOM errors or remove weight logging.
# Change sample_size if user observe frequent OOM errors or remove weight logging.
norms = emb_weight_tensor[sample_mask].cpu().norm(dim=1).to(torch.float32)
logging.info(f"Norm shape before reduction: {norms.shape}", rank=-1)
norms = norms.mean().to(torch.device(f"cuda:{dist.get_rank()}"))