batch_size = 32
# Load batch converter
batch_converter = alphabet.get_batch_converter()
# Open the HDF5 file
with h5py.File(hdf5_file_path, “r+”) as hdf:
sequences = [] # Retailer sequences for batching
accessions = [] # Retailer protein IDs for batching
for acc in hdf.keys():
group = hdf[acc]
# Skip already processed information
if “options” in group:
proceed
# Load sequence
sequence = group[“sequence”][()]
sequence = sequence.decode(“utf-8”) if isinstance(sequence, bytes) else sequence
sequences.append((acc, sequence))
accessions.append(acc)
# Course of when sufficient information accumulates for a batch
if len(sequences) == batch_size:
batch_labels, batch_strs, batch_tokens = batch_converter(sequences)
batch_tokens = batch_tokens.to(gadget) # Transfer information to GPU
with torch.no_grad():
outcomes = mannequin(batch_tokens, repr_layers=[30])
embeddings = outcomes[“representations”][30].imply(dim=1).cpu().numpy() # Transfer information GPU -> CPU
# Save embeddings for every ID
for accession, embedding in zip(accessions, embeddings):
group = hdf[accession]
group.create_dataset(“options”, information=embedding)
# Print batch save completion
print(f”Batch processed and saved for accessions: {‘, ‘.be part of(accessions)}”)
# Reset batch
sequences = []
accessions = []
# Course of remaining information
if sequences:
batch_labels, batch_strs, batch_tokens = batch_converter(sequences)
batch_tokens = batch_tokens.to(gadget) # Transfer information to GPU
with torch.no_grad():
outcomes = mannequin(batch_tokens, repr_layers=[30])
embeddings = outcomes[“representations”][30].imply(dim=1).cpu().numpy() # Transfer information GPU -> CPU
for accession, embedding in zip(accessions, embeddings):
group = hdf[accession]
group.create_dataset(“options”, information=embedding)
# Print remaining batch save completion
print(f”Remaining batch processed and saved for accessions: {‘, ‘.be part of(accessions)}”)
print(“All batches processed and saved.”)
