from transformers import LlamaTokenizer

tokenizer = LlamaTokenizer.from_pretrained('decapoda-research/llama-7b-hf')

def count(text):
  return len(tokenizer(text)['input_ids'])

def parallel_count(texts):
  from joblib import Parallel, delayed
  results = Parallel(n_jobs=-1)(delayed(count)(text) for text in texts)
  return sum(results)