William Held Helw150

Helw150 / stream_gradio_audio.py

Last active August 27, 2024 07:05

	import gradio as gr
	import math
	import numpy as np
	import time
	import io
	import wave


	def wave_header_chunk(frame_input=b"", channels=1, sample_width=2, sample_rate=24000):
	# This will create a wave header then append the frame input

Helw150 / data_generation.py

Created May 23, 2024 00:06

	from time import sleep

	from datasets import load_dataset
	from huggingface_hub import InferenceClient
	from ratelimit import limits, sleep_and_retry
	from transformers import AutoTokenizer

	dataset = load_dataset("yijingwu/HeySQuAD_human", split="train")

	tokenizer = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3-8B-Instruct")

Helw150 / via.py

Last active May 10, 2024 19:58

	text = # Tokenized Text Corresponding to Recording Transcript
	audio = # Mel Spectrogram of the Recording

	# Only Train Connector and Projection
	self.encoder.freeze()
	self.llama.freeze()

	# Convert Raw Audio Signal to 1500 Embeddings with Whisper Encoder (CNN+Transformer)
	audio_features = self.encoder(audio)

Helw150 / intermediate_push_parquet.py

Created May 7, 2024 21:41

	def _push_parquet_shards_to_hub( [1071/1877]
	self,
	repo_id: str,
	data_dir: str = "data",
	split: Optional[str] = None,
	token: Optional[str] = None,
	revision: Optional[str] = None,
	create_pr: Optional[bool] = False,
	max_shard_size: Optional[Union[int, str]] = None,
	num_shards: Optional[int] = None,

Helw150 / process_parses.py

Last active May 6, 2024 17:33

Helw150 / ot_loss.py

Last active April 27, 2023 22:02

OT TADA Loss

	from typing import List, Optional, Tuple, Union
	from torchtyping import TensorType

	from transformers.adapters.modeling import Adapter
	from transformers.adapters import (
	BartAdapterModel,
	RobertaAdapterModel,
	BertAdapterModel,
	AdapterConfig,
	)

Helw150 / parallel_t5.py

Last active May 10, 2023 14:52

Flan T5 Parallel Usage

	from transformers import AutoTokenizer, T5ForConditionalGeneration

	# Model Init
	n_gpu = 8
	tokenizer = AutoTokenizer.from_pretrained("google/flan-ul2")
	model = T5ForConditionalGeneration.from_pretrained("google/flan-ul2")
	heads_per_gpu = len(model.encoder.block) // n_gpu
	device_map = {
	gpu: list(
	range(

Helw150 / upload_csv.py

Created September 16, 2022 15:50

Lab Meeting Dataset upload Code

	# See https://huggingface.co/docs/datasets/upload_dataset for more details

	from datasets import load_dataset

	dataset_name = "PUT_YOUR_NAME_HERE"
	data_files = {"train": "train.csv", "dev": "dev.csv", "test": "test.csv"}
	dataset = load_dataset("namespace/your_dataset_name", data_files=data_files)
	datasets.push_to_hub(f"SALT-NLP/{dataset_name}", private=True)

Helw150 / save2gensim.py

Last active April 13, 2019 12:32

Saves a dictionary of vectors into the Gensim KeyedVectors format

	from gensim import utils

	def save2gensim(fname, word2vec_dict):
	vectors = list(word2vec_dict.values())
	vector_size = vectors[0].shape[0]
	total_vec = len(vectors)
	with utils.smart_open(fname, 'wb') as fout:
	fout.write(utils.to_utf8("%s %s\n" % (total_vec, vector_size)))
	# store in sorted order: most frequent words at the top
	for word, vector in word2vec_dict.items():

Helw150 / large-file-processing.py

Last active July 28, 2018 22:34

A Python Script which multi-processes large files with a rough progress bar

	#!/usr/bin/env python
	"""Counts the number of times a word occurs in a very large text file"""

	from __future__ import print_function
	import os
	import sys
	import argparse
	import textacy
	import multiprocessing
	from tqdm import tqdm