Skip to content

Instantly share code, notes, and snippets.

@msampathkumar

msampathkumar/glue_job.py

Created October 31, 2019 06:53
Show Gist options
  • Save msampathkumar/38c4777180ee522b7876ee5221e65278 to your computer and use it in GitHub Desktop.
Save msampathkumar/38c4777180ee522b7876ee5221e65278 to your computer and use it in GitHub Desktop.
Download ZIP
Read & Write data from S3 AWS Python Glue
Raw
glue_job.py
import io
import pandas as pd
def split_s3_name(s3_path):
bucket, key = s3_path.strip('s3://').split('/', 1)
return (bucket, key)
def read_from_s3(file_name):
bucket, key = split_s3_name(file_name)
s3 = boto3.client('s3')
obj = s3.get_object(Bucket=bucket, Key=key)
df = pd.read_csv(io.BytesIO(obj['Body'].read()))
return df
return obj
def write_to_s3(df, file_name):
bucket, key = split_s3_name(file_name)
csv_buffer = io.BytesIO()
df.to_csv(csv_buffer)
s3_resource = boto3.resource('s3')
s3_resource.Object(bucket, key).put(Body=csv_buffer.getvalue())
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment