Skip to content

Instantly share code, notes, and snippets.

@sanealytics

sanealytics/sample_function_sql.py

Last active June 20, 2020 19:11
Show Gist options
  • Save sanealytics/39884f087e2046e88c8ffce19a2ef1ce to your computer and use it in GitHub Desktop.
Save sanealytics/39884f087e2046e88c8ffce19a2ef1ce to your computer and use it in GitHub Desktop.
Download ZIP
Raw
sample_function_sql.py
def get_elt_queries():
"""Gets parameterised queries for ELT"""
return {
# Parameters: service
'input_data': """
select
data,
added_ts
from
XXX.TABLE
where
service = @service
order by
added_ts desc
"""
}
# Deploy a wrapper around this function to wake up on file drop of some file
def sample_transform_table():
input_data = get_input_df('input_data') # Returns pandas dataframe
r = service_trace_id('sample_transform_table', ...., input_data['tid'].unique().tolist())
tid = json.loads(r).get('tid')
# When writing data back,
# For new rows, insert this new tid into into tid column.
# For updating rows, insert this new tid into prev_tid column.
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment