Prepare files & libraries

Download selenium standalone server from: http://goo.gl/rQhaxb, Create local folder at ~/selenium and copy or move jar to that path:

$ mkdir ~/selenium
$ cd ~/selenium
$ wget http://goo.gl/rQhaxb
$ mv selenium-server-standalone-2.49.1.jar ~/selenium/

josesaribeiro / ssl_everywhere.py

Created January 9, 2020 14:54 — forked from neovintage/ssl_everywhere.py

SSL Support for Airflow

	from flask import Blueprint, redirect, url_for, request, current_app
	from airflow.plugins_manager import AirflowPlugin


	YEAR_IN_SECS = 31536000
	ssl_bp = Blueprint('ssl_everything', __name__)

	@ssl_bp.before_app_request
	def before_request():
	app = current_app._get_current_object()

josesaribeiro / query_athena.py

Created November 26, 2019 18:48 — forked from sysboss/query_athena.py

SQL Query Amazon Athena using Python

	#!/usr/bin/env python3
	#
	# Query AWS Athena using SQL
	# Copyright (c) Alexey Baikov <sysboss[at]mail.ru>
	#
	# This snippet is a basic example to query Athen and load the results
	# to a variable.
	#
	# Requirements:
	# > pip3 install boto3 botocore retrying

josesaribeiro / 0_register_planet.sql

Created October 17, 2019 03:51 — forked from mojodna/0_register_planet.sql

Sample OSM Athena queries

	--
	-- This will register the "planet" table within your AWS account
	--
	CREATE EXTERNAL TABLE planet (
	id BIGINT,
	type STRING,
	tags MAP<STRING,STRING>,
	lat DECIMAL(9,7),
	lon DECIMAL(10,7),
	nds ARRAY<STRUCT<ref: BIGINT>>,

josesaribeiro / gzip_s3_and_json_py3.py

Created October 11, 2019 03:56 — forked from a-hisame/gzip_s3_and_json_py3.py

To use gzip file between python application and S3 directly for Python3

	#!/usr/bin/python
	# -- coding: utf-8 --

	'''To use gzip file between python application and S3 directly for Python3.
	Python 2 version - https://gist.github.com/a-hisame/f90815f4fae695ad3f16cb48a81ec06e
	'''

	import io
	import gzip
	import json

josesaribeiro / airflowPostgresqlInstall.sh

Created September 26, 2019 02:55 — forked from cronosnull/airflowPostgresqlInstall.sh

Install Airflow on a new Ubuntu server 18.04

	#!/bin/bash
	USUARIO_SO="$(whoami)"
	ANACONDA_URL="https://repo.anaconda.com/archive/Anaconda3-5.2.0-Linux-x86_64.sh"
	_DB_PASSWORD="la contraseña"
	_IP=$(hostname -I \| cut -d' ' -f1)
	while getopts "a:p:h" opt; do
	case $opt in
	a) ANACONDA_URL="$OPTARG";;
	p) _DB_PASSWORD="$OPTARG";;
	h) cat <<EOF

josesaribeiro / to_redshift.py

Created September 22, 2019 17:59 — forked from TomAugspurger/to_redshift.py

to_redshift.py

	# see also https://github.com/wrobstory/pgshift

	import gzip
	from io import StringIO, BytesIO
	from functools import wraps

	import boto
	from sqlalchemy import MetaData
	from pandas import DataFrame
	from pandas.io.sql import SQLTable, pandasSQL_builder

josesaribeiro / airflow-dag-csv-to-mysql.py

Created September 21, 2019 13:04

Airflow Ftp CSV to SQL

	"""
	Code that goes along with the Airflow tutorial located at:
	https://github.com/airbnb/airflow/blob/master/airflow/example_dags/tutorial.py
	"""
	from airflow import DAG
	from airflow.operators.python_operator import PythonOperator
	from airflow.operators.generic_transfer import GenericTransfer
	from airflow.contrib.hooks import FTPHook
	from airflow.hooks.mysql_hook import MySqlHook

josesaribeiro / flask_drive_example.py

Created August 7, 2019 20:17 — forked from prahladyeri/flask_drive_example.py

google drive api implementation in python-flask framework

	##
	# Flask Drive Example App
	#
	# @author Prahlad Yeri <[email protected]>
	# @date 30-12-2016
	# Dependency:
	# 1. pip install flask google-api-python-client
	# 2. make sure you have client_id.json in this same directory.

	import os

josesaribeiro / airflow-supervisord.conf

Created July 16, 2019 02:23 — forked from criccomini/airflow-supervisord.conf

airflow-supervisord.conf

	; Configuration for Airflow webserver and scheduler in Supervisor

	[program:airflow]
	command=/bin/airflow webserver
	stopsignal=QUIT
	stopasgroup=true
	user=airflow
	stdout_logfile=/var/log/airflow/airflow-stdout.log
	stderr_logfile=/var/log/airflow/airflow-stderr.log
	environment=HOME="/home/airflow",AIRFLOW_HOME="/etc/airflow",TMPDIR="/storage/airflow_tmp"

Jose Ribeiro josesaribeiro