| df.write | |
| .format("parquet") | |
| .mode(SaveMode.Append) | |
| .option("compression", "snappy") | |
| .insertInto(table) |
| @transient private lazy val logger: Logger = LogManager.getRootLogger |
| val serviceResponsesRDD: RDD[List[ServiceResponse]] = requestsRDD.mapPartitions{ requestIterator => | |
| val requests = requestIterator.toList | |
| logger.info(s"[START] Query service with ${requests.length} requests") | |
| // Provides a CloseableHttpClient with back off should the service fail | |
| val httpClient = HttpClientBuilder.withBackoff() | |
| val responsesToAwait = requests.map(request => Service.call(request, httpClient)) | |
| val sequenceResponses = Future.sequence(responsesToAwait) | |
| val responses = Await.result(sequenceResponses, 2 minutes) |
| import org.apache.spark.sql.{DataFrame, SparkSession} | |
| object ElasticSearch extends Serializable { | |
| def query(spark: SparkSession, query: String, index: String): DataFrame = | |
| spark.read.format("es") | |
| .option("es.nodes", Configuration.elasticSearch.host) | |
| .option("es.port", Configuration.elasticSearch.port) | |
| .option("es.nodes.wan.only", "true") | |
| .option("pushdown", "true") |
| name := "etl-store-availability" | |
| version := "0.1" | |
| scalaVersion := "2.11.8" | |
| libraryDependencies ++= Seq( | |
| // Spark | |
| "org.apache.spark" %% "spark-core" % "2.3.2" % "provided", | |
| "org.apache.spark" %% "spark-sql" % "2.3.2" % "provided", |
| import boto3 | |
| def create_session(profile_name, region_name): | |
| return boto3.session.Session(profile_name=profile_name, region_name=region_name) | |
| def create_client(profile_name, region_name, service_name): | |
| session = create_session(profile_name, region_name) | |
| return session.client(service_name) |
| #!/usr/bin/env python | |
| import boto | |
| import sys, os | |
| from boto.s3.key import Key | |
| from boto.exception import S3ResponseError | |
| DOWNLOAD_LOCATION_PATH = os.path.expanduser("~") + "/s3-backup/" | |
| if not os.path.exists(DOWNLOAD_LOCATION_PATH): |
| import boto3 | |
| ACCESS_KEY_ID = "xxxxxxxx" | |
| SECRET_ACCESS_KEY = "xxxxxxxx" | |
| client = boto3.client('emr', | |
| region_name='region', | |
| aws_access_key_id=ACCESS_KEY_ID, | |
| aws_secret_access_key=SECRET_ACCESS_KEY) |
Copyright © 2017 Fantasyland Institute of Learning. All rights reserved.
A function is a mapping from one set, called a domain, to another set, called the codomain. A function associates every element in the domain with exactly one element in the codomain. In Scala, both domain and codomain are types.
val square : Int => Int = x => x * x
| import scala.io.Source | |
| val lines = Source.fromFile("your-path-here").getLines |