Frédéric Laudarin flaudanum

PySpark examples

This gist is a collection of small scripts illustrating the basic usage of Apache Spark with the python API PySpark. I wrote them while reading the book Learning Spark from H. Karau, et al. (O'Reilly, 2015). The scripts are named after the examples in the book, although their content is merely inspired by the topics of the examples.

	<?xml version="1.0" encoding="UTF8"?>
	<toolchains>
	<toolchain>
	<type>jdk</type>
	<provides>
	<version>1.8</version>
	<vendor>Oracle</vendor>
	</provides>
	<configuration>
	<jdkHome>C:\Program Files\Java\jdk1.8.0_241\</jdkHome>

	from pathlib import Path

	def tested_function(path_obj: Path):

	# fobj is the return value of path_obj.open().__enter__()
	with path_obj.open('r') as fobj:
	for line in fobj.readlines():
	# do stuffs w/ line

	{
	"version": "0.2.0",
	"configurations": [
	{
	"type": "node",
	"request": "launch",
	"name": "Launch Program",
	"skipFiles": [
	"<node_internals>/**"
	],

	package com.my.domain;

	import java.io.BufferedReader;
	import java.io.IOException;
	import java.io.InputStreamReader;

	/**
	* Static methods for running external command sub-processes
	*/
	public class CmdProcUtils {

	String winPath = "C:\\Users\\f.laudanum\\Documents\\My-Directory\\ ";
	String cleanPath;

	/* REGEXP pattern: '\\\s' with doubled backslashes /
	Pattern pattern = Pattern.compile("\\\\\\s*$");
	Matcher match = pattern.matcher(winPath);

	/* Call to find(0) triggers the pattern search from text's start*/
	if (match.find(0)) {
	cleanPath = match.replaceAll("");

	BackendRunner mockedRunner = mock(BackendRunner.class);

	// Mock method runProcess() to do nothing
	doAnswer(new Answer<Void>() {
	public Void answer(InvocationOnMock invocation) {
	Object[] args = invocation.getArguments();
	System.out.println("called with arguments: " + Arrays.toString(args));
	return null;
	}
	}).when(mockedRunner).runProcess(any());

	package com.javatpoint;

	import java.awt.event.*;
	import javax.swing.*;

	public class ButtonExample {

	public static void main(String[] args) {
	/* New frame with title 'Button Example' */
	JFrame f = new JFrame("Button Example");

	server {
	# Listening to port 80 (default port for HTTP protocol)
	listen 80 default_server;
	listen [::]:80 default_server;

	# Default local root path for web resources
	root /var/www/html;

	# Add index.php to the list if you are using PHP
	index index.html index.htm index.nginx-debian.html;

	prune Tests
	graft Licenses
	exclude pytest.ini README.md