Install jupyter
$pip install -U jupyter
Create virtual environment
| <?xml version="1.0" encoding="UTF-8"?> | |
| <Invoice xmlns="urn:oasis:names:specification:ubl:schema:xsd:Invoice-2" xmlns:cac="urn:oasis:names:specification:ubl:schema:xsd:CommonAggregateComponents-2" xmlns:cbc="urn:oasis:names:specification:ubl:schema:xsd:CommonBasicComponents-2" xmlns:ds="http://www.w3.org/2000/09/xmldsig#" xmlns:ext="urn:oasis:names:specification:ubl:schema:xsd:CommonExtensionComponents-2"> | |
| <ext:UBLExtensions> | |
| <ext:UBLExtension /> | |
| </ext:UBLExtensions> | |
| <cbc:UBLVersionID>2.1</cbc:UBLVersionID> | |
| <cbc:CustomizationID>2.0</cbc:CustomizationID> | |
| <cbc:ID>B001-1</cbc:ID> | |
| <cbc:IssueDate>2018-10-15</cbc:IssueDate> | |
| <cbc:IssueTime>00:44:53</cbc:IssueTime> |
| """ | |
| Create train, valid, test iterators for CIFAR-10 [1]. | |
| Easily extended to MNIST, CIFAR-100 and Imagenet. | |
| [1]: https://discuss.pytorch.org/t/feedback-on-pytorch-for-kaggle-competitions/2252/4 | |
| """ | |
| import torch | |
| import numpy as np |
| #!/bin/bash | |
| # License: Public Domain. | |
| echo "export PYSPARK_PYTHON=python3" | tee -a /etc/profile.d/spark_config.sh /etc/*bashrc /usr/lib/spark/conf/spark-env.sh | |
| echo "export PYTHONHASHSEED=0" | tee -a /etc/profile.d/spark_config.sh /etc/*bashrc /usr/lib/spark/conf/spark-env.sh | |
| echo "spark.executorEnv.PYTHONHASHSEED=0" >> /etc/spark/conf/spark-defaults.conf | |
| # Only run on the master node | |
| ROLE=$(/usr/share/google/get_metadata_value attributes/dataproc-role) | |
| if [[ "${ROLE}" == 'Master' ]]; then |
| #!/bin/bash | |
| # from https://gist.github.com/nehalecky/9258c01fb2077f51545a/raw/789f08141dc681cf1ad5da05455c2cd01d1649e8/install-py3-dataproc.sh | |
| apt-get -y install python3 | |
| echo "export PYSPARK_PYTHON=python3" | tee -a /etc/profile.d/spark_config.sh /etc/*bashrc /usr/lib/spark/conf/spark-env.sh | |
| echo "Adding PYTHONHASHSEED=0 to profiles and spark-defaults.conf..." | |
| echo "export PYTHONHASHSEED=0" | tee -a /etc/profile.d/spark_config.sh /etc/*bashrc /usr/lib/spark/conf/spark-env.sh | |
| echo "spark.executorEnv.PYTHONHASHSEED=0" >> /etc/spark/conf/spark-defaults.conf |
| # Install R | |
| sudo apt-get update | |
| sudo apt-get install gdebi libxml2-dev libssl-dev libcurl4-openssl-dev libopenblas-dev r-base r-base-dev | |
| # Install RStudio | |
| cd ~/Downloads | |
| wget https://download1.rstudio.org/rstudio-xenial-1.1.383-amd64.deb | |
| sudo gdebi rstudio-xenial-1.1.383-amd64.deb | |
| printf '\nexport QT_STYLE_OVERRIDE=gtk\n' | sudo tee -a ~/.profile |
| +--------+--------------------+-----+--------------------+--------------------+--------------------+--------------------+----------+ | |
| |category| text|label| words| features| rawPrediction| probability|prediction| | |
| +--------+--------------------+-----+--------------------+--------------------+--------------------+--------------------+----------+ | |
| | 3001|Плойки и наборы V...| 24.0|[плойки, и, набор...|(10000,[326,796,1...|[-174.67716870697...|[6.63481663197049...| 24.0| | |
| | 833|"Чехол-обложка дл...| 1.0|["чехол-обложка, ...|(10000,[514,986,1...|[-379.37151502387...|[5.32678001676623...| 1.0| | |
| | 833|"Чехол-обложка дл...| 1.0|["чехол-обложка, ...|(10000,[514,986,1...|[-379.84825219376...|[2.15785456821554...| 1.0| | |
| | 833|"Чехол-обложка дл...| 1.0|["чехол-обложка, ...|(10000,[290,514,9...|[-395.42735009477...|[6.44323423370500...| 1.0| | |
| | 833|"Чехол-обложка дл...| 1.0|["чехол-обложка, ...|(10000,[290,514,9...|[-396.10251348 |
| #run this in any directory add -v for verbose | |
| #get Pillow (fork of PIL) from pip before running --> pip install Pillow | |
| import os | |
| import sys | |
| from PIL import Image | |
| def compressMe(file, verbose=False): | |
| filepath = os.path.join(os.getcwd(), file) | |
| oldsize = os.stat(filepath).st_size |
| import sqlite3 | |
| class Model: | |
| def __init__(self, db, table): | |
| self.db = db | |
| self.table = table | |
| self.connection = sqlite3.connect(db + '.db') | |
| self.connection.row_factory = sqlite3.Row | |
| def create(self, row): |