kingspp · August 4, 2025 14:13 · Nov 21, 2024 · Oct 20, 2024 · Aug 23, 2024 · Nov 3, 2023
diff --git a/databricks.md b/databricks.md
@@ -37,4 +37,20 @@ print(views)
 ```python
 databricksURL = dbutils.notebook.entry_point.getDbutils().notebook().getContext().apiUrl().getOrElse(None)
 myToken = dbutils.notebook.entry_point.getDbutils().notebook().getContext().apiToken().getOrElse(None)
+```
+
+### Alter Schema of a column
+```sql
+-- Try
+ALTER TABLE <catalog>.<schema>.<table_name> SET TBLPROPERTIES ('delta.columnMapping.mode' = 'name')
+-- If the above does not work use,
+ALTER TABLE <catalog>.<schema>.<table_name> SET TBLPROPERTIES (
+   'delta.columnMapping.mode' = 'name',
+   'delta.minReaderVersion' = '2',
+   'delta.minWriterVersion' = '5')
+
+-- Change column name to make sure the schema evolution works in the subsequent run
+ALTER TABLE <catalog>.<schema>.<table_name>  RENAME COLUMN <col_of_interest> TO <col_of_interest>r_v1;
+
+-- Merge once the run is successfull if necessary
 ```
diff --git a/databricks.md b/databricks.md
@@ -10,16 +10,31 @@
 # dbutils.library.restartPython()
 ```
 
-# Load a JAR after cluster start
+### Load a JAR after cluster start
 
 ```sql
 ADD jar dbfs:/<path to jar>.jar
 ```
 
-# Mount a bucket
-```
+### Mount a bucket
+```python
 aws_bucket_name = "bucket_name"
 mount_name = "research"
 dbutils.fs.mount(f"s3a://{aws_bucket_name}", f"/mnt/{mount_name}")
 display(dbutils.fs.ls(f"/mnt/{mount_name}/"))
+```
+
+### Fetch Views from a catalog
+```python
+catalog = "prod"
+schema = "ml"
+spark.sql(f"USE CATALOG {catalog}")
+views = [f.viewName for f in spark.sql(f"SHOW VIEWS IN {catalog}.{schema}").collect()]
+print(views)
+```
+
+### Get token
+```python
+databricksURL = dbutils.notebook.entry_point.getDbutils().notebook().getContext().apiUrl().getOrElse(None)
+myToken = dbutils.notebook.entry_point.getDbutils().notebook().getContext().apiToken().getOrElse(None)
 ```
diff --git a/macos-snippets.sh b/macos-snippets.sh
@@ -50,3 +50,6 @@ sudo spctl --master-disable
 
 # Install Tree View for listing file structure, Linux Core Utils
 brew install tree coreutils
+
+# Identify thermal throttling
+pmset -g thermlog
diff --git a/databricks.md b/databricks.md
@@ -5,13 +5,21 @@
 
 
 ```python
-# !/databricks/python3/bin/python -m pip uninstall cmxdt -y
-# !/databricks/python3/bin/python -m pip install -e /dbfs/code/
+# !/databricks/python3/bin/python -m pip uninstall <package> -y
+# !/databricks/python3/bin/python -m pip install -e /dbfs/<path>/
 # dbutils.library.restartPython()
 ```
 
 # Load a JAR after cluster start
 
 ```sql
 ADD jar dbfs:/<path to jar>.jar
+```
+
+# Mount a bucket
+```
+aws_bucket_name = "bucket_name"
+mount_name = "research"
+dbutils.fs.mount(f"s3a://{aws_bucket_name}", f"/mnt/{mount_name}")
+display(dbutils.fs.ls(f"/mnt/{mount_name}/"))
 ```
diff --git a/databricks.md b/databricks.md
@@ -1,11 +1,17 @@
 # Databricks Snippets
 
 
-### Load A python library dynamically
+### Load A python library dynamically (Cluster restarts not required)
 
 
 ```python
 # !/databricks/python3/bin/python -m pip uninstall cmxdt -y
 # !/databricks/python3/bin/python -m pip install -e /dbfs/code/
 # dbutils.library.restartPython()
+```
+
+# Load a JAR after cluster start
+
+```sql
+ADD jar dbfs:/<path to jar>.jar
 ```
diff --git a/databricks.md b/databricks.md
@@ -0,0 +1,11 @@
+# Databricks Snippets
+
+
+### Load A python library dynamically
+
+
+```python
+# !/databricks/python3/bin/python -m pip uninstall cmxdt -y
+# !/databricks/python3/bin/python -m pip install -e /dbfs/code/
+# dbutils.library.restartPython()
+```
diff --git a/py-snippet.py b/py-snippet.py
@@ -639,4 +639,12 @@ def internal_writer(self):
   def close(self):
     self.queue.join()
     self.finished = True
-    self.filewriter.close()
+    self.filewriter.close()
+
+
+# Timeout Whileloop
+import time
+timeout = 10   # [seconds]
+timeout_start = time.time()
+while time.time() < timeout_start + timeout:    
+    pass
diff --git a/py-snippet.py b/py-snippet.py
@@ -614,4 +614,29 @@ def batch(iterable, n=1):
 finder.run_script("./main.py")
 for name, mod in finder.modules.items():
     print(name)
-
+
+
+# Thread Safe Writer
+class SafeWriter:
+  def __init__(self, *args):
+    self.filewriter = open(*args)
+    self.queue = Queue()
+    self.finished = False
+    Thread(name="SafeWriter", target=self.internal_writer).start()
+
+  def write(self, data):
+    self.queue.put(data)
+
+  def internal_writer(self):
+    while not self.finished:
+      try:
+        data = self.queue.get(True, 1)
+      except Empty:
+        continue
+      self.filewriter.write(data)
+      self.queue.task_done()
+
+  def close(self):
+    self.queue.join()
+    self.finished = True
+    self.filewriter.close()
diff --git a/docker_utils.sh b/docker_utils.sh
@@ -6,6 +6,11 @@ sudo docker load -i [image_name:tag].tar.gz
 
 # Delete all containers including its volumes use
 docker rm -vf $(docker ps -a -q)
+docker rm -vf  $(docker-compose ps -q)
 
 # Delete all the images
 docker rmi -f $(docker images -a -q)
+docker rmi -f $(docker-compose images -q)
+
+# Delete images and containers from docker compose
+DOCKER_CONTAINERS=$(docker-compose ps -q) DOCKER_IMAGES=$(docker-compose images -q) docker rm -vf $DOCKER_CONTAINERS && docker rmi -f $DOCKER_IMAGES
diff --git a/docker_utils.sh b/docker_utils.sh
@@ -2,4 +2,10 @@
 sudo docker save [image_name:tag] >  [image_name:tag].tar.gz
 
 # Load Image
-sudo docker load -i [image_name:tag].tar.gz
+sudo docker load -i [image_name:tag].tar.gz
+
+# Delete all containers including its volumes use
+docker rm -vf $(docker ps -a -q)
+
+# Delete all the images
+docker rmi -f $(docker images -a -q)
diff --git a/py-snippet.py b/py-snippet.py
@@ -608,5 +608,10 @@ def batch(iterable, n=1):
 for x in batch(list(range(0, 10)), 3):
     print(x)
 
-
+# Find all the imported modules
+from modulefinder import ModuleFinder
+finder = ModuleFinder()
+finder.run_script("./main.py")
+for name, mod in finder.modules.items():
+    print(name)
 
diff --git a/tensorflow-utils.py b/tensorflow-utils.py
@@ -12,4 +12,16 @@ def get_available_devices(cpu: bool = True, gpu: bool = True):
     return devices
 
 # Check CUDA Installation and GPU Availability
-print(tf.config.list_physical_devices('GPU'))
+print(tf.config.list_physical_devices('GPU'))
+
+
+
+# Fetch row indices
+x = tf.random.normal([3,2])
+x = tf.convert_to_tensor(x)
+
+indices = tf.convert_to_tensor([0,1,0])
+one_hot_indices = tf.expand_dims(indices, 1)
+range = tf.expand_dims(tf.range(tf.shape(indices)[0]), 1)
+ind = tf.concat([range, one_hot_indices], 1)
+tf.gather_nd(x, ind)
diff --git a/latex.tex b/latex.tex
@@ -67,3 +67,7 @@
 \end{aligned}
 \end{equation}
 \end{minipage}
+
+% Add Paragraph break
+\par% 
+\bigskip
diff --git a/latex.tex b/latex.tex
@@ -0,0 +1,69 @@
+% Required Packages
+% Package Declarations
+\usepackage{arxiv}
+\usepackage[utf8]{inputenc} % allow utf-8 input
+\usepackage[T1]{fontenc}    % use 8-bit T1 fonts
+\usepackage{hyperref}       % hyperlinks
+\usepackage{url}            % simple URL typesetting
+\usepackage{booktabs}       % professional-quality tables
+\usepackage{amsfonts}       % blackboard math symbols
+\usepackage{nicefrac}       % compact symbols for 1/2, etc.
+\usepackage{microtype}      % microtypography
+\usepackage{lipsum}         % Lorem Ipsum fill text
+\usepackage{multicol}       % Support for Multi columns for tables
+\usepackage{multirow}       % Support for Multi rows fot tables
+\usepackage{mathtools}      % Advanced mathtools
+\usepackage{caption}        % Advanced caption configuration
+\usepackage{amsmath}        % Math package for equations       
+\usepackage{titlesec}       % Title section
+\usepackage{graphicx}       % For adding labels to parts of the equation
+\usepackage{stackrel}       % For adding labels to parts of the equation
+\usepackage[ruled,vlined]{algorithm2e}  % For Algorithms
+\usepackage{algorithm}
+\usepackage{algpseudocode}
+
+
+% Theme Configurations
+
+% Set section depth to 4
+% \setcounter{secnumdepth}{4}
+
+% \titleformat{\paragraph}
+% {\normalfont\normalsize\bfseries}{\theparagraph}{1em}{}
+% \titlespacing*{\paragraph}
+% {0pt}{3.25ex plus 1ex minus .2ex}{1.5ex plus .2ex}
+
+% Add padding to text below table
+\captionsetup[table]{skip=10pt}
+
+% Configure hat tex
+\let\oldhat\hat
+\renewcommand{\hat}[1]{\oldhat{\mathbf{#1}}}
+
+% Image
+
+\begin{figure}[H]
+\centering
+\includegraphics[scale=0.2]{assets/sdk.png}
+\captionof{figure}{Comparison between Tensorflow and Keras}
+\label{fig:workflow}
+\end{figure}
+
+
+% Two column [Image | Text]
+\noindent\begin{minipage}{.45\textwidth}
+   \centering
+   \includegraphics[scale=0.09]{SimpleFFN.png}
+   \captionof{figure}{Feed Forward Network}
+   \label{fig:SimpleFFN.png}
+\end{minipage}
+\begin{minipage}{.45\textwidth}
+\begin{equation}
+\label{eq:ffn_math_representation}
+\begin{aligned}
+   Dense_{1} &= \sigma(Input \cdot \hat{W}_{1} + \hat{b}_{1}) &\\
+   Dense_{2} &= \sigma(Dense_{1} \cdot \hat{W}_{2} + \hat{b}_{2}) &\\
+   Dense_{3} &= \sigma(Dense_{2} \cdot \hat{W}_{3} + \hat{b}_{3}) 
+\end{aligned}
+\end{equation}
+\end{minipage}
diff --git a/tensorflow-utils.py b/tensorflow-utils.py
@@ -9,4 +9,7 @@ def get_available_devices(cpu: bool = True, gpu: bool = True):
         devices = [x.name for x in local_device_protos if x.device_type == 'CPU']
     if gpu:
         devices += [x.name for x in local_device_protos if x.device_type == 'GPU']
-    return devices
+    return devices
+
+# Check CUDA Installation and GPU Availability
+print(tf.config.list_physical_devices('GPU'))
diff --git a/linux_snippets.sh b/linux_snippets.sh
@@ -0,0 +1,2 @@
+# Fix unmet dependencies error
+sudo apt-get -o Dpkg::Options::="--force-overwrite" install --fix-broken
diff --git a/jupyter_snippets.py b/jupyter_snippets.py
@@ -10,6 +10,7 @@
 env = gym.make('Breakout-v0')
 env.reset()
 img = plt.imshow(env.render(mode='rgb_array')) # only call this once
+plt.xticks([]),plt.yticks([])
 for _ in range(100):
     img.set_data(env.render(mode='rgb_array')) # just update the data
     display.display(plt.gcf())

diff --git a/bash-snippet.sh b/bash-snippet.sh
@@ -194,4 +194,7 @@ lsof -i tcp:<port number>
 shuf -n N input > output
 
 # Generate a random number between 1-10
-$(( ( RANDOM % 10 )  + 1 ))
+$(( ( RANDOM % 10 )  + 1 ))
+
+# SSH Based Port forwarding
+ssh -N -f -L localhost:3000:localhost:3000 username@ip
diff --git a/jupyter_snippets.py b/jupyter_snippets.py
@@ -0,0 +1,18 @@
+# Notebook Snippets
+
+# Render environment
+import gym
+from IPython import display
+import matplotlib
+import matplotlib.pyplot as plt
+%matplotlib inline
+
+env = gym.make('Breakout-v0')
+env.reset()
+img = plt.imshow(env.render(mode='rgb_array')) # only call this once
+for _ in range(100):
+    img.set_data(env.render(mode='rgb_array')) # just update the data
+    display.display(plt.gcf())
+    display.clear_output(wait=True)
+    action = env.action_space.sample()
+    env.step(action)
diff --git a/py-snippet.py b/py-snippet.py
@@ -607,3 +607,6 @@ def batch(iterable, n=1):
 
 for x in batch(list(range(0, 10)), 3):
     print(x)
+
+
+
diff --git a/pandas.py b/pandas.py
@@ -6,4 +6,7 @@
 df.columns =df.columns.map(lambda x: str(float(x)))
 
 # Convert a column to String
-df['ColumnID'] = df['ColumnID'].astype(str)
+df['ColumnID'] = df['ColumnID'].astype(str)
+
+# Check for missing values (NaN)
+df.isnull().sum().sum()
diff --git a/py-snippet.py b/py-snippet.py
@@ -599,3 +599,11 @@ def custom_metric():
     logger.error('My code logged an error')
 assert 'My code logged an error' in str(logs)
 
+# Simplest form of batching
+def batch(iterable, n=1):
+    l = len(iterable)
+    for ndx in range(0, l, n):
+        yield iterable[ndx:min(ndx + n, l)]
+
+for x in batch(list(range(0, 10)), 3):
+    print(x)
diff --git a/bash-snippet.sh b/bash-snippet.sh
@@ -191,4 +191,7 @@ set -o xtrace # to revert to normal - set +o xtrace
 lsof -i tcp:<port number> 
 
 #Select random lines from a file
-shuf -n N input > output
+shuf -n N input > output
+
+# Generate a random number between 1-10
+$(( ( RANDOM % 10 )  + 1 ))
diff --git a/bash-snippet.sh b/bash-snippet.sh
@@ -188,4 +188,7 @@ set -o xtrace # to revert to normal - set +o xtrace
 # or bash -x myscript.sh
 
 #list process on port
-lsof -i tcp:<port number> 
+lsof -i tcp:<port number> 
+
+#Select random lines from a file
+shuf -n N input > output
diff --git a/macos-snippets.sh b/macos-snippets.sh
@@ -48,5 +48,5 @@ sudo codesign -f -s - /Library/Frameworks/Python.framework/Versions/3.6/Resource
 # Disable Gatekeeper for installing apps from unidentified devs
 sudo spctl --master-disable
 
-# Install Tree View for listing file structure
-brew install tree
+# Install Tree View for listing file structure, Linux Core Utils
+brew install tree coreutils
diff --git a/macos-snippets.sh b/macos-snippets.sh
@@ -47,3 +47,6 @@ sudo codesign -f -s - /Library/Frameworks/Python.framework/Versions/3.6/Resource
 
 # Disable Gatekeeper for installing apps from unidentified devs
 sudo spctl --master-disable
+
+# Install Tree View for listing file structure
+brew install tree
diff --git a/macos-snippets.sh b/macos-snippets.sh
@@ -45,3 +45,5 @@ caffeinate -u -t <seconds>
 # Get rid of allow incoming connections for Python.app for Pycharm
 sudo codesign -f -s - /Library/Frameworks/Python.framework/Versions/3.6/Resources/Python.app/
 
+# Disable Gatekeeper for installing apps from unidentified devs
+sudo spctl --master-disable
diff --git a/mongo_utils.js b/mongo_utils.js
@@ -2,3 +2,21 @@
 # false for upsert
 # true for multiple documents
 db.<collection_name>.update({"<original_key.h1.h2>": {$exists: true}}, {$rename:{"<original_key.h1.h2>":"<new_name>"}}, false, true);
+
+# Merge two collections
+db.c1.find().forEach(function(item) {
+    db.c2.insert(item);
+    db.c1.remove(item);
+});
+
+# Get Unique keys in a collection
+mr = db.runCommand({
+  "mapreduce" : "my_collection",
+  "map" : function() {
+    for (var key in this) { emit(key, null); }
+  },
+  "reduce" : function(key, stuff) { return null; }, 
+  "out": "my_collection" + "_keys"
+})
+db[mr.result].distinct("_id")
+["foo", "bar", "baz", "_id", ...]
diff --git a/mongo_utils.js b/mongo_utils.js
@@ -0,0 +1,4 @@
+# Rename Fields
+# false for upsert
+# true for multiple documents
+db.<collection_name>.update({"<original_key.h1.h2>": {$exists: true}}, {$rename:{"<original_key.h1.h2>":"<new_name>"}}, false, true);
diff --git a/py-snippet.py b/py-snippet.py
@@ -590,3 +590,12 @@ def custom_metric():
 import tensorflow as tf
 print(tf.metrics)
 
+# Test if a function is printing the required log
+import logging
+from testfixtures import LogCapture
+logger = logging.getLogger('')
+with LogCapture() as logs:
+    # my awesome code
+    logger.error('My code logged an error')
+assert 'My code logged an error' in str(logs)
+
diff --git a/bash-snippet.sh b/bash-snippet.sh
@@ -186,3 +186,6 @@ exit 0
 # To log every command before execution
 set -o xtrace # to revert to normal - set +o xtrace
 # or bash -x myscript.sh
+
+#list process on port
+lsof -i tcp:<port number> 
diff --git a/mac.sh b/mac.sh
@@ -1,2 +0,0 @@
-#list process on port
-lsof -i tcp:<port number> 

diff --git a/macos-snippets.sh b/macos-snippets.sh
@@ -42,3 +42,6 @@ caffeinate
 # Timeout
 caffeinate -u -t <seconds>
 
+# Get rid of allow incoming connections for Python.app for Pycharm
+sudo codesign -f -s - /Library/Frameworks/Python.framework/Versions/3.6/Resources/Python.app/
+
diff --git a/py-snippet.py b/py-snippet.py
@@ -567,3 +567,26 @@ def __getitem__(self, item):
 
 # Check if -ve no in python list
 any(n < 0 for n in any_list)
+
+# Disable import for a function
+from contextlib import contextmanager
+
+@contextmanager
+def custom_metric():
+    import tensorflow
+    t = tensorflow.metrics
+    delattr(tensorflow, "metrics")
+    yield None
+    tensorflow.metrics = t
+
+
+with custom_metric():
+    import tensorflow as tf
+    try:
+        print(tf.metrics)
+    except AttributeError:
+        print("Cannot import metrics inside custom metric")
+
+import tensorflow as tf
+print(tf.metrics)
+
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		# Fix unmet dependencies error
		sudo apt-get -o Dpkg::Options::="--force-overwrite" install --fix-broken
Original file line number	Diff line number	Diff line change
		@@ -607,3 +607,6 @@ def batch(iterable, n=1):

		for x in batch(list(range(0, 10)), 3):
		print(x)
Original file line number	Diff line number	Diff line change
		@@ -1,2 +0,0 @@
		#list process on port
		lsof -i tcp:<port number>