Skip to content

Instantly share code, notes, and snippets.

@matthewdeanmartin

matthewdeanmartin/randomish_gpt5.py

Created August 25, 2025 02:06
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save matthewdeanmartin/d46f4bbdf81306f6a13f5b6e6b20e7a6 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save matthewdeanmartin/d46f4bbdf81306f6a13f5b6e6b20e7a6 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
randomish_gpt5.py
# Update: fix the unbiased bit extractor to use *disjoint* pairs of independent samples,
# which makes the 4-bit blocks uniform (so digits 0..9 after rejection are uniform).
# Re-run the whole pipeline and tests.
import hashlib
import math
from collections import Counter
from dataclasses import dataclass
from typing import Iterable, List, Sequence, Tuple
BASE_TEXT = """
Alice was beginning to get very tired of sitting by her sister on the bank, and of having nothing to do:
once or twice she had peeped into the book her sister was reading, but it had no pictures or conversations
in it, “and what is the use of a book,” thought Alice “without pictures or conversations?”
So she was considering in her own mind (as well as she could, for the hot day made her feel very sleepy
and stupid), whether the pleasure of making a daisy-chain would be worth the trouble of getting up and
picking the daisies, when suddenly a White Rabbit with pink eyes ran close by her.
There was nothing so very remarkable in that; nor did Alice think it so very much out of the way to hear the
Rabbit say to itself, “Oh dear! Oh dear! I shall be late!” (when she thought it over afterwards, it occurred
to her that she ought to have wondered at this, but at the time it all seemed quite natural); but when the
Rabbit actually took a watch out of its waistcoat-pocket, and looked at it, and then hurried on, Alice started
to her feet, for it flashed across her mind that she had never before seen a rabbit with either a waistcoat-pocket,
or a watch to take out of it, and burning with curiosity, she ran across the field after it, and fortunately
was just in time to see it pop down a large rabbit-hole under the hedge.
"""
TEXT = (BASE_TEXT.strip() + "\n") * 200
def _sha256_int(data: bytes) -> int:
return int.from_bytes(hashlib.sha256(data).digest(), "big")
def keyed_index_stream(n: int, key: str) -> Iterable[int]:
if n <= 0:
raise ValueError("n must be positive")
k = key.encode("utf-8")
counter = 0
buf: List[int] = []
while True:
if not buf:
h = hashlib.sha256(counter.to_bytes(16, "big") + b"|" + k).digest()
for j in range(0, 32, 4):
chunk = int.from_bytes(h[j:j+4], "big")
buf.append(chunk)
counter += 1
val = buf.pop() % n
yield val
def words_from_text(text: str) -> List[str]:
import re
return re.findall(r"[A-Za-z']+", text)
def letters_from_text(text: str) -> List[str]:
import re
return re.findall(r"[A-Za-z]", text)
def word_length(word: str) -> int:
return sum(1 for ch in word if ch.isalpha())
def letter_index(letter: str) -> int:
return ord(letter.upper()) - ord("A")
def unbiased_bits_from_stream_disjoint(values: Sequence[int], key: str, needed_bits: int) -> List[int]:
"""
Produce unbiased bits using *disjoint* pairs (v1, v2), (v3, v4), ... sampled by a keyed index stream.
If v1 > v2 -> 1, if v2 > v1 -> 0, if equal -> discard the pair.
"""
bits: List[int] = []
n = len(values)
idx_stream = keyed_index_stream(n, key)
while len(bits) < needed_bits:
i1 = next(idx_stream)
i2 = next(idx_stream)
v1, v2 = values[i1], values[i2]
if v1 == v2:
continue
bits.append(1 if v1 > v2 else 0)
return bits
def bits_to_digits_0_9(bits: Sequence[int]) -> List[int]:
out: List[int] = []
for i in range(0, len(bits) - 3, 4):
v = (bits[i] << 3) | (bits[i+1] << 2) | (bits[i+2] << 1) | bits[i+3]
if v < 10:
out.append(v)
return out
def derive_digit_permutation(key: str) -> List[int]:
digits = list(range(10))
stream: List[int] = []
i = 0
while len(stream) < 64:
stream.append(_sha256_int((key + f"|perm|{i}").encode("utf-8")) & 0xFFFFFFFF)
i += 1
j_stream = iter(stream)
for j in range(9, 0, -1):
r = next(j_stream)
k = r % (j + 1)
digits[j], digits[k] = digits[k], digits[j]
return digits
def apply_permutation(digits: Sequence[int], perm: Sequence[int]) -> List[int]:
mapping = {i: perm[i] for i in range(10)}
return [mapping[d] for d in digits]
@dataclass
class GenerationResult:
name: str
key_used: str
perm: List[int]
digits: List[int]
def word_length_walk_digits(text: str, key: str, target_digits: int = 1000) -> GenerationResult:
words = words_from_text(text)
vals = [word_length(w) for w in words]
needed_bits = int(target_digits * 4 * 1.4) # smaller multiplier now that pairs are disjoint
bits = unbiased_bits_from_stream_disjoint(vals, key + "|WL", needed_bits)
raw_digits = bits_to_digits_0_9(bits)
while len(raw_digits) < target_digits:
# top up if needed
bits += unbiased_bits_from_stream_disjoint(vals, key + f"|WL+{len(bits)}", needed_bits // 2 + 16)
raw_digits = bits_to_digits_0_9(bits)
raw_digits = raw_digits[:target_digits]
perm = derive_digit_permutation(key + "|perm")
masked = apply_permutation(raw_digits, perm)
return GenerationResult("word_length_walk", key, perm, masked)
def letter_index_walk_digits(text: str, key: str, target_digits: int = 1000) -> GenerationResult:
letters = letters_from_text(text)
vals = [letter_index(ch) for ch in letters]
needed_bits = int(target_digits * 4 * 1.4)
bits = unbiased_bits_from_stream_disjoint(vals, key + "|LET", needed_bits)
raw_digits = bits_to_digits_0_9(bits)
while len(raw_digits) < target_digits:
bits += unbiased_bits_from_stream_disjoint(vals, key + f"|LET+{len(bits)}", needed_bits // 2 + 16)
raw_digits = bits_to_digits_0_9(bits)
raw_digits = raw_digits[:target_digits]
perm = derive_digit_permutation(key + "|perm")
masked = apply_permutation(raw_digits, perm)
return GenerationResult("crib_card_letter_walk", key, perm, masked)
def normal_cdf(z: float) -> float:
return 0.5 * (1.0 + math.erf(z / math.sqrt(2.0)))
def chi_square_test_freq(digits: Sequence[int], k: int = 10) -> Tuple[float, int, float]:
n = len(digits)
counts = Counter(digits)
expected = n / k
chi2 = sum((counts[i] - expected) ** 2 / expected for i in range(k))
df = k - 1
if chi2 <= 0:
p = 1.0
else:
c = (chi2 / df) ** (1 / 3)
mu = 1 - 2 / (9 * df)
sigma = math.sqrt(2 / (9 * df))
z = (c - mu) / sigma
p = 1 - normal_cdf(z)
return chi2, df, p
def adjacent_equal_test(digits: Sequence[int]) -> Tuple[int, float, float]:
n = len(digits) - 1
matches = sum(1 for i in range(n) if digits[i] == digits[i + 1])
p = 0.1
mean = n * p
var = n * p * (1 - p)
if var == 0:
return matches, 0.0, 1.0
z = (matches - mean) / math.sqrt(var)
p_two = 2 * (1 - normal_cdf(abs(z)))
return matches, z, p_two
def runs_test_above_below_median(digits: Sequence[int]) -> Tuple[int, float, float]:
median = 4.5
seq = [1 if d > median else 0 for d in digits]
runs = 1
for i in range(1, len(seq)):
if seq[i] != seq[i - 1]:
runs += 1
n1 = sum(seq)
n0 = len(seq) - n1
if n1 == 0 or n0 == 0:
return runs, 0.0, 1.0
N = n1 + n0
mu = 1 + (2 * n1 * n0) / N
var = (2 * n1 * n0 * (2 * n1 * n0 - N)) / (N ** 2 * (N - 1))
if var <= 0:
return runs, 0.0, 1.0
z = (runs - mu) / math.sqrt(var)
p_two = 2 * (1 - normal_cdf(abs(z)))
return runs, z, p_two
def serial_correlation_test(digits: Sequence[int]) -> Tuple[float, float]:
n = len(digits)
xbar = sum(digits) / n
num = sum((digits[i] - xbar) * (digits[i + 1] - xbar) for i in range(n - 1))
den = sum((d - xbar) ** 2 for d in digits)
if den == 0:
return 0.0, 1.0
r = num / den
if n <= 3:
return r, 1.0
t = r * math.sqrt((n - 2) / (1 - r ** 2 + 1e-12))
p_two = 2 * (1 - normal_cdf(abs(t)))
return r, p_two
def summarize_tests(digits: Sequence[int]) -> dict:
counts = Counter(digits)
chi2, df, p_chi = chi_square_test_freq(digits)
matches, z_adj, p_adj = adjacent_equal_test(digits)
runs, z_runs, p_runs = runs_test_above_below_median(digits)
r, p_r = serial_correlation_test(digits)
return {
"n": len(digits),
"counts": {k: counts.get(k, 0) for k in range(10)},
"chi_square": round(chi2, 3),
"chi_df": df,
"chi_p_approx": round(p_chi, 4),
"adjacent_equal": matches,
"adjacent_equal_z": round(z_adj, 3),
"adjacent_equal_p_approx": round(p_adj, 4),
"runs": runs,
"runs_z": round(z_runs, 3),
"runs_p_approx": round(p_runs, 4),
"lag1_r": round(r, 4),
"lag1_p_approx": round(p_r, 4),
}
WORD_KEY = "isopod-walk-key-1.1"
CRIB_KEY = "isopod-crib-key-1.1"
res_word = word_length_walk_digits(TEXT, WORD_KEY, target_digits=1000)
res_crib = letter_index_walk_digits(TEXT, CRIB_KEY, target_digits=1000)
tests_word = summarize_tests(res_word.digits)
tests_crib = summarize_tests(res_crib.digits)
from pathlib import Path
out1 = Path("/mnt/data/word_length_digits.txt")
out2 = Path("/mnt/data/crib_card_digits.txt")
out1.write_text("".join(map(str, res_word.digits)))
out2.write_text("".join(map(str, res_crib.digits)))
print("Word-length walk (masked) — first 100 digits:\n", "".join(map(str, res_word.digits[:100])))
print("\nCrib-card letter walk (masked) — first 100 digits:\n", "".join(map(str, res_crib.digits[:100])))
print("\n=== Tests: word-length walk ===")
for k, v in tests_word.items():
print(f"{k}: {v}")
print("\n=== Tests: crib-card letter walk ===")
for k, v in tests_crib.items():
print(f"{k}: {v}")
print("\nSaved files:")
print(out1)
print(out2)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment