Last active
August 24, 2025 21:32
-
-
Save mikeymckay/9bd6a28c0b5bc21898a95bd2efaa4587 to your computer and use it in GitHub Desktop.
Revisions
-
mikeymckay revised this gist
Aug 24, 2025 . No changes.There are no files selected for viewing
-
mikeymckay revised this gist
Aug 24, 2025 . 1 changed file with 280 additions and 74 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -6,32 +6,110 @@ const KENYA_BBOX = { maxLon: 42.5 // east }; const TARGET_ALBUM_NAME = 'Found Kenya Pics'; // Throttling and pagination controls const MAX_REQUESTS_PER_SECOND = 100; // throttle for getItemInfoExt calls const CHUNK_SLEEP_MS = 500; // delay between chunks within a page const SAFETY_MAX_RESULTS = 500000; // safety bound on total fetched items // LocalStorage keys const LS_KEYS = { resumeMode: 'kenya_resume_mode', // 'processed' | 'added' flushSize: 'kenya_flush_size', // number as string lastProcessedTs: 'kenya_resume_ts_processed', // number as string lastAddedTs: 'kenya_resume_ts_added', // number as string albumMediaKey: 'kenya_album_media_key' // album id for the target album }; // Helpers: localStorage get/set function lsGetNumber(key, fallback = null) { const v = localStorage.getItem(key); if (v == null) return fallback; const n = Number(v); return Number.isFinite(n) ? n : fallback; } function lsGetString(key, fallback = null) { const v = localStorage.getItem(key); return v == null ? fallback : v; } function lsSet(key, val) { if (val == null) return; localStorage.setItem(key, String(val)); } // Settings that can be persisted function getFlushSize() { const persisted = lsGetNumber(LS_KEYS.flushSize, null); return Number.isFinite(persisted) && persisted > 0 ? persisted : 100; // default 100 } function getResumeMode() { const mode = lsGetString(LS_KEYS.resumeMode, 'processed'); // default 'processed' return mode === 'added' ? 'added' : 'processed'; } function getResumeTimestamp() { const mode = getResumeMode(); if (mode === 'added') { const t = lsGetNumber(LS_KEYS.lastAddedTs, null); if (t != null) return t; } // fallback or default return lsGetNumber(LS_KEYS.lastProcessedTs, null); } function setLastProcessedTs(ts) { lsSet(LS_KEYS.lastProcessedTs, ts); } function setLastAddedTs(ts) { lsSet(LS_KEYS.lastAddedTs, ts); } // Coordinate helpers // Updated extractLatLon: handle E7/E6/E5 scaling and bbox-aware swap function extractLatLon(coords) { if (!coords) return null; const scaleToDegrees = (v) => { if (!Number.isFinite(v)) return null; if (Math.abs(v) <= 180) return v; // already degrees const tryScales = [1e7, 1e6, 1e5]; for (const s of tryScales) { const candidate = v / s; if (Math.abs(candidate) <= 180) return candidate; } return v; }; const inLatLonRange = (lat, lon) => Number.isFinite(lat) && Number.isFinite(lon) && Math.abs(lat) <= 90 && Math.abs(lon) <= 180; const chooseWithHeuristic = (lat, lon) => { const candidate = inLatLonRange(lat, lon) ? { lat, lon } : null; const swapped = inLatLonRange(lon, lat) ? { lat: lon, lon: lat } : null; // Prefer the one inside the Kenya bbox, if any if (candidate && isInBBox(candidate, KENYA_BBOX)) return candidate; if (swapped && isInBBox(swapped, KENYA_BBOX)) return swapped; // Otherwise prefer any in-range candidate; keep original ordering if both valid if (candidate) return candidate; if (swapped) return swapped; return null; }; // 1) Array form: [lon, lat] if (Array.isArray(coords) && coords.length >= 2) { const lon = scaleToDegrees(coords[0]); const lat = scaleToDegrees(coords[1]); return chooseWithHeuristic(lat, lon); } // 2) Object form: { latitude, longitude } or { lat, lon } or { lat, lng } const latRaw = coords.latitude ?? coords.lat; const lonRaw = coords.longitude ?? coords.lon ?? coords.lng; const lat = scaleToDegrees(latRaw); const lon = scaleToDegrees(lonRaw); return chooseWithHeuristic(lat, lon); } function isInBBox({ lat, lon }, bbox) { @@ -47,68 +125,196 @@ function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } // Album helpers using gptkApiUtils async function findAlbumByMediaKey(mediaKey) { const albums = (await gptkApi.getAlbums()).items; return albums.find(a => a.mediaKey === mediaKey) || null; } async function findAlbumByTitle(title) { const albums = (await gptkApi.getAlbums()).items; // If multiple have the same title, just pick the first. You can refine this if needed. return albums.find(a => a.title === title) || null; } async function ensureTargetAlbumAndExistingSet() { // Try by persisted mediaKey first let album = null; const persistedKey = lsGetString(LS_KEYS.albumMediaKey, null); if (persistedKey) { album = await findAlbumByMediaKey(persistedKey); } // If not found, try by title if (!album) { album = await findAlbumByTitle(TARGET_ALBUM_NAME); } // If still not found, create it (with an empty add) if (!album) { console.log(`Album "${TARGET_ALBUM_NAME}" not found. Creating...`); await gptkApiUtils.addToNewAlbum([], TARGET_ALBUM_NAME); // Re-fetch by title album = await findAlbumByTitle(TARGET_ALBUM_NAME); if (!album) { throw new Error(`Failed to create or locate album "${TARGET_ALBUM_NAME}"`); } } // Persist album mediaKey for future runs lsSet(LS_KEYS.albumMediaKey, album.mediaKey); // Load existing items to dedupe adds across runs const existingItems = await gptkApiUtils.getAllMediaInAlbum(album.mediaKey); const existingMediaKeySet = new Set((existingItems || []).map(it => it.mediaKey)); console.log(`Using album "${album.title}" (mediaKey=${album.mediaKey}) with ${existingMediaKeySet.size} existing items`); return { album, existingMediaKeySet }; } (async () => { const FLUSH_BATCH_SIZE = getFlushSize(); const RESUME_MODE = getResumeMode(); const filteredBuffer = []; // pending Kenya items not yet flushed let { album, existingMediaKeySet } = await ensureTargetAlbumAndExistingSet(); // Dedupe within this run const seenThisRun = new Set(); let nextPageId = null; let totalSeen = 0; // Determine resume timestamp let resumeTs = getResumeTimestamp(); console.log( resumeTs != null ? `Resuming from ${RESUME_MODE} timestamp: ${resumeTs} (${new Date(resumeTs).toISOString()})` : 'Starting from the beginning (no resume timestamp found)' ); async function flushToAlbum(force = false) { // Build a batch to add if (filteredBuffer.length >= FLUSH_BATCH_SIZE || (force && filteredBuffer.length > 0)) { const countToAdd = force ? filteredBuffer.length : FLUSH_BATCH_SIZE; const candidate = filteredBuffer.splice(0, countToAdd); // Dedupe against existing album and this run const finalBatch = []; for (const item of candidate) { if (!item || !item.mediaKey) continue; if (existingMediaKeySet.has(item.mediaKey)) continue; if (seenThisRun.has(item.mediaKey)) continue; seenThisRun.add(item.mediaKey); finalBatch.push(item); } if (finalBatch.length === 0) { console.log(`No new unique items to add in this batch (requested ${countToAdd}).`); return; } // Add to existing album to avoid creating duplicates await gptkApiUtils.addToExistingAlbum(finalBatch, album, /* preserveOrder */ false); // Update existing set for future dedupe for (const it of finalBatch) { existingMediaKeySet.add(it.mediaKey); } // Persist and print the last added timestamp const lastAddedTs = finalBatch[finalBatch.length - 1].timestamp; setLastAddedTs(lastAddedTs); console.log(`Added ${finalBatch.length} items to album "${album.title}". Last added timestamp: ${lastAddedTs} (${new Date(lastAddedTs).toISOString()})`); } } gptkCore.isProcessRunning = true; try { do { const page = await gptkApi.getItemsByTakenDate( /* timestamp = */ resumeTs ?? null, /* source = */ null, /* pageId = */ nextPageId ); const items = page?.items || []; totalSeen += items.length; if (items.length > 0) { const lastOnPageTs = items[items.length - 1].timestamp; console.log(`Fetched ${items.length} items; total seen: ${totalSeen}; last photo ts on page: ${lastOnPageTs} (${new Date(lastOnPageTs).toISOString()})`); } else { console.log('Fetched page with 0 items.'); } // Process items in throttled batches for (let i = 0; i < items.length; i += MAX_REQUESTS_PER_SECOND) { if (!gptkCore.isProcessRunning) break; const chunk = items.slice(i, i + MAX_REQUESTS_PER_SECOND); const promises = chunk.map(async (item) => { try { const ext = await gptkApi.getItemInfoExt(item.mediaKey); const coords = extractLatLon(ext?.geoLocation?.coordinates); if (!coords) return null; if (!isInBBox(coords, KENYA_BBOX)) return null; return item; } catch (err) { console.error(`Error processing item ${item.mediaKey}:`, err); return null; } }); const resolved = await Promise.all(promises); for (const it of resolved) { if (it) { filteredBuffer.push(it); await flushToAlbum(false); // flush as soon as we reach the configured size } } if (i + MAX_REQUESTS_PER_SECOND < items.length) { await sleep(CHUNK_SLEEP_MS); } } // After processing this page, persist last processed timestamp if (items.length > 0) { const lastProcessedTs = items[items.length - 1].timestamp; setLastProcessedTs(lastProcessedTs); // For next page calls, keep using the same resumeTs but advance via nextPageId. // If we restart mid-run, we will resume from lastProcessedTs. } nextPageId = page?.nextPageId; if (totalSeen >= SAFETY_MAX_RESULTS) { console.warn(`Stopping due to SAFETY_MAX_RESULTS=${SAFETY_MAX_RESULTS}`); break; } } while (nextPageId); // Final flush of any remaining items await flushToAlbum(true); // Summary logs const persistedProcessed = lsGetNumber(LS_KEYS.lastProcessedTs, null); const persistedAdded = lsGetNumber(LS_KEYS.lastAddedTs, null); console.log( persistedProcessed != null ? `Saved last processed timestamp: ${persistedProcessed} (${new Date(persistedProcessed).toISOString()})` : 'No processed timestamp saved.' ); console.log( persistedAdded != null ? `Saved last added timestamp: ${persistedAdded} (${new Date(persistedAdded).toISOString()})` : 'No added timestamp saved (no items were added).' ); } catch (e) { console.error('Fatal error:', e); } finally { gptkCore.isProcessRunning = false; } console.log('DONE'); })(); -
mikeymckay revised this gist
Aug 22, 2025 . 1 changed file with 3 additions and 2 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -62,7 +62,8 @@ try { // Process items in batches throttled to MAX_REQUESTS_PER_SECOND const items = page.items; // Items to process index += page.items.length console.log(index + ", last photo timestamp:" + page.items[page.items.length-1].timestamp); for (let i = 0; i < items.length; i += MAX_REQUESTS_PER_SECOND) { const chunk = items.slice(i, i + MAX_REQUESTS_PER_SECOND); // 10 items per batch @@ -94,7 +95,7 @@ try { // Wait for 1 second before processing the next batch if (i + MAX_REQUESTS_PER_SECOND < items.length) { await sleep(500); // 1-second delay } } -
mikeymckay revised this gist
Aug 22, 2025 . 1 changed file with 44 additions and 11 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -1,5 +1,3 @@ // Define a rectangle that fully contains Kenya (with a small buffer). const KENYA_BBOX = { minLat: -5.0, // south @@ -10,6 +8,7 @@ const KENYA_BBOX = { const newAlbumName = 'Kenya'; const filteredItems = []; const MAX_REQUESTS_PER_SECOND = 100; // Limit to 10 requests per second function extractLatLon(coords) { if (!coords) return null; @@ -44,25 +43,59 @@ function isInBBox({ lat, lon }, bbox) { ); } function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } let nextPageId = null; gptkCore.isProcessRunning = true; index = 0; try { do { const page = await gptkApi.getItemsByTakenDate( /* timestamp = */ null, /* source = */ null, /* pageId = */ nextPageId ); // Process items in batches throttled to MAX_REQUESTS_PER_SECOND const items = page.items; // Items to process console.log(index += page.items.length); for (let i = 0; i < items.length; i += MAX_REQUESTS_PER_SECOND) { const chunk = items.slice(i, i + MAX_REQUESTS_PER_SECOND); // 10 items per batch // Map the requests for the current batch const promises = chunk.map(async (item) => { try { const extended_info_item = await gptkApi.getItemInfoExt(item.mediaKey); const coords = extractLatLon(extended_info_item?.geoLocation?.coordinates); if (!coords) return null; // skip if no coordinates if (!isInBBox(coords, KENYA_BBOX)) return null; // skip if outside Kenya bbox return item; // Pass valid item } catch (error) { console.error(`Error processing item with mediaKey ${item.mediaKey}:`, error); return null; // Skip failed items } }); // Wait for all the promises in the current batch to resolve const resolvedItems = await Promise.all(promises); // Add successfully processed items to the filtered list resolvedItems.forEach((item) => { if (item) { filteredItems.push(item); console.log("# Kenya Pictures:" + filteredItems.length); } }); // Wait for 1 second before processing the next batch if (i + MAX_REQUESTS_PER_SECOND < items.length) { await sleep(1000); // 1-second delay } } nextPageId = page.nextPageId; -
mikeymckay revised this gist
Aug 14, 2025 . 1 changed file with 2 additions and 1 deletion.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -57,7 +57,8 @@ try { console.log(index+=1); for (const item of page.items) { console.log("."); extended_info_item = await gptkApi.getItemInfoExt(item.mediaKey) const coords = extractLatLon(extended_info_item?.geoLocation?.coordinates); if (!coords) continue; // skip if no coordinates if (!isInBBox(coords, KENYA_BBOX)) continue; // skip if outside Kenya bbox filteredItems.push(item); -
mikeymckay revised this gist
Aug 13, 2025 . 1 changed file with 2 additions and 0 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -1,3 +1,5 @@ //albumPage = await gptkApi.getAlbumPage(a.items.filter(a => a.title == "Kenya Wall")[0].mediaKey) // Define a rectangle that fully contains Kenya (with a small buffer). const KENYA_BBOX = { minLat: -5.0, // south -
mikeymckay revised this gist
Aug 12, 2025 . 1 changed file with 1 addition and 1 deletion.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -52,7 +52,7 @@ try { /* source = */ null, /* pageId = */ nextPageId ); console.log(index+=1); for (const item of page.items) { console.log("."); const coords = extractLatLon(item?.geoLocation?.coordinates); -
mikeymckay revised this gist
Aug 12, 2025 . 1 changed file with 2 additions and 2 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -44,15 +44,15 @@ function isInBBox({ lat, lon }, bbox) { let nextPageId = null; gptkCore.isProcessRunning = true; index = 0; try { do { const page = await gptkApi.getItemsByTakenDate( /* timestamp = */ null, /* source = */ null, /* pageId = */ nextPageId ); console.log(index); for (const item of page.items) { console.log("."); const coords = extractLatLon(item?.geoLocation?.coordinates); -
mikeymckay revised this gist
Aug 12, 2025 . 1 changed file with 2 additions and 2 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -59,11 +59,11 @@ try { if (!coords) continue; // skip if no coordinates if (!isInBBox(coords, KENYA_BBOX)) continue; // skip if outside Kenya bbox filteredItems.push(item); console.log(filteredItems.length); } nextPageId = page.nextPageId; } while (nextPageId && filteredItems.length < 20000); if (filteredItems.length > 0) { await gptkApiUtils.addToNewAlbum(filteredItems, /* albumName = */ newAlbumName); -
mikeymckay revised this gist
Aug 12, 2025 . 1 changed file with 1 addition and 1 deletion.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -63,7 +63,7 @@ try { } nextPageId = page.nextPageId; } while (nextPageId && filterItems.length < 20000); if (filteredItems.length > 0) { await gptkApiUtils.addToNewAlbum(filteredItems, /* albumName = */ newAlbumName); -
mikeymckay created this gist
Aug 12, 2025 .There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -0,0 +1,77 @@ // Define a rectangle that fully contains Kenya (with a small buffer). const KENYA_BBOX = { minLat: -5.0, // south maxLat: 5.5, // north minLon: 33.5, // west maxLon: 42.5 // east }; const newAlbumName = 'Kenya'; const filteredItems = []; function extractLatLon(coords) { if (!coords) return null; // Common patterns: // 1) [lon, lat] if (Array.isArray(coords) && coords.length >= 2) { const [lon, lat] = coords; if (Number.isFinite(lat) && Number.isFinite(lon)) return { lat, lon }; } // 2) { latitude, longitude } or { lat, lon } or { lat, lng } const lat = coords.latitude ?? coords.lat; const lon = coords.longitude ?? coords.lon ?? coords.lng; if (Number.isFinite(lat) && Number.isFinite(lon)) return { lat, lon }; return null; } function isInBBox({ lat, lon }, bbox) { return ( lat >= bbox.minLat && lat <= bbox.maxLat && lon >= bbox.minLon && lon <= bbox.maxLon ); } let nextPageId = null; gptkCore.isProcessRunning = true; try { do { const page = await gptkApi.getItemsByTakenDate( /* timestamp = */ null, /* source = */ null, /* pageId = */ nextPageId ); console.log("\n"); for (const item of page.items) { console.log("."); const coords = extractLatLon(item?.geoLocation?.coordinates); if (!coords) continue; // skip if no coordinates if (!isInBBox(coords, KENYA_BBOX)) continue; // skip if outside Kenya bbox filteredItems.push(item); console.log(filterItems.length); } nextPageId = page.nextPageId; } while (nextPageId and filterItems.length < 20000); if (filteredItems.length > 0) { await gptkApiUtils.addToNewAlbum(filteredItems, /* albumName = */ newAlbumName); } else { console.warn('No items found within the Kenya bounding box.'); } } finally { gptkCore.isProcessRunning = false; } console.log('DONE');