Skip to content

Instantly share code, notes, and snippets.

@xflr6

xflr6/walk_gdrive.py

Last active October 11, 2025 16:03
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save xflr6/d106aa5b561fbac4ce1a9969eba728bb to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save xflr6/d106aa5b561fbac4ce1a9969eba728bb to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. xflr6 revised this gist Oct 11, 2025. 1 changed file with 13 additions and 3 deletions.
    16 changes: 13 additions & 3 deletions walk_gdrive.py
    View file
    Original file line number Diff line number Diff line change
    @@ -1,8 +1,9 @@
    """os.walk() variation with Google Drive API v3."""

    from collections.abc import Sequence
    from collections.abc import Iterator, Sequence
    import os
    import pathlib
    from typing import TypedDict

    # $ pip install google-api-python-client google-auth-oauthlib
    from apiclient import discovery
    @@ -43,7 +44,7 @@ def get_credentials(scopes: Sequence[str], *,
    def iterfiles(name: str | None = None, *,
    is_folder: bool | None = None,
    parent: str | None = None,
    order_by: str = 'folder,name,createdTime'):
    order_by: str = 'folder,name,createdTime') -> Iterator['File']:
    q = []
    if name is not None:
    q.append("name = '{}'".format(name.replace("'", "\\'")))
    @@ -66,7 +67,16 @@ def iterfiles(name: str | None = None, *,
    return


    def walk(top: str = 'root', *, by_name: bool = False):
    class File(TypedDict):
    id: str
    kind: str
    name: str
    mimeType: str
    resourceKey: str


    def walk(top: str = 'root', *,
    by_name: bool = False) -> Iterator[tuple[str, list[File], list[File]]]:
    if by_name:
    (top,) = iterfiles(name=top, is_folder=True)
    else:
  2. xflr6 revised this gist Oct 11, 2025. 1 changed file with 29 additions and 16 deletions.
    45 changes: 29 additions & 16 deletions walk_gdrive.py
    View file
    Original file line number Diff line number Diff line change
    @@ -1,31 +1,43 @@
    """os.walk() variation with Google Drive API."""
    """os.walk() variation with Google Drive API v3."""

    from collections.abc import Sequence
    import os
    import pathlib

    from apiclient.discovery import build # pip install google-api-python-client
    # $ pip install google-api-python-client google-auth-oauthlib
    from apiclient import discovery
    from google.oauth2 import credentials
    from google_auth_oauthlib import flow as flow_lib

    SCOPES = ['https://www.googleapis.com/auth/drive.metadata.readonly']

    FOLDER = 'application/vnd.google-apps.folder'


    def get_credentials(scopes: Sequence[str], *,
    secrets: os.PathLike[str] | str = '~/client_secrets.json',
    storage: os.PathLike[str] | str = '~/storage.json'):
    from oauth2client import file, client, tools

    store = file.Storage(os.path.expanduser(storage))
    creds = store.get()

    if creds is None or creds.invalid:
    flow = client.flow_from_clientsecrets(os.path.expanduser(secrets), scopes)
    flags = tools.argparser.parse_args([])
    creds = tools.run_flow(flow, store, flags)
    storage: os.PathLike[str] | str | None = '~/authorized_user.json'
    ) -> credentials.Credentials:
    creds = None
    if storage is not None:
    storage = pathlib.Path(storage).expanduser()
    if storage.exists():
    creds = credentials.Credentials.from_authorized_user_file(storage, scopes=scopes)

    if creds is None or creds.token_state.name == 'INVALID':
    secrets = pathlib.Path(secrets).expanduser()
    flow = flow_lib.InstalledAppFlow.from_client_secrets_file(secrets, scopes=scopes)
    flow.run_local_server()
    creds = flow.credentials
    if storage is not None:
    authorized_user_info = creds.to_json()
    storage.write_text(authorized_user_info)
    return creds


    creds = get_credentials(scopes=['https://www.googleapis.com/auth/drive.metadata.readonly'])
    creds = get_credentials(SCOPES)

    service = build('drive', version='v3', credentials=creds)
    service = discovery.build('drive', version='v3', credentials=creds)


    def iterfiles(name: str | None = None, *,
    @@ -64,7 +76,7 @@ def walk(top: str = 'root', *, by_name: bool = False):

    stack = [((top['name'],), top)]
    while stack:
    path, top = stack.pop()
    (path, top) = stack.pop()

    (dirs, files) = is_file = ([], [])
    for f in iterfiles(parent=top['id']):
    @@ -77,5 +89,6 @@ def walk(top: str = 'root', *, by_name: bool = False):


    for kwargs in [{'top': 'spam', 'by_name': True}, {}]:
    print('', f'walk(**{kwargs!r})', sep='\n')
    for path, root, dirs, files in walk(**kwargs):
    print('/'.join(path), f'{len(dirs):d} {len(files):d}', sep='\t')
    print('/'.join(path), f'{len(dirs):d}', f'{len(files):d}', sep='\t')
  3. xflr6 revised this gist Oct 4, 2025. 1 changed file with 11 additions and 8 deletions.
    19 changes: 11 additions & 8 deletions walk_gdrive.py
    View file
    Original file line number Diff line number Diff line change
    @@ -1,15 +1,16 @@
    """os.walk() variation with Google Drive API."""

    from collections.abc import Sequence
    import os

    from apiclient.discovery import build # pip install google-api-python-client

    FOLDER = 'application/vnd.google-apps.folder'


    def get_credentials(scopes, *,
    secrets='~/client_secrets.json',
    storage='~/storage.json'):
    def get_credentials(scopes: Sequence[str], *,
    secrets: os.PathLike[str] | str = '~/client_secrets.json',
    storage: os.PathLike[str] | str = '~/storage.json'):
    from oauth2client import file, client, tools

    store = file.Storage(os.path.expanduser(storage))
    @@ -27,8 +28,10 @@ def get_credentials(scopes, *,
    service = build('drive', version='v3', credentials=creds)


    def iterfiles(name=None, *, is_folder=None, parent=None,
    order_by='folder,name,createdTime'):
    def iterfiles(name: str | None = None, *,
    is_folder: bool | None = None,
    parent: str | None = None,
    order_by: str = 'folder,name,createdTime'):
    q = []
    if name is not None:
    q.append("name = '{}'".format(name.replace("'", "\\'")))
    @@ -51,9 +54,9 @@ def iterfiles(name=None, *, is_folder=None, parent=None,
    return


    def walk(top='root', *, by_name: bool = False):
    def walk(top: str = 'root', *, by_name: bool = False):
    if by_name:
    top, = iterfiles(name=top, is_folder=True)
    (top,) = iterfiles(name=top, is_folder=True)
    else:
    top = service.files().get(fileId=top).execute()
    if top['mimeType'] != FOLDER:
    @@ -63,7 +66,7 @@ def walk(top='root', *, by_name: bool = False):
    while stack:
    path, top = stack.pop()

    dirs, files = is_file = [], []
    (dirs, files) = is_file = ([], [])
    for f in iterfiles(parent=top['id']):
    is_file[f['mimeType'] != FOLDER].append(f)

  4. xflr6 revised this gist Jun 4, 2022. 1 changed file with 12 additions and 1 deletion.
    13 changes: 12 additions & 1 deletion walk_gdrive.py
    View file
    Original file line number Diff line number Diff line change
    @@ -7,16 +7,21 @@
    FOLDER = 'application/vnd.google-apps.folder'


    def get_credentials(scopes, *, secrets='~/client_secrets.json', storage='~/storage.json'):
    def get_credentials(scopes, *,
    secrets='~/client_secrets.json',
    storage='~/storage.json'):
    from oauth2client import file, client, tools

    store = file.Storage(os.path.expanduser(storage))
    creds = store.get()

    if creds is None or creds.invalid:
    flow = client.flow_from_clientsecrets(os.path.expanduser(secrets), scopes)
    flags = tools.argparser.parse_args([])
    creds = tools.run_flow(flow, store, flags)
    return creds


    creds = get_credentials(scopes=['https://www.googleapis.com/auth/drive.metadata.readonly'])

    service = build('drive', version='v3', credentials=creds)
    @@ -31,9 +36,11 @@ def iterfiles(name=None, *, is_folder=None, parent=None,
    q.append("mimeType {} '{}'".format('=' if is_folder else '!=', FOLDER))
    if parent is not None:
    q.append("'{}' in parents".format(parent.replace("'", "\\'")))

    params = {'pageToken': None, 'orderBy': order_by}
    if q:
    params['q'] = ' and '.join(q)

    while True:
    response = service.files().list(**params).execute()
    for f in response['files']:
    @@ -51,13 +58,17 @@ def walk(top='root', *, by_name: bool = False):
    top = service.files().get(fileId=top).execute()
    if top['mimeType'] != FOLDER:
    raise ValueError(f'not a folder: {top!r}')

    stack = [((top['name'],), top)]
    while stack:
    path, top = stack.pop()

    dirs, files = is_file = [], []
    for f in iterfiles(parent=top['id']):
    is_file[f['mimeType'] != FOLDER].append(f)

    yield path, top, dirs, files

    if dirs:
    stack.extend((path + (d['name'],), d) for d in reversed(dirs))

  5. xflr6 revised this gist Dec 18, 2021. 1 changed file with 5 additions and 1 deletion.
    6 changes: 5 additions & 1 deletion walk_gdrive.py
    View file
    Original file line number Diff line number Diff line change
    @@ -1,11 +1,12 @@
    # walk_gdrive.py - os.walk variation with Google Drive API
    """os.walk() variation with Google Drive API."""

    import os

    from apiclient.discovery import build # pip install google-api-python-client

    FOLDER = 'application/vnd.google-apps.folder'


    def get_credentials(scopes, *, secrets='~/client_secrets.json', storage='~/storage.json'):
    from oauth2client import file, client, tools
    store = file.Storage(os.path.expanduser(storage))
    @@ -20,6 +21,7 @@ def get_credentials(scopes, *, secrets='~/client_secrets.json', storage='~/stora

    service = build('drive', version='v3', credentials=creds)


    def iterfiles(name=None, *, is_folder=None, parent=None,
    order_by='folder,name,createdTime'):
    q = []
    @@ -41,6 +43,7 @@ def iterfiles(name=None, *, is_folder=None, parent=None,
    except KeyError:
    return


    def walk(top='root', *, by_name: bool = False):
    if by_name:
    top, = iterfiles(name=top, is_folder=True)
    @@ -58,6 +61,7 @@ def walk(top='root', *, by_name: bool = False):
    if dirs:
    stack.extend((path + (d['name'],), d) for d in reversed(dirs))


    for kwargs in [{'top': 'spam', 'by_name': True}, {}]:
    for path, root, dirs, files in walk(**kwargs):
    print('/'.join(path), f'{len(dirs):d} {len(files):d}', sep='\t')
  6. xflr6 revised this gist Aug 1, 2021. 1 changed file with 5 additions and 5 deletions.
    10 changes: 5 additions & 5 deletions walk_gdrive.py
    View file
    Original file line number Diff line number Diff line change
    @@ -24,11 +24,11 @@ def iterfiles(name=None, *, is_folder=None, parent=None,
    order_by='folder,name,createdTime'):
    q = []
    if name is not None:
    q.append("name = '%s'" % name.replace("'", "\\'"))
    q.append("name = '{}'".format(name.replace("'", "\\'")))
    if is_folder is not None:
    q.append("mimeType %s '%s'" % ('=' if is_folder else '!=', FOLDER))
    q.append("mimeType {} '{}'".format('=' if is_folder else '!=', FOLDER))
    if parent is not None:
    q.append("'%s' in parents" % parent.replace("'", "\\'"))
    q.append("'{}' in parents".format(parent.replace("'", "\\'")))
    params = {'pageToken': None, 'orderBy': order_by}
    if q:
    params['q'] = ' and '.join(q)
    @@ -47,7 +47,7 @@ def walk(top='root', *, by_name: bool = False):
    else:
    top = service.files().get(fileId=top).execute()
    if top['mimeType'] != FOLDER:
    raise ValueError('not a folder: %r' % top)
    raise ValueError(f'not a folder: {top!r}')
    stack = [((top['name'],), top)]
    while stack:
    path, top = stack.pop()
    @@ -60,4 +60,4 @@ def walk(top='root', *, by_name: bool = False):

    for kwargs in [{'top': 'spam', 'by_name': True}, {}]:
    for path, root, dirs, files in walk(**kwargs):
    print('%s\t%d %d' % ('/'.join(path), len(dirs), len(files)))
    print('/'.join(path), f'{len(dirs):d} {len(files):d}', sep='\t')
  7. xflr6 revised this gist Aug 1, 2021. 1 changed file with 5 additions and 3 deletions.
    8 changes: 5 additions & 3 deletions walk_gdrive.py
    View file
    Original file line number Diff line number Diff line change
    @@ -6,7 +6,7 @@

    FOLDER = 'application/vnd.google-apps.folder'

    def get_credentials(scopes, secrets='~/client_secrets.json', storage='~/storage.json'):
    def get_credentials(scopes, *, secrets='~/client_secrets.json', storage='~/storage.json'):
    from oauth2client import file, client, tools
    store = file.Storage(os.path.expanduser(storage))
    creds = store.get()
    @@ -17,9 +17,11 @@ def get_credentials(scopes, secrets='~/client_secrets.json', storage='~/storage.
    return creds

    creds = get_credentials(scopes=['https://www.googleapis.com/auth/drive.metadata.readonly'])

    service = build('drive', version='v3', credentials=creds)

    def iterfiles(name=None, is_folder=None, parent=None, order_by='folder,name,createdTime'):
    def iterfiles(name=None, *, is_folder=None, parent=None,
    order_by='folder,name,createdTime'):
    q = []
    if name is not None:
    q.append("name = '%s'" % name.replace("'", "\\'"))
    @@ -39,7 +41,7 @@ def iterfiles(name=None, is_folder=None, parent=None, order_by='folder,name,crea
    except KeyError:
    return

    def walk(top='root', by_name=False):
    def walk(top='root', *, by_name: bool = False):
    if by_name:
    top, = iterfiles(name=top, is_folder=True)
    else:
  8. xflr6 revised this gist Mar 1, 2020. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion walk_gdrive.py
    View file
    Original file line number Diff line number Diff line change
    @@ -54,7 +54,7 @@ def walk(top='root', by_name=False):
    is_file[f['mimeType'] != FOLDER].append(f)
    yield path, top, dirs, files
    if dirs:
    stack.extend((path + (d['name'],), d) for d in dirs)
    stack.extend((path + (d['name'],), d) for d in reversed(dirs))

    for kwargs in [{'top': 'spam', 'by_name': True}, {}]:
    for path, root, dirs, files in walk(**kwargs):
  9. xflr6 revised this gist Dec 18, 2019. 1 changed file with 10 additions and 5 deletions.
    15 changes: 10 additions & 5 deletions walk_gdrive.py
    View file
    Original file line number Diff line number Diff line change
    @@ -16,7 +16,7 @@ def get_credentials(scopes, secrets='~/client_secrets.json', storage='~/storage.
    creds = tools.run_flow(flow, store, flags)
    return creds

    creds = get_credentials(['https://www.googleapis.com/auth/drive.metadata.readonly'])
    creds = get_credentials(scopes=['https://www.googleapis.com/auth/drive.metadata.readonly'])
    service = build('drive', version='v3', credentials=creds)

    def iterfiles(name=None, is_folder=None, parent=None, order_by='folder,name,createdTime'):
    @@ -39,8 +39,13 @@ def iterfiles(name=None, is_folder=None, parent=None, order_by='folder,name,crea
    except KeyError:
    return

    def walk(top):
    top, = iterfiles(name=top, is_folder=True)
    def walk(top='root', by_name=False):
    if by_name:
    top, = iterfiles(name=top, is_folder=True)
    else:
    top = service.files().get(fileId=top).execute()
    if top['mimeType'] != FOLDER:
    raise ValueError('not a folder: %r' % top)
    stack = [((top['name'],), top)]
    while stack:
    path, top = stack.pop()
    @@ -51,6 +56,6 @@ def walk(top):
    if dirs:
    stack.extend((path + (d['name'],), d) for d in dirs)

    for testdir in ['spam', 'folders']:
    for path, root, dirs, files in walk(testdir):
    for kwargs in [{'top': 'spam', 'by_name': True}, {}]:
    for path, root, dirs, files in walk(**kwargs):
    print('%s\t%d %d' % ('/'.join(path), len(dirs), len(files)))
  10. xflr6 revised this gist Dec 3, 2019. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion walk_gdrive.py
    View file
    Original file line number Diff line number Diff line change
    @@ -16,7 +16,7 @@ def get_credentials(scopes, secrets='~/client_secrets.json', storage='~/storage.
    creds = tools.run_flow(flow, store, flags)
    return creds

    creds = get_credentials('https://www.googleapis.com/auth/drive.metadata.readonly')
    creds = get_credentials(['https://www.googleapis.com/auth/drive.metadata.readonly'])
    service = build('drive', version='v3', credentials=creds)

    def iterfiles(name=None, is_folder=None, parent=None, order_by='folder,name,createdTime'):
  11. xflr6 revised this gist Sep 30, 2018. 1 changed file with 12 additions and 12 deletions.
    24 changes: 12 additions & 12 deletions walk_gdrive.py
    View file
    Original file line number Diff line number Diff line change
    @@ -41,16 +41,16 @@ def iterfiles(name=None, is_folder=None, parent=None, order_by='folder,name,crea

    def walk(top):
    top, = iterfiles(name=top, is_folder=True)
    stack = [((top['name'],), [top])]
    stack = [((top['name'],), top)]
    while stack:
    path, tops = stack.pop()
    for top in tops:
    dirs, files = is_file = [], []
    for f in iterfiles(parent=top['id']):
    is_file[f['mimeType'] != FOLDER].append(f)
    yield path, top, dirs, files
    if dirs:
    stack.append((path + (top['name'],), dirs))

    for path, root, dirs, files in walk('spam'):
    print('%s\t%d %d' % ('/'.join(path), len(dirs), len(files)))
    path, top = stack.pop()
    dirs, files = is_file = [], []
    for f in iterfiles(parent=top['id']):
    is_file[f['mimeType'] != FOLDER].append(f)
    yield path, top, dirs, files
    if dirs:
    stack.extend((path + (d['name'],), d) for d in dirs)

    for testdir in ['spam', 'folders']:
    for path, root, dirs, files in walk(testdir):
    print('%s\t%d %d' % ('/'.join(path), len(dirs), len(files)))
  12. xflr6 revised this gist Aug 8, 2017. 1 changed file with 2 additions and 1 deletion.
    3 changes: 2 additions & 1 deletion walk_gdrive.py
    View file
    Original file line number Diff line number Diff line change
    @@ -12,7 +12,8 @@ def get_credentials(scopes, secrets='~/client_secrets.json', storage='~/storage.
    creds = store.get()
    if creds is None or creds.invalid:
    flow = client.flow_from_clientsecrets(os.path.expanduser(secrets), scopes)
    creds = tools.run_flow(flow, store)
    flags = tools.argparser.parse_args([])
    creds = tools.run_flow(flow, store, flags)
    return creds

    creds = get_credentials('https://www.googleapis.com/auth/drive.metadata.readonly')
  13. xflr6 revised this gist Dec 5, 2016. 1 changed file with 1 addition and 4 deletions.
    5 changes: 1 addition & 4 deletions walk_gdrive.py
    View file
    Original file line number Diff line number Diff line change
    @@ -23,10 +23,7 @@ def iterfiles(name=None, is_folder=None, parent=None, order_by='folder,name,crea
    if name is not None:
    q.append("name = '%s'" % name.replace("'", "\\'"))
    if is_folder is not None:
    if is_folder:
    q.append("mimeType = '%s'" % FOLDER)
    else:
    q.append("mimeType != '%s'" % FOLDER)
    q.append("mimeType %s '%s'" % ('=' if is_folder else '!=', FOLDER))
    if parent is not None:
    q.append("'%s' in parents" % parent.replace("'", "\\'"))
    params = {'pageToken': None, 'orderBy': order_by}
  14. xflr6 created this gist Dec 4, 2016.
    58 changes: 58 additions & 0 deletions walk_gdrive.py
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,58 @@
    # walk_gdrive.py - os.walk variation with Google Drive API

    import os

    from apiclient.discovery import build # pip install google-api-python-client

    FOLDER = 'application/vnd.google-apps.folder'

    def get_credentials(scopes, secrets='~/client_secrets.json', storage='~/storage.json'):
    from oauth2client import file, client, tools
    store = file.Storage(os.path.expanduser(storage))
    creds = store.get()
    if creds is None or creds.invalid:
    flow = client.flow_from_clientsecrets(os.path.expanduser(secrets), scopes)
    creds = tools.run_flow(flow, store)
    return creds

    creds = get_credentials('https://www.googleapis.com/auth/drive.metadata.readonly')
    service = build('drive', version='v3', credentials=creds)

    def iterfiles(name=None, is_folder=None, parent=None, order_by='folder,name,createdTime'):
    q = []
    if name is not None:
    q.append("name = '%s'" % name.replace("'", "\\'"))
    if is_folder is not None:
    if is_folder:
    q.append("mimeType = '%s'" % FOLDER)
    else:
    q.append("mimeType != '%s'" % FOLDER)
    if parent is not None:
    q.append("'%s' in parents" % parent.replace("'", "\\'"))
    params = {'pageToken': None, 'orderBy': order_by}
    if q:
    params['q'] = ' and '.join(q)
    while True:
    response = service.files().list(**params).execute()
    for f in response['files']:
    yield f
    try:
    params['pageToken'] = response['nextPageToken']
    except KeyError:
    return

    def walk(top):
    top, = iterfiles(name=top, is_folder=True)
    stack = [((top['name'],), [top])]
    while stack:
    path, tops = stack.pop()
    for top in tops:
    dirs, files = is_file = [], []
    for f in iterfiles(parent=top['id']):
    is_file[f['mimeType'] != FOLDER].append(f)
    yield path, top, dirs, files
    if dirs:
    stack.append((path + (top['name'],), dirs))

    for path, root, dirs, files in walk('spam'):
    print('%s\t%d %d' % ('/'.join(path), len(dirs), len(files)))