Skip to content

Instantly share code, notes, and snippets.

@tracphil
Forked from Potherca/README.md
Created June 27, 2023 11:28
Show Gist options
  • Save tracphil/94d3780793f7d8f92bb499d58702b395 to your computer and use it in GitHub Desktop.
Save tracphil/94d3780793f7d8f92bb499d58702b395 to your computer and use it in GitHub Desktop.

Revisions

  1. @Potherca Potherca revised this gist Jun 16, 2023. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion github-clone-projects.bash
    Original file line number Diff line number Diff line change
    @@ -90,7 +90,7 @@ github-clone-projects() {
    ) || true

    if [[ "${sPaginationUrl}" == '' ]]; then
    echo -e "${sPrevious}\n${sResult}"
    printf '%s\n%s' "${sPrevious}" "${sResult}"
    else
    call-url "${sPaginationUrl}" "${sResult}"
    fi
  2. @Potherca Potherca revised this gist Jun 16, 2023. 1 changed file with 3 additions and 2 deletions.
    5 changes: 3 additions & 2 deletions github-clone-projects.bash
    Original file line number Diff line number Diff line change
    @@ -85,8 +85,9 @@ github-clone-projects() {
    "${sUrl}")

    sPaginationUrl=$(echo "${sHeaderResult}" \
    | grep -oE 'link: <[^>]+>; rel="next"' \
    | grep -oE 'https?://[^>]+')
    | grep -oE '<[^>]+>; rel="next"' \
    | grep -oE 'https?://[^>]+'
    ) || true

    if [[ "${sPaginationUrl}" == '' ]]; then
    echo -e "${sPrevious}\n${sResult}"
  3. @Potherca Potherca revised this gist Apr 6, 2023. 1 changed file with 9 additions and 9 deletions.
    18 changes: 9 additions & 9 deletions github-clone-projects.bash
    Original file line number Diff line number Diff line change
    @@ -25,11 +25,11 @@ set -o pipefail # Return exit status of the last command in the pipe that exited
    # ==============================================================================
    # Git Clone All Projects in GitHub Organisation
    # ------------------------------------------------------------------------------
    ## Usage: $0 [-dhu] <github-domain> <group-id> <github-token>
    ## Usage: $0 [-dhu] <github-domain> <organisation-id> <github-token>
    ##
    ## Where:
    ## - <github-domain> is the domain where github lives (for instance: 'github.com')
    ## - <group-id> is the ID of the group who's repos should be cloned
    ## - <organisation-id> is the ID of the organisation who's repos should be cloned
    ## - <github-token> is the API access token to make REST API calls with
    ##
    ## Options:
    @@ -38,12 +38,12 @@ set -o pipefail # Return exit status of the last command in the pipe that exited
    ## -u|--user The given ID is a user, not an organisation
    ##
    ## The repositories will be cloned into a sub-directory under the path from Where
    ## this script has been called. The repository will be cloned into ./${group-id}/${repo-name}
    ## this script has been called. The repository will be cloned into ./${organisation-id}/${repo-name}
    ##
    ## The git and cUrl executable can be overridden by setting their respective environmental variable
    ## before calling this script:
    ##
    ## CURL=/usr/local/curl GIT=/usr/local/git-plus $0 <github-domain> <group-id> <github-token>
    ## CURL=/usr/local/curl GIT=/usr/local/git-plus $0 <github-domain> <organisation-id> <github-token>
    # ==============================================================================

    : readonly "${CURL:=curl}"
    @@ -117,7 +117,7 @@ github-clone-projects() {
    | cut -d '"' -f4
    }

    fetch-group-projects() {
    fetch-organisation-projects() {
    local -r iId="${1?One parameters required: <id>}"
    fetch-projects 'orgs' "${iId}"
    }
    @@ -157,14 +157,14 @@ github-clone-projects() {
    done
    readonly aParameters

    readonly sGithubDomain="${aParameters[0]?Three parameters required: <github-domain> <group-id> <github-token>}"
    readonly sId="${aParameters[1]?Three parameters required: <github-domain> <group-id> <github-token>}"
    readonly sGithubToken="${aParameters[2]?Three parameters required: <github-domain> <group-id> <github-token>}"
    readonly sGithubDomain="${aParameters[0]?Three parameters required: <github-domain> <organisation-id> <github-token>}"
    readonly sId="${aParameters[1]?Three parameters required: <github-domain> <organisation-id> <github-token>}"
    readonly sGithubToken="${aParameters[2]?Three parameters required: <github-domain> <organisation-id> <github-token>}"

    if [[ "${bIsUser}" = 'true' ]];then
    readonly sRepos=$(fetch-user-projects "${sId}")
    else
    readonly sRepos=$(fetch-group-projects "${sId}")
    readonly sRepos=$(fetch-organisation-projects "${sId}")
    fi

    aRepos=()
  4. @Potherca Potherca revised this gist Apr 6, 2023. 1 changed file with 10 additions and 3 deletions.
    13 changes: 10 additions & 3 deletions github-clone-projects.bash
    Original file line number Diff line number Diff line change
    @@ -65,19 +65,26 @@ github-clone-projects() {
    local sDirectory sGithubDomain sGithubToken sId sRepo sRepos

    call-url() {
    local sPaginationUrl sPrevious sResult sUrl
    local sHeaderResult sPaginationUrl sPrevious sResult sUrl

    readonly sUrl="${1?One parameter required: <url> [previous-content]}"
    readonly sPrevious="${2:-''}"

    sHeaderResult=$("${CURL}" \
    --head \
    --header "Accept: application/vnd.github.v3+json" \
    --header "Authorization: token ${sGithubToken}" \
    --request 'GET' \
    --silent \
    "${sUrl}")

    sResult=$("${CURL}" \
    --dump-header - \
    --header "Accept: application/vnd.github.v3+json" \
    --header "Authorization: token ${sGithubToken}" \
    --silent \
    "${sUrl}")

    sPaginationUrl=$(echo "${sResult}" \
    sPaginationUrl=$(echo "${sHeaderResult}" \
    | grep -oE 'link: <[^>]+>; rel="next"' \
    | grep -oE 'https?://[^>]+')

  5. @Potherca Potherca revised this gist Apr 6, 2023. 1 changed file with 46 additions and 15 deletions.
    61 changes: 46 additions & 15 deletions github-clone-projects.bash
    Original file line number Diff line number Diff line change
    @@ -1,7 +1,7 @@
    #!/usr/bin/env bash

    # ==============================================================================
    # Copyright (C) 2021 Potherca
    # Copyright (C) 2021-2023 Potherca
    #
    # This Source Code Form is subject to the terms of the Mozilla Public
    # License, v. 2.0. If a copy of the MPL was not distributed with this
    @@ -25,7 +25,7 @@ set -o pipefail # Return exit status of the last command in the pipe that exited
    # ==============================================================================
    # Git Clone All Projects in GitHub Organisation
    # ------------------------------------------------------------------------------
    ## Usage: $0 [-dh] <github-domain> <group-id> <github-token>
    ## Usage: $0 [-dhu] <github-domain> <group-id> <github-token>
    ##
    ## Where:
    ## - <github-domain> is the domain where github lives (for instance: 'github.com')
    @@ -35,17 +35,18 @@ set -o pipefail # Return exit status of the last command in the pipe that exited
    ## Options:
    ## -d|--dry-run Only list the repositories, without actually cloning them
    ## -h|--help Print this help dialogue and exit
    ## -u|--user The given ID is a user, not a organisation
    ## -u|--user The given ID is a user, not an organisation
    ##
    ## The repositories will be cloned into a sub-directory under the path from Where
    ## this script has been called. The repository will be cloned into ./${group-id}/${repo-name}
    ##
    ## The git executable can be overridden by setting the GIT environmental variable
    ## The git and cUrl executable can be overridden by setting their respective environmental variable
    ## before calling this script:
    ##
    ## GIT=/usr/local/git-plus $0 <github-domain> <group-id> <github-token>
    ## CURL=/usr/local/curl GIT=/usr/local/git-plus $0 <github-domain> <group-id> <github-token>
    # ==============================================================================

    : readonly "${CURL:=curl}"
    : readonly "${GIT:=git}"

    usage() {
    @@ -60,13 +61,39 @@ usage() {
    github-clone-projects() {

    local -a aParameters aRepos
    local g_sGithubDomain g_sGithubToken g_sId
    local bIsUser bDryRun
    local sDirectory sRepo sRepos
    local sDirectory sGithubDomain sGithubToken sId sRepo sRepos

    call-url() {
    local sPaginationUrl sPrevious sResult sUrl

    readonly sUrl="${1?One parameter required: <url> [previous-content]}"
    readonly sPrevious="${2:-''}"

    sResult=$("${CURL}" \
    --dump-header - \
    --header "Accept: application/vnd.github.v3+json" \
    --header "Authorization: token ${sGithubToken}" \
    --silent \
    "${sUrl}")

    sPaginationUrl=$(echo "${sResult}" \
    | grep -oE 'link: <[^>]+>; rel="next"' \
    | grep -oE 'https?://[^>]+')

    if [[ "${sPaginationUrl}" == '' ]]; then
    echo -e "${sPrevious}\n${sResult}"
    else
    call-url "${sPaginationUrl}" "${sResult}"
    fi
    }

    call-api() {
    local -r sSubject="${1?One parameter required: <api-subject>}"
    curl --silent -u "username:${g_sGithubToken}" "https://api.${g_sGithubDomain}/${sSubject}?per_page=100"
    local sSubject

    readonly sSubject="${1?One parameter required: <api-subject>}"

    call-url "https://api.${sGithubDomain}/${sSubject}?per_page=100"
    }

    fetch-projects() {
    @@ -75,6 +102,9 @@ github-clone-projects() {
    readonly sSubject="${1?Two parameters required: <subject> <id>}"
    readonly iId="${2?Two parameters required: <subject> <id>}"

    # @TODO: Add param that allows skipping archived repos
    # @TODO: Add param that allows skipping forks

    call-api "${sSubject}/${iId}/repos" \
    | grep -E -o '"ssh_url"\s*:\s*"[^"]+"' \
    | cut -d '"' -f4
    @@ -107,6 +137,7 @@ github-clone-projects() {
    ;;

    -u|--user )
    # @TODO: Is there a way we can detect if this is a user or organisation?
    readonly bIsUser=true
    shift
    ;;
    @@ -119,14 +150,14 @@ github-clone-projects() {
    done
    readonly aParameters

    readonly g_sGithubDomain="${aParameters[0]?Three parameters required: <github-domain> <group-id> <github-token>}"
    readonly g_sId="${aParameters[1]?Three parameters required: <github-domain> <group-id> <github-token>}"
    readonly g_sGithubToken="${aParameters[2]?Three parameters required: <github-domain> <group-id> <github-token>}"
    readonly sGithubDomain="${aParameters[0]?Three parameters required: <github-domain> <group-id> <github-token>}"
    readonly sId="${aParameters[1]?Three parameters required: <github-domain> <group-id> <github-token>}"
    readonly sGithubToken="${aParameters[2]?Three parameters required: <github-domain> <group-id> <github-token>}"

    if [[ "${bIsUser}" = 'true' ]];then
    readonly sRepos=$(fetch-user-projects "${g_sId}")
    readonly sRepos=$(fetch-user-projects "${sId}")
    else
    readonly sRepos=$(fetch-group-projects "${g_sId}")
    readonly sRepos=$(fetch-group-projects "${sId}")
    fi

    aRepos=()
    @@ -169,4 +200,4 @@ else
    exit $?
    fi

    #EOF
    #EOF
  6. @Potherca Potherca revised this gist Apr 6, 2023. 1 changed file with 10 additions and 6 deletions.
    16 changes: 10 additions & 6 deletions gitlab-clone-projects.bash
    Original file line number Diff line number Diff line change
    @@ -1,7 +1,7 @@
    #!/usr/bin/env bash

    # ==============================================================================
    # Copyright (C) 2018, 2020 Potherca
    # Copyright (C) 2018, 2020-2023 Potherca
    #
    # This Source Code Form is subject to the terms of the Mozilla Public
    # License, v. 2.0. If a copy of the MPL was not distributed with this
    @@ -25,7 +25,7 @@ set -o pipefail # Return exit status of the last command in the pipe that exited
    # ==============================================================================
    # Git Clone All Projects in Gitlab Group
    # ------------------------------------------------------------------------------
    ## Usage: $0 [-dh] <gitlab-domain> <group-id> <gitlab-token>
    ## Usage: $0 [-dhu] <gitlab-domain> <group-id> <gitlab-token>
    ##
    ## Where:
    ## - <gitlab-domain> is the domain where gitlab lives (for instance: 'gitlab.com')
    @@ -40,12 +40,13 @@ set -o pipefail # Return exit status of the last command in the pipe that exited
    ## The repositories will be cloned into a sub-directory under the path from Where
    ## this script has been called. The repository will be cloned into ./${group-id}/${repo-name}
    ##
    ## The git executable can be overridden by setting the GIT environmental variable
    ## The git and cUrl executable can be overridden by setting their respective environmental variable
    ## before calling this script:
    ##
    ## GIT=/usr/local/git-plus $0 <gitlab-domain> <group-id> <gitlab-token>
    ## CURL=/usr/local/curl GIT=/usr/local/git-plus $0 <github-domain> <group-id> <github-token>
    # ==============================================================================

    : readonly "${CURL:=curl}"
    : readonly "${GIT:=git}"

    usage() {
    @@ -66,7 +67,7 @@ gitlab-clone-projects() {

    call-api() {
    local -r sSubject="${1?One parameter required: <api-subject>}"
    curl --silent --header "PRIVATE-TOKEN: ${g_sGitlabToken}" "https://${g_sGitlabDomain}/api/v4/${sSubject}?per_page=100"
    "${CURL}" --silent --header "PRIVATE-TOKEN: ${g_sGitlabToken}" "https://${g_sGitlabDomain}/api/v4/${sSubject}?per_page=100"
    }

    fetch-projects() {
    @@ -76,8 +77,10 @@ gitlab-clone-projects() {
    readonly iId="${2?Two parameters required: <subject> <id>}"

    call-api "${sSubject}/${iId}/projects" \
    | grep -E -o '"ssh_url_to_repo":"[^"]+"' \
    | grep -E -o '"ssh_url_to_repo"\s*:\s*"[^"]+"' \
    | cut -d '"' -f4

    # @TODO: Take pagination into account.
    }

    fetch-group-projects() {
    @@ -107,6 +110,7 @@ gitlab-clone-projects() {
    ;;

    -u|--user )
    # @TODO: Is there a way we can detect if this is a user or organisation?
    readonly bIsUser=true
    shift
    ;;
  7. @Potherca Potherca revised this gist Apr 15, 2021. 2 changed files with 175 additions and 12 deletions.
    172 changes: 172 additions & 0 deletions github-clone-projects.bash
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,172 @@
    #!/usr/bin/env bash

    # ==============================================================================
    # Copyright (C) 2021 Potherca
    #
    # This Source Code Form is subject to the terms of the Mozilla Public
    # License, v. 2.0. If a copy of the MPL was not distributed with this
    # file, You can obtain one at https://mozilla.org/MPL/2.0/.
    # ==============================================================================
    # There are a few standards this code tries to adhere to, these are listed below.
    #
    # - Code follows the BASH style-guide described at:
    # http://guides.dealerdirect.io/code-styling/bash/
    #
    # - Variables are named using an adaption of Systems Hungarian explained at:
    # http://blog.pother.ca/VariableNamingConvention
    #
    # ==============================================================================

    set -o errexit # Exit script when a command exits with non-zero status.
    set -o errtrace # Exit on error inside any functions or sub-shells.
    set -o nounset # Exit script on use of an undefined variable.
    set -o pipefail # Return exit status of the last command in the pipe that exited with a non-zero exit code

    # ==============================================================================
    # Git Clone All Projects in GitHub Organisation
    # ------------------------------------------------------------------------------
    ## Usage: $0 [-dh] <github-domain> <group-id> <github-token>
    ##
    ## Where:
    ## - <github-domain> is the domain where github lives (for instance: 'github.com')
    ## - <group-id> is the ID of the group who's repos should be cloned
    ## - <github-token> is the API access token to make REST API calls with
    ##
    ## Options:
    ## -d|--dry-run Only list the repositories, without actually cloning them
    ## -h|--help Print this help dialogue and exit
    ## -u|--user The given ID is a user, not a organisation
    ##
    ## The repositories will be cloned into a sub-directory under the path from Where
    ## this script has been called. The repository will be cloned into ./${group-id}/${repo-name}
    ##
    ## The git executable can be overridden by setting the GIT environmental variable
    ## before calling this script:
    ##
    ## GIT=/usr/local/git-plus $0 <github-domain> <group-id> <github-token>
    # ==============================================================================

    : readonly "${GIT:=git}"

    usage() {
    local sScript sUsage

    readonly sScript="$(basename "$0")"
    readonly sUsage="$(grep '^##' <"$0" | cut -c4-)"

    echo -e "${sUsage//\$0/${sScript}}"
    }

    github-clone-projects() {

    local -a aParameters aRepos
    local g_sGithubDomain g_sGithubToken g_sId
    local bIsUser bDryRun
    local sDirectory sRepo sRepos

    call-api() {
    local -r sSubject="${1?One parameter required: <api-subject>}"
    curl --silent -u "username:${g_sGithubToken}" "https://api.${g_sGithubDomain}/${sSubject}?per_page=100"
    }

    fetch-projects() {
    local iId sSubject

    readonly sSubject="${1?Two parameters required: <subject> <id>}"
    readonly iId="${2?Two parameters required: <subject> <id>}"

    call-api "${sSubject}/${iId}/repos" \
    | grep -E -o '"ssh_url"\s*:\s*"[^"]+"' \
    | cut -d '"' -f4
    }

    fetch-group-projects() {
    local -r iId="${1?One parameters required: <id>}"
    fetch-projects 'orgs' "${iId}"
    }

    fetch-user-projects() {
    local -r iId="${1?One parameters required: <id>}"
    fetch-projects 'users' "${iId}"
    }

    bIsUser=false
    bDryRun=false
    aParameters=()

    for arg in "$@";do
    case $arg in
    -h|--help )
    usage
    exit
    ;;

    -d|--dry-run )
    readonly bDryRun=true
    shift
    ;;

    -u|--user )
    readonly bIsUser=true
    shift
    ;;

    * )
    aParameters+=( "$1" )
    shift
    ;;
    esac
    done
    readonly aParameters

    readonly g_sGithubDomain="${aParameters[0]?Three parameters required: <github-domain> <group-id> <github-token>}"
    readonly g_sId="${aParameters[1]?Three parameters required: <github-domain> <group-id> <github-token>}"
    readonly g_sGithubToken="${aParameters[2]?Three parameters required: <github-domain> <group-id> <github-token>}"

    if [[ "${bIsUser}" = 'true' ]];then
    readonly sRepos=$(fetch-user-projects "${g_sId}")
    else
    readonly sRepos=$(fetch-group-projects "${g_sId}")
    fi

    aRepos=()
    for sRepo in ${sRepos[*]}; do
    aRepos+=("${sRepo}")
    done

    echo ' =====> Found ' ${#aRepos[@]} ' repositories'

    for sRepo in "${aRepos[@]}"; do
    # Grab repo name
    sDirectory="$(echo "${sRepo}" | grep -o -E ':(.*)\.')"
    # Lowercase the name
    sDirectory="$(echo "${sDirectory}" | tr '[:upper:]' '[:lower:]')"
    # Prepend the current location
    sDirectory="$(realpath --canonicalize-missing --relative-to=./ "${sDirectory:1:-1}")"

    if [[ -d "${sDirectory}" ]];then
    echo " -----> Skipping '${sRepo}', directory '${sDirectory}' already exists"
    else
    echo " -----> Cloning '${sRepo}' into directory '${sDirectory}'"

    if [[ "${bDryRun}" != 'true' ]];then
    mkdir -p "${sDirectory}"
    "${GIT}" clone --recursive "${sRepo}" "${sDirectory}" \
    || {
    rm -rf "${sDirectory}"
    echo -e "\n ! ERROR !\n Could not clone ${sRepo}"
    }
    echo ""
    fi
    fi
    done
    }

    if [[ "${BASH_SOURCE[0]}" != "$0" ]]; then
    export -f github-clone-projects
    else
    github-clone-projects "${@}"
    exit $?
    fi

    #EOF
    15 changes: 3 additions & 12 deletions gitlab-clone-projects.sh → gitlab-clone-projects.bash
    Original file line number Diff line number Diff line change
    @@ -3,18 +3,9 @@
    # ==============================================================================
    # Copyright (C) 2018, 2020 Potherca
    #
    # This program is free software: you can redistribute it and/or modify
    # it under the terms of the GNU General Public License as published by
    # the Free Software Foundation, either version 3 of the License, or
    # (at your option) any later version.
    #
    # This program is distributed in the hope that it will be useful,
    # but WITHOUT ANY WARRANTY; without even the implied warranty of
    # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    # GNU General Public License for more details.

    # You should have received a copy of the GNU General Public License
    # along with this program. If not, see <http://www.gnu.org/licenses/>.
    # This Source Code Form is subject to the terms of the Mozilla Public
    # License, v. 2.0. If a copy of the MPL was not distributed with this
    # file, You can obtain one at https://mozilla.org/MPL/2.0/.
    # ==============================================================================
    # There are a few standards this code tries to adhere to, these are listed below.
    #
  8. @Potherca Potherca revised this gist Apr 22, 2020. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion gitlab-clone-projects.sh
    Original file line number Diff line number Diff line change
    @@ -1,7 +1,7 @@
    #!/usr/bin/env bash

    # ==============================================================================
    # Copyright (C) 2018 Potherca
    # Copyright (C) 2018, 2020 Potherca
    #
    # This program is free software: you can redistribute it and/or modify
    # it under the terms of the GNU General Public License as published by
  9. @Potherca Potherca revised this gist Apr 22, 2020. 2 changed files with 17 additions and 14 deletions.
    29 changes: 16 additions & 13 deletions README.md
    Original file line number Diff line number Diff line change
    @@ -18,21 +18,24 @@ Call `bash gitlab-clone-projects.sh <gitlab-domain> <group-name> <gitlab-token>`

    For full usage details `gitlab-clone-projects --help` is available:

    Usage: git-clone-all-projects-in-gitlab-group.sh [-lh] <gitlab-domain> <group-name> <gitlab-token>
    ```
    Usage: gitlab-clone-projects.sh [-dh] <gitlab-domain> <group-id> <gitlab-token>
    Where:
    - <gitlab-domain> is the domain where gitlab lives (for instance: 'gitlab.com')
    - <group-name> is the name of the group who's repos should be cloned
    - <gitlab-token> is the API access token to make REST API calls with
    Where:
    - <gitlab-domain> is the domain where gitlab lives (for instance: 'gitlab.com')
    - <group-id> is the ID of the group who's repos should be cloned
    - <gitlab-token> is the API access token to make REST API calls with
    Options:
    -h|--help Print this help dialogue and exit
    -L|--list List the repositories that would be cloned and exit
    Options:
    -d|--dry-run Only list the repositories, without actually cloning them
    -h|--help Print this help dialogue and exit
    -u|--user The given ID is a user, not a group
    The repositories will be cloned into a sub-directory under the path from Where
    this script has been called. The repository will be cloned into ./${group-name}/${repo-name}
    The repositories will be cloned into a sub-directory under the path from Where
    this script has been called. The repository will be cloned into ./${group-id}/${repo-name}
    The git executable can be overridden by setting the GIT environmental variable
    before calling this script:
    The git executable can be overridden by setting the GIT environmental variable
    before calling this script:
    GIT=/usr/local/git-plus git-clone-all-projects-in-gitlab-group.sh <gitlab-domain> <group-name> <gitlab-token>
    GIT=/usr/local/git-plus gitlab-clone-projects.sh <gitlab-domain> <group-id> <gitlab-token>
    ```
    2 changes: 1 addition & 1 deletion gitlab-clone-projects.sh
    Original file line number Diff line number Diff line change
    @@ -34,7 +34,7 @@ set -o pipefail # Return exit status of the last command in the pipe that exited
    # ==============================================================================
    # Git Clone All Projects in Gitlab Group
    # ------------------------------------------------------------------------------
    ## Usage: $0 [-lh] <gitlab-domain> <group-id> <gitlab-token>
    ## Usage: $0 [-dh] <gitlab-domain> <group-id> <gitlab-token>
    ##
    ## Where:
    ## - <gitlab-domain> is the domain where gitlab lives (for instance: 'gitlab.com')
  10. @Potherca Potherca revised this gist Apr 22, 2020. 1 changed file with 53 additions and 24 deletions.
    77 changes: 53 additions & 24 deletions gitlab-clone-projects.sh
    Original file line number Diff line number Diff line change
    @@ -34,24 +34,25 @@ set -o pipefail # Return exit status of the last command in the pipe that exited
    # ==============================================================================
    # Git Clone All Projects in Gitlab Group
    # ------------------------------------------------------------------------------
    ## Usage: $0 [-lh] <gitlab-domain> <group-name> <gitlab-token>
    ## Usage: $0 [-lh] <gitlab-domain> <group-id> <gitlab-token>
    ##
    ## Where:
    ## - <gitlab-domain> is the domain where gitlab lives (for instance: 'gitlab.com')
    ## - <group-name> is the name of the group who's repos should be cloned
    ## - <group-id> is the ID of the group who's repos should be cloned
    ## - <gitlab-token> is the API access token to make REST API calls with
    ##
    ## Options:
    ## -d|--dry-run Only list the repositories, without actually cloning them
    ## -h|--help Print this help dialogue and exit
    ## -L|--list List the repositories that would be cloned and exit
    ## -u|--user The given ID is a user, not a group
    ##
    ## The repositories will be cloned into a sub-directory under the path from Where
    ## this script has been called. The repository will be cloned into ./${group-name}/${repo-name}
    ## this script has been called. The repository will be cloned into ./${group-id}/${repo-name}
    ##
    ## The git executable can be overridden by setting the GIT environmental variable
    ## before calling this script:
    ##
    ## GIT=/usr/local/git-plus $0 <gitlab-domain> <group-name> <gitlab-token>
    ## GIT=/usr/local/git-plus $0 <gitlab-domain> <group-id> <gitlab-token>
    # ==============================================================================

    : readonly "${GIT:=git}"
    @@ -68,21 +69,38 @@ usage() {
    gitlab-clone-projects() {

    local -a aParameters aRepos
    local g_sGitlabDomain g_sGitlabToken g_sGroupName
    local bListOnly
    local g_sGitlabDomain g_sGitlabToken g_sId
    local bIsUser bDryRun
    local sDirectory sRepo

    call-api() {
    local -r sSubject="${1?One parameter required: <api-subject>}"
    curl --silent --header "PRIVATE-TOKEN: ${g_sGitlabToken}" "https://${g_sGitlabDomain}/api/v4/${sSubject}?per_page=100"
    }

    fetch-projects() {
    call-api "groups/${g_sGroupName}/projects" \
    local iId sSubject

    readonly sSubject="${1?Two parameters required: <subject> <id>}"
    readonly iId="${2?Two parameters required: <subject> <id>}"

    call-api "${sSubject}/${iId}/projects" \
    | grep -E -o '"ssh_url_to_repo":"[^"]+"' \
    | cut -d '"' -f4
    }

    bListOnly=false
    fetch-group-projects() {
    local -r iId="${1?One parameters required: <id>}"
    fetch-projects 'groups' "${iId}"
    }

    fetch-user-projects() {
    local -r iId="${1?One parameters required: <id>}"
    fetch-projects 'users' "${iId}"
    }

    bIsUser=false
    bDryRun=false
    aParameters=()

    for arg in "$@";do
    @@ -92,8 +110,13 @@ gitlab-clone-projects() {
    exit
    ;;

    -l|--list )
    readonly bListOnly=true
    -d|--dry-run )
    readonly bDryRun=true
    shift
    ;;

    -u|--user )
    readonly bIsUser=true
    shift
    ;;

    @@ -105,11 +128,15 @@ gitlab-clone-projects() {
    done
    readonly aParameters

    readonly g_sGitlabDomain="${aParameters[0]?Three parameters required: <gitlab-domain> <group-name> <gitlab-token>}"
    readonly g_sGroupName="${aParameters[1]?Three parameters required: <gitlab-domain> <group-name> <gitlab-token>}"
    readonly g_sGitlabToken="${aParameters[2]?Three parameters required: <gitlab-domain> <group-name> <gitlab-token>}"
    readonly g_sGitlabDomain="${aParameters[0]?Three parameters required: <gitlab-domain> <group-id> <gitlab-token>}"
    readonly g_sId="${aParameters[1]?Three parameters required: <gitlab-domain> <group-id> <gitlab-token>}"
    readonly g_sGitlabToken="${aParameters[2]?Three parameters required: <gitlab-domain> <group-id> <gitlab-token>}"

    readonly sRepos=$(fetch-projects)
    if [[ "${bIsUser}" = 'true' ]];then
    readonly sRepos=$(fetch-user-projects "${g_sId}")
    else
    readonly sRepos=$(fetch-group-projects "${g_sId}")
    fi

    aRepos=()
    for sRepo in ${sRepos[*]}; do
    @@ -128,16 +155,18 @@ gitlab-clone-projects() {

    if [[ -d "${sDirectory}" ]];then
    echo " -----> Skipping '${sRepo}', directory '${sDirectory}' already exists"
    elif [[ "${bListOnly}" = 'true' ]];then
    echo " -----> Repository '${sRepo}' would be cloned into directory '${sDirectory}'"
    else
    echo -e "\n -----> Cloning '${sRepo}' into directory '${sDirectory}'"
    mkdir -p "${sDirectory}"
    "${GIT}" clone --recursive "${sRepo}" "${sDirectory}" \
    || {
    rm -rf "${sDirectory}"
    echo -e "\n ! ERROR !\n Could not clone ${sRepo}"
    }
    echo " -----> Cloning '${sRepo}' into directory '${sDirectory}'"

    if [[ "${bDryRun}" != 'true' ]];then
    mkdir -p "${sDirectory}"
    "${GIT}" clone --recursive "${sRepo}" "${sDirectory}" \
    || {
    rm -rf "${sDirectory}"
    echo -e "\n ! ERROR !\n Could not clone ${sRepo}"
    }
    echo ""
    fi
    fi
    done
    }
  11. @Potherca Potherca revised this gist Apr 22, 2020. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion gitlab-clone-projects.sh
    Original file line number Diff line number Diff line change
    @@ -129,7 +129,7 @@ gitlab-clone-projects() {
    if [[ -d "${sDirectory}" ]];then
    echo " -----> Skipping '${sRepo}', directory '${sDirectory}' already exists"
    elif [[ "${bListOnly}" = 'true' ]];then
    echo " -----> Cloning '${sRepo}' into directory '${sDirectory}'"
    echo " -----> Repository '${sRepo}' would be cloned into directory '${sDirectory}'"
    else
    echo -e "\n -----> Cloning '${sRepo}' into directory '${sDirectory}'"
    mkdir -p "${sDirectory}"
  12. @Potherca Potherca revised this gist May 3, 2018. 1 changed file with 7 additions and 2 deletions.
    9 changes: 7 additions & 2 deletions gitlab-clone-projects.sh
    Original file line number Diff line number Diff line change
    @@ -111,14 +111,19 @@ gitlab-clone-projects() {

    readonly sRepos=$(fetch-projects)

    aRepos=()
    for sRepo in ${sRepos[*]}; do
    aRepos+=("${sRepo}")
    done

    echo ' =====> Found ' ${#aRepos[@]} ' repositories'

    for sRepo in "${aRepos[@]}"; do
    # Grab repo name
    sDirectory="$(echo "${sRepo}" | grep -o -E ':(.*)\.')"
    # Lowercase the name
    sDirectory="$(echo "${sDirectory}" | tr '[:upper:]' '[:lower:]')"
    # Prepend the current location
    sDirectory="$(realpath --canonicalize-missing --relative-to=./ "${sDirectory:1:-1}")"

    if [[ -d "${sDirectory}" ]];then
    @@ -128,11 +133,11 @@ gitlab-clone-projects() {
    else
    echo -e "\n -----> Cloning '${sRepo}' into directory '${sDirectory}'"
    mkdir -p "${sDirectory}"
    "${GIT}" --recursive clone "${sRepo}" "${sDirectory}" \
    "${GIT}" clone --recursive "${sRepo}" "${sDirectory}" \
    || {
    rm -rf "${sDirectory}"
    echo -e "\n ! ERROR !\n Could not clone ${sRepo}"
    }
    }
    fi
    done
    }
  13. @Potherca Potherca revised this gist May 3, 2018. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion gitlab-clone-projects.sh
    Original file line number Diff line number Diff line change
    @@ -128,7 +128,7 @@ gitlab-clone-projects() {
    else
    echo -e "\n -----> Cloning '${sRepo}' into directory '${sDirectory}'"
    mkdir -p "${sDirectory}"
    "${GIT}" clone "${sRepo}" "${sDirectory}" \
    "${GIT}" --recursive clone "${sRepo}" "${sDirectory}" \
    || {
    rm -rf "${sDirectory}"
    echo -e "\n ! ERROR !\n Could not clone ${sRepo}"
  14. @Potherca Potherca revised this gist May 2, 2018. 1 changed file with 7 additions and 1 deletion.
    8 changes: 7 additions & 1 deletion gitlab-clone-projects.sh
    Original file line number Diff line number Diff line change
    @@ -128,7 +128,11 @@ gitlab-clone-projects() {
    else
    echo -e "\n -----> Cloning '${sRepo}' into directory '${sDirectory}'"
    mkdir -p "${sDirectory}"
    "${GIT}" clone "${sRepo}" "${sDirectory}"
    "${GIT}" clone "${sRepo}" "${sDirectory}" \
    || {
    rm -rf "${sDirectory}"
    echo -e "\n ! ERROR !\n Could not clone ${sRepo}"
    }
    fi
    done
    }
    @@ -139,3 +143,5 @@ else
    gitlab-clone-projects "${@}"
    exit $?
    fi

    #EOF
  15. @Potherca Potherca revised this gist May 2, 2018. 1 changed file with 15 additions and 7 deletions.
    22 changes: 15 additions & 7 deletions gitlab-clone-projects.sh
    Original file line number Diff line number Diff line change
    @@ -65,13 +65,12 @@ usage() {
    echo -e "${sUsage//\$0/${sScript}}"
    }

    git-clone-projects-from-gitlab() {
    gitlab-clone-projects() {

    local -a aParameters
    local -a aParameters aRepos
    local g_sGitlabDomain g_sGitlabToken g_sGroupName
    local bListOnly
    local sDirectory sRepo

    call-api() {
    local -r sSubject="${1?One parameter required: <api-subject>}"
    curl --silent --header "PRIVATE-TOKEN: ${g_sGitlabToken}" "https://${g_sGitlabDomain}/api/v4/${sSubject}?per_page=100"
    @@ -110,24 +109,33 @@ git-clone-projects-from-gitlab() {
    readonly g_sGroupName="${aParameters[1]?Three parameters required: <gitlab-domain> <group-name> <gitlab-token>}"
    readonly g_sGitlabToken="${aParameters[2]?Three parameters required: <gitlab-domain> <group-name> <gitlab-token>}"

    for sRepo in $(fetch-projects); do
    readonly sRepos=$(fetch-projects)

    for sRepo in ${sRepos[*]}; do
    aRepos+=("${sRepo}")
    done

    echo ' =====> Found ' ${#aRepos[@]} ' repositories'

    for sRepo in "${aRepos[@]}"; do
    sDirectory="$(echo "${sRepo}" | grep -o -E ':(.*)\.')"
    sDirectory="$(realpath --canonicalize-missing --relative-to=./ "${sDirectory:1:-1}")"

    if [[ -d "${sDirectory}" ]];then
    echo " -----> Skipping '${sRepo}', directory '${sDirectory}' already exists"
    elif [[ "${bListOnly}" = 'true' ]];then
    echo "git clone ${sRepo} ${sDirectory}"
    echo " -----> Cloning '${sRepo}' into directory '${sDirectory}'"
    else
    echo -e "\n -----> Cloning '${sRepo}' into directory '${sDirectory}'"
    mkdir -p "${sDirectory}"
    "${GIT}" clone "${sRepo}" "${sDirectory}"
    fi
    done
    }

    if [[ "${BASH_SOURCE[0]}" != "$0" ]]; then
    export -f git-clone-projects-from-gitlab
    export -f gitlab-clone-projects
    else
    git-clone-projects-from-gitlab "${@}"
    gitlab-clone-projects "${@}"
    exit $?
    fi
  16. @Potherca Potherca created this gist May 2, 2018.
    38 changes: 38 additions & 0 deletions README.md
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,38 @@
    ## Introduction

    Starting at a new employer always mean checking out various git repositories.

    As the amount of repositories a company has grows, the time needed to clone all of those repositories also grows.

    This script automates this task.

    In order for this script to work, [a personal access token](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html)
    is needed.

    To avoid unexpected behaviour, use Group's ID, rather than its key/name. Informations about Groups can be found in the API at
    `https://gitlab.example.com/api/v4/groups`.

    ## Usage

    Call `bash gitlab-clone-projects.sh <gitlab-domain> <group-name> <gitlab-token>`

    For full usage details `gitlab-clone-projects --help` is available:

    Usage: git-clone-all-projects-in-gitlab-group.sh [-lh] <gitlab-domain> <group-name> <gitlab-token>

    Where:
    - <gitlab-domain> is the domain where gitlab lives (for instance: 'gitlab.com')
    - <group-name> is the name of the group who's repos should be cloned
    - <gitlab-token> is the API access token to make REST API calls with

    Options:
    -h|--help Print this help dialogue and exit
    -L|--list List the repositories that would be cloned and exit

    The repositories will be cloned into a sub-directory under the path from Where
    this script has been called. The repository will be cloned into ./${group-name}/${repo-name}

    The git executable can be overridden by setting the GIT environmental variable
    before calling this script:

    GIT=/usr/local/git-plus git-clone-all-projects-in-gitlab-group.sh <gitlab-domain> <group-name> <gitlab-token>
    133 changes: 133 additions & 0 deletions gitlab-clone-projects.sh
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,133 @@
    #!/usr/bin/env bash

    # ==============================================================================
    # Copyright (C) 2018 Potherca
    #
    # This program is free software: you can redistribute it and/or modify
    # it under the terms of the GNU General Public License as published by
    # the Free Software Foundation, either version 3 of the License, or
    # (at your option) any later version.
    #
    # This program is distributed in the hope that it will be useful,
    # but WITHOUT ANY WARRANTY; without even the implied warranty of
    # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    # GNU General Public License for more details.

    # You should have received a copy of the GNU General Public License
    # along with this program. If not, see <http://www.gnu.org/licenses/>.
    # ==============================================================================
    # There are a few standards this code tries to adhere to, these are listed below.
    #
    # - Code follows the BASH style-guide described at:
    # http://guides.dealerdirect.io/code-styling/bash/
    #
    # - Variables are named using an adaption of Systems Hungarian explained at:
    # http://blog.pother.ca/VariableNamingConvention
    #
    # ==============================================================================

    set -o errexit # Exit script when a command exits with non-zero status.
    set -o errtrace # Exit on error inside any functions or sub-shells.
    set -o nounset # Exit script on use of an undefined variable.
    set -o pipefail # Return exit status of the last command in the pipe that exited with a non-zero exit code

    # ==============================================================================
    # Git Clone All Projects in Gitlab Group
    # ------------------------------------------------------------------------------
    ## Usage: $0 [-lh] <gitlab-domain> <group-name> <gitlab-token>
    ##
    ## Where:
    ## - <gitlab-domain> is the domain where gitlab lives (for instance: 'gitlab.com')
    ## - <group-name> is the name of the group who's repos should be cloned
    ## - <gitlab-token> is the API access token to make REST API calls with
    ##
    ## Options:
    ## -h|--help Print this help dialogue and exit
    ## -L|--list List the repositories that would be cloned and exit
    ##
    ## The repositories will be cloned into a sub-directory under the path from Where
    ## this script has been called. The repository will be cloned into ./${group-name}/${repo-name}
    ##
    ## The git executable can be overridden by setting the GIT environmental variable
    ## before calling this script:
    ##
    ## GIT=/usr/local/git-plus $0 <gitlab-domain> <group-name> <gitlab-token>
    # ==============================================================================

    : readonly "${GIT:=git}"

    usage() {
    local sScript sUsage

    readonly sScript="$(basename "$0")"
    readonly sUsage="$(grep '^##' <"$0" | cut -c4-)"

    echo -e "${sUsage//\$0/${sScript}}"
    }

    git-clone-projects-from-gitlab() {

    local -a aParameters
    local g_sGitlabDomain g_sGitlabToken g_sGroupName
    local bListOnly
    local sDirectory sRepo

    call-api() {
    local -r sSubject="${1?One parameter required: <api-subject>}"
    curl --silent --header "PRIVATE-TOKEN: ${g_sGitlabToken}" "https://${g_sGitlabDomain}/api/v4/${sSubject}?per_page=100"
    }

    fetch-projects() {
    call-api "groups/${g_sGroupName}/projects" \
    | grep -E -o '"ssh_url_to_repo":"[^"]+"' \
    | cut -d '"' -f4
    }

    bListOnly=false
    aParameters=()

    for arg in "$@";do
    case $arg in
    -h|--help )
    usage
    exit
    ;;

    -l|--list )
    readonly bListOnly=true
    shift
    ;;

    * )
    aParameters+=( "$1" )
    shift
    ;;
    esac
    done
    readonly aParameters

    readonly g_sGitlabDomain="${aParameters[0]?Three parameters required: <gitlab-domain> <group-name> <gitlab-token>}"
    readonly g_sGroupName="${aParameters[1]?Three parameters required: <gitlab-domain> <group-name> <gitlab-token>}"
    readonly g_sGitlabToken="${aParameters[2]?Three parameters required: <gitlab-domain> <group-name> <gitlab-token>}"

    for sRepo in $(fetch-projects); do
    sDirectory="$(echo "${sRepo}" | grep -o -E ':(.*)\.')"
    sDirectory="$(realpath --canonicalize-missing --relative-to=./ "${sDirectory:1:-1}")"

    if [[ -d "${sDirectory}" ]];then
    echo " -----> Skipping '${sRepo}', directory '${sDirectory}' already exists"
    elif [[ "${bListOnly}" = 'true' ]];then
    echo "git clone ${sRepo} ${sDirectory}"
    else
    mkdir -p "${sDirectory}"
    "${GIT}" clone "${sRepo}" "${sDirectory}"
    fi
    done
    }

    if [[ "${BASH_SOURCE[0]}" != "$0" ]]; then
    export -f git-clone-projects-from-gitlab
    else
    git-clone-projects-from-gitlab "${@}"
    exit $?
    fi