Copy_repos

import gitlab
import shutil
import subprocess
import pathlib
import warnings
import contextlib
from datetime import datetime as dt
import datetime, time
import requests
from urllib3.exceptions import InsecureRequestWarning
import pandas as pd

old_merge_environment_settings = requests.Session.merge_environment_settings


@contextlib.contextmanager
def no_ssl_verification():
    opened_adapters = set()

    def merge_environment_settings(self, url, proxies, stream, verify, cert):
        # Verification happens only once per connection so we need to close
        # all the opened adapters once we're done. Otherwise, the effects of
        # verify=False persist beyond the end of this context manager.
        opened_adapters.add(self.get_adapter(url))

        settings = old_merge_environment_settings(self, url, proxies, stream, verify, cert)
        settings['verify'] = False

        return settings

    requests.Session.merge_environment_settings = merge_environment_settings

    try:
        with warnings.catch_warnings():
            warnings.simplefilter('ignore', InsecureRequestWarning)
            yield
    finally:
        requests.Session.merge_environment_settings = old_merge_environment_settings

        for adapter in opened_adapters:
            try:
                adapter.close()
            except:
                pass


def delete_folders(dir_path, hours, fn):
    if fn == "":
        fn = today.strftime("%m-%d-%Y0-%H-%M")
    hours = datetime.timedelta(hours=hours)     #converting
    for path in dir_path:
        if dt.strptime(fn, "%m-%d-%Y0-%H-%M") - dt.strptime(path, "%m-%d-%Y0-%H-%M") > hours:
            shutil.rmtree(f"C:/tmp/{path}")                 #remove recursivaly dir
            #pathlib.Path(f"C:/tmp/{path}/").unlink()       remove only specific file
            #pathlib.Path(f"C:/tmp/{path}").rmdir()         remove only empty dir


# https://stackoverflow.com/questions/15445981/how-do-i-disable-the-security-certificate-check-in-python-requests

# https://code-maven.com/slides/gitlab/gitlab-api-using-python
# https://www.cyberforum.ru/python/thread1723215.html
# https://python-gitlab.readthedocs.io/en/stable/gl_objects/projects.html


all_dir_path = []
for path in pathlib.Path('C:/tmp').iterdir():
    all_dir_path.append(pathlib.PurePosixPath(path).name)
    # print(parent.path)                             #todo check all folders

today = dt.now()

p = pathlib.Path("C:/tmp").resolve()
fn = today.strftime("%m-%d-%Y0-%H-%M")
# temppath = str(p / fn).replace('\\', '/')
print(fn)
delete_folders(all_dir_path, 72, "")
temppath = p / fn
path = pathlib.Path(temppath)
path.mkdir(parents=True, exist_ok=True)
check = path / "out.csv"  # for save csv files
print(path)
full_projects = []
namewithspace = []
rep = []
created_at = []
last_activity_at = []
count_is = []
df = pd.DataFrame()
projects = {}

with no_ssl_verification():
    gl = gitlab.Gitlab(url= 'ur url', private_token='ur token')
    gl.auth()
    groups = gl.groups.list()
    for x in groups:
        full_projects.append(x.name)
    for i, values in enumerate(gl.projects.list(iterator=True)):
        # if i>0: continue
        # pd.DataFrame.from_dict(values.attributes, orient='index').to_excel('1.xlsx')
        print(values.attributes)
        if values.namespace['name'] in full_projects:
            namewithspace.append(values.namespace['name'])
            rep.append(values.name)
            created_at.append(values.created_at)
            last_activity_at.append(values.last_activity_at)
            count_is.append(values.open_issues_count)
            print(values.namespace['name'], values.name, values.default_branch, sep=" ")
            # print(values.name)     #TODO this or full path(including group.name)
            # print(values.default_branch)  # TODO check differences between .name & .path
            # print(values.issues)                       #TODO count them
    # df = pd.concat([pd.DataFrame(group),pd.DataFrame(rep),pd.DataFrame(branch), pd.DataFrame(count_is)], axis= 1)  #todo check this constrution
    df['group'] = pd.DataFrame(namewithspace)
    df['repo'] = pd.DataFrame(rep)
    df['created_at'] = pd.DataFrame(created_at)
    df['last_activity_at'] = pd.DataFrame(last_activity_at)
    df['count_issues'] = pd.DataFrame(count_is)
    print(df)
    df.to_csv(check)
    # groupname = "validation"        #or could be massive
    for group in groups:
        # if group.name == groupname:        add if it isn't necessary to clone all groups
        projects[group] = group.projects.list(all=True)
        # projects = group.projects.list(all=True)
        print(projects)

    for group, repos in projects.items():
        for repo in repos:
            if repo.name == 'ttttttttt': continue
            command = f'git clone {repo.ssh_url_to_repo}'
            process = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True,
                                       cwd=f"{path}")  # your path
            output, _ = process.communicate()
            process.wait()