updated helm script

conf/application.conf

 [SERVICE]
-port = 3973
+port = 5987
 host = 0.0.0.0
 
 [SQL_DB]

helm_automate_script.py

@@ -94,54 +94,33 @@ def convert_yaml_to_json(yaml_file_path):
         logging.exception(f"Exception Occurred while reading the yaml file {e.args}")
         return {}
     
-def push_helm_deployments(repo_link:str, private_token:str, branch: str):
+def push_helm_deployments(repo: git.Repo):
     try:
-        base_url = os.environ.get("GIT_BASE_URL", default=default_link)
-        repo_link_split = repo_link.split(base_url)
-        if not repo_link_split:
-            return False
-        gl = gitlab.Gitlab(url=base_url, private_token=private_token)
-        search_str = repo_link_split[-1].replace(".git", "")
-        pl = gl.projects.list(search=search_str)
-        if not pl:
-            return False
-        pl = pl[0]
-        commit_actions = []
         files_list = os.listdir(HELM_STORE_PATH)
         for file in files_list:
-            _action = {
-                'action': 'create',
-                'file_path': f'{HELM_PATH}/{file}',
-                'content': open(f'{HELM_STORE_PATH}/{file}').read()
-            }
-            commit_actions.append(_action)
-        commit_data = {'branch': branch, 'commit_message': f"{branch} helm creation"} | {'actions': commit_actions}
-
-        pl.commits.create(commit_data)
+            shutil.copy(f'{HELM_STORE_PATH}/{file}', f'{REPO_CLONE_PATH}{HELM_PATH}/{file}')
+        repo.git.add("-A")
+        repo.git.add(update=True)
+        repo.index.commit("helm files update")
+        repo.remotes.origin.push()
     except Exception as e:
         logging.exception(f'Exception while pushing helm deployments: {e.args}')
         
-def pull_global_config(repo_link: str, private_token:str, branch: str, file_name: str):
+def pull_global_config(repo_link: str, branch: str, file_name: str):
     if repo_link.split("https://")[-1].startswith("gitlab-pm"):
         repo_link = repo_link.replace("https://", f"https://{git_user_name}:{git_access_token}@")
-    repo = git.Repo.clone_from(url=repo_link, to_path=REPO_CLONE_PATH)
+    repo = git.Repo.clone_from(url=f'{repo_link}.git', to_path=REPO_CLONE_PATH)
     repo.git.checkout(branch)
     shutil.copy(f'{REPO_CLONE_PATH}{HELM_PATH}/{file_name}', ".")
-    print("I can break here")
+    return repo
 
-def remove_all_files_from_repo(repo_link: str, private_token: str, branch: str, exclude_file: list):
-    base_url = os.environ.get("GIT_BASE_URL", default=default_link)
-    repo_link_split = repo_link.split(base_url)
-    if not repo_link_split:
-        return False
-    gl = gitlab.Gitlab(url=base_url, private_token=private_token)
-    search_str = repo_link_split[-1].replace(".git", "")
-    if pl := gl.projects.list(search=search_str):
-        pl = pl[0]
-    else:
-        return False
-    items = pl.repository_tree(path=HELM_PATH, ref=branch)
-    print(items)
+def remove_all_files_from_repo(exclude_file: list):
+    dir_list = os.listdir(f'{REPO_CLONE_PATH}{HELM_PATH}')
+    remove_list = list(set(dir_list).difference(exclude_file))
+    for file in remove_list:
+        if os.path.isfile(f'{REPO_CLONE_PATH}{HELM_PATH}/{file}'):
+            os.remove(f'{REPO_CLONE_PATH}{HELM_PATH}/{file}')
+    return True
 
 
 ap = argparse.ArgumentParser()
@@ -184,61 +163,76 @@ if __name__ == '__main__':
         help="Module names to be added in helm",
         nargs="+"
     )
+    ap.add_argument(
+        "--exclude_files",
+        "-ef",
+        required=False,
+        default=[],
+        help="Files to be excluded from deleting from repo",
+        nargs="+"
+    )
     arguments = vars(ap.parse_args())
     _ilens_version = arguments["ilens_version"]
     _release_version = arguments["release_version"]
     _client_name = arguments['client_name']
     _git_repos = arguments["git_repos"]
     _module_names = arguments["module_names"]
+    _exclude_files = arguments["exclude_files"]
     if not _ilens_version or not _release_version or not _client_name or not (_git_repos or _module_names) or not global_configmap:
         print("global_configmap, git_repos, module_names, client_name, ilens_version and release_version details not found!!!!!")
         sys.exit()
     _branch = f"{_client_name}_{_ilens_version}.{_release_version}"
-    if not pull_global_config(repo_link=helm_repo, private_token=git_access_token, branch=_branch, file_name=global_configmap):
-        logging.error(f"Cannot clone helm repo with branch: {_branch}")
-        sys.exit()
-    with open("config.json", "r") as f:
-        data = json.load(f)
-    if _git_repos:
-        data['git_modules'] = [x for x in data['git_modules'] if x['git_link'] in _git_repos]
-    else:
-        data['git_modules'] = [x for x in data['git_modules'] if x['module_name'] in _module_names]
-    global_config_data = convert_yaml_to_json(global_configmap)
-    remove_all_files_from_repo(repo_link=helm_repo, private_token=git_access_token, branch=_branch, exclude_file=[])
-    for _data in data.get('git_modules'):
-        _ilens_version = _data.get("ilens_version", _ilens_version)
-        _release_version = _data.get("ilens_version", _release_version)
-        git_link = _data["git_link"]
-        branch = _data["branch"]
-        variables_file = _data.get("variables_file") or "variables.yml"
-
-        module_path = os.path.join("tmp")
-
-        module_name = _data['module_name']
-        module_path = os.path.join(module_path, module_name)
-        if not os.path.exists(module_path):
-            os.makedirs(module_path)
-        variables_file_path = os.path.join(module_path, variables_file)
-        if not clone_repository_with_defined_file(repo_link=git_link, clone_branch=branch,
-                                                  file_output_path=variables_file_path,
-                                                  private_token=git_access_token, clone_file_path=variables_file):
-            logging.debug("Failed to clone module!! Skipping Helm File Preparation")
-            continue
-        _module_data = convert_yaml_to_json(variables_file_path)
-        global_config_vars = global_config_data['data']
-        env_variables_from_yml = _module_data.get('deployment', {}).get('environmentVar', [])
-        env_variables_from_yml = {_v['name']:_v['value'] for _v in env_variables_from_yml if
-                          {'name', 'value'}.issubset(set(list(_v.keys())))}
-        env_variables_from_yml |= global_config_vars
-        template_file = _data.get("template_file") or "helm_service_deployment.yaml"
-        session_obj = get_db_for_func()
-        module_info = db_handler.get_module_versions(
-            input_data=GetRequest(module_name=module_name, client='iLens', ilens_version=_ilens_version,
-                                  release_version=_release_version), db=session_obj)
-        session_obj.close()
-        _data["image_tag"] = module_info.get("image_tag", "-")
-        render_helm_chart(helm_name=module_name, data_dict=_data, variables_list=env_variables_from_yml,
-                          helm_template_file=template_file)
-    push_helm_deployments(helm_repo, git_access_token, _branch)
-    shutil.rmtree("./tmp")
-    shutil.rmtree(HELM_STORE_PATH)
+    try:
+        if not (helm_push_repo:=pull_global_config(repo_link=helm_repo, branch=_branch, file_name=global_configmap)):
+            logging.error(f"Cannot clone helm repo with branch: {_branch}")
+            sys.exit()
+        with open("config.json", "r") as f:
+            data = json.load(f)
+        if _git_repos:
+            data['git_modules'] = [x for x in data['git_modules'] if x['git_link'] in _git_repos]
+        else:
+            data['git_modules'] = [x for x in data['git_modules'] if x['module_name'] in _module_names]
+        global_config_data = convert_yaml_to_json(global_configmap)
+        remove_all_files_from_repo(exclude_file=_exclude_files)
+        for _data in data.get('git_modules'):
+            _ilens_version = _data.get("ilens_version", _ilens_version)
+            _release_version = _data.get("ilens_version", _release_version)
+            git_link = _data["git_link"]
+            branch = _data["branch"]
+            variables_file = _data.get("variables_file") or "variables.yml"
+
+            module_path = os.path.join("tmp")
+
+            module_name = _data['module_name']
+            module_path = os.path.join(module_path, module_name)
+            if not os.path.exists(module_path):
+                os.makedirs(module_path)
+            variables_file_path = os.path.join(module_path, variables_file)
+            if not clone_repository_with_defined_file(repo_link=git_link, clone_branch=branch,
+                                                    file_output_path=variables_file_path,
+                                                    private_token=git_access_token, clone_file_path=variables_file):
+                logging.debug("Failed to clone module!! Skipping Helm File Preparation")
+                continue
+            _module_data = convert_yaml_to_json(variables_file_path)
+            global_config_vars = global_config_data['data']
+            env_variables_from_yml = _module_data.get('deployment', {}).get('environmentVar', [])
+            env_variables_from_yml = {_v['name']:_v['value'] for _v in env_variables_from_yml if
+                            {'name', 'value'}.issubset(set(list(_v.keys())))}
+            env_variables_from_yml |= global_config_vars
+            template_file = _data.get("template_file") or "helm_service_deployment.yaml"
+            session_obj = get_db_for_func()
+            module_info = db_handler.get_module_versions(
+                input_data=GetRequest(module_name=module_name, client='iLens', ilens_version=_ilens_version,
+                                    release_version=_release_version), db=session_obj)
+            session_obj.close()
+            _data["image_tag"] = module_info.get("image_tag", "-")
+            render_helm_chart(helm_name=module_name, data_dict=_data, variables_list=env_variables_from_yml,
+                            helm_template_file=template_file)
+        push_helm_deployments(helm_push_repo)
+    except Exception as e:
\ No newline at end of file
+        logging.exception(f'Unable to update helms: {e.args}')
+    finally:
+        shutil.rmtree(REPO_CLONE_PATH)
+        shutil.rmtree("./tmp")
+        shutil.rmtree(HELM_STORE_PATH)
+        os.remove(f"./{global_configmap}")
\ No newline at end of file

requirements.txt

-GitPython==3.1.27
+anyio==3.6.1
+bandit==1.7.4
+certifi @ file:///opt/conda/conda-bld/certifi_1663615672595/work/certifi
+charset-normalizer==2.1.1
+click==8.1.3
+fastapi==0.73.0
 gitdb==4.0.9
-python-dotenv~=0.21.0
-jinja2~=3.1.2
-fastapi~=0.73.0
-sqlalchemy~=1.3.24
-python-gitlab~=3.9.0
-ruamel.yaml~=0.17.21
+GitPython==3.1.27
+h11==0.14.0
+httptools==0.5.0
+idna==3.4
+Jinja2==3.1.2
+MarkupSafe==2.1.1
\ No newline at end of file
+pbr==5.10.0
+pendulum==2.1.2
+pydantic==1.10.2
+python-dateutil==2.8.2
+python-dotenv==0.21.0
+python-gitlab==3.9.0
+pytzdata==2020.1
+PyYAML==6.0
+requests==2.28.1
+requests-toolbelt==0.9.1
+ruamel.yaml==0.17.21
+ruamel.yaml.clib==0.2.6
+six==1.16.0
+smmap==5.0.0
+sniffio==1.3.0
+SQLAlchemy==1.3.24
+SQLAlchemy-Utils==0.38.3
+starlette==0.17.1
+stevedore==4.0.0
+typing_extensions==4.3.0
+urllib3==1.26.12
+uvicorn==0.18.3
+uvloop==0.17.0
+watchfiles==0.17.0
+websockets==10.3