updated templates

.env

@@ -3,3 +3,4 @@ CONFIG_MAP_VARIABLES=MONGO_URI
 HELM_REPO = https://gitlab-pm.knowledgelens.com/faizan.azim/utest
 GIT_USERNAME = faizan.azim
 GIT_TOKEN = MFz2FuGSasbQtBkpdfr6
+GLOBAL_VARIABLES_FILE=ilens-global-configmap.yml

config.json

@@ -6,7 +6,8 @@
       "command": [],
       "enableMountVolume": true,
       "module_port": 45678,
-      "node_port": 98765
+      "node_port": 98765,
+      "module_name": "workflow-management"
     },
     {
       "git_link": "https://gitlab-pm.knowledgelens.com/KnowledgeLens/Products/iLens-2.0/core/server/ilens-assistant/form-management.git",
@@ -14,7 +15,8 @@
       "command": [],
       "enableMountVolume": true,
       "module_port": 45678,
-      "node_port": 98765
+      "node_port": 98765,
+      "module_name": "form-management"
     },
     {
       "git_link": "https://gitlab-pm.knowledgelens.com/KnowledgeLens/Products/iLens-2.0/core/server/ebpr-report-engine.git",
@@ -22,7 +24,8 @@
       "command": [],
       "enableMountVolume": true,
       "module_port": 45678,
-      "node_port": 98765
+      "node_port": 98765,
+      "module_name":"ebpr-report-engine"
     },
     {
       "git_link": "https://gitlab-pm.knowledgelens.com/KnowledgeLens/Products/iLens-2.0/core/server/ebpr-report-engine.git",
@@ -38,7 +41,8 @@
         "-c",
         "2"
       ],
-      "enableMountVolume": true
+      "enableMountVolume": true,
+      "module_name": "ebpr-report-engine-celery"
     }
   ]
 }
\ No newline at end of file

helm_automate_script.py

@@ -18,10 +18,12 @@ from scripts.core import ILensVersionHandler
 from scripts.db.psql.databases import get_db_for_func
 from scripts.schemas import GetRequest
 
+default_link = "https://gitlab-pm.knowledgelens.com/"
 git_user_name = os.environ.get("GIT_USERNAME", default="harshavardhan.c")
 git_access_token = os.environ.get("GIT_TOKEN", default="FEMA6PnP63fJCs6DrtZJ")
 config_variables = os.environ.get("CONFIG_MAP_VARIABLES", default="").split(",")
 helm_repo = os.environ.get("HELM_REPO", default="")
+global_configmap = os.environ.get("GLOBAL_VARIABLES_FILE", default="")
 
 HELM_PATH = "/ilens-core/ilens-modules"
 HELM_STORE_PATH = "./helm-charts"
@@ -38,7 +40,7 @@ def render_helm_chart(helm_name, data_dict, variables_list, helm_template_file):
             loader=jinja2.FileSystemLoader(searchpath='./templates'),
             trim_blocks=True,
             variable_start_string='<{', variable_end_string='}>')
-        _render = environment.get_template(helm_template_file).render(module_name=helm_name, **data_dict,
+        _render = environment.get_template(helm_template_file).render(**data_dict,
                                                                       variables=variables_list)
         with open(helm_path, "w") as fp:
             fp.write(_render)
@@ -61,7 +63,7 @@ def clone_repository(repo_link, module_output_path, clone_branch):
 
 def clone_repository_with_defined_file(repo_link: str, file_output_path, clone_branch, private_token, clone_file_path):
     try:
-        base_url = os.environ.get("GIT_BASE_URL", default="https://gitlab-pm.knowledgelens.com/")
+        base_url = os.environ.get("GIT_BASE_URL", default=default_link)
         repo_link_split = repo_link.split(base_url)
         if not repo_link_split:
             return False
@@ -91,9 +93,9 @@ def convert_yaml_to_json(yaml_file_path):
         logging.exception(f"Exception Occurred while reading the yaml file {e.args}")
         return {}
     
-def push_helm_deployments(repo_link:str, private_token:str, ilens_version, release_version, client_name):
+def push_helm_deployments(repo_link:str, private_token:str, branch: str):
     try:
-        base_url = os.environ.get("GIT_BASE_URL", default="https://gitlab-pm.knowledgelens.com/")
+        base_url = os.environ.get("GIT_BASE_URL", default=default_link)
         repo_link_split = repo_link.split(base_url)
         if not repo_link_split:
             return False
@@ -103,11 +105,6 @@ def push_helm_deployments(repo_link:str, private_token:str, ilens_version, relea
         if not pl:
             return False
         pl = pl[0]
-        pl.branches.create({ 'branch': f"{client_name}_{ilens_version}.{release_version}", 'ref': pl.default_branch})
-        commit_data = {
-            'branch': f"{client_name}_{ilens_version}.{release_version}",
-            'commit_message': f"{client_name}_{ilens_version}.{release_version} helm creation"
-        }
         commit_actions = []
         files_list = os.listdir(HELM_STORE_PATH)
         for file in files_list:
@@ -117,11 +114,41 @@ def push_helm_deployments(repo_link:str, private_token:str, ilens_version, relea
                 'content': open(f'{HELM_STORE_PATH}/{file}').read()
             }
             commit_actions.append(_action)
-        commit_data |= {'actions': commit_actions}
+        commit_data = {'branch': branch, 'commit_message': f"{branch} helm creation"} | {'actions': commit_actions}
+
         pl.commits.create(commit_data)
     except Exception as e:
         logging.exception(f'Exception while pushing helm deployments: {e.args}')
         
+def pull_global_config(repo_link: str, private_token:str, branch: str, file_name: str):
+    base_url = os.environ.get("GIT_BASE_URL", default=default_link)
+    repo_link_split = repo_link.split(base_url)
+    if not repo_link_split:
+        return False
+    gl = gitlab.Gitlab(url=base_url, private_token=private_token)
+    search_str = repo_link_split[-1].replace(".git", "")
+    if pl := gl.projects.list(search=search_str):
+        pl = pl[0]
+    else:
+        return False
+    with open(file_name, 'wb') as f:
+        pl.files.raw(file_path=f'{HELM_PATH[1:]}/{file_name}', ref=branch, streamed=True, action=f.write)
+    return True
+
+def remove_all_files_from_repo(repo_link: str, private_token: str, branch: str, exclude_file: list):
+    base_url = os.environ.get("GIT_BASE_URL", default=default_link)
+    repo_link_split = repo_link.split(base_url)
+    if not repo_link_split:
+        return False
+    gl = gitlab.Gitlab(url=base_url, private_token=private_token)
+    search_str = repo_link_split[-1].replace(".git", "")
+    if pl := gl.projects.list(search=search_str):
+        pl = pl[0]
+    else:
+        return False
+    items = pl.repository_tree(path=HELM_PATH, ref=branch)
+    print(items)
+
 
 ap = argparse.ArgumentParser()
 db_handler = ILensVersionHandler()
@@ -155,17 +182,35 @@ if __name__ == '__main__':
         help="Git repos to be added in helm",
         nargs="+"
     )
+    ap.add_argument(
+        "--module_names",
+        "-mn",
+        required=False,
+        default=None,
+        help="Module names to be added in helm",
+        nargs="+"
+    )
     arguments = vars(ap.parse_args())
     _ilens_version = arguments["ilens_version"]
     _release_version = arguments["release_version"]
     _client_name = arguments['client_name']
     _git_repos = arguments["git_repos"]
-    if not _ilens_version or not _release_version or not _client_name or not _git_repos:
-        print("git_repos, client_name, ilens_version and release_version details not found!!!!!")
+    _module_names = arguments["module_names"]
+    if not _ilens_version or not _release_version or not _client_name or not (_git_repos or _module_names) or not global_configmap:
+        print("global_configmap, git_repos, module_names, client_name, ilens_version and release_version details not found!!!!!")
+        sys.exit()
+    _branch = f"{_client_name}_{_ilens_version}.{_release_version}"
+    if not pull_global_config(repo_link=helm_repo, private_token=git_access_token, branch=_branch, file_name=global_configmap):
+        logging.error(f"Cannot clone helm repo with branch: {_branch}")
         sys.exit()
     with open("config.json", "r") as f:
         data = json.load(f)
+    if _git_repos:
         data['git_modules'] = [x for x in data['git_modules'] if x['git_link'] in _git_repos]
+    else:
+        data['git_modules'] = [x for x in data['git_modules'] if x['module_name'] in _module_names]
+    global_config_data = convert_yaml_to_json(global_configmap)
+    remove_all_files_from_repo(repo_link=helm_repo, private_token=git_access_token, branch=_branch, exclude_file=[])
     for _data in data.get('git_modules'):
         _ilens_version = _data.get("ilens_version", _ilens_version)
         _release_version = _data.get("ilens_version", _release_version)
@@ -175,7 +220,7 @@ if __name__ == '__main__':
 
         module_path = os.path.join("tmp")
 
-        module_name = git_link.split("/")[-1].split(".git")[0]
+        module_name = _data['module_name']
         module_path = os.path.join(module_path, module_name)
         if not os.path.exists(module_path):
             os.makedirs(module_path)
@@ -186,9 +231,11 @@ if __name__ == '__main__':
             logging.debug("Failed to clone module!! Skipping Helm File Preparation")
             continue
         _module_data = convert_yaml_to_json(variables_file_path)
+        global_config_vars = global_config_data['data']
         env_variables_from_yml = _module_data.get('deployment', {}).get('environmentVar', [])
-        env_variables_from_yml = [_v for _v in env_variables_from_yml if
-                          {'name', 'value'}.issubset(set(list(_v.keys())))]
+        env_variables_from_yml = {_v['name']:_v['value'] for _v in env_variables_from_yml if
+                          {'name', 'value'}.issubset(set(list(_v.keys())))}
+        env_variables_from_yml |= global_config_vars
         template_file = _data.get("template_file") or "helm_service_deployment.yaml"
         session_obj = get_db_for_func()
         module_info = db_handler.get_module_versions(
@@ -198,6 +245,6 @@ if __name__ == '__main__':
         _data["image_tag"] = module_info.get("image_tag", "-")
         render_helm_chart(helm_name=module_name, data_dict=_data, variables_list=env_variables_from_yml,
                           helm_template_file=template_file)
-    push_helm_deployments(helm_repo, git_access_token, _ilens_version, _release_version, _client_name)
+    push_helm_deployments(helm_repo, git_access_token, _branch)
     shutil.rmtree("./tmp")
     shutil.rmtree(HELM_STORE_PATH)
\ No newline at end of file

logs/helm-automation-script.log

@@ -605,3 +605,48 @@ Traceback (most recent call last):
   File "C:\Users\faizan.azim\Anaconda3\envs\helm-script\lib\site-packages\gitlab\exceptions.py", line 327, in wrapped_f
     raise error(e.error_message, e.response_code, e.response_body) from e
 gitlab.exceptions.GitlabCreateError: 400: Branch already exists
+2022-09-27 15:19:53 - ERROR  - [MainThread:render_helm_chart():48] - Exception occurred while rendering the helm file workflow-management - ("jinja2.environment.Template.render() got multiple values for keyword argument 'module_name'",)
+Traceback (most recent call last):
+  File "helm_automate_script.py", line 42, in render_helm_chart
+    _render = environment.get_template(helm_template_file).render(module_name=helm_name, **data_dict,
+TypeError: jinja2.environment.Template.render() got multiple values for keyword argument 'module_name'
+2022-09-27 15:19:53 - ERROR  - [MainThread:render_helm_chart():48] - Exception occurred while rendering the helm file ebpr-report-engine - ("jinja2.environment.Template.render() got multiple values for keyword argument 'module_name'",)
+Traceback (most recent call last):
+  File "helm_automate_script.py", line 42, in render_helm_chart
+    _render = environment.get_template(helm_template_file).render(module_name=helm_name, **data_dict,
+TypeError: jinja2.environment.Template.render() got multiple values for keyword argument 'module_name'
+2022-09-27 15:19:53 - ERROR  - [MainThread:render_helm_chart():48] - Exception occurred while rendering the helm file ebpr-report-engine - ("jinja2.environment.Template.render() got multiple values for keyword argument 'module_name'",)
+Traceback (most recent call last):
+  File "helm_automate_script.py", line 42, in render_helm_chart
+    _render = environment.get_template(helm_template_file).render(module_name=helm_name, **data_dict,
+TypeError: jinja2.environment.Template.render() got multiple values for keyword argument 'module_name'
+2022-09-27 15:22:15 - ERROR  - [MainThread:render_helm_chart():48] - Exception occurred while rendering the helm file workflow-management - ("jinja2.environment.Template.render() got multiple values for keyword argument 'module_name'",)
+Traceback (most recent call last):
+  File "helm_automate_script.py", line 42, in render_helm_chart
+    _render = environment.get_template(helm_template_file).render(module_name=helm_name, **data_dict,
+TypeError: jinja2.environment.Template.render() got multiple values for keyword argument 'module_name'
+2022-09-27 15:26:15 - ERROR  - [MainThread:render_helm_chart():48] - Exception occurred while rendering the helm file workflow-management - ("jinja2.environment.Template.render() got multiple values for keyword argument 'module_name'",)
+Traceback (most recent call last):
+  File "helm_automate_script.py", line 42, in render_helm_chart
+    _render = environment.get_template(helm_template_file).render(module_name=helm_name, **data_dict,
+TypeError: jinja2.environment.Template.render() got multiple values for keyword argument 'module_name'
+2022-09-27 16:37:45 - ERROR  - [MainThread:push_helm_deployments():124] - Exception while pushing helm deployments: ('Branch already exists',)
+Traceback (most recent call last):
+  File "C:\Users\faizan.azim\Anaconda3\envs\helm-script\lib\site-packages\gitlab\exceptions.py", line 325, in wrapped_f
+    return f(*args, **kwargs)
+  File "C:\Users\faizan.azim\Anaconda3\envs\helm-script\lib\site-packages\gitlab\mixins.py", line 300, in create
+    server_data = self.gitlab.http_post(path, post_data=data, files=files, **kwargs)
+  File "C:\Users\faizan.azim\Anaconda3\envs\helm-script\lib\site-packages\gitlab\client.py", line 1015, in http_post
+    result = self.http_request(
+  File "C:\Users\faizan.azim\Anaconda3\envs\helm-script\lib\site-packages\gitlab\client.py", line 798, in http_request
+    raise gitlab.exceptions.GitlabHttpError(
+gitlab.exceptions.GitlabHttpError: 400: Branch already exists
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "helm_automate_script.py", line 107, in push_helm_deployments
+    pl.branches.create({ 'branch': f"{client_name}_{ilens_version}.{release_version}", 'ref': pl.default_branch})
+  File "C:\Users\faizan.azim\Anaconda3\envs\helm-script\lib\site-packages\gitlab\exceptions.py", line 327, in wrapped_f
+    raise error(e.error_message, e.response_code, e.response_body) from e
+gitlab.exceptions.GitlabCreateError: 400: Branch already exists

templates/helm_deployment.yaml

@@ -32,10 +32,10 @@ deployment:
         secretKeyRef:
           name: mongo-uri
           key: MONGO_URI
-{% for i in variables %}
-{% if 'name' in i and 'value' in i %}
-    - name: '<{ i.name }>',
-      value: '<{ i.value }>'
+{% for k,v in variables.items() %}
+{% if k and v %}
+    - name: '<{ k }>',
+      value: '<{ v }>'
 {% endif %}
 {% endfor %}
 

templates/helm_service_deployment.yaml

@@ -46,10 +46,10 @@ deployment:
         secretKeyRef:
           name: mongo-uri
           key: MONGO_URI
-{% for i in variables %}
-{% if 'name' in i and 'value' in i %}
-    - name: '<{ i.name }>',
-      value: '<{ i.value }>'
+{% for k, v in variables.items() %}
+{% if k and v %}
+    - name: '<{ k }>',
+      value: '<{ v }>'
 {% endif %}
 {% endfor %}