Skip to content

H
Helm-Automation-Script
Commit 29ed9a0f authored by harshavardhan.c's avatar harshavardhan.c
Browse files
Options

Dev: integration fixes while generating helm-scripts dynamically.

parent f413d47e
Hide whitespace changes
Inline Side-by-side
Showing with 288 additions and 354 deletions
automation_script.py
View file @ 29ed9a0f
...@@ -4,9 +4,11 @@ import sys ...@@ -4,9 +4,11 @@ import sys
from dotenv import load_dotenv from dotenv import load_dotenv
from docker_automation.docker_automate_script import DockerVersionUpgrade
load_dotenv() load_dotenv()
from helm_automation.helm_register import HelmRegistration
from docker_automation.docker_automate_script import DockerVersionUpgrade
from helm_automation.helm_automate_script import HelmVersionUpgrade
from docker_automation.docker_register import DockerRegistration from docker_automation.docker_register import DockerRegistration
from scripts.config import EnvironmentVariables from scripts.config import EnvironmentVariables
from scripts.logging import logger from scripts.logging import logger
...@@ -25,7 +27,11 @@ def func_mapper(operation_type, parameters): ...@@ -25,7 +27,11 @@ def func_mapper(operation_type, parameters):
"docker_registration": DockerRegistration(arguments=parameters, git_user_name=git_user_name, "docker_registration": DockerRegistration(arguments=parameters, git_user_name=git_user_name,
git_access_token=git_access_token).module_registration, git_access_token=git_access_token).module_registration,
"docker_version_upgrade": DockerVersionUpgrade(arguments=parameters, git_user_name=git_user_name, "docker_version_upgrade": DockerVersionUpgrade(arguments=parameters, git_user_name=git_user_name,
git_access_token=git_access_token).module_registration git_access_token=git_access_token).module_registration,
"helm_version_upgrade": HelmVersionUpgrade(arguments=parameters, git_user_name=git_user_name,
git_access_token=git_access_token).module_registration,
"helm_registration": HelmRegistration(arguments=parameters, git_user_name=git_user_name,
git_access_token=git_access_token).module_registration
} }
return func_mapper_dict.get(operation_type) return func_mapper_dict.get(operation_type)
......
docker_automation/docker_automate_script.py
View file @ 29ed9a0f
...@@ -37,9 +37,9 @@ class DockerVersionUpgrade: ...@@ -37,9 +37,9 @@ class DockerVersionUpgrade:
_module_names = self.arguments["module_names"] _module_names = self.arguments["module_names"]
_branch_name = self.arguments['branch_name'] or "master" _branch_name = self.arguments['branch_name'] or "master"
_ilens_version = self.arguments["ilens_version"] _ilens_version = self.arguments["ilens_version"]
if not _ilens_version or not _release_version or not _client_name or not _git_repos and not _module_names or not global_configmap: if not _ilens_version or not _release_version or not _client_name or not global_configmap:
print( print(
"global_configmap, git_repos, module_names, client_name, ilens_version and release_version details " "global_configmap, client_name, ilens_version and release_version details "
"not found!!!!!") "not found!!!!!")
sys.exit() sys.exit()
variables_file = "variables.yml" variables_file = "variables.yml"
......
helm_automation/helm_automate_script.py
View file @ 29ed9a0f
from scripts.config import EnvironmentVariables
from scripts.core.helm_handler import HelmHandler
if __name__ == "__main__": if __name__ == "__main__":
from dotenv import load_dotenv from dotenv import load_dotenv
load_dotenv() load_dotenv()
import argparse
import logging import logging
import os import os
import sys import sys
...@@ -10,7 +12,6 @@ import time ...@@ -10,7 +12,6 @@ import time
from scripts.core import ILensVersionHandler from scripts.core import ILensVersionHandler
from scripts.core.git_handler import GitHandler from scripts.core.git_handler import GitHandler
from scripts.core.helm_handler import HelmHandler
from scripts.db.psql.databases import get_db_for_func from scripts.db.psql.databases import get_db_for_func
from scripts.schemas import GetRequest from scripts.schemas import GetRequest
from scripts.utils.common_utils import CommonUtils from scripts.utils.common_utils import CommonUtils
...@@ -18,157 +19,123 @@ from scripts.utils.common_utils import CommonUtils ...@@ -18,157 +19,123 @@ from scripts.utils.common_utils import CommonUtils
HELM_PATH = "/ilens-core/ilens-modules" HELM_PATH = "/ilens-core/ilens-modules"
HELM_STORE_PATH = "./helm-charts" HELM_STORE_PATH = "./helm-charts"
git_handler_obj = GitHandler(user_name=git_user_name, access_token=git_access_token) global_configmap = EnvironmentVariables.global_configmap
helm_repo = EnvironmentVariables.helm_repo
class HelmVersionUpgrade:
def __init__(self, arguments: dict, git_user_name: str, git_access_token: str):
self.arguments = arguments
self.db_handler = ILensVersionHandler()
self.helm_handler = HelmHandler()
self.git_handler_obj = GitHandler(user_name=git_user_name, access_token=git_access_token)
self.common_util = CommonUtils()
ap = argparse.ArgumentParser() def module_registration(self):
db_handler = ILensVersionHandler() helm_temp_path = f"{int(time.time())}_helm_tmp_path"
common_util = CommonUtils() general_temp_path = f"{int(time.time())}_tmp"
helm_handler = HelmHandler() output_path = f"{int(time.time())}_helm-charts"
if __name__ == '__main__': try:
ap.add_argument( _ilens_version = self.arguments["ilens_version"]
"--ilens_version", _release_version = self.arguments["release_version"]
"-iv", _client_name = self.arguments['client_name']
required=False, _git_repos = self.arguments["git_repos"]
default=None, _module_names = self.arguments["module_names"]
help="ILens Version Tag", _branch_name = self.arguments['branch_name'] or "master"
) if not _ilens_version or not _release_version or not _client_name or not (
ap.add_argument( _git_repos or _module_names) or not global_configmap:
"--release_version", print(
"-rv", "global_configmap, git_repos, client_name, ilens_version and release_version details not found!!!!!")
required=False, sys.exit()
default=None, _branch = f"{_client_name}_{_ilens_version}.{_release_version}"
help="ILens Release Tag", if not os.path.exists(helm_temp_path):
) os.makedirs(helm_temp_path)
ap.add_argument( if not os.path.exists(output_path):
"--client_name", os.makedirs(output_path)
"-cn", helm_path = os.path.join(helm_temp_path, "helm-charts")
required=False, if not self.git_handler_obj.clone_repository(repo_link=helm_repo, module_output_path=helm_path,
default=None, clone_branch=_client_name):
help="Client Name Tag" logging.error(f"Cannot clone helm repo with branch: {_client_name}")
) sys.exit()
ap.add_argument( base_helm_directory_path = os.path.join(helm_temp_path, "helm-charts", "ilens-core", "ilens-modules")
"--branch_name", if len(_module_names) == 1 and _module_names[0].lower() == "all":
"-b", updated_list = []
required=False, files_info = os.listdir(base_helm_directory_path)
default=None, sorted_files = list(filter(lambda f: f.endswith(".yml"), files_info))
help="Branch Name" _module_names = [_each.replace(".yml", "") for _each in sorted_files]
)
ap.add_argument(
"--git_repos",
"-gr",
required=False,
default=None,
help="Git repos to be added in helm",
nargs="+"
)
ap.add_argument(
"--module_names",
"-mn",
required=False,
default=None,
help="Module names to be added in helm",
nargs="+"
)
HELM_TEMP_PATH = f"{int(time.time())}_helm_tmp_path"
GENERAL_TEMP_PATH = f"{int(time.time())}_tmp"
OUTPUT_PATH = f"{int(time.time())}_helm-charts"
try:
arguments = vars(ap.parse_args())
_ilens_version = arguments["ilens_version"]
_release_version = arguments["release_version"]
_client_name = arguments['client_name']
_git_repos = arguments["git_repos"]
_module_names = arguments["module_names"]
_branch_name = arguments['branch_name'] or "master"
if not _ilens_version or not _release_version or not _client_name or not (
_git_repos or _module_names) or not global_configmap:
print(
"global_configmap, git_repos, module_names, client_name, ilens_version and release_version details not found!!!!!")
sys.exit()
_branch = f"{_client_name}_{_ilens_version}.{_release_version}"
if not os.path.exists(HELM_TEMP_PATH):
os.makedirs(HELM_TEMP_PATH)
if not os.path.exists(OUTPUT_PATH):
os.makedirs(OUTPUT_PATH)
helm_path = os.path.join(HELM_TEMP_PATH, "helm-charts")
if not git_handler_obj.clone_repository(repo_link=helm_repo, module_output_path=helm_path,
clone_branch=_client_name):
logging.error(f"Cannot clone helm repo with branch: {_client_name}")
sys.exit()
base_helm_directory_path = os.path.join(HELM_TEMP_PATH, "helm-charts", "ilens-core", "ilens-modules")
if len(_module_names) == 1 and _module_names[0].lower() == "all":
updated_list = []
files_info = os.listdir(base_helm_directory_path)
sorted_files = list(filter(lambda f: f.endswith(".yml"), files_info))
_module_names = [_each.replace(".yml", "") for _each in sorted_files]
global_config_data = common_util.convert_yaml_to_define_obj( global_config_data = self.common_util.convert_yaml_to_define_obj(
os.path.join(base_helm_directory_path, global_configmap)) os.path.join(base_helm_directory_path, global_configmap))
# global_config_data = common_util.convert_yaml_to_json("ilens-global-configmap.yml") # global_config_data = common_util.convert_yaml_to_json("ilens-global-configmap.yml")
variables_file = "variables.yml" variables_file = "variables.yml"
template_path = os.path.join(GENERAL_TEMP_PATH, "../templates") template_path = os.path.join(general_temp_path, "../templates")
if not os.path.exists(template_path): if not os.path.exists(template_path):
os.makedirs(template_path) os.makedirs(template_path)
for _module in _module_names: for _module in _module_names:
module_path = os.path.join(GENERAL_TEMP_PATH) module_path = os.path.join(general_temp_path)
module_path = os.path.join(module_path, _module) module_path = os.path.join(module_path, _module)
if not os.path.exists(module_path): if not os.path.exists(module_path):
os.makedirs(module_path) os.makedirs(module_path)
helm_out_file_path = os.path.join(OUTPUT_PATH, f'{_module}.yml') helm_out_file_path = os.path.join(output_path, f'{_module}.yml')
if os.path.exists(helm_out_file_path): if os.path.exists(helm_out_file_path):
logging.debug(f"Helm Deployment File found for selected the module {_module}") logging.debug(f"Helm Deployment File found for selected the module {_module}")
continue continue
variables_file_path = os.path.join(module_path, variables_file) variables_file_path = os.path.join(module_path, variables_file)
git_info = git_handler_obj.get_git_url_by_module_name(module_name=_module) git_info = self.git_handler_obj.get_git_url_by_module_name(module_name=_module)
if not git_info: if not git_info:
logging.debug("Failed to fetch module info!! Skipping Helm File Preparation") logging.debug("Failed to fetch module info!! Skipping Helm File Preparation")
continue continue
if not git_handler_obj.clone_repository_with_defined_file(repo_link=git_info, clone_branch=_branch_name, if not self.git_handler_obj.clone_repository_with_defined_file(repo_link=git_info,
file_output_path=variables_file_path, clone_branch=_branch_name,
clone_file_path=variables_file): file_output_path=variables_file_path,
logging.debug("Failed to clone module!! Skipping Helm File Preparation") clone_file_path=variables_file):
continue logging.debug("Failed to clone module!! Skipping Helm File Preparation")
_module_data = common_util.convert_yaml_to_define_obj(variables_file_path) continue
module_env_variables = _module_data.get('deployment', {}).get('environmentVar', []) _module_data = self.common_util.convert_yaml_to_define_obj(variables_file_path)
module_env_variables = {_v['name']: _v for _v in module_env_variables} module_env_variables = _module_data.get('deployment', {}).get('environmentVar', [])
template_file = os.path.join(template_path, f'{_module}.yml') module_env_variables = {_v['name']: _v for _v in module_env_variables}
session_obj = get_db_for_func() template_file = os.path.join(template_path, f'{_module}.yml')
module_info = db_handler.get_module_versions( session_obj = get_db_for_func()
input_data=GetRequest(module_name=_module, client='iLens', ilens_version=_ilens_version, module_info = self.db_handler.get_module_versions(
release_version=_release_version), db=session_obj) input_data=GetRequest(module_name=_module, client='iLens', ilens_version=_ilens_version,
session_obj.close() release_version=_release_version), db=session_obj)
image_url = module_info.get("image_url", '') if module_info else '' session_obj.close()
existing_yml_path = os.path.join(base_helm_directory_path, f'{_module}.yml') image_url = module_info.get("image_url", '') if module_info else ''
helm_handler.create_existing_helm_deployment_file(template_yml_path=existing_yml_path, image_tag=image_url, existing_yml_path = os.path.join(base_helm_directory_path, f'{_module}.yml')
module_env_variables=module_env_variables, self.helm_handler.create_existing_helm_deployment_file(template_yml_path=existing_yml_path,
template_file=template_file, template_path=template_path, image_tag=image_url,
helm_out_file_path=helm_out_file_path, module_env_variables=module_env_variables,
global_config_data=global_config_data, template_file=template_file,
module_name=_module) template_path=template_path,
files_info = os.listdir(base_helm_directory_path) helm_out_file_path=helm_out_file_path,
sorted_files = list(filter(lambda f: f.startswith(_module), files_info)) global_config_data=global_config_data,
updated_files_info = [x for x in sorted_files if f'{_module}.yml' != x] module_name=_module)
for _file in updated_files_info: files_info = os.listdir(base_helm_directory_path)
sub_file_path = os.path.join(base_helm_directory_path, _file) sorted_files = list(filter(lambda f: f.startswith(_module), files_info))
helm_out_file_path = os.path.join(OUTPUT_PATH, _file) updated_files_info = [x for x in sorted_files if f'{_module}.yml' != x]
template_file = os.path.join(template_path, _file) for _file in updated_files_info:
helm_handler.create_existing_helm_deployment_file(template_yml_path=sub_file_path, image_tag=image_url, sub_file_path = os.path.join(base_helm_directory_path, _file)
module_env_variables=module_env_variables, helm_out_file_path = os.path.join(output_path, _file)
template_file=template_file, template_file = os.path.join(template_path, _file)
template_path=template_path, self.helm_handler.create_existing_helm_deployment_file(template_yml_path=sub_file_path,
helm_out_file_path=helm_out_file_path, image_tag=image_url,
global_config_data=global_config_data, module_env_variables=module_env_variables,
module_name=_file.split(".yml")[0]) template_file=template_file,
git_handler_obj.push_deployments_to_git(helm_repo, git_access_token, _branch, folder_path=OUTPUT_PATH, template_path=template_path,
base_path=helm_path) helm_out_file_path=helm_out_file_path,
git_handler_obj.create_merge_request(repo_link=helm_repo, source_branch=_branch, global_config_data=global_config_data,
destination_branch=_client_name) module_name=_file.split(".yml")[0])
except Exception as e: self.git_handler_obj.push_deployments_to_git(repo_link=helm_repo, branch=_branch, folder_path=output_path,
logging.exception(f"Exception Occurred while processing the Helm-Script Preparation {e.args}") base_path=helm_path,
finally: private_token=EnvironmentVariables.git_access_token)
# shutil.rmtree(HELM_STORE_PATH) self.git_handler_obj.create_merge_request(repo_link=helm_repo, source_branch=_branch,
# shutil.rmtree(HELM_TEMP_PATH) destination_branch=_client_name)
# shutil.rmtree(GENERAL_TEMP_PATH) except Exception as e:
# shutil.rmtree(OUTPUT_PATH) logging.exception(f"Exception Occurred while processing the Helm-Script Preparation {e.args}")
... finally:
# shutil.rmtree(HELM_STORE_PATH)
# shutil.rmtree(helm_temp_path)
# shutil.rmtree(general_temp_path)
# shutil.rmtree(output_path)
...
helm_automation/helm_register.py
View file @ 29ed9a0f
...@@ -10,7 +10,6 @@ if __name__ == "__main__": ...@@ -10,7 +10,6 @@ if __name__ == "__main__":
from scripts.db.psql.databases import get_db_for_func from scripts.db.psql.databases import get_db_for_func
from scripts.schemas import GetRequest from scripts.schemas import GetRequest
import argparse
import logging import logging
import os import os
import sys import sys
...@@ -22,201 +21,147 @@ from scripts.core.helm_handler import HelmHandler ...@@ -22,201 +21,147 @@ from scripts.core.helm_handler import HelmHandler
from scripts.utils.common_utils import CommonUtils from scripts.utils.common_utils import CommonUtils
default_link = EnvironmentVariables.default_link default_link = EnvironmentVariables.default_link
git_user_name = EnvironmentVariables.git_user_name
git_access_token = EnvironmentVariables.git_access_token
config_variables = EnvironmentVariables.config_variables
helm_repo = EnvironmentVariables.helm_repo helm_repo = EnvironmentVariables.helm_repo
global_configmap = EnvironmentVariables.global_configmap global_configmap = EnvironmentVariables.global_configmap
HELM_PATH = "/ilens-core/ilens-modules" HELM_PATH = "/ilens-core/ilens-modules"
HELM_STORE_PATH = "./helm-charts" HELM_STORE_PATH = "./helm-charts"
git_handler_obj = GitHandler(user_name=git_user_name, access_token=git_access_token)
ap = argparse.ArgumentParser() class HelmRegistration:
db_handler = ILensVersionHandler() def __init__(self, arguments: dict, git_user_name: str, git_access_token: str):
common_util = CommonUtils() self.arguments = arguments
helm_handler = HelmHandler() self.db_handler = ILensVersionHandler()
if __name__ == '__main__': self.helm_handler = HelmHandler()
ap.add_argument( self.git_handler_obj = GitHandler(user_name=git_user_name, access_token=git_access_token)
"--ilens_version", self.common_util = CommonUtils()
"-iv",
required=False,
default=None,
help="ILens Version Tag",
)
ap.add_argument(
"--release_version",
"-rv",
required=False,
default=None,
help="ILens Release Tag",
)
ap.add_argument(
"--reference_client_name",
"-rcn",
required=False,
default=None,
help="Client Name Tag"
)
ap.add_argument(
"--client_name",
"-cn",
required=False,
default=None,
help="Client Name Tag"
)
ap.add_argument(
"--branch_name",
"-b",
required=False,
default=None,
help="Branch Name"
)
ap.add_argument(
"--git_repos",
"-gr",
required=False,
default=None,
help="Git repos to be added in helm",
nargs="+"
)
ap.add_argument(
"--module_names",
"-mn",
required=False,
default=None,
help="Module names to be added in helm",
nargs="+"
)
ap.add_argument(
"--repo_info",
"-ri",
required=False,
default=None,
help="Module names to be added in helm",
nargs="+"
)
HELM_TEMP_PATH = f"{int(time.time())}_helm_tmp_path"
GENERAL_TEMP_PATH = f"{int(time.time())}_tmp"
OUTPUT_PATH = f"{int(time.time())}_helm-charts"
try:
arguments = vars(ap.parse_args())
_ilens_version = arguments["ilens_version"]
_release_version = arguments["release_version"]
_client_name = arguments['client_name']
_reference_branch = arguments['reference_client_name']
_git_repos = arguments["git_repos"]
repo_info = arguments["repo_info"]
_module_names = arguments["module_names"]
_branch_name = arguments['branch_name'] or "master"
if repo_info:
repo_info = json.loads(repo_info)
if not _ilens_version or not _release_version or not _client_name or not (
_git_repos or _module_names) or not global_configmap:
print(
"global_configmap, git_repos, module_names, client_name, ilens_version and release_version details not found!!!!!")
sys.exit()
_branch = f"{_client_name}_{_ilens_version}.{_release_version}"
if not os.path.exists(HELM_TEMP_PATH):
os.makedirs(HELM_TEMP_PATH)
if not os.path.exists(OUTPUT_PATH):
os.makedirs(OUTPUT_PATH)
helm_path = os.path.join(HELM_TEMP_PATH, "helm-charts")
if not git_handler_obj.clone_repository(repo_link=helm_repo, module_output_path=helm_path,
clone_branch=_reference_branch or _client_name):
logging.error(f"Cannot clone helm repo with branch: {_reference_branch or _client_name}")
base_helm_directory_path = os.path.join(HELM_TEMP_PATH, "helm-charts", "ilens-core", "ilens-modules")
if os.path.exists(base_helm_directory_path) and len(_module_names) == 1 and _module_names[0].lower() == "all":
updated_list = []
files_info = os.listdir(base_helm_directory_path)
sorted_files = list(filter(lambda f: f.endswith(".yml"), files_info))
_module_names = [_each.replace(".yml", "") for _each in sorted_files]
global_config_data = common_util.convert_yaml_to_define_obj( def module_registration(self):
os.path.join(base_helm_directory_path, global_configmap)) helm_temp_path = f"{int(time.time())}_helm_tmp_path"
# global_config_data = common_util.convert_yaml_to_json("ilens-global-configmap.yml") general_temp_path = f"{int(time.time())}_tmp"
if repo_info: output_path = f"{int(time.time())}_helm-charts"
global_config_data.update(repo_info) try:
variables_file = "variables.yml" _ilens_version = self.arguments["ilens_version"]
template_path = os.path.join(GENERAL_TEMP_PATH, "../templates") _release_version = self.arguments["release_version"]
if not os.path.exists(template_path): _client_name = self.arguments['client_name']
os.makedirs(template_path) _reference_branch = self.arguments['reference_client_name']
for _module in _module_names: _git_repos = self.arguments["git_repos"]
module_path = os.path.join(GENERAL_TEMP_PATH) _module_names = self.arguments["module_names"]
module_path = os.path.join(module_path, _module) _branch_name = self.arguments['branch_name'] or "master"
if not os.path.exists(module_path): repo_info = self.arguments["repo_info"]
os.makedirs(module_path) if repo_info:
helm_out_file_path = os.path.join(OUTPUT_PATH, f'{_module}.yml') repo_info = json.loads(repo_info)
if os.path.exists(helm_out_file_path): if not _ilens_version or not _release_version or not _client_name or not _git_repos and not _module_names or not global_configmap:
logging.debug(f"Helm Deployment File found for selected the module {_module}") print(
continue "global_configmap, git_repos, module_names, client_name, ilens_version and release_version details not found!!!!!")
variables_file_path = os.path.join(module_path, variables_file) sys.exit()
git_info = git_handler_obj.get_git_url_by_module_name(module_name=_module) _branch = f"{_client_name}_{_ilens_version}.{_release_version}"
if not git_info: if not os.path.exists(helm_temp_path):
logging.debug("Failed to fetch module info!! Skipping Helm File Preparation") os.makedirs(helm_temp_path)
continue if not os.path.exists(output_path):
if not git_handler_obj.clone_repository_with_defined_file(repo_link=git_info, clone_branch=_branch_name, os.makedirs(output_path)
file_output_path=variables_file_path, helm_path = os.path.join(helm_temp_path, "helm-charts")
clone_file_path=variables_file): if not self.git_handler_obj.clone_repository(repo_link=helm_repo, module_output_path=helm_path,
logging.debug("Failed to clone module!! Skipping Helm File Preparation") clone_branch=_reference_branch or _client_name):
continue logging.error(f"Cannot clone helm repo with branch: {_reference_branch or _client_name}")
_module_data = common_util.convert_yaml_to_define_obj(variables_file_path) base_helm_directory_path = os.path.join(helm_temp_path, "helm-charts", "ilens-core", "ilens-modules")
module_env_variables = _module_data.get('deployment', {}).get('environmentVar', []) if os.path.exists(base_helm_directory_path) and len(_module_names) == 1 and _module_names[
0].lower() == "all":
updated_list = []
files_info = os.listdir(base_helm_directory_path)
sorted_files = list(filter(lambda f: f.endswith(".yml"), files_info))
_module_names = [_each.replace(".yml", "") for _each in sorted_files]
global_config_data = self.common_util.convert_yaml_to_define_obj(
os.path.join(base_helm_directory_path, global_configmap))
# global_config_data = common_util.convert_yaml_to_json("ilens-global-configmap.yml")
if repo_info:
global_config_data.update(repo_info)
variables_file = "variables.yml"
template_path = os.path.join(general_temp_path, "../templates")
if not os.path.exists(template_path):
os.makedirs(template_path)
for _module in _module_names:
module_path = os.path.join(general_temp_path)
module_path = os.path.join(module_path, _module)
if not os.path.exists(module_path):
os.makedirs(module_path)
helm_out_file_path = os.path.join(output_path, f'{_module}.yml')
if os.path.exists(helm_out_file_path):
logging.debug(f"Helm Deployment File found for selected the module {_module}")
continue
variables_file_path = os.path.join(module_path, variables_file)
git_info = self.git_handler_obj.get_git_url_by_module_name(module_name=_module)
if not git_info:
logging.debug("Failed to fetch module info!! Skipping Helm File Preparation")
continue
if not self.git_handler_obj.clone_repository_with_defined_file(repo_link=git_info,
clone_branch=_branch_name,
file_output_path=variables_file_path,
clone_file_path=variables_file):
logging.debug("Failed to clone module!! Skipping Helm File Preparation")
continue
_module_data = self.common_util.convert_yaml_to_define_obj(variables_file_path)
module_env_variables = _module_data.get('deployment', {}).get('environmentVar', [])
module_env_variables = {_v['name']: _v['value'] for _v in module_env_variables if module_env_variables = {_v['name']: _v['value'] for _v in module_env_variables if
{'name', 'value'}.issubset(set(list(_v.keys())))} {'name', 'value'}.issubset(set(list(_v.keys())))}
template_file = os.path.join(template_path, f'{_module}.yml') template_file = os.path.join(template_path, f'{_module}.yml')
session_obj = get_db_for_func() session_obj = get_db_for_func()
module_info = db_handler.get_module_versions( module_info = self.db_handler.get_module_versions(
input_data=GetRequest(module_name=_module, client='iLens', ilens_version=_ilens_version, input_data=GetRequest(module_name=_module, client='iLens', ilens_version=_ilens_version,
release_version=_release_version), db=session_obj) release_version=_release_version), db=session_obj)
session_obj.close() session_obj.close()
image_url = module_info.get("image_url", '') if module_info else '' image_url = module_info.get("image_url", '') if module_info else ''
existing_yml_path = os.path.join(base_helm_directory_path, f'{_module}.yml') existing_yml_path = os.path.join(base_helm_directory_path, f'{_module}.yml')
new_module = deepcopy(_module) new_module = deepcopy(_module)
worker_type = True if "worker" in _module or "celery" in _module else False worker_type = "worker" in _module or "celery" in _module
if not os.path.exists(existing_yml_path): if not os.path.exists(existing_yml_path):
existing_yml_path = os.path.join("../templates", "helm_deployment.yml") existing_yml_path = os.path.join("../templates", "helm_deployment.yml")
new_module = "helm_deployment" new_module = "helm_deployment"
helm_handler.create_helm_deployment_file(template_yml_path=existing_yml_path, image_tag=image_url, self.helm_handler.create_helm_deployment_file(template_yml_path=existing_yml_path,
module_env_variables=module_env_variables,
helm_out_file_path=helm_out_file_path,
global_config_data=global_config_data, module_name=new_module)
else:
helm_handler.create_existing_helm_deployment_file(template_yml_path=existing_yml_path,
image_tag=image_url, image_tag=image_url,
module_env_variables=module_env_variables, module_env_variables=module_env_variables,
helm_out_file_path=helm_out_file_path, helm_out_file_path=helm_out_file_path,
global_config_data=global_config_data, global_config_data=global_config_data,
module_name=new_module, template_file=template_file, module_name=new_module)
template_path=template_path) else:
if os.path.exists(base_helm_directory_path): self.helm_handler.create_existing_helm_deployment_file(template_yml_path=existing_yml_path,
files_info = os.listdir(base_helm_directory_path) image_tag=image_url,
sorted_files = list(filter(lambda f: f.startswith(_module), files_info)) module_env_variables=module_env_variables,
updated_files_info = [x for x in sorted_files if f'{_module}.yml' != x] helm_out_file_path=helm_out_file_path,
for _file in updated_files_info: global_config_data=global_config_data,
sub_file_path = os.path.join(base_helm_directory_path, _file) module_name=new_module,
helm_out_file_path = os.path.join(OUTPUT_PATH, _file) template_file=template_file,
template_file = os.path.join(template_path, _file) template_path=template_path)
worker_type = True if "worker" in _module or "celery" in _module else False if os.path.exists(base_helm_directory_path):
helm_handler.create_existing_helm_deployment_file(template_yml_path=sub_file_path, files_info = os.listdir(base_helm_directory_path)
image_tag=image_url, sorted_files = list(filter(lambda f: f.startswith(_module), files_info))
module_env_variables=module_env_variables, updated_files_info = [x for x in sorted_files if f'{_module}.yml' != x]
template_file=template_file, for _file in updated_files_info:
template_path=template_path, sub_file_path = os.path.join(base_helm_directory_path, _file)
helm_out_file_path=helm_out_file_path, helm_out_file_path = os.path.join(output_path, _file)
global_config_data=global_config_data, template_file = os.path.join(template_path, _file)
service_type=False, worker_type = "worker" in _module or "celery" in _module
module_name=_file.split(".yml")[0]) self.helm_handler.create_existing_helm_deployment_file(template_yml_path=sub_file_path,
# git_handler_obj.push_helm_deployments(helm_repo, git_access_token, _client_name, final_helm_path=OUTPUT_PATH, image_tag=image_url,
# base_path=helm_path) module_env_variables=module_env_variables,
template_file=template_file,
template_path=template_path,
helm_out_file_path=helm_out_file_path,
global_config_data=global_config_data,
service_type=False,
module_name=_file.split(".yml")[0])
# self.git_handler_obj.push_deployments_to_git(repo_link=helm_repo,
# private_token=git_access_token,
# branch=_client_name,
# folder_path=output_path, base_path=helm_path)
except Exception as e: except Exception as e:
logging.exception(f"Exception Occurred while processing the Helm-Script Preparation {e.args}") logging.exception(f"Exception Occurred while processing the Helm-Script Preparation {e.args}")
finally: finally:
# shutil.rmtree(HELM_STORE_PATH) # shutil.rmtree(HELM_STORE_PATH)
# shutil.rmtree(HELM_TEMP_PATH) # shutil.rmtree(HELM_TEMP_PATH)
# shutil.rmtree(GENERAL_TEMP_PATH) # shutil.rmtree(GENERAL_TEMP_PATH)
# shutil.rmtree(OUTPUT_PATH) # shutil.rmtree(OUTPUT_PATH)
... ...
scripts/core/docker_handler.py
View file @ 29ed9a0f
...@@ -3,8 +3,9 @@ import os ...@@ -3,8 +3,9 @@ import os
from scripts.config import EnvironmentVariables from scripts.config import EnvironmentVariables
from scripts.core import ILensVersionHandler from scripts.core import ILensVersionHandler
from scripts.core.git_handler import GitHandler from scripts.core.git_handler import GitHandler
from scripts.db.psql.databases import get_db_for_func
from scripts.logging import logging from scripts.logging import logging
from scripts.schemas import DockerComposeSchema from scripts.schemas import DockerComposeSchema, GetRequest
from scripts.utils.common_utils import CommonUtils from scripts.utils.common_utils import CommonUtils
...@@ -39,13 +40,12 @@ class DockerHandler: ...@@ -39,13 +40,12 @@ class DockerHandler:
logging.debug("Failed to clone module!! Skipping Docker File Preparation") logging.debug("Failed to clone module!! Skipping Docker File Preparation")
continue continue
_module_data = self.common_util.convert_yaml_to_define_obj(variables_file_path) _module_data = self.common_util.convert_yaml_to_define_obj(variables_file_path)
# session_obj = get_db_for_func() session_obj = get_db_for_func()
# module_info = self.db_handler.get_module_versions( module_info = self.db_handler.get_module_versions(
# input_data=GetRequest(module_name=_module, client='iLens', ilens_version=ilens_version, input_data=GetRequest(module_name=_module, client='iLens', ilens_version=ilens_version,
# release_version=release_version), db=session_obj) release_version=release_version), db=session_obj)
# session_obj.close() session_obj.close()
_module_data = self.common_util.convert_yaml_to_define_obj(variables_file_path) _module_data = self.common_util.convert_yaml_to_define_obj(variables_file_path)
module_info = {}
module_env_variables = _module_data.get('deployment', {}).get('environmentVar', []) module_env_variables = _module_data.get('deployment', {}).get('environmentVar', [])
module_env_variables = {_v['name']: _v.get('value') for _v in module_env_variables} module_env_variables = {_v['name']: _v.get('value') for _v in module_env_variables}
image_url = module_info.get("image_url", '') if module_info else '' image_url = module_info.get("image_url", '') if module_info else ''
...@@ -105,11 +105,20 @@ class DockerHandler: ...@@ -105,11 +105,20 @@ class DockerHandler:
def process_compose_data_by_template(self, modules, compose_info: dict, variables_file, arguments, def process_compose_data_by_template(self, modules, compose_info: dict, variables_file, arguments,
tmp_path, output_path): tmp_path, output_path):
ilens_version = arguments.get("ilens_version")
branch_name = arguments.get("branch_name")
release_version = arguments.get("release_version")
compose_dict = {"services": {}} compose_dict = {"services": {}}
try: try:
for module in modules: for module in modules:
session_obj = get_db_for_func()
module_info = self.db_handler.get_module_versions(
input_data=GetRequest(module_name=module, client='iLens', ilens_version=ilens_version,
release_version=release_version), db=session_obj)
session_obj.close()
image_url = module_info.get("image_url", '') if module_info else ''
compose_dict['services'].update({ compose_dict['services'].update({
module: DockerComposeSchema().dict() module: DockerComposeSchema(image=image_url).dict()
}) })
compose_out_file_path = os.path.join(output_path, "docker-compose.yml") compose_out_file_path = os.path.join(output_path, "docker-compose.yml")
jinja_template_file = 'docker_deployment.yml' jinja_template_file = 'docker_deployment.yml'
......
templates/docker_deployment.yml
View file @ 29ed9a0f
version: "2" {% for k,v in modules.items() %}
x-{{ k }}-image: &{{k}}-image {{v['image']}}
{% endfor %}
services: services:
{% for k,v in modules.items() %} {% for k,v in modules.items() %}
{{k}}: {{k}}:
image: {{v['image']}} image: *{{k}}-image
ports: ports:
- {{v['ports']}} - {{v['ports']}}
environment: environment:
...@@ -15,4 +17,9 @@ services: ...@@ -15,4 +17,9 @@ services:
{% for volume in v['volumes'] %} {% for volume in v['volumes'] %}
- {{volume}} - {{volume}}
{% endfor %} {% endfor %}
{% endfor %} {% endfor %}
\ No newline at end of file
networks:
default:
external:
name: "ilens"
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment