Skip to content

H
Helm-Automation-Script
Commit ba823dcc authored by harshavardhan.c's avatar harshavardhan.c
Browse files
Options

Dev: code optimization fixes

parent 29ed9a0f
Hide whitespace changes
Inline Side-by-side
Showing with 120 additions and 85 deletions
docker_automation/docker_automate_script.py
View file @ ba823dcc
......@@ -31,34 +31,38 @@ class DockerVersionUpgrade:
general_temp_path = f"{int(time.time())}_tmp"
output_path = f"{int(time.time())}_docker-compose"
try:
_release_version = self.arguments["release_version"]
_client_name = self.arguments['client_name']
_git_repos = self.arguments["git_repos"]
_module_names = self.arguments["module_names"]
_branch_name = self.arguments['branch_name'] or "master"
_ilens_version = self.arguments["ilens_version"]
if not _ilens_version or not _release_version or not _client_name or not global_configmap:
print(
"global_configmap, client_name, ilens_version and release_version details "
"not found!!!!!")
sys.exit()
variables_file = "variables.yml"
if not os.path.exists(docker_temp_path):
os.makedirs(docker_temp_path)
if not os.path.exists(output_path):
os.makedirs(output_path)
docker_compose_path = os.path.join(docker_temp_path, "docker-compose")
_branch = f"{_client_name}_{_ilens_version}.{_release_version}"
if not self.git_handler_obj.clone_repository(repo_link=docker_repo, module_output_path=docker_compose_path,
clone_branch=_client_name):
logging.error(f"Cannot clone docker repo with branch: {_client_name}")
sys.exit()
self.docker_handler.process_compose_data_for_existing_files(destination_branch=_client_name,
source_branch=_branch,
variables_file=variables_file,
arguments=self.arguments,
tmp_path=general_temp_path,
output_path=output_path,
docker_compose_path=docker_compose_path)
self.process_docker_automation(docker_temp_path, output_path, general_temp_path)
except Exception as e:
logging.error(f'Exception occurred while preparing the docker compose {e.args}')
def process_docker_automation(self, docker_temp_path, output_path, general_temp_path):
_release_version = self.arguments["release_version"]
_client_name = self.arguments['client_name']
_git_repos = self.arguments["git_repos"]
_module_names = self.arguments["module_names"]
_branch_name = self.arguments['branch_name'] or "master"
_ilens_version = self.arguments["ilens_version"]
if not _ilens_version or not _release_version or not _client_name or not global_configmap:
print(
"global_configmap, client_name, ilens_version and release_version details "
"not found!!!!!")
sys.exit()
variables_file = "variables.yml"
if not os.path.exists(docker_temp_path):
os.makedirs(docker_temp_path)
if not os.path.exists(output_path):
os.makedirs(output_path)
docker_compose_path = os.path.join(docker_temp_path, "docker-compose")
_branch = f"{_client_name}_{_ilens_version}.{_release_version}"
if not self.git_handler_obj.clone_repository(repo_link=docker_repo, module_output_path=docker_compose_path,
clone_branch=_client_name):
logging.error(f"Cannot clone docker repo with branch: {_client_name}")
sys.exit()
self.docker_handler.process_compose_data_for_existing_files(destination_branch=_client_name,
source_branch=_branch,
variables_file=variables_file,
arguments=self.arguments,
tmp_path=general_temp_path,
output_path=output_path,
docker_compose_path=docker_compose_path)
docker_automation/docker_register.py
View file @ ba823dcc
import json
from scripts.config import EnvironmentVariables
if __name__ == "__main__":
from dotenv import load_dotenv
......@@ -69,6 +71,12 @@ class DockerRegistration:
arguments=self.arguments, compose_info=repo_info,
tmp_path=general_temp_path,
output_path=output_path, modules=_module_names)
# self.git_handler_obj.push_deployments_to_git(repo_link=docker_repo,
# private_token=EnvironmentVariables.git_access_token,
# branch=_branch,
# folder_path=output_path, helm_deployment=False,
# base_path=docker_compose_path)
# self.git_handler_obj.create_merge_request(repo_link=docker_repo, source_branch=_branch,
# destination_branch=_client_name)
except Exception as e:
logging.exception(f"Exception Occurred while generating the docker compose file {e.args}")
scripts/core/docker_handler.py
View file @ ba823dcc
......@@ -16,20 +16,22 @@ class DockerHandler:
self.common_util = CommonUtils()
self.db_handler = ILensVersionHandler()
def process_module_request(self, compose_data: dict, variables_file, general_path,
def process_module_request(self, compose_data: dict, variables_file, general_path, image_mapping_dict: dict,
**kwargs):
try:
ilens_version = kwargs.get("ilens_version")
client_version = kwargs.get("client", "iLens")
branch_name = kwargs.get("branch_name")
release_version = kwargs.get("release_version")
for _module, module_dict in compose_data['services'].items():
variable_name = f'x-{_module}-image'
updated_module_name = self.get_defined_module_name(module_name=_module,
module_mapping_dict=image_mapping_dict)
module_path = os.path.join(general_path)
module_path = os.path.join(module_path, _module)
if not os.path.exists(module_path):
os.makedirs(module_path)
variables_file_path = os.path.join(module_path, variables_file)
git_info = self.git_handler_obj.get_git_url_by_module_name(module_name=_module)
git_info = self.git_handler_obj.get_git_url_by_module_name(module_name=updated_module_name)
if not git_info:
logging.debug("Failed to fetch module info!! Skipping Helm File Preparation")
continue
......@@ -40,18 +42,14 @@ class DockerHandler:
logging.debug("Failed to clone module!! Skipping Docker File Preparation")
continue
_module_data = self.common_util.convert_yaml_to_define_obj(variables_file_path)
session_obj = get_db_for_func()
module_info = self.db_handler.get_module_versions(
input_data=GetRequest(module_name=_module, client='iLens', ilens_version=ilens_version,
release_version=release_version), db=session_obj)
session_obj.close()
_module_data = self.common_util.convert_yaml_to_define_obj(variables_file_path)
if updated_module_name in image_mapping_dict:
image_url = image_mapping_dict.get(updated_module_name, '')
else:
image_url = self.get_image_tag_from_db(module=_module, ilens_version=ilens_version,
release_version=release_version, client=client_version)
module_env_variables = _module_data.get('deployment', {}).get('environmentVar', [])
module_env_variables = {_v['name']: _v.get('value') for _v in module_env_variables}
image_url = module_info.get("image_url", '') if module_info else ''
if image_url:
module_dict['image'] = image_url
compose_data[variable_name] = image_url
image_mapping_dict[_module] = image_url
existing_env_variables = module_dict.get('environment', {})
diff_keys = list(
set(existing_env_variables.keys()).symmetric_difference(set(module_env_variables.keys())))
......@@ -72,63 +70,88 @@ class DockerHandler:
tmp_path, output_path, source_branch, destination_branch):
docker_repo = EnvironmentVariables.docker_repo
global_configmap = EnvironmentVariables.global_configmap
jinja_template_file = 'docker_deployment.yml'
template_path = "./templates"
try:
files_info = os.listdir(docker_compose_path)
sorted_files = list(filter(lambda f: f.endswith(".yml") and "docker" in f, files_info))
global_config_data = self.common_util.convert_yaml_to_define_obj(
os.path.join(docker_compose_path, global_configmap))
for _file in sorted_files:
image_mapping_dict = {}
for index, _file in enumerate(sorted_files):
docker_compose_data = self.common_util.convert_yaml_to_define_obj(
os.path.join(docker_compose_path, _file),
load_type=None)
os.path.join(docker_compose_path, _file))
service_dict = docker_compose_data.get("services")
if not service_dict:
logging.debug(f'Services not found for current docker compose file - {_file}')
if index + 1 == len(sorted_files):
modules = self.get_modules_dict_for_registration(modules=arguments.get('module_names', []),
existing_modules=list(
docker_compose_data['services'].keys()))
docker_compose_data['services'].update(modules)
if response_data := self.process_module_request(compose_data=docker_compose_data,
variables_file=variables_file,
image_mapping_dict=image_mapping_dict,
general_path=tmp_path, **arguments):
compose_out_file_path = os.path.join(output_path, _file)
self.common_util.convert_json_to_yaml(json_data=response_data,
output_file_path=compose_out_file_path)
self.git_handler_obj.push_deployments_to_git(repo_link=docker_repo,
private_token=EnvironmentVariables.git_access_token,
branch=source_branch,
folder_path=output_path, helm_deployment=False,
base_path=docker_compose_path)
self.git_handler_obj.create_merge_request(repo_link=docker_repo, source_branch=source_branch,
destination_branch=destination_branch)
# self.common_util.convert_json_to_yaml(json_data=response_data,
# output_file_path=compose_out_file_path)
self.common_util.render_deployment_chart(data_dict={"modules": response_data.get('services', {})},
jinja_template_file=jinja_template_file,
template_path=template_path,
outfile_path=compose_out_file_path)
# self.git_handler_obj.push_deployments_to_git(repo_link=docker_repo,
# private_token=EnvironmentVariables.git_access_token,
# branch=source_branch,
# folder_path=output_path, helm_deployment=False,
# base_path=docker_compose_path)
# self.git_handler_obj.create_merge_request(repo_link=docker_repo, source_branch=source_branch,
# destination_branch=destination_branch)
except Exception as e:
logging.exception(f'Exception Occurred while processing the compose data info {e.args}')
def process_compose_data_by_template(self, modules, compose_info: dict, variables_file, arguments,
tmp_path, output_path):
ilens_version = arguments.get("ilens_version")
branch_name = arguments.get("branch_name")
release_version = arguments.get("release_version")
compose_dict = {"services": {}}
compose_dict = {"services": self.get_modules_dict_for_registration(modules=modules)}
try:
for module in modules:
session_obj = get_db_for_func()
module_info = self.db_handler.get_module_versions(
input_data=GetRequest(module_name=module, client='iLens', ilens_version=ilens_version,
release_version=release_version), db=session_obj)
session_obj.close()
image_url = module_info.get("image_url", '') if module_info else ''
compose_dict['services'].update({
module: DockerComposeSchema(image=image_url).dict()
})
compose_out_file_path = os.path.join(output_path, "docker-compose.yml")
jinja_template_file = 'docker_deployment.yml'
template_path = "./templates"
compose_data = self.process_module_request(compose_data=compose_dict,
image_mapping_dict = {}
compose_data = self.process_module_request(compose_data=compose_dict, image_mapping_dict=image_mapping_dict,
variables_file=variables_file,
general_path=tmp_path, **arguments)
self.common_util.render_helm_chart(data_dict={"modules": compose_data.get('services', {})},
jinja_template_file=jinja_template_file, template_path=template_path,
outfile_path=compose_out_file_path)
self.common_util.render_deployment_chart(data_dict={"modules": compose_data.get('services', {})},
jinja_template_file=jinja_template_file,
template_path=template_path,
outfile_path=compose_out_file_path)
except Exception as e:
logging.exception(f'Exception Occurred while processing the compose data by template {e.args}')
@staticmethod
def get_modules_dict_for_registration(modules, existing_modules=None):
if existing_modules is None:
existing_modules = []
return {module: DockerComposeSchema().dict() for module in modules if module not in existing_modules}
@staticmethod
def get_defined_module_name(module_name, module_mapping_dict):
for key in module_mapping_dict:
if key.startswith(module_name):
module_name = key
break
return module_name
def get_image_tag_from_db(self, module, ilens_version, client, release_version):
image_url = ''
try:
session_obj = get_db_for_func()
module_info = self.db_handler.get_module_versions(
input_data=GetRequest(module_name=module, client=client, ilens_version=ilens_version,
release_version=release_version), db=session_obj)
session_obj.close()
image_url = module_info.get("image_url", '') if module_info else ''
except Exception as e:
logging.exception(f'Exception occurred while fetching the image details {e.args}')
return image_url
scripts/core/helm_handler.py
View file @ ba823dcc
......@@ -33,9 +33,9 @@ class HelmHandler:
existing_data['deployment']['imageName'] = image_tag or existing_data['deployment']['imageName']
self.common_utils.convert_json_to_yaml(json_data=existing_data, output_file_path=template_file)
self.common_utils.render_helm_chart(data_dict=global_config_vars, jinja_template_file=f'{module_name}.yml',
outfile_path=helm_out_file_path, template_path=template_path,
service_type=service_type)
self.common_utils.render_deployment_chart(data_dict=global_config_vars, jinja_template_file=f'{module_name}.yml',
outfile_path=helm_out_file_path, template_path=template_path,
service_type=service_type)
return True
except Exception as e:
logging.exception(f'Exception occurred while preparing the helm deployment file {e.args}')
......@@ -61,11 +61,11 @@ class HelmHandler:
value_key = v.strip("<{ }>")
global_config_vars.update({k: global_config_vars.get(value_key, v)})
self.common_utils.render_helm_chart(data_dict={}, jinja_template_file=f'{module_name}.yml',
outfile_path=helm_out_file_path, template_path="templates",
service_type=service_type, image_url=image_tag,
variables=global_config_vars,
module_port=module_port, node_port=node_port)
self.common_utils.render_deployment_chart(data_dict={}, jinja_template_file=f'{module_name}.yml',
outfile_path=helm_out_file_path, template_path="templates",
service_type=service_type, image_url=image_tag,
variables=global_config_vars,
module_port=module_port, node_port=node_port)
return True
except Exception as e:
logging.exception(f'Exception occurred while preparing the helm deployment file {e.args}')
......
scripts/utils/common_utils.py
View file @ ba823dcc
......@@ -35,7 +35,7 @@ class CommonUtils:
return False
@staticmethod
def render_helm_chart(data_dict, jinja_template_file, template_path, outfile_path, **kwargs):
def render_deployment_chart(data_dict, jinja_template_file, template_path, outfile_path, **kwargs):
try:
environment = jinja2.Environment(
loader=jinja2.FileSystemLoader(searchpath=template_path),
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment