Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
H
Helm-Automation-Script
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
CI / CD Analytics
Repository Analytics
Value Stream Analytics
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
harshavardhan.c
Helm-Automation-Script
Commits
29ed9a0f
Commit
29ed9a0f
authored
Nov 27, 2022
by
harshavardhan.c
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Dev: integration fixes while generating helm-scripts dynamically.
parent
f413d47e
Changes
6
Show whitespace changes
Inline
Side-by-side
Showing
6 changed files
with
288 additions
and
354 deletions
+288
-354
automation_script.py
automation_script.py
+9
-3
docker_automation/docker_automate_script.py
docker_automation/docker_automate_script.py
+2
-2
helm_automation/helm_automate_script.py
helm_automation/helm_automate_script.py
+120
-153
helm_automation/helm_register.py
helm_automation/helm_register.py
+130
-185
scripts/core/docker_handler.py
scripts/core/docker_handler.py
+17
-8
templates/docker_deployment.yml
templates/docker_deployment.yml
+10
-3
No files found.
automation_script.py
View file @
29ed9a0f
...
...
@@ -4,9 +4,11 @@ import sys
from
dotenv
import
load_dotenv
from
docker_automation.docker_automate_script
import
DockerVersionUpgrade
load_dotenv
()
from
helm_automation.helm_register
import
HelmRegistration
from
docker_automation.docker_automate_script
import
DockerVersionUpgrade
from
helm_automation.helm_automate_script
import
HelmVersionUpgrade
from
docker_automation.docker_register
import
DockerRegistration
from
scripts.config
import
EnvironmentVariables
from
scripts.logging
import
logger
...
...
@@ -25,6 +27,10 @@ def func_mapper(operation_type, parameters):
"docker_registration"
:
DockerRegistration
(
arguments
=
parameters
,
git_user_name
=
git_user_name
,
git_access_token
=
git_access_token
)
.
module_registration
,
"docker_version_upgrade"
:
DockerVersionUpgrade
(
arguments
=
parameters
,
git_user_name
=
git_user_name
,
git_access_token
=
git_access_token
)
.
module_registration
,
"helm_version_upgrade"
:
HelmVersionUpgrade
(
arguments
=
parameters
,
git_user_name
=
git_user_name
,
git_access_token
=
git_access_token
)
.
module_registration
,
"helm_registration"
:
HelmRegistration
(
arguments
=
parameters
,
git_user_name
=
git_user_name
,
git_access_token
=
git_access_token
)
.
module_registration
}
return
func_mapper_dict
.
get
(
operation_type
)
...
...
docker_automation/docker_automate_script.py
View file @
29ed9a0f
...
...
@@ -37,9 +37,9 @@ class DockerVersionUpgrade:
_module_names
=
self
.
arguments
[
"module_names"
]
_branch_name
=
self
.
arguments
[
'branch_name'
]
or
"master"
_ilens_version
=
self
.
arguments
[
"ilens_version"
]
if
not
_ilens_version
or
not
_release_version
or
not
_client_name
or
not
_git_repos
and
not
_module_names
or
not
global_configmap
:
if
not
_ilens_version
or
not
_release_version
or
not
_client_name
or
not
global_configmap
:
print
(
"global_configmap,
git_repos, module_names,
client_name, ilens_version and release_version details "
"global_configmap, client_name, ilens_version and release_version details "
"not found!!!!!"
)
sys
.
exit
()
variables_file
=
"variables.yml"
...
...
helm_automation/helm_automate_script.py
View file @
29ed9a0f
from
scripts.config
import
EnvironmentVariables
from
scripts.core.helm_handler
import
HelmHandler
if
__name__
==
"__main__"
:
from
dotenv
import
load_dotenv
load_dotenv
()
import
argparse
import
logging
import
os
import
sys
...
...
@@ -10,7 +12,6 @@ import time
from
scripts.core
import
ILensVersionHandler
from
scripts.core.git_handler
import
GitHandler
from
scripts.core.helm_handler
import
HelmHandler
from
scripts.db.psql.databases
import
get_db_for_func
from
scripts.schemas
import
GetRequest
from
scripts.utils.common_utils
import
CommonUtils
...
...
@@ -18,131 +19,95 @@ from scripts.utils.common_utils import CommonUtils
HELM_PATH
=
"/ilens-core/ilens-modules"
HELM_STORE_PATH
=
"./helm-charts"
git_handler_obj
=
GitHandler
(
user_name
=
git_user_name
,
access_token
=
git_access_token
)
global_configmap
=
EnvironmentVariables
.
global_configmap
helm_repo
=
EnvironmentVariables
.
helm_repo
ap
=
argparse
.
ArgumentParser
()
db_handler
=
ILensVersionHandler
()
common_util
=
CommonUtils
()
helm_handler
=
HelmHandler
()
if
__name__
==
'__main__'
:
ap
.
add_argument
(
"--ilens_version"
,
"-iv"
,
required
=
False
,
default
=
None
,
help
=
"ILens Version Tag"
,
)
ap
.
add_argument
(
"--release_version"
,
"-rv"
,
required
=
False
,
default
=
None
,
help
=
"ILens Release Tag"
,
)
ap
.
add_argument
(
"--client_name"
,
"-cn"
,
required
=
False
,
default
=
None
,
help
=
"Client Name Tag"
)
ap
.
add_argument
(
"--branch_name"
,
"-b"
,
required
=
False
,
default
=
None
,
help
=
"Branch Name"
)
ap
.
add_argument
(
"--git_repos"
,
"-gr"
,
required
=
False
,
default
=
None
,
help
=
"Git repos to be added in helm"
,
nargs
=
"+"
)
ap
.
add_argument
(
"--module_names"
,
"-mn"
,
required
=
False
,
default
=
None
,
help
=
"Module names to be added in helm"
,
nargs
=
"+"
)
HELM_TEMP_PATH
=
f
"{int(time.time())}_helm_tmp_path"
GENERAL_TEMP_PATH
=
f
"{int(time.time())}_tmp"
OUTPUT_PATH
=
f
"{int(time.time())}_helm-charts"
class
HelmVersionUpgrade
:
def
__init__
(
self
,
arguments
:
dict
,
git_user_name
:
str
,
git_access_token
:
str
):
self
.
arguments
=
arguments
self
.
db_handler
=
ILensVersionHandler
()
self
.
helm_handler
=
HelmHandler
()
self
.
git_handler_obj
=
GitHandler
(
user_name
=
git_user_name
,
access_token
=
git_access_token
)
self
.
common_util
=
CommonUtils
()
def
module_registration
(
self
):
helm_temp_path
=
f
"{int(time.time())}_helm_tmp_path"
general_temp_path
=
f
"{int(time.time())}_tmp"
output_path
=
f
"{int(time.time())}_helm-charts"
try
:
arguments
=
vars
(
ap
.
parse_args
())
_ilens_version
=
arguments
[
"ilens_version"
]
_release_version
=
arguments
[
"release_version"
]
_client_name
=
arguments
[
'client_name'
]
_git_repos
=
arguments
[
"git_repos"
]
_module_names
=
arguments
[
"module_names"
]
_branch_name
=
arguments
[
'branch_name'
]
or
"master"
_ilens_version
=
self
.
arguments
[
"ilens_version"
]
_release_version
=
self
.
arguments
[
"release_version"
]
_client_name
=
self
.
arguments
[
'client_name'
]
_git_repos
=
self
.
arguments
[
"git_repos"
]
_module_names
=
self
.
arguments
[
"module_names"
]
_branch_name
=
self
.
arguments
[
'branch_name'
]
or
"master"
if
not
_ilens_version
or
not
_release_version
or
not
_client_name
or
not
(
_git_repos
or
_module_names
)
or
not
global_configmap
:
print
(
"global_configmap, git_repos, module_name
s, client_name, ilens_version and release_version details not found!!!!!"
)
"global_configmap, git_repo
s, client_name, ilens_version and release_version details not found!!!!!"
)
sys
.
exit
()
_branch
=
f
"{_client_name}_{_ilens_version}.{_release_version}"
if
not
os
.
path
.
exists
(
HELM_TEMP_PATH
):
os
.
makedirs
(
HELM_TEMP_PATH
)
if
not
os
.
path
.
exists
(
OUTPUT_PATH
):
os
.
makedirs
(
OUTPUT_PATH
)
helm_path
=
os
.
path
.
join
(
HELM_TEMP_PATH
,
"helm-charts"
)
if
not
git_handler_obj
.
clone_repository
(
repo_link
=
helm_repo
,
module_output_path
=
helm_path
,
if
not
os
.
path
.
exists
(
helm_temp_path
):
os
.
makedirs
(
helm_temp_path
)
if
not
os
.
path
.
exists
(
output_path
):
os
.
makedirs
(
output_path
)
helm_path
=
os
.
path
.
join
(
helm_temp_path
,
"helm-charts"
)
if
not
self
.
git_handler_obj
.
clone_repository
(
repo_link
=
helm_repo
,
module_output_path
=
helm_path
,
clone_branch
=
_client_name
):
logging
.
error
(
f
"Cannot clone helm repo with branch: {_client_name}"
)
sys
.
exit
()
base_helm_directory_path
=
os
.
path
.
join
(
HELM_TEMP_PATH
,
"helm-charts"
,
"ilens-core"
,
"ilens-modules"
)
base_helm_directory_path
=
os
.
path
.
join
(
helm_temp_path
,
"helm-charts"
,
"ilens-core"
,
"ilens-modules"
)
if
len
(
_module_names
)
==
1
and
_module_names
[
0
]
.
lower
()
==
"all"
:
updated_list
=
[]
files_info
=
os
.
listdir
(
base_helm_directory_path
)
sorted_files
=
list
(
filter
(
lambda
f
:
f
.
endswith
(
".yml"
),
files_info
))
_module_names
=
[
_each
.
replace
(
".yml"
,
""
)
for
_each
in
sorted_files
]
global_config_data
=
common_util
.
convert_yaml_to_define_obj
(
global_config_data
=
self
.
common_util
.
convert_yaml_to_define_obj
(
os
.
path
.
join
(
base_helm_directory_path
,
global_configmap
))
# global_config_data = common_util.convert_yaml_to_json("ilens-global-configmap.yml")
variables_file
=
"variables.yml"
template_path
=
os
.
path
.
join
(
GENERAL_TEMP_PATH
,
"../templates"
)
template_path
=
os
.
path
.
join
(
general_temp_path
,
"../templates"
)
if
not
os
.
path
.
exists
(
template_path
):
os
.
makedirs
(
template_path
)
for
_module
in
_module_names
:
module_path
=
os
.
path
.
join
(
GENERAL_TEMP_PATH
)
module_path
=
os
.
path
.
join
(
general_temp_path
)
module_path
=
os
.
path
.
join
(
module_path
,
_module
)
if
not
os
.
path
.
exists
(
module_path
):
os
.
makedirs
(
module_path
)
helm_out_file_path
=
os
.
path
.
join
(
OUTPUT_PATH
,
f
'{_module}.yml'
)
helm_out_file_path
=
os
.
path
.
join
(
output_path
,
f
'{_module}.yml'
)
if
os
.
path
.
exists
(
helm_out_file_path
):
logging
.
debug
(
f
"Helm Deployment File found for selected the module {_module}"
)
continue
variables_file_path
=
os
.
path
.
join
(
module_path
,
variables_file
)
git_info
=
git_handler_obj
.
get_git_url_by_module_name
(
module_name
=
_module
)
git_info
=
self
.
git_handler_obj
.
get_git_url_by_module_name
(
module_name
=
_module
)
if
not
git_info
:
logging
.
debug
(
"Failed to fetch module info!! Skipping Helm File Preparation"
)
continue
if
not
git_handler_obj
.
clone_repository_with_defined_file
(
repo_link
=
git_info
,
clone_branch
=
_branch_name
,
if
not
self
.
git_handler_obj
.
clone_repository_with_defined_file
(
repo_link
=
git_info
,
clone_branch
=
_branch_name
,
file_output_path
=
variables_file_path
,
clone_file_path
=
variables_file
):
logging
.
debug
(
"Failed to clone module!! Skipping Helm File Preparation"
)
continue
_module_data
=
common_util
.
convert_yaml_to_define_obj
(
variables_file_path
)
_module_data
=
self
.
common_util
.
convert_yaml_to_define_obj
(
variables_file_path
)
module_env_variables
=
_module_data
.
get
(
'deployment'
,
{})
.
get
(
'environmentVar'
,
[])
module_env_variables
=
{
_v
[
'name'
]:
_v
for
_v
in
module_env_variables
}
template_file
=
os
.
path
.
join
(
template_path
,
f
'{_module}.yml'
)
session_obj
=
get_db_for_func
()
module_info
=
db_handler
.
get_module_versions
(
module_info
=
self
.
db_handler
.
get_module_versions
(
input_data
=
GetRequest
(
module_name
=
_module
,
client
=
'iLens'
,
ilens_version
=
_ilens_version
,
release_version
=
_release_version
),
db
=
session_obj
)
session_obj
.
close
()
image_url
=
module_info
.
get
(
"image_url"
,
''
)
if
module_info
else
''
existing_yml_path
=
os
.
path
.
join
(
base_helm_directory_path
,
f
'{_module}.yml'
)
helm_handler
.
create_existing_helm_deployment_file
(
template_yml_path
=
existing_yml_path
,
image_tag
=
image_url
,
self
.
helm_handler
.
create_existing_helm_deployment_file
(
template_yml_path
=
existing_yml_path
,
image_tag
=
image_url
,
module_env_variables
=
module_env_variables
,
template_file
=
template_file
,
template_path
=
template_path
,
template_file
=
template_file
,
template_path
=
template_path
,
helm_out_file_path
=
helm_out_file_path
,
global_config_data
=
global_config_data
,
module_name
=
_module
)
...
...
@@ -151,24 +116,26 @@ if __name__ == '__main__':
updated_files_info
=
[
x
for
x
in
sorted_files
if
f
'{_module}.yml'
!=
x
]
for
_file
in
updated_files_info
:
sub_file_path
=
os
.
path
.
join
(
base_helm_directory_path
,
_file
)
helm_out_file_path
=
os
.
path
.
join
(
OUTPUT_PATH
,
_file
)
helm_out_file_path
=
os
.
path
.
join
(
output_path
,
_file
)
template_file
=
os
.
path
.
join
(
template_path
,
_file
)
helm_handler
.
create_existing_helm_deployment_file
(
template_yml_path
=
sub_file_path
,
image_tag
=
image_url
,
self
.
helm_handler
.
create_existing_helm_deployment_file
(
template_yml_path
=
sub_file_path
,
image_tag
=
image_url
,
module_env_variables
=
module_env_variables
,
template_file
=
template_file
,
template_path
=
template_path
,
helm_out_file_path
=
helm_out_file_path
,
global_config_data
=
global_config_data
,
module_name
=
_file
.
split
(
".yml"
)[
0
])
git_handler_obj
.
push_deployments_to_git
(
helm_repo
,
git_access_token
,
_branch
,
folder_path
=
OUTPUT_PATH
,
base_path
=
helm_path
)
git_handler_obj
.
create_merge_request
(
repo_link
=
helm_repo
,
source_branch
=
_branch
,
self
.
git_handler_obj
.
push_deployments_to_git
(
repo_link
=
helm_repo
,
branch
=
_branch
,
folder_path
=
output_path
,
base_path
=
helm_path
,
private_token
=
EnvironmentVariables
.
git_access_token
)
self
.
git_handler_obj
.
create_merge_request
(
repo_link
=
helm_repo
,
source_branch
=
_branch
,
destination_branch
=
_client_name
)
except
Exception
as
e
:
logging
.
exception
(
f
"Exception Occurred while processing the Helm-Script Preparation {e.args}"
)
finally
:
# shutil.rmtree(HELM_STORE_PATH)
# shutil.rmtree(HELM_TEMP_PATH
)
# shutil.rmtree(GENERAL_TEMP_PATH
)
# shutil.rmtree(OUTPUT_PATH
)
# shutil.rmtree(helm_temp_path
)
# shutil.rmtree(general_temp_path
)
# shutil.rmtree(output_path
)
...
helm_automation/helm_register.py
View file @
29ed9a0f
...
...
@@ -10,7 +10,6 @@ if __name__ == "__main__":
from
scripts.db.psql.databases
import
get_db_for_func
from
scripts.schemas
import
GetRequest
import
argparse
import
logging
import
os
import
sys
...
...
@@ -22,174 +21,118 @@ from scripts.core.helm_handler import HelmHandler
from
scripts.utils.common_utils
import
CommonUtils
default_link
=
EnvironmentVariables
.
default_link
git_user_name
=
EnvironmentVariables
.
git_user_name
git_access_token
=
EnvironmentVariables
.
git_access_token
config_variables
=
EnvironmentVariables
.
config_variables
helm_repo
=
EnvironmentVariables
.
helm_repo
global_configmap
=
EnvironmentVariables
.
global_configmap
HELM_PATH
=
"/ilens-core/ilens-modules"
HELM_STORE_PATH
=
"./helm-charts"
git_handler_obj
=
GitHandler
(
user_name
=
git_user_name
,
access_token
=
git_access_token
)
ap
=
argparse
.
ArgumentParser
()
db_handler
=
ILensVersionHandler
()
common_util
=
CommonUtils
()
helm_handler
=
HelmHandler
()
if
__name__
==
'__main__'
:
ap
.
add_argument
(
"--ilens_version"
,
"-iv"
,
required
=
False
,
default
=
None
,
help
=
"ILens Version Tag"
,
)
ap
.
add_argument
(
"--release_version"
,
"-rv"
,
required
=
False
,
default
=
None
,
help
=
"ILens Release Tag"
,
)
ap
.
add_argument
(
"--reference_client_name"
,
"-rcn"
,
required
=
False
,
default
=
None
,
help
=
"Client Name Tag"
)
ap
.
add_argument
(
"--client_name"
,
"-cn"
,
required
=
False
,
default
=
None
,
help
=
"Client Name Tag"
)
ap
.
add_argument
(
"--branch_name"
,
"-b"
,
required
=
False
,
default
=
None
,
help
=
"Branch Name"
)
ap
.
add_argument
(
"--git_repos"
,
"-gr"
,
required
=
False
,
default
=
None
,
help
=
"Git repos to be added in helm"
,
nargs
=
"+"
)
ap
.
add_argument
(
"--module_names"
,
"-mn"
,
required
=
False
,
default
=
None
,
help
=
"Module names to be added in helm"
,
nargs
=
"+"
)
ap
.
add_argument
(
"--repo_info"
,
"-ri"
,
required
=
False
,
default
=
None
,
help
=
"Module names to be added in helm"
,
nargs
=
"+"
)
HELM_TEMP_PATH
=
f
"{int(time.time())}_helm_tmp_path"
GENERAL_TEMP_PATH
=
f
"{int(time.time())}_tmp"
OUTPUT_PATH
=
f
"{int(time.time())}_helm-charts"
class
HelmRegistration
:
def
__init__
(
self
,
arguments
:
dict
,
git_user_name
:
str
,
git_access_token
:
str
):
self
.
arguments
=
arguments
self
.
db_handler
=
ILensVersionHandler
()
self
.
helm_handler
=
HelmHandler
()
self
.
git_handler_obj
=
GitHandler
(
user_name
=
git_user_name
,
access_token
=
git_access_token
)
self
.
common_util
=
CommonUtils
()
def
module_registration
(
self
):
helm_temp_path
=
f
"{int(time.time())}_helm_tmp_path"
general_temp_path
=
f
"{int(time.time())}_tmp"
output_path
=
f
"{int(time.time())}_helm-charts"
try
:
arguments
=
vars
(
ap
.
parse_args
())
_ilens_version
=
arguments
[
"ilens_version"
]
_release_version
=
arguments
[
"release_version"
]
_client_name
=
arguments
[
'client_name'
]
_reference_branch
=
arguments
[
'reference_client_name'
]
_git_repos
=
arguments
[
"git_repos"
]
repo_info
=
arguments
[
"repo_info"
]
_module_names
=
arguments
[
"module_names"
]
_branch_name
=
arguments
[
'branch_name'
]
or
"master"
_ilens_version
=
self
.
arguments
[
"ilens_version"
]
_release_version
=
self
.
arguments
[
"release_version"
]
_client_name
=
self
.
arguments
[
'client_name'
]
_reference_branch
=
self
.
arguments
[
'reference_client_name'
]
_git_repos
=
self
.
arguments
[
"git_repos"
]
_module_names
=
self
.
arguments
[
"module_names"
]
_branch_name
=
self
.
arguments
[
'branch_name'
]
or
"master"
repo_info
=
self
.
arguments
[
"repo_info"
]
if
repo_info
:
repo_info
=
json
.
loads
(
repo_info
)
if
not
_ilens_version
or
not
_release_version
or
not
_client_name
or
not
(
_git_repos
or
_module_names
)
or
not
global_configmap
:
if
not
_ilens_version
or
not
_release_version
or
not
_client_name
or
not
_git_repos
and
not
_module_names
or
not
global_configmap
:
print
(
"global_configmap, git_repos, module_names, client_name, ilens_version and release_version details not found!!!!!"
)
sys
.
exit
()
_branch
=
f
"{_client_name}_{_ilens_version}.{_release_version}"
if
not
os
.
path
.
exists
(
HELM_TEMP_PATH
):
os
.
makedirs
(
HELM_TEMP_PATH
)
if
not
os
.
path
.
exists
(
OUTPUT_PATH
):
os
.
makedirs
(
OUTPUT_PATH
)
helm_path
=
os
.
path
.
join
(
HELM_TEMP_PATH
,
"helm-charts"
)
if
not
git_handler_obj
.
clone_repository
(
repo_link
=
helm_repo
,
module_output_path
=
helm_path
,
if
not
os
.
path
.
exists
(
helm_temp_path
):
os
.
makedirs
(
helm_temp_path
)
if
not
os
.
path
.
exists
(
output_path
):
os
.
makedirs
(
output_path
)
helm_path
=
os
.
path
.
join
(
helm_temp_path
,
"helm-charts"
)
if
not
self
.
git_handler_obj
.
clone_repository
(
repo_link
=
helm_repo
,
module_output_path
=
helm_path
,
clone_branch
=
_reference_branch
or
_client_name
):
logging
.
error
(
f
"Cannot clone helm repo with branch: {_reference_branch or _client_name}"
)
base_helm_directory_path
=
os
.
path
.
join
(
HELM_TEMP_PATH
,
"helm-charts"
,
"ilens-core"
,
"ilens-modules"
)
if
os
.
path
.
exists
(
base_helm_directory_path
)
and
len
(
_module_names
)
==
1
and
_module_names
[
0
]
.
lower
()
==
"all"
:
base_helm_directory_path
=
os
.
path
.
join
(
helm_temp_path
,
"helm-charts"
,
"ilens-core"
,
"ilens-modules"
)
if
os
.
path
.
exists
(
base_helm_directory_path
)
and
len
(
_module_names
)
==
1
and
_module_names
[
0
]
.
lower
()
==
"all"
:
updated_list
=
[]
files_info
=
os
.
listdir
(
base_helm_directory_path
)
sorted_files
=
list
(
filter
(
lambda
f
:
f
.
endswith
(
".yml"
),
files_info
))
_module_names
=
[
_each
.
replace
(
".yml"
,
""
)
for
_each
in
sorted_files
]
global_config_data
=
common_util
.
convert_yaml_to_define_obj
(
global_config_data
=
self
.
common_util
.
convert_yaml_to_define_obj
(
os
.
path
.
join
(
base_helm_directory_path
,
global_configmap
))
# global_config_data = common_util.convert_yaml_to_json("ilens-global-configmap.yml")
if
repo_info
:
global_config_data
.
update
(
repo_info
)
variables_file
=
"variables.yml"
template_path
=
os
.
path
.
join
(
GENERAL_TEMP_PATH
,
"../templates"
)
template_path
=
os
.
path
.
join
(
general_temp_path
,
"../templates"
)
if
not
os
.
path
.
exists
(
template_path
):
os
.
makedirs
(
template_path
)
for
_module
in
_module_names
:
module_path
=
os
.
path
.
join
(
GENERAL_TEMP_PATH
)
module_path
=
os
.
path
.
join
(
general_temp_path
)
module_path
=
os
.
path
.
join
(
module_path
,
_module
)
if
not
os
.
path
.
exists
(
module_path
):
os
.
makedirs
(
module_path
)
helm_out_file_path
=
os
.
path
.
join
(
OUTPUT_PATH
,
f
'{_module}.yml'
)
helm_out_file_path
=
os
.
path
.
join
(
output_path
,
f
'{_module}.yml'
)
if
os
.
path
.
exists
(
helm_out_file_path
):
logging
.
debug
(
f
"Helm Deployment File found for selected the module {_module}"
)
continue
variables_file_path
=
os
.
path
.
join
(
module_path
,
variables_file
)
git_info
=
git_handler_obj
.
get_git_url_by_module_name
(
module_name
=
_module
)
git_info
=
self
.
git_handler_obj
.
get_git_url_by_module_name
(
module_name
=
_module
)
if
not
git_info
:
logging
.
debug
(
"Failed to fetch module info!! Skipping Helm File Preparation"
)
continue
if
not
git_handler_obj
.
clone_repository_with_defined_file
(
repo_link
=
git_info
,
clone_branch
=
_branch_name
,
if
not
self
.
git_handler_obj
.
clone_repository_with_defined_file
(
repo_link
=
git_info
,
clone_branch
=
_branch_name
,
file_output_path
=
variables_file_path
,
clone_file_path
=
variables_file
):
logging
.
debug
(
"Failed to clone module!! Skipping Helm File Preparation"
)
continue
_module_data
=
common_util
.
convert_yaml_to_define_obj
(
variables_file_path
)
_module_data
=
self
.
common_util
.
convert_yaml_to_define_obj
(
variables_file_path
)
module_env_variables
=
_module_data
.
get
(
'deployment'
,
{})
.
get
(
'environmentVar'
,
[])
module_env_variables
=
{
_v
[
'name'
]:
_v
[
'value'
]
for
_v
in
module_env_variables
if
{
'name'
,
'value'
}
.
issubset
(
set
(
list
(
_v
.
keys
())))}
template_file
=
os
.
path
.
join
(
template_path
,
f
'{_module}.yml'
)
session_obj
=
get_db_for_func
()
module_info
=
db_handler
.
get_module_versions
(
module_info
=
self
.
db_handler
.
get_module_versions
(
input_data
=
GetRequest
(
module_name
=
_module
,
client
=
'iLens'
,
ilens_version
=
_ilens_version
,
release_version
=
_release_version
),
db
=
session_obj
)
session_obj
.
close
()
image_url
=
module_info
.
get
(
"image_url"
,
''
)
if
module_info
else
''
existing_yml_path
=
os
.
path
.
join
(
base_helm_directory_path
,
f
'{_module}.yml'
)
new_module
=
deepcopy
(
_module
)
worker_type
=
True
if
"worker"
in
_module
or
"celery"
in
_module
else
Fals
e
worker_type
=
"worker"
in
_module
or
"celery"
in
_modul
e
if
not
os
.
path
.
exists
(
existing_yml_path
):
existing_yml_path
=
os
.
path
.
join
(
"../templates"
,
"helm_deployment.yml"
)
new_module
=
"helm_deployment"
helm_handler
.
create_helm_deployment_file
(
template_yml_path
=
existing_yml_path
,
image_tag
=
image_url
,
self
.
helm_handler
.
create_helm_deployment_file
(
template_yml_path
=
existing_yml_path
,
image_tag
=
image_url
,
module_env_variables
=
module_env_variables
,
helm_out_file_path
=
helm_out_file_path
,
global_config_data
=
global_config_data
,
module_name
=
new_module
)
global_config_data
=
global_config_data
,
module_name
=
new_module
)
else
:
helm_handler
.
create_existing_helm_deployment_file
(
template_yml_path
=
existing_yml_path
,
self
.
helm_handler
.
create_existing_helm_deployment_file
(
template_yml_path
=
existing_yml_path
,
image_tag
=
image_url
,
module_env_variables
=
module_env_variables
,
helm_out_file_path
=
helm_out_file_path
,
global_config_data
=
global_config_data
,
module_name
=
new_module
,
template_file
=
template_file
,
module_name
=
new_module
,
template_file
=
template_file
,
template_path
=
template_path
)
if
os
.
path
.
exists
(
base_helm_directory_path
):
files_info
=
os
.
listdir
(
base_helm_directory_path
)
...
...
@@ -197,10 +140,10 @@ if __name__ == '__main__':
updated_files_info
=
[
x
for
x
in
sorted_files
if
f
'{_module}.yml'
!=
x
]
for
_file
in
updated_files_info
:
sub_file_path
=
os
.
path
.
join
(
base_helm_directory_path
,
_file
)
helm_out_file_path
=
os
.
path
.
join
(
OUTPUT_PATH
,
_file
)
helm_out_file_path
=
os
.
path
.
join
(
output_path
,
_file
)
template_file
=
os
.
path
.
join
(
template_path
,
_file
)
worker_type
=
True
if
"worker"
in
_module
or
"celery"
in
_module
else
Fals
e
helm_handler
.
create_existing_helm_deployment_file
(
template_yml_path
=
sub_file_path
,
worker_type
=
"worker"
in
_module
or
"celery"
in
_modul
e
self
.
helm_handler
.
create_existing_helm_deployment_file
(
template_yml_path
=
sub_file_path
,
image_tag
=
image_url
,
module_env_variables
=
module_env_variables
,
template_file
=
template_file
,
...
...
@@ -209,8 +152,10 @@ if __name__ == '__main__':
global_config_data
=
global_config_data
,
service_type
=
False
,
module_name
=
_file
.
split
(
".yml"
)[
0
])
# git_handler_obj.push_helm_deployments(helm_repo, git_access_token, _client_name, final_helm_path=OUTPUT_PATH,
# base_path=helm_path)
# self.git_handler_obj.push_deployments_to_git(repo_link=helm_repo,
# private_token=git_access_token,
# branch=_client_name,
# folder_path=output_path, base_path=helm_path)
except
Exception
as
e
:
logging
.
exception
(
f
"Exception Occurred while processing the Helm-Script Preparation {e.args}"
)
...
...
scripts/core/docker_handler.py
View file @
29ed9a0f
...
...
@@ -3,8 +3,9 @@ import os
from
scripts.config
import
EnvironmentVariables
from
scripts.core
import
ILensVersionHandler
from
scripts.core.git_handler
import
GitHandler
from
scripts.db.psql.databases
import
get_db_for_func
from
scripts.logging
import
logging
from
scripts.schemas
import
DockerComposeSchema
from
scripts.schemas
import
DockerComposeSchema
,
GetRequest
from
scripts.utils.common_utils
import
CommonUtils
...
...
@@ -39,13 +40,12 @@ class DockerHandler:
logging
.
debug
(
"Failed to clone module!! Skipping Docker File Preparation"
)
continue
_module_data
=
self
.
common_util
.
convert_yaml_to_define_obj
(
variables_file_path
)
#
session_obj = get_db_for_func()
#
module_info = self.db_handler.get_module_versions(
#
input_data=GetRequest(module_name=_module, client='iLens', ilens_version=ilens_version,
#
release_version=release_version), db=session_obj)
#
session_obj.close()
session_obj
=
get_db_for_func
()
module_info
=
self
.
db_handler
.
get_module_versions
(
input_data
=
GetRequest
(
module_name
=
_module
,
client
=
'iLens'
,
ilens_version
=
ilens_version
,
release_version
=
release_version
),
db
=
session_obj
)
session_obj
.
close
()
_module_data
=
self
.
common_util
.
convert_yaml_to_define_obj
(
variables_file_path
)
module_info
=
{}
module_env_variables
=
_module_data
.
get
(
'deployment'
,
{})
.
get
(
'environmentVar'
,
[])
module_env_variables
=
{
_v
[
'name'
]:
_v
.
get
(
'value'
)
for
_v
in
module_env_variables
}
image_url
=
module_info
.
get
(
"image_url"
,
''
)
if
module_info
else
''
...
...
@@ -105,11 +105,20 @@ class DockerHandler:
def
process_compose_data_by_template
(
self
,
modules
,
compose_info
:
dict
,
variables_file
,
arguments
,
tmp_path
,
output_path
):
ilens_version
=
arguments
.
get
(
"ilens_version"
)
branch_name
=
arguments
.
get
(
"branch_name"
)
release_version
=
arguments
.
get
(
"release_version"
)
compose_dict
=
{
"services"
:
{}}
try
:
for
module
in
modules
:
session_obj
=
get_db_for_func
()
module_info
=
self
.
db_handler
.
get_module_versions
(
input_data
=
GetRequest
(
module_name
=
module
,
client
=
'iLens'
,
ilens_version
=
ilens_version
,
release_version
=
release_version
),
db
=
session_obj
)
session_obj
.
close
()
image_url
=
module_info
.
get
(
"image_url"
,
''
)
if
module_info
else
''
compose_dict
[
'services'
]
.
update
({
module
:
DockerComposeSchema
()
.
dict
()
module
:
DockerComposeSchema
(
image
=
image_url
)
.
dict
()
})
compose_out_file_path
=
os
.
path
.
join
(
output_path
,
"docker-compose.yml"
)
jinja_template_file
=
'docker_deployment.yml'
...
...
templates/docker_deployment.yml
View file @
29ed9a0f
version
:
"
2"
{
%
for k
,
v in modules.items() %
}
x-{{ k }}-image
:
&
{{
k
}}
-image {{v['image']}}
{
%
endfor %
}
services
:
{
%
for k
,
v in modules.items() %
}
{{
k
}}:
image
:
{{
v
[
'
image'
]}}
image
:
*
{{
k
}}
-image
ports
:
-
{{
v
[
'
ports'
]}}
environment
:
...
...
@@ -16,3 +18,8 @@ services:
-
{{
volume
}}
{
%
endfor %
}
{
%
endfor %
}
networks
:
default
:
external
:
name
:
"
ilens"
\ No newline at end of file
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment