Commit 2a0fd1ea authored by 金凯强's avatar 金凯强 🎨

初始化

parents
*.pyo
*.pyc
__pycache__
.idea/*
logs*
migrations/*
/.vscode/settings.json
venv/*
celerybeat_schedule.*
\ No newline at end of file
image: huansi/docker-compose:latest
stages:
- build
- push
- deploy
variables:
APP_NAME: # 这里写上app名称
APP_CODE: # 这里写上appCode
APP_IMAGE: "${HUANSI_REGISTRY_URL}/huansi/${APP_NAME}"
before_script:
- docker login -u $HUANSI_USER -p $HUANSI_PASSWORD $HUANSI_REGISTRY_URL
build:
stage: build
retry: 2
tags:
- dev
script:
- python /sync_release_info_to_db.py
- docker build -t ${APP_IMAGE}:${CI_COMMIT_REF_NAME} .
except:
- master
push:
stage: push
retry: 2
tags:
- dev
script:
- docker build -t ${APP_IMAGE}:${APP_VERSION} .
- docker push ${APP_IMAGE}:${APP_VERSION}
- echo "success"
only:
- web
deploy-dev:
stage: deploy
retry: 2
tags:
- dev
variables:
APP_CNAME: "${APP_NAME}-${CI_COMMIT_REF_NAME}"
APP_PORT: # app的端口号
script:
- docker ps -a | grep -i ${APP_CNAME} > /dev/null 2>&1 && docker stop ${APP_CNAME} && docker rm ${APP_CNAME}
- docker images|grep none|grep ${APP_IMAGE} > /dev/null 2>&1 && docker images|grep none|grep ${APP_IMAGE}|awk '{print $3}'|xargs docker rmi
- docker run --name ${APP_CNAME} -d --env-file variable.env -p ${APP_PORT}:8080 -v /huansi/temp_data/mes_api:/huansi_app/static_file/temp ${APP_IMAGE}:${CI_COMMIT_REF_NAME}
only:
- dev
\ No newline at end of file
hs.module.AppDepends.122.sql
\ No newline at end of file
FROM 47.110.145.204:8084/huansi/python3:slim-util-0.0.3
COPY . .
EXPOSE 8080
CMD ["/bin/bash","docker_entry.sh"]
# pip install requests-unixsocket
\ No newline at end of file
from huansi_utils.common.modeloader import dynamic_import
from huansi_utils.webapi.HSWebApi import HSWebApi
_webApi = HSWebApi("api", __name__)
# 导出简单的类装饰器
api = _webApi.api
# dynamic_import(import_prefix=_webApi.import_name, root_dir=_webApi.root_path, file_suffix="_api.py")
dynamic_import(import_prefix=_webApi.import_name, root_dir=_webApi.root_path, file_suffix=["_api.py", "_api.pyc"])
# -*- coding:utf-8 -*-
\ No newline at end of file
# -*- coding:utf-8 -*-
from flask import request
from huansi_utils.webapi import ApiController
from app import api
from app.authorization.authorization_service import AuthorizationService
@api('authorization')
class AuthorizationAPI(ApiController):
@api()
def get_session_info(self):
'''
获取session信息
:return:
'''
return AuthorizationService().get_session_info()
@api()
def post_set_esssion_info(self):
'''
设置session信息
:return:
'''
return AuthorizationService().set_esssion_info(request.json)
@api('login')
def post_login(self):
'''
登录
:return:
'''
return AuthorizationService().login(request.json)
# -*- coding:utf-8 -*-
import json
import os
from huansi_utils.exception.exception import HSException
from huansi_utils.server.service_uc import HSBaseUCService
from static_file import temp_file_dir
session_info_path = os.path.join(temp_file_dir, 'session.info')
class AuthorizationService(HSBaseUCService):
def get_session_info(self):
'''
查询是否有session信息
:return:
'''
if os.path.exists(session_info_path):
return True
else:
return False
def set_esssion_info(self, json_data):
'''
设置session信息
:param json_data:
:return:
'''
session_info = json.dumps(json_data)
with open(session_info_path, 'w') as f:
f.write(session_info)
def login(self, json_data):
'''
登录
:param json_data:
:return:
'''
if not self.get_session_info():
raise HSException('账号或密码错误')
with open(session_info_path, 'r') as f:
session_info = f.read()
session_info_dict = json.loads(session_info)
if session_info_dict.get('user_name') == json_data.get('user_name') and \
session_info_dict.get('password') == json_data.get('password'):
return {'message': '登录成功'}
else:
raise HSException('账号或密码错误')
# -*- coding:utf-8 -*-
\ No newline at end of file
# -*- coding:utf-8 -*-
from flask import request
from huansi_utils.webapi import ApiController
from app import api
from app.conncetion.conncetion_service import ConnectionService
@api('connection')
class ConnectionAPI(ApiController):
@api()
def get_project_info(self):
'''
获取项目信息
:return:
'''
return ConnectionService().get_project_info()
@api()
def post_set_project_info(self):
'''
设置项目信息
:return:
'''
return ConnectionService().set_project_info(request.json)
@api('<string:project_no>/<string:db_name>')
def get_test_conncetion(self, project_no, db_name):
'''
测试项目数据库连接
:param project_no:
:param db_name:
:return:
'''
return ConnectionService().test_connection(project_no, db_name)
@api('remote_server')
def get_remote_server(self):
'''
获取远程服务器连接
:return:
'''
return ConnectionService().get_remote_server_info()
@api('remote_server')
def post_set_remote_server_info(self):
'''
设置远程服务器连接
:return:
'''
return ConnectionService().set_remote_server_info(request.json)
@api('test_port/<int:port>')
def get_test_port(self, port):
'''
测试端口是否开放
:param port:
:return:
'''
return ConnectionService().test_port(port)
# -*- coding:utf-8 -*-
from _socket import timeout
from huansi_utils.db.db import new_id
from huansi_utils.exception.exception import HSException
from huansi_utils.server.service_uc import HSBaseUCService
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from app.utils.db_tools import db_driver
class ConnectionService(HSBaseUCService):
def get_project_info(self):
'''
获取项目信息
:return:
'''
with db_driver as session:
project_info = session.retrive_sql('select * from project_info')
return project_info
def set_project_info(self, json_data):
'''
设置项目信息
:param json:{"project_no": "j1kq", "db_ip": "47.97.206.38", "db_port": "9610", "db_user": "000",
"db_password": "000", "mes_db_name": "HSGmtMes", "tiip_db_name": "HSTIIP"}
:return:
'''
id = json_data.get('id')
with db_driver as session:
if id:
update_sql = '''update project_info
set project_no=:project_no,db_ip=:db_ip,db_port=:db_port,db_user=:db_user
,db_password=:db_password,mes_db_name=:mes_db_name,tiip_db_name=:tiip_db_name
where id=:id'''
session.exec_sql(update_sql, json_data)
else:
id = new_id()
json_data['id'] = id
insert_sql = '''insert into project_info
(id,project_no,db_ip,db_port,db_user
,db_password,mes_db_name,tiip_db_name)
values
(:id,:project_no,:db_ip,:db_port,:db_user
,:db_password,:mes_db_name,:tiip_db_name)'''
session.exec_sql(insert_sql, json_data)
data = session.retrive_sql('select * from project_info where id=:id',
{"id": id})
if not data:
raise HSException('保存失败')
import os
from static_file import profile_dir
project_info_path = os.path.join(profile_dir, 'huansi.sh')
# 写入
with open(project_info_path, "w") as f:
for k, v in data.items():
k = self.math_name(k)
if k in ['ID', 'CREATE_TIME']:
continue
f.writelines("export {}={}\n".format(k, v))
return {'message': "保存成功"}
def math_name(self, k):
'''
匹配名称
:param k:
:return:
'''
if k == 'project_no':
return 'HSCUSCODE'
if k == 'db_ip':
return 'HSDB_HOST'
if k == 'db_port':
return 'HSDB_PORT'
if k == 'db_user':
return 'HSDB_USER'
if k == 'db_password':
return 'HSDB_PASSWORD'
if k == 'mes_db_name':
return 'HSDB_NAME'
else:
return k.upper()
def test_connection(self, project_no, db_name):
'''
测试数据库连接
:param project_no:
:param db_name:
:return:
'''
with db_driver as session:
project_info = session.retrive_sql(f"select * from project_info where project_no='{project_no}'")
if not project_info:
raise HSException(f'未找到【{project_no}】的信息')
db_ip = project_info['db_ip']
db_port = project_info['db_port']
db_user = project_info['db_user']
db_password = project_info['db_password']
test_db_name = db_name
con_str = f'mssql+pymssql://{db_user}:{db_password}@{db_ip}:{db_port}/{test_db_name}'
engine = create_engine(con_str)
DBSession = sessionmaker(bind=engine)
try:
session = DBSession()
result = session.execute("select test='huansi'")
data = result.fetchone()
if data.test == 'huansi':
return {'message': '连接成功'}
except Exception as e:
orig = getattr(e, 'orig')
args = getattr(orig, 'args', ((),))[0]
error_message = ''
for e_item in args:
if isinstance(e_item, bytes):
try:
e_item = e_item.decode()
except:
e_item = str(e_item)
error_message += str(e_item)
if not error_message:
error_message = str(e)
print("数据库连接失败,失败原因:{}".format(error_message))
return {'message': '连接失败'}
def get_remote_server_info(self):
'''
获取远端服务器信息
:return:
'''
with db_driver as session:
remote_server_info = session.retrive_sql('select * from remote_server_info')
return remote_server_info
def set_remote_server_info(self, json_data):
'''
设置远端服务器信息
:return:
'''
id = json_data.get('id')
with db_driver as session:
if id:
update_sql = '''update remote_server_info
set project_no=:project_no,server_ip=:server_ip,server_ssh_port=:server_ssh_port,server_user=:server_user
,server_password=:server_password
where id=:id'''
session.exec_sql(update_sql, json_data)
else:
id = new_id()
json_data['id'] = id
insert_sql = '''insert into remote_server_info
(id,project_no,server_ip,server_ssh_port,server_user
,server_password)
values
(:id,:project_no,:server_ip,:server_ssh_port,:server_user
,:server_password)'''
session.exec_sql(insert_sql, json_data)
return {'message': "保存成功"}
def test_port(self, port):
project_info = self.get_project_info()
if not project_info:
raise HSException('项目信息未查到,请先配置')
import telnetlib
host_ip = project_info.get('host_ip')
try:
telnetlib.Telnet(host=host_ip, port=port, timeout=2)
except timeout:
raise HSException(f'{host_ip}:{port}连接超时')
except Exception as e:
raise HSException(f'{host_ip}:{port}连接失败')
# -*- coding:utf-8 -*-
\ No newline at end of file
# -*- coding:utf-8 -*-
from flask import request
from huansi_utils.webapi import ApiController
from app import api
from app.info.info_service import InfoService
@api('info')
class InfoAPI(ApiController):
@api('app_list')
def get_app_list(self):
'''
获取app_list
:return:
'''
return InfoService().get_app_list()
@api('app_upgrade')
def post_set_app_upgrade_info(self):
'''
配置app升级信息
:return:
'''
return InfoService().set_upgrade_info(request.json)
@api('upgrade_log')
def get_upgrade_log(self):
'''
获取app升级日志列表
:return:
'''
return InfoService().get_upgrade_log()
@api('upgrade_log_dtl/<int:log_id>')
def get_upgrade_log_dtl(self, log_id):
'''
获取app单次升级日志明细
:param log_id:
:return:
'''
return InfoService().get_upgrade_log_dtl(log_id)
# -*- coding:utf-8 -*-
import os
import shutil
import time
from huansi_utils.app.apploader import logger
from huansi_utils.db.db import new_id
from huansi_utils.exception.exception import HSException
from huansi_utils.server.service_uc import HSBaseUCService
import docker
from app.conncetion.conncetion_service import ConnectionService
from app.utils.db_tools import db_driver
from static_file import builds_dir, back_up_dir
# docker_client = docker.APIClient(base_url='tcp://localhost:2375')
docker_client = docker.APIClient(base_url='unix:///var/run/docker.sock')
class InfoService(HSBaseUCService):
def get_app_list(self):
project_info = ConnectionService().get_project_info()
if not project_info:
raise HSException('请先设置项目信息')
host_ip = project_info.get('host_ip')
# host_ip = 'localhost'
containers = docker_client.containers()
app_info_list = []
for container in containers:
app_info = {}
app_info['app_name'] = container['Names'][0][1:] # "/***"这种格式
port = container['Ports']
if port:
app_info['app_url'] = f"{host_ip}:{port[0].get('PublicPort')}"
else:
app_info['app_url'] = ''
image_name = container['Image']
# 区分版本号
if ':' in image_name:
verison = image_name.split(':')[-1:][0]
else:
verison = 'latest'
app_info['version'] = verison
app_info['state'] = container['State']
app_info['status'] = container['Status']
app_info['creat_time'] = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(container['Created']))
app_info_list.append(app_info)
return app_info_list
def set_upgrade_info(self, app_list):
'''
设置app升级信息
:arg app_list [{app_code,app_name,app_image}]
:return:
'''
logger.info(f'获取到的升级app列表:{app_list}')
log_id = new_id()
sql_list = []
if not app_list:
raise HSException('app信息为空')
import time
upgrade_no = time.strftime('%Y%m%d%H%M%S', time.localtime(time.time()))
# 新增表头
sql = f'''insert into app_upgrade_log
(id,status,default_version,upgrade_no)
values
({log_id},'成功',1,'{upgrade_no}')'''
sql_list.append(sql)
# 更新之前的升级app状态
sql = f'''update app_upgrade_log
set default_version=0
where id <> {log_id}'''
sql_list.append(sql)
for app in app_list:
app_image = app.get('app_image', '')
app_code = app.get('app_code', '')
app_name = app.get('app_name', '')
app_image_info = docker_client.images(name=app_image)
if not app_image_info:
raise HSException(f'找不到[{app_image}]的具体镜像')
app_image_id = app_image_info[0]['Id'].replace('sha256:', '')
app_image_name = app_image_info[0]['RepoTags'][0]
_sql = f'''insert into app_upgrade_log_dtl
(log_id,app_code,app_name,app_image_id,app_image_name)
values
({log_id},'{app_code}','{app_name}','{app_image_id}','{app_image_name}')'''
sql_list.append(_sql)
with db_driver as session:
for sql_str in sql_list:
session.exec_sql(sql_str)
# 备份升级文件
upgrade_back_up_dir = os.path.join(back_up_dir, upgrade_no)
if not os.path.exists(upgrade_back_up_dir):
os.mkdir(upgrade_back_up_dir)
deploy_dir = os.path.join(builds_dir, 'deploy')
for files in os.listdir(deploy_dir):
name = os.path.join(deploy_dir, files)
back_name = os.path.join(upgrade_back_up_dir, files)
shutil.copy(name, back_name)
return {'message': '写入成功'}
def get_upgrade_log(self):
'''
获取升级历史信息
:return:
'''
sql = '''select A.*,(select count(*) from app_upgrade_log_dtl where log_id=A.id) as count
from app_upgrade_log A
order by A.default_version desc, A.upgrade_time desc
'''
with db_driver as session:
data = session.query_sql(sql)
return data
def get_upgrade_log_dtl(self, log_id):
'''
获取升级历史信息详情
:param log_id:
:return:
'''
sql = f'''select A.*
from app_upgrade_log_dtl A
where A.log_id={log_id}
'''
with db_driver as session:
data = session.query_sql(sql)
return data
# -*- coding:utf-8 -*-
\ No newline at end of file
# -*- coding:utf-8 -*-
from huansi_utils.webapi import ApiController
from app import api
from app.install.install_service import InstallService
@api('install')
class InstallAPI(ApiController):
@api('runner')
def get_list(self):
'''
安装runner
:return:
'''
return InstallService().install_runner()
@api('remote_server/docker/')
def get_install_remote_server_docker(self):
'''
远端服务器安装docker
:return:
'''
return InstallService().install_remote_service_docker()
# -*- coding:utf-8 -*-
import os
from huansi_utils.exception.exception import HSException
from huansi_utils.server.service_uc import HSBaseUCService
from app.conncetion.conncetion_service import ConnectionService
from app.utils.ssh_tools import SSHConnect
class InstallService(HSBaseUCService):
def install_runner(self):
'''
安装runner
:return:
'''
conncetion_service = ConnectionService()
project_info = conncetion_service.get_project_info()
if not project_info:
raise HSException('项目信息未查到,请先配置')
project_code = project_info['project_code']
self.install_huansi_runner(project_code)
def install_huansi_runner(self, HSCUSCODE):
'''
安装
:param HSCUSCODE:
:return:
'''
self.delete_setting_file()
s1 = os.system("sudo docker run --rm -t -i -v /etc/gitlab-runner:/etc/gitlab-runner gitlab/gitlab-runner register \
-n -u http://47.110.145.204:8085/ -r WwpzH4qk19KjvAjEwoTz --executor docker --docker-image docker \
--description huansi.{hscode} \
--tag-list huansi.{hscode}".format(hscode=HSCUSCODE))
if s1 != 0:
raise HSException('gitlab-runner安装失败')
self._edit_runner_setting()
self._remove_old_runner_container()
self.start_runner()
def _edit_runner_setting(self):
os.system("sudo chmod 777 /etc/gitlab-runner/config.toml")
with open('/etc/gitlab-runner/config.toml', 'rw') as f:
s = f.read()
a = s.replace(' volumes = ["/cache"]',
'volumes = ["/var/run/docker.sock:/var/run/docker.sock","/etc/profile.d/huansi.sh:/etc/profile.d/huansi.sh" ,"/cache","/huansi/gitlab-runner/builds:/builds/hs"]')
f.write(a)
def _remove_old_runner_container(self):
'''
移除旧runner容器,主要作用于第二次第三次的重复安装
:return:
'''
os.system("sudo docker stop gitlab-runner && docker rm gitlab-runner")
def start_runner(self):
s = os.system("sudo docker run -d --name gitlab-runner --restart always \
-v /etc/gitlab-runner:/etc/gitlab-runner \
-v /var/run/docker.sock:/var/run/docker.sock \
-v /etc/profile.d/huansi.sh:/etc/profile.d/huansi.sh \
gitlab/gitlab-runner:latest")
if s != 0:
raise HSException('启动runner失败')
def delete_setting_file(self):
os.system('rm -f /etc/gitlab-runner/config.toml')
def install_remote_service_docker(self):
'''
安装远端服务器的docker
:return:
'''
remote_server_info = ConnectionService().get_remote_server_info()
if not remote_server_info:
raise HSException('远端服务器信息未查到,请先配置')
ssh_conenct = SSHConnect(host_ip=remote_server_info['server_ip'],
host_port=remote_server_info['server_ssh_port'],
user_name=remote_server_info['server_user'],
password=remote_server_info['server_password'])
with ssh_conenct as ssh:
InstallDocker(ssh).run()
docker_service = '''[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network-online.target firewalld.service
Wants=network-online.target
[Service]
Type=notify
# the default is not to use systemd for cgroups because the delegate issues still
# exists and systemd currently does not support the cgroup feature set required
# for containers run by docker
ExecStart=/usr/bin/dockerd
ExecReload=/bin/kill -s HUP $MAINPID
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
# Uncomment TasksMax if your systemd version supports it.
# Only systemd 226 and above support this version.
#TasksMax=infinity
TimeoutStartSec=0
# set delegate yes so that systemd does not reset the cgroups of docker containers
Delegate=yes
# kill only the docker process, not all processes in the cgroup
KillMode=process
# restart the docker process if it exits prematurely
Restart=on-failure
StartLimitBurst=3
StartLimitInterval=60s
[Install]
WantedBy=multi-user.target'''
class InstallDocker(object):
def __init__(self, ssh: SSHConnect):
self.ssh = ssh
def run(self):
self._close_firewalld()
self._install_docker()
self._install_docker_compose()
self.ssh.exec_command('sudo mkdir -p /huansi && sudo mkdir -p /huansi/upgrade')
print('*****************************************************')
print('********服务器需要重启,请输入reboot重启服务器*************')
print('*****************************************************')
def _close_firewalld(self):
print('关闭防火墙...')
out, err = self.ssh.exec_command('systemctl stop firewalld && systemctl disable firewalld')
print(out)
if err: raise RuntimeError('关闭防火墙失败')
def _install_docker_compose(self):
print('修改/etc/selinux/config下的SELINUX属性')
out, err = self.ssh.exec_command("sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config")
print(out)
if err: raise RuntimeError('修改/etc/selinux/config下的SELINUX属性失败')
print('移动文件至/usr/local/bin/docker-compose...')
out, err = self.ssh.exec_command('sudo mv docker-compose-Linux-x86_64 /usr/local/bin/docker-compose')
print(out)
if err: raise RuntimeError('移动文件至/usr/local/bin/docker-compose失败')
print('添加可执行权限...')
out, err = self.ssh.exec_command('sudo chmod +x /usr/local/bin/docker-compose')
print(out)
if err: raise RuntimeError('添加可执行权限失败')
print('查看docker-compose版本...')
out, err = self.ssh.exec_command('docker-compose -v')
print(out)
if err: raise RuntimeError('查看docker-compose版本失败')
def _install_docker(self):
print('解压tar包...')
out, err = self.ssh.exec_command('sudo tar -xvf docker-19.03.4.tgz')
print(out)
if err: raise RuntimeError('解压tar包失败')
print('将docker目录移到/usr/bin目录下...')
out, err = self.ssh.exec_command('sudo cp docker/* /usr/bin/')
print(out)
if err: raise RuntimeError('将docker目录移到/usr/bin目录下失败')
print('在/etc/systemd/system/目录下创建docekr.service...')
out, err = self.ssh.exec_command(f"""sudo tee /etc/systemd/system/docker.service <<-'EOF'
{docker_service}
EOF""")
print(out)
if err: raise RuntimeError('在/etc/systemd/system/目录下创建docekr.service失败')
print('添加文件权限...')
out, err = self.ssh.exec_command('sudo chmod +x /etc/systemd/system/docker.service')
print(out)
if err: raise RuntimeError('添加文件权限失败')
print('新增配置文件/etc/docker/daemon.json...')
out, err = self.ssh.exec_command("""sudo mkdir -p /etc/docker && sudo tee /etc/docker/daemon.json <<-'EOF'
{
"registry-mirrors":["https://m6wlkecl.mirror.aliyuncs.com"],
"insecure-registries": ["http://47.110.145.204:8084","http://183.134.73.2:8084"],
"log-driver": "json-file",
"log-opts": {
"max-size": "50m",
"max-file": "3"
}
}
EOF""")
print(out)
if err: raise RuntimeError('新增配置文件/etc/docker/daemon.json失败')
print('重新加载配置文件...')
out, err = self.ssh.exec_command('sudo systemctl daemon-reload')
print(out)
if err: raise RuntimeError('重新加载配置文件失败')
print('启动docker...')
out, err = self.ssh.exec_command('sudo systemctl start docker')
print(out)
if err: raise RuntimeError('启动docker失败')
print('设置开机自启...')
out, err = self.ssh.exec_command('sudo systemctl enable docker.service')
print(out)
if err: raise RuntimeError('设置开机自启失败')
print('查看docker版本...')
out, err = self.ssh.exec_command('docker -v')
print(out)
if err: raise RuntimeError('查看docker版本失败')
# -*- coding:utf-8 -*-
\ No newline at end of file
# -*- coding:utf-8 -*-
from flask import request
from huansi_utils.webapi import ApiController
from app import api
from app.upgrade.upgrade_service import UpgradeService
@api('upgrade')
class UpgradeAPI(ApiController):
'''
1、回滚升级
2、远程升级
3、导出安装包
4、导入安装包升级(不紧急)
'''
@api('rollback/<int:log_id>')
def get_rollback_upgrade(self, log_id):
'''
回滚升级
:return:
'''
return UpgradeService().rollback_upgrade(log_id)
@api('remote')
def get_remote_upgrade(self):
'''
远程升级
:return:
'''
log_id = request.args.get('log_id')
return UpgradeService().remote_upgrade(log_id)
@api('package_images/<int:log_id>')
def get_package_images(self, log_id):
'''
导出镜像
:param log_id:
:return:
'''
return UpgradeService().package_images(log_id)
This diff is collapsed.
# -*- coding:utf-8 -*-
\ No newline at end of file
import os
import sqlite3
from collections import OrderedDict
from static_file import temp_file_dir
create_table_sql_list = ['''create table if not exists project_info
(id bigint primary key
, project_no varchar(50)
, db_ip varchar(50)
, db_port varchar(50)
, db_user varchar(50)
, db_password varchar(50)
, mes_db_name varchar(100)
, host_ip varchar(50)
, tiip_db_name varchar(100) default ''
, right_db_name varchar(100) default ''
, create_time date default (datetime('now','localtime'))
)''', '''create table if not exists app_upgrade_log
(id bigint primary key
, status varchar(50)
, default_version bit
, upgrade_time date default (datetime('now','localtime'))
, upgrade_no varchar(50)
)''', '''create table if not exists app_upgrade_log_dtl
(id integer primary key autoincrement
, log_id bigint
, app_code varchar(50)
, app_name varchar(100)
, app_image_id varchar(200)
, app_image_name varchar(200)
)''', '''create table if not exists remote_server_info
(id bigint
, project_no varchar(50)
, server_ip varchar(50)
, server_ssh_port varchar(50)
, server_user varchar(50)
, server_password varchar(50)
)''']
# createtabsql1 = "create table if not exists scriptdata(id integer primary key autoincrement, name varchar(128), info varchar(128))"
class DBDriver:
def __init__(self, dbfile=None):
if not dbfile:
dbfile = os.path.join(temp_file_dir, 'data.db')
self.dbfile = dbfile
def cerateDB(self):
self.conn = sqlite3.connect(self.dbfile)
self.conn.isolation_level = None
for create_table_sql in create_table_sql_list:
self.conn.execute(create_table_sql) ####create new table
return
def execDB(self, execsql):
self.conn.execute(execsql)
self.conn.commit()
return
def exec_sql(self, sql, *args, **kwargs):
cur = self.conn.cursor()
cur.execute(sql, *args, **kwargs)
return cur
def retrive_sql(self, sql, *args, **kwargs):
cur = self.exec_sql(sql, *args, **kwargs)
fields = [item[0] for item in cur.description]
res = cur.fetchone()
if res is None:
return res
res_dict = dict(map(lambda x, y: [x, y], fields, res))
return res_dict
def query_sql(self, sql, *args, **kwargs):
cur = self.exec_sql(sql, *args, **kwargs)
fields = [item[0] for item in cur.description]
res = cur.fetchall()
res_list = []
for r in res:
res_dict = OrderedDict(map(lambda x, y: [x, y], fields, r))
res_list.append(res_dict)
return res_list
def __enter__(self):
self.cerateDB()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
if exc_val:
self.conn.rollback()
else:
self.conn.commit()
self.conn.close()
db_driver = DBDriver()
# -*- coding:utf-8 -*-
import paramiko, getpass # getpass是隐藏密码
from paramiko import AuthenticationException
class SSHConnect():
def __init__(self, host_ip, host_port, user_name, password):
self.host_ip = host_ip
self.host_port = host_port
self.user_name = user_name
self.password = password
def conncet(self):
try:
# SSH远程连接
self.ssh = paramiko.SSHClient() # 创建sshclient
self.ssh.set_missing_host_key_policy(
paramiko.AutoAddPolicy()) # 指定当对方主机没有本机公钥的情况时应该怎么办,AutoAddPolicy表示自动在对方主机保存下本机的秘钥
self.ssh.connect(self.host_ip, self.host_port, self.user_name, self.password)
except AuthenticationException:
raise RuntimeError('登录信息不正确,请检查')
except Exception:
raise RuntimeError(f'{self.host_ip}连不上,请检查')
def __enter__(self):
self.conncet()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.ssh.close()
def exec_command(self, command):
if not self.ssh:
raise RuntimeError('请先连接')
# 执行命令并获取执行结果
stdin, stdout, stderr = self.ssh.exec_command(command)
out = stdout.read()
err = stderr.read()
return out, err
def upload(self, local_path, target_path):
"""
本地上传文件到服务器
:param local_path: 本地文件地址
:param target_path: 目标文件路径
"""
sftp = paramiko.SFTPClient.from_transport(self.ssh)
sftp.put(local_path, target_path)
def download(self, remote_path, local_path):
"""
服务器下载文件到本地
:param remote_path: 服务器文件路径
:param local_path: 本地目标路径
"""
sftp = paramiko.SFTPClient.from_transport(self.ssh)
sftp.get(remote_path, local_path)
#!/usr/bin/env bash
# app所需SQL升级
nohup python -u upgradeDB.py > upgrade_log.log 2>&1 &
if [ -z "$THREADS" ]; then
THREADS=50
fi
if [ -z "$WORKER" ]; then
WORKER=2
fi
# 默认2个进程 50个线程
gunicorn -b 0.0.0.0:23280 -w $WORKER --threads=$THREADS startup:global_app -t 60 --log-level debug
\ No newline at end of file
from flask import Flask
from huansi_utils.app.apploader import AppLoaderBase
from huansi_utils.db.sqlalchemy import db
from huansi_utils.flask_docs import ApiDoc
from flask_config import Config
class FlashAppLoader(AppLoaderBase):
def init(self, app: Flask):
app.config.from_object(Config)
# 因为之前的基类不规范,到处引用,这样强行将import_name修改成当前文件
app.import_name = 'flask_app'
def register_blueprint(self, app):
# 新版本api doc
ApiDoc(app)
def init_db(self, app):
db.init_app(app)
def register_webapi(self, app):
from app import _webApi
_webApi.init_app(app)
global_app = FlashAppLoader()()
import os
# api请求加密的密文key
import time
# 签名key
secretKey = '018f162e804f945ee6b23aebfa863639'
# 报表签名key
report_secretKey = 'report028!f162e804f945#eesdkljfdskljfdsXa'
from static_file import base_dir
BASE_DIR = os.path.abspath(os.path.dirname(__file__))
class Config:
JSON_AS_ASCII = False
with open(os.path.join(BASE_DIR, "sql_config.txt"), 'r') as f:
import json
temp = json.loads(f.read())
user = os.environ.get('DB_USER') or temp['config']['DB_USER']
pwd = os.environ.get('DB_PASSWORD') or temp['config']['DB_PASSWORD']
host = os.environ.get('DB_HOST') or temp['config']['DB_HOST']
port = int(os.environ.get('DB_PORT', 0)) or temp['config']['DB_PORT']
db = os.environ.get('DB_NAME') or temp['config']['DB_NAME']
IP_PORT = temp['config']['IP_PORT'] or 5000
IP_ADDRESS = temp['config']['IP_ADDRESS']
data = dict(user=user, pwd=pwd, host=host, port=port, db=db)
con_str = 'mssql+pymssql://{user}:{pwd}@{host}:{port}/{db}'
SQLALCHEMY_DATABASE_URI = os.environ.get('DATABASE_URL') or con_str.format(**data)
# SQLALCHEMY_POOL_SIZE = 5
SQLALCHEMY_POOL_TIMEOUT = 60
# SQLALCHEMY_POOL_RECYCLE = -1
# SQLALCHEMY_MAX_OVERFLOW = 10
PROJECT_INFO_PATH = os.getenv('PROJECT_INFO_PATH', '/etc/profile.d/huansi.sh')
# **********全局SQL日志************
# 全局SQL日志记录
GLOBAL_SQL_LOG = os.getenv('GLOBAL_SQL_LOG', False)
# 是否打印全局SQL日志
GLOBAL_SQL_ECHO = os.getenv('GLOBAL_SQL_ECHO', False)
# APPCode
APP_CODE = ''
# 代理地址
PROXY_URL = os.getenv('PROXY_URL')
# rpc代理地址
RPC_PROXY_URL = os.getenv('RPC_PROXY_URL')
# rpc服务
HSRIGHT_RPC = os.getenv('HSRIGHT_RPC')
# authorize开关
START_AUTHORIZE = os.getenv('START_AUTHORIZE', False)
# 日志配置
syslog_tag = "huansi"
LOGCONFIG = {
'version': 1,
'disable_existing_loggers': False,
'filters': {},
'formatters': {
"defualt": {
"format": "[%(asctime)s] %(levelname)s - %(message)s"
},
"simple": {
"format": "[%(asctime)s] %(levelname)s - [FileName:%(filename)s] - [FuncName:%(funcName)s] - [LineNo:%(lineno)s} - %(message)s"
}
},
'handlers': {
'console': {
'()': 'logging.StreamHandler',
'formatter': 'defualt'
},
'file': {
'()': 'logging.handlers.RotatingFileHandler',
'formatter': 'simple',
'filename': '{}/log/log_{}.txt'.format(base_dir, time.strftime("%Y-%m-%d")),
'mode': 'a',
'maxBytes': 50000, # 5 MB
'backupCount': 1,
"encoding": "utf8"
},
},
'loggers': {
'myapp': {
'handlers': ['file'],
'level': 'DEBUG'
},
},
'root': {
'handlers': ['console'],
'level': 'DEBUG'
}
}
# 数据卷挂载说明
/etc/profile.d/:/huansi_app/static_file/profile/
/huansi/gitlab-runner/builds/:/huansi_app/static_file/builds/
/data/upgrade_tools_data/:/huansi_app/static_file/temp/
/data/upgrade/:/data/upgrade
{
"config":{
"DB_USER":"huansi",
"DB_PASSWORD":"huansi",
"DB_HOST":"47.97.206.38",
"DB_PORT":"9610",
"DB_NAME":"HsGmtMES",
"IP_PORT":"5000",
"IP_ADDRESS": "localhost"
}
}
\ No newline at end of file
from flask_cors import CORS
from flask_app import global_app
# 允许跨域请求
CORS(global_app, supports_credentials=True)
from huansi_utils.route import _route_register_instance
_route_register_instance.init_app(global_app)
'''新的一套API运行方式'''
if __name__ == '__main__':
global_app.run(host="0.0.0.0", port=int(global_app.config.get('IP_PORT')) or 5000, debug=True)
import os
base_dir = os.path.abspath(os.path.dirname(__file__))
temp_file_dir = os.path.join(base_dir, 'temp')
excel_file_dir = os.path.join(base_dir, 'import_excel')
excel__dir = os.path.join(base_dir, 'import_excel')
export_dir = os.path.join(base_dir, 'temp')
builds_dir = os.path.join(base_dir, 'builds')
back_up_dir = os.path.join(temp_file_dir, 'backup')
profile_dir = os.path.join(base_dir, 'profile')
system_file_dir = os.path.join(base_dir, 'system_file')
# -*- coding:utf-8 -*-
\ No newline at end of file
import os
file_dir = os.path.abspath(os.path.dirname(__file__))
# -*- coding:utf-8 -*-
\ No newline at end of file
# -*- coding:utf-8 -*-
\ No newline at end of file
#!/usr/bin/env bash
cd /huansi/upgrade
now_time=`date +%Y%m%d_%H%M%S`
python -u load_handle.py > ${now_time}.log
sleep 3
python -u analysis_file_to_hsright.py
sleep 3
python -u analysis_file_to_nginx.py
\ No newline at end of file
# -*- coding:utf-8 -*-
import os
import re
if __name__ == '__main__':
with os.popen('ls | grep .tar') as f:
cmd_txt = f.read()
if not cmd_txt:
raise RuntimeError('请检查文件,tar文件为空')
app_name_list = cmd_txt.split('\n')
for app_name in app_name_list:
if not app_name:
continue
print('加载镜像:{}'.format(app_name[:-4].replace('___', "/").replace("__", ":")))
cmd_res = os.system('docker load -i {}'.format(app_name))
if cmd_res == 1:
raise RuntimeError('加载失败')
print('镜像加载完毕')
print('开始升级程序')
with open('docker-compose.yml', 'r') as f:
compose_content = f.read()
app_list_str = re.findall('# app_list: ".*"', compose_content)
if app_list_str:
app_list_str = app_list_str[0]
else:
raise RuntimeError('未设置app_list')
app_str = app_list_str[len('# app_list: "'):-1]
if app_str == '*':
res = os.system('''export HUANSI_REGISTRY_URL=47.110.145.204:8084 &&\
source /etc/profile.d/huansi.sh &&\
docker-compose up -p deploy -d --force-recreate''')
else:
res = os.system('''export HUANSI_REGISTRY_URL=47.110.145.204:8084 &&\
source /etc/profile.d/huansi.sh &&\
docker-compose up -p deploy -d --force-recreate --no-deps {}'''.format(app_str))
if res == 1: raise RuntimeError('程序升级失败')
# -*- coding:utf-8 -*-
\ No newline at end of file
# -*- coding:utf-8 -*-
\ No newline at end of file
# -*- coding:utf-8 -*-
import json
import os
import unittest
from flask_app import global_app
from static_file import temp_file_dir
session_info_path = os.path.join(temp_file_dir, 'session.info')
class TestAuthorization(unittest.TestCase):
def setUp(self):
self.client = global_app.test_client()
def test_get_session_info(self):
# 删除文件
if os.path.exists(session_info_path):
os.remove(session_info_path)
res = self.client.get('/authorization/')
self.assertEqual(res.status_code, 200)
self.assertFalse(res.json)
# 缓存文件不存在
self.assertFalse(os.path.exists(session_info_path))
def test_set_session_info(self):
headers = {
'Content-Type': "application/json",
}
# 正常新增
json_data = {"user_name": "123", "password": "321"}
res = self.client.post('/authorization/', headers=headers, json=json_data)
self.assertEqual(res.status_code, 200)
# 缓存文件存在
self.assertTrue(os.path.exists(session_info_path))
# 判断是否正常的文件
with open(session_info_path, 'r') as f:
session_info = f.read()
session_info_dict = json.loads(session_info)
self.assertEqual(session_info_dict, json_data)
def test_login(self):
self.test_set_session_info()
headers = {
'Content-Type': "application/json",
}
# 正常新增
json_data = {"user_name": "124", "password": "321"}
res = self.client.post('/authorization/login/', headers=headers, json=json_data)
self.assertEqual(res.status_code, 501)
self.assertEqual(res.json['error_title'], '账号或密码错误')
json_data = {"user_name": "123", "password": "321"}
res = self.client.post('/authorization/login/', headers=headers, json=json_data)
self.assertEqual(res.status_code, 200)
self.assertEqual(res.json['message'], '登录成功')
# -*- coding:utf-8 -*-
import unittest
from flask_app import global_app
from app.utils.db_tools import db_driver
delete_conncetion_sql = 'delete from project_info'
test_insert_connection_sql = '''insert into project_info
(id,project_no,db_ip,db_port,db_user
,db_password,mes_db_name,tiip_db_name,host_ip)
values
('1','jkq_test','47.97.206.38',9610,'huansi'
,'huansi','HSGmtMes','HSTIIP','192.168.4.116')
'''
delete_remote_conncetion_sql = 'delete from remote_server_info'
test_insert_remote_connection_sql = '''insert into remote_server_info
(id,project_no,server_ip,server_ssh_port,server_user
,server_password)
values
(1,'jkq_test','192.168.4.116','1111','root'
,'huansi.net')
'''
class TestDataMaintain(unittest.TestCase):
def setUp(self):
self.client = global_app.test_client()
def tearDown(self):
pass
def test_get_db_connection(self):
# 删除数据
with db_driver as session:
session.exec_sql(delete_conncetion_sql)
session.exec_sql(test_insert_connection_sql)
res = self.client.get('/connection/')
self.assertEqual(res.status_code, 200)
self.assertIsInstance(res.json, dict)
self.assertEqual(res.json.get('db_password'), 'huansi')
self.assertEqual(res.json.get('mes_db_name'), 'HSGmtMes')
self.assertEqual(res.json.get('db_ip'), '47.97.206.38')
self.assertEqual(res.json.get('project_no'), 'jkq_test')
def test_set_db_conncetion(self):
with db_driver as session:
session.exec_sql(delete_conncetion_sql)
headers = {
'Content-Type': "application/json",
}
# 正常新增
json_data = {"project_no": "jkq", "db_ip": "47.97.206.38", "db_port": "9610", "db_user": "huansi",
"db_password": "huansi", "mes_db_name": "HSGmtMes", "tiip_db_name": "HSTIIP"}
res = self.client.post('/connection/', headers=headers, json=json_data)
self.assertEqual(res.status_code, 200)
self.assertEqual(res.json['message'], '保存成功')
with db_driver as session:
project_info = session.retrive_sql('select * from project_info')
self.assertEqual(project_info.get('db_password'), 'huansi')
self.assertEqual(project_info.get('mes_db_name'), 'HSGmtMes')
self.assertEqual(project_info.get('db_ip'), '47.97.206.38')
self.assertEqual(project_info.get('project_no'), 'jkq')
id = project_info['id']
json_data = {"id": id, "project_no": "jkq_test", "db_ip": "47.97.206.38", "db_port": "9610",
"db_user": "huansi",
"db_password": "pwd", "mes_db_name": "HSGmtMes22", "tiip_db_name": "HSTIIP"}
res = self.client.post('/connection/', headers=headers, json=json_data)
self.assertEqual(res.status_code, 200)
self.assertEqual(res.json['message'], '保存成功')
with db_driver as session:
project_info = session.retrive_sql('select * from project_info')
self.assertEqual(project_info.get('db_password'), 'pwd')
self.assertEqual(project_info.get('mes_db_name'), 'HSGmtMes22')
self.assertEqual(project_info.get('db_ip'), '47.97.206.38')
self.assertEqual(project_info.get('project_no'), 'jkq_test')
self.assertEqual(project_info.get('id'), id)
def test_connecton_test(self):
res = self.client.get('/connection/jkq_test/HSGmtMes/')
self.assertEqual(res.status_code, 200)
self.assertEqual(res.json['message'], '连接成功')
res = self.client.get('/connection/jkq_test/HSGmtMes2/')
self.assertEqual(res.status_code, 200)
self.assertEqual(res.json['message'], '连接失败')
res = self.client.get('/connection/jkq1/HSGmtMes/')
self.assertEqual(res.status_code, 501)
self.assertEqual(res.json['error_title'], '未找到【jkq1】的信息')
def test_port(self):
res = self.client.get('/connection/test_port/83/')
self.assertEqual(res.status_code, 501)
self.assertIn('连接', res.json['error_title'])
res = self.client.get('/connection/test_port/82/')
self.assertEqual(res.status_code, 200)
def test_get_remote_server_connection(self):
# 删除数据
with db_driver as session:
session.exec_sql(delete_remote_conncetion_sql)
session.exec_sql(test_insert_remote_connection_sql)
res = self.client.get('/connection/remote_server/')
self.assertEqual(res.status_code, 200)
self.assertIsInstance(res.json, dict)
self.assertEqual(res.json.get('server_ssh_port'), '1111')
self.assertEqual(res.json.get('project_no'), 'jkq_test')
def test_set_remote_conncetion(self):
with db_driver as session:
session.exec_sql(delete_remote_conncetion_sql)
headers = {
'Content-Type': "application/json",
}
# 正常新增
json_data = {"project_no": "jkq", "server_ip": "47.97.206.38", "server_ssh_port": "9610",
"server_user": "huansi",
"server_password": "huansi"}
res = self.client.post('/connection/remote_server/', headers=headers, json=json_data)
self.assertEqual(res.status_code, 200)
self.assertEqual(res.json['message'], '保存成功')
with db_driver as session:
project_info = session.retrive_sql('select * from remote_server_info')
self.assertEqual(project_info.get('server_password'), 'huansi')
self.assertEqual(project_info.get('server_ssh_port'), '9610')
self.assertEqual(project_info.get('server_ip'), '47.97.206.38')
self.assertEqual(project_info.get('project_no'), 'jkq')
id = project_info['id']
json_data = {"id": id, "project_no": "jkq_test", "server_ip": "192.168.4.116", "server_ssh_port": "1111",
"server_user": "root",
"server_password": "huansi.net"}
res = self.client.post('/connection/remote_server/', headers=headers, json=json_data)
self.assertEqual(res.status_code, 200)
self.assertEqual(res.json['message'], '保存成功')
with db_driver as session:
project_info = session.retrive_sql('select * from remote_server_info')
self.assertEqual(project_info.get('server_password'), 'huansi.net')
self.assertEqual(project_info.get('server_ssh_port'), '1111')
self.assertEqual(project_info.get('server_ip'), '192.168.4.116')
self.assertEqual(project_info.get('project_no'), 'jkq_test')
self.assertEqual(project_info.get('id'), id)
# -*- coding:utf-8 -*-
import json
import unittest
from app.utils.db_tools import db_driver
from flask_app import global_app
class TestGetInfo(unittest.TestCase):
def setUp(self):
self.client = global_app.test_client()
def test_app_list(self):
# TestDataMaintain().test_get_db_connection()
res = self.client.get('/info/app_list/')
self.assertEqual(res.status_code, 200)
print(json.dumps(res.json, indent=4))
def test_set_app_list_info(self):
'''[{app_code,app_name,app_image}]'''
with db_driver as session:
session.exec_sql('delete from app_upgrade_log')
session.exec_sql('delete from app_upgrade_log_dtl')
headers = {
'Content-Type': "application/json",
}
# 正常新增
json_data = [
{"app_code": "rabbitMQ", "app_name": "rabbitMQ", "app_image": "47.110.145.204:8084/huansi/mq:latest"},
{"app_code": "Redis", "app_name": "Redis", "app_image": "47.110.145.204:8084/huansi/redis:latest"},
{"app_code": "Tool", "app_name": "tools工具集", "app_image": "47.110.145.204:8084/huansi/hs_tools:1.1"}]
res = self.client.post('/info/app_upgrade/', headers=headers, json=json_data)
self.assertEqual(res.status_code, 200)
self.assertEqual(res.json['message'], '写入成功')
def test_get_upgrade_log(self):
res = self.client.get('/info/upgrade_log/')
self.assertEqual(res.status_code, 200)
print(res.json)
def test_get_upgrade_log_dtl(self):
log_id = 1237310513316270080
res = self.client.get('/info/upgrade_log_dtl/{}/'.format(log_id))
self.assertEqual(res.status_code, 200)
print(res.json)
# -*- coding:utf-8 -*-
import unittest
from flask_app import global_app
class TestAuthorization(unittest.TestCase):
def setUp(self):
self.client = global_app.test_client()
# -*- coding:utf-8 -*-
import os
from flask_app import global_app
if __name__ == "__main__":
with global_app.app_context():
from huansi_utils.db.sqlUpgradeTool import upgrade_db
upgrade_db(os.path.join(os.getcwd(), 'DB', 'sql_list.txt'))
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment